feat(wasm)!: support large-file downloads and disk streaming

Cargo.toml

@@ -1,4 +1,3 @@
 [workspace]
 members = ["ant-core", "ant-cli"]
 resolver = "2"
-

README.md

@@ -35,6 +35,50 @@ transports use `record.rs` for BLAKE3 verification. Native
 `Client::data_download_range` and the browser media reader call the same range
 implementation.
 
+Browser downloads and range reads have no 1 GB file cap. File sizes and offsets
+use 64-bit arithmetic internally; the JavaScript API accepts exact, nonnegative
+`number` positions through `Number.MAX_SAFE_INTEGER` (just under 8 PiB). The
+resolved DataMap and seek index still occupy memory proportional to the number
+of chunks; content buffers stay bounded, including for seeks past 4 GiB.
+Uploads retain their existing size limits.
+
+`downloadPublicFile(file, concurrency?, onProgress?, { maxMemoryBytes?, signal? })`
+and its private-file counterpart allocate the output as one JavaScript
+`Uint8Array` and copy each decrypted chunk into place as it arrives, so WASM
+holds only the records in flight. Progress is reported as
+`Downloaded chunk n/total`, and an `AbortSignal` cancels the download. `maxMemoryBytes` limits the output buffer,
+not total process memory; without it, allocation is attempted up to the
+JavaScript engine's buffer limit. Allocation errors reject with guidance to use
+streaming, since browsers do not expose a reliable amount of free memory.
+
+For large downloads, pass a `WritableStream` to `BrowserFileReader.pipeTo`.
+For example, after the application obtains a file handle from a user gesture:
+
+```js
+const reader = await client.openPublicFile(address);
+try {
+  const writable = await fileHandle.createWritable();
+  const { bytesWritten, hash } = await reader.pipeTo(writable);
+} finally {
+  reader.close();
+}
+```
+
+The same API works with `openPrivateFile({ data_map })`. `pipeTo(writable,
+{ start?, end?, signal?, onProgress? })` optionally streams a half-open byte
+range and reports `(bytesWritten, totalBytes)` after each write. Chunks are
+fetched concurrently and written in order, at most 4 MiB per write; fetching
+continues during writes until 32 MiB of plaintext is held. Invalid calls reject
+without touching the destination; otherwise the destination is closed on
+success, aborted on failure (an abort during its close may come too late to
+undo the commit), and its writer lock released. The result contains
+the byte count and BLAKE3 of the written range, and an `AbortSignal` cancels
+the call. `reader.close()` releases the records read-ahead holds and makes later
+calls fail. `readRange(start, length)` remains available for media and custom
+sinks, with a 4 MiB per-call limit. The application chooses a memory budget and
+supplies its disk destination; the core does not open file pickers or silently
+select a path.
+
 The adapters supply QUIC or WebRTC discovery/GET requests, runtime timers,
 caches, progress callbacks, and output handling. Browser descriptors and wallet
 callbacks remain browser API concerns. Native filesystem streaming still uses

ant-core/Cargo.toml

@@ -24,7 +24,7 @@ rmp-serde = "1"
 serde = { version = "1", features = ["derive"] }
 serde_bytes = "0.11"
 serde_json = "1"
-self_encryption = "0.36"
+self_encryption = "0.36.2"
 # Portable WebRTC Direct profile shared with ant-node.
 saorsa-transport = { version = "0.37.0", default-features = false, features = ["webrtc"] }
 thiserror = "2"
@@ -91,6 +91,7 @@ serde-wasm-bindgen = { version = "0.6", optional = true }
 wasm-bindgen = { version = "0.2", optional = true }
 wasm-bindgen-futures = { version = "0.4", optional = true }
 web-sys = { version = "0.3", optional = true, features = [
+    "AbortSignal",
     "Event",
     "MessageEvent",
     "RtcConfiguration",
@@ -104,6 +105,8 @@ web-sys = { version = "0.3", optional = true, features = [
     "RtcSdpType",
     "RtcSessionDescription",
     "RtcSessionDescriptionInit",
+    "WritableStream",
+    "WritableStreamDefaultWriter",
 ] }
 gloo-timers = { version = "0.3", features = ["futures"], optional = true }
 # Deterministic mock-node signing for generated-WASM transport regression tests.

ant-core/browser-tests/README.md

@@ -75,3 +75,26 @@ or node RPC mocks are installed. It also checks that a closed network client
 rejects reconnection. Temporary node data is removed on shutdown.
 The suite uses local funds only and verifies that the RPC belongs to the local
 Anvil devnet before invoking the wallet.
+
+## Large-file disk streaming
+
+`large-file.mjs` is an opt-in check, not run in CI, that a file larger than
+4 GiB streams to disk in real Chromium. Build the production bindings as above
+and a release CLI (`cargo build --release --bin ant` in the workspace root),
+start `node start-devnet.mjs` in another terminal, then run:
+
+```sh
+node large-file.mjs [path to ant]
+```
+
+It writes the 4,303,347,835-byte plaintext of the
+[`large-file.json`](../wasm-tests/fixtures/large-file.json) fixture to a
+temporary file and uploads it with `ant`, under a throwaway home so the upload
+stays out of your upload history, and checks it lands on the fixture's
+address. Chromium then opens the file, reads across 4 GiB, and pipes all of it
+into the origin private file system with `pipeTo`. The run checks the BLAKE3
+hash and the stored size and bytes, and that a complete in-memory download over
+a 64 MiB budget fails and points to `pipeTo`. It exits non-zero if any check
+fails and needs about 5 GB of free disk. The file is one repeated byte, so its
+records compress to about 67 KB: the run checks correctness and local
+decryption and writing, not network throughput.

ant-core/browser-tests/large-file.mjs

@@ -0,0 +1,122 @@
+// Opt-in check, not run in CI, that a >4 GiB public file streams to disk in real Chromium.
+// With `node start-devnet.mjs` running, production bindings in ../wasm-tests/pkg and a release `ant`:
+// node large-file.mjs [path to ant, default ../../target/release/ant]
+// Uploads the fixture's plaintext natively, then pipes it into the origin private file system.
+import { chromium } from "@playwright/test";
+import { spawnSync } from "node:child_process";
+import { createWriteStream } from "node:fs";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { createServer } from "node:http";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+
+const DEVNET_API = "http://127.0.0.1:35000/api";
+const PKG = resolve("../wasm-tests/pkg");
+const WRITE_BLOCK_BYTES = 4 * 1024 * 1024;
+// A complete in-memory download over this budget must fail before any transfer.
+const BUDGET_BYTES = 64 * 1024 * 1024;
+const FOUR_GIB = 2 ** 32;
+// Bytes read on each side of a probed position.
+const PROBE_BYTES = 8;
+
+const ant = resolve(process.argv[2] ?? "../../target/release/ant");
+const file = JSON.parse(await readFile("../wasm-tests/fixtures/large-file.json", "utf8"));
+const devnet = await (await fetch(`${DEVNET_API}/devnet-manifest.json`)).json();
+const { endpoints } = await (await fetch(`${DEVNET_API}/browser-manifest.json`)).json();
+const directory = await mkdtemp(join(tmpdir(), "ant-large-file-"));
+const server = createServer(async (request, response) => {
+  const name = request.url?.split("?")[0];
+  if (name === "/") { response.setHeader("Content-Type", "text/html"); response.end("<!doctype html><title>Large file</title>"); return; }
+  if (!["/ant_core.js", "/ant_core_bg.wasm"].includes(name)) { response.writeHead(404).end(); return; }
+  response.setHeader("Content-Type", name.endsWith(".wasm") ? "application/wasm" : "text/javascript");
+  response.end(await readFile(join(PKG, name.slice(1))));
+});
+let context;
+try {
+  // The fixture's plaintext, so the native upload must land on the fixture's address.
+  const source = join(directory, "large-file.bin");
+  const block = Buffer.alloc(WRITE_BLOCK_BYTES, file.byte);
+  await pipeline(Readable.from((function* () {
+    for (let left = file.size; left > 0; left -= block.length) yield block.subarray(0, Math.min(left, block.length));
+  })()), createWriteStream(source));
+  const manifest = join(directory, "devnet-manifest.json");
+  await writeFile(manifest, JSON.stringify(devnet));
+  // A throwaway home keeps the upload out of the real upload history.
+  const home = join(directory, "home");
+  const upload = spawnSync(ant, ["--json", "--devnet-manifest", manifest, "--allow-loopback", "--evm-network", "local",
+    "file", "upload", source, "--public"], { encoding: "utf8", stdio: ["ignore", "pipe", "inherit"],
+    env: { ...process.env, HOME: home, XDG_DATA_HOME: home, XDG_CONFIG_HOME: home, SECRET_KEY: devnet.evm.wallet_private_key } });
+  if (upload.status !== 0) throw new Error(`ant file upload failed with ${upload.status ?? upload.error}`);
+  const uploaded = JSON.parse(upload.stdout);
+  await rm(source);
+
+  await new Promise(done => server.listen(0, "127.0.0.1", done));
+  // A persistent profile gives the origin private file system a normal disk quota.
+  context = await chromium.launchPersistentContext(join(directory, "profile"), { headless: true,
+    args: ["--force-fieldtrials=WebRTC-NoSdpMangleUfrag/Enabled/"] });
+  const page = await context.newPage();
+  const errors = [];
+  page.on("pageerror", error => errors.push(error.message));
+  await page.goto(`http://127.0.0.1:${server.address().port}/`);
+  const result = await page.evaluate(async ({ endpoint, file, budget, boundary, probe }) => {
+    const { default: init, BrowserNetworkClient } = await import("/ant_core.js");
+    await init();
+    const seconds = since => Math.round(performance.now() - since) / 1000;
+    const filled = bytes => bytes.every(value => value === file.byte);
+    const client = new BrowserNetworkClient([endpoint]);
+    let started = performance.now();
+    const reader = await client.openPublicFile(file.address);
+    const openSeconds = seconds(started);
+    const seek = await reader.readRange(boundary - probe, 2 * probe);
+
+    const root = await navigator.storage.getDirectory();
+    const handle = await root.getFileHandle("large-file.bin", { create: true });
+    let progressCalls = 0, lastProgress;
+    started = performance.now();
+    const piped = await reader.pipeTo(await handle.createWritable(),
+      { onProgress: written => { progressCalls++; lastProgress = written; } });
+    const pipeSeconds = seconds(started);
+    const stored = await handle.getFile();
+    const around = new Uint8Array(await stored.slice(boundary - probe, boundary + probe).arrayBuffer());
+    const tail = new Uint8Array(await stored.slice(stored.size - 2 * probe).arrayBuffer());
+    const size = reader.size;
+    reader.close();
+
+    let budgetError;
+    try { await client.downloadPublicFile(file.address, undefined, undefined, { maxMemoryBytes: budget }); }
+    catch (error) { budgetError = String(error); }
+    // Informational: whether this browser could hold the whole file in one buffer at all.
+    let allocation;
+    try { allocation = `allocated ${new Uint8Array(file.size).length} bytes`; }
+    catch (error) { allocation = `${error.name}: ${error.message}`; }
+    client.close(); client.free();
+    await root.removeEntry("large-file.bin");
+    return { size, seekFilled: seek.length === 2 * probe && filled(seek), bytesWritten: piped.bytesWritten,
+      hash: piped.hash, storedSize: stored.size, storedFilled: filled(around) && filled(tail),
+      progressCalls, lastProgress, budgetError, allocation, openSeconds, pipeSeconds };
+  }, { endpoint: endpoints[0], file: { address: file.address, size: file.size, byte: file.byte },
+    budget: BUDGET_BYTES, boundary: FOUR_GIB, probe: PROBE_BYTES });
+
+  const checks = {
+    uploadAddress: uploaded.address === file.address,
+    size: result.size === file.size,
+    seekAcross4GiB: result.seekFilled,
+    bytesWritten: result.bytesWritten === file.size,
+    hash: result.hash === file.hash,
+    storedSize: result.storedSize === file.size,
+    storedBytes: result.storedFilled,
+    progress: result.lastProgress === file.size,
+    budgetPointsAtPipeTo: result.budgetError?.includes("pipeTo") ?? false,
+    noPageErrors: errors.length === 0,
+  };
+  console.log(JSON.stringify({ browser: context.browser()?.version() ?? chromium.name(),
+    uploadSeconds: uploaded.elapsed_secs, result, errors, checks }, null, 2));
+  const failed = Object.keys(checks).filter(name => !checks[name]);
+  if (failed.length > 0) { console.error(`failed: ${failed.join(", ")}`); process.exitCode = 1; }
+} finally {
+  await context?.close();
+  server.close();
+  await rm(directory, { recursive: true, force: true });
+}

ant-core/examples/generate-browser-large-file.rs

@@ -0,0 +1,55 @@
+//! Regenerate the compact native fixture used by the WASM large-file tests:
+//! cargo run -p ant-core --release --example generate-browser-large-file
+//!
+//! Encrypts >4 GiB with bounded memory. Repeated plaintext compresses well;
+//! all encrypted records fit in a small fixture without allocating the file.
+use bytes::Bytes;
+use std::collections::HashMap;
+
+/// Plaintext byte repeated through the file; the tests check it on every read.
+const FILL_BYTE: u8 = 0x5a;
+/// Whole chunks past the 4 GiB position, so reads and seeks cross it.
+const CHUNKS_PAST_4_GIB: u64 = 2;
+/// A short final chunk, so the file does not end on a chunk boundary.
+const TAIL_BYTES: u64 = 123;
+
+fn main() {
+    let size =
+        (1u64 << 32) + CHUNKS_PAST_4_GIB * self_encryption::MAX_CHUNK_SIZE as u64 + TAIL_BYTES;
+    let block = Bytes::from(vec![FILL_BYTE; self_encryption::MAX_CHUNK_SIZE]);
+    let mut remaining = size as usize;
+    let mut hasher = blake3::Hasher::new();
+    let input = std::iter::from_fn(|| {
+        if remaining == 0 {
+            return None;
+        }
+        let length = remaining.min(block.len());
+        remaining -= length;
+        hasher.update(&block[..length]);
+        Some(block.slice(..length))
+    });
+    let mut encryptor = self_encryption::stream_encrypt(size as usize, input).unwrap();
+    let records: HashMap<_, _> = encryptor.chunks().map(|item| item.unwrap()).collect();
+    let map = encryptor.into_datamap().unwrap();
+    let root =
+        self_encryption::get_root_data_map(map.clone(), &mut |hash| Ok(records[&hash].clone()))
+            .unwrap();
+    assert_eq!(root.original_file_size() as u64, size);
+    let mut retained = records;
+    let map_bytes = rmp_serde::to_vec(&map).unwrap();
+    let address = ant_protocol::compute_address(&map_bytes);
+    retained.insert(self_encryption::XorName(address), Bytes::from(map_bytes));
+    let mut records: Vec<_> = retained.into_iter().map(|(hash, content)| {
+        serde_json::json!({"address": hex::encode(hash.0), "content": hex::encode(content)})
+    }).collect();
+    records.sort_by(|a, b| a["address"].as_str().cmp(&b["address"].as_str()));
+    let fixture = serde_json::json!({
+        "size": size, "byte": FILL_BYTE, "hash": hasher.finalize().to_hex().to_string(),
+        "address": hex::encode(address), "records": records,
+    });
+    let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("wasm-tests/fixtures/large-file.json");
+    std::fs::create_dir_all(path.parent().unwrap()).unwrap();
+    std::fs::write(&path, serde_json::to_vec_pretty(&fixture).unwrap()).unwrap();
+    eprintln!("Wrote {}-byte file fixture to {}", size, path.display());
+}

ant-core/src/browser.rs

@@ -27,14 +27,16 @@ pub use protocol::{
 mod wasm_transport;
 
 use bytes::Bytes;
-use self_encryption::{DataMap, EncryptedChunk};
+use self_encryption::{ChunkInfo, DataMap, EncryptedChunk};
 use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
 
-/// Maximum file size accepted by the browser API (1 GB decimal).
+/// Maximum file size accepted by the browser upload API (1 GB decimal).
 ///
-/// The page upload path streams through a worker and browser storage. Complete
-/// downloads and the legacy whole-buffer encryption binding remain memory-bound.
+/// The page upload path streams through a worker and browser storage; the
+/// legacy encryption binding remains memory-bound. Downloads are not limited by
+/// this constant: a complete download needs one buffer of the file's size,
+/// while `pipeTo` and range reads hold a bounded amount of plaintext.
 pub const MAX_BROWSER_FILE_BYTES: usize = 1_000_000_000;
 
 /// One native self-encryption chunk descriptor exposed to the browser.
@@ -255,16 +257,16 @@ pub fn decrypt_public_file(
 }
 
 fn chunk_infos(data_map: &DataMap) -> Vec<BrowserChunkInfo> {
-    data_map
-        .infos()
-        .iter()
-        .map(|info| BrowserChunkInfo {
-            index: info.index,
-            dst_hash: hex::encode(info.dst_hash.0),
-            src_hash: hex::encode(info.src_hash.0),
-            src_size: info.src_size,
-        })
-        .collect()
+    data_map.infos().iter().map(browser_chunk_info).collect()
+}
+
+fn browser_chunk_info(info: &ChunkInfo) -> BrowserChunkInfo {
+    BrowserChunkInfo {
+        index: info.index,
+        dst_hash: hex::encode(info.dst_hash.0),
+        src_hash: hex::encode(info.src_hash.0),
+        src_size: info.src_size,
+    }
 }
 
 #[cfg(all(target_arch = "wasm32", feature = "browser-wasm"))]