From 41dac79defd4498141a33389d7106de47e7d30b3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 04:00:02 +0000 Subject: [PATCH 01/12] chore: record implementation plan From b90b66089ce2eeb4c2616c61b514379d57c4b75e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 04:17:42 +0000 Subject: [PATCH 02/12] feat(release): generate artifact inventory scope Co-authored-by: ScriptedAlchemy <25274700+ScriptedAlchemy@users.noreply.github.com> --- scripts/generate-release-inventory.py | 656 +++++++++++++++++++++ scripts/test-generate-release-inventory.py | 244 ++++++++ 2 files changed, 900 insertions(+) create mode 100644 scripts/generate-release-inventory.py create mode 100644 scripts/test-generate-release-inventory.py diff --git a/scripts/generate-release-inventory.py b/scripts/generate-release-inventory.py new file mode 100644 index 0000000000..f8f217f363 --- /dev/null +++ b/scripts/generate-release-inventory.py @@ -0,0 +1,656 @@ +#!/usr/bin/env python3 +"""Generate a deterministic SPDX inventory for a shipped CLI release target.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +import tarfile +import tomllib +import zipfile +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import quote + + +TARGET = "//crates/tracedecay-cli:tracedecay" +FONT_LICENSES = { + "@fontsource-variable/archivo": "OFL-1.1", + "@fontsource-variable/ibm-plex-sans": "OFL-1.1", + "@fontsource/ibm-plex-mono": "OFL-1.1", +} + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def canonical_json(value: object) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n").encode() + + +def label_name(label: str) -> str: + label = label.split(" (", 1)[0] + return label.strip('"') + + +def cquery(source_root: Path) -> tuple[str, str]: + env = os.environ.copy() + graph = subprocess.run( + [ + "bazel", + "cquery", + "--config=dist", + "--config=ci", + f"deps({TARGET})", + "--output=graph", + ], + cwd=source_root, + env=env, + check=True, + capture_output=True, + text=True, + ) + kinds = subprocess.run( + [ + "bazel", + "cquery", + "--config=dist", + "--config=ci", + f"deps({TARGET})", + "--output=label_kind", + ], + cwd=source_root, + env=env, + check=True, + capture_output=True, + text=True, + ) + return graph.stdout, kinds.stdout + + +def parse_graph(graph: str, kinds: str) -> list[dict[str, str]]: + target_kinds: dict[str, str] = {} + for line in kinds.splitlines(): + match = re.match(r"(.+?) rule (.+)$", line.strip()) + if match: + target_kinds[label_name(match.group(2))] = match.group(1) + + edges = [ + (label_name(parent), label_name(child)) + for parent, child in re.findall(r'"([^"]+)"\s*->\s*"([^"]+)"', graph) + ] + adjacency: dict[str, list[str]] = {} + for parent, child in edges: + adjacency.setdefault(parent, []).append(child) + + root = label_name(TARGET) + pending = [root] + visited: set[str] = set() + included: set[str] = set() + while pending: + current = pending.pop() + if current in visited: + continue + visited.add(current) + kind = target_kinds.get(current, "") + if kind in {"cargo_build_script", "rust_proc_macro", "rust_test"}: + continue + if kind == "rust_library": + if current.startswith("@@") or current.startswith("@"): + included.add(current) + elif current.startswith("//crates/") or current.startswith("//patches/"): + included.add(current) + pending.extend(adjacency.get(current, ())) + + return [{"label": label, "kind": target_kinds.get(label, "")} for label in sorted(included)] + + +def lock_package_index(source_root: Path) -> tuple[dict[str, dict[str, object]], dict[str, dict[str, object]]]: + lock = tomllib.loads((source_root / "Cargo.lock").read_text(encoding="utf-8")) + by_repo: dict[str, dict[str, object]] = {} + by_name: dict[str, dict[str, object]] = {} + for package in lock.get("package", []): + if package.get("source"): + by_repo[f"c__{package['name']}-{package['version']}"] = package + else: + by_name[package["name"]] = package + return by_repo, by_name + + +def cargo_package( + source_root: Path, + output_base: Path, + label: str, + lock_packages: tuple[dict[str, dict[str, object]], dict[str, dict[str, object]]], +) -> dict[str, object] | None: + by_repo, by_name = lock_packages + if label.startswith("@"): + repository = label.split("//", 1)[0].lstrip("@") + repository = repository.split("+", 1)[0] if repository.startswith("c+") else repository + package = by_repo.get(repository) + if package is None: + # Canonical repository labels may include a module-version suffix. + package = next( + ( + value + for key, value in by_repo.items() + if repository.endswith(key.removeprefix("c__")) + ), + None, + ) + if package is None: + return { + "name": repository, + "version": "NOASSERTION", + "source": "NOASSERTION", + "license": "NOASSERTION", + "checksum": None, + "local_path": None, + } + manifest = output_base / "external" / repository / "Cargo.toml" + source = str(package.get("source", "NOASSERTION")) + return package_record(package, manifest, source_root, source) + + package_path = label.split("//", 1)[-1].split(":", 1)[0] + if not package_path: + return None + manifest = source_root / package_path / "Cargo.toml" + if not manifest.is_file(): + return None + with manifest.open("rb") as source_file: + metadata = tomllib.load(source_file).get("package", {}) + package_name = str(metadata.get("name", "")) + package = by_name.get(package_name, {}) + source = f"git+https://github.com/ScriptedAlchemy/tracedecay#source-revision" + local_path = package_path + return package_record( + { + "name": package_name, + "version": metadata.get("version", package.get("version", "NOASSERTION")), + "checksum": package.get("checksum"), + }, + manifest, + source_root, + source, + local_path=local_path, + ) + + +def package_record( + package: dict[str, object], + manifest: Path, + source_root: Path, + source: str, + *, + local_path: str | None = None, +) -> dict[str, object]: + metadata: dict[str, object] = {} + if manifest.is_file(): + with manifest.open("rb") as source_file: + metadata = tomllib.load(source_file).get("package", {}) + license_value = metadata.get("license") + name = str(package.get("name", metadata.get("name", "unknown"))) + version = str(package.get("version", metadata.get("version", "NOASSERTION"))) + source_url = source.removeprefix("git+") + if source.startswith("registry+"): + source_url = f"https://crates.io/crates/{quote(name, safe='')}/{quote(version, safe='')}" + record: dict[str, object] = { + "name": name, + "version": version, + "source": source, + "downloadLocation": source_url or "NOASSERTION", + "license": license_value if isinstance(license_value, str) and license_value else "NOASSERTION", + "checksum": package.get("checksum"), + "purl": f"pkg:cargo/{quote(name, safe='')}@{quote(version, safe='')}", + } + if local_path: + record["local_path"] = local_path + record["source"] = f"{local_path} at the release source revision" + record["downloadLocation"] = "NOASSERTION" + return record + + +def component_id(purl: str) -> str: + return "SPDXRef-" + hashlib.sha256(purl.encode()).hexdigest()[:24] + + +def checksum_from_sri(integrity: str) -> dict[str, str] | None: + for item in integrity.split(): + algorithm, separator, encoded = item.partition("-") + if not separator or algorithm.lower() not in {"sha1", "sha256", "sha384", "sha512"}: + continue + import base64 + + try: + digest = base64.b64decode(encoded, validate=True).hex() + except ValueError: + continue + return {"algorithm": algorithm.upper().replace("SHA", "SHA"), "checksumValue": digest} + return None + + +def npm_component_key(name: str, version: str, package_keys: set[str]) -> str | None: + exact = f"{name}@{version}" + if exact in package_keys: + return exact + candidates = sorted( + key + for key in package_keys + if key.startswith(exact + "(") + ) + return candidates[0] if candidates else None + + +def npm_components(source_root: Path, platform: str) -> list[dict[str, object]]: + try: + import yaml + except ImportError as error: + raise RuntimeError("PyYAML is required to read the locked dashboard package graph") from error + + lock = yaml.safe_load((source_root / "pnpm-lock.yaml").read_text(encoding="utf-8")) + dashboard = lock.get("importers", {}).get("dashboard", {}) + roots = dashboard.get("dependencies", {}) + packages = lock.get("packages", {}) + snapshots = lock.get("snapshots", {}) + package_keys = set(packages) + os_name, arch = { + "aarch64-macos": ("darwin", "arm64"), + "x86_64-linux": ("linux", "x64"), + "aarch64-linux": ("linux", "arm64"), + "x86_64-windows": ("win32", "x64"), + }[platform] + + pending: list[str] = [] + for name, record in roots.items(): + version = record.get("version") + if version: + key = npm_component_key(name, str(version), package_keys) + if key: + pending.append(key) + + visited: set[str] = set() + result: list[dict[str, object]] = [] + while pending: + key = pending.pop() + if key in visited: + continue + visited.add(key) + metadata = packages.get(key, {}) + version = key.rsplit("@", 1)[-1].split("(", 1)[0] + name = key[: -(len(key.rsplit("@", 1)[-1]) + 1)] + if not name: + continue + package = { + "name": name, + "version": version, + "source": metadata.get("resolution", {}).get("tarball") + or f"https://registry.npmjs.org/{quote(name, safe='@/')}/-/{name.rsplit('/', 1)[-1]}-{version}.tgz", + "license": FONT_LICENSES.get(name, "NOASSERTION"), + "checksum": checksum_from_sri( + str(metadata.get("resolution", {}).get("integrity", "")) + ), + "purl": f"pkg:npm/{quote(name, safe='@/') }@{quote(version, safe='')}", + } + result.append(package) + node = snapshots.get(key, {}) + for group in ("dependencies", "optionalDependencies"): + for dependency, child_version in node.get(group, {}).items(): + child = npm_component_key(dependency, str(child_version), package_keys) + if child is None: + continue + constraints = packages.get(child, {}) + if constraints.get("os") and os_name not in constraints["os"] and "!" + os_name not in constraints["os"]: + continue + if constraints.get("cpu") and arch not in constraints["cpu"] and "!" + arch not in constraints["cpu"]: + continue + pending.append(child) + return sorted(result, key=lambda component: str(component["purl"])) + + +def scope_inventory( + source_root: Path, + output: Path, + platform: str, + source_sha: str, + dashboard_dist: Path, + graph_file: Path | None, + kinds_file: Path | None, + output_base_arg: Path | None, +) -> None: + if platform not in { + "aarch64-macos", + "x86_64-linux", + "aarch64-linux", + "x86_64-windows", + }: + raise ValueError(f"unsupported release platform: {platform}") + if graph_file and kinds_file: + graph = graph_file.read_text(encoding="utf-8") + kinds = kinds_file.read_text(encoding="utf-8") + else: + graph, kinds = cquery(source_root) + labels = parse_graph(graph, kinds) + output_base = output_base_arg + if output_base is None: + output_base = Path( + subprocess.run( + ["bazel", "info", "--config=dist", "--config=ci", "output_base"], + cwd=source_root, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + ) + cargo_lock = lock_package_index(source_root) + rust_components = { + str(record["purl"]): record + for item in labels + if ( + record := cargo_package(source_root, output_base, item["label"], cargo_lock) + ) + is not None + } + assets = [] + if not dashboard_dist.is_dir(): + raise FileNotFoundError(f"built dashboard asset directory is missing: {dashboard_dist}") + for path in sorted(item for item in dashboard_dist.rglob("*") if item.is_file()): + assets.append( + { + "path": path.relative_to(dashboard_dist).as_posix(), + "sha256": sha256(path), + "size": path.stat().st_size, + } + ) + adapted_sources = [] + module_file = source_root / "MODULE.bazel" + if module_file.is_file(): + module_text = module_file.read_text(encoding="utf-8") + patch_paths = sorted(set(re.findall(r'"//scripts:([^"]+\.patch)"', module_text))) + for relative in patch_paths: + path = source_root / "scripts" / relative + if path.is_file(): + adapted_sources.append( + { + "path": path.relative_to(source_root).as_posix(), + "sha256": sha256(path), + } + ) + scope = { + "format": "tracedecay-release-component-scope-v1", + "platform": platform, + "bazel_target": TARGET, + "source_sha": source_sha, + "rust_components": [rust_components[key] for key in sorted(rust_components)], + "dashboard_assets": assets, + "adapted_sources": adapted_sources, + } + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(canonical_json(scope)) + + +def artifact_binary(archive: Path, platform: str) -> bytes: + member = "tracedecay.exe" if platform == "x86_64-windows" else "tracedecay" + if archive.name.endswith(".zip"): + with zipfile.ZipFile(archive) as packaged: + return packaged.read(member) + with tarfile.open(archive, "r:gz") as packaged: + content = packaged.extractfile(member) + if content is None: + raise ValueError(f"archive does not contain {member}") + return content.read() + + +def spdx_package(record: dict[str, object]) -> dict[str, object]: + purl = str(record["purl"]) + license_value = str(record.get("license") or "NOASSERTION") + package: dict[str, object] = { + "name": record["name"], + "SPDXID": component_id(purl), + "versionInfo": str(record.get("version") or "NOASSERTION"), + "downloadLocation": str(record.get("downloadLocation") or record.get("source") or "NOASSERTION"), + "filesAnalyzed": False, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": license_value, + "copyrightText": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": purl, + } + ], + } + checksum = record.get("checksum") + if isinstance(checksum, dict): + package["checksums"] = [checksum] + elif isinstance(checksum, str) and checksum: + package["checksums"] = [{"algorithm": "SHA256", "checksumValue": checksum}] + source_info = record.get("source") + if source_info and source_info != package["downloadLocation"]: + package["sourceInfo"] = str(source_info) + return package + + +def release_inventory( + source_root: Path, + scope_path: Path, + archive: Path, + mcpb: Path, + tag: str, + source_sha: str, + source_epoch: int, + repository: str, + output: Path, +) -> None: + scope = json.loads(scope_path.read_text(encoding="utf-8")) + platform = str(scope["platform"]) + npm = npm_components(source_root, platform) + components = [*scope["rust_components"], *npm] + packages = [spdx_package(item) for item in components] + package_ids = {str(package["SPDXID"]) for package in packages} + archive_package_id = "SPDXRef-Archive" + mcpb_package_id = "SPDXRef-MCPB" + binary_id = "SPDXRef-CLI-Binary" + asset_package = lambda name, path, identifier: { + "name": name, + "SPDXID": identifier, + "versionInfo": tag.removeprefix("v"), + "downloadLocation": f"https://github.com/{repository}/releases/download/{quote(tag, safe='')}/{quote(path.name, safe='.-_')}", + "filesAnalyzed": False, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + "checksums": [{"algorithm": "SHA256", "checksumValue": sha256(path)}], + } + packages.extend( + [ + asset_package(archive.name, archive, archive_package_id), + asset_package(mcpb.name, mcpb, mcpb_package_id), + { + "name": "tracedecay CLI binary", + "SPDXID": binary_id, + "versionInfo": tag.removeprefix("v"), + "downloadLocation": f"https://github.com/{repository}", + "filesAnalyzed": False, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "copyrightText": "NOASSERTION", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": hashlib.sha256( + artifact_binary(archive, platform) + ).hexdigest(), + } + ], + }, + ] + ) + file_entries = [] + for item in scope["dashboard_assets"]: + path = str(item["path"]) + file_id = "SPDXRef-Asset-" + hashlib.sha256(path.encode()).hexdigest()[:20] + font_package = next( + ( + name + for name in FONT_LICENSES + if path.startswith(name.split("/", 1)[-1] + "-") + or name.split("/", 1)[-1] in path + ), + None, + ) + file_entries.append( + { + "fileName": f"dashboard/app-dist/{path}", + "SPDXID": file_id, + "checksums": [{"algorithm": "SHA256", "checksumValue": item["sha256"]}], + "licenseConcluded": "NOASSERTION", + "licenseInfoInFiles": [FONT_LICENSES[font_package]] if font_package else ["NOASSERTION"], + "copyrightText": "NOASSERTION", + } + ) + + relationships = [ + {"spdxElementId": archive_package_id, "relationshipType": "CONTAINS", "relatedSpdxElement": binary_id}, + {"spdxElementId": mcpb_package_id, "relationshipType": "CONTAINS", "relatedSpdxElement": binary_id}, + ] + for identifier in sorted(package_ids): + relationships.extend( + [ + {"spdxElementId": binary_id, "relationshipType": "DEPENDS_ON", "relatedSpdxElement": identifier}, + {"spdxElementId": archive_package_id, "relationshipType": "CONTAINS", "relatedSpdxElement": identifier}, + {"spdxElementId": mcpb_package_id, "relationshipType": "CONTAINS", "relatedSpdxElement": identifier}, + ] + ) + for item in file_entries: + for parent in (binary_id, archive_package_id, mcpb_package_id): + relationships.append( + { + "spdxElementId": parent, + "relationshipType": "CONTAINS", + "relatedSpdxElement": item["SPDXID"], + } + ) + review_items = sorted( + f"{package['name']}@{package.get('versionInfo', 'NOASSERTION')}" + for package in packages + if package.get("licenseDeclared") == "NOASSERTION" + or package.get("downloadLocation") == "NOASSERTION" + ) + annotations = [ + { + "annotationDate": datetime.fromtimestamp( + source_epoch, timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ"), + "annotationType": "OTHER", + "annotator": "Tool: tracedecay-release-inventory", + "comment": f"Review required: license or source metadata is unknown for {item}.", + } + for item in review_items + ] + file_entries.extend( + { + "fileName": str(item["path"]), + "SPDXID": "SPDXRef-Adapted-" + hashlib.sha256(str(item["path"]).encode()).hexdigest()[:20], + "checksums": [{"algorithm": "SHA256", "checksumValue": item["sha256"]}], + "licenseConcluded": "NOASSERTION", + "licenseInfoInFiles": ["NOASSERTION"], + "copyrightText": "NOASSERTION", + } + for item in scope.get("adapted_sources", []) + ) + document_name = f"TraceDecay {tag} {platform} component inventory" + namespace = ( + "https://spdx.org/spdxdocs/" + + quote(document_name.replace(" ", "-"), safe="-_.") + + "/" + + hashlib.sha256((source_sha + sha256(archive) + sha256(mcpb)).encode()).hexdigest() + ) + document = { + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": document_name, + "documentNamespace": namespace, + "creationInfo": { + "creators": ["Tool: tracedecay-release-inventory"], + "created": datetime.fromtimestamp( + source_epoch, timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ"), + "comment": f"Source revision {source_sha}; Bazel target {scope['bazel_target']}. Unknown metadata is called out in annotations.", + }, + "packages": sorted(packages, key=lambda package: str(package["SPDXID"])), + "files": sorted(file_entries, key=lambda item: str(item["SPDXID"])), + "relationships": sorted( + relationships, + key=lambda item: ( + item["spdxElementId"], + item["relationshipType"], + item["relatedSpdxElement"], + ), + ), + "annotations": annotations, + } + output.parent.mkdir(parents=True, exist_ok=True) + output.write_bytes(canonical_json(document)) + + +def main() -> None: + parser = argparse.ArgumentParser() + commands = parser.add_subparsers(dest="command", required=True) + scope = commands.add_parser("scope") + scope.add_argument("--source-root", type=Path, default=Path(".")) + scope.add_argument("--platform", required=True) + scope.add_argument("--source-sha", required=True) + scope.add_argument("--dashboard-dist", type=Path, required=True) + scope.add_argument("--graph", type=Path) + scope.add_argument("--kinds", type=Path) + scope.add_argument("--output-base", type=Path) + scope.add_argument("--output", type=Path, required=True) + document = commands.add_parser("document") + document.add_argument("--source-root", type=Path, default=Path(".")) + document.add_argument("--scope", type=Path, required=True) + document.add_argument("--archive", type=Path, required=True) + document.add_argument("--mcpb", type=Path, required=True) + document.add_argument("--tag", required=True) + document.add_argument("--source-sha", required=True) + document.add_argument("--source-epoch", type=int, required=True) + document.add_argument("--repository", required=True) + document.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.command == "scope": + scope_inventory( + args.source_root, + args.output, + args.platform, + args.source_sha, + args.dashboard_dist, + args.graph, + args.kinds, + args.output_base, + ) + else: + release_inventory( + args.source_root, + args.scope, + args.archive, + args.mcpb, + args.tag, + args.source_sha, + args.source_epoch, + args.repository, + args.output, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/test-generate-release-inventory.py b/scripts/test-generate-release-inventory.py new file mode 100644 index 0000000000..5e1d1ee8e3 --- /dev/null +++ b/scripts/test-generate-release-inventory.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +"""Behavioral tests for release component inventory scope and artifact digests.""" + +from __future__ import annotations + +import importlib.util +import json +import tarfile +import tempfile +import zipfile +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location( + "release_inventory", Path(__file__).with_name("generate-release-inventory.py") +) +assert SPEC is not None and SPEC.loader is not None +INVENTORY = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(INVENTORY) + + +def write_package(path: Path, name: str, version: str, license_value: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + f'[package]\nname = "{name}"\nversion = "{version}"\nlicense = "{license_value}"\n', + encoding="utf-8", + ) + + +def make_archive(path: Path, payload: bytes) -> None: + with tarfile.open(path, "w:gz") as archive: + entry = tarfile.TarInfo("tracedecay") + entry.size = len(payload) + archive.addfile(entry, __import__("io").BytesIO(payload)) + + +def main() -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "source" + output_base = root / "bazel" + dashboard = root / "app-dist" + dashboard.mkdir(parents=True) + (dashboard / "index.js").write_bytes(b"bundled npm") + (dashboard / "archivo-latin.woff2").write_bytes(b"font bytes") + (source / "crates/tracedecay-cli").mkdir(parents=True) + (source / "crates/tracedecay").mkdir(parents=True) + (source / "patches/tokensave-medium-treesitters").mkdir(parents=True) + (source / "scripts/bazel/patches").mkdir(parents=True) + (source / "scripts/bazel/patches/rust.patch").write_text("adaptation\n") + (source / "MODULE.bazel").write_text( + 'patches = ["//scripts:bazel/patches/rust.patch"]\n', encoding="utf-8" + ) + write_package( + source / "crates/tracedecay-cli/Cargo.toml", + "tracedecay-cli", + "1.2.3", + "MIT", + ) + write_package(source / "crates/tracedecay/Cargo.toml", "tracedecay", "1.2.3", "MIT") + write_package( + source / "patches/tokensave-medium-treesitters/Cargo.toml", + "tokensave-medium-treesitters", + "0.1.0", + "NOASSERTION", + ) + (source / "Cargo.lock").write_text( + """version = 4 + +[[package]] +name = "tracedecay-cli" +version = "1.2.3" +dependencies = ["tracedecay"] + +[[package]] +name = "tracedecay" +version = "1.2.3" +dependencies = ["asupersync 0.6.0 (git+https://github.com/vendor/asupersync?rev=abc#abc)", "serde 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)"] + +[[package]] +name = "asupersync" +version = "0.6.0" +source = "git+https://github.com/vendor/asupersync?rev=abc#abc" + +[[package]] +name = "serde" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0123456789abcdef" + +[[package]] +name = "tokensave-medium-treesitters" +version = "0.1.0" +""", + encoding="utf-8", + ) + (output_base / "external/c__asupersync-0.6.0").mkdir(parents=True) + (output_base / "external/c__serde-1.0.0").mkdir(parents=True) + write_package( + output_base / "external/c__asupersync-0.6.0/Cargo.toml", + "asupersync", + "0.6.0", + "MIT OR Apache-2.0", + ) + write_package( + output_base / "external/c__serde-1.0.0/Cargo.toml", + "serde", + "1.0.0", + "MIT OR Apache-2.0", + ) + graph = root / "graph.dot" + graph.write_text( + '\n'.join( + ( + '"//crates/tracedecay-cli:tracedecay" -> "//crates/tracedecay:tracedecay";', + '"//crates/tracedecay:tracedecay" -> "@@c__serde-1.0.0//:serde";', + '"//crates/tracedecay:tracedecay" -> "@@c__asupersync-0.6.0//:asupersync";', + '"//crates/tracedecay:tracedecay" -> "//patches/tokensave-medium-treesitters:tokensave-medium-treesitters";', + '"//crates/tracedecay:tracedecay" -> "//scripts:build_script_build";', + '"//scripts:build_script_build" -> "@@c__build-tool-1.0.0//:build_tool";', + ) + ), + encoding="utf-8", + ) + kinds = root / "kinds.txt" + kinds.write_text( + """rust_binary rule //crates/tracedecay-cli:tracedecay +rust_library rule //crates/tracedecay:tracedecay +rust_library rule @@c__serde-1.0.0//:serde +rust_library rule @@c__asupersync-0.6.0//:asupersync +rust_library rule //patches/tokensave-medium-treesitters:tokensave-medium-treesitters +cargo_build_script rule //scripts:build_script_build +rust_library rule @@c__build-tool-1.0.0//:build_tool +""", + encoding="utf-8", + ) + scope = root / "scope.json" + INVENTORY.scope_inventory( + source, + scope, + "x86_64-linux", + "a" * 40, + dashboard, + graph, + kinds, + output_base, + ) + scope_bytes = scope.read_bytes() + assert scope_bytes == INVENTORY.canonical_json(json.loads(scope_bytes)) + scoped = json.loads(scope_bytes) + rust = {component["name"]: component for component in scoped["rust_components"]} + assert {"tracedecay", "serde", "asupersync", "tokensave-medium-treesitters"} <= set(rust) + assert "build-tool" not in rust + assert "https://github.com/vendor/asupersync" in rust["asupersync"]["downloadLocation"] + assert scoped["adapted_sources"][0]["path"] == "scripts/bazel/patches/rust.patch" + + (source / "pnpm-lock.yaml").write_text( + """lockfileVersion: '9.0' +importers: + dashboard: + dependencies: + '@fontsource-variable/archivo': + version: 5.3.0 + react: + version: 19.0.0 + devDependencies: + playwright: + version: 1.0.0 +packages: + '@fontsource-variable/archivo@5.3.0': + resolution: + integrity: sha512-YWJj + react@19.0.0: + resolution: + integrity: sha512-cmVhY3Q= + playwright@1.0.0: + resolution: + integrity: sha512-cGxheXdyaWdodA== +snapshots: + '@fontsource-variable/archivo@5.3.0': {} + react@19.0.0: {} + playwright@1.0.0: {} +""", + encoding="utf-8", + ) + binary = b"release-binary" + archive = root / "tracedecay-v1.2.3-x86_64-linux.tar.gz" + make_archive(archive, binary) + mcpb = root / "tracedecay-v1.2.3-x86_64-linux.mcpb" + with zipfile.ZipFile(mcpb, "w") as bundle: + bundle.writestr("manifest.json", "{}") + bundle.writestr("server/tracedecay", binary) + inventory = root / "inventory.spdx.json" + arguments = ( + source, + scope, + archive, + mcpb, + "v1.2.3", + "a" * 40, + 1_700_000_000, + "ScriptedAlchemy/tracedecay", + inventory, + ) + INVENTORY.release_inventory(*arguments) + first = inventory.read_bytes() + INVENTORY.release_inventory(*arguments) + assert inventory.read_bytes() == first + + document = json.loads(first) + assert document["spdxVersion"] == "SPDX-2.3" + packages = {package["name"]: package for package in document["packages"]} + assert "react" in packages + assert "@fontsource-variable/archivo" in packages + assert "playwright" not in packages + assert packages["asupersync"]["sourceInfo"].startswith("git+https://github.com/vendor/") + assert packages["tokensave-medium-treesitters"]["licenseDeclared"] == "NOASSERTION" + assert any("tokensave-medium-treesitters" in item["comment"] for item in document["annotations"]) + assert any(item["fileName"].endswith("archivo-latin.woff2") for item in document["files"]) + artifact_checksums = { + package["name"]: package["checksums"][0]["checksumValue"] + for package in document["packages"] + if package["SPDXID"] in {"SPDXRef-Archive", "SPDXRef-MCPB"} + } + assert artifact_checksums[archive.name] == INVENTORY.sha256(archive) + assert artifact_checksums[mcpb.name] == INVENTORY.sha256(mcpb) + + (dashboard / "index.js").write_bytes(b"changed bundled npm") + INVENTORY.scope_inventory( + source, + scope, + "x86_64-linux", + "a" * 40, + dashboard, + graph, + kinds, + output_base, + ) + assert scope.read_bytes() != scope_bytes + + print("release component inventory tests passed") + + +if __name__ == "__main__": + main() From f0cbac1dd4ef40110e0b3b46a16fd4a06aca765e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 04:38:05 +0000 Subject: [PATCH 03/12] feat(release): publish artifact-scoped component inventories Co-authored-by: ScriptedAlchemy <25274700+ScriptedAlchemy@users.noreply.github.com> --- .github/workflows/ci.yml | 1 + .github/workflows/release-beta.yml | 121 +++++- .github/workflows/release.yml | 130 ++++++- SECURITY.md | 15 +- scripts/BUILD.bazel | 2 + scripts/assemble-release-artifacts.sh | 44 ++- scripts/check-release-artifacts.py | 8 + scripts/generate-release-inventory.py | 433 +++++++++++++++++---- scripts/plan-release-recovery.py | 38 +- scripts/test-check-release-artifacts.py | 25 +- scripts/test-generate-release-inventory.py | 163 ++++++-- scripts/test-plan-release-recovery.py | 70 +++- 12 files changed, 895 insertions(+), 155 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77c8ed4556..de47327d8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -212,6 +212,7 @@ jobs: python3 scripts/test-package-release-archive.py python3 scripts/test-plan-release-recovery.py python3 scripts/test-check-release-artifacts.py + python3 scripts/test-generate-release-inventory.py python3 scripts/test-update-release-checksums.py python3 scripts/test-neutralize-release-pr-closing-keywords.py bash scripts/test-check-distribution-clean-source.sh diff --git a/.github/workflows/release-beta.yml b/.github/workflows/release-beta.yml index 8dba727562..b098d5754e 100644 --- a/.github/workflows/release-beta.yml +++ b/.github/workflows/release-beta.yml @@ -38,6 +38,7 @@ jobs: source_epoch: ${{ steps.source.outputs.source_epoch }} matrix: ${{ steps.source.outputs.matrix }} build_required: ${{ steps.source.outputs.build_required }} + package_build_required: ${{ steps.source.outputs.package_build_required }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -45,6 +46,17 @@ jobs: fetch-depth: 0 persist-credentials: false + - name: Checkout release recovery planner + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.workflow_sha }} + path: .release-automation + persist-credentials: false + sparse-checkout-cone-mode: false + sparse-checkout: | + scripts/plan-release-recovery.py + scripts/verify-retained-release-assets.sh + - name: Validate prerelease env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -75,7 +87,7 @@ jobs: echo "source_epoch=${source_epoch}" >> "$GITHUB_OUTPUT" gh api "repos/${{ github.repository }}/releases/tags/${RELEASE_TAG}" \ --jq '.assets[].name' > remote-asset-names - python3 scripts/plan-release-recovery.py \ + python3 .release-automation/scripts/plan-release-recovery.py \ --manifest .github/release-targets.json \ --tag "$RELEASE_TAG" \ --profile beta \ @@ -83,7 +95,7 @@ jobs: --retained-output retained-asset-names \ --github-output "$GITHUB_OUTPUT" mkdir retained-assets - scripts/verify-retained-release-assets.sh \ + .release-automation/scripts/verify-retained-release-assets.sh \ --tag "$RELEASE_TAG" \ --repo "${{ github.repository }}" \ --signer-workflow "${{ github.repository }}/.github/workflows/release-beta.yml" \ @@ -126,8 +138,22 @@ jobs: with: python-version: "3.15.0" + - name: Install the locked-inventory YAML reader + run: python3 -m pip install pyyaml==6.0.2 + + - name: Set up Bazel for inventory-only recovery + if: matrix.inventory_only == 'true' + uses: ./.github/actions/setup-bazel + with: + cache-key: release-${{ matrix.target }} + + - name: Build dashboard bundle for inventory-only recovery + if: matrix.inventory_only == 'true' + run: bazel build --config=dist --config=ci //dashboard:app_dist + # Bazel builds the dashboard bundle the binary embeds (//dashboard:app_dist). - name: Build release binary for packaging + if: matrix.inventory_only != 'true' id: cli uses: ./.github/actions/build-cli with: @@ -144,9 +170,12 @@ jobs: ref: ${{ github.workflow_sha }} path: .release-automation persist-credentials: false - sparse-checkout: .github/actions/package-release + sparse-checkout: | + .github/actions/package-release + scripts/generate-release-inventory.py - name: Package and verify native archive and MCPB + if: matrix.inventory_only != 'true' uses: ./.release-automation/.github/actions/package-release with: binary: ${{ steps.cli.outputs.path }} @@ -156,7 +185,29 @@ jobs: archive: ${{ matrix.archive }} epoch: ${{ needs.validate.outputs.source_epoch }} + - name: Record shipping component scope + shell: bash + env: + PLATFORM: ${{ matrix.name }} + SOURCE_SHA: ${{ needs.validate.outputs.source_sha }} + run: | + set -euo pipefail + python3 .release-automation/scripts/generate-release-inventory.py scope \ + --source-root . \ + --platform "$PLATFORM" \ + --source-sha "$SOURCE_SHA" \ + --dashboard-dist "$(bazel info --config=dist --config=ci bazel-bin)/dashboard/app-dist" \ + --output "inventory-scope-${PLATFORM}.json" + + - name: Upload component scope + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: inventory-scope-${{ matrix.name }} + path: inventory-scope-${{ matrix.name }}.json + if-no-files-found: error + - name: Upload MCPB artifact + if: matrix.inventory_only != 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: mcpb-beta-${{ matrix.name }} @@ -164,6 +215,7 @@ jobs: if-no-files-found: error - name: Upload binary archive artifact + if: matrix.inventory_only != 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: beta-${{ matrix.name }} @@ -193,17 +245,23 @@ jobs: ref: ${{ needs.validate.outputs.source_sha }} persist-credentials: false - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - if: needs.validate.outputs.build_required == 'true' + if: needs.validate.outputs.package_build_required == 'true' with: pattern: beta-* path: new-binaries merge-multiple: true - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - if: needs.validate.outputs.build_required == 'true' + if: needs.validate.outputs.package_build_required == 'true' with: pattern: mcpb-beta-* path: new-mcpbs merge-multiple: true + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + if: needs.validate.outputs.build_required == 'true' + with: + pattern: inventory-scope-* + path: new-inventory-scopes + merge-multiple: true # Use the helpers from the running workflow revision for older-tag recovery. - name: Checkout release recovery helpers @@ -215,7 +273,20 @@ jobs: sparse-checkout-cone-mode: false sparse-checkout: | scripts/assemble-release-artifacts.sh + scripts/check-release-artifacts.py + scripts/generate-release-inventory.py + scripts/plan-release-recovery.py scripts/update-release-checksums.py + scripts/verify-retained-release-assets.sh + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + if: needs.validate.outputs.build_required == 'true' + with: + python-version: "3.15.0" + + - name: Install the locked-inventory YAML reader + if: needs.validate.outputs.build_required == 'true' + run: python3 -m pip install pyyaml==6.0.2 - name: Assemble retained and newly built prerelease assets env: @@ -239,8 +310,10 @@ jobs: shopt -s nullglob archives=(new-binaries/tracedecay-beta-*) bundles=(new-mcpbs/*.mcpb) + inventories=(new-inventories/*.spdx.json) echo "archives=${#archives[@]}" >>"$GITHUB_OUTPUT" echo "bundles=${#bundles[@]}" >>"$GITHUB_OUTPUT" + echo "inventories=${#inventories[@]}" >>"$GITHUB_OUTPUT" - name: Attest newly built prerelease archives if: steps.built.outputs.archives != '0' @@ -254,6 +327,12 @@ jobs: with: subject-path: new-mcpbs/*.mcpb + - name: Attest newly generated component inventories + if: steps.built.outputs.inventories != '0' + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-path: new-inventories/*.spdx.json + - name: Upload missing prerelease assets env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -267,16 +346,22 @@ jobs: cd mcpbs sha256sum -- *.mcpb ) >> SHA256SUMS + ( + cd inventories + sha256sum -- *.spdx.json + ) >> SHA256SUMS mapfile -t archives < <(find binaries -maxdepth 1 -type f -name 'tracedecay-beta-*' -print | sort) test "${#archives[@]}" -gt 0 mapfile -t bundles < <(find mcpbs -maxdepth 1 -type f -name '*.mcpb' -print | sort) test "${#bundles[@]}" -gt 0 + mapfile -t inventories < <(find inventories -maxdepth 1 -type f -name '*.spdx.json' -print | sort) + test "${#inventories[@]}" -gt 0 # `install.sh` ships with the prerelease as it does with a stable # release. README tells users to curl the installer from a release, # and only release.yml published one, so the hardened script on # master reached nobody following the beta line. The checkout is # pinned to the tag's source sha, so this is that tag's installer. - release_assets=("${archives[@]}" "${bundles[@]}" SHA256SUMS install.sh) + release_assets=("${archives[@]}" "${bundles[@]}" "${inventories[@]}" SHA256SUMS install.sh) gh api "repos/${{ github.repository }}/releases/tags/${RELEASE_TAG}" \ --jq '.assets[].name' > remote-asset-names for release_asset in "${release_assets[@]}"; do @@ -294,7 +379,7 @@ jobs: # Verify by name, not by glob: a partial publish may have no .zip # (Windows) or no .tar.gz yet, and `gh release download --pattern` # fails on an empty match. - mkdir verified-binaries verified-mcpbs + mkdir verified-binaries verified-mcpbs verified-inventories gh api "repos/${{ github.repository }}/releases/tags/${RELEASE_TAG}" \ --jq '.assets[].name' > remote-asset-names while IFS= read -r name; do @@ -302,22 +387,36 @@ jobs: tracedecay-beta-"${RELEASE_TAG}"-*.mcpb) gh release download "$RELEASE_TAG" --repo "${{ github.repository }}" \ --pattern "$name" --dir verified-mcpbs ;; + tracedecay-beta-"${RELEASE_TAG}"-*.spdx.json) + gh release download "$RELEASE_TAG" --repo "${{ github.repository }}" \ + --pattern "$name" --dir verified-inventories ;; tracedecay-beta-"${RELEASE_TAG}"-*.tar.gz | tracedecay-beta-"${RELEASE_TAG}"-*.zip) gh release download "$RELEASE_TAG" --repo "${{ github.repository }}" \ --pattern "$name" --dir verified-binaries ;; esac done < remote-asset-names - python3 scripts/check-release-artifacts.py \ + python3 .release-automation/scripts/check-release-artifacts.py \ --manifest .github/release-targets.json \ --profile beta \ --tag "$RELEASE_TAG" \ --binaries verified-binaries \ - --mcpbs verified-mcpbs - scripts/verify-retained-release-assets.sh \ + --mcpbs verified-mcpbs \ + --inventories verified-inventories + for inventory in verified-inventories/*.spdx.json; do + platform="${inventory#verified-inventories/tracedecay-beta-${RELEASE_TAG}-}" + platform="${platform%.spdx.json}" + archive_format="$(jq -r --arg platform "$platform" \ + '.include[] | select(.name == $platform) | .archive' .github/release-targets.json)" + python3 .release-automation/scripts/generate-release-inventory.py verify \ + --inventory "$inventory" \ + --archive "verified-binaries/tracedecay-beta-${RELEASE_TAG}-${platform}.${archive_format}" \ + --mcpb "verified-mcpbs/tracedecay-beta-${RELEASE_TAG}-${platform}.mcpb" + done + .release-automation/scripts/verify-retained-release-assets.sh \ --tag "$RELEASE_TAG" \ --repo "${{ github.repository }}" \ --signer-workflow "${{ github.repository }}/.github/workflows/release-beta.yml" \ --source-digest "${{ needs.validate.outputs.source_sha }}" \ --signer-ref "refs/tags/${RELEASE_TAG}" \ --signer-ref "refs/heads/master" \ - --files verified-binaries/* verified-mcpbs/* + --files verified-binaries/* verified-mcpbs/* verified-inventories/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6272427c18..e6cac508dc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,6 +40,7 @@ jobs: prerelease: ${{ steps.release.outputs.prerelease }} matrix: ${{ steps.release.outputs.matrix }} build_required: ${{ steps.release.outputs.build_required }} + package_build_required: ${{ steps.release.outputs.package_build_required }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -47,6 +48,17 @@ jobs: ref: ${{ env.RELEASE_TAG }} persist-credentials: false + - name: Checkout release recovery planner + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.workflow_sha }} + path: .release-automation + persist-credentials: false + sparse-checkout-cone-mode: false + sparse-checkout: | + scripts/plan-release-recovery.py + scripts/verify-retained-release-assets.sh + - name: Validate stable release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -85,7 +97,7 @@ jobs: } >> "$GITHUB_OUTPUT" gh api "repos/${{ github.repository }}/releases/tags/${tag}" \ --jq '.assets[].name' > remote-asset-names - python3 scripts/plan-release-recovery.py \ + python3 .release-automation/scripts/plan-release-recovery.py \ --manifest .github/release-targets.json \ --tag "$tag" \ --profile stable \ @@ -93,7 +105,7 @@ jobs: --retained-output retained-asset-names \ --github-output "$GITHUB_OUTPUT" mkdir retained-assets - scripts/verify-retained-release-assets.sh \ + .release-automation/scripts/verify-retained-release-assets.sh \ --tag "$tag" \ --repo "${{ github.repository }}" \ --signer-workflow "${{ github.repository }}/.github/workflows/release.yml" \ @@ -123,9 +135,27 @@ jobs: with: ref: ${{ needs.validate-release.outputs.source_sha }} + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.15.0" + + - name: Install the locked-inventory YAML reader + run: python3 -m pip install pyyaml==6.0.2 + + - name: Set up Bazel for inventory-only recovery + if: matrix.inventory_only == 'true' + uses: ./.github/actions/setup-bazel + with: + cache-key: release-${{ matrix.target }} + + - name: Build dashboard bundle for inventory-only recovery + if: matrix.inventory_only == 'true' + run: bazel build --config=dist --config=ci //dashboard:app_dist + # Bazel builds and validates //dashboard:app_dist, and the CLI build # script refuses to embed it unless it matches //dashboard:bundle_sha256. - name: Build release binary for packaging + if: matrix.inventory_only != 'true' id: cli uses: ./.github/actions/build-cli with: @@ -135,6 +165,7 @@ jobs: # verify-release requires every target to embed the same bundle. - name: Record the embedded dashboard digest + if: matrix.inventory_only != 'true' shell: bash run: | set -euo pipefail @@ -144,6 +175,7 @@ jobs: "dashboard-sha256-${{ matrix.name }}.txt" - name: Upload embedded dashboard digest + if: matrix.inventory_only != 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: dashboard-sha256-${{ matrix.name }} @@ -159,9 +191,12 @@ jobs: ref: ${{ github.workflow_sha }} path: .release-automation persist-credentials: false - sparse-checkout: .github/actions/package-release + sparse-checkout: | + .github/actions/package-release + scripts/generate-release-inventory.py - name: Package and verify native archive and MCPB + if: matrix.inventory_only != 'true' uses: ./.release-automation/.github/actions/package-release with: binary: ${{ steps.cli.outputs.path }} @@ -171,7 +206,29 @@ jobs: archive: ${{ matrix.archive }} epoch: ${{ needs.validate-release.outputs.source_epoch }} + - name: Record shipping component scope + shell: bash + env: + PLATFORM: ${{ matrix.name }} + SOURCE_SHA: ${{ needs.validate-release.outputs.source_sha }} + run: | + set -euo pipefail + python3 .release-automation/scripts/generate-release-inventory.py scope \ + --source-root . \ + --platform "$PLATFORM" \ + --source-sha "$SOURCE_SHA" \ + --dashboard-dist "$(bazel info --config=dist --config=ci bazel-bin)/dashboard/app-dist" \ + --output "inventory-scope-${PLATFORM}.json" + + - name: Upload component scope + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: inventory-scope-${{ matrix.name }} + path: inventory-scope-${{ matrix.name }}.json + if-no-files-found: error + - name: Upload MCPB artifact + if: matrix.inventory_only != 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: mcpb-${{ matrix.name }} @@ -179,6 +236,7 @@ jobs: if-no-files-found: error - name: Upload binary archive artifact + if: matrix.inventory_only != 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: binary-${{ matrix.name }} @@ -214,7 +272,7 @@ jobs: merge-multiple: true - name: Require one embedded dashboard across targets - if: needs.validate-release.outputs.build_required == 'true' + if: needs.validate-release.outputs.package_build_required == 'true' run: scripts/check-dashboard-digests.sh dashboard-digests - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable @@ -264,8 +322,17 @@ jobs: ref: ${{ needs.validate-release.outputs.source_sha }} persist-credentials: false - - name: Download newly built binary artifacts + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 if: needs.validate-release.outputs.build_required == 'true' + with: + python-version: "3.15.0" + + - name: Install the locked-inventory YAML reader + if: needs.validate-release.outputs.build_required == 'true' + run: python3 -m pip install pyyaml==6.0.2 + + - name: Download newly built binary artifacts + if: needs.validate-release.outputs.package_build_required == 'true' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: binary-* @@ -273,13 +340,21 @@ jobs: merge-multiple: true - name: Download newly built MCPB artifacts - if: needs.validate-release.outputs.build_required == 'true' + if: needs.validate-release.outputs.package_build_required == 'true' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: pattern: mcpb-* path: new-mcpbs merge-multiple: true + - name: Download shipping component scopes + if: needs.validate-release.outputs.build_required == 'true' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: inventory-scope-* + path: new-inventory-scopes + merge-multiple: true + # Use the helpers from the running workflow revision for older-tag recovery. - name: Checkout release recovery helpers uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -290,7 +365,11 @@ jobs: sparse-checkout-cone-mode: false sparse-checkout: | scripts/assemble-release-artifacts.sh + scripts/check-release-artifacts.py + scripts/generate-release-inventory.py + scripts/plan-release-recovery.py scripts/update-release-checksums.py + scripts/verify-retained-release-assets.sh - name: Assemble exact retained and newly built artifacts env: @@ -313,6 +392,8 @@ jobs: bundles=(new-mcpbs/*.mcpb) echo "archives=${#archives[@]}" >>"$GITHUB_OUTPUT" echo "bundles=${#bundles[@]}" >>"$GITHUB_OUTPUT" + inventories=(new-inventories/*.spdx.json) + echo "inventories=${#inventories[@]}" >>"$GITHUB_OUTPUT" - name: Attest newly built binary archives if: steps.built.outputs.archives != '0' @@ -326,6 +407,12 @@ jobs: with: subject-path: new-mcpbs/*.mcpb + - name: Attest newly generated component inventories + if: steps.built.outputs.inventories != '0' + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-path: new-inventories/*.spdx.json + - name: Upload validated artifacts to release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -334,7 +421,8 @@ jobs: shopt -s nullglob binaries=(binaries/tracedecay-*) bundles=(mcpbs/*.mcpb) - assets=("${binaries[@]}" "${bundles[@]}") + inventories=(inventories/*.spdx.json) + assets=("${binaries[@]}" "${bundles[@]}" "${inventories[@]}") ( cd binaries sha256sum tracedecay-* @@ -343,6 +431,10 @@ jobs: cd mcpbs sha256sum -- *.mcpb ) >> SHA256SUMS + ( + cd inventories + sha256sum -- *.spdx.json + ) >> SHA256SUMS assets+=(SHA256SUMS install.sh) TAG="${{ needs.validate-release.outputs.tag }}" SOURCE_SHA="${{ needs.validate-release.outputs.source_sha }}" @@ -388,7 +480,7 @@ jobs: # Verify by name, not by glob: a partial publish may have no .zip # or no .tar.gz yet, and `gh release download --pattern` fails on # an empty match. - mkdir -p remote-assets/{binaries,mcpbs} + mkdir -p remote-assets/{binaries,mcpbs,inventories} gh api "repos/${{ github.repository }}/releases/tags/${TAG}" \ --jq '.assets[].name' > remote-asset-names while IFS= read -r name; do @@ -396,24 +488,38 @@ jobs: tracedecay-"${TAG}"-*.mcpb) gh release download "$TAG" --repo "${{ github.repository }}" \ --pattern "$name" --dir remote-assets/mcpbs ;; + tracedecay-"${TAG}"-*.spdx.json) + gh release download "$TAG" --repo "${{ github.repository }}" \ + --pattern "$name" --dir remote-assets/inventories ;; tracedecay-"${TAG}"-*.tar.gz | tracedecay-"${TAG}"-*.zip) gh release download "$TAG" --repo "${{ github.repository }}" \ --pattern "$name" --dir remote-assets/binaries ;; esac done < remote-asset-names - python3 scripts/check-release-artifacts.py \ + python3 .release-automation/scripts/check-release-artifacts.py \ --manifest .github/release-targets.json \ --tag "$TAG" \ --binaries remote-assets/binaries \ - --mcpbs remote-assets/mcpbs - scripts/verify-retained-release-assets.sh \ + --mcpbs remote-assets/mcpbs \ + --inventories remote-assets/inventories + for inventory in remote-assets/inventories/*.spdx.json; do + platform="${inventory#remote-assets/inventories/tracedecay-${TAG}-}" + platform="${platform%.spdx.json}" + archive_format="$(jq -r --arg platform "$platform" \ + '.include[] | select(.name == $platform) | .archive' .github/release-targets.json)" + python3 .release-automation/scripts/generate-release-inventory.py verify \ + --inventory "$inventory" \ + --archive "remote-assets/binaries/tracedecay-${TAG}-${platform}.${archive_format}" \ + --mcpb "remote-assets/mcpbs/tracedecay-${TAG}-${platform}.mcpb" + done + .release-automation/scripts/verify-retained-release-assets.sh \ --tag "$TAG" \ --repo "${{ github.repository }}" \ --signer-workflow "${{ github.repository }}/.github/workflows/release.yml" \ --source-digest "$SOURCE_SHA" \ --signer-ref "refs/tags/${TAG}" \ --signer-ref "refs/heads/master" \ - --files remote-assets/binaries/* remote-assets/mcpbs/* + --files remote-assets/binaries/* remote-assets/mcpbs/* remote-assets/inventories/* - name: Mark stable release as latest if: needs.validate-release.outputs.prerelease == 'false' diff --git a/SECURITY.md b/SECURITY.md index 2d0d2dce1d..febdad5b74 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -143,7 +143,20 @@ consistency, not independent publisher authenticity. Other distribution/update paths have their own verification; do not assume installer checks apply to all paths. -**Windows:** Authenticode code signing via the [SignPath.io Foundation](https://signpath.io/foundation) program is being rolled out so Windows binaries are signed as part of the release workflow (addresses the Smart App Control block reported in #79). Until that lands in a published release, Windows binaries remain unsigned. +**Release component inventory:** Each native archive and MCPB is accompanied by a +platform-specific SPDX 2.3 inventory asset. The inventory names and SHA-256 hashes +both final package files; `SHA256SUMS` and the release build-provenance attestation +cover the downloaded inventory asset itself. `NOASSERTION` licenses or unknown +sources are explicitly annotated for review. The inventory is a traceability report, +not a license scan or a claim that all components have been legally reviewed. + +**Windows:** Windows release binaries are currently unsigned. The repository contains a +SignPath artifact-configuration example, but no release-workflow signing integration or +verified SignPath project/certificate provisioning. Signing is not enforced by the release +workflow; consumers must not assume an Authenticode signature or Smart App Control +acceptance. Enabling signing remains blocked on external signing-service and certificate +provisioning, followed by workflow and published-artifact verification. The previously cited +PR #79 was unrelated and does not provide evidence for a Smart App Control report. ### Opt-in background daemon diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index ad89936906..65049bcbd8 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -62,6 +62,8 @@ exports_files( "test-package-release-archive.py", "test-plan-release-recovery.py", "test-check-release-artifacts.py", + "generate-release-inventory.py", + "test-generate-release-inventory.py", "test-update-release-checksums.py", "bazel/gen_builds.py", "bazel/rust_test_environment_probe.txt", diff --git a/scripts/assemble-release-artifacts.sh b/scripts/assemble-release-artifacts.sh index 5afefa5521..f52853bfad 100644 --- a/scripts/assemble-release-artifacts.sh +++ b/scripts/assemble-release-artifacts.sh @@ -13,14 +13,15 @@ esac retained_dir="$(mktemp -d)" trap 'rm -rf -- "$retained_dir"' EXIT mkdir binaries mcpbs +mkdir inventories new-inventories gh api "repos/${repo}/releases/tags/${tag}" --jq '.assets[].name' > "$retained_dir/remote-names" -python3 scripts/plan-release-recovery.py \ +python3 .release-automation/scripts/plan-release-recovery.py \ --manifest .github/release-targets.json --tag "$tag" --profile "$profile" \ --asset-names "$retained_dir/remote-names" \ --retained-output "$retained_dir/names" --github-output "$retained_dir/planner-output" mkdir "$retained_dir/assets" -scripts/verify-retained-release-assets.sh \ +.release-automation/scripts/verify-retained-release-assets.sh \ --tag "$tag" --repo "$repo" --signer-workflow "$signer_workflow" \ --source-digest "$source_sha" \ --signer-ref "refs/tags/${tag}" --signer-ref refs/heads/master \ @@ -29,6 +30,8 @@ while IFS= read -r asset; do [[ -z "$asset" ]] && continue if [[ "$asset" == *.mcpb ]]; then cp "$retained_dir/assets/$asset" "mcpbs/$asset" + elif [[ "$asset" == *.spdx.json ]]; then + cp "$retained_dir/assets/$asset" "inventories/$asset" else cp "$retained_dir/assets/$asset" "binaries/$asset" fi @@ -43,13 +46,46 @@ if [[ "$build_required" == true ]]; then destination="mcpbs/$(basename "$artifact")" [[ -e "$destination" ]] || cp "$artifact" "$destination" done + for scope in new-inventory-scopes/*.json; do + [[ -f "$scope" ]] || continue + platform="$(jq -r '.platform' "$scope")" + archive_format="$(jq -r --arg platform "$platform" \ + '.include[] | select(.name == $platform) | .archive' .github/release-targets.json)" + archive="${prefix}-${tag}-${platform}.${archive_format}" + mcpb="${prefix}-${tag}-${platform}.mcpb" + inventory="${prefix}-${tag}-${platform}.spdx.json" + if [[ -e "inventories/$inventory" ]]; then + continue + fi + python3 .release-automation/scripts/generate-release-inventory.py document \ + --source-root . \ + --scope "$scope" \ + --archive "binaries/$archive" \ + --mcpb "mcpbs/$mcpb" \ + --tag "$tag" \ + --source-sha "$source_sha" \ + --source-epoch "$(git show -s --format=%ct "$source_sha")" \ + --repository "$repo" \ + --output "new-inventories/$inventory" + cp "new-inventories/$inventory" "inventories/$inventory" + done fi -python3 scripts/check-release-artifacts.py \ +python3 .release-automation/scripts/check-release-artifacts.py \ --manifest .github/release-targets.json --tag "$tag" --profile "$profile" \ - --binaries binaries --mcpbs mcpbs + --binaries binaries --mcpbs mcpbs --inventories inventories for bundle in mcpbs/*.mcpb; do platform="${bundle#mcpbs/"${prefix}-${tag}-"}" platform="${platform%.mcpb}" python3 scripts/build-mcpb.py verify \ --bundle "$bundle" --version "${tag#v}" --platform "$platform" done +for inventory in inventories/*.spdx.json; do + platform="${inventory#inventories/${prefix}-${tag}-}" + platform="${platform%.spdx.json}" + archive_format="$(jq -r --arg platform "$platform" \ + '.include[] | select(.name == $platform) | .archive' .github/release-targets.json)" + python3 .release-automation/scripts/generate-release-inventory.py verify \ + --inventory "$inventory" \ + --archive "binaries/${prefix}-${tag}-${platform}.${archive_format}" \ + --mcpb "mcpbs/${prefix}-${tag}-${platform}.mcpb" +done diff --git a/scripts/check-release-artifacts.py b/scripts/check-release-artifacts.py index abc8b15f23..dd4d7424c1 100644 --- a/scripts/check-release-artifacts.py +++ b/scripts/check-release-artifacts.py @@ -57,6 +57,7 @@ def main() -> int: parser.add_argument("--binaries", type=Path, required=True) parser.add_argument("--profile", choices=("stable", "beta"), default="stable") parser.add_argument("--mcpbs", type=Path) + parser.add_argument("--inventories", type=Path) arguments = parser.parse_args() targets = target_matrix(arguments.manifest) prefix = "tracedecay-beta" if arguments.profile == "beta" else "tracedecay" @@ -68,6 +69,10 @@ def main() -> int: target["name"]: f"{prefix}-{arguments.tag}-{target['name']}.mcpb" for target in targets } + inventory_name = { + target["name"]: f"{prefix}-{arguments.tag}-{target['name']}.spdx.json" + for target in targets + } if arguments.mcpbs is None: raise SystemExit("release validation requires an MCPB directory") binaries = files(arguments.binaries) @@ -75,6 +80,9 @@ def main() -> int: require_exact("binary", binaries, set(binary_name.values())) require_exact("MCPB", mcpbs, set(mcpb_name.values())) + if arguments.inventories is not None: + inventories = files(arguments.inventories) + require_exact("component inventory", inventories, set(inventory_name.values())) print("release artifact coverage matches target manifest") return 0 diff --git a/scripts/generate-release-inventory.py b/scripts/generate-release-inventory.py index f8f217f363..2b38cf850b 100644 --- a/scripts/generate-release-inventory.py +++ b/scripts/generate-release-inventory.py @@ -37,6 +37,125 @@ def canonical_json(value: object) -> bytes: return (json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n").encode() +def source_tree_digest(path: Path) -> str: + digest = hashlib.sha256() + for file_path in sorted(item for item in path.rglob("*") if item.is_file()): + relative = file_path.relative_to(path).as_posix().encode() + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + digest.update(bytes.fromhex(sha256(file_path))) + return digest.hexdigest() + + +def embedded_source_components( + source_root: Path, rust_components: list[dict[str, object]] +) -> list[dict[str, object]]: + local_owners = { + str(component["local_path"]) + for component in rust_components + if str(component.get("local_path", "")).startswith("crates/") + } + runtime_names = {str(component["name"]) for component in rust_components} + components: list[dict[str, object]] = [] + for owner in sorted(local_owners): + for provenance in sorted((source_root / owner).rglob("vendor/**/PROVENANCE.md")): + vendored = provenance.parent + text = provenance.read_text(encoding="utf-8") + repository = re.search( + r"(?im)^\|\s*upstream (?:repository|URL)\s*\|\s*<([^>]+)>", + text, + ) + license_match = re.search( + r"(?im)^\|\s*licen[cs]e\s*\|\s*([^|]+)\|", text + ) + license_value = ( + license_match.group(1).split("—", 1)[0].strip() + if license_match + else "NOASSERTION" + ) + version_match = re.search( + r"(?im)^\|\s*upstream release current at retrieval\s*\|\s*`?([^|`]+)", + text, + ) + if version_match is None: + version_match = re.search( + r"(?i)crates\.io package `[^`]+`\s+([0-9][^ )|]*)", text + ) + version = version_match.group(1).strip() if version_match else "NOASSERTION" + package_checksum = re.search( + r"(?im)^\|\s*package sha256\s*\|\s*`?([0-9a-f]{64})", text + ) + source_digest = source_tree_digest(vendored) + name = vendored.name + purl = f"pkg:generic/{quote(name, safe='')}" + if version != "NOASSERTION": + purl += f"@{quote(version, safe='')}" + components.append( + { + "name": name, + "version": version, + "source": repository.group(1) if repository else "NOASSERTION", + "downloadLocation": repository.group(1) if repository else "NOASSERTION", + "sourceInfo": ( + f"Vendored at {vendored.relative_to(source_root).as_posix()}; " + f"provenance {provenance.relative_to(source_root).as_posix()}; " + f"source tree SHA256 {source_digest}" + ), + "license": license_value, + "license_evidence": provenance.relative_to(source_root).as_posix(), + "checksum": ( + { + "algorithm": "SHA256", + "checksumValue": package_checksum.group(1), + } + if package_checksum + else { + "algorithm": "SHA256", + "checksumValue": source_digest, + } + ), + "purl": purl, + } + ) + + notice_path = source_root / "NOTICE" + memory_path = source_root / "crates/tracedecay-session-memory/src/memory/diff.rs" + if ( + "tracedecay-session-memory" in runtime_names + and notice_path.is_file() + and memory_path.is_file() + ): + notice = notice_path.read_text(encoding="utf-8") + source_url = re.search(r"(?im)^mnemon\s+\((https?://[^)]+)\)", notice) + license_name = ( + "Apache-2.0" + if re.search(r"(?i)Apache License, Version 2\.0", notice) + else "NOASSERTION" + ) + purl = "pkg:generic/mnemon-adapted-content-cues" + components.append( + { + "name": "mnemon adapted content cues", + "version": "NOASSERTION", + "source": source_url.group(1) if source_url else "NOASSERTION", + "downloadLocation": source_url.group(1) if source_url else "NOASSERTION", + "sourceInfo": ( + "Adaptation identified in NOTICE at " + "crates/tracedecay-session-memory/src/memory/diff.rs; " + "the upstream revision is not recorded." + ), + "license": license_name, + "license_evidence": "NOTICE", + "checksum": { + "algorithm": "SHA256", + "checksumValue": sha256(memory_path), + }, + "purl": purl, + } + ) + return components + + def label_name(label: str) -> str: label = label.split(" (", 1)[0] return label.strip('"') @@ -105,12 +224,14 @@ def parse_graph(graph: str, kinds: str) -> list[dict[str, str]]: if kind in {"cargo_build_script", "rust_proc_macro", "rust_test"}: continue if kind == "rust_library": - if current.startswith("@@") or current.startswith("@"): + if current.startswith("@"): included.add(current) elif current.startswith("//crates/") or current.startswith("//patches/"): included.add(current) pending.extend(adjacency.get(current, ())) + if root not in visited or not included: + raise ValueError("Bazel query did not expose the configured CLI Rust dependency graph") return [{"label": label, "kind": target_kinds.get(label, "")} for label in sorted(included)] @@ -135,28 +256,33 @@ def cargo_package( by_repo, by_name = lock_packages if label.startswith("@"): repository = label.split("//", 1)[0].lstrip("@") - repository = repository.split("+", 1)[0] if repository.startswith("c+") else repository package = by_repo.get(repository) + matched_repository = repository if package is not None else "" if package is None: # Canonical repository labels may include a module-version suffix. - package = next( - ( - value - for key, value in by_repo.items() - if repository.endswith(key.removeprefix("c__")) - ), - None, - ) + matches = [ + (key, value) + for key, value in by_repo.items() + if repository.endswith(key) + or repository.endswith(key.removeprefix("c__")) + ] + if matches: + matched_repository, package = max(matches, key=lambda item: len(item[0])) if package is None: + unknown_name = repository.split("+")[-1].removeprefix("c__") return { - "name": repository, + "name": unknown_name, "version": "NOASSERTION", "source": "NOASSERTION", + "downloadLocation": "NOASSERTION", "license": "NOASSERTION", "checksum": None, "local_path": None, + "purl": f"pkg:cargo/{quote(unknown_name, safe='')}@NOASSERTION", } manifest = output_base / "external" / repository / "Cargo.toml" + if not manifest.is_file(): + manifest = output_base / "external" / matched_repository / "Cargo.toml" source = str(package.get("source", "NOASSERTION")) return package_record(package, manifest, source_root, source) @@ -216,11 +342,13 @@ def package_record( record["local_path"] = local_path record["source"] = f"{local_path} at the release source revision" record["downloadLocation"] = "NOASSERTION" + record["source_digest"] = source_tree_digest(source_root / local_path) return record -def component_id(purl: str) -> str: - return "SPDXRef-" + hashlib.sha256(purl.encode()).hexdigest()[:24] +def component_id(purl: str, source: object) -> str: + identity = f"{purl}\0{source or ''}" + return "SPDXRef-" + hashlib.sha256(identity.encode()).hexdigest()[:24] def checksum_from_sri(integrity: str) -> dict[str, str] | None: @@ -239,15 +367,28 @@ def checksum_from_sri(integrity: str) -> dict[str, str] | None: def npm_component_key(name: str, version: str, package_keys: set[str]) -> str | None: + exact = f"{name}@{version.split('(', 1)[0]}" + return exact if exact in package_keys else None + + +def npm_snapshot_key(name: str, version: str, snapshot_keys: set[str]) -> str | None: exact = f"{name}@{version}" - if exact in package_keys: + if exact in snapshot_keys: return exact - candidates = sorted( - key - for key in package_keys - if key.startswith(exact + "(") - ) - return candidates[0] if candidates else None + base = f"{name}@{version.split('(', 1)[0]}" + candidates = sorted(key for key in snapshot_keys if key.startswith(base + "(")) + return candidates[0] if candidates else (base if base in snapshot_keys else None) + + +def npm_package_identity(key: str) -> tuple[str, str]: + delimiter = key.find("@", 1 if key.startswith("@") else 0) + if delimiter < 0: + raise ValueError(f"invalid pnpm package key: {key}") + name = key[:delimiter] + version = key[delimiter + 1 :].split("(", 1)[0] + if not name or not version: + raise ValueError(f"invalid pnpm package key: {key}") + return name, version def npm_components(source_root: Path, platform: str) -> list[dict[str, object]]: @@ -256,12 +397,29 @@ def npm_components(source_root: Path, platform: str) -> list[dict[str, object]]: except ImportError as error: raise RuntimeError("PyYAML is required to read the locked dashboard package graph") from error - lock = yaml.safe_load((source_root / "pnpm-lock.yaml").read_text(encoding="utf-8")) + documents = yaml.safe_load_all( + (source_root / "pnpm-lock.yaml").read_text(encoding="utf-8") + ) + lock = next( + ( + document + for document in reversed(list(documents)) + if "dashboard" in document.get("importers", {}) + ), + None, + ) + if lock is None: + raise ValueError("pnpm lockfile has no dashboard importer") + return npm_components_from_lock(lock, platform) + + +def npm_components_from_lock(lock: dict[str, object], platform: str) -> list[dict[str, object]]: dashboard = lock.get("importers", {}).get("dashboard", {}) roots = dashboard.get("dependencies", {}) packages = lock.get("packages", {}) snapshots = lock.get("snapshots", {}) package_keys = set(packages) + snapshot_keys = set(snapshots) os_name, arch = { "aarch64-macos": ("darwin", "arm64"), "x86_64-linux": ("linux", "x64"), @@ -269,50 +427,80 @@ def npm_components(source_root: Path, platform: str) -> list[dict[str, object]]: "x86_64-windows": ("win32", "x64"), }[platform] - pending: list[str] = [] + pending: list[tuple[str, str]] = [] for name, record in roots.items(): version = record.get("version") if version: - key = npm_component_key(name, str(version), package_keys) - if key: - pending.append(key) + package_key = npm_component_key(name, str(version), package_keys) + snapshot_key = npm_snapshot_key(name, str(version), snapshot_keys) + if package_key and snapshot_key: + pending.append((package_key, snapshot_key)) - visited: set[str] = set() + visited_packages: set[str] = set() + visited_snapshots: set[str] = set() result: list[dict[str, object]] = [] while pending: - key = pending.pop() - if key in visited: - continue - visited.add(key) - metadata = packages.get(key, {}) - version = key.rsplit("@", 1)[-1].split("(", 1)[0] - name = key[: -(len(key.rsplit("@", 1)[-1]) + 1)] - if not name: + package_key, snapshot_key = pending.pop() + if snapshot_key in visited_snapshots: continue + visited_snapshots.add(snapshot_key) + metadata = packages.get(package_key, {}) + name, version = npm_package_identity(package_key) + resolution = metadata.get("resolution", {}) + source = resolution.get("tarball") or "NOASSERTION" package = { "name": name, "version": version, - "source": metadata.get("resolution", {}).get("tarball") - or f"https://registry.npmjs.org/{quote(name, safe='@/')}/-/{name.rsplit('/', 1)[-1]}-{version}.tgz", + "source": source, + "downloadLocation": source, + "sourceInfo": ( + f"pnpm-lock.yaml package {package_key}; " + f"integrity {resolution.get('integrity', 'NOASSERTION')}; " + "source URL is not recorded in the lockfile" + ), "license": FONT_LICENSES.get(name, "NOASSERTION"), + "license_evidence": ( + "dashboard/src/theme/fonts.css" if name in FONT_LICENSES else None + ), "checksum": checksum_from_sri( - str(metadata.get("resolution", {}).get("integrity", "")) + str(resolution.get("integrity", "")) ), - "purl": f"pkg:npm/{quote(name, safe='@/') }@{quote(version, safe='')}", + "purl": f"pkg:npm/{quote(name, safe='@/')}@{quote(version, safe='')}", } - result.append(package) - node = snapshots.get(key, {}) + if package_key not in visited_packages: + result.append(package) + visited_packages.add(package_key) + node = snapshots.get(snapshot_key, {}) for group in ("dependencies", "optionalDependencies"): for dependency, child_version in node.get(group, {}).items(): - child = npm_component_key(dependency, str(child_version), package_keys) - if child is None: + child_package = npm_component_key( + dependency, str(child_version), package_keys + ) + child_snapshot = npm_snapshot_key( + dependency, str(child_version), snapshot_keys + ) + if child_package is None or child_snapshot is None: continue - constraints = packages.get(child, {}) - if constraints.get("os") and os_name not in constraints["os"] and "!" + os_name not in constraints["os"]: + constraints = packages.get(child_package, {}) + allowed_os = constraints.get("os", []) + if allowed_os and ( + os_name in {item[1:] for item in allowed_os if item.startswith("!")} + or ( + any(not item.startswith("!") for item in allowed_os) + and os_name not in allowed_os + ) + ): continue - if constraints.get("cpu") and arch not in constraints["cpu"] and "!" + arch not in constraints["cpu"]: + allowed_cpu = constraints.get("cpu", []) + if allowed_cpu and ( + arch in {item[1:] for item in allowed_cpu if item.startswith("!")} + or ( + any(not item.startswith("!") for item in allowed_cpu) + and arch not in allowed_cpu + ) + ): continue - pending.append(child) + pending.append((child_package, child_snapshot)) return sorted(result, key=lambda component: str(component["purl"])) @@ -375,7 +563,14 @@ def scope_inventory( if module_file.is_file(): module_text = module_file.read_text(encoding="utf-8") patch_paths = sorted(set(re.findall(r'"//scripts:([^"]+\.patch)"', module_text))) + git_packages = { + str(component["name"]) + for component in rust_components.values() + if str(component.get("source", "")).startswith("git+") + } for relative in patch_paths: + if not any(Path(relative).name.startswith(f"{name}-") for name in git_packages): + continue path = source_root / "scripts" / relative if path.is_file(): adapted_sources.append( @@ -390,6 +585,9 @@ def scope_inventory( "bazel_target": TARGET, "source_sha": source_sha, "rust_components": [rust_components[key] for key in sorted(rust_components)], + "embedded_sources": embedded_source_components( + source_root, list(rust_components.values()) + ), "dashboard_assets": assets, "adapted_sources": adapted_sources, } @@ -409,12 +607,22 @@ def artifact_binary(archive: Path, platform: str) -> bytes: return content.read() +def mcpb_contents(bundle: Path, platform: str) -> tuple[bytes, bytes]: + member = "tracedecay.exe" if platform == "x86_64-windows" else "tracedecay" + with zipfile.ZipFile(bundle) as packaged: + expected = {"manifest.json", f"server/{member}"} + if set(packaged.namelist()) != expected: + raise ValueError(f"MCPB does not contain the expected entries: {bundle.name}") + return packaged.read("manifest.json"), packaged.read(f"server/{member}") + + def spdx_package(record: dict[str, object]) -> dict[str, object]: purl = str(record["purl"]) + source = record.get("source") license_value = str(record.get("license") or "NOASSERTION") package: dict[str, object] = { "name": record["name"], - "SPDXID": component_id(purl), + "SPDXID": component_id(purl, source), "versionInfo": str(record.get("version") or "NOASSERTION"), "downloadLocation": str(record.get("downloadLocation") or record.get("source") or "NOASSERTION"), "filesAnalyzed": False, @@ -434,9 +642,19 @@ def spdx_package(record: dict[str, object]) -> dict[str, object]: package["checksums"] = [checksum] elif isinstance(checksum, str) and checksum: package["checksums"] = [{"algorithm": "SHA256", "checksumValue": checksum}] - source_info = record.get("source") + source_info = record.get("sourceInfo") or record.get("source") if source_info and source_info != package["downloadLocation"]: package["sourceInfo"] = str(source_info) + if record.get("source_digest"): + package["sourceInfo"] = ( + str(package.get("sourceInfo", "")) + + f"; local source tree SHA256 {record['source_digest']}" + ).lstrip("; ") + if record.get("license_evidence"): + package["sourceInfo"] = ( + str(package.get("sourceInfo", "")) + + f"; license metadata recorded in {record['license_evidence']}" + ).lstrip("; ") return package @@ -453,13 +671,25 @@ def release_inventory( ) -> None: scope = json.loads(scope_path.read_text(encoding="utf-8")) platform = str(scope["platform"]) + if scope.get("format") != "tracedecay-release-component-scope-v1": + raise ValueError("component scope has an unsupported format") + if scope.get("source_sha") != source_sha: + raise ValueError("component scope does not match the immutable release source") + if not archive.is_file() or not mcpb.is_file(): + raise FileNotFoundError("final archive and MCPB are required to generate an inventory") + binary_bytes = artifact_binary(archive, platform) + manifest_bytes, bundled_binary = mcpb_contents(mcpb, platform) + if binary_bytes != bundled_binary: + raise ValueError("native archive and MCPB do not contain identical CLI binaries") npm = npm_components(source_root, platform) - components = [*scope["rust_components"], *npm] + components = [*scope["rust_components"], *scope.get("embedded_sources", []), *npm] packages = [spdx_package(item) for item in components] package_ids = {str(package["SPDXID"]) for package in packages} archive_package_id = "SPDXRef-Archive" mcpb_package_id = "SPDXRef-MCPB" binary_id = "SPDXRef-CLI-Binary" + with (source_root / "crates/tracedecay-cli/Cargo.toml").open("rb") as source_file: + product_manifest = tomllib.load(source_file).get("package", {}) asset_package = lambda name, path, identifier: { "name": name, "SPDXID": identifier, @@ -467,8 +697,15 @@ def release_inventory( "downloadLocation": f"https://github.com/{repository}/releases/download/{quote(tag, safe='')}/{quote(path.name, safe='.-_')}", "filesAnalyzed": False, "licenseConcluded": "NOASSERTION", - "licenseDeclared": "NOASSERTION", + "licenseDeclared": product_manifest.get("license") or "NOASSERTION", "copyrightText": "NOASSERTION", + "sourceInfo": ( + f"Root LICENSE SHA256 {sha256(source_root / 'LICENSE')}; " + f"root NOTICE SHA256 {sha256(source_root / 'NOTICE')}" + if (source_root / "LICENSE").is_file() + and (source_root / "NOTICE").is_file() + else "Root LICENSE or NOTICE file is unavailable" + ), "checksums": [{"algorithm": "SHA256", "checksumValue": sha256(path)}], } packages.extend( @@ -482,14 +719,12 @@ def release_inventory( "downloadLocation": f"https://github.com/{repository}", "filesAnalyzed": False, "licenseConcluded": "NOASSERTION", - "licenseDeclared": "MIT", + "licenseDeclared": product_manifest.get("license") or "NOASSERTION", "copyrightText": "NOASSERTION", "checksums": [ { "algorithm": "SHA256", - "checksumValue": hashlib.sha256( - artifact_binary(archive, platform) - ).hexdigest(), + "checksumValue": hashlib.sha256(binary_bytes).hexdigest(), } ], }, @@ -519,6 +754,33 @@ def release_inventory( } ) + manifest_file_id = "SPDXRef-MCPB-Manifest" + manifest_entry = { + "fileName": "mcpb/manifest.json", + "SPDXID": manifest_file_id, + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": hashlib.sha256(manifest_bytes).hexdigest(), + } + ], + "licenseConcluded": "NOASSERTION", + "licenseInfoInFiles": ["NOASSERTION"], + "copyrightText": "NOASSERTION", + } + file_entries.append(manifest_entry) + patch_file_entries = [ + { + "fileName": str(item["path"]), + "SPDXID": "SPDXRef-Adapted-" + hashlib.sha256(str(item["path"]).encode()).hexdigest()[:20], + "checksums": [{"algorithm": "SHA256", "checksumValue": item["sha256"]}], + "licenseConcluded": "NOASSERTION", + "licenseInfoInFiles": ["NOASSERTION"], + "copyrightText": "NOASSERTION", + } + for item in scope.get("adapted_sources", []) + ] + file_entries.extend(patch_file_entries) relationships = [ {"spdxElementId": archive_package_id, "relationshipType": "CONTAINS", "relatedSpdxElement": binary_id}, {"spdxElementId": mcpb_package_id, "relationshipType": "CONTAINS", "relatedSpdxElement": binary_id}, @@ -532,7 +794,21 @@ def release_inventory( ] ) for item in file_entries: - for parent in (binary_id, archive_package_id, mcpb_package_id): + if item in patch_file_entries: + relationships.append( + { + "spdxElementId": binary_id, + "relationshipType": "GENERATED_FROM", + "relatedSpdxElement": item["SPDXID"], + } + ) + continue + parents = ( + (mcpb_package_id,) + if item["SPDXID"] == manifest_file_id + else (binary_id, archive_package_id, mcpb_package_id) + ) + for parent in parents: relationships.append( { "spdxElementId": parent, @@ -546,6 +822,12 @@ def release_inventory( if package.get("licenseDeclared") == "NOASSERTION" or package.get("downloadLocation") == "NOASSERTION" ) + review_items.extend( + str(item["fileName"]) + for item in file_entries + if "NOASSERTION" in item.get("licenseInfoInFiles", []) + ) + review_items.sort() annotations = [ { "annotationDate": datetime.fromtimestamp( @@ -557,17 +839,6 @@ def release_inventory( } for item in review_items ] - file_entries.extend( - { - "fileName": str(item["path"]), - "SPDXID": "SPDXRef-Adapted-" + hashlib.sha256(str(item["path"]).encode()).hexdigest()[:20], - "checksums": [{"algorithm": "SHA256", "checksumValue": item["sha256"]}], - "licenseConcluded": "NOASSERTION", - "licenseInfoInFiles": ["NOASSERTION"], - "copyrightText": "NOASSERTION", - } - for item in scope.get("adapted_sources", []) - ) document_name = f"TraceDecay {tag} {platform} component inventory" namespace = ( "https://spdx.org/spdxdocs/" @@ -604,6 +875,28 @@ def release_inventory( output.write_bytes(canonical_json(document)) +def verify_inventory(inventory: Path, archive: Path, mcpb: Path) -> None: + document = json.loads(inventory.read_text(encoding="utf-8")) + by_id = {package["SPDXID"]: package for package in document.get("packages", [])} + expected = { + "SPDXRef-Archive": archive, + "SPDXRef-MCPB": mcpb, + } + for identifier, path in expected.items(): + package = by_id.get(identifier) + if package is None or package.get("name") != path.name: + raise ValueError(f"inventory does not identify the expected artifact: {path.name}") + checksums = { + checksum.get("algorithm"): checksum.get("checksumValue") + for checksum in package.get("checksums", []) + } + if checksums.get("SHA256") != sha256(path): + raise ValueError(f"inventory digest does not match the final artifact: {path.name}") + platform = "x86_64-windows" if archive.name.endswith(".zip") else "x86_64-linux" + if artifact_binary(archive, platform) != mcpb_contents(mcpb, platform)[1]: + raise ValueError("native archive and MCPB do not contain identical CLI binaries") + + def main() -> None: parser = argparse.ArgumentParser() commands = parser.add_subparsers(dest="command", required=True) @@ -626,6 +919,10 @@ def main() -> None: document.add_argument("--source-epoch", type=int, required=True) document.add_argument("--repository", required=True) document.add_argument("--output", type=Path, required=True) + verify = commands.add_parser("verify") + verify.add_argument("--inventory", type=Path, required=True) + verify.add_argument("--archive", type=Path, required=True) + verify.add_argument("--mcpb", type=Path, required=True) args = parser.parse_args() if args.command == "scope": scope_inventory( @@ -638,7 +935,7 @@ def main() -> None: args.kinds, args.output_base, ) - else: + elif args.command == "document": release_inventory( args.source_root, args.scope, @@ -650,6 +947,8 @@ def main() -> None: args.repository, args.output, ) + else: + verify_inventory(args.inventory, args.archive, args.mcpb) if __name__ == "__main__": diff --git a/scripts/plan-release-recovery.py b/scripts/plan-release-recovery.py index ea88c4bb92..3b9bfd18c4 100644 --- a/scripts/plan-release-recovery.py +++ b/scripts/plan-release-recovery.py @@ -21,14 +21,11 @@ def target_assets( ) -> tuple[str, ...]: name = target["name"] archive = target["archive"] - if profile == "beta": - return ( - f"tracedecay-beta-{tag}-{name}.{archive}", - f"tracedecay-beta-{tag}-{name}.mcpb", - ) + prefix = "tracedecay-beta" if profile == "beta" else "tracedecay" return ( - f"tracedecay-{tag}-{name}.{archive}", - f"tracedecay-{tag}-{name}.mcpb", + f"{prefix}-{tag}-{name}.{archive}", + f"{prefix}-{tag}-{name}.mcpb", + f"{prefix}-{tag}-{name}.spdx.json", ) @@ -55,13 +52,28 @@ def plan( "unexpected existing release assets: " + ", ".join(unexpected) ) + missing_packages = { + target["name"]: any( + asset not in existing + for asset in target_assets(target, tag, profile)[:2] + ) + for target in targets + } missing_targets = [ - target + { + **target, + **( + {"inventory_only": True} + if not missing_packages[target["name"]] + else {} + ), + } for target in targets - if any(asset not in existing for asset in target_assets(target, tag, profile)) + if missing_packages[target["name"]] + or target_assets(target, tag, profile)[2] not in existing ] finalized = sorted(existing & fixed) - if finalized and missing_targets: + if finalized and any(missing_packages[target["name"]] for target in missing_targets): raise SystemExit( "final release metadata exists before all immutable artifacts " f"({', '.join(finalized)}); refusing destructive recovery" @@ -92,9 +104,15 @@ def main() -> None: encoding="utf-8", ) matrix = json.dumps({"include": missing}, separators=(",", ":")) + package_build_required = any( + not target.get("inventory_only", False) for target in missing + ) with arguments.github_output.open("a", encoding="utf-8") as output: output.write(f"matrix={matrix}\n") output.write(f"build_required={'true' if missing else 'false'}\n") + output.write( + f"package_build_required={'true' if package_build_required else 'false'}\n" + ) if __name__ == "__main__": diff --git a/scripts/test-check-release-artifacts.py b/scripts/test-check-release-artifacts.py index 706e10bf82..af28c0dd24 100755 --- a/scripts/test-check-release-artifacts.py +++ b/scripts/test-check-release-artifacts.py @@ -28,6 +28,8 @@ def run(root: Path, profile: str = "stable") -> subprocess.CompletedProcess[str] str(root / "binaries"), "--mcpbs", str(root / "mcpbs"), + "--inventories", + str(root / "inventories"), ], capture_output=True, text=True, @@ -67,7 +69,7 @@ def main() -> int: ] } (root / "targets.json").write_text(json.dumps(manifest), encoding="utf-8") - for child in ("binaries", "mcpbs"): + for child in ("binaries", "mcpbs", "inventories"): (root / child).mkdir() expected = { "binaries": ( @@ -78,11 +80,22 @@ def main() -> int: "tracedecay-v1.2.3-linux.mcpb", "tracedecay-v1.2.3-windows.mcpb", ), + "inventories": ( + "tracedecay-v1.2.3-linux.spdx.json", + "tracedecay-v1.2.3-windows.spdx.json", + ), } for child, names in expected.items(): for name in names: (root / child / name).write_bytes(b"artifact") expect(root, 0, "release artifact coverage matches target manifest") + (root / "inventories" / expected["inventories"][0]).unlink() + expect( + root, + 1, + "component inventory coverage mismatch: missing tracedecay-v1.2.3-linux.spdx.json", + ) + (root / "inventories" / expected["inventories"][0]).write_bytes(b"inventory") (root / "mcpbs" / expected["mcpbs"][0]).unlink() expect(root, 1, "MCPB coverage mismatch: missing tracedecay-v1.2.3-linux.mcpb") (root / "mcpbs" / expected["mcpbs"][0]).write_bytes(b"artifact") @@ -92,6 +105,8 @@ def main() -> int: item.unlink() for item in (root / "mcpbs").iterdir(): item.unlink() + for item in (root / "inventories").iterdir(): + item.unlink() for target in manifest["include"]: (root / "binaries" / ( f"tracedecay-beta-v1.2.3-{target['name']}.{target['archive']}" @@ -99,6 +114,9 @@ def main() -> int: (root / "mcpbs" / ( f"tracedecay-beta-v1.2.3-{target['name']}.mcpb" )).write_bytes(b"artifact") + (root / "inventories" / ( + f"tracedecay-beta-v1.2.3-{target['name']}.spdx.json" + )).write_bytes(b"inventory") expect(root, 0, "release artifact coverage matches target manifest", profile="beta") (root / "binaries" / "tracedecay-beta-v1.2.3-linux.tar.gz").unlink() expect( @@ -118,7 +136,7 @@ def main() -> int: ) # An MCPB that leaked into the binaries directory (a `tracedecay-beta-*` # artifact glob did this) is foreign there. - for child in ("binaries", "mcpbs"): + for child in ("binaries", "mcpbs", "inventories"): for item in (root / child).iterdir(): item.unlink() for target in manifest["include"]: @@ -128,6 +146,9 @@ def main() -> int: (root / "mcpbs" / ( f"tracedecay-beta-v1.2.3-{target['name']}.mcpb" )).write_bytes(b"artifact") + (root / "inventories" / ( + f"tracedecay-beta-v1.2.3-{target['name']}.spdx.json" + )).write_bytes(b"inventory") (root / "binaries" / "tracedecay-beta-v1.2.3-linux.mcpb").write_bytes(b"artifact") expect( root, diff --git a/scripts/test-generate-release-inventory.py b/scripts/test-generate-release-inventory.py index 5e1d1ee8e3..908a88b949 100644 --- a/scripts/test-generate-release-inventory.py +++ b/scripts/test-generate-release-inventory.py @@ -43,12 +43,32 @@ def main() -> None: (dashboard / "index.js").write_bytes(b"bundled npm") (dashboard / "archivo-latin.woff2").write_bytes(b"font bytes") (source / "crates/tracedecay-cli").mkdir(parents=True) - (source / "crates/tracedecay").mkdir(parents=True) + (source / "crates/tracedecay/vendor/sample-vendor").mkdir(parents=True) + (source / "crates/tracedecay-session-memory/src/memory").mkdir(parents=True) (source / "patches/tokensave-medium-treesitters").mkdir(parents=True) (source / "scripts/bazel/patches").mkdir(parents=True) - (source / "scripts/bazel/patches/rust.patch").write_text("adaptation\n") + (source / "LICENSE").write_text("MIT License\n", encoding="utf-8") + (source / "NOTICE").write_text( + "mnemon (https://github.com/mnemon-dev/mnemon)\n" + "Licensed under the Apache License, Version 2.0\n" + "Adapted from crates/tracedecay-session-memory/src/memory/diff.rs\n", + encoding="utf-8", + ) + (source / "crates/tracedecay-session-memory/src/memory/diff.rs").write_text( + "adapted cues\n", encoding="utf-8" + ) + (source / "crates/tracedecay/vendor/sample-vendor/PROVENANCE.md").write_text( + "| upstream repository | |\n" + "| licence | MIT — see LICENSE |\n" + "| upstream release current at retrieval | `v2.3.4` |\n", + encoding="utf-8", + ) + (source / "crates/tracedecay/vendor/sample-vendor/LICENSE").write_text( + "MIT License\n", encoding="utf-8" + ) + (source / "scripts/bazel/patches/asupersync-test.patch").write_text("adaptation\n") (source / "MODULE.bazel").write_text( - 'patches = ["//scripts:bazel/patches/rust.patch"]\n', encoding="utf-8" + 'patches = ["//scripts:bazel/patches/asupersync-test.patch"]\n', encoding="utf-8" ) write_package( source / "crates/tracedecay-cli/Cargo.toml", @@ -57,6 +77,12 @@ def main() -> None: "MIT", ) write_package(source / "crates/tracedecay/Cargo.toml", "tracedecay", "1.2.3", "MIT") + write_package( + source / "crates/tracedecay-session-memory/Cargo.toml", + "tracedecay-session-memory", + "1.2.3", + "MIT", + ) write_package( source / "patches/tokensave-medium-treesitters/Cargo.toml", "tokensave-medium-treesitters", @@ -74,7 +100,11 @@ def main() -> None: [[package]] name = "tracedecay" version = "1.2.3" -dependencies = ["asupersync 0.6.0 (git+https://github.com/vendor/asupersync?rev=abc#abc)", "serde 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)"] +dependencies = ["asupersync 0.6.0 (git+https://github.com/vendor/asupersync?rev=abc#abc)", "serde 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "tracedecay-session-memory"] + +[[package]] +name = "tracedecay-session-memory" +version = "1.2.3" [[package]] name = "asupersync" @@ -114,6 +144,7 @@ def main() -> None: '"//crates/tracedecay-cli:tracedecay" -> "//crates/tracedecay:tracedecay";', '"//crates/tracedecay:tracedecay" -> "@@c__serde-1.0.0//:serde";', '"//crates/tracedecay:tracedecay" -> "@@c__asupersync-0.6.0//:asupersync";', + '"//crates/tracedecay:tracedecay" -> "//crates/tracedecay-session-memory:tracedecay-session-memory";', '"//crates/tracedecay:tracedecay" -> "//patches/tokensave-medium-treesitters:tokensave-medium-treesitters";', '"//crates/tracedecay:tracedecay" -> "//scripts:build_script_build";', '"//scripts:build_script_build" -> "@@c__build-tool-1.0.0//:build_tool";', @@ -125,6 +156,7 @@ def main() -> None: kinds.write_text( """rust_binary rule //crates/tracedecay-cli:tracedecay rust_library rule //crates/tracedecay:tracedecay +rust_library rule //crates/tracedecay-session-memory:tracedecay-session-memory rust_library rule @@c__serde-1.0.0//:serde rust_library rule @@c__asupersync-0.6.0//:asupersync rust_library rule //patches/tokensave-medium-treesitters:tokensave-medium-treesitters @@ -151,37 +183,54 @@ def main() -> None: assert {"tracedecay", "serde", "asupersync", "tokensave-medium-treesitters"} <= set(rust) assert "build-tool" not in rust assert "https://github.com/vendor/asupersync" in rust["asupersync"]["downloadLocation"] - assert scoped["adapted_sources"][0]["path"] == "scripts/bazel/patches/rust.patch" - - (source / "pnpm-lock.yaml").write_text( - """lockfileVersion: '9.0' -importers: - dashboard: - dependencies: - '@fontsource-variable/archivo': - version: 5.3.0 - react: - version: 19.0.0 - devDependencies: - playwright: - version: 1.0.0 -packages: - '@fontsource-variable/archivo@5.3.0': - resolution: - integrity: sha512-YWJj - react@19.0.0: - resolution: - integrity: sha512-cmVhY3Q= - playwright@1.0.0: - resolution: - integrity: sha512-cGxheXdyaWdodA== -snapshots: - '@fontsource-variable/archivo@5.3.0': {} - react@19.0.0: {} - playwright@1.0.0: {} -""", - encoding="utf-8", + embedded = {component["name"]: component for component in scoped["embedded_sources"]} + assert embedded["sample-vendor"]["version"] == "v2.3.4" + assert embedded["sample-vendor"]["license"] == "MIT" + assert embedded["mnemon adapted content cues"]["license"] == "Apache-2.0" + assert embedded["mnemon adapted content cues"]["version"] == "NOASSERTION" + assert scoped["adapted_sources"][0]["path"] == ( + "scripts/bazel/patches/asupersync-test.patch" ) + + npm_lock = { + "importers": { + "dashboard": { + "dependencies": { + "@fontsource-variable/archivo": {"version": "5.3.0"}, + "react": {"version": "19.0.0"}, + "react-dom": {"version": "19.0.0(react@19.0.0)"}, + }, + "devDependencies": {"playwright": {"version": "1.0.0"}}, + } + }, + "packages": { + "@fontsource-variable/archivo@5.3.0": { + "resolution": {"integrity": "sha512-YWJj"} + }, + "react@19.0.0": { + "resolution": {"integrity": "sha512-cmVhY3Q="} + }, + "react-dom@19.0.0": { + "resolution": {"integrity": "sha512-cmVhY3Qy" + } + }, + "scheduler@0.23.2": { + "resolution": {"integrity": "sha512-c2NoZWR1bGVy"} + }, + "playwright@1.0.0": { + "resolution": {"integrity": "sha512-cGxheXdyaWdodA=="} + }, + }, + "snapshots": { + "@fontsource-variable/archivo@5.3.0": {}, + "react@19.0.0": {"dependencies": {"scheduler": "0.23.2"}}, + "react-dom@19.0.0(react@19.0.0)": { + "dependencies": {"react": "19.0.0"} + }, + "scheduler@0.23.2": {}, + "playwright@1.0.0": {}, + }, + } binary = b"release-binary" archive = root / "tracedecay-v1.2.3-x86_64-linux.tar.gz" make_archive(archive, binary) @@ -201,6 +250,22 @@ def main() -> None: "ScriptedAlchemy/tracedecay", inventory, ) + npm_components = INVENTORY.npm_components + INVENTORY.npm_components = lambda _source_root, platform: ( + INVENTORY.npm_components_from_lock(npm_lock, platform) + ) + npm_roots = INVENTORY.npm_components_from_lock(npm_lock, "x86_64-linux") + npm_lock["importers"]["dashboard"]["devDependencies"]["playwright"] = { + "version": "1.0.0" + } + npm_lock["packages"]["dev-only@2.0.0"] = { + "resolution": {"integrity": "sha512-ZGV2"} + } + npm_lock["snapshots"]["dev-only@2.0.0"] = {} + npm_lock["importers"]["dashboard"]["devDependencies"]["dev-only"] = { + "version": "2.0.0" + } + assert INVENTORY.npm_components_from_lock(npm_lock, "x86_64-linux") == npm_roots INVENTORY.release_inventory(*arguments) first = inventory.read_bytes() INVENTORY.release_inventory(*arguments) @@ -210,12 +275,20 @@ def main() -> None: assert document["spdxVersion"] == "SPDX-2.3" packages = {package["name"]: package for package in document["packages"]} assert "react" in packages + assert "react-dom" in packages + assert "scheduler" in packages assert "@fontsource-variable/archivo" in packages assert "playwright" not in packages assert packages["asupersync"]["sourceInfo"].startswith("git+https://github.com/vendor/") assert packages["tokensave-medium-treesitters"]["licenseDeclared"] == "NOASSERTION" + assert packages["sample-vendor"]["licenseDeclared"] == "MIT" + assert packages["mnemon adapted content cues"]["licenseDeclared"] == "Apache-2.0" assert any("tokensave-medium-treesitters" in item["comment"] for item in document["annotations"]) assert any(item["fileName"].endswith("archivo-latin.woff2") for item in document["files"]) + assert any( + "dashboard/app-dist/index.js" in item["comment"] + for item in document["annotations"] + ) artifact_checksums = { package["name"]: package["checksums"][0]["checksumValue"] for package in document["packages"] @@ -223,8 +296,30 @@ def main() -> None: } assert artifact_checksums[archive.name] == INVENTORY.sha256(archive) assert artifact_checksums[mcpb.name] == INVENTORY.sha256(mcpb) + INVENTORY.verify_inventory(inventory, archive, mcpb) + npm_lock["packages"]["react@19.0.0"]["resolution"]["integrity"] = ( + "sha512-cmVhY3Qx" + ) + INVENTORY.release_inventory(*arguments) + INVENTORY.npm_components = npm_components + assert inventory.read_bytes() != first + with tempfile.TemporaryDirectory() as tampered_dir: + tampered_archive = Path(tampered_dir) / archive.name + tampered_archive.write_bytes(archive.read_bytes() + b"changed") + try: + INVENTORY.verify_inventory(inventory, tampered_archive, mcpb) + except ValueError as error: + assert "digest does not match" in str(error) + else: + raise AssertionError("inventory accepted a different package digest") (dashboard / "index.js").write_bytes(b"changed bundled npm") + (source / "Cargo.lock").write_text( + (source / "Cargo.lock").read_text(encoding="utf-8").replace( + "rev=abc#abc", "rev=def#def" + ), + encoding="utf-8", + ) INVENTORY.scope_inventory( source, scope, diff --git a/scripts/test-plan-release-recovery.py b/scripts/test-plan-release-recovery.py index f8d6a255a7..e33dfe0e7b 100755 --- a/scripts/test-plan-release-recovery.py +++ b/scripts/test-plan-release-recovery.py @@ -106,29 +106,60 @@ def main() -> None: build_required, matrix, retained = run(root, (linux_binary, linux_mcpb)) assert build_required == "true" - assert matrix == {"include": [TARGETS["include"][1]]} + assert matrix == { + "include": [ + {**TARGETS["include"][0], "inventory_only": True}, + TARGETS["include"][1], + ] + } assert retained == sorted((linux_binary, linux_mcpb)) stable_assets = ( linux_binary, linux_mcpb, + "tracedecay-v1.2.3-linux.spdx.json", "tracedecay-v1.2.3-windows.zip", "tracedecay-v1.2.3-windows.mcpb", + "tracedecay-v1.2.3-windows.spdx.json", "SHA256SUMS", "install.sh", ) build_required, matrix, retained = run(root, stable_assets) assert build_required == "false" assert matrix == {"include": []} - assert retained == [ - linux_mcpb, - linux_binary, - "tracedecay-v1.2.3-windows.mcpb", - "tracedecay-v1.2.3-windows.zip", - ] + assert retained == sorted( + ( + linux_mcpb, + linux_binary, + "tracedecay-v1.2.3-linux.spdx.json", + "tracedecay-v1.2.3-windows.mcpb", + "tracedecay-v1.2.3-windows.zip", + "tracedecay-v1.2.3-windows.spdx.json", + ) + ) refuses(root, (linux_binary, "SHA256SUMS"), premature_metadata("SHA256SUMS")) refuses(root, (linux_binary, "install.sh"), premature_metadata("install.sh")) + build_required, matrix, retained = run( + root, + ( + linux_binary, + linux_mcpb, + "tracedecay-v1.2.3-windows.zip", + "tracedecay-v1.2.3-windows.mcpb", + "SHA256SUMS", + "install.sh", + ), + ) + assert build_required == "true" + assert matrix == { + "include": [ + {**TARGETS["include"][0], "inventory_only": True}, + {**TARGETS["include"][1], "inventory_only": True}, + ] + } + assert retained == sorted((linux_binary, linux_mcpb, + "tracedecay-v1.2.3-windows.zip", "tracedecay-v1.2.3-windows.mcpb")) refuses( root, ("unexpected.tar.gz",), "unexpected existing release assets: unexpected.tar.gz" ) @@ -141,26 +172,37 @@ def main() -> None: profile="beta", ) assert build_required == "true" - assert matrix == {"include": [TARGETS["include"][1]]} + assert matrix == { + "include": [ + {**TARGETS["include"][0], "inventory_only": True}, + TARGETS["include"][1], + ] + } assert retained == sorted((beta_linux, beta_linux_mcpb)) beta_assets = ( beta_linux, beta_linux_mcpb, + "tracedecay-beta-v1.2.3-linux.spdx.json", "tracedecay-beta-v1.2.3-windows.zip", "tracedecay-beta-v1.2.3-windows.mcpb", + "tracedecay-beta-v1.2.3-windows.spdx.json", "SHA256SUMS", "install.sh", ) build_required, matrix, retained = run(root, beta_assets, profile="beta") assert build_required == "false" assert matrix == {"include": []} - assert retained == [ - beta_linux_mcpb, - beta_linux, - "tracedecay-beta-v1.2.3-windows.mcpb", - "tracedecay-beta-v1.2.3-windows.zip", - ] + assert retained == sorted( + ( + beta_linux_mcpb, + beta_linux, + "tracedecay-beta-v1.2.3-linux.spdx.json", + "tracedecay-beta-v1.2.3-windows.mcpb", + "tracedecay-beta-v1.2.3-windows.zip", + "tracedecay-beta-v1.2.3-windows.spdx.json", + ) + ) refuses(root, (beta_linux, "install.sh"), premature_metadata("install.sh"), profile="beta") From 0d40e67ba78d6450b4c54fdc477087a385ffd256 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 19 Oct 2026 04:40:44 +0000 Subject: [PATCH 04/12] fix(release): disable cache saves in inventory recovery Co-authored-by: ScriptedAlchemy <25274700+ScriptedAlchemy@users.noreply.github.com> --- .github/actions/setup-bazel/action.yml | 5 ++++- .github/workflows/release-beta.yml | 29 +++++++++++++------------- .github/workflows/release.yml | 29 +++++++++++++------------- 3 files changed, 34 insertions(+), 29 deletions(-) diff --git a/.github/actions/setup-bazel/action.yml b/.github/actions/setup-bazel/action.yml index 009d721800..bcbbecae30 100644 --- a/.github/actions/setup-bazel/action.yml +++ b/.github/actions/setup-bazel/action.yml @@ -18,6 +18,9 @@ inputs: prefix misses. Use when two jobs share rustc flags and runner image so Clippy can reuse Test Linux action outputs. default: "" + cache-save: + description: Whether this setup may save Bazelisk and repository caches. + default: "true" tailscale-client-id: description: OIDC client for trusted Linux CI; empty keeps the hosted-only build. default: "" @@ -79,7 +82,7 @@ runs: disk-cache: false bazelrc: >- build --disk_cache=${{ steps.disk-cache-config.outputs.path }} - cache-save: ${{ github.ref == 'refs/heads/master' }} + cache-save: ${{ github.ref == 'refs/heads/master' && inputs.cache-save == 'true' }} # Windows build-script runfiles nest each vendored crate's sources # under its repository name twice; a short output base (default # D:/_bazel) keeps MSVC's header paths under MAX_PATH. diff --git a/.github/workflows/release-beta.yml b/.github/workflows/release-beta.yml index b098d5754e..9ec82c6acc 100644 --- a/.github/workflows/release-beta.yml +++ b/.github/workflows/release-beta.yml @@ -141,11 +141,25 @@ jobs: - name: Install the locked-inventory YAML reader run: python3 -m pip install pyyaml==6.0.2 + # Recovery automation is pinned to this workflow revision, not the release + # source being inventoried. + - name: Checkout release packaging helpers + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.workflow_sha }} + path: .release-automation + persist-credentials: false + sparse-checkout: | + .github/actions/package-release + .github/actions/setup-bazel + scripts/generate-release-inventory.py + - name: Set up Bazel for inventory-only recovery if: matrix.inventory_only == 'true' - uses: ./.github/actions/setup-bazel + uses: ./.release-automation/.github/actions/setup-bazel with: cache-key: release-${{ matrix.target }} + cache-save: "false" - name: Build dashboard bundle for inventory-only recovery if: matrix.inventory_only == 'true' @@ -161,19 +175,6 @@ jobs: dest: target/${{ matrix.target }}/release cache-key: release-${{ matrix.target }} - # Packaging orchestration comes from this immutable workflow revision, - # so recovery also works for tags predating the shared helper. The CLI - # and its packager inputs still come from the validated release source. - - name: Checkout release packaging helpers - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.workflow_sha }} - path: .release-automation - persist-credentials: false - sparse-checkout: | - .github/actions/package-release - scripts/generate-release-inventory.py - - name: Package and verify native archive and MCPB if: matrix.inventory_only != 'true' uses: ./.release-automation/.github/actions/package-release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e6cac508dc..7ed8974dac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -142,11 +142,25 @@ jobs: - name: Install the locked-inventory YAML reader run: python3 -m pip install pyyaml==6.0.2 + # Recovery automation is pinned to this workflow revision, not the release + # source being inventoried. + - name: Checkout release packaging helpers + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.workflow_sha }} + path: .release-automation + persist-credentials: false + sparse-checkout: | + .github/actions/package-release + .github/actions/setup-bazel + scripts/generate-release-inventory.py + - name: Set up Bazel for inventory-only recovery if: matrix.inventory_only == 'true' - uses: ./.github/actions/setup-bazel + uses: ./.release-automation/.github/actions/setup-bazel with: cache-key: release-${{ matrix.target }} + cache-save: "false" - name: Build dashboard bundle for inventory-only recovery if: matrix.inventory_only == 'true' @@ -182,19 +196,6 @@ jobs: path: dashboard-sha256-${{ matrix.name }}.txt if-no-files-found: error - # Packaging orchestration comes from this immutable workflow revision, - # so recovery also works for tags predating the shared helper. The CLI - # and its packager inputs still come from the validated release source. - - name: Checkout release packaging helpers - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.workflow_sha }} - path: .release-automation - persist-credentials: false - sparse-checkout: | - .github/actions/package-release - scripts/generate-release-inventory.py - - name: Package and verify native archive and MCPB if: matrix.inventory_only != 'true' uses: ./.release-automation/.github/actions/package-release From 6ad658a0122edffb284334752d1cc18c82b44f35 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:04:10 -0700 Subject: [PATCH 05/12] fix(release): resolve inventory versions and recovery flags --- .github/workflows/release-beta.yml | 12 ++++++------ .github/workflows/release.yml | 16 ++++++++-------- scripts/generate-release-inventory.py | 3 ++- scripts/test-generate-release-inventory.py | 16 +++++++++++++++- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release-beta.yml b/.github/workflows/release-beta.yml index 9ec82c6acc..4484c42866 100644 --- a/.github/workflows/release-beta.yml +++ b/.github/workflows/release-beta.yml @@ -155,19 +155,19 @@ jobs: scripts/generate-release-inventory.py - name: Set up Bazel for inventory-only recovery - if: matrix.inventory_only == 'true' + if: matrix.inventory_only == true uses: ./.release-automation/.github/actions/setup-bazel with: cache-key: release-${{ matrix.target }} cache-save: "false" - name: Build dashboard bundle for inventory-only recovery - if: matrix.inventory_only == 'true' + if: matrix.inventory_only == true run: bazel build --config=dist --config=ci //dashboard:app_dist # Bazel builds the dashboard bundle the binary embeds (//dashboard:app_dist). - name: Build release binary for packaging - if: matrix.inventory_only != 'true' + if: matrix.inventory_only != true id: cli uses: ./.github/actions/build-cli with: @@ -176,7 +176,7 @@ jobs: cache-key: release-${{ matrix.target }} - name: Package and verify native archive and MCPB - if: matrix.inventory_only != 'true' + if: matrix.inventory_only != true uses: ./.release-automation/.github/actions/package-release with: binary: ${{ steps.cli.outputs.path }} @@ -208,7 +208,7 @@ jobs: if-no-files-found: error - name: Upload MCPB artifact - if: matrix.inventory_only != 'true' + if: matrix.inventory_only != true uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: mcpb-beta-${{ matrix.name }} @@ -216,7 +216,7 @@ jobs: if-no-files-found: error - name: Upload binary archive artifact - if: matrix.inventory_only != 'true' + if: matrix.inventory_only != true uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: beta-${{ matrix.name }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7ed8974dac..e81fbe31d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -156,20 +156,20 @@ jobs: scripts/generate-release-inventory.py - name: Set up Bazel for inventory-only recovery - if: matrix.inventory_only == 'true' + if: matrix.inventory_only == true uses: ./.release-automation/.github/actions/setup-bazel with: cache-key: release-${{ matrix.target }} cache-save: "false" - name: Build dashboard bundle for inventory-only recovery - if: matrix.inventory_only == 'true' + if: matrix.inventory_only == true run: bazel build --config=dist --config=ci //dashboard:app_dist # Bazel builds and validates //dashboard:app_dist, and the CLI build # script refuses to embed it unless it matches //dashboard:bundle_sha256. - name: Build release binary for packaging - if: matrix.inventory_only != 'true' + if: matrix.inventory_only != true id: cli uses: ./.github/actions/build-cli with: @@ -179,7 +179,7 @@ jobs: # verify-release requires every target to embed the same bundle. - name: Record the embedded dashboard digest - if: matrix.inventory_only != 'true' + if: matrix.inventory_only != true shell: bash run: | set -euo pipefail @@ -189,7 +189,7 @@ jobs: "dashboard-sha256-${{ matrix.name }}.txt" - name: Upload embedded dashboard digest - if: matrix.inventory_only != 'true' + if: matrix.inventory_only != true uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: dashboard-sha256-${{ matrix.name }} @@ -197,7 +197,7 @@ jobs: if-no-files-found: error - name: Package and verify native archive and MCPB - if: matrix.inventory_only != 'true' + if: matrix.inventory_only != true uses: ./.release-automation/.github/actions/package-release with: binary: ${{ steps.cli.outputs.path }} @@ -229,7 +229,7 @@ jobs: if-no-files-found: error - name: Upload MCPB artifact - if: matrix.inventory_only != 'true' + if: matrix.inventory_only != true uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: mcpb-${{ matrix.name }} @@ -237,7 +237,7 @@ jobs: if-no-files-found: error - name: Upload binary archive artifact - if: matrix.inventory_only != 'true' + if: matrix.inventory_only != true uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: binary-${{ matrix.name }} diff --git a/scripts/generate-release-inventory.py b/scripts/generate-release-inventory.py index 2b38cf850b..75e8656faa 100644 --- a/scripts/generate-release-inventory.py +++ b/scripts/generate-release-inventory.py @@ -301,7 +301,8 @@ def cargo_package( return package_record( { "name": package_name, - "version": metadata.get("version", package.get("version", "NOASSERTION")), + # Cargo.lock resolves workspace-inherited package versions to strings. + "version": package.get("version", "NOASSERTION"), "checksum": package.get("checksum"), }, manifest, diff --git a/scripts/test-generate-release-inventory.py b/scripts/test-generate-release-inventory.py index 908a88b949..704f91a92d 100644 --- a/scripts/test-generate-release-inventory.py +++ b/scripts/test-generate-release-inventory.py @@ -70,13 +70,21 @@ def main() -> None: (source / "MODULE.bazel").write_text( 'patches = ["//scripts:bazel/patches/asupersync-test.patch"]\n', encoding="utf-8" ) + (source / "Cargo.toml").write_text( + '[workspace]\nmembers = ["crates/*"]\n\n' + '[workspace.package]\nversion = "1.2.3"\n', + encoding="utf-8", + ) write_package( source / "crates/tracedecay-cli/Cargo.toml", "tracedecay-cli", "1.2.3", "MIT", ) - write_package(source / "crates/tracedecay/Cargo.toml", "tracedecay", "1.2.3", "MIT") + (source / "crates/tracedecay/Cargo.toml").write_text( + '[package]\nname = "tracedecay"\nversion.workspace = true\nlicense = "MIT"\n', + encoding="utf-8", + ) write_package( source / "crates/tracedecay-session-memory/Cargo.toml", "tracedecay-session-memory", @@ -182,6 +190,8 @@ def main() -> None: rust = {component["name"]: component for component in scoped["rust_components"]} assert {"tracedecay", "serde", "asupersync", "tokensave-medium-treesitters"} <= set(rust) assert "build-tool" not in rust + assert rust["tracedecay"]["version"] == "1.2.3" + assert rust["tracedecay"]["purl"] == "pkg:cargo/tracedecay@1.2.3" assert "https://github.com/vendor/asupersync" in rust["asupersync"]["downloadLocation"] embedded = {component["name"]: component for component in scoped["embedded_sources"]} assert embedded["sample-vendor"]["version"] == "v2.3.4" @@ -279,6 +289,10 @@ def main() -> None: assert "scheduler" in packages assert "@fontsource-variable/archivo" in packages assert "playwright" not in packages + assert packages["tracedecay"]["versionInfo"] == "1.2.3" + assert packages["tracedecay"]["externalRefs"][0]["referenceLocator"] == ( + "pkg:cargo/tracedecay@1.2.3" + ) assert packages["asupersync"]["sourceInfo"].startswith("git+https://github.com/vendor/") assert packages["tokensave-medium-treesitters"]["licenseDeclared"] == "NOASSERTION" assert packages["sample-vendor"]["licenseDeclared"] == "MIT" From 8060f45ae917de3ca892c67f5f224f16772f0130 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:28:57 -0700 Subject: [PATCH 06/12] fix(release): preserve configured inventory dependency graphs --- scripts/generate-release-inventory.py | 36 +++++++++++------- scripts/test-generate-release-inventory.py | 44 ++++++++++++++++++++++ 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/scripts/generate-release-inventory.py b/scripts/generate-release-inventory.py index 75e8656faa..4e67d495b1 100644 --- a/scripts/generate-release-inventory.py +++ b/scripts/generate-release-inventory.py @@ -171,6 +171,8 @@ def cquery(source_root: Path) -> tuple[str, str]: "--config=ci", f"deps({TARGET})", "--output=graph", + "--nograph:factored", + "--graph:node_limit=-1", ], cwd=source_root, env=env, @@ -201,18 +203,23 @@ def parse_graph(graph: str, kinds: str) -> list[dict[str, str]]: for line in kinds.splitlines(): match = re.match(r"(.+?) rule (.+)$", line.strip()) if match: - target_kinds[label_name(match.group(2))] = match.group(1) - - edges = [ - (label_name(parent), label_name(child)) - for parent, child in re.findall(r'"([^"]+)"\s*->\s*"([^"]+)"', graph) - ] + target_kinds[match.group(2)] = match.group(1) + + # Configuration is part of a node's identity: a crate used by both the + # executable and a proc macro can have different target/exec dependencies. + # Collapsing those nodes before traversal would include build-only crates. + edges = re.findall(r'"([^"]+)"\s*->\s*"([^"]+)"', graph) + nodes = {node for edge in edges for node in edge} + if any("\\n" in node or "\n" in node for node in nodes): + raise ValueError("Bazel inventory graph must use --nograph:factored") adjacency: dict[str, list[str]] = {} for parent, child in edges: adjacency.setdefault(parent, []).append(child) - root = label_name(TARGET) - pending = [root] + roots = [node for node in nodes if label_name(node) == TARGET] + if len(roots) != 1: + raise ValueError("Bazel query did not expose one configured CLI target") + pending = roots.copy() visited: set[str] = set() included: set[str] = set() while pending: @@ -224,15 +231,16 @@ def parse_graph(graph: str, kinds: str) -> list[dict[str, str]]: if kind in {"cargo_build_script", "rust_proc_macro", "rust_test"}: continue if kind == "rust_library": - if current.startswith("@"): - included.add(current) - elif current.startswith("//crates/") or current.startswith("//patches/"): - included.add(current) + label = label_name(current) + if label.startswith("@"): + included.add(label) + elif label.startswith("//crates/") or label.startswith("//patches/"): + included.add(label) pending.extend(adjacency.get(current, ())) - if root not in visited or not included: + if not included: raise ValueError("Bazel query did not expose the configured CLI Rust dependency graph") - return [{"label": label, "kind": target_kinds.get(label, "")} for label in sorted(included)] + return [{"label": label, "kind": "rust_library"} for label in sorted(included)] def lock_package_index(source_root: Path) -> tuple[dict[str, dict[str, object]], dict[str, dict[str, object]]]: diff --git a/scripts/test-generate-release-inventory.py b/scripts/test-generate-release-inventory.py index 704f91a92d..0cdfeb6c5c 100644 --- a/scripts/test-generate-release-inventory.py +++ b/scripts/test-generate-release-inventory.py @@ -33,7 +33,51 @@ def make_archive(path: Path, payload: bytes) -> None: archive.addfile(entry, __import__("io").BytesIO(payload)) +def configured_graph_preserves_target_and_exec_dependency_boundaries() -> None: + root = "//crates/tracedecay-cli:tracedecay (123abc)" + shared_target = "@@c__shared-1.0.0//:shared (123abc)" + shared_exec = "@@c__shared-1.0.0//:shared (456def)" + macro = "@@c__derive-1.0.0//:derive (456def)" + shipped = "@@c__runtime-only-1.0.0//:runtime_only (123abc)" + build_only = "@@c__build-only-1.0.0//:build_only (456def)" + graph = "\n".join( + f'"{parent}" -> "{child}";' + for parent, child in ( + (root, shared_target), + (root, macro), + (shared_target, shipped), + (macro, shared_exec), + (shared_exec, build_only), + ) + ) + kinds = "\n".join( + ( + f"rust_binary rule {root}", + f"rust_library rule {shared_target}", + f"rust_library rule {shared_exec}", + f"rust_proc_macro rule {macro}", + f"rust_library rule {shipped}", + f"rust_library rule {build_only}", + ) + ) + assert INVENTORY.parse_graph(graph, kinds) == [ + {"label": "@@c__runtime-only-1.0.0//:runtime_only", "kind": "rust_library"}, + {"label": "@@c__shared-1.0.0//:shared", "kind": "rust_library"}, + ] + + # Bazel's default DOT output combines equivalent nodes. An externally + # supplied factored graph must not silently lose all but the first crate. + factored = f'"{root}" -> "{shared_target}\\n{shipped}";' + try: + INVENTORY.parse_graph(factored, kinds) + except ValueError as error: + assert "--nograph:factored" in str(error) + else: + raise AssertionError("inventory accepted a factored dependency graph") + + def main() -> None: + configured_graph_preserves_target_and_exec_dependency_boundaries() with tempfile.TemporaryDirectory() as directory: root = Path(directory) source = root / "source" From 5a120bec2bb6ead38e6b31f2416cd778f9503afa Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:55:34 -0700 Subject: [PATCH 07/12] fix(release): include inventory helpers in sparse checkouts --- .github/workflows/release-beta.yml | 1 + .github/workflows/release.yml | 1 + scripts/generate-release-inventory.py | 15 +++++++++------ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release-beta.yml b/.github/workflows/release-beta.yml index 4484c42866..008852f4ca 100644 --- a/.github/workflows/release-beta.yml +++ b/.github/workflows/release-beta.yml @@ -153,6 +153,7 @@ jobs: .github/actions/package-release .github/actions/setup-bazel scripts/generate-release-inventory.py + sparse-checkout-cone-mode: false - name: Set up Bazel for inventory-only recovery if: matrix.inventory_only == true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e81fbe31d7..cc319fa982 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -154,6 +154,7 @@ jobs: .github/actions/package-release .github/actions/setup-bazel scripts/generate-release-inventory.py + sparse-checkout-cone-mode: false - name: Set up Bazel for inventory-only recovery if: matrix.inventory_only == true diff --git a/scripts/generate-release-inventory.py b/scripts/generate-release-inventory.py index 4e67d495b1..46e3a7c754 100644 --- a/scripts/generate-release-inventory.py +++ b/scripts/generate-release-inventory.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import base64 import hashlib import json import os @@ -16,6 +17,12 @@ from pathlib import Path from urllib.parse import quote +# Scope-only generation does not need the dashboard lockfile reader. +try: + import yaml +except ImportError: + yaml = None + TARGET = "//crates/tracedecay-cli:tracedecay" FONT_LICENSES = { @@ -365,8 +372,6 @@ def checksum_from_sri(integrity: str) -> dict[str, str] | None: algorithm, separator, encoded = item.partition("-") if not separator or algorithm.lower() not in {"sha1", "sha256", "sha384", "sha512"}: continue - import base64 - try: digest = base64.b64decode(encoded, validate=True).hex() except ValueError: @@ -401,10 +406,8 @@ def npm_package_identity(key: str) -> tuple[str, str]: def npm_components(source_root: Path, platform: str) -> list[dict[str, object]]: - try: - import yaml - except ImportError as error: - raise RuntimeError("PyYAML is required to read the locked dashboard package graph") from error + if yaml is None: + raise RuntimeError("PyYAML is required to read the locked dashboard package graph") documents = yaml.safe_load_all( (source_root / "pnpm-lock.yaml").read_text(encoding="utf-8") From e435ec68dda73d7994543de250e61907fe5ea347 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:07:46 -0700 Subject: [PATCH 08/12] fix(memory): correct native index build admission --- .../src/native_integration/journey_tests.rs | 45 ++++- .../src/code_index_scheduler.rs | 3 +- .../src/code_index_scheduler/serving.rs | 156 ++++++------------ .../tests/code_index_scheduler/serving.rs | 39 ----- .../src/resident_memory.rs | 66 +++++++- .../src/resident_memory/tests.rs | 127 ++++++++++++++ 6 files changed, 288 insertions(+), 148 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs b/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs index 7a624dba01..1e2788add4 100644 --- a/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs +++ b/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs @@ -495,17 +495,56 @@ async fn mount( .mount_worktree(project_id.clone(), &repository_root, index_store) .await .expect("mount canonical code-index scheduler"); - tokio::time::timeout(Duration::from_secs(5), async { + let memory = schedulers.process_resident_memory(); + let mut last_observation = "no owner observation completed".to_owned(); + let initial_generation = tokio::time::timeout(Duration::from_secs(5), async { + let owner_activity = schedulers.subscribe_owner_activity(&repository_root).await; while schedulers .latest_generation_id(&repository_root) .await .is_none() { + let parked = schedulers + .convergence_park(&repository_root) + .await + .map(|parked| { + ( + parked.reason.chars().take(200).collect::(), + parked.blocked_reason, + parked.retries_on_wake, + parked.observed_passes, + ) + }); + let activity = owner_activity.as_ref().map(|activity| { + ( + activity.worker_phase(), + activity.passes(), + activity.wake_pending(), + ) + }); + let pressure = memory.pressure().state(); + let ledger = memory.snapshot(); + last_observation = format!( + "parked={parked:?}; activity={activity:?}; pressure={pressure:?}; \ + ledger_used_bytes={}; ledger_limit_bytes={}; ledger_charge_count={}; \ + process_shared_charge_count={}", + ledger.used_bytes, + ledger.limit_bytes, + ledger.charges.len(), + ledger.process_shared_charges.len() + ); tokio::time::sleep(Duration::from_millis(10)).await; } }) - .await - .expect("initial code-index generation seat"); + .await; + if let Err(error) = initial_generation { + // The bounded observation was captured inside the original deadline; + // reporting starts no new reads, admission, or readiness wait. + // A failed fixture still owns the worker it mounted. Signal its + // canonical cancellation before unwinding out of the temporary root. + schedulers.cancel(); + panic!("initial code-index generation seat: {error}; {last_observation}"); + } let analysis = Arc::new(DaemonNativeIntegrationAnalysisV1::new( schedulers, analysis_scope, diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs index baca65a33f..a325e59813 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler.rs @@ -300,8 +300,7 @@ pub use serving::{ use serving::{ CodeIndexCommittedProgressSampleV1, CodeTextProjectionSlotV1, map_sealed_page_source_error, sha256_private_file_and_size, text_artifact_admitted_build_budget, - text_artifact_builder_budget, text_artifact_resident_memory_charge, - text_artifact_source_batch_limits, + text_artifact_builder_budget, text_artifact_source_batch_limits, }; #[cfg(test)] use tracedecay_query::retrieval::ports::TEXT_ARTIFACT_PAGE_CHUNKS_V1; diff --git a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs index de14bd949d..e6a5be1c4a 100644 --- a/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/src/code_index_scheduler/serving.rs @@ -5,7 +5,6 @@ mod family_report; use std::{ collections::{BTreeSet, VecDeque}, io::Read, - num::NonZeroU64, path::{Path, PathBuf}, sync::{ Arc, Condvar, Mutex, MutexGuard, OnceLock, PoisonError, RwLock, @@ -947,29 +946,6 @@ pub struct DaemonCodeTextArtifactStoreV1 { worktree_id: WorktreeId, } -/// The bytes one artifact admission reserves: its ceiling plus the band above -/// the admission watermark, so the reserve call enforces that watermark -/// against both the ledger and the measured resident set. The band is not -/// memory the artifact owns and is released right after the reserve; -/// retaining it in every overlapping build would charge the same -/// process-wide margin repeatedly. The process's live set is not charged: -/// the reserve already compares the request with the measured resident set, -/// which contains it. -pub(super) fn text_artifact_resident_memory_charge( - requested: NonZeroU64, - watermark_headroom: u64, -) -> Result { - requested - .get() - .checked_add(watermark_headroom) - .and_then(NonZeroU64::new) - .ok_or_else(|| { - RetrievalPortError::Contract( - "text-artifact resident-memory accounting overflowed".to_owned(), - ) - }) -} - pub(super) fn text_artifact_admitted_build_budget( preferred_bytes: u64, minimum_bytes: u64, @@ -1170,10 +1146,8 @@ impl DaemonCodeTextArtifactStoreV1 { } } - /// Reserve one artifact memory ceiling. The atomic reserve also includes - /// the process-wide high-watermark headroom, then releases that - /// check-only margin before returning while the component ceiling stays - /// charged. + /// Charge the artifact's ceiling below the process admission watermark. + /// Sizing and charging share one measured headroom reading. fn reserve_resident_memory( &self, generation_id: &CodeGenerationId, @@ -1184,32 +1158,6 @@ impl DaemonCodeTextArtifactStoreV1 { .map(|(reservation, _)| reservation) } - /// Measure headroom and size the build: `(observed, watermark headroom, - /// admitted)` bytes. - fn text_artifact_admission( - &self, - preferred: NonZeroU64, - minimum: NonZeroU64, - ) -> Result<(u64, u64), RetrievalPortError> { - let admission_watermark = self.resident_memory.admission_watermark_bytes(); - let headroom = self.resident_memory.headroom_below(admission_watermark); - let watermark_headroom = self - .resident_memory - .snapshot() - .limit_bytes - .saturating_sub(admission_watermark); - let admitted_bytes = text_artifact_admitted_build_budget( - preferred.get(), - minimum.get(), - headroom.available_bytes, - ) - .inspect_err(|_| { - self.resident_memory - .wait_for_headroom(minimum.get(), admission_watermark); - })?; - Ok((watermark_headroom, admitted_bytes)) - } - fn reserve_resident_memory_up_to( &self, generation_id: &CodeGenerationId, @@ -1235,57 +1183,59 @@ impl DaemonCodeTextArtifactStoreV1 { "text-artifact minimum resident-memory reservation must be nonzero".to_owned(), ) })?; - let (watermark_headroom, admitted_bytes) = - match self.text_artifact_admission(preferred, minimum) { - Err(RetrievalPortError::ResidentMemoryRefused(detail)) => { - let released = self - .resident_owners - .shed(minimum.get(), std::time::Instant::now()); - for release in &released { - log_resident_owner_release_v1(release); - } - // Freed pages sit in the pool workers' allocator heaps - // whether or not an owner was shed. - let trim = release_process_allocator_memory_v1(); - tracing::info!( - event = "code_text_artifact_build_shed_retained_state", - released_owners = released.len(), - trimmed_bytes = trim.released_bytes(), - refused = %detail, - "text-artifact build shed retained state before re-measuring headroom" - ); - self.text_artifact_admission(preferred, minimum) + let admission_watermark = self.resident_memory.admission_watermark_bytes(); + let reserve = || { + let reservation = self + .resident_memory + .reserve_up_to( + ResidentMemoryKeyV1 { + project_id: self.project_id.clone(), + worktree_id: self.worktree_id.clone(), + generation_id: generation_id.clone(), + component, + }, + preferred, + admission_watermark, + ) + .map_err(|error| { + RetrievalPortError::ResidentMemoryRefused(format!( + "text-artifact resident-memory admission was refused: {error}" + )) + })?; + let admitted = text_artifact_admitted_build_budget( + preferred.get(), + minimum.get(), + reservation.reserved_bytes(), + )?; + Ok((reservation, admitted)) + }; + let (reservation, admitted) = match reserve() { + Err(RetrievalPortError::ResidentMemoryRefused(detail)) => { + let released = self + .resident_owners + .shed(minimum.get(), std::time::Instant::now()); + for release in &released { + log_resident_owner_release_v1(release); } - admission => admission, - }?; - let admitted = NonZeroU64::new(admitted_bytes).ok_or_else(|| { - RetrievalPortError::Contract( - "text-artifact admitted resident-memory reservation must be nonzero".to_owned(), - ) - })?; - let accounted = text_artifact_resident_memory_charge(admitted, watermark_headroom)?; - let mut reservation = self - .resident_memory - .reserve( - ResidentMemoryKeyV1 { - project_id: self.project_id.clone(), - worktree_id: self.worktree_id.clone(), - generation_id: generation_id.clone(), - component, - }, - accounted, - ) - .map_err(|error| { - RetrievalPortError::ResidentMemoryRefused(format!( - "text-artifact resident-memory admission was refused: {error}" - )) - })?; - reservation.shrink_to(admitted.get()).map_err(|error| { - RetrievalPortError::Contract(format!( - "text-artifact resident-memory headroom release failed: {error}" - )) + let trim = release_process_allocator_memory_v1(); + tracing::info!( + event = "code_text_artifact_build_shed_retained_state", + released_owners = released.len(), + trimmed_bytes = trim.released_bytes(), + refused = %detail, + "text-artifact build shed retained state before re-measuring headroom" + ); + reserve() + } + result => result, + } + .inspect_err(|error| { + if matches!(error, RetrievalPortError::ResidentMemoryRefused(_)) { + self.resident_memory + .wait_for_headroom(minimum.get(), admission_watermark); + } })?; - let admitted = usize::try_from(admitted.get()).map_err(|error| { + let admitted = usize::try_from(admitted).map_err(|error| { RetrievalPortError::Contract(format!( "text-artifact admitted reservation exceeds the platform limit: {error}" )) diff --git a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs index 05bf0e812e..aca2a7f7bb 100644 --- a/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs +++ b/crates/tracedecay-code-index-runtime/tests/code_index_scheduler/serving.rs @@ -2085,45 +2085,6 @@ fn reader_reservation_refusal_precedes_missing_artifact_access() { assert!(latest.text_projection_needs_work()); } -#[test] -fn overlapping_text_builds_share_one_admission_watermark_headroom() { - let limit_bytes = 1_000_u64; - let watermark_headroom = 100_u64; - let requested = NonZeroU64::new(300).expect("nonzero build request"); - let mut used_bytes = 0_u64; - - for _ in 0..3 { - let accounted = - super::super::text_artifact_resident_memory_charge(requested, watermark_headroom) - .expect("bounded admission accounting"); - assert!( - used_bytes + accounted.get() <= limit_bytes, - "each overlapping build fits beneath the same 900-byte high watermark" - ); - used_bytes += requested.get(); - } - - assert_eq!( - used_bytes, 900, - "the retained ledger owns the three build ceilings, not three margins" - ); - for overflow in [ - super::super::text_artifact_resident_memory_charge( - NonZeroU64::new(u64::MAX).expect("maximum nonzero request"), - 1, - ), - super::super::text_artifact_resident_memory_charge(requested, u64::MAX), - ] { - assert!( - matches!( - overflow, - Err(tracedecay_query::retrieval::RetrievalPortError::Contract(_)) - ), - "overflow must remain a typed contract refusal: {overflow:?}" - ); - } -} - #[test] fn text_build_budget_shrinks_to_available_headroom_without_dropping_below_its_floor() { const GIB: u64 = 1024 * 1024 * 1024; diff --git a/crates/tracedecay-runtime-core/src/resident_memory.rs b/crates/tracedecay-runtime-core/src/resident_memory.rs index 34a9c445be..a16e42ca7d 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory.rs @@ -1531,6 +1531,62 @@ impl ProcessResidentMemoryV1 { Err(self.refused(self.admission_failure(requested_bytes))) } + /// Size and charge new work from the same measured headroom reading. + /// The returned ceiling can be smaller than `preferred_bytes`; consumers + /// with a minimum working set must release a reservation below that floor. + /// Concurrent ledger charges are checked under the charge's own lock. + /// The consumer registers its working-set minimum with `wait_for_headroom` + /// when the returned budget cannot support its next operation. + #[tracing::instrument( + name = "runtime_core.resident.reserve_up_to", + level = "trace", + skip_all + )] + pub fn reserve_up_to( + self: &Arc, + key: ResidentMemoryKeyV1, + preferred_bytes: NonZeroU64, + ceiling: u64, + ) -> Result { + self.pressure.measure_admission_bytes(); + let ceiling = ceiling.min(self.limit_bytes.get()); + let mut state = self.lock_state(); + let headroom = self.headroom_from_state(&state, ceiling); + let admitted = preferred_bytes + .get() + .min(headroom.available_bytes) + .min(ceiling.saturating_sub(state.used_bytes)); + if admitted == 0 { + let failure = if self.pressure.state().is_over_budget() + || headroom.observed_bytes >= ceiling.min(self.pressure.limit_bytes()) + { + ResidentMemoryAdmissionFailureV1::ObservedOverBudget { + observed_bytes: headroom.observed_bytes, + limit_bytes: self.pressure.limit_bytes(), + high_watermark_bytes: self.pressure.high_watermark_bytes(), + requested_bytes: preferred_bytes.get(), + floor_bytes: RESIDENT_MEMORY_PRESSURE_ADMISSION_FLOOR_BYTES_V1, + } + } else { + ResidentMemoryAdmissionFailureV1::ReservationCeiling { + used_bytes: state.used_bytes, + requested_bytes: preferred_bytes.get(), + limit_bytes: ceiling, + } + }; + drop(state); + return Err(failure); + } + // `admitted` is at most ceiling - used_bytes, so both ledger sums fit. + state.used_bytes += admitted; + *state.charges.entry(key.clone()).or_default() += admitted; + Ok(ResidentMemoryReservationV1 { + authority: Arc::clone(self), + key, + reserved_bytes: admitted, + }) + } + /// Reserves one process-shared component without fabricating a project, /// worktree, or code-generation owner. These reservations use the same /// process ceiling and RAII release authority as project generations. @@ -1596,12 +1652,20 @@ impl ProcessResidentMemoryV1 { /// [`Self::reserve`] holds them. pub fn headroom_below(&self, ceiling: u64) -> ResidentMemoryHeadroomV1 { self.pressure.measure_admission_bytes(); + self.headroom_from_state(&self.lock_state(), ceiling) + } + + fn headroom_from_state( + &self, + state: &ResidentMemoryStateV1, + ceiling: u64, + ) -> ResidentMemoryHeadroomV1 { let measured_ceiling = ceiling.min(self.pressure.limit_bytes()); let observed_bytes = self .pressure .observed_bytes_for_limit(measured_ceiling) .unwrap_or(0); - let used_bytes = self.lock_state().used_bytes; + let used_bytes = state.used_bytes; let available_bytes = if self.pressure.state().is_over_budget() { 0 } else { diff --git a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs index 557293b4ec..10fe13f033 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory/tests.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory/tests.rs @@ -1335,6 +1335,35 @@ fn cgroup_headroom_respects_a_larger_but_fuller_ancestor() { ); } +#[test] +fn sized_reservations_charge_only_the_measured_cgroup_headroom() { + const MIB: u64 = 1024 * 1024; + let limit = bytes(800 * MIB); + let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( + limit, + Arc::new(|| { + Some(ProcessResidentSampleV1 { + resident_bytes: 100 * MIB, + unreclaimable_bytes: 100 * MIB, + swapped_bytes: 0, + cgroup_headroom_bytes: Some(200 * MIB), + }) + }), + )); + let authority = Arc::new(ProcessResidentMemoryV1::with_pressure(limit, pressure)); + let watermark = authority.admission_watermark_bytes(); + let headroom = authority.headroom_below(watermark); + assert_eq!(headroom.available_bytes, 200 * MIB); + let owner = key("project-a", "worktree-a", "generation-a", "text-build"); + let held = authority + .reserve_up_to(owner, bytes(headroom.available_bytes + 1), watermark) + .expect("the actual allocation fits without charging the 80 MiB watermark band"); + assert_eq!(held.reserved_bytes(), 200 * MIB); + assert_eq!(authority.snapshot().used_bytes, 200 * MIB); + drop(held); + assert_eq!(authority.snapshot().used_bytes, 0); +} + #[test] fn narrowed_budgets_and_waiters_share_cgroup_capacity_without_charging_peers() { const MIB: u64 = 1024 * 1024; @@ -1619,3 +1648,101 @@ fn allocator_reclaim_preserves_cgroup_pressure_until_sample_recovers() { .is_ok() ); } + +#[test] +fn sized_admission_remeasures_and_charges_the_supported_smaller_budget() { + const MIB: u64 = 1024 * 1024; + let measured = Arc::new(AtomicU64::new(300 * MIB)); + let sampled = Arc::clone(&measured); + let reads = Arc::new(AtomicU64::new(0)); + let sampled_reads = Arc::clone(&reads); + let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( + bytes(1000 * MIB), + Arc::new(move || { + sampled_reads.fetch_add(1, Ordering::SeqCst); + let observed = sampled.load(Ordering::SeqCst); + Some(ProcessResidentSampleV1 { + resident_bytes: observed, + unreclaimable_bytes: observed, + swapped_bytes: 0, + cgroup_headroom_bytes: None, + }) + }), + )); + let authority = Arc::new(ProcessResidentMemoryV1::with_pressure( + bytes(1000 * MIB), + Arc::clone(&pressure), + )); + let watermark = authority.admission_watermark_bytes(); + assert_eq!(watermark, 900 * MIB); + assert_eq!( + authority.headroom_below(watermark).available_bytes, + 600 * MIB + ); + measured.store(700 * MIB, Ordering::SeqCst); + let before = reads.load(Ordering::SeqCst); + let reservation = authority + .reserve_up_to( + key("project-a", "worktree-a", "generation-a", "text-build"), + bytes(600 * MIB), + watermark, + ) + .expect("the smaller build fits despite growth since planning"); + assert_eq!(reads.load(Ordering::SeqCst), before + 1); + assert_eq!(reservation.reserved_bytes(), 200 * MIB); + assert_eq!(authority.snapshot().used_bytes, 200 * MIB); + drop(reservation); + assert_eq!(authority.snapshot().used_bytes, 0); + + measured.store(950 * MIB, Ordering::SeqCst); + let refused = authority + .reserve_up_to( + key("project-a", "worktree-a", "generation-b", "text-build"), + bytes(600 * MIB), + watermark, + ) + .expect_err("sizing must still refuse measured pressure"); + assert!(refused.is_observed_over_budget()); + assert_eq!(authority.snapshot().used_bytes, 0); +} + +#[test] +fn sized_admissions_share_the_watermark_without_charging_its_margin() { + let pressure = Arc::new(ResidentMemoryPressureV1::with_sampler( + bytes(1000), + Arc::new(|| None), + )); + let authority = Arc::new(ProcessResidentMemoryV1::with_pressure( + bytes(1000), + pressure, + )); + let watermark = authority.admission_watermark_bytes(); + let owner = key("project-a", "worktree-a", "generation-a", "text-build"); + let mut reservations = (0..3) + .map(|_| { + authority + .reserve_up_to(owner.clone(), bytes(300), watermark) + .unwrap() + }) + .collect::>(); + assert_eq!(authority.snapshot().used_bytes, 900); + let refused = authority + .reserve_up_to(owner.clone(), bytes(300), watermark) + .unwrap_err(); + assert_eq!( + refused, + ResidentMemoryAdmissionFailureV1::ReservationCeiling { + used_bytes: 900, + requested_bytes: 300, + limit_bytes: 900, + } + ); + drop(reservations.pop()); + let maximum = authority + .reserve_up_to(owner, bytes(u64::MAX), watermark) + .expect("bounded sizing cannot overflow while reserving available bytes"); + assert_eq!(maximum.reserved_bytes(), 300); + assert_eq!(authority.snapshot().used_bytes, 900); + drop((reservations, maximum)); + assert_eq!(authority.snapshot().used_bytes, 0); +} From 1da4f8d5151ca6097466c833ab4d0a2f0a1bb8f2 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:42:42 -0700 Subject: [PATCH 09/12] fix(tests): retire native journey code-index workers --- .../src/native_integration/journey_tests.rs | 49 ++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs b/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs index 1e2788add4..2a7f6684de 100644 --- a/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs +++ b/crates/tracedecay-agent-hosts/src/native_integration/journey_tests.rs @@ -469,6 +469,25 @@ fn approval_for( .expect("approval") } +struct JourneyCodeIndex { + schedulers: CodeIndexSchedulerRegistryV1, +} + +impl JourneyCodeIndex { + async fn shutdown(&self) { + self.schedulers.shutdown().await; + } +} + +impl Drop for JourneyCodeIndex { + fn drop(&mut self) { + // A failed assertion must also release the fixture's native workers. + // The analysis port uses the daemon's scheduler authority; retiring + // the transaction registry does not stop that independent authority. + self.schedulers.cancel(); + } +} + async fn mount( database: RegisteredGlobalDbLeaseV1, repository_root: std::path::PathBuf, @@ -476,6 +495,7 @@ async fn mount( DaemonNativeIntegrationServiceRegistry, super::registry::DaemonNativeIntegrationOwner, Arc, + JourneyCodeIndex, ) { let project_id = ProjectId::new("project.native.journey").expect("project id"); let identity = IndexingIdentityV1::resolve(&repository_root).expect("indexing identity"); @@ -486,7 +506,10 @@ async fn mount( identity.head_ref().cloned(), ) .expect("analysis scope"); - let schedulers = CodeIndexSchedulerRegistryV1::new(1); + let code_index = JourneyCodeIndex { + schedulers: CodeIndexSchedulerRegistryV1::new(1), + }; + let schedulers = &code_index.schedulers; let index_store = repository_root .parent() .expect("repository parent") @@ -540,13 +563,10 @@ async fn mount( if let Err(error) = initial_generation { // The bounded observation was captured inside the original deadline; // reporting starts no new reads, admission, or readiness wait. - // A failed fixture still owns the worker it mounted. Signal its - // canonical cancellation before unwinding out of the temporary root. - schedulers.cancel(); panic!("initial code-index generation seat: {error}; {last_observation}"); } let analysis = Arc::new(DaemonNativeIntegrationAnalysisV1::new( - schedulers, + schedulers.clone(), analysis_scope, )); let registry = DaemonNativeIntegrationServiceRegistry::default(); @@ -564,7 +584,7 @@ async fn mount( ) .await .expect("mount native integration owner"); - (registry, owner, analysis) + (registry, owner, analysis, code_index) } async fn preflight( @@ -793,7 +813,8 @@ async fn checked_out_declared_stack_conflict_enqueues_without_moving_refs() { let database = runtime .registered_database_lease(HostAdmissionScope::Project) .expect("registered project database"); - let (registry, owner, _analysis) = mount(database.clone(), repository_root.clone()).await; + let (registry, owner, _analysis, code_index) = + mount(database.clone(), repository_root.clone()).await; let now = UtcMicros( i64::try_from( SystemTime::now() @@ -1038,6 +1059,7 @@ async fn checked_out_declared_stack_conflict_enqueues_without_moving_refs() { drop(stack_runtime); registry.shutdown().await.expect("shutdown owner registry"); + code_index.shutdown().await; } #[tokio::test(flavor = "multi_thread")] @@ -1066,7 +1088,8 @@ async fn independent_pair_applies_supported_modes_and_survives_daemon_restart() .registered_database_lease(HostAdmissionScope::Project) .expect("registered project database"); - let (registry, owner, analysis) = mount(database.clone(), repository_root.clone()).await; + let (registry, owner, analysis, code_index) = + mount(database.clone(), repository_root.clone()).await; let request_id = format!("request.native.journey.{index}"); let request = if index == 0 { declared_preflight_request(&repository_root, &database, mode, &request_id) @@ -1410,7 +1433,8 @@ async fn independent_pair_applies_supported_modes_and_survives_daemon_restart() } registry.shutdown().await.expect("shutdown owner registry"); - let (restarted_registry, restarted_owner, _analysis) = + code_index.shutdown().await; + let (restarted_registry, restarted_owner, _analysis, restarted_code_index) = mount(database, repository_root).await; let durable = restarted_owner .service() @@ -1428,6 +1452,7 @@ async fn independent_pair_applies_supported_modes_and_survives_daemon_restart() .shutdown() .await .expect("shutdown restarted owner registry"); + restarted_code_index.shutdown().await; } } @@ -1447,7 +1472,7 @@ async fn generated_only_change_requires_semantic_review() { let database = runtime .registered_database_lease(HostAdmissionScope::Project) .expect("registered project database"); - let (registry, owner, _analysis) = mount(database, repository_root.clone()).await; + let (registry, owner, _analysis, code_index) = mount(database, repository_root.clone()).await; let preview = preflight( owner, @@ -1473,6 +1498,7 @@ async fn generated_only_change_requires_semantic_review() { vec![NativeIntegrationAnalysisGapV1::WithheldSource] ); registry.shutdown().await.expect("shutdown owner registry"); + code_index.shutdown().await; } #[tokio::test(flavor = "multi_thread")] @@ -1492,7 +1518,7 @@ async fn foreign_destination_ref_drift_terminates_without_mutating_the_foreign_t let database = runtime .registered_database_lease(HostAdmissionScope::Project) .expect("registered project database"); - let (registry, owner, _analysis) = mount(database, repository_root.clone()).await; + let (registry, owner, _analysis, code_index) = mount(database, repository_root.clone()).await; let request = preflight_request( &repository_root, MechanicalIntegrationModeV1::FastForward, @@ -1571,6 +1597,7 @@ async fn foreign_destination_ref_drift_terminates_without_mutating_the_foreign_t ) ); registry.shutdown().await.expect("shutdown owner registry"); + code_index.shutdown().await; } /// Cleanup inspection of an owned checkout, asked while another thread walks From de7c0d09aa8f3d3ca69077b30e74e1b1e3ef65d1 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:02:19 -0700 Subject: [PATCH 10/12] fix(tests): await index readiness before broker client burst --- .../tests/storage_suite/multi_connection_test/harness.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/tracedecay/tests/storage_suite/multi_connection_test/harness.rs b/crates/tracedecay/tests/storage_suite/multi_connection_test/harness.rs index 00a51165f7..8fb0c7d7af 100644 --- a/crates/tracedecay/tests/storage_suite/multi_connection_test/harness.rs +++ b/crates/tracedecay/tests/storage_suite/multi_connection_test/harness.rs @@ -96,13 +96,15 @@ pub(super) fn init_project(home: &Path, project: &Path, socket_path: &Path) -> P ); } + // Establish a ready generation before the client burst: ordinary init only + // queues reconciliation, while the later Doctor probe requires it complete. let output = common::tracedecay_command_with_home(home) .env("TRACEDECAY_DAEMON_SOCKET", socket_path) - .arg("init") + .args(["init", "--wait"]) .current_dir(project) .output() - .expect("tracedecay init should run"); - assert_command_success("tracedecay init", &output); + .expect("tracedecay init --wait should run"); + assert_command_success("tracedecay init --wait", &output); home.join(".tracedecay").join("global.db") } From 4364b009746b2ea56a4cd311b7c1d30149dc68a8 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:05:50 -0700 Subject: [PATCH 11/12] fix(memory): report admission pressure without claiming process RSS --- .../src/doctor/sources.rs | 9 +++--- .../src/resident_memory.rs | 30 +++++++++---------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/crates/tracedecay-contracts/src/doctor/sources.rs b/crates/tracedecay-contracts/src/doctor/sources.rs index da87bf3376..bd672add26 100644 --- a/crates/tracedecay-contracts/src/doctor/sources.rs +++ b/crates/tracedecay-contracts/src/doctor/sources.rs @@ -1740,7 +1740,8 @@ pub struct ResidentMemoryOwnerReadV1 { #[serde(rename_all = "snake_case", tag = "kind")] pub enum ResidentMemoryReadV1 { Observed { - /// Last sampled process RSS, `None` before the first sample. + /// Last sampled admission pressure: process allocations constrained by + /// shared cgroup headroom, `None` before the first sample. resident_bytes: Option, limit_bytes: u64, high_watermark_bytes: u64, @@ -1782,12 +1783,12 @@ pub fn resident_memory_findings( )?]); }; let unmeasured = owners.iter().filter(|owner| owner.bytes.is_none()).count(); - let resident = resident_bytes.map_or_else( + let admission = resident_bytes.map_or_else( || "unsampled".to_owned(), |bytes| format!("{} MiB", bytes / MIB), ); let summary = format!( - "resident {resident} of a {} MiB limit (pressure line {} MiB); {} owners retain {} MiB measured, {unmeasured} unmeasured", + "admission pressure {admission} of a {} MiB limit (pressure line {} MiB; process allocations or shared cgroup pressure); {} owners retain {} MiB measured, {unmeasured} unmeasured", limit_bytes / MIB, high_watermark_bytes / MIB, owners.len(), @@ -2174,7 +2175,7 @@ mod tests { [ ( DoctorEvidenceStateV1::Partial, - "resident 3072 MiB of a 6144 MiB limit (pressure line 5120 MiB); 2 owners retain 300 MiB measured, 1 unmeasured".to_owned() + "admission pressure 3072 MiB of a 6144 MiB limit (pressure line 5120 MiB; process allocations or shared cgroup pressure); 2 owners retain 300 MiB measured, 1 unmeasured".to_owned() ), ( DoctorEvidenceStateV1::HealthyCompleteCoverage, diff --git a/crates/tracedecay-runtime-core/src/resident_memory.rs b/crates/tracedecay-runtime-core/src/resident_memory.rs index a16e42ca7d..f212bf0252 100644 --- a/crates/tracedecay-runtime-core/src/resident_memory.rs +++ b/crates/tracedecay-runtime-core/src/resident_memory.rs @@ -581,22 +581,22 @@ fn psi_some_avg10_v1(text: &str) -> Option { .filter(|value| value.is_finite()) } -/// What the last measured RSS sample says about this process. +/// Admission pressure from the last process sample and shared cgroup headroom. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ResidentMemoryPressureStateV1 { /// No sample has been published yet, so admission has nothing measured to /// consult and falls back to the reservation ceiling alone. An abstention, /// never a claim that the process is small. Unobserved, - /// Measured RSS is below the pressure watermarks, or between them with the - /// latch clear. + /// Measured admission pressure is below the watermarks, or between them + /// with the latch clear. Nominal { observed_bytes: u64, limit_bytes: u64, high_watermark_bytes: u64, }, - /// Measured RSS reached the high watermark and has not yet fallen back to - /// the low watermark. + /// Measured admission pressure reached the high watermark and has not yet + /// fallen back to the low watermark. OverBudget { observed_bytes: u64, limit_bytes: u64, @@ -611,7 +611,7 @@ impl ResidentMemoryPressureStateV1 { matches!(self, Self::OverBudget { .. }) } - /// The last measured RSS, or `None` when nothing has been sampled. + /// Last sampled admission pressure, or `None` before the first sample. #[must_use] pub const fn observed_bytes(self) -> Option { match self { @@ -1285,9 +1285,9 @@ pub struct ResidentMemoryKeyV1 { /// Typed refusal after one bounded reclaim pass. /// /// Two distinct refusals, never collapsed into one: the modeled reservations -/// are full, or the *measured* process is over budget while the model still -/// claims room. Both name the exact bytes that produced them so no caller has -/// to guess, and neither is ever a silent stall. +/// are full, or measured admission pressure is over budget while the model +/// still claims room. Both name the exact bytes that produced them so callers +/// need not guess, and neither is ever a silent stall. #[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] pub enum ResidentMemoryAdmissionFailureV1 { /// Admitted reservations plus this request exceed the configured limit. @@ -1299,10 +1299,10 @@ pub enum ResidentMemoryAdmissionFailureV1 { requested_bytes: u64, limit_bytes: u64, }, - /// Measured process RSS is at or above the high watermark. Reservations - /// say there is room; the kernel says otherwise, and the measurement wins. + /// Measured admission pressure reached the high watermark. Process + /// allocations or shared cgroup pressure take precedence over reservations. #[error( - "resident-memory admission over budget: observed_rss={observed_bytes} configured_limit={limit_bytes} high_watermark={high_watermark_bytes} requested={requested_bytes} floor={floor_bytes}" + "resident-memory admission over budget: observed_admission={observed_bytes} configured_limit={limit_bytes} high_watermark={high_watermark_bytes} requested={requested_bytes} floor={floor_bytes}" )] ObservedOverBudget { observed_bytes: u64, @@ -1334,9 +1334,9 @@ impl ResidentMemoryAdmissionFailureV1 { } } - /// Whether this refusal came from measured RSS rather than the modeled - /// ceiling. Over-budget refusals clear as pressure falls, so callers retry - /// them instead of treating the input as permanently unservable. + /// Whether this refusal came from measured admission pressure rather than + /// the modeled ceiling. These refusals clear as pressure falls, so callers + /// retry them instead of treating the input as permanently unservable. #[must_use] pub const fn is_observed_over_budget(&self) -> bool { matches!(self, Self::ObservedOverBudget { .. }) From c891bb9602a09b096b538a28d03cc61dbf3dc752 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:54:11 -0700 Subject: [PATCH 12/12] test(sessions): release setup runtimes before census Prepare the twenty persisted project stores with bounded daemon lifetimes. The census assertions need cold stores, not twenty concurrent background indexing runtimes competing with initialization. --- .../stale_project_sessions_census.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_project_sessions_census.rs b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_project_sessions_census.rs index 475230c6f1..e34cd6c230 100644 --- a/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_project_sessions_census.rs +++ b/crates/tracedecay/tests/transport_acceptance_suite/typed_terminal_restart_acceptance/stale_project_sessions_census.rs @@ -352,14 +352,17 @@ fn census_opens_only_project_sessions_stores_with_a_stale_admission_stamp() { let doctor_project = projects[0].1.as_path(); let stale_id = projects[1].2.as_str(); - let mut daemon = spawn_tracedecay_daemon_with(&home_path, |_| {}); + // The census observes persisted stores. Release each setup project's + // background owners before admitting the next one, so preparing the + // profile does not require all twenty indexing runtimes to stay resident. for (_, path, _) in &projects { + let mut daemon = spawn_tracedecay_daemon_with(&home_path, |_| {}); super::initialize_project(&home_path, path, "census-stamp"); wait_for_code_index_hit(&home_path, path, "probe"); + daemon + .kill_and_wait() + .expect("stop the daemon that wrote the project store"); } - daemon - .kill_and_wait() - .expect("stop the daemon that wrote every store"); let mut daemon = spawn_tracedecay_daemon_with(&home_path, |_| {}); assert_eq!( doctor_store_resets(&home_path, doctor_project).1,