diff --git a/deploy/compose/smoke-pins.json b/deploy/compose/smoke-pins.json index f448930cd..5d7cd61e8 100644 --- a/deploy/compose/smoke-pins.json +++ b/deploy/compose/smoke-pins.json @@ -1,5 +1,4 @@ { - "revision": "cc7e1aad3bde47598d161d6372e2e211bb61d9cd", - "release_base": "https://github.com/MiniMax-AI/OpenAgentCore/releases/download/v0.0.3/", - "archive_checksum": "579d2d43accfc45a0a8567db5bc33b407c48b05b0d731bea3fed73e6ade558ae" + "tag": "v0.0.11", + "compose_sha256": "0368e256f8f1830017e5440820cbdb0f148a613fa5e50abdf3a459e20fd72a5e" } diff --git a/docs/maintainers.md b/docs/maintainers.md index 0247fbf63..8e787804f 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -37,7 +37,7 @@ make build-core-distribution | `CORE_DISTRIBUTION_MICROSANDBOX_ARCHIVE` | Cached microsandbox release archive. Default: `~/.oac/cache/microsandbox-v0.7.8-linux-x86_64.tar.gz`, downloaded when missing | | `CORE_DISTRIBUTION_DATABASE_IMAGE` | PostgreSQL 16 image; the default is pinned by its linux/amd64 manifest digest | -The build reuses the Core, Web, Runtime, SDK and helper builders. The manifest records the commit and source tree, image config and OCI manifest digests, the Runtime OCI manifest digest, the microsandbox runtime and firmware hashes, and the size and SHA-256 of every Runtime and node artifact; native installers carry only their SHA-256 in the [catalog](#native-installers). Output is the control archive and its `.sha256`, the optional offline archive, and the versioned Runtime, node and native installer assets. Nothing is published. Rebuilding into a directory that already holds this commit's distribution is refused. +The build reuses the Core, Web, Runtime, SDK and helper builders. The manifest records the commit, its commit time (`source_time`, which orders releases for upgrades) and source tree, image config and OCI manifest digests, the Runtime OCI manifest digest, the microsandbox runtime and firmware hashes, and the size and SHA-256 of every Runtime and node artifact; native installers carry only their SHA-256 in the [catalog](#native-installers). Output is the control archive and its `.sha256`, the optional offline archive, and the versioned Runtime, node and native installer assets. Nothing is published. Rebuilding into a directory that already holds this commit's distribution is refused. The ingress image carries only `oac` and four node metadata files: `manifest.json`, `SHA256SUMS`, `node-install.pyz` and `runtime/seccomp.json`. The build prepares their manifest from the same Runtime and artifact identities as the distribution, then verifies initialization with networking disabled. The node manifest omits control-plane image identities so it can be packaged before the ingress image itself exists. @@ -132,6 +132,8 @@ git push origin v1.2.3 Tags use `vMAJOR.MINOR.PATCH`, optionally with a prerelease suffix such as `-rc.1` and build metadata such as `+build.1`. A prerelease suffix creates a GitHub prerelease. Pushing the tag is the release decision. Automated checks establish build and test results, not real-model qualification: assess live execution evidence before you push the tag. Model credentials and private certificate authorities never enter CI or release inputs, including acceptance images that contain them. +The release notes list every protocol version change (the node wire protocol, the microsandbox helper or the Core–Runtime protocol) with the actions it requires of operators. After the Release is published, set `deploy/compose/smoke-pins.json` to its tag and the SHA-256 of its `compose.yaml`, so the Compose smoke upgrades from the latest published release. + The workflow runs `check` on the tagged commit, including the full local gate, official-client and image acceptance, and the native matrix with its packaging artifacts enabled. After checks succeed, one `build` job on `blacksmith-4vcpu-ubuntu-2204` prepares the pinned Runtime inputs, reuses the native installers, builds the distribution and publishes directly from its local files. This combined job has `contents: write` and `packages: write`; checkout does not persist credentials. It retains the assets as an uncompressed Actions artifact only when publication fails, for recovery, or when a manual build does not publish. Distribution and Runtime archives use `pigz` level 6 with at most four compression workers and no filename or timestamp in the gzip header. The publisher verifies archive and native installer checksums, resolves the repository identity, refuses an existing Release or draft for the tag and creates one draft with a fixed ID. Up to four assets upload concurrently, largest first, without retries. After confirming the complete remote inventory, the publisher validates all image archives and existing registry tags before pushing up to four images concurrently. Each image config and registry manifest is verified; any error leaves the Release unpublished. In-flight transfers finish before a failed operation returns. The publisher rechecks the version tag before publishing the draft by its ID. @@ -172,7 +174,7 @@ The planner compares the PR event's tested merge commit with its verified first | --- | --- | | `hygiene` | Names, repository links, bundled documentation integrity, and CI planner/gate tests; runs for every change | | `distribution` | Harness catalog and installer schema, install/apply/recovery/cleanup tests, Compose parsing and initialization fixtures, release/download and bundle contracts, Go console tests and build; needs Docker Compose for template parsing, no pnpm install or browser | -| `compose` | Real startup from empty volumes using images built from the checkout, sign-in and API access, file upload and node installer download, then URL reconfiguration and container recreation with preserved credentials and data; needs Docker, Go and network access, no model credentials | +| `compose` | Real startup from empty volumes using images built from the checkout, sign-in and API access, file upload and node installer download, then URL reconfiguration and container recreation with preserved credentials and data, and an in-place upgrade from the pinned published release; needs Docker, Go and network access, no model credentials | | `backend` | Parallel parts, each with a dedicated PostgreSQL guard: `runtime` (sqlc freshness, Runtime/shared Go tests, Linux microsandbox helper, daemon build), `core` (standalone Core build, Core service and client tests) and three `integration` shards of the serial Core integration test package | | `harness` | Claude SDK tests and packaging, MiniMax companion scripts | | `example` | Optional application typecheck, tests, build and isolated browser acceptance | @@ -185,7 +187,7 @@ The planner compares the PR event's tested merge commit with its verified first `.github/actionlint.yaml` selects hygiene and lint. Known workflow changes select their consumers: the CI review and actionlint workflows run hygiene and lint; native workflow changes add native checks; API acceptance workflow changes add API checks with container acceptance enabled; website workflow changes add website checks. The shared Node action selects every job that uses it plus lint. A new or unclassified workflow/action selects the full gate until its consumers are declared in the planner. Planner tests and CI measurement scripts run hygiene; changing the planner itself runs the full gate. -Compose template and Compose test changes select both `distribution` fixtures and the `compose` smoke job; Core, Web, shared Go packages and the image Dockerfiles also select the smoke job. Run `python3 scripts/compose-smoke.py` locally with Docker available to repeat it. The script uses a unique project, an automatically assigned loopback port and artifacts under `~/.oac/tests/`; it removes its containers and volumes on exit. CI also performs cleanup after a failed or interrupted smoke step. Diagnostics show container status without printing HTTP response bodies or sign-in keys. Core, Web and the ingress image are built from the checkout; Web serves a placeholder page instead of the console build. Build-time node metadata comes from the release pinned in `deploy/compose/smoke-pins.json`; the initialization container runs with networking disabled. The smoke matrix runs on native Linux amd64 and arm64 runners; the native matrix builds and tests the shared Core installer on Linux, macOS and Windows. +Compose template and Compose test changes select both `distribution` fixtures and the `compose` smoke job; Core, Web, shared Go packages and the image Dockerfiles also select the smoke job. Run `python3 scripts/compose-smoke.py` locally with Docker available to repeat it. The script uses a unique project, an automatically assigned loopback port and artifacts under `~/.oac/tests/`; it removes its containers and volumes on exit. CI also performs cleanup after a failed or interrupted smoke step. Diagnostics show container status without printing HTTP response bodies or sign-in keys. Core, Web and the ingress image are built from the checkout, with node metadata for the checkout's commit; Web serves a placeholder page instead of the console build, and the initialization container runs with networking disabled. The upgrade phase starts a new installation from the published `compose.yaml` of the release pinned by tag and SHA-256 in `deploy/compose/smoke-pins.json`, creates a Project, a Project key and a file, then runs the installer's preflight with the checkout's images (`core check-config`, then `init`) and `up --wait`. It checks that the installation ID, Core key, Project, Project key and file survive, that Core and the node payload report the checkout's commit and that the schema is at the latest migration. The smoke matrix runs on native Linux amd64 and arm64 runners; the native matrix builds and tests the shared Core installer on Linux, macOS and Windows. Go module and workspace inputs select backend, API (including the container), native and distribution checks. Each Node module owns its manifest and lockfile. Website dependencies select website checks; Web dependencies select Web and browser checks; example dependencies select example checks; shared TypeScript client dependencies select Web, browser and example checks; Claude adapter dependencies select Harness, native and distribution checks. Shared package-manager configuration selects all Node consumers. The root TypeScript configuration selects Web and example checks; the adapter TypeScript configuration selects Harness and native checks. Each selected set includes hygiene. Mixed changes accumulate their consumers, and every job reads the same plan instead of maintaining its own path list. For example, a notification-only PR skips database, browser and native jobs, while a notification plus Core change adds backend and API checks. diff --git a/docs/zh/maintainers.md b/docs/zh/maintainers.md index b7529b589..44f90e4c7 100644 --- a/docs/zh/maintainers.md +++ b/docs/zh/maintainers.md @@ -1,7 +1,7 @@ --- title: "构建并发布 OpenAgentCore" source: docs/maintainers.md -source_hash: 402a990e859cb3fb0155e2b4e60421cd841de88f84a49b678eb918450b253114 +source_hash: ae368641567f256f0579182f65cc565e5c6d10161bdb35e6b473306b03167b53 --- 本指南面向负责构建和发布 OpenAgentCore 的维护者。要安装 Core 和 Web,请使用 [安装指南](getting-started/install.md)。安装器代码遵循的规则见 [部署](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/README.md) 和 [节点安装器](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/node/README.md);必需检查见 [CONTRIBUTING](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/CONTRIBUTING.md#required-checks)。 @@ -39,7 +39,7 @@ make build-core-distribution | `CORE_DISTRIBUTION_MICROSANDBOX_ARCHIVE` | 已缓存的 microsandbox 发布归档。默认值:`~/.oac/cache/microsandbox-v0.7.8-linux-x86_64.tar.gz`,缺失时下载 | | `CORE_DISTRIBUTION_DATABASE_IMAGE` | PostgreSQL 16 镜像;默认值通过其 linux/amd64 清单摘要固定 | -构建过程会复用 Core、Web、Runtime、SDK 和辅助程序构建器。清单会记录提交和源代码树、镜像配置及 OCI 清单摘要、Runtime OCI 清单摘要、microsandbox 运行时和固件哈希,以及每个 Runtime 和节点构件的大小与 SHA-256;原生安装器在[目录](#native-installers)中仅记录其 SHA-256。输出包括控制归档及其 `.sha256`、可选的离线归档,以及带版本号的 Runtime、节点和原生安装器资源。此过程不会发布任何内容。如果目标目录中已包含此提交的分发包,重建会拒绝执行。 +构建过程会复用 Core、Web、Runtime、SDK 和辅助程序构建器。清单会记录提交、其提交时间(`source_time`,用于在升级时为发布版本排序)和源代码树、镜像配置及 OCI 清单摘要、Runtime OCI 清单摘要、microsandbox 运行时和固件哈希,以及每个 Runtime 和节点构件的大小与 SHA-256;原生安装器在[目录](#native-installers)中仅记录其 SHA-256。输出包括控制归档及其 `.sha256`、可选的离线归档,以及带版本号的 Runtime、节点和原生安装器资源。此过程不会发布任何内容。如果目标目录中已包含此提交的分发包,重建会拒绝执行。 ingress 镜像仅包含 `oac` 和四个节点元数据文件:`manifest.json`、`SHA256SUMS`、`node-install.pyz` 与 `runtime/seccomp.json`。构建流程根据与分发包相同的 Runtime 和构件身份生成其清单,再禁用网络验证初始化。节点清单不包含控制平面镜像身份,因此可在 ingress 镜像生成前打包。 @@ -134,6 +134,8 @@ git push origin v1.2.3 标签使用 `vMAJOR.MINOR.PATCH` 格式,可选用 `-rc.1` 等预发布后缀以及 `+build.1` 等构建元数据。预发布后缀会创建 GitHub 预发布版。推送标签即表示发布决定。自动检查用于确定构建和测试结果,而不是真实模型资格:推送标签前应评估实际执行证据。模型凭据和私有证书颁发机构绝不能进入 CI 或发布输入,包含它们的验收镜像也不例外。 +发布说明会列出每一项协议版本变更(节点线协议、microsandbox 辅助程序或 Core–Runtime 协议),以及它要求运维人员执行的操作。Release 发布后,将 `deploy/compose/smoke-pins.json` 设为它的标签及其 `compose.yaml` 的 SHA-256,使 Compose 冒烟测试从最新的已发布版本升级。 + 工作流会在带标签的提交上运行 `check`,包括完整的本地门禁、官方客户端和镜像验收,以及启用打包构件的原生平台矩阵。检查成功后,`blacksmith-4vcpu-ubuntu-2204` 上的一个 `build` 作业会准备固定的 Runtime 输入、复用原生安装器、构建分发包,并直接从本地文件发布。此合并作业具有 `contents: write` 和 `packages: write` 权限;检出过程不会保留凭据。只有发布失败或手动构建不发布时,才会把这些文件保留为未压缩的 Actions 构建产物,供恢复使用。 分发归档和 Runtime 归档使用 `pigz` 级别 6,最多使用四个压缩工作线程,并且 gzip 头部中不包含文件名或时间戳。发布器会验证归档和原生安装器校验和、解析仓库身份、拒绝使用该标签已有的 Release 或草稿,并创建一个具有固定 ID 的草稿。最多四个资源可并发上传,按从大到小的顺序进行,且不会重试。确认完整的远程资源清单后,发布器会验证所有镜像归档和现有注册表标签,再并发推送最多四个镜像。每个镜像配置和注册表清单都会接受验证;任何错误都会使 Release 保持未发布状态。失败操作返回前,正在进行的传输会完成。发布器会按 ID 发布草稿,并在发布前重新检查版本标签。 @@ -174,7 +176,7 @@ gh workflow run core-release --repo MiniMax-AI/OpenAgentCore --ref main \ | --- | --- | | `hygiene` | 名称、仓库链接、随包文档完整性以及 CI 计划器/门禁测试;每次变更都会运行 | | `distribution` | Harness 目录和安装器模式、安装/应用/恢复/清理测试、Compose 解析和初始化固定数据、发布/下载和捆绑包契约、Go 控制台测试与构建;模板解析需要 Docker Compose,不需要 pnpm install 或浏览器 | -| `compose` | 使用从当前检出构建的镜像从空数据卷实际启动、登录和 API 访问、上传文件和下载节点安装器,然后在保留凭据和数据的同时重新配置 URL 并重新创建容器;需要 Docker、Go 和网络访问,不需要模型凭据 | +| `compose` | 使用从当前检出构建的镜像从空数据卷实际启动、登录和 API 访问、上传文件和下载节点安装器,然后在保留凭据和数据的同时重新配置 URL 并重新创建容器,并从固定的已发布版本原地升级;需要 Docker、Go 和网络访问,不需要模型凭据 | | `backend` | 并行部分,每部分都有专用 PostgreSQL 保护检查:`runtime`(sqlc 新鲜度、Runtime/共享 Go 测试、Linux microsandbox 辅助程序、守护进程构建)、`core`(独立 Core 构建、Core 服务和客户端测试),以及串行 Core 集成测试包的三个 `integration` 分片 | | `harness` | Claude SDK 测试和打包、MiniMax 配套脚本 | | `example` | 可选的应用程序类型检查、测试、构建和隔离的浏览器验收 | @@ -187,7 +189,7 @@ gh workflow run core-release --repo MiniMax-AI/OpenAgentCore --ref main \ `.github/actionlint.yaml` 会选择 hygiene 和 lint。已知工作流变更会选择其使用方:CI review 和 actionlint 工作流运行 hygiene 和 lint;原生工作流变更会添加原生检查;API 验收工作流变更会添加启用容器验收的 API 检查;网站工作流变更会添加网站检查。共享 Node 操作会选择使用它的每个作业以及 lint。新工作流或未分类的工作流/操作会选择完整门禁,直至在计划器中声明其使用方。计划器测试和 CI 测量脚本运行 hygiene;更改计划器本身会运行完整门禁。 -Compose 模板和 Compose 测试发生变更时,会同时选择 `distribution` 固定数据和 `compose` 冒烟作业;Core、Web、共享 Go 软件包和镜像 Dockerfile 的变更也会选择冒烟作业。安装 Docker 后,可在本地运行 `python3 scripts/compose-smoke.py` 重复该测试。该脚本使用唯一的项目、自动分配的回环端口,并将在 `~/.oac/tests/` 下生成构件;退出时移除其容器和数据卷。CI 还会在冒烟步骤失败或中断后执行清理。诊断信息会显示容器状态,但不会打印 HTTP 响应正文或登录密钥。Core、Web 和 ingress 镜像都从当前检出构建;Web 提供占位页面而不是控制台构建。构建时的节点元数据来自 `deploy/compose/smoke-pins.json` 固定的发布版本;初始化容器禁用网络运行。冒烟矩阵使用 Linux amd64 和 arm64 原生 runner;原生矩阵在 Linux、macOS 和 Windows 上构建并测试共享的 Core 安装器。 +Compose 模板和 Compose 测试发生变更时,会同时选择 `distribution` 固定数据和 `compose` 冒烟作业;Core、Web、共享 Go 软件包和镜像 Dockerfile 的变更也会选择冒烟作业。安装 Docker 后,可在本地运行 `python3 scripts/compose-smoke.py` 重复该测试。该脚本使用唯一的项目、自动分配的回环端口,并将在 `~/.oac/tests/` 下生成构件;退出时移除其容器和数据卷。CI 还会在冒烟步骤失败或中断后执行清理。诊断信息会显示容器状态,但不会打印 HTTP 响应正文或登录密钥。Core、Web 和 ingress 镜像都从当前检出构建,并带有当前检出提交的节点元数据;Web 提供占位页面而不是控制台构建,初始化容器禁用网络运行。升级阶段使用 `deploy/compose/smoke-pins.json` 中按标签和 SHA-256 固定的发布版本的已发布 `compose.yaml` 启动一个新安装,创建一个 Project、一个 Project 密钥和一个文件,然后使用当前检出的镜像运行安装器的预检(先 `core check-config`,再 `init`)和 `up --wait`。它会检查安装 ID、Core 密钥、Project、Project 密钥和文件均被保留,Core 和节点载荷报告当前检出的提交,并且数据库架构处于最新迁移。冒烟矩阵使用 Linux amd64 和 arm64 原生 runner;原生矩阵在 Linux、macOS 和 Windows 上构建并测试共享的 Core 安装器。 Go 模块和工作区输入会选择后端、API(包括容器)、原生和分发检查。每个 Node 模块都拥有自己的清单和锁文件。网站依赖项会选择网站检查;Web 依赖项会选择 Web 和浏览器检查;示例依赖项会选择示例检查;共享 TypeScript 客户端依赖项会选择 Web、浏览器和示例检查;Claude 适配器依赖项会选择 Harness、原生和分发检查。共享包管理器配置会选择所有 Node 使用方。根 TypeScript 配置会选择 Web 和示例检查;适配器 TypeScript 配置会选择 Harness 和原生检查。每个所选集合都包含 hygiene。混合变更会累加其使用方,并且每个作业都读取同一计划,而不是维护各自的路径列表。例如,仅修改通知的 PR 会跳过数据库、浏览器和原生作业,而同时修改通知和 Core 的 PR 会添加后端和 API 检查。 diff --git a/scripts/build-core-distribution.sh b/scripts/build-core-distribution.sh index fe2e4d97b..75df48957 100755 --- a/scripts/build-core-distribution.sh +++ b/scripts/build-core-distribution.sh @@ -222,7 +222,7 @@ msb=(docker run --rm --network none --user "$(id -u):$(id -g)" \ --entrypoint /opt/microsandbox/msb "$core_image") "${msb[@]}" image load --input /runtime.tar --tag oac-runtime:distribution --quiet "${msb[@]}" image inspect oac-runtime:distribution --format json > "$stage/runtime-inspect.json" -python3 scripts/core-distribution-manifest.py node-payload "$bundle" "$stage" "$revision" "$source_tree" "$release_base_url" "$offline" +python3 scripts/core-distribution-manifest.py node-payload "$bundle" "$stage" "$revision" "$source_tree" "$source_epoch" "$release_base_url" "$offline" mkdir -p "$stage/ingress" cp deploy/distribution/Ingress.Dockerfile "$stage/ingress/Dockerfile" cp "$stage/core/bin/oac" "$stage/ingress/oac" diff --git a/scripts/compose-smoke.py b/scripts/compose-smoke.py index cd402ba79..1b3bb0754 100644 --- a/scripts/compose-smoke.py +++ b/scripts/compose-smoke.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """Exercise the Compose installation in an isolated Docker project. -Core, Web and the gateway image are built from this checkout. Web serves a -placeholder page instead of the console build. Node metadata comes from the -release pinned in deploy/compose/smoke-pins.json. +Core, Web and the ingress image are built from this checkout, with node +metadata for this commit. Web serves a placeholder page instead of the console +build. A second phase starts the published release pinned in +deploy/compose/smoke-pins.json and upgrades it in place to this checkout. """ import hashlib @@ -13,8 +14,8 @@ import os from pathlib import Path import re +import shutil import signal -import tarfile import subprocess import tempfile import urllib.error @@ -22,44 +23,42 @@ import uuid ROOT = Path(__file__).resolve().parents[1] -render_spec = importlib.util.spec_from_file_location("render_compose", ROOT / "scripts/render-compose.py") -render_compose = importlib.util.module_from_spec(render_spec) -render_spec.loader.exec_module(render_compose) - - -def prepare_pinned_payload(destination): - """Build fixtures use verified release metadata; initialization stays offline.""" - pins = json.loads((ROOT / 'deploy/compose/smoke-pins.json').read_text()) - stem = 'oac-' + pins['revision'] + '-linux-amd64' - destination.mkdir(parents=True, exist_ok=True) - with tempfile.TemporaryDirectory(dir=destination.parent) as temporary: - archive = Path(temporary) / 'release.tar.gz' - urllib.request.urlretrieve(pins['release_base'] + stem + '.tar.gz', archive) - digest = hashlib.sha256() - with archive.open('rb') as stream: - for block in iter(lambda: stream.read(1024 * 1024), b''): - digest.update(block) - if digest.hexdigest() != pins['archive_checksum']: - raise ValueError('Pinned smoke metadata checksum mismatch') - with tarfile.open(archive, 'r:gz') as release: - for name in ('manifest.json', 'SHA256SUMS', 'node-install.pyz', 'runtime/seccomp.json'): - member = release.getmember(stem + '/' + name) - if not member.isfile() or member.size > 1024 * 1024: - raise ValueError('Invalid smoke metadata member') - target = destination / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(release.extractfile(member).read()) - return pins['revision'] - - -def build_images(directory, tag): - revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip() +RELEASES = 'https://github.com/MiniMax-AI/OpenAgentCore/releases/download/' + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, ROOT / path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +render_compose = load('render_compose', 'scripts/render-compose.py') +distribution = load('distribution', 'scripts/core-distribution-manifest.py') + + +def prepare_payload(destination, revision, source_time): + """Node metadata for this commit, built with the distribution's own builders.""" + (destination / 'runtime').mkdir(parents=True) + with tempfile.TemporaryDirectory(dir=destination.parent) as sources: + sources = Path(sources) + for path in (ROOT / 'deploy/node').glob('*.py'): + shutil.copyfile(path, sources / path.name) + distribution.bootstraps(sources, source_time, revision) + shutil.copyfile(sources / 'node-install.pyz', destination / 'node-install.pyz') + shutil.copyfile(ROOT / 'services/core/deploy/codex/seccomp.json', destination / 'runtime/seccomp.json') + (destination / 'manifest.json').write_text(json.dumps( + {'source_commit': revision, 'source_time': int(source_time), 'platform': 'linux/amd64'})) + distribution.checksums(destination) + + +def build_images(directory, tag, revision, source_time): protocol = re.search(r'const Version = "([^"]+)"', (ROOT / 'internal/agentdaemon/proto/version.go').read_text()).group(1) go_env = {**os.environ, 'CGO_ENABLED': '0', 'GOOS': 'linux', 'GOARCH': os.environ.get('GOARCH', 'amd64')} - def go_build(package, output, build_revision=revision): + def go_build(package, output): output.parent.mkdir(parents=True, exist_ok=True) - subprocess.run(['go', 'build', '-trimpath', '-ldflags', '-X main.buildRevision=' + build_revision, + subprocess.run(['go', 'build', '-trimpath', '-ldflags', '-X main.buildRevision=' + revision, '-o', str(output), './' + package], cwd=ROOT, env=go_env, check=True) contexts = {name: directory / ('image-' + name) for name in ('core', 'web', 'ingress')} @@ -80,8 +79,8 @@ def go_build(package, output, build_revision=revision): (web / 'dist/index.html').write_text('Compose smoke\n') (web / 'Dockerfile').write_bytes((ROOT / 'services/web/Dockerfile').read_bytes()) ingress = contexts['ingress'] - payload_revision = prepare_pinned_payload(ingress / 'node-payload') - go_build('services/core/cmd/oac', ingress / 'oac', payload_revision) + prepare_payload(ingress / 'node-payload', revision, source_time) + go_build('services/core/cmd/oac', ingress / 'oac') (ingress / 'Dockerfile').write_bytes((ROOT / 'deploy/distribution/Ingress.Dockerfile').read_bytes()) for path in directory.glob('image-*/**/*'): path.chmod(0o755 if path.is_dir() or os.access(path, os.X_OK) else 0o644) @@ -103,33 +102,40 @@ def main(): directory = Path(tempfile.mkdtemp(prefix=project + '-', dir=artifacts)) data = directory / 'data' data.mkdir(mode=0o700) - pins = json.loads((ROOT / 'deploy/compose/smoke-pins.json').read_text()) + revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip() + source_time = subprocess.check_output(['git', 'show', '-s', '--format=%ct', 'HEAD'], cwd=ROOT, text=True).strip() rendered = directory / 'compose.yaml' rendered.write_text(render_compose.render({ - 'REVISION': pins['revision'], + 'REVISION': revision, 'INIT_IMAGE': 'ghcr.io/minimax-ai/openagentcore/ingress@sha256:' + '0' * 64, 'CORE_IMAGE': 'ghcr.io/minimax-ai/openagentcore/core@sha256:' + '0' * 64, 'WEB_IMAGE': 'ghcr.io/minimax-ai/openagentcore/web@sha256:' + '0' * 64, })) override = directory / 'offline-init.json' - images = build_images(directory, project.removeprefix('oac-smoke-')) + images = build_images(directory, project.removeprefix('oac-smoke-'), revision, source_time) override.write_text(json.dumps({'services': {'init': {'network_mode': 'none'}}})) env = {**os.environ, 'COMPOSE_PROGRESS': 'plain', 'OAC_HOST': '127.0.0.1', 'OAC_WEB_PORT': '0', **{'OAC_IMAGE_' + name.upper(): image for name, image in images.items()}} env.pop('OAC_PUBLIC_URL', None) - command = ['docker', 'compose', '--env-file', os.devnull, '-p', project, - '-f', str(rendered), '-f', str(override)] - - def compose(*args, timeout=120): - result = subprocess.run(command + list(args), env=env, cwd=ROOT, capture_output=True, timeout=timeout) - if result.returncode: - # Startup/configuration stderr helps diagnose failures before containers exist. - # Tool output and application logs may contain credentials and stay private. - detail = '' if args[0] in ('run', 'logs') else result.stderr.decode(errors='replace')[-4096:] - raise RuntimeError(f'Compose {args[0]} failed with exit code {result.returncode}\n{detail}') - return result.stdout + + def runner(files, environment): + command = ['docker', 'compose', '--env-file', os.devnull, '-p', project] + for file in files: + command += ['-f', str(file)] + + def compose(*args, timeout=120): + result = subprocess.run(command + list(args), env=environment, cwd=ROOT, capture_output=True, timeout=timeout) + if result.returncode: + # Startup/configuration stderr helps diagnose failures before containers exist. + # Tool output and application logs may contain credentials and stay private. + detail = '' if args[0] in ('run', 'logs') else result.stderr.decode(errors='replace')[-4096:] + raise RuntimeError(f'Compose {args[0]} failed with exit code {result.returncode}\n{detail}') + return result.stdout + return compose + + compose = runner([rendered, override], env) def client(): return urllib.request.build_opener(urllib.request.ProxyHandler({}), @@ -163,6 +169,15 @@ def private_logs(*keys): assert all(key not in logs for key in keys), 'Credentials appeared in container logs' return logs + def upload(api, content): + boundary = 'oac-compose-smoke' + body = (f'--{boundary}\r\nContent-Disposition: form-data; name="purpose"\r\n\r\nuser_data\r\n' + f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="smoke.txt"\r\n' + 'Content-Type: text/plain\r\n\r\n').encode() + content + f'\r\n--{boundary}--\r\n'.encode() + uploaded = get('/v1/files', body=body, headers={**api, 'Content-Type': 'multipart/form-data; boundary=' + boundary}) + assert uploaded['bytes'] == len(content), 'Upload was truncated' + return uploaded + def terminate(_signum, _frame): raise SystemExit(1) @@ -188,13 +203,8 @@ def terminate(_signum, _frame): installer = request('/node-install/node-install.pyz') checksums = dict(line.split(' ', 1)[::-1] for line in request('/node-install/SHA256SUMS').decode().splitlines()) assert hashlib.sha256(installer).hexdigest() == checksums['node-install.pyz'], 'Node installer checksum mismatch' - boundary = 'oac-compose-smoke' content = b'x' * (5 * 1024 * 1024) - body = (f'--{boundary}\r\nContent-Disposition: form-data; name="purpose"\r\n\r\nuser_data\r\n' - f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="smoke.txt"\r\n' - 'Content-Type: text/plain\r\n\r\n').encode() + content + f'\r\n--{boundary}--\r\n'.encode() - uploaded = get('/v1/files', body=body, headers={**api, 'Content-Type': 'multipart/form-data; boundary=' + boundary}) - assert uploaded['bytes'] == len(content), 'Upload was truncated' + uploaded = upload(api, content) private_logs(key, project_key) print('Configuring a reachable URL and recreating containers with the same data directory', flush=True) @@ -226,9 +236,53 @@ def terminate(_signum, _frame): assert compose('exec', '-T', 'web', '/usr/local/bin/oac-web', 'core-key').decode().strip() == rotated, 'Rotated key was not retained' private_logs(key, rotated, project_key) print('PASS: startup, origin validation, sign-in, API, upload, node installer, key rotation and persistent installation', flush=True) + + pins = json.loads((ROOT / 'deploy/compose/smoke-pins.json').read_text()) + print(f'Starting the published {pins["tag"]} release in a new installation and upgrading it to this checkout', flush=True) + compose('down', '--volumes') + published = directory / 'published-compose.yaml' + urllib.request.urlretrieve(RELEASES + pins['tag'] + '/compose.yaml', published) + if hashlib.sha256(published.read_bytes()).hexdigest() != pins['compose_sha256']: + raise ValueError('Pinned release compose.yaml checksum mismatch') + env['OAC_WEB_PORT'] = '0' + env.pop('OAC_PUBLIC_URL') + released = runner([published], {name: value for name, value in env.items() if not name.startswith('OAC_IMAGE_')}) + released('up', '-d', '--wait', '--wait-timeout', '600', timeout=900) + address = 'http://' + released('port', 'web', '8080').decode().strip() + origin = 'http://localhost:8080' + browser = client() + key = released('exec', '-T', 'web', '/usr/local/bin/oac-web', 'core-key').decode().strip() + request('/console/auth/login', {'core_key': key}) + facts = get('/core/v1/installation') + project_data = get('/core/v1/projects', body={'name': 'Upgrade smoke'}, status=201) + project_key = get('/core/v1/projects/' + project_data['id'] + '/keys', body={'name': 'smoke'}, status=201)['key'] + api = {'Authorization': 'Bearer ' + project_key} + content = b'upgrade smoke\n' + uploaded = upload(api, content) + + # The installer's preflight with the new release's images, then its start. + compose('run', '--rm', '--no-deps', '--entrypoint', '/usr/local/bin/oac-core', 'core', 'check-config') + compose('run', '--rm', '--no-deps', 'init') + compose('up', '-d', '--wait', '--wait-timeout', '180', '--pull', 'never', '--remove-orphans', timeout=240) + address = 'http://' + compose('port', 'web', '8080').decode().strip() + browser = client() + assert compose('exec', '-T', 'web', '/usr/local/bin/oac-web', 'core-key').decode().strip() == key, 'Upgrade changed the Core key' + request('/console/auth/login', {'core_key': key}) + upgraded = get('/core/v1/installation') + assert upgraded['installation_id'] == facts['installation_id'], 'Upgrade changed the installation identity' + assert upgraded['source_commit'] == revision, 'Core does not run this checkout' + assert json.loads(request('/node-install/manifest.json'))['source_commit'] == revision, 'Web serves another release' + assert any(p['id'] == project_data['id'] for p in get('/core/v1/projects')['data']), 'Upgrade lost the Project' + assert get('/v1/files/' + uploaded['id'], headers=api)['bytes'] == len(content), 'Upgrade lost the Project key or file' + schema = compose('exec', '-T', 'database', 'psql', '-U', 'agents_api', '-d', 'agents_api', '-Atc', + 'SELECT max(version_id) FROM agents_api_schema_version').decode().strip() + latest = max(int(path.name.split('_', 1)[0]) for path in (ROOT / 'services/core/migrations').glob('*.sql')) + assert int(schema) == latest, 'Upgrade did not migrate the schema' + private_logs(key, project_key) + print(f'PASS: in-place upgrade from {pins["tag"]} keeps the installation, Core key, Project, Project key and file', flush=True) except BaseException: # Service status identifies failed containers without dumping secret-bearing logs. - status = subprocess.run(command + ['ps', '--all'], env=env, capture_output=True, timeout=30) + status = subprocess.run(['docker', 'compose', '-p', project, 'ps', '--all'], env=env, capture_output=True, timeout=30) print(status.stdout.decode(), flush=True) raise finally: diff --git a/scripts/core-distribution-manifest.py b/scripts/core-distribution-manifest.py index 6cffe3b41..c8fcd05a4 100644 --- a/scripts/core-distribution-manifest.py +++ b/scripts/core-distribution-manifest.py @@ -320,7 +320,7 @@ def bootstraps(bundle, epoch, revision): zipapp.create_archive(directory, bundle / "node-install.pyz", compressed=True) -def node_payload(bundle, stage, revision, source_tree, artifact_base_url="", offline="0"): +def node_payload(bundle, stage, revision, source_tree, source_time, artifact_base_url="", offline="0"): bundle, stage = pathlib.Path(bundle), pathlib.Path(stage) artifact_base_url = release_base(artifact_base_url) if not artifact_base_url and offline != "1": @@ -339,6 +339,8 @@ def node_payload(bundle, stage, revision, source_tree, artifact_base_url="", off metadata = { "source_commit": revision, "source_tree": source_tree, + # The commit time orders releases: initialization refuses an older one. + "source_time": int(source_time), "platform": "linux/amd64", "artifact_base_url": artifact_base_url, "artifacts": package_artifacts(bundle, stage, revision), diff --git a/scripts/core-distribution-manifest.test.py b/scripts/core-distribution-manifest.test.py index cb4b78ce8..5d3730f8f 100644 --- a/scripts/core-distribution-manifest.test.py +++ b/scripts/core-distribution-manifest.test.py @@ -21,6 +21,7 @@ REVISION = "a" * 40 TREE = "b" * 40 +SOURCE_TIME = "1760000000" RELEASE_BASE = "https://example.com/releases/" + REVISION @@ -112,7 +113,7 @@ def setUp(self): self.runtime_bytes = (self.bundle / "images/runtime.tar").read_bytes() def manifest(self, base=RELEASE_BASE, offline="0"): - distribution.node_payload(self.bundle, self.stage, REVISION, TREE, base, offline) + distribution.node_payload(self.bundle, self.stage, REVISION, TREE, SOURCE_TIME, base, offline) distribution.manifest(self.bundle, self.stage) def write_inspection(self): @@ -128,8 +129,9 @@ def test_init_payload_contains_only_verified_node_metadata(self): self.assertEqual(distribution.sha256(payload / name), checksum) bundled = json.loads((payload / "manifest.json").read_text()) full = json.loads((self.bundle / "manifest.json").read_text()) - for name in ("source_commit", "platform", "artifacts", "runtime_ref", "microsandbox"): + for name in ("source_commit", "source_time", "platform", "artifacts", "runtime_ref", "microsandbox"): self.assertEqual(bundled[name], full[name]) + self.assertEqual(bundled["source_time"], 1760000000) self.assertEqual(list(bundled["images"]), ["runtime"]) def test_oci_manifest_identity_is_distinct_from_docker_config_identity(self): diff --git a/services/core/cmd/oac/init.go b/services/core/cmd/oac/init.go index 9128bc68b..e2772e2d4 100644 --- a/services/core/cmd/oac/init.go +++ b/services/core/cmd/oac/init.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "fmt" "io/fs" "os" "path/filepath" @@ -32,6 +33,14 @@ type releaseIdentity struct { revision string } +// releaseManifest holds the node manifest fields initialization reads. +// source_time, the commit time in Unix seconds, orders releases. +type releaseManifest struct { + SourceCommit string `json:"source_commit"` + SourceTime int64 `json:"source_time"` + Platform string `json:"platform"` +} + func initCommand() error { revision := os.Getenv("OAC_REVISION") if revision == "" { @@ -83,10 +92,7 @@ func readRelease(root string, release releaseIdentity) (map[string][]byte, error return nil, errors.New("release metadata checksum mismatch") } } - var manifest struct { - SourceCommit string `json:"source_commit"` - Platform string `json:"platform"` - } + var manifest releaseManifest if err := json.Unmarshal(files["manifest.json"], &manifest); err != nil { return nil, err } @@ -175,15 +181,27 @@ func initialize(root string, release releaseIdentity, fetch func() (map[string][ nextStep("verify_existing_installation") marker := filepath.Join(root, "installation.json") - if raw, err := os.ReadFile(marker); err == nil { - var receipt installReceipt - if err := json.Unmarshal(raw, &receipt); err != nil { - return err + var previous installReceipt + raw, err := os.ReadFile(marker) + switch { + case errors.Is(err, fs.ErrNotExist): + nextStep("verify_empty_data") + for _, name := range []string{"database", "state"} { + entries, err := os.ReadDir(filepath.Join(root, name)) + if err != nil { + return err + } + if len(entries) > 0 { + return errors.New("existing data requires its original installation files") + } } - if receipt.SourceCommit != release.revision { - return errors.New("this data directory belongs to another release; create a new installation") + case err != nil: + return err + default: + if err := json.Unmarshal(raw, &previous); err != nil { + return err } - for name, checksum := range receipt.Files { + for name, checksum := range previous.Files { actual, err := fileDigest(filepath.Join(root, name)) if err != nil { return err @@ -192,22 +210,16 @@ func initialize(root string, release releaseIdentity, fetch func() (map[string][ return errors.New("installation files changed; restore the matching data directory") } } - if err := syncCoreKeyDigest(root); err != nil { - return err - } - logger.InfoContext(ctx, "Existing installation verified", "file_count", len(receipt.Files)) - return nil - } else if !errors.Is(err, fs.ErrNotExist) { - return err - } - nextStep("verify_empty_data") - for _, name := range []string{"database", "state"} { - entries, err := os.ReadDir(filepath.Join(root, name)) - if err != nil { - return err - } - if len(entries) > 0 { - return errors.New("existing data requires its original installation files") + if previous.SourceCommit == release.revision { + // The pointer follows the receipt, so a crash between the two heals here. + if err := writePointer(root, release); err != nil { + return err + } + if err := syncCoreKeyDigest(root); err != nil { + return err + } + logger.InfoContext(ctx, "Existing installation verified", "file_count", len(previous.Files)) + return nil } } nextStep("verify_bundled_metadata") @@ -216,9 +228,23 @@ func initialize(root string, release releaseIdentity, fetch func() (map[string][ return err } logger.InfoContext(ctx, "Bundled node installation metadata verified", "file_count", len(files)) + if previous.SourceCommit != "" { + nextStep("verify_release_order") + if err := requireNewer(root, previous.SourceCommit, release, files["manifest.json"]); err != nil { + return err + } + logger.InfoContext(ctx, "Upgrading installation", "from_revision", previous.SourceCommit) + } nextStep("publish_node_metadata") - prefix := "node-payload/releases/" + release.revision + "/" + // The receipt keeps every retained release authenticated; old releases stay + // for the nodes that still run them. names := []string{} + for name := range previous.Files { + if strings.HasPrefix(name, "node-payload/releases/") { + names = append(names, name) + } + } + prefix := "node-payload/releases/" + release.revision + "/" for _, name := range releaseMembers { if err := writeOwned(filepath.Join(root, prefix+name), files[name]); err != nil { return err @@ -226,10 +252,6 @@ func initialize(root string, release releaseIdentity, fetch func() (map[string][ logger.InfoContext(ctx, "Node metadata file copied", "file", name) names = append(names, prefix+name) } - active, _ := json.Marshal(map[string]string{"source_commit": release.revision}) - if err := writeOwned(filepath.Join(root, "node-payload", "active.json"), active); err != nil { - return err - } if err := filepath.WalkDir(filepath.Join(root, "node-payload"), func(path string, entry fs.DirEntry, err error) error { if err != nil || !entry.IsDir() { return err @@ -277,7 +299,6 @@ func initialize(root string, release releaseIdentity, fetch func() (map[string][ return err } - names = append(names, "node-payload/active.json") nextStep("write_installation_receipt") receipt := installReceipt{SourceCommit: release.revision, Files: map[string]string{}} for _, name := range names { @@ -287,15 +308,50 @@ func initialize(root string, release releaseIdentity, fetch func() (map[string][ } receipt.Files[name] = checksum } - raw, _ := json.Marshal(receipt) + raw, _ = json.Marshal(receipt) + // The receipt is the commit point; the pointer follows it. if err := writeOwned(marker, raw); err != nil { return err } + if err := writePointer(root, release); err != nil { + return err + } logger.InfoContext(ctx, "Installation initialized", "sign_in_key_command", "docker compose exec web oac-web core-key") return nil }) } +// writePointer selects the installation's release for Web and node installs. +func writePointer(root string, release releaseIdentity) error { + active, _ := json.Marshal(map[string]string{"source_commit": release.revision}) + return writeOwned(filepath.Join(root, "node-payload", "active.json"), active) +} + +// requireNewer refuses a release that is not newer than the installed one by +// commit time. A manifest without source_time reads as 0, the oldest. +func requireNewer(root, installed string, release releaseIdentity, manifest []byte) error { + raw, err := os.ReadFile(filepath.Join(root, "node-payload", "releases", installed, "manifest.json")) + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("installation receipt names release %.12s, whose node metadata is missing; restore the matching data directory", installed) + } + if err != nil { + return err + } + var current, next releaseManifest + if err := json.Unmarshal(raw, ¤t); err != nil { + return err + } + if err := json.Unmarshal(manifest, &next); err != nil { + return err + } + if next.SourceTime > current.SourceTime { + return nil + } + date := func(seconds int64) string { return time.Unix(seconds, 0).UTC().Format("2006-01-02 15:04 UTC") } + return fmt.Errorf("This installation runs release %.12s (%s); this release, %.12s (%s), is not newer. Downgrades are not supported: run that release or a newer one, or restore the backup taken before the upgrade.", + installed, date(current.SourceTime), release.revision, date(next.SourceTime)) +} + func randomBytes(n int) []byte { value := make([]byte, n) if _, err := rand.Read(value); err != nil { diff --git a/services/core/cmd/oac/init_test.go b/services/core/cmd/oac/init_test.go index c005a2fc3..5f0864978 100644 --- a/services/core/cmd/oac/init_test.go +++ b/services/core/cmd/oac/init_test.go @@ -25,13 +25,23 @@ func initFixture(t *testing.T) (string, releaseIdentity, map[string][]byte) { previous := chown chown = func(string, int, int) error { return nil } t.Cleanup(func() { chown = previous }) - release := releaseIdentity{strings.Repeat("d", 40)} + release, files := releaseFixture(strings.Repeat("d", 40), 1760000000) + return t.TempDir(), release, files +} + +// releaseFixture returns one release's metadata. A zero sourceTime omits +// source_time, as v0.0.10 and v0.0.11 manifests do. +func releaseFixture(revision string, sourceTime int64) (releaseIdentity, map[string][]byte) { files := map[string][]byte{} for _, name := range releaseMembers { - files[name] = []byte("fixture") + files[name] = []byte("fixture " + revision) } - files["manifest.json"], _ = json.Marshal(map[string]string{"source_commit": release.revision, "platform": "linux/amd64"}) - return t.TempDir(), release, files + manifest := map[string]any{"source_commit": revision, "platform": "linux/amd64"} + if sourceTime != 0 { + manifest["source_time"] = sourceTime + } + files["manifest.json"], _ = json.Marshal(manifest) + return releaseIdentity{revision}, files } func snapshot(t *testing.T, root string) map[string]string { @@ -110,7 +120,7 @@ func TestInitializeRefusesDataWithoutItsInstallationFiles(t *testing.T) { } } -func TestInitializeRefusesChangedMissingOrForeignFiles(t *testing.T) { +func TestInitializeRefusesChangedOrMissingFiles(t *testing.T) { root, release, files := initFixture(t) if err := initialize(root, release, fixed(files)); err != nil { t.Fatal(err) @@ -126,9 +136,93 @@ func TestInitializeRefusesChangedMissingOrForeignFiles(t *testing.T) { t.Fatalf("missing: %v", err) } _ = os.WriteFile(path, original, 0o600) - release.revision = strings.Repeat("f", 40) - if err := initialize(root, release, fixed(files)); err == nil || !strings.Contains(err.Error(), "another release") { - t.Fatalf("foreign: %v", err) + if err := initialize(root, release, refuseDownload(t)); err != nil { + t.Fatalf("restored: %v", err) + } +} + +func readReceipt(t *testing.T, root string) installReceipt { + t.Helper() + var receipt installReceipt + raw, err := os.ReadFile(filepath.Join(root, "installation.json")) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(raw, &receipt); err != nil { + t.Fatal(err) + } + return receipt +} + +func TestInitializeUpgradesAnOlderInstallationInPlace(t *testing.T) { + root, release, files := initFixture(t) + old, oldFiles := releaseFixture(strings.Repeat("c", 40), 0) + if err := initialize(root, old, fixed(oldFiles)); err != nil { + t.Fatal(err) + } + // Receipts written by v0.0.10 and v0.0.11 also pin the release pointer. + receipt := readReceipt(t, root) + pointer := filepath.Join(root, "node-payload", "active.json") + receipt.Files["node-payload/active.json"], _ = fileDigest(pointer) + raw, _ := json.Marshal(receipt) + if err := os.WriteFile(filepath.Join(root, "installation.json"), raw, 0o600); err != nil { + t.Fatal(err) + } + before := snapshot(t, root) + + if err := initialize(root, release, fixed(files)); err != nil { + t.Fatal(err) + } + after := snapshot(t, root) + for name, data := range before { + if (strings.HasPrefix(name, "secrets/") || strings.HasPrefix(name, "node-payload/releases/")) && after[name] != data { + t.Fatalf("upgrade changed %s", name) + } + } + receipt = readReceipt(t, root) + if receipt.SourceCommit != release.revision { + t.Fatalf("receipt release = %s", receipt.SourceCommit) + } + for _, revision := range []string{old.revision, release.revision} { + for _, name := range releaseMembers { + if receipt.Files["node-payload/releases/"+revision+"/"+name] == "" { + t.Fatalf("receipt lacks %s of %s", name, revision) + } + } + } + if _, pinned := receipt.Files["node-payload/active.json"]; pinned { + t.Fatal("receipt still pins the release pointer") + } + want := `{"source_commit":"` + release.revision + `"}` + if after["node-payload/active.json"] != want { + t.Fatalf("pointer = %s", after["node-payload/active.json"]) + } + + // A crash between the receipt and the pointer heals on the next start. + if err := os.WriteFile(pointer, []byte(before["node-payload/active.json"]), 0o600); err != nil { + t.Fatal(err) + } + if err := initialize(root, release, refuseDownload(t)); err != nil { + t.Fatal(err) + } + if healed, _ := os.ReadFile(pointer); string(healed) != want { + t.Fatalf("pointer = %s", healed) + } +} + +func TestInitializeRefusesAnOlderRelease(t *testing.T) { + root, release, files := initFixture(t) + if err := initialize(root, release, fixed(files)); err != nil { + t.Fatal(err) + } + before := snapshot(t, root) + older, olderFiles := releaseFixture(strings.Repeat("e", 40), 1700000000) + want := "This installation runs release dddddddddddd (2025-10-09 08:53 UTC); this release, eeeeeeeeeeee (2023-11-14 22:13 UTC), is not newer. Downgrades are not supported: run that release or a newer one, or restore the backup taken before the upgrade." + if err := initialize(root, older, fixed(olderFiles)); err == nil || err.Error() != want { + t.Fatalf("err = %v", err) + } + if !maps.Equal(before, snapshot(t, root)) { + t.Fatal("a refused downgrade changed installation files") } } diff --git a/services/core/migrations/migrations.go b/services/core/migrations/migrations.go index b3802d763..121ba1a84 100644 --- a/services/core/migrations/migrations.go +++ b/services/core/migrations/migrations.go @@ -5,6 +5,7 @@ import ( "context" "database/sql" "embed" + "fmt" _ "github.com/jackc/pgx/v5/stdlib" "github.com/pressly/goose/v3" @@ -15,7 +16,8 @@ import ( var files embed.FS // Apply upgrades the dedicated execution database without consulting product -// data. A session advisory lock serializes concurrent Core starts. +// data and refuses a schema newer than its migrations. A session advisory lock +// serializes concurrent Core starts. func Apply(ctx context.Context, databaseURL string) error { db, err := sql.Open("pgx", databaseURL) if err != nil { @@ -31,6 +33,17 @@ func Apply(ctx context.Context, databaseURL string) error { if err != nil { return err } - _, err = provider.Up(ctx) - return err + if _, err = provider.Up(ctx); err != nil { + return err + } + // Up ignores versions above the embedded ones, so an older Core would start + // on a schema it does not know. + current, target, err := provider.GetVersions(ctx) + if err != nil { + return err + } + if current > target { + return fmt.Errorf("Agents API database schema version %d is newer than this Core supports (%d). Downgrades are not supported: run the release that upgraded this database or a newer one, or restore the backup taken before the upgrade.", current, target) + } + return nil } diff --git a/services/core/migrations/migrations_test.go b/services/core/migrations/migrations_test.go new file mode 100644 index 000000000..bf176591c --- /dev/null +++ b/services/core/migrations/migrations_test.go @@ -0,0 +1,24 @@ +package migrations_test + +import ( + "strings" + "testing" + + "github.com/jackc/pgx/v5/stdlib" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/persistence/postgres/pgtest" + "github.com/MiniMax-AI/OpenAgentCore/services/core/migrations" +) + +func TestApplyRefusesANewerSchema(t *testing.T) { + pool := pgtest.OpenIsolated(t, nil) + if _, err := pool.Exec(t.Context(), "INSERT INTO agents_api_schema_version (version_id, is_applied) VALUES (999999, true)"); err != nil { + t.Fatal(err) + } + connection := stdlib.RegisterConnConfig(pool.Config().ConnConfig) + defer stdlib.UnregisterConnConfig(connection) + err := migrations.Apply(t.Context(), connection) + if err == nil || !strings.HasPrefix(err.Error(), "Agents API database schema version 999999 is newer than this Core supports (") { + t.Fatalf("err = %v", err) + } +} diff --git a/services/web/distribution_test.go b/services/web/distribution_test.go index 7e36414b7..51e0608df 100644 --- a/services/web/distribution_test.go +++ b/services/web/distribution_test.go @@ -1,6 +1,8 @@ package main import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "net/http/httptest" "net/url" @@ -170,6 +172,13 @@ func TestRetainedPayloadsRemainHTTPReachableAfterPublication(t *testing.T) { if code, body := request("node-install.pyz"); code != 200 || body != "installer-"+revision { t.Fatal("active release not served", code, body) } + // Add node commands verify the active release's installer without a Web restart. + config := httptest.NewRecorder() + h.serveConsoleConfiguration(config, httptest.NewRequest("GET", "/console/config", nil)) + digest := sha256.Sum256([]byte("installer-" + revision)) + if !strings.Contains(config.Body.String(), `"node_installer_sha256":"`+hex.EncodeToString(digest[:])+`"`) { + t.Fatal("installer digest does not follow the active release", config.Body.String()) + } if code, body := request("releases/" + a + "/artifacts/node"); code != 200 || body != "binary-"+a { t.Fatal("retained release unreachable", code, body) } diff --git a/services/web/node_installation.go b/services/web/node_installation.go index 2747292ba..4825af720 100644 --- a/services/web/node_installation.go +++ b/services/web/node_installation.go @@ -181,11 +181,17 @@ func (h *console) nodeArtifacts() []string { return available } +// serveConsoleConfiguration reads the active release on every request, so an +// upgrade that moves the pointer needs no Web restart. func (h *console) serveConsoleConfiguration(w http.ResponseWriter, _ *http.Request) { + digest := "" + if h.nodePayload != nil { + digest, _ = installerDigest(h.nodePayload, "node-install.pyz") + } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(struct { NodeInstaller bool `json:"node_installer"` NodeInstallerSHA256 string `json:"node_installer_sha256"` NodeArtifacts []string `json:"node_artifacts"` - }{h.nodePayload != nil, h.nodeInstallerDigest, h.nodeArtifacts()}) + }{digest != "", digest, h.nodeArtifacts()}) } diff --git a/services/web/server.go b/services/web/server.go index 01d43dd7c..84bc4f8c2 100644 --- a/services/web/server.go +++ b/services/web/server.go @@ -15,13 +15,12 @@ import ( type console struct { config - root *os.Root - nodePayload *os.Root - nodeInstallerDigest string - proxy, direct *httputil.ReverseProxy - transport *http.Transport - host string - auth *consoleAuth + root *os.Root + nodePayload *os.Root + proxy, direct *httputil.ReverseProxy + transport *http.Transport + host string + auth *consoleAuth } type consoleActorContextKey struct{} @@ -47,8 +46,7 @@ func newConsole(c config) (*console, error) { root.Close() return nil, errors.New("cannot open node installation payload") } - h.nodeInstallerDigest, err = installerDigest(h.nodePayload, "node-install.pyz") - if err != nil { + if _, err = installerDigest(h.nodePayload, "node-install.pyz"); err != nil { h.nodePayload.Close() root.Close() return nil, err