From 0e8f59138089100d55fe9d5e5fb5d3535dce7f76 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Wed, 14 Oct 2026 13:44:40 +0000 Subject: [PATCH 1/3] Upgrade installations with the installer oac install --upgrade and install.py --upgrade move an existing installation to the installer's release in place. They stage the new files, check them with oac-core check-config and init against the installation's .env and override, and only then replace oac and compose.yaml and start the services. Without --upgrade, another release is refused before any Docker change; the same release resumes through the start oac apply uses. The release checksum is verified in the stage only, and starts recreate changed containers and remove orphans. node_payload is split into publish and activate so a bundle upgrade keeps earlier releases and moves the active pointer after the checks. --- deploy/README.md | 8 +- deploy/install-source.py | 132 ++++++--- deploy/node/README.md | 2 +- deploy/node/node_payload.py | 178 ++++++------- .../test_generation_review_regressions.py | 20 +- deploy/test_install_source.py | 51 +++- services/core/cmd/oac/install.go | 251 ++++++++++++------ services/core/cmd/oac/install_test.go | 161 ++++++++--- services/core/cmd/oac/main.go | 13 +- 9 files changed, 562 insertions(+), 254 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 01a50d6ea..4d10338d0 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -12,9 +12,13 @@ `install.sh` and `install.ps1` select a native `oac` release binary, verify its checksum and invoke `oac install`. That Go command owns the same installation lifecycle on Linux, macOS and Windows: validate Docker, verify release configuration, prepare `.env` and the host command, pull images, publish the installation directory, initialize the data volume and start Compose. [Installation](../docs/getting-started/install.md#prerequisites) owns platform prerequisites; [Configuration](../docs/configuration.md) owns settings and storage. -Preparation happens in `.staging`. On entry, the installer removes an empty stage or one with the empty ownership directory `.oac-installer-`; it preserves unrecognized directories. Ordinary failures remove owned staging immediately. The ownership marker is created atomically before downloading files. A killed process leaves staging for the next attempt. Publication happens only after downloads, checks and pulls succeed. Later failures preserve saved configuration, containers and the Docker data volume. Retries reuse those settings and cached images, pull missing images and start services with `--pull never --no-recreate`. `.lock` is a persistent lock directory shared by installation and mutating operator commands; the existing `runtimefs` platform adapter owns locking. +Preparation happens in `.staging`. On entry, the installer removes an empty stage or one with the empty ownership directory `.oac-installer-`; it preserves unrecognized directories. Ordinary failures remove owned staging immediately. The ownership marker is created atomically before downloading files. A killed process leaves staging for the next attempt. The release's `compose.yaml` is verified against its published checksum in the stage and must name the installer's own release in `OAC_REVISION`. Publication happens only after downloads, checks and pulls succeed. Later failures preserve saved configuration, containers and the Docker data volume. `.lock` is a persistent lock directory shared by installation and mutating operator commands; the existing `runtimefs` platform adapter owns locking. -The host `oac` command also implements `apply`, `core-key` and `rotate-core-key`. `apply` checks Core configuration before recreating services. Reading the key executes `oac-web core-key` in Web. Rotation runs in the initialization container, where Linux file ownership is identical on every host, then restarts Core and Web. Start, stop, logs and removal use `docker compose`. The initialization image verifies and copies bundled node metadata without network access. Its receipt authenticates immutable metadata and credentials; the operator-rotatable Core key and its digest remain mutable. No service receives a Docker socket. Initialization emits structured step logs without credential values; view them with `docker compose logs --timestamps init`. +An existing directory runs the release named by `OAC_REVISION` in its `compose.yaml`. When that is the installer's release, the installer resumes: it keeps the saved settings, pulls only missing images and starts the services the way `oac apply` does. The port check applies only to new directories. Another release is refused unless `--upgrade` is given. An upgrade stages the new `compose.yaml` and `oac`, pulls the new images, then runs `oac-core check-config` and init against the staged `compose.yaml` together with the directory's `.env` and `compose.override.yaml`. Init upgrades the data volume or refuses, for example a downgrade; until it succeeds, the running installation is unchanged. The installer then renames `oac` and `compose.yaml` into place and starts the services with `--pull never --remove-orphans`, recreating every changed container. Rerunning the same command retries an interrupted upgrade. The upgrade leaves the deployment's Runtime unchanged; the final message names the console step that applies the new one. + +`install.py --upgrade` upgrades a source-bundle installation the same way, under the same lock. It publishes the bundle's node payload beside earlier releases, renders the new `compose.yaml` and rewrites only its own `compose.override.yaml` entries, the four service images and Web's `/node-payload` bind, keeping every other entry. After the same two checks it moves the payload's `active.json`, renames `oac`, `compose.override.yaml` and `compose.yaml` into place and starts the services. Without `--upgrade` it refuses a non-empty directory; `--public-url` is required only for a new one. + +The host `oac` command also implements `apply`, `core-key` and `rotate-core-key`. `apply` checks Core configuration before starting services and recreating those that changed. Reading the key executes `oac-web core-key` in Web. Rotation runs in the initialization container, where Linux file ownership is identical on every host, then restarts Core and Web. Start, stop, logs and removal use `docker compose`. The initialization image verifies and copies bundled node metadata without network access. Its receipt authenticates immutable metadata and credentials; the operator-rotatable Core key and its digest remain mutable. No service receives a Docker socket. Initialization emits structured step logs without credential values; view them with `docker compose logs --timestamps init`. Web serves the console and forwards `/v1` and `/api/v1` to Core, so it is the only published service. HTTPS is terminated by the operator's reverse proxy or hosting platform, which routes to `web:8080`; `OAC_PUBLIC_URL` records that origin. diff --git a/deploy/install-source.py b/deploy/install-source.py index 9f92d65b0..8139e69bc 100644 --- a/deploy/install-source.py +++ b/deploy/install-source.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Start an offline source distribution with the production Compose services.""" import argparse -import hashlib +import fcntl import ipaddress import json import os @@ -9,6 +9,7 @@ import shutil import subprocess import sys +import tempfile from urllib.parse import urlsplit import uuid @@ -17,48 +18,99 @@ def run(*args, **kwargs): return subprocess.run(args, check=True, text=True, **kwargs) -def install(bundle, destination, host, port, public_url): +def compose_revision(text): + """Return the release a Compose file starts, from init's OAC_REVISION.""" + return next((line.split(':', 1)[1].strip() for line in text.splitlines() if line.strip().startswith('OAC_REVISION:')), '') + + +def install(bundle, destination, host, port, public_url, upgrade): # Use the distribution's own node download and publication implementation. sys.path.insert(0, str(bundle)) from distribution import ensure_docker_image, load_manifest - from node_payload import prepare_node_payload + import node_payload manifest = load_manifest(offline_root=bundle) revision = manifest['source_commit'] - if destination.exists() and any(destination.iterdir()): - raise ValueError('Installation directory is not empty. Use docker compose up for an existing installation; use a fresh directory and data volume for another commit.') for name in manifest['artifacts'].values(): if not (bundle / 'artifacts' / name['filename']).is_file(): raise ValueError('An offline distribution with all node artifacts is required') - # Docker validates both daemon access and the production template's required features. - images = {name: ensure_docker_image(manifest, name, lambda name=name: bundle / 'images' / (name + '.tar')) - for name in ('core', 'web', 'ingress', 'database')} - destination.mkdir(parents=True, exist_ok=True, mode=0o700) - prepare_node_payload(destination, None, bundle) - payload = destination / 'node-payload' - # These files contain only public distribution bytes and must be readable by Web. - for root, dirs, files in os.walk(payload): - os.chmod(root, 0o755) - for name in files: - os.chmod(Path(root) / name, 0o644) - template = (bundle / 'compose.yaml.template').read_text().replace('__OAC_REVISION__', revision).replace('__OAC_INIT_IMAGE__', images['ingress']) - for name in ('core', 'web'): - template = template.replace('__OAC_' + name.upper() + '_IMAGE__', images[name]) - override = {'services': {name: {'image': images[image]} for name, image in - (('core', 'core'), ('web', 'web'), ('init', 'ingress'), ('database', 'database'))}} - override['services']['web']['volumes'] = [{'type': 'bind', 'source': str(payload), 'target': '/node-payload', 'read_only': True}] - (destination / 'compose.yaml').write_text(template) - (destination / 'compose.override.yaml').write_text(json.dumps(override, indent=2) + '\n') - (destination / 'compose-sha256sums.txt').write_text(hashlib.sha256(template.encode()).hexdigest() + ' compose.yaml\n') - (destination / '.env').write_text(f"COMPOSE_PROJECT_NAME=oac-{uuid.uuid4().hex[:10]}\nOAC_HOST='{host}'\nOAC_WEB_PORT='{port}'\nOAC_PUBLIC_URL='{public_url}'\n") - os.chmod(destination / '.env', 0o600) - shutil.copyfile(bundle / 'oac', destination / 'oac') - os.chmod(destination / 'oac', 0o700) - run('docker', 'compose', 'config', '--quiet', cwd=destination) - print(f'Configuration ready in {destination}. Starting services…', flush=True) - run('docker', 'compose', 'up', '-d', '--wait', '--wait-timeout', '180', '--pull', 'never', cwd=destination) - print(f'Console: {public_url}\nCore key:', flush=True) - run('docker', 'compose', 'exec', '-T', 'web', '/usr/local/bin/oac-web', 'core-key', cwd=destination) + lock = Path(str(destination) + '.lock') + lock.mkdir(parents=True, exist_ok=True, mode=0o700) + descriptor = os.open(lock, os.O_RDONLY | os.O_DIRECTORY) + try: + # The same lock as oac apply and the other mutating oac commands. + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise ValueError('Another operation is using this installation') from None + existing = destination.exists() and any(destination.iterdir()) + if existing and not upgrade: + raise ValueError('Installation directory is not empty. To move it to this release, back up the installation (Upgrade in the operations guide), then rerun with --upgrade.') + if existing and not all((destination / name).is_file() for name in ('compose.yaml', '.env')): + raise ValueError(f'Incomplete installation: preserve {destination} and choose another directory') + if not existing and public_url is None: + raise ValueError('--public-url is required for a new installation') + # Docker validates both daemon access and the production template's required features. + images = {name: ensure_docker_image(manifest, name, lambda name=name: bundle / 'images' / (name + '.tar')) + for name in ('core', 'web', 'ingress', 'database')} + destination.mkdir(parents=True, exist_ok=True, mode=0o700) + payload = destination / 'node-payload' + node_payload.publish(payload, bundle) + # These files contain only public distribution bytes and must be readable by Web. + for root, dirs, files in os.walk(payload): + os.chmod(root, 0o755) + for name in files: + os.chmod(Path(root) / name, 0o644) + previous = compose_revision((destination / 'compose.yaml').read_text()) if existing else '' + if not existing: + (destination / '.env').write_text(f"COMPOSE_PROJECT_NAME=oac-{uuid.uuid4().hex[:10]}\nOAC_HOST='{host}'\nOAC_WEB_PORT='{port}'\nOAC_PUBLIC_URL='{public_url}'\n") + os.chmod(destination / '.env', 0o600) + with tempfile.TemporaryDirectory(prefix='.staging-', dir=destination) as stage: + stage = Path(stage) + template = (bundle / 'compose.yaml.template').read_text().replace('__OAC_REVISION__', revision).replace('__OAC_INIT_IMAGE__', images['ingress']) + for name in ('core', 'web'): + template = template.replace('__OAC_' + name.upper() + '_IMAGE__', images[name]) + (stage / 'compose.yaml').write_text(template) + # The installer owns only the service images and Web's payload bind; other entries are the operator's. + saved = destination / 'compose.override.yaml' + try: + override = json.loads(saved.read_text()) if saved.is_file() else {} + except ValueError: + raise ValueError('compose.override.yaml must stay JSON so the installer can replace its own entries') from None + services = override.setdefault('services', {}) + for name, image in (('core', 'core'), ('web', 'web'), ('init', 'ingress'), ('database', 'database')): + services.setdefault(name, {})['image'] = images[image] + web = services['web'] + web['volumes'] = [volume for volume in web.get('volumes', []) if not (isinstance(volume, dict) and volume.get('target') == '/node-payload')] + web['volumes'].append({'type': 'bind', 'source': str(payload), 'target': '/node-payload', 'read_only': True}) + (stage / 'compose.override.yaml').write_text(json.dumps(override, indent=2) + '\n') + shutil.copyfile(bundle / 'oac', stage / 'oac') + os.chmod(stage / 'oac', 0o700) + compose = ('docker', 'compose', '-f', str(stage / 'compose.yaml'), '-f', str(stage / 'compose.override.yaml'), '--project-directory', str(destination)) + if existing: + # Init upgrades the data volume or refuses; until then the running release is untouched. + try: + run(*compose, 'run', '--rm', '--no-deps', '--entrypoint', '/usr/local/bin/oac-core', 'core', 'check-config', cwd=destination) + run(*compose, 'run', '--rm', '--no-deps', 'init', cwd=destination) + except subprocess.CalledProcessError: + raise ValueError('Upgrade refused; the installation is unchanged') from None + else: + run(*compose, 'config', '--quiet', cwd=destination) + node_payload.activate(payload, revision) + for name in ('oac', 'compose.override.yaml', 'compose.yaml'): + os.replace(stage / name, destination / name) + # Installations from earlier releases kept a checksum that no longer describes compose.yaml. + (destination / 'compose-sha256sums.txt').unlink(missing_ok=True) + print(f'Configuration ready in {destination}. Starting services…', flush=True) + run('docker', 'compose', 'up', '-d', '--wait', '--wait-timeout', '180', '--pull', 'never', '--remove-orphans', cwd=destination) + if not existing: + print(f'Console: {public_url}') + if previous and previous != revision: + print(f"Upgraded from {previous[:12]} to {revision[:12]}. Settings → Sandbox and scheduling offers this release's Runtime; sandboxes keep their current Runtime until you apply it.") + print('Core key:', flush=True) + run('docker', 'compose', 'exec', '-T', 'web', '/usr/local/bin/oac-web', 'core-key', cwd=destination) + finally: + os.close(descriptor) def main(): @@ -67,17 +119,19 @@ def main(): parser.add_argument('--install-dir', type=Path, default=Path.home() / '.oac' / 'local') parser.add_argument('--host', default='127.0.0.1') parser.add_argument('--web-port', type=int, default=8080) - parser.add_argument('--public-url', required=True, help='HTTP(S) origin reachable by browsers and nodes') + parser.add_argument('--public-url', help='HTTP(S) origin reachable by browsers and nodes; required for a new installation') + parser.add_argument('--upgrade', action='store_true', help='upgrade an existing installation to this release') args = parser.parse_args() if not args.install_dir.is_absolute() or args.install_dir == Path('/') or not args.bundle_dir.is_absolute(): parser.error('bundle and installation directories must be absolute; installation cannot be /') ipaddress.ip_address(args.host) if not 1 <= args.web_port <= 65535: parser.error('web port must be between 1 and 65535') - url = urlsplit(args.public_url) - if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password or url.path not in ('', '/') or url.query or url.fragment or any(c in args.public_url for c in "\r\n'\\$"): - parser.error('public URL must be an HTTP(S) origin without credentials') - install(args.bundle_dir.resolve(), args.install_dir.resolve(), args.host, args.web_port, args.public_url) + if args.public_url is not None: + url = urlsplit(args.public_url) + if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password or url.path not in ('', '/') or url.query or url.fragment or any(c in args.public_url for c in "\r\n'\\$"): + parser.error('public URL must be an HTTP(S) origin without credentials') + install(args.bundle_dir.resolve(), args.install_dir.resolve(), args.host, args.web_port, args.public_url, args.upgrade) if __name__ == '__main__': diff --git a/deploy/node/README.md b/deploy/node/README.md index 74814031c..a1804d37e 100644 --- a/deploy/node/README.md +++ b/deploy/node/README.md @@ -8,7 +8,7 @@ These modules are packaged as `node-install.pyz`, which Web serves at `/node-ins | `distribution.py` | Verified artifacts, private temporary files, release metadata and Docker image identity | | `install_display.py`, `node_output.py` | Terminal output | | `provider_assets.py` | Node provider artifact names, generated by `go run ./services/core/cmd/provider-artifacts` | -| `node_payload.py` | Publish a release's node files without replacing bytes already installed | +| `node_payload.py` | Publish a release's node files without replacing bytes already installed, then activate a release | ## Accounts and permissions diff --git a/deploy/node/node_payload.py b/deploy/node/node_payload.py index 5265e3b9c..6cbe8826b 100644 --- a/deploy/node/node_payload.py +++ b/deploy/node/node_payload.py @@ -1,4 +1,4 @@ -"""Publish one release's node payload without replacing bytes already installed.""" +"""Publish release node payloads without replacing installed bytes, then activate one.""" import json import os from pathlib import Path @@ -13,102 +13,100 @@ class PayloadError(Exception): pass -def prepare_node_payload(root, state, bundle): - destination = root / "node-payload" - # Each release remains immutable and addressable while old nodes retain it. - # The only mutable publication is a small, atomically replaced active pointer. - def publish(source): - manifest = json.loads((source / "manifest.json").read_text()) - revision = manifest.get("source_commit", "") - if not re.fullmatch(r"[0-9a-f]{40}", revision): - raise PayloadError("Invalid node payload release identity") - metadata_names = ("node-install.pyz", "manifest.json", "SHA256SUMS", "runtime/seccomp.json") - names = list(metadata_names) +def publish(destination, source): + """Publish the release in source under destination/releases/ and return the commit. + + Each release stays immutable and addressable while nodes retain it. The active pointer is not moved. + """ + if destination.is_symlink(): + raise PayloadError("Invalid node payload directory") + destination.mkdir(mode=0o700, exist_ok=True) + manifest = json.loads((source / "manifest.json").read_text()) + revision = manifest.get("source_commit", "") + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise PayloadError("Invalid node payload release identity") + metadata_names = ("node-install.pyz", "manifest.json", "SHA256SUMS", "runtime/seccomp.json") + names = list(metadata_names) + for logical in manifest.get("artifacts", {}): + entry = artifact(manifest, logical) + name = "artifacts/" + entry["filename"] + path = source / name + if path.exists(): + if (path.is_symlink() or not path.is_file() + or not path.resolve().is_relative_to(source.resolve()) + or path.stat().st_size != entry["size"] or digest(path) != entry["sha256"]): + raise PayloadError("Offline artifact verification failed: " + logical) + names.append(name) + target = destination / "releases" / revision + if target.is_symlink() or target.parent.is_symlink(): + raise PayloadError("Installed node payload differs; preserve it and inspect the distribution") + target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + if target.exists(): + # Validate the complete published metadata and every existing declared + # artifact before filling any absence. Existing bytes are immutable. + for name in metadata_names: + previous = target / name + if (previous.parent.is_symlink() or previous.is_symlink() or not previous.is_file() + or digest(previous) != digest(source / name)): + raise PayloadError("Installed node payload differs; preserve it and inspect the distribution") + artifacts = target / "artifacts" + if artifacts.is_symlink() or artifacts.exists() and not artifacts.is_dir(): + raise PayloadError("Installed node artifact directory differs") + missing = [] for logical in manifest.get("artifacts", {}): entry = artifact(manifest, logical) name = "artifacts/" + entry["filename"] - path = source / name - if path.exists(): - if (path.is_symlink() or not path.is_file() - or not path.resolve().is_relative_to(source.resolve()) - or path.stat().st_size != entry["size"] or digest(path) != entry["sha256"]): - raise PayloadError("Offline artifact verification failed: " + logical) - names.append(name) - target = destination / "releases" / revision - if target.is_symlink() or target.parent.is_symlink(): - raise PayloadError("Installed node payload differs; preserve it and inspect the distribution") - target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - if target.exists(): - # Validate the complete published metadata and every existing declared - # artifact before filling any absence. Existing bytes are immutable. - for name in metadata_names: - previous = target / name - if (previous.parent.is_symlink() or previous.is_symlink() or not previous.is_file() - or digest(previous) != digest(source / name)): - raise PayloadError("Installed node payload differs; preserve it and inspect the distribution") - artifacts = target / "artifacts" - if artifacts.is_symlink() or artifacts.exists() and not artifacts.is_dir(): - raise PayloadError("Installed node artifact directory differs") - missing = [] - for logical in manifest.get("artifacts", {}): - entry = artifact(manifest, logical) - name = "artifacts/" + entry["filename"] - previous = target / name - if previous.is_symlink() or previous.exists() and (not previous.is_file() - or previous.stat().st_size != entry["size"] or digest(previous) != entry["sha256"]): - raise PayloadError("Installed node artifact differs; refusing repair") - if not previous.exists() and name in names: - missing.append((name, entry)) - if missing: - artifacts.mkdir(mode=0o700, exist_ok=True) - for name, entry in missing: - descriptor, temporary = tempfile.mkstemp(prefix=".payload-", dir=artifacts) - try: - with os.fdopen(descriptor, "wb") as outgoing, (source / name).open("rb") as incoming: - shutil.copyfileobj(incoming, outgoing) - outgoing.flush() - os.fsync(outgoing.fileno()) - if Path(temporary).stat().st_size != entry["size"] or digest(Path(temporary)) != entry["sha256"]: - raise PayloadError("Node artifact changed during repair") - # Publish without replacing bytes introduced concurrently. - os.link(temporary, target / name) - finally: - os.unlink(temporary) - for directory in (artifacts, target): - descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) - try: - os.fsync(descriptor) - finally: - os.close(descriptor) - return revision - with tempfile.TemporaryDirectory(prefix=".payload-", dir=target.parent) as temporary: - stage = Path(temporary) / "release" - stage.mkdir(mode=0o700) - for name in names: - path = source / name - if path.is_symlink() or not path.is_file(): - raise PayloadError("Invalid node payload source file") - copied = stage / name - copied.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - with path.open("rb") as incoming, copied.open("xb") as outgoing: - os.chmod(copied, 0o600) - shutil.copyfileobj(incoming, outgoing) - outgoing.flush() - os.fsync(outgoing.fileno()) - os.rename(stage, target) + previous = target / name + if previous.is_symlink() or previous.exists() and (not previous.is_file() + or previous.stat().st_size != entry["size"] or digest(previous) != entry["sha256"]): + raise PayloadError("Installed node artifact differs; refusing repair") + if not previous.exists() and name in names: + missing.append((name, entry)) + if missing: + artifacts.mkdir(mode=0o700, exist_ok=True) + for name, entry in missing: + descriptor, temporary = tempfile.mkstemp(prefix=".payload-", dir=artifacts) + try: + with os.fdopen(descriptor, "wb") as outgoing, (source / name).open("rb") as incoming: + shutil.copyfileobj(incoming, outgoing) + outgoing.flush() + os.fsync(outgoing.fileno()) + if Path(temporary).stat().st_size != entry["size"] or digest(Path(temporary)) != entry["sha256"]: + raise PayloadError("Node artifact changed during repair") + # Publish without replacing bytes introduced concurrently. + os.link(temporary, target / name) + finally: + os.unlink(temporary) + for directory in (artifacts, target): + descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) return revision + with tempfile.TemporaryDirectory(prefix=".payload-", dir=target.parent) as temporary: + stage = Path(temporary) / "release" + stage.mkdir(mode=0o700) + for name in names: + path = source / name + if path.is_symlink() or not path.is_file(): + raise PayloadError("Invalid node payload source file") + copied = stage / name + copied.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + with path.open("rb") as incoming, copied.open("xb") as outgoing: + os.chmod(copied, 0o600) + shutil.copyfileobj(incoming, outgoing) + outgoing.flush() + os.fsync(outgoing.fileno()) + os.rename(stage, target) + return revision - if destination.is_symlink(): - raise PayloadError("Invalid node payload directory") - destination.mkdir(mode=0o700, exist_ok=True) - revision = publish(bundle) + +def activate(destination, revision): + """Atomically point destination/active.json at a published release.""" pointer = destination / "active.json" if pointer.is_symlink(): raise PayloadError("Invalid active node payload pointer") - if pointer.exists(): - if json.loads(pointer.read_text()) != {"source_commit": revision}: - raise PayloadError("Installed node payload differs; preserve it and inspect the distribution") - return descriptor, temporary = tempfile.mkstemp(prefix=".active-", dir=destination) try: with os.fdopen(descriptor, "w") as stream: @@ -116,6 +114,8 @@ def publish(source): stream.write("\n") stream.flush() os.fsync(stream.fileno()) + # Web reads the pointer as another user while it serves the previous release. + os.chmod(temporary, 0o644) os.replace(temporary, pointer) descriptor = os.open(destination, os.O_RDONLY | os.O_DIRECTORY) try: diff --git a/deploy/node/test_generation_review_regressions.py b/deploy/node/test_generation_review_regressions.py index f2f730ff1..f160c2386 100644 --- a/deploy/node/test_generation_review_regressions.py +++ b/deploy/node/test_generation_review_regressions.py @@ -135,9 +135,21 @@ def payload_fixture(self, directory): files = {'node-install.pyz': b'node', 'SHA256SUMS': b'fixture', 'runtime/seccomp.json': b'{}', 'manifest.json': json.dumps({'source_commit': source, 'artifacts': entries}).encode()} for name, raw in files.items(): path = bundle / name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(raw) - node_payload.prepare_node_payload(root, {'mode': 'all'}, bundle) + node_payload.publish(root / 'node-payload', bundle) return root, bundle, root / 'node-payload/releases' / source, entries + def test_publish_keeps_releases_and_activate_swaps_the_pointer(self): + with tempfile.TemporaryDirectory() as temp: + root, bundle, old, _ = self.payload_fixture(Path(temp)) + payload = root / 'node-payload' + node_payload.activate(payload, old.name) + (bundle / 'manifest.json').write_text(json.dumps({'source_commit': 'b' * 40, 'artifacts': {}})) + self.assertEqual(node_payload.publish(payload, bundle), 'b' * 40) + self.assertEqual(json.loads((payload / 'active.json').read_text()), {'source_commit': old.name}) + node_payload.activate(payload, 'b' * 40) + self.assertEqual(json.loads((payload / 'active.json').read_text()), {'source_commit': 'b' * 40}) + self.assertEqual(sorted(path.name for path in (payload / 'releases').iterdir()), [old.name, 'b' * 40]) + def test_thin_release_adds_and_repairs_verified_missing_artifact(self): with tempfile.TemporaryDirectory() as temp: root, bundle, release, entries = self.payload_fixture(Path(temp)) @@ -145,11 +157,11 @@ def test_thin_release_adds_and_repairs_verified_missing_artifact(self): for name, entry in entries.items(): (bundle / 'artifacts' / entry['filename']).write_bytes((name + ' verified artifact').encode()) original = (release / 'manifest.json').read_bytes() - node_payload.prepare_node_payload(root, {'mode': 'all'}, bundle) + node_payload.publish(root / 'node-payload', bundle) artifact = release / 'artifacts' / entries['node']['filename'] self.assertEqual(artifact.read_bytes(), b'node verified artifact') artifact.unlink() - node_payload.prepare_node_payload(root, {'mode': 'all'}, bundle) + node_payload.publish(root / 'node-payload', bundle) self.assertEqual(artifact.read_bytes(), b'node verified artifact') self.assertEqual((release / 'manifest.json').read_bytes(), original) self.assertEqual(artifact.stat().st_nlink, 1) @@ -162,7 +174,7 @@ def test_published_conflict_refuses_every_missing_repair(self): (bundle / 'artifacts' / node).write_bytes(b'node verified artifact') (release / 'artifacts' / helper).write_bytes(b'changed') with self.assertRaises(node_payload.PayloadError): - node_payload.prepare_node_payload(root, {'mode': 'all'}, bundle) + node_payload.publish(root / 'node-payload', bundle) self.assertFalse((release / 'artifacts' / node).exists()) self.assertEqual((release / 'artifacts' / helper).read_bytes(), b'changed') diff --git a/deploy/test_install_source.py b/deploy/test_install_source.py index 1544d7131..b8b0aeb86 100644 --- a/deploy/test_install_source.py +++ b/deploy/test_install_source.py @@ -39,9 +39,20 @@ def setUp(self): def write_sums(self): (self.bundle / 'SHA256SUMS').write_text(hashlib.sha256((self.bundle / 'manifest.json').read_bytes()).hexdigest() + ' manifest.json\n') - def install(self): - with patch('distribution.ensure_docker_image', return_value='sha256:' + '1' * 64), patch.object(installer, 'run', side_effect=lambda *a, **k: self.calls.append((a, k))): - installer.install(self.bundle, self.destination, '127.0.0.1', 19873, 'http://127.0.0.1:19873') + def install(self, upgrade=False, image='sha256:' + '1' * 64, failure=None): + def run(*args, **kwargs): + self.calls.append((args, kwargs)) + if failure and failure in args: + raise subprocess.CalledProcessError(1, args) + with patch('distribution.ensure_docker_image', return_value=image), patch.object(installer, 'run', side_effect=run): + installer.install(self.bundle, self.destination, '127.0.0.1', 19873, None if upgrade else 'http://127.0.0.1:19873', upgrade) + + def commands(self): + """Compose subcommands, without the staged file arguments.""" + return [args[args.index('--project-directory') + 2:] if '--project-directory' in args else args[2:] for args, _ in self.calls] + + def snapshot(self): + return {name: (self.destination / name).read_bytes() for name in ('compose.yaml', 'compose.override.yaml', '.env', 'oac', 'node-payload/active.json')} def test_complete_compose_preserves_secrets_and_serves_whole_payload(self): self.install() @@ -66,11 +77,43 @@ def test_complete_compose_preserves_secrets_and_serves_whole_payload(self): def test_existing_installation_is_not_replaced(self): self.destination.mkdir() (self.destination / 'keep').write_text('existing') - with self.assertRaisesRegex(ValueError, 'not empty'): + with self.assertRaisesRegex(ValueError, 'rerun with --upgrade'): self.install() self.assertEqual((self.destination / 'keep').read_text(), 'existing') self.assertEqual(self.calls, []) + def test_upgrade_checks_the_new_release_before_replacing_it_and_keeps_operator_entries(self): + self.install() + override = json.loads((self.destination / 'compose.override.yaml').read_text()) + nfs = {'type': 'bind', 'source': '/srv/workspaces', 'target': '/srv/workspaces'} + override['services']['core']['volumes'] = [nfs] + (self.destination / 'compose.override.yaml').write_text(json.dumps(override)) + before = self.snapshot() + upgraded = 'b' * 40 + self.manifest['source_commit'] = upgraded + (self.bundle / 'manifest.json').write_text(json.dumps(self.manifest)) + self.write_sums() + (self.bundle / 'oac').write_bytes(b'upgraded operator') + + self.calls = [] + with self.assertRaisesRegex(ValueError, 'Upgrade refused; the installation is unchanged'): + self.install(upgrade=True, image='sha256:' + '2' * 64, failure='init') + self.assertEqual(self.snapshot(), before) + + self.calls = [] + self.install(upgrade=True, image='sha256:' + '2' * 64) + self.assertEqual([command[-1] for command in self.commands()], ['check-config', 'init', '--remove-orphans', 'core-key']) + services = json.loads((self.destination / 'compose.override.yaml').read_text())['services'] + self.assertEqual(services['core']['volumes'], [nfs]) + self.assertEqual({services[name]['image'] for name in ('core', 'web', 'init', 'database')}, {'sha256:' + '2' * 64}) + self.assertEqual([volume['target'] for volume in services['web']['volumes']], ['/node-payload']) + self.assertIn('OAC_REVISION: ' + upgraded, (self.destination / 'compose.yaml').read_text()) + self.assertEqual((self.destination / 'oac').read_bytes(), b'upgraded operator') + self.assertEqual((self.destination / '.env').read_bytes(), before['.env']) + payload = self.destination / 'node-payload' + self.assertEqual(json.loads((payload / 'active.json').read_text()), {'source_commit': upgraded}) + self.assertEqual(sorted(path.name for path in (payload / 'releases').iterdir()), [self.revision, upgraded]) + def test_thin_bundle_rejected_before_creating_installation(self): self.manifest['artifacts'] = {'native/bin/node': {'filename': 'missing'}} (self.bundle / 'manifest.json').write_text(json.dumps(self.manifest)) diff --git a/services/core/cmd/oac/install.go b/services/core/cmd/oac/install.go index efe7d5955..5832ad7fd 100644 --- a/services/core/cmd/oac/install.go +++ b/services/core/cmd/oac/install.go @@ -25,10 +25,11 @@ import ( type installOptions struct { dir, version, publicURL, host string port int + upgrade bool } type installer struct { docker func(context.Context, string, ...string) ([]byte, error) - download func(context.Context, string, string) error + download func(context.Context, string) ([]byte, error) executable string } @@ -44,6 +45,7 @@ func installCommand(ctx context.Context, args []string) error { flags.StringVar(&o.publicURL, "public-url", "", "URL reachable by browsers and nodes") flags.StringVar(&o.host, "host", "0.0.0.0", "Web bind address") flags.IntVar(&o.port, "web-port", 8080, "Web port") + flags.BoolVar(&o.upgrade, "upgrade", false, "upgrade an existing installation to this release") if err := flags.Parse(args); err != nil { return err } @@ -68,7 +70,7 @@ func dockerOutput(ctx context.Context, dir string, args ...string) ([]byte, erro return output, nil } -func downloadAsset(ctx context.Context, address, path string) error { +func downloadAsset(ctx context.Context, address string) ([]byte, error) { client := &http.Client{Timeout: 2 * time.Minute, CheckRedirect: func(req *http.Request, via []*http.Request) error { if req.URL.Scheme != "https" || len(via) >= 10 { return errors.New("invalid release redirect") @@ -77,24 +79,24 @@ func downloadAsset(ctx context.Context, address, path string) error { }} req, err := http.NewRequestWithContext(ctx, http.MethodGet, address, nil) if err != nil { - return err + return nil, err } res, err := client.Do(req) if err != nil { - return err + return nil, err } defer res.Body.Close() if res.StatusCode != http.StatusOK { - return fmt.Errorf("download %s: HTTP %d", address, res.StatusCode) + return nil, fmt.Errorf("download %s: HTTP %d", address, res.StatusCode) } raw, err := io.ReadAll(io.LimitReader(res.Body, (1<<20)+1)) if err != nil { - return err + return nil, err } if len(raw) > 1<<20 { - return errors.New("release configuration exceeds size limit") + return nil, errors.New("release configuration exceeds size limit") } - return os.WriteFile(path, raw, 0o600) + return raw, nil } func (i installer) install(ctx context.Context, o installOptions) error { @@ -179,14 +181,36 @@ func (i installer) installLocked(ctx context.Context, o installOptions) error { if err != nil && !errors.Is(err, os.ErrNotExist) { return err } - resume := len(entries) > 0 - working := o.dir - if !resume { - listener, err := net.Listen("tcp", net.JoinHostPort(o.host, strconv.Itoa(o.port))) + fresh := len(entries) == 0 + installed := buildRevision + if !fresh { + for _, name := range []string{"compose.yaml", ".env", cliName()} { + info, err := os.Lstat(filepath.Join(o.dir, name)) + if err != nil || !info.Mode().IsRegular() { + return fmt.Errorf("incomplete installation: preserve %s and choose another directory", o.dir) + } + } + raw, err := os.ReadFile(filepath.Join(o.dir, "compose.yaml")) if err != nil { - return fmt.Errorf("Web port unavailable; choose --web-port: %w", err) + return err + } + if installed = composeRevision(raw); installed == "" { + return fmt.Errorf("incomplete installation: preserve %s and choose another directory", o.dir) + } + } + upgrade := installed != buildRevision + if upgrade && !o.upgrade { + return fmt.Errorf("This installation runs release %s; this installer is %s. Back up the installation (Upgrade in the operations guide), then rerun with --upgrade.", short(installed), short(buildRevision)) + } + compose := installCompose{docker: i.docker, dir: o.dir} + if fresh || upgrade { + if fresh { + listener, err := net.Listen("tcp", net.JoinHostPort(o.host, strconv.Itoa(o.port))) + if err != nil { + return fmt.Errorf("Web port unavailable; choose --web-port: %w", err) + } + listener.Close() } - listener.Close() if err := os.Mkdir(stage, 0o700); err != nil { return err } @@ -195,76 +219,68 @@ func (i installer) installLocked(ctx context.Context, o installOptions) error { if err := os.Mkdir(filepath.Join(stage, stageMarker(o.dir)), 0o700); err != nil { return err } - working = stage - repository := os.Getenv("OAC_REPOSITORY") - if repository == "" { - repository = "MiniMax-AI/OpenAgentCore" - } - base := "https://github.com/" + repository + "/releases/latest/download/" - if o.version != "latest" { - base = "https://github.com/" + repository + "/releases/download/" + url.PathEscape(o.version) + "/" - } - fmt.Println("Downloading release configuration...") - for _, name := range []string{"compose-sha256sums.txt", "compose.yaml"} { - if err := i.download(ctx, base+name, filepath.Join(stage, name)); err != nil { - return err - } - } - contents := fmt.Sprintf("COMPOSE_PROJECT_NAME=oac-%s\nOAC_HOST='%s'\nOAC_WEB_PORT='%d'\nOAC_PUBLIC_URL='%s'\n", randomHex(5), o.host, o.port, o.publicURL) - if err := os.WriteFile(filepath.Join(stage, ".env"), []byte(contents), 0o600); err != nil { - return err - } - source, err := os.Open(i.executable) + release, err := i.releaseCompose(ctx, o.version) if err != nil { return err } - defer source.Close() - target, err := os.OpenFile(filepath.Join(stage, cliName()), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o700) - if err != nil { + if err := os.WriteFile(filepath.Join(stage, "compose.yaml"), release, 0o600); err != nil { return err } - _, copyErr := io.Copy(target, source) - closeErr := target.Close() - if copyErr != nil { - return copyErr + if fresh { + contents := fmt.Sprintf("COMPOSE_PROJECT_NAME=oac-%s\nOAC_HOST='%s'\nOAC_WEB_PORT='%d'\nOAC_PUBLIC_URL='%s'\n", randomHex(5), o.host, o.port, o.publicURL) + if err := os.WriteFile(filepath.Join(stage, ".env"), []byte(contents), 0o600); err != nil { + return err + } + compose.dir = stage + } else { + // The staged release runs with the installation's settings, project name and override. + compose.files = []string{"-f", filepath.Join(stage, "compose.yaml")} + if _, err := os.Stat(filepath.Join(o.dir, "compose.override.yaml")); err == nil { + compose.files = append(compose.files, "-f", filepath.Join(o.dir, "compose.override.yaml")) + } + compose.files = append(compose.files, "--project-directory", o.dir) } - if closeErr != nil { - return closeErr + if err := copyExecutable(i.executable, filepath.Join(stage, cliName())); err != nil { + return err } } else { fmt.Println("Using saved settings and retaining existing data.") } - for _, name := range []string{"compose.yaml", "compose-sha256sums.txt", ".env", cliName()} { - info, err := os.Lstat(filepath.Join(working, name)) - if err != nil || !info.Mode().IsRegular() { - return fmt.Errorf("incomplete installation: preserve %s and choose another directory", o.dir) - } - } - if err := verifyCompose(working); err != nil { + if err := compose.Run(ctx, "config", "--quiet"); err != nil { return err } - compose := func(args ...string) ([]byte, error) { - return i.docker(ctx, working, append([]string{"compose"}, args...)...) - } - if _, err := compose("config", "--quiet"); err != nil { - return err - } - images, err := compose("config", "--images") + images, err := compose.Output(ctx, "config", "--images") if err != nil { return err } fmt.Println("Checking and downloading images...") for _, image := range strings.Fields(string(images)) { - if resume { - if _, err := i.docker(ctx, working, "image", "inspect", image); err == nil { + if !fresh && !upgrade { + if _, err := i.docker(ctx, "", "image", "inspect", image); err == nil { continue } } - if _, err := i.docker(ctx, working, "pull", image); err != nil { + if _, err := i.docker(ctx, "", "pull", image); err != nil { return err } } - if !resume { + if upgrade { + // Init upgrades the data volume or refuses; until then the running release is untouched. + fmt.Println("Checking the upgrade...") + for _, check := range [][]string{checkConfig, {"run", "--rm", "--no-deps", "init"}} { + if err := compose.Run(ctx, check...); err != nil { + return fmt.Errorf("Upgrade refused; the installation is unchanged: %w", err) + } + } + for _, name := range []string{cliName(), "compose.yaml"} { + if err := os.Rename(filepath.Join(stage, name), filepath.Join(o.dir, name)); err != nil { + return err + } + } + // Installations from earlier releases kept a checksum that no longer describes compose.yaml. + _ = os.Remove(filepath.Join(o.dir, "compose-sha256sums.txt")) + } + if fresh { if _, err := os.Stat(o.dir); err == nil { if err := os.Remove(o.dir); err != nil { return err @@ -273,18 +289,23 @@ func (i installer) installLocked(ctx context.Context, o installOptions) error { if err := os.Rename(stage, o.dir); err != nil { return err } - working = o.dir _ = os.Remove(filepath.Join(o.dir, stageMarker(o.dir))) } + compose = installCompose{docker: i.docker, dir: o.dir} fmt.Println("Starting services...") - if _, err := compose("up", "-d", "--wait", "--wait-timeout", "180", "--pull", "never", "--no-recreate"); err != nil { + if fresh || upgrade { + err = compose.Run(ctx, startServices...) + } else { + err = apply(ctx, compose) + } + if err != nil { return fmt.Errorf("startup failed; data retained, rerun the installer: %w", err) } - key, err := compose("exec", "-T", "web", "/usr/local/bin/oac-web", "core-key") + key, err := compose.Output(ctx, "exec", "-T", "web", "/usr/local/bin/oac-web", "core-key") if err != nil { return err } - environment, err := compose("config", "--environment") + environment, err := compose.Output(ctx, "config", "--environment") if err != nil { return err } @@ -294,12 +315,96 @@ func (i installer) installLocked(ctx context.Context, o installOptions) error { } } fmt.Printf("\nOpenAgentCore is running.\n\nConsole: %s\nCore key: %s\nCommand: %s\n", o.publicURL, strings.TrimSpace(string(key)), filepath.Join(o.dir, cliName())) + if upgrade { + fmt.Printf("Upgraded from %s to %s. Settings → Sandbox and scheduling offers this release's Runtime; sandboxes keep their current Runtime until you apply it.\n", short(installed), short(buildRevision)) + } if u, _ := url.Parse(o.publicURL); u != nil && (u.Hostname() == "localhost" || u.Hostname() == "127.0.0.1") { fmt.Println("For remote access, set OAC_PUBLIC_URL in .env to a reachable origin and run oac apply.") } return nil } +// releaseCompose downloads the requested release's compose.yaml, verifies its published checksum and requires it to start this installer's release. +func (i installer) releaseCompose(ctx context.Context, version string) ([]byte, error) { + repository := os.Getenv("OAC_REPOSITORY") + if repository == "" { + repository = "MiniMax-AI/OpenAgentCore" + } + base := "https://github.com/" + repository + "/releases/latest/download/" + if version != "latest" { + base = "https://github.com/" + repository + "/releases/download/" + url.PathEscape(version) + "/" + } + fmt.Println("Downloading release configuration...") + sums, err := i.download(ctx, base+"compose-sha256sums.txt") + if err != nil { + return nil, err + } + compose, err := i.download(ctx, base+"compose.yaml") + if err != nil { + return nil, err + } + digest := sha256.Sum256(compose) + if strings.TrimSpace(string(sums)) != hex.EncodeToString(digest[:])+" compose.yaml" { + return nil, errors.New("Compose checksum mismatch; rerun the installer") + } + // A moving latest release can differ from the release that published this installer. + if revision := composeRevision(compose); revision != buildRevision { + return nil, fmt.Errorf("release configuration %s does not match this installer %s; rerun the installer", short(revision), short(buildRevision)) + } + return compose, nil +} + +// composeRevision returns the release a Compose file starts, from init's OAC_REVISION. +func composeRevision(compose []byte) string { + for _, line := range strings.Split(string(compose), "\n") { + if value, ok := strings.CutPrefix(strings.TrimSpace(line), "OAC_REVISION:"); ok { + return strings.TrimSpace(value) + } + } + return "" +} + +func short(revision string) string { + if len(revision) > 12 { + return revision[:12] + } + return revision +} + +func copyExecutable(from, to string) error { + source, err := os.Open(from) + if err != nil { + return err + } + defer source.Close() + target, err := os.OpenFile(to, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o700) + if err != nil { + return err + } + _, copyErr := io.Copy(target, source) + closeErr := target.Close() + if copyErr != nil { + return copyErr + } + return closeErr +} + +// installCompose runs Docker Compose through the installer's Docker command, so installation shares oac apply's start. +type installCompose struct { + docker func(context.Context, string, ...string) ([]byte, error) + dir string + files []string +} + +func (c installCompose) Output(ctx context.Context, args ...string) ([]byte, error) { + return c.docker(ctx, c.dir, append(append([]string{"compose"}, c.files...), args...)...) +} + +func (c installCompose) Run(ctx context.Context, args ...string) error { + _, err := c.Output(ctx, args...) + return err +} + func cliName() string { if runtime.GOOS == "windows" { return "oac.exe" @@ -337,22 +442,6 @@ func cleanStage(stage, owner string) error { return os.RemoveAll(stage) } -func verifyCompose(dir string) error { - raw, err := os.ReadFile(filepath.Join(dir, "compose.yaml")) - if err != nil { - return err - } - sums, err := os.ReadFile(filepath.Join(dir, "compose-sha256sums.txt")) - if err != nil { - return err - } - digest := sha256.Sum256(raw) - if strings.TrimSpace(string(sums)) != hex.EncodeToString(digest[:])+" compose.yaml" { - return errors.New("Compose checksum mismatch; restore the matching release configuration") - } - return nil -} - func stageMarker(owner string) string { return fmt.Sprintf(".oac-installer-%x", sha256.Sum256([]byte(owner))) } diff --git a/services/core/cmd/oac/install_test.go b/services/core/cmd/oac/install_test.go index be48b1bfc..2e9a8c6c0 100644 --- a/services/core/cmd/oac/install_test.go +++ b/services/core/cmd/oac/install_test.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "fmt" + "maps" "net" "os" "os/exec" @@ -21,6 +22,12 @@ type installFixture struct { cached bool } +// installedRevision reads the release the installation directory starts. +func (f *installFixture) installedRevision() string { + raw, _ := os.ReadFile(filepath.Join(f.options.dir, "compose.yaml")) + return composeRevision(raw) +} + func newInstallFixture(t *testing.T) *installFixture { t.Helper() root, err := filepath.EvalSymlinks(t.TempDir()) @@ -39,21 +46,42 @@ func newInstallFixture(t *testing.T) *installFixture { } f := &installFixture{options: installOptions{dir: filepath.Join(root, "installation with spaces"), version: "latest", host: "127.0.0.1", port: port}} f.executable = exe - compose := []byte("services: {}\n") - f.download = func(_ context.Context, address, path string) error { + // The release served for download is always the running installer's. + f.download = func(_ context.Context, address string) ([]byte, error) { if f.failure == "download" { - return os.ErrPermission + return nil, os.ErrPermission } - raw := compose + compose := []byte("services:\n init:\n environment:\n OAC_REVISION: " + buildRevision + "\n") if strings.HasSuffix(address, "sums.txt") { - raw = []byte(fmt.Sprintf("%x compose.yaml\n", sha256.Sum256(compose))) + return []byte(fmt.Sprintf("%x compose.yaml\n", sha256.Sum256(compose))), nil } - return os.WriteFile(path, raw, 0o600) + return compose, nil } + // Calls against the staged release are recorded with a "staged " prefix and without their file arguments. f.docker = func(_ context.Context, dir string, args ...string) ([]byte, error) { - command := strings.Join(args, " ") - f.calls = append(f.calls, command) - if f.failure != "" && strings.HasPrefix(command, f.failure) { + if _, err := os.Stat(dir); dir != "" && err != nil { + t.Fatalf("docker %v ran in a missing directory", args) + } + var words []string + staged := false + for i := 0; i < len(args); i++ { + if args[i] == "-f" || args[i] == "--project-directory" { + staged = staged || strings.HasPrefix(args[i+1], f.options.dir+".staging") + i++ + continue + } + words = append(words, args[i]) + } + command := strings.Join(words, " ") + if staged { + if f.installedRevision() == buildRevision { + t.Fatal("release published before its checks") + } + f.calls = append(f.calls, "staged "+command) + } else { + f.calls = append(f.calls, command) + } + if f.failure != "" && strings.HasPrefix(f.calls[len(f.calls)-1], f.failure) { return nil, os.ErrPermission } switch command { @@ -72,7 +100,7 @@ func newInstallFixture(t *testing.T) *installFixture { return nil, os.ErrNotExist } if strings.HasPrefix(command, "compose up") { - if _, err := os.Stat(filepath.Join(f.options.dir, ".env")); err != nil { + if _, err := os.Stat(filepath.Join(f.options.dir, ".env")); err != nil || f.installedRevision() != buildRevision { t.Fatal("services started before configuration publication") } } @@ -103,6 +131,7 @@ func TestInstallFailureBeforePublicationAndRetry(t *testing.T) { } } func TestInstallResumePreservesConfigAndUsesCachedImages(t *testing.T) { + // The same release resumes with or without --upgrade. f := newInstallFixture(t) f.failure = "compose up" if err := f.run(); err == nil { @@ -115,11 +144,12 @@ func TestInstallResumePreservesConfigAndUsesCachedImages(t *testing.T) { f.failure = "" f.cached = true f.calls = nil - f.download = func(context.Context, string, string) error { + f.download = func(context.Context, string) ([]byte, error) { t.Fatal("resume downloaded replacement release") - return nil + return nil, nil } f.options.publicURL = "https://replacement.invalid" + f.options.upgrade = true if err := f.run(); err != nil { t.Fatal(err) } @@ -128,12 +158,97 @@ func TestInstallResumePreservesConfigAndUsesCachedImages(t *testing.T) { t.Fatal("saved configuration replaced") } for _, call := range f.calls { - if strings.HasPrefix(call, "pull") || strings.Contains(call, "down") { + if strings.HasPrefix(call, "pull") || strings.Contains(call, "down") || strings.Contains(call, "init") { t.Fatal(call) } } - if !strings.Contains(strings.Join(f.calls, "\n"), "--pull never --no-recreate") { - t.Fatal(f.calls) + assertCallsInOrder(t, f.calls, "compose "+strings.Join(checkConfig, " "), "compose "+strings.Join(startServices, " ")) +} + +func TestInstallUpgradesAnotherReleaseOnlyWhenRequested(t *testing.T) { + f := newInstallFixture(t) + setBuildRevision(t, "a1b2c3d4e5f6a7b8c9d0a1b2c3d4e5f6a7b8c9d0") + if err := f.run(); err != nil { + t.Fatal(err) + } + os.WriteFile(filepath.Join(f.options.dir, "compose-sha256sums.txt"), []byte("previous release"), 0o600) + before := installationFiles(t, f.options.dir) + setBuildRevision(t, "development") + os.WriteFile(f.executable, []byte("upgraded binary"), 0o700) + + f.calls = nil + err := f.run() + if err == nil || err.Error() != "This installation runs release a1b2c3d4e5f6; this installer is development. Back up the installation (Upgrade in the operations guide), then rerun with --upgrade." { + t.Fatal(err) + } + if len(f.calls) != 3 { + t.Fatalf("refusal ran Docker beyond its checks: %v", f.calls) + } + + f.options.upgrade = true + f.failure = "staged compose run --rm --no-deps init" + if err := f.run(); err == nil || !strings.HasPrefix(err.Error(), "Upgrade refused; the installation is unchanged") { + t.Fatal(err) + } + if after := installationFiles(t, f.options.dir); !maps.Equal(before, after) { + t.Fatal("refused upgrade changed the installation directory") + } + if _, err := os.Stat(f.options.dir + ".staging"); !os.IsNotExist(err) { + t.Fatal("refused upgrade left its stage") + } + + f.failure = "" + f.calls = nil + if err := f.run(); err != nil { + t.Fatal(err) + } + assertCallsInOrder(t, f.calls, + "pull example/core:latest", + "staged compose "+strings.Join(checkConfig, " "), + "staged compose run --rm --no-deps init", + "compose up -d --wait --wait-timeout 180 --pull never --remove-orphans") + after := installationFiles(t, f.options.dir) + if after["oac"] != "upgraded binary" || after[".env"] != before[".env"] || f.installedRevision() != "development" { + t.Fatal("upgrade did not publish the release with the saved settings") + } + if _, ok := after["compose-sha256sums.txt"]; ok { + t.Fatal("upgrade kept the previous release checksum") + } +} + +func setBuildRevision(t *testing.T, revision string) { + previous := buildRevision + buildRevision = revision + t.Cleanup(func() { buildRevision = previous }) +} + +func installationFiles(t *testing.T, dir string) map[string]string { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + files := map[string]string{} + for _, entry := range entries { + raw, err := os.ReadFile(filepath.Join(dir, entry.Name())) + if err != nil { + t.Fatal(err) + } + files[entry.Name()] = string(raw) + } + return files +} + +func assertCallsInOrder(t *testing.T, calls []string, want ...string) { + t.Helper() + next := 0 + for _, call := range calls { + if next < len(want) && call == want[next] { + next++ + } + } + if next != len(want) { + t.Fatalf("missing %q in order in %v", want[next], calls) } } func TestInstallRecoversRecognizedStageAndPreservesUnknownData(t *testing.T) { @@ -189,22 +304,6 @@ func TestInstallRejectsBusyPortAndLock(t *testing.T) { t.Fatal(err) } } -func TestInstallRefusesModifiedCompose(t *testing.T) { - f := newInstallFixture(t) - if err := f.run(); err != nil { - t.Fatal(err) - } - os.WriteFile(filepath.Join(f.options.dir, "compose.yaml"), []byte("changed"), 0o600) - f.calls = nil - if err := f.run(); err == nil { - t.Fatal("changed compose accepted") - } - for _, call := range f.calls { - if strings.HasPrefix(call, "compose up") { - t.Fatal("started modified compose") - } - } -} func TestInstallerInterruptedProcess(t *testing.T) { if root := os.Getenv("OAC_TEST_INSTALL_KILL"); root != "" { err := withLock(root, func() error { diff --git a/services/core/cmd/oac/main.go b/services/core/cmd/oac/main.go index 69e15bad7..ce3d20a88 100644 --- a/services/core/cmd/oac/main.go +++ b/services/core/cmd/oac/main.go @@ -112,11 +112,18 @@ func withLock(root string, fn func() error) error { return fn() } +var ( + checkConfig = []string{"run", "--rm", "--no-deps", "--entrypoint", "/usr/local/bin/oac-core", "core", "check-config"} + // Starting recreates changed services, so a resumed upgrade leaves no container on the previous release. + startServices = []string{"up", "-d", "--wait", "--wait-timeout", "180", "--pull", "never", "--remove-orphans"} +) + +// apply is the start shared by oac apply and a resumed installation. func apply(ctx context.Context, runner Runner) error { - if err := runner.Run(ctx, "run", "--rm", "--no-deps", "--entrypoint", "/usr/local/bin/oac-core", "core", "check-config"); err != nil { - return errors.New("configuration check failed; no service was changed") + if err := runner.Run(ctx, checkConfig...); err != nil { + return fmt.Errorf("configuration check failed; no service was changed: %w", err) } - return runner.Run(ctx, "up", "-d", "--wait") + return runner.Run(ctx, startServices...) } func coreKeyCommand(ctx context.Context, runner Runner, args []string) error { From ee075d3e8305891edbd35c35d326d900bc56690d Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Wed, 14 Oct 2026 13:53:38 +0000 Subject: [PATCH 2/3] Read the Windows command name in the upgrade test --- services/core/cmd/oac/install_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/core/cmd/oac/install_test.go b/services/core/cmd/oac/install_test.go index 2e9a8c6c0..2f16ab086 100644 --- a/services/core/cmd/oac/install_test.go +++ b/services/core/cmd/oac/install_test.go @@ -208,7 +208,7 @@ func TestInstallUpgradesAnotherReleaseOnlyWhenRequested(t *testing.T) { "staged compose run --rm --no-deps init", "compose up -d --wait --wait-timeout 180 --pull never --remove-orphans") after := installationFiles(t, f.options.dir) - if after["oac"] != "upgraded binary" || after[".env"] != before[".env"] || f.installedRevision() != "development" { + if after[cliName()] != "upgraded binary" || after[".env"] != before[".env"] || f.installedRevision() != "development" { t.Fatal("upgrade did not publish the release with the saved settings") } if _, ok := after["compose-sha256sums.txt"]; ok { From e1eacb2eb33a845e6eca9a4be63656f7073649f1 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Thu, 15 Oct 2026 06:18:26 +0000 Subject: [PATCH 3/3] Run init before the configuration check on resume and address review findings --- deploy/README.md | 4 ++-- deploy/install-source.py | 24 ++++++++++++++++------ deploy/test_install_source.py | 15 ++++++++++++-- docs/configuration.md | 5 +++-- docs/getting-started/install-options.md | 2 +- docs/zh/configuration.md | 7 ++++--- docs/zh/getting-started/install-options.md | 4 ++-- services/core/cmd/oac/install.go | 12 +++++------ services/core/cmd/oac/install_test.go | 7 ++++--- services/core/cmd/oac/main.go | 8 +++++++- services/core/cmd/oac/oac_test.go | 4 ++-- 11 files changed, 62 insertions(+), 30 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 4d10338d0..9fdc3258f 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -16,9 +16,9 @@ Preparation happens in `.staging`. On entry, the installer removes An existing directory runs the release named by `OAC_REVISION` in its `compose.yaml`. When that is the installer's release, the installer resumes: it keeps the saved settings, pulls only missing images and starts the services the way `oac apply` does. The port check applies only to new directories. Another release is refused unless `--upgrade` is given. An upgrade stages the new `compose.yaml` and `oac`, pulls the new images, then runs `oac-core check-config` and init against the staged `compose.yaml` together with the directory's `.env` and `compose.override.yaml`. Init upgrades the data volume or refuses, for example a downgrade; until it succeeds, the running installation is unchanged. The installer then renames `oac` and `compose.yaml` into place and starts the services with `--pull never --remove-orphans`, recreating every changed container. Rerunning the same command retries an interrupted upgrade. The upgrade leaves the deployment's Runtime unchanged; the final message names the console step that applies the new one. -`install.py --upgrade` upgrades a source-bundle installation the same way, under the same lock. It publishes the bundle's node payload beside earlier releases, renders the new `compose.yaml` and rewrites only its own `compose.override.yaml` entries, the four service images and Web's `/node-payload` bind, keeping every other entry. After the same two checks it moves the payload's `active.json`, renames `oac`, `compose.override.yaml` and `compose.yaml` into place and starts the services. Without `--upgrade` it refuses a non-empty directory; `--public-url` is required only for a new one. +`install.py --upgrade` upgrades a source-bundle installation the same way, under the same lock. It publishes the bundle's node payload beside earlier releases, renders the new `compose.yaml` and rewrites only its own `compose.override.yaml` entries, the four service images and Web's `/node-payload` bind, keeping every other entry. After the same two checks it moves the payload's `active.json`, renames `oac`, `compose.override.yaml` and `compose.yaml` into place and starts the services. On its own release it resumes like `oac install`; another release is refused without `--upgrade`, and `--public-url` is required only for a new directory. -The host `oac` command also implements `apply`, `core-key` and `rotate-core-key`. `apply` checks Core configuration before starting services and recreating those that changed. Reading the key executes `oac-web core-key` in Web. Rotation runs in the initialization container, where Linux file ownership is identical on every host, then restarts Core and Web. Start, stop, logs and removal use `docker compose`. The initialization image verifies and copies bundled node metadata without network access. Its receipt authenticates immutable metadata and credentials; the operator-rotatable Core key and its digest remain mutable. No service receives a Docker socket. Initialization emits structured step logs without credential values; view them with `docker compose logs --timestamps init`. +The host `oac` command also implements `apply`, `core-key` and `rotate-core-key`. `apply` runs init, which finishes a first start that stopped before it and refuses a data volume committed to another release, then checks Core configuration before starting services and recreating those that changed. Reading the key executes `oac-web core-key` in Web. Rotation runs in the initialization container, where Linux file ownership is identical on every host, then restarts Core and Web. Start, stop, logs and removal use `docker compose`. The initialization image verifies and copies bundled node metadata without network access. Its receipt authenticates immutable metadata and credentials; the operator-rotatable Core key and its digest remain mutable. No service receives a Docker socket. Initialization emits structured step logs without credential values; view them with `docker compose logs --timestamps init`. Web serves the console and forwards `/v1` and `/api/v1` to Core, so it is the only published service. HTTPS is terminated by the operator's reverse proxy or hosting platform, which routes to `web:8080`; `OAC_PUBLIC_URL` records that origin. diff --git a/deploy/install-source.py b/deploy/install-source.py index 8139e69bc..cbe27bf47 100644 --- a/deploy/install-source.py +++ b/deploy/install-source.py @@ -43,11 +43,17 @@ def install(bundle, destination, host, port, public_url, upgrade): fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: raise ValueError('Another operation is using this installation') from None + # A staging directory left by a killed run is never published. + for leftover in destination.glob('.staging-*'): + shutil.rmtree(leftover) existing = destination.exists() and any(destination.iterdir()) - if existing and not upgrade: - raise ValueError('Installation directory is not empty. To move it to this release, back up the installation (Upgrade in the operations guide), then rerun with --upgrade.') if existing and not all((destination / name).is_file() for name in ('compose.yaml', '.env')): raise ValueError(f'Incomplete installation: preserve {destination} and choose another directory') + previous = compose_revision((destination / 'compose.yaml').read_text()) if existing else '' + if existing and not previous: + raise ValueError(f'Incomplete installation: preserve {destination} and choose another directory') + if previous and previous != revision and not upgrade: + raise ValueError(f'This installation runs release {previous[:12]}; this bundle is {revision[:12]}. Back up the installation (Upgrade in the operations guide), then rerun with --upgrade.') if not existing and public_url is None: raise ValueError('--public-url is required for a new installation') # Docker validates both daemon access and the production template's required features. @@ -61,7 +67,6 @@ def install(bundle, destination, host, port, public_url, upgrade): os.chmod(root, 0o755) for name in files: os.chmod(Path(root) / name, 0o644) - previous = compose_revision((destination / 'compose.yaml').read_text()) if existing else '' if not existing: (destination / '.env').write_text(f"COMPOSE_PROJECT_NAME=oac-{uuid.uuid4().hex[:10]}\nOAC_HOST='{host}'\nOAC_WEB_PORT='{port}'\nOAC_PUBLIC_URL='{public_url}'\n") os.chmod(destination / '.env', 0o600) @@ -84,16 +89,23 @@ def install(bundle, destination, host, port, public_url, upgrade): web['volumes'] = [volume for volume in web.get('volumes', []) if not (isinstance(volume, dict) and volume.get('target') == '/node-payload')] web['volumes'].append({'type': 'bind', 'source': str(payload), 'target': '/node-payload', 'read_only': True}) (stage / 'compose.override.yaml').write_text(json.dumps(override, indent=2) + '\n') + os.chmod(stage / 'compose.override.yaml', 0o600) shutil.copyfile(bundle / 'oac', stage / 'oac') os.chmod(stage / 'oac', 0o700) compose = ('docker', 'compose', '-f', str(stage / 'compose.yaml'), '-f', str(stage / 'compose.override.yaml'), '--project-directory', str(destination)) if existing: # Init upgrades the data volume or refuses; until then the running release is untouched. + # On the same release init comes first: it finishes a first start that stopped before it, + # so check-config finds the secrets. + checks = [('run', '--rm', '--no-deps', '--entrypoint', '/usr/local/bin/oac-core', 'core', 'check-config'), + ('run', '--rm', '--no-deps', 'init')] + if previous == revision: + checks.reverse() try: - run(*compose, 'run', '--rm', '--no-deps', '--entrypoint', '/usr/local/bin/oac-core', 'core', 'check-config', cwd=destination) - run(*compose, 'run', '--rm', '--no-deps', 'init', cwd=destination) + for check in checks: + run(*compose, *check, cwd=destination) except subprocess.CalledProcessError: - raise ValueError('Upgrade refused; the installation is unchanged') from None + raise ValueError('Upgrade stopped; the running services are unchanged') from None else: run(*compose, 'config', '--quiet', cwd=destination) node_payload.activate(payload, revision) diff --git a/deploy/test_install_source.py b/deploy/test_install_source.py index b8b0aeb86..64d6dfe13 100644 --- a/deploy/test_install_source.py +++ b/deploy/test_install_source.py @@ -77,11 +77,18 @@ def test_complete_compose_preserves_secrets_and_serves_whole_payload(self): def test_existing_installation_is_not_replaced(self): self.destination.mkdir() (self.destination / 'keep').write_text('existing') - with self.assertRaisesRegex(ValueError, 'rerun with --upgrade'): + with self.assertRaisesRegex(ValueError, 'Incomplete installation'): self.install() self.assertEqual((self.destination / 'keep').read_text(), 'existing') self.assertEqual(self.calls, []) + def test_same_release_reruns_init_before_the_configuration_check(self): + # A first start that stopped before init left no secrets for check-config. + self.install() + self.calls = [] + self.install() + self.assertEqual([command[-1] for command in self.commands()], ['init', 'check-config', '--remove-orphans', 'core-key']) + def test_upgrade_checks_the_new_release_before_replacing_it_and_keeps_operator_entries(self): self.install() override = json.loads((self.destination / 'compose.override.yaml').read_text()) @@ -96,7 +103,10 @@ def test_upgrade_checks_the_new_release_before_replacing_it_and_keeps_operator_e (self.bundle / 'oac').write_bytes(b'upgraded operator') self.calls = [] - with self.assertRaisesRegex(ValueError, 'Upgrade refused; the installation is unchanged'): + with self.assertRaisesRegex(ValueError, 'rerun with --upgrade'): + self.install(image='sha256:' + '2' * 64) + self.assertEqual(self.snapshot(), before) + with self.assertRaisesRegex(ValueError, 'Upgrade stopped; the running services are unchanged'): self.install(upgrade=True, image='sha256:' + '2' * 64, failure='init') self.assertEqual(self.snapshot(), before) @@ -105,6 +115,7 @@ def test_upgrade_checks_the_new_release_before_replacing_it_and_keeps_operator_e self.assertEqual([command[-1] for command in self.commands()], ['check-config', 'init', '--remove-orphans', 'core-key']) services = json.loads((self.destination / 'compose.override.yaml').read_text())['services'] self.assertEqual(services['core']['volumes'], [nfs]) + self.assertEqual(os.stat(self.destination / 'compose.override.yaml').st_mode & 0o777, 0o600) self.assertEqual({services[name]['image'] for name in ('core', 'web', 'init', 'database')}, {'sha256:' + '2' * 64}) self.assertEqual([volume['target'] for volume in services['web']['volumes']], ['/node-payload']) self.assertIn('OAC_REVISION: ' + upgraded, (self.destination / 'compose.yaml').read_text()) diff --git a/docs/configuration.md b/docs/configuration.md index 80f4e6858..4cc8ef6b8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -22,8 +22,9 @@ Installer flags in [installation options](./getting-started/install-options.md) ### How oac apply works {#how-oac-apply-works} -1. It runs `oac-core check-config` with the `.env` you edited and changes nothing if a value is invalid. -2. It runs `docker compose up -d --wait`. Compose recreates only the services whose configuration changed. +1. It runs the `init` service, which checks that the data volume belongs to the installed release and finishes initializing it if the first start stopped early. +2. It runs `oac-core check-config` with the `.env` you edited and changes no service if a value is invalid. +3. It runs `docker compose up -d --wait` with the images already on the host. Compose recreates only the services whose configuration changed and removes containers the release no longer defines. Use `docker compose ps` to check the services. See [stop and restart](./getting-started/operations.md#stop-and-restart) for what a restart interrupts. diff --git a/docs/getting-started/install-options.md b/docs/getting-started/install-options.md index a0c43c7f2..251f0b0fe 100644 --- a/docs/getting-started/install-options.md +++ b/docs/getting-started/install-options.md @@ -45,7 +45,7 @@ Stop with `docker compose stop` using the same files and environment. Back up th ## Process settings -These flags are written to `.env` once. After installation, edit that file and run `oac apply`. Rerunning the installer does not change an installation that has already started. +These flags are written to `.env` once. After installation, edit that file and run `oac apply`. Rerunning the installer on an existing installation ignores these flags and starts it with the saved settings. | Flag | `.env` variable | | --- | --- | diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index 432d30886..bad35339b 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -1,7 +1,7 @@ --- title: "配置参考" source: docs/configuration.md -source_hash: 8a7d64ed4be902844e1cbd5c7725e8364599131f3d7df6ce409300159db0ce47 +source_hash: da6efd48db2be8e2b7afafceeb825be946d25faf606e1ed8b749f036fa3ed793 --- Core 安装的每项设置都恰好只有一个归属位置，分属以下三类： @@ -24,8 +24,9 @@ Core 安装的每项设置都恰好只有一个归属位置，分属以下三类 ### oac apply 的工作方式 {#how-oac-apply-works} -1. 它用你改过的 `.env` 运行 `oac-core check-config`。值无效时什么都不改。 -2. 它运行 `docker compose up -d --wait`。Compose 只重新创建配置有变化的服务。 +1. 它运行 `init` 服务，确认数据卷属于已安装的版本；如果首次启动中途停止，就完成初始化。 +2. 它用你改过的 `.env` 运行 `oac-core check-config`。值无效时不改动任何服务。 +3. 它用主机上已有的镜像运行 `docker compose up -d --wait`。Compose 只重新创建配置有变化的服务，并移除该版本不再定义的容器。 用 `docker compose ps` 检查服务。重启会中断哪些操作，见[停止和重启](getting-started/operations.md#stop-and-restart)。 diff --git a/docs/zh/getting-started/install-options.md b/docs/zh/getting-started/install-options.md index 08e23edf0..5604fa82b 100644 --- a/docs/zh/getting-started/install-options.md +++ b/docs/zh/getting-started/install-options.md @@ -1,7 +1,7 @@ --- title: "安装选项" source: docs/getting-started/install-options.md -source_hash: 23c43bed7e7368fe9e106572cd0061d6fea306e418e34340cafed34a47398bc5 +source_hash: 2fa7ecf80a4f78813dee915b9f327137b286bda61c706022c29512ba9e282e62 --- [默认安装](install.md)无需任何选项。本页介绍安装选项、Compose 部署和反向代理配置。 @@ -47,7 +47,7 @@ docker compose exec web oac-web core-key ## 进程设置 {#process-settings} -这些标志只会一次性写入 `.env`。安装完成后，编辑该文件并运行 `oac apply`。安装程序不会更改已经启动的安装。 +这些标志只会一次性写入 `.env`。安装完成后，编辑该文件并运行 `oac apply`。对已有安装重新运行安装程序时，这些标志会被忽略，安装以已保存的设置启动。 以下表格是由脚本自动生成的参考信息，因此保留为英文原文： | Flag | `.env` variable | diff --git a/services/core/cmd/oac/install.go b/services/core/cmd/oac/install.go index 5832ad7fd..52292a588 100644 --- a/services/core/cmd/oac/install.go +++ b/services/core/cmd/oac/install.go @@ -255,10 +255,8 @@ func (i installer) installLocked(ctx context.Context, o installOptions) error { } fmt.Println("Checking and downloading images...") for _, image := range strings.Fields(string(images)) { - if !fresh && !upgrade { - if _, err := i.docker(ctx, "", "image", "inspect", image); err == nil { - continue - } + if _, err := i.docker(ctx, "", "image", "inspect", image); err == nil { + continue } if _, err := i.docker(ctx, "", "pull", image); err != nil { return err @@ -267,9 +265,9 @@ func (i installer) installLocked(ctx context.Context, o installOptions) error { if upgrade { // Init upgrades the data volume or refuses; until then the running release is untouched. fmt.Println("Checking the upgrade...") - for _, check := range [][]string{checkConfig, {"run", "--rm", "--no-deps", "init"}} { + for _, check := range [][]string{checkConfig, initVolume} { if err := compose.Run(ctx, check...); err != nil { - return fmt.Errorf("Upgrade refused; the installation is unchanged: %w", err) + return fmt.Errorf("Upgrade stopped; the running services are unchanged: %w", err) } } for _, name := range []string{cliName(), "compose.yaml"} { @@ -277,6 +275,8 @@ func (i installer) installLocked(ctx context.Context, o installOptions) error { return err } } + } + if !fresh { // Installations from earlier releases kept a checksum that no longer describes compose.yaml. _ = os.Remove(filepath.Join(o.dir, "compose-sha256sums.txt")) } diff --git a/services/core/cmd/oac/install_test.go b/services/core/cmd/oac/install_test.go index 2f16ab086..8c69009ec 100644 --- a/services/core/cmd/oac/install_test.go +++ b/services/core/cmd/oac/install_test.go @@ -158,11 +158,12 @@ func TestInstallResumePreservesConfigAndUsesCachedImages(t *testing.T) { t.Fatal("saved configuration replaced") } for _, call := range f.calls { - if strings.HasPrefix(call, "pull") || strings.Contains(call, "down") || strings.Contains(call, "init") { + if strings.HasPrefix(call, "pull") || strings.Contains(call, "down") { t.Fatal(call) } } - assertCallsInOrder(t, f.calls, "compose "+strings.Join(checkConfig, " "), "compose "+strings.Join(startServices, " ")) + // Init finishes a first start that stopped before it, so check-config finds the secrets. + assertCallsInOrder(t, f.calls, "compose "+strings.Join(initVolume, " "), "compose "+strings.Join(checkConfig, " "), "compose "+strings.Join(startServices, " ")) } func TestInstallUpgradesAnotherReleaseOnlyWhenRequested(t *testing.T) { @@ -187,7 +188,7 @@ func TestInstallUpgradesAnotherReleaseOnlyWhenRequested(t *testing.T) { f.options.upgrade = true f.failure = "staged compose run --rm --no-deps init" - if err := f.run(); err == nil || !strings.HasPrefix(err.Error(), "Upgrade refused; the installation is unchanged") { + if err := f.run(); err == nil || !strings.HasPrefix(err.Error(), "Upgrade stopped; the running services are unchanged") { t.Fatal(err) } if after := installationFiles(t, f.options.dir); !maps.Equal(before, after) { diff --git a/services/core/cmd/oac/main.go b/services/core/cmd/oac/main.go index ce3d20a88..d27b34fb4 100644 --- a/services/core/cmd/oac/main.go +++ b/services/core/cmd/oac/main.go @@ -113,13 +113,19 @@ func withLock(root string, fn func() error) error { } var ( + initVolume = []string{"run", "--rm", "--no-deps", "init"} checkConfig = []string{"run", "--rm", "--no-deps", "--entrypoint", "/usr/local/bin/oac-core", "core", "check-config"} // Starting recreates changed services, so a resumed upgrade leaves no container on the previous release. startServices = []string{"up", "-d", "--wait", "--wait-timeout", "180", "--pull", "never", "--remove-orphans"} ) -// apply is the start shared by oac apply and a resumed installation. +// apply is the start shared by oac apply and a resumed installation. Init runs +// first: it refuses a data volume committed to another release and finishes one +// whose first start was interrupted, before check-config reads its secrets. func apply(ctx context.Context, runner Runner) error { + if err := runner.Run(ctx, initVolume...); err != nil { + return fmt.Errorf("data volume check failed; no service was changed: %w", err) + } if err := runner.Run(ctx, checkConfig...); err != nil { return fmt.Errorf("configuration check failed; no service was changed: %w", err) } diff --git a/services/core/cmd/oac/oac_test.go b/services/core/cmd/oac/oac_test.go index d3ccd058d..a6e169393 100644 --- a/services/core/cmd/oac/oac_test.go +++ b/services/core/cmd/oac/oac_test.go @@ -13,7 +13,7 @@ func TestApplyDoesNotStartWhenTheConfigurationCheckFails(t *testing.T) { var calls [][]string runner := scriptedRunner{run: func(args ...string) error { calls = append(calls, args) - if args[0] == "run" { + if args[len(args)-1] == "check-config" { return os.ErrInvalid } return nil @@ -21,7 +21,7 @@ func TestApplyDoesNotStartWhenTheConfigurationCheckFails(t *testing.T) { if err := apply(context.Background(), runner); err == nil { t.Fatal("apply continued") } - if len(calls) != 1 || calls[0][0] != "run" { + if len(calls) != 2 || strings.Join(calls[0], " ") != strings.Join(initVolume, " ") { t.Fatal(calls) } }