test(protect-ffi): migrate live EQL v3 data to v4 end to end

.github/workflows/integration-protect-ffi.yml

@@ -97,6 +97,35 @@ on:
       - 'packages/eql/crates/eql-bindings/src/v3/**'
       - 'packages/eql/crates/eql-bindings/sql/**'
       - 'packages/eql/Cargo.toml'
+      # EQL v4 and the codec the migration suites (`tests/migrate-v*-to-v4.test.ts`)
+      # re-encrypt through. `eql:v4:install` builds the v4 bundle from the SQL
+      # source with `tasks/build.sh` (which runs eql-codegen), and `eql:v4:codec`
+      # builds `tests/encryption/examples/v4_codec.rs` over eql-bindings' v4
+      # targets, stack-encrypt and stack-kms. No published package carries any of
+      # it, so these paths are the only inputs that change what those suites run.
+      # The crate entries are the eql-bindings modules the codec compiles and the
+      # native binding does not (`v4`, `encryption`, `encoding`), and the
+      # eql-codegen / eql-domains sources `tasks/build.sh` renders the v4 bundle
+      # from. test-eql.yml runs the Rust and SQL suites on those files but not
+      # this one, so without them a v4 change skips the migration suites.
+      # Derived in scripts/__tests__/lib/eql-v4-suite-inputs.mjs and held here by
+      # eql-v4-suite-triggers.test.mjs; wasm-build-inputs-paths-filter.test.mjs
+      # allows exactly these beyond the native binding's files, and only here.
+      - 'packages/eql/crates/eql-bindings/src/encoding.rs'
+      - 'packages/eql/crates/eql-bindings/src/encryption.rs'
+      - 'packages/eql/crates/eql-bindings/src/encryption/**'
+      - 'packages/eql/crates/eql-bindings/src/v4/**'
+      - 'packages/eql/crates/eql-codegen/Cargo.toml'
+      - 'packages/eql/crates/eql-codegen/src/**'
+      - 'packages/eql/crates/eql-codegen/templates/**'
+      - 'packages/eql/crates/eql-domains/Cargo.toml'
+      - 'packages/eql/crates/eql-domains/src/**'
+      - 'packages/eql/Cargo.lock'
+      - 'packages/eql/src/**'
+      - 'packages/eql/tasks/**'
+      - 'packages/eql/tests/encryption/**'
+      - 'packages/stack-encrypt/**'
+      - 'packages/stack-kms/**'
       - '.github/workflows/integration-protect-ffi.yml'
       - '.github/actions/build-ffi-binding/**'
       - '.github/actions/build-auth-binding/**'
@@ -146,6 +175,21 @@ on:
       - 'packages/eql/crates/eql-bindings/src/v3/**'
       - 'packages/eql/crates/eql-bindings/sql/**'
       - 'packages/eql/Cargo.toml'
+      - 'packages/eql/crates/eql-bindings/src/encoding.rs'
+      - 'packages/eql/crates/eql-bindings/src/encryption.rs'
+      - 'packages/eql/crates/eql-bindings/src/encryption/**'
+      - 'packages/eql/crates/eql-bindings/src/v4/**'
+      - 'packages/eql/crates/eql-codegen/Cargo.toml'
+      - 'packages/eql/crates/eql-codegen/src/**'
+      - 'packages/eql/crates/eql-codegen/templates/**'
+      - 'packages/eql/crates/eql-domains/Cargo.toml'
+      - 'packages/eql/crates/eql-domains/src/**'
+      - 'packages/eql/Cargo.lock'
+      - 'packages/eql/src/**'
+      - 'packages/eql/tasks/**'
+      - 'packages/eql/tests/encryption/**'
+      - 'packages/stack-encrypt/**'
+      - 'packages/stack-kms/**'
       - '.github/workflows/integration-protect-ffi.yml'
       - '.github/actions/build-ffi-binding/**'
       - '.github/actions/build-auth-binding/**'
@@ -361,11 +405,24 @@ jobs:
           # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs
           cache_key_prefix: mise-v1-ubuntu-2404
 
-      # One task, four things the suite cannot run without (see
+      # The EQL workspace's Rust, which `mise run setup` compiles for the
+      # migration suites: eql-codegen (the v4 bundle) and the v4 codec. Without
+      # a cache that is a cold build of stack-encrypt and its HTTP stack on every
+      # run. Keyed apart from the `sqlx-tests` cache test-eql.yml saves, which
+      # carries the SQLx graph this job never builds.
+      - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+        with:
+          workspaces: packages/eql
+          shared-key: eql-v4-codec
+
+      # One task, six things the suite cannot run without (see
       # integration-tests/tasks.toml): `pnpm install --frozen-lockfile` (a
       # no-op here — the step above already ran it), `docker compose up` for
-      # Postgres, the EQL **v2** bundle from a GitHub release, and EQL **v3**
-      # built from the in-tree `@cipherstash/eql`.
+      # Postgres, the EQL **v2** bundle from a GitHub release, EQL **v3**
+      # built from the in-tree `@cipherstash/eql`, EQL **v4** built from the
+      # in-tree EQL source, and the Rust v4 codec. The last two serve
+      # `tests/migrate-v*-to-v4.test.ts`: Stack Encrypt is the only v4
+      # producer, and no TypeScript SDK reaches it yet.
       #
       # BOTH EQL versions are required and nothing else in this repo installs
       # v2: `tests/postgres.test.ts` creates `eql_v2_encrypted` columns and
@@ -381,7 +438,7 @@ jobs:
       # `integration-db`'s per-job project names deliberately do not produce.
       # The fixed host port (5436) that comes with it cannot collide with the
       # shared stacks, which publish on ephemeral ports.
-      - name: Provision the suite (Postgres, EQL v2 + v3)
+      - name: Provision the suite (Postgres, EQL v2 + v3 + v4, v4 codec)
         working-directory: languages/typescript/packages/protect-ffi
         run: mise run setup
 

AGENTS.md

@@ -225,10 +225,10 @@ stays optional for everyone else.
 
 `languages/typescript/packages/protect-ffi/integration-tests/` is 19 files of **live** coverage —
 encrypt/decrypt, lock context, keysets, JS auth strategies, JSON SteVec,
-Postgres (EQL v2 *and* v3), and a WASM round trip — and it is the only place
-several of those paths are exercised at all. It needs three things a normal
-`pnpm test` does not have: **Docker**, **CipherStash credentials**, and **both
-EQL versions installed** in the database.
+Postgres (EQL v2 *and* v3), EQL v3 -> v4 migration, and a WASM round trip — and
+it is the only place several of those paths are exercised at all. It needs
+three things a normal `pnpm test` does not have: **Docker**, **CipherStash
+credentials**, and **EQL v2, v3 and v4 installed** in the database.
 
 - **It is a pnpm workspace member, and `pnpm test` must never reach it.** Named
   literally in `pnpm-workspace.yaml` (the `languages/typescript/packages/*` glob is one level deep and
@@ -290,6 +290,16 @@ EQL versions installed** in the database.
   `eql_v2.add_encrypted_constraint`) while `tests/postgres-v3.test.ts` needs the
   `eql_v3_*` domains. Skip either and half the suite fails on missing SQL
   functions.
+- **The migration suites need EQL v4 and a Rust codec.**
+  `tests/migrate-v*-to-v4.test.ts` decrypt legacy rows with protect-ffi and
+  re-encrypt them as v4, and Stack Encrypt is the only v4 producer, so they call
+  `packages/eql/tests/encryption/examples/v4_codec.rs` through
+  `tests/v4-codec.ts`. `eql:v4:install` builds the v4 bundle from the in-tree
+  EQL source (`bash tasks/build.sh`, called directly so it needs cargo and not
+  `packages/eql`'s mise config) and `eql:v4:codec` builds the codec; both are
+  part of `mise run setup`. The CI filter cannot name `packages/eql/crates/**`
+  (`wasm-build-inputs-paths-filter.test.mjs`), so an edit to eql-codegen or
+  eql-bindings' v4 module reaches this suite only through `test-eql.yml`.
 - **`eql:v3:install` builds `@cipherstash/eql` first, and has to.** The task
   reads the bundle through `@cipherstash/eql/sql`, which the package's `exports`
   map resolves to `dist/sql.js` — a tsup output. That was free while `npm ci`

languages/typescript/packages/protect-ffi/integration-tests/package.json

@@ -2,7 +2,7 @@
   "name": "@cipherstash/ffi-integration-tests",
   "version": "1.0.2",
   "private": true,
-  "description": "Live integration coverage for @cipherstash/protect-ffi — real ZeroKMS, real Postgres, EQL v2 and v3",
+  "description": "Live integration coverage for @cipherstash/protect-ffi — real ZeroKMS, real Postgres, EQL v2, v3 and v4",
   "license": "MIT",
   "scripts": {
     "typecheck": "tsc",

languages/typescript/packages/protect-ffi/integration-tests/tasks.toml

@@ -45,6 +45,30 @@ fi
 docker exec -i protect-ffi-postgres psql postgresql://cipherstash:password@postgres:5432/cipherstash -f- < "$tmp"
 """
 
+# EQL v4, for the migration suites (`tests/migrate-v*-to-v4.test.ts`). No npm
+# package ships the v4 bundle yet, so it is built from the in-tree EQL source:
+# `tasks/build.sh` is what `mise run build` runs in `packages/eql`, called
+# directly so this task needs only cargo, not that directory's mise config (which
+# a runner has not trusted). It writes both bundles; only v4 is installed here,
+# v3 comes from `eql:v3:install` above.
+["eql:v4:install"]
+description = "Build EQL v4 from the in-tree source and install it beside v2 and v3"
+dir = "{{ config_root }}/../../../../packages/eql"
+run = """
+set -e
+bash tasks/build.sh
+test -s release/cipherstash-encrypt-v4.sql
+docker exec -i protect-ffi-postgres psql -v ON_ERROR_STOP=1 -q postgresql://cipherstash:password@postgres:5432/cipherstash -f- < release/cipherstash-encrypt-v4.sql
+"""
+
+# The Rust codec the migration suites re-encrypt through: Stack Encrypt is the
+# only EQL v4 producer and no TypeScript SDK reaches it yet. See
+# `tests/v4-codec.ts` and `packages/eql/tests/encryption/examples/v4_codec.rs`.
+["eql:v4:codec"]
+description = "Build the EQL v4 codec the migration suites call"
+dir = "{{ config_root }}/../../../../packages/eql"
+run = "cargo build --locked -p eql-encryption-tests --example v4_codec"
+
 # `pnpm install`, not `npm ci`: this directory is a pnpm workspace member as of
 # CIP-3744, so its dependencies come from the repo lockfile and its
 # `@cipherstash/protect-ffi` and `@cipherstash/eql` resolve to the packages in
@@ -62,6 +86,8 @@ run = [
     "mise run eql:download",
     "mise run eql:install",
     "mise run eql:v3:install",
+    "mise run eql:v4:install",
+    "mise run eql:v4:codec",
 ]
 
 ["test:integration"]