diff --git a/.github/workflows/bench-eql.yml b/.github/workflows/bench-eql.yml index 8d46945ae..390745c0f 100644 --- a/.github/workflows/bench-eql.yml +++ b/.github/workflows/bench-eql.yml @@ -64,6 +64,7 @@ on: - "packages/eql/docker/**" - "packages/eql/README.md" - "packages/eql/SUPABASE.md" + - "conformance/stack-encrypt/**" - "packages/stack-encrypt/**" - "packages/stack-encrypt-derive/**" - "packages/stack-kms/**" diff --git a/.github/workflows/crap-crates.yml b/.github/workflows/crap-crates.yml index 8a325ab1a..79020aa40 100644 --- a/.github/workflows/crap-crates.yml +++ b/.github/workflows/crap-crates.yml @@ -56,10 +56,12 @@ env: CARGO_NET_GIT_FETCH_WITH_CLI: true NEXTEST_PROFILE: ci +# cargo-crap 0.6.1's prebuilt GNU/Linux binary needs GLIBC 2.39 (Ubuntu 24.04). +# Keep the runner and its mise cache on that image. The gate itself is unchanged. jobs: crap-stack-auth: name: CRAP (stack-auth) - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 45 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -74,16 +76,19 @@ jobs: working_directory: . cache: true # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs - cache_key_prefix: mise-v1-ubuntu-2204 + cache_key_prefix: mise-v1-ubuntu-2404 - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + - name: Check the CRAP executable before collecting coverage + run: cargo crap --version + - name: Run the CRAP gate run: mise run crap:stack-auth -- --format github crap-stack-encrypt: name: CRAP (stack-encrypt) - runs-on: blacksmith-16vcpu-ubuntu-2204 + runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 45 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -97,9 +102,12 @@ jobs: working_directory: . cache: true # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs - cache_key_prefix: mise-v1-ubuntu-2204 + cache_key_prefix: mise-v1-ubuntu-2404 - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + - name: Check the CRAP executable before collecting coverage + run: cargo crap --version + - name: Run the CRAP gate run: mise run crap:stack-encrypt -- --format github diff --git a/.github/workflows/test-eql.yml b/.github/workflows/test-eql.yml index 438cfb3bf..fc88db912 100644 --- a/.github/workflows/test-eql.yml +++ b/.github/workflows/test-eql.yml @@ -109,6 +109,7 @@ on: # cannot see them (a task body names a cargo package, not a path); the # manifest walk in eql-suite-ci.test.mjs does, and fails when one of them # is missing from this list. + - "conformance/stack-encrypt/**" - "packages/stack-encrypt/**" - "packages/stack-encrypt-derive/**" - "packages/stack-kms/**" @@ -208,6 +209,7 @@ jobs: - "packages/eql/docker/**" - "packages/eql/README.md" - "packages/eql/SUPABASE.md" + - "conformance/stack-encrypt/**" - "packages/stack-encrypt/**" - "packages/stack-encrypt-derive/**" - "packages/stack-kms/**" diff --git a/.github/workflows/tests-golang.yml b/.github/workflows/tests-golang.yml index 0fa4a6967..62bd9453a 100644 --- a/.github/workflows/tests-golang.yml +++ b/.github/workflows/tests-golang.yml @@ -60,6 +60,7 @@ on: - scripts/go-binding-test.sh - scripts/stack-conformance.sh - scripts/stack-conformance-selection.mjs + - scripts/stack-eql-conformance.mjs - Cargo.toml - Cargo.lock - .cargo/** @@ -99,6 +100,7 @@ on: - scripts/go-binding-test.sh - scripts/stack-conformance.sh - scripts/stack-conformance-selection.mjs + - scripts/stack-eql-conformance.mjs - Cargo.toml - Cargo.lock - .cargo/** @@ -144,6 +146,18 @@ jobs: with: persist-credentials: false + # Native EQL conformance uses the EQL workspace's toolchain. Install it + # before root setup, so ordinary guest commands retain the root pin. + # Its local DATABASE_URL must not enable database tests in this job. + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + working_directory: packages/eql + env: false + cache: true + cache_key_prefix: mise-v1-ubuntu-2204 + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 with: version: 2026.4.0 @@ -258,6 +272,17 @@ jobs: echo "RLIMIT_MEMLOCK: $(ulimit -l) KiB" mise run go:test + - name: Required EQL and JSON conformance + env: + STACK_CONFORMANCE_REPORT_DIR: ${{ runner.temp }}/eql-conformance-fast + run: mise run test:conformance:eql + + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: eql-conformance-fast + path: ${{ runner.temp }}/eql-conformance-fast/*.json + if-no-files-found: error + - name: Fresh Go scalar artifacts open in Rust env: STACK_CONFORMANCE_INPUT: ${{ runner.temp }}/stack-conformance-go.json @@ -392,6 +417,8 @@ jobs: # v4 bundle under packages/eql's mise config, which pins its own Rust; the # guests are the ones wasi-check built and checked. eql-postgres: + env: + MISE_ENV: test name: Go rows and queries in PostgreSQL with EQL v4 needs: wasi-check runs-on: blacksmith-16vcpu-ubuntu-2204 @@ -434,6 +461,16 @@ jobs: # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs cache_key_prefix: mise-v1-ubuntu-2204 + # mise caches Rust symlinks, not ~/.rustup. A cache hit can leave a + # toolchain directory without cargo and still report "all tools installed". + # Reconcile both workspaces' configured toolchains before cargo is used. + - name: Restore Rust toolchains behind the cached mise links + run: | + mise install --force rust + mise -C packages/eql install --force rust + mise x -- cargo --version + mise -C packages/eql x -- cargo --version + # `mise run build` compiles eql-codegen to write the bundle. - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: @@ -459,11 +496,18 @@ jobs: # Builds the bundle from the tree, installs it, and runs the tests; a # skip fails the task, so a missing guest or database cannot pass. - - name: Go rows and queries through EQL v4 + - name: Go rows, queries and required JSON conformance through EQL v4 env: EQL_TEST_DATABASE_URL: "host=localhost port=${{ job.services.postgres.ports['5432'] }} user=postgres password=postgres dbname=postgres sslmode=disable" + STACK_CONFORMANCE_REPORT_DIR: ${{ runner.temp }}/eql-conformance-postgres run: mise run go:test:postgres + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: eql-conformance-postgres + path: ${{ runner.temp }}/eql-conformance-postgres/*.json + if-no-files-found: error + # Round trips through real ZeroKMS. The Go live tests skip without their # STACK_ENCRYPT_TEST_* variables, so the secrets pre-flight runs first and # turns a missing credential into a failure rather than a silent skip. The diff --git a/AGENTS.md b/AGENTS.md index 73a0a56c3..20a056d9e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -752,8 +752,9 @@ exact pin would stop the suite sharing one `cts-common` with it. `tests/ui` binary, doctests, rustdoc, the node bindings and `stack-auth-wasm`), `tests-golang.yml` (WASI checks, the guests, the Go module on Linux, macOS and Windows, its PostgreSQL tests against the EQL v4 - bundle built from the tree — `mise run go:test:postgres` locally — and the - live tests), `crap-crates.yml`, + bundle built from the tree — `mise run go:test:postgres` locally, which also + runs the Rust and Go EQL/JSON conformance lane and so needs `psql` and the + `MISE_ENV=test` cargo tools — and the live tests), `crap-crates.yml`, `mutants.yml`, `fuzz.yml`, `miri.yml` and `udeps.yml`. `scripts/__tests__/crates-ci.test.mjs` fails when a mise task in the root `mise.toml` or the five crate `tasks.toml` files loses its last CI caller. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c288acd22..a902291e6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -95,8 +95,12 @@ mise run wasm:auth-guest:build mise run go:test ``` -For EQL v4 PostgreSQL coverage, set `EQL_TEST_DATABASE_URL` and run -`mise run go:test:postgres`. Go examples that connect to ZeroKMS need +For EQL v4 PostgreSQL coverage, set `EQL_TEST_DATABASE_URL` to a disposable +database and run `mise run go:test:postgres`. The task builds the EQL v4 bundle +from this tree and installs it into that database, so it replaces the EQL there. +It needs `psql`, the Rust test tools (`mise install --env test`) and both +deterministic guests. It refuses to run while a fixture-update variable, such as +`STACK_ENCRYPT_UPDATE_FIXTURES`, is set. Go examples that connect to ZeroKMS need CipherStash credentials. The [Go module guide](languages/golang/README.md) links to the SDK, generator, credentials, examples and release notes. Commit generated `*_stash.go` files when changing Go model tags; CI checks for drift. diff --git a/conformance/stack-encrypt/README.md b/conformance/stack-encrypt/README.md index d4dcb6651..635a99185 100644 --- a/conformance/stack-encrypt/README.md +++ b/conformance/stack-encrypt/README.md @@ -20,6 +20,8 @@ from each profile's own version. `` is one of: producer regenerates this file, deliberately; runners compare against it. - `inputs.`: extra inputs keyed by case ID that only `` needs, such as Go host values that have no language-neutral form. +- `enrollment`: the profile's requirement IDs mapped onto existing compiled + tests that are its evidence, rather than cases of its own. Every file has a `version` and `profile` header. An `expected` file also names its `producer`, and an `inputs` file names its `runner`. @@ -37,11 +39,14 @@ these rules. | `record-contexts-v1.cases.json` | cases | `tests/context_conformance.rs` | `context_conformance_test.go` | | `record-contexts-v1.expected.rust-native.json` | expected | `tests/context_conformance.rs` (writes it) | `context_conformance_test.go` | | `record-context-boundaries-v1.cases.json` | cases | `tests/context_conformance.rs` | `context_conformance_test.go` | +| `eql-json-v1.cases.json` | cases | `tests/json_operators.rs`; EQL's `tests/encryption/tests/json_tree_data_plan.rs` | `json_conformance_test.go` | +| `eql-json-v1.enrollment.json` | enrollment | `scripts/stack-eql-conformance.mjs` runs the tests it names | `scripts/stack-eql-conformance.mjs` runs the tests it names | | `fixture-enrollment.json` | index | `tests/common/mod.rs` | `fixture_conformance_test.go` | Rust readers are under `packages/stack-encrypt/`; Go readers are under -`languages/golang/encrypt/`. `fixture-enrollment.json` is not a profile: it maps -requirement IDs onto existing fixtures and tests (see +`languages/golang/encrypt/`; EQL's are under `packages/eql/`. +`fixture-enrollment.json` is the one file not named after a profile: it maps +requirement IDs of several profiles onto existing fixtures and tests (see [the fixture contract](fixture-contract.md)), and `scripts/stack-conformance-selection.mjs` reads it to choose the Go tests that `mise run test:conformance` runs. @@ -70,6 +75,15 @@ is `languages/golang/encrypt/conformance_test.go`. Go calls the record interface used by generated code. Testing public SDK declarations is a separate layer. Python and TypeScript runners are pending; their absence is not a passing result. +## EQL and JSON operations + +The [EQL/JSON contract](eql-json-contract.md) enrolls resolver capabilities, +document writes, queries, extraction/subtree reads, element-match and predicate +matching. It reuses the native, Go and independent PostgreSQL suites and shares +write/predicate case data. Run `mise run test:conformance:eql` and the separate +required `test:conformance:eql:postgres` task; see the contract for guest builds, +disposable database setup, intentional SQL differences and version reports. + ## Context composition and expectations The [context contract](context-contract.md) covers ordered extensions, canonical @@ -93,8 +107,8 @@ checks. The task runs the existing tests directly, and a missing guest fails when conformance is required. The ordinary CI suites already execute them. This is representative fixture coverage, not an exhaustive type/operator -matrix. Wider fresh artifact exchange, shared query refusals and EQL/JSON -operations remain separate work. +matrix. Wider fresh artifact exchange and shared scalar query refusals remain +separate work. The EQL/JSON profiles are described below. ## `record-reads-v1` contract diff --git a/conformance/stack-encrypt/coverage.md b/conformance/stack-encrypt/coverage.md index ca8c1bc04..37160f81e 100644 --- a/conformance/stack-encrypt/coverage.md +++ b/conformance/stack-encrypt/coverage.md @@ -106,3 +106,19 @@ Go proves canonical AAD indirectly through its actual guest plan frame and nativ ciphertext opening; provider-call instrumentation remains a separate gap. Nested composite extensions remain a native API capability rather than an unsupported feature silently advertised by the Go/dynamic record profile. + +## EQL/JSON checkpoint + +The [EQL/JSON contract](eql-json-contract.md) and its enrollment index cover +resolver/catalog capabilities, JSON writes and queries, request-bound reads, +element-match and predicate-match. Existing Rust, Go and PostgreSQL suites stay +in place. The common document and predicate-refusal tables now live in +`v1/eql-json-v1.cases.json`: Rust and Go execute them, retaining native key-call checks, +Go diagnostic mapping and all extra cases from the original suites. + +The database lane preserves independent plaintext oracles and generated +properties. Required test inventory/no-skip checks and engine/guest/SQL identity +reports make enrollment observable. This consolidates data and requirements; +it does not replace adapter tests or claim whole-database authentication. +Fresh JSON artifact exchange, released-version matrices and Python/TypeScript +runners remain open. diff --git a/conformance/stack-encrypt/eql-json-contract.md b/conformance/stack-encrypt/eql-json-contract.md new file mode 100644 index 000000000..5cef1a74e --- /dev/null +++ b/conformance/stack-encrypt/eql-json-contract.md @@ -0,0 +1,134 @@ +# EQL and JSON operation profiles + +These experimental profiles enroll the existing EQL v4 JSON tree operations. +They do not describe EQL v3 SteVec, or certify an entire SDK. The +[enrollment index](v1/eql-json-v1.enrollment.json) names mandatory compiled tests; +[shared cases](v1/eql-json-v1.cases.json) supply document text and predicate refusals to +native Rust and Go. Python and TypeScript have no enrolled runner yet. + +## Requirements and evidence + +| Requirement | Contract | Existing evidence retained | +|---|---|---| +| `eql-resolver-catalog-v1` | The resolver names catalog domains, their producibility and operators. Unknown names, unavailable EQL builds and unsupported capabilities are refused. JSON options require a JSON target. | Native `targets` and `json_tree_data_plan`; Go catalog and plain-guest refusal tests | +| `json-write-v1` | JSON documents, including scalar roots, empty containers, JSON null and integer boundaries, round-trip as values. SQL null stores no document. Invalid text is refused without exposing it; native tests observe no key request. Typed and dynamic plans agree on search terms and can open each other's output. | Native typed/data-plan suites, Go raw and generated typed document APIs; existing seeded derivation fixture | +| `json-query-v1` | Containment, reverse containment, equality at a path, ordered bounds, selectors and child operations derive their documented operand shapes. Unsupported requests fail. | Native engine/EQL operand comparisons, Go generated queries, independent PostgreSQL `jsonb` containment oracles | +| `json-request-bound-reads-v1` | Extraction and subtree/wildcard reads open against the prepared path, column, position and bounds. Wrong requests and detectable tampering fail. | Native adversarial read suite, EQL request/open tests, Go selector reads, PostgreSQL extraction/subtree and hostile-row suites | +| `json-extraction-v1` | Single-node extraction opens the requested scalar. | Native scalar extraction, Go selectors and PostgreSQL plaintext extraction oracle | +| `json-subtree-v1` | Complete bounded subtrees and wildcard reads match plaintext queries. | Native/EQL prepared reads and Go/PostgreSQL read paths | +| `json-tamper-v1` | Detectable mixed-document, truncated-chain and substituted-column answers are refused. | Native and PostgreSQL adversarial suites; Go observes wrong-column rejection | +| `json-element-match-v1` | Every condition must hold on one element. Ordering paths and resolver capability are required; a probe beyond the declared bound raises rather than silently truncating. | Native operator/capability tests, Go generated member APIs, SQL domain and independent plaintext predicate tests | +| `json-predicate-match-v1` | Supported text compiles to existing operators; refusals preserve reason, construct and byte offset, with no caller text in diagnostics. Missing and null paths obey the documented truth rules. | Shared negative corpus, native compiler/property tests, Go generated `Match`, SQL `jsonb_path_match` and handwritten-operator oracles | + +The index is an evidence inventory, not a second copy of every existing test. +Native execution retains whole binaries, including their unindexed properties +and regressions. Go selects the exact enrolled tests and rejects missing passes, +failures and skips, including skipped subtests. Native inventory checks reject +missing names, filtered tests and a wrong `#[ignore]` state before execution. +Only the PostgreSQL lane executes the database tests marked `#[ignore]`. + +## Predicate grammar + +The current grammar in `packages/stack-encrypt/src/dynamic/json_query.rs` and +`json_predicate.rs` is the executable specification. This profile pins: + +```text +predicate := comparison | exists(path) | exists(path ? (filter)) + | (predicate) | !(predicate) + | predicate && predicate | predicate || predicate +comparison := path op value | value op path +op := == | != | <> | < | <= | > | >= +path := $ followed by .key, ."quoted key", or [*] steps +value := JSON scalar literal | $variable | $"quoted variable" +filter := comparison on @ or @.member, optionally joined with && +``` + +Negation binds before `&&`, which binds before `||`. Variables are a JSON +object; absence of `vars` means an empty object. Scalar literals are strings, +numbers, booleans and null; objects/arrays cannot be comparison values. +Orderings and multi-condition element filters require numbers or strings and +configured ordered paths. A filter requires a wildcard in its outer path; +its member cannot contain a wildcard. Predicate nesting is limited to 32. +A wildcard bound must be positive; its probe prevents an undersized bound from +silently producing a partial answer. + +Arithmetic, regex, `starts with`, `is unknown`, mode words, methods, `.*`, `.**`, +slices, `last`, path-to-path/value-to-value comparisons, array positions, +filters outside `exists`, and `||`, `!`, `!=` or nested `exists` inside filters +are unsupported. `!=` at a wildcard is refused. The corpus pins representative +syntax, unsupported-construct, variable, type, depth and path diagnostics. +Existing parser tests retain the wider grammar matrix and adapter-specific +invalid-input checks; the shared corpus is not an exhaustive grammar fuzzer. +Offsets are UTF-8 byte offsets, not character indices. Each refusal also names +`at_text`, the suffix beginning at that offset. Omitted `vars` exercises absent +variables rather than an explicit empty object. + +## Nulls, oracles and intentional differences + +SQL null (no encrypted document), JSON null (a document/node), and an absent +path are distinct. The SQL predicate suites compare both selected rows and +three-valued answers. An absent path does not satisfy a comparison. Ordering a +JSON null can produce SQL unknown where plaintext jsonpath returns false; +the compiled predicate adds the null test under negation where it affects +selected rows. This exception is checked explicitly, not normalized away. + +The independent oracles use PostgreSQL `jsonb @>`, `<@`, `jsonb_path_query` and +`jsonb_path_match`, alongside handcrafted plaintext conditions. They do not +calculate expected results from encrypted operands. Their supported-domain +assumptions and exceptions remain explicit: + +- Paths are literal. There is no lax implicit array unwrapping or scalar-to-array + wrapping. Use `[*]` and keep array shapes consistent. +- Keep one scalar kind at an ordered path. Terms do not preserve the plaintext + type distinctions used by jsonpath's cross-kind comparisons. +- Order terms fold text case/accents and compare numbers as float64. Equality in + a multi-condition element filter uses those terms; standalone `==` is exact. +- Reverse containment excludes the plaintext root-scalar-in-root-array special + case; the generated oracle logs how many such comparisons it excludes. +- Storage and projection list order is randomized. Compare projections as sets + and stored selector collections as multisets. The existing deterministic + `packages/stack-encrypt/tests/fixtures/json_tree_vectors.json` pins a seeded + permutation, tags and tokens, not a production ordering promise. Fresh + ciphertext bytes are not equality fixtures. +- Read authentication detects substituted paths, wrong columns, mixed-document + subtrees, forged tags and incomplete returned chains. It does not establish + database completeness or row identity: withholding a whole answer can look + like no node, and another row's node at the same path can open. Those limits + have named native evidence in the index. + +## Required execution and identity + +From the repository root: + +```sh +mise run wasm:guest:build +mise run wasm:guest:build:deterministic +mise run wasm:guest:build:eql +mise run wasm:guest:build:eql:deterministic +mise run test:conformance:eql + +# A disposable database: this command installs/replaces EQL v4 there. +EQL_TEST_DATABASE_URL='host=localhost port=5432 user=postgres password=postgres dbname=postgres' \ + mise run test:conformance:eql:postgres +``` + +No CipherStash credentials are required. Both lanes are required in +`tests-golang.yml` (the PostgreSQL lane is called by `go:test:postgres`); conformance changes trigger them. The database lane rebuilds +SQL from this tree, installs exactly the bundle whose SHA-256 it records, and +passes the same connection to both languages. It fails without a database or +required guests. Required mode deliberately applies to every Go test using +`deterministicEQLClient`, including tests outside the enrollment index. A fast-lane pass alone is not SQL conformance. + +Reports include the native engine and EQL binding package versions, root/EQL +lockfile hashes, corpus/enrollment hashes, deterministic guest hashes, executed +requirement test names, and (for PostgreSQL) the installed EQL version and schema name, the expected schema name, PostgreSQL version and actual SQL bundle hash. Development bundles can +report `DEV`; their hash distinguishes them. CI uploads reports and fails if +none were produced. Locally set `STACK_CONFORMANCE_REPORT_DIR` to retain them; +the same report is always printed after success. These describe this checkout +and its guest artifacts, not a claim of compatibility across released versions. + +Required checks reject all fixture-update environment variables. Extending a +language binding means running the applicable shared cases and providing its +own adapter evidence for each claimed requirement. Fresh cross-language JSON +artifact exchange, a release compatibility matrix, and Python/TypeScript +runners remain separate work. diff --git a/conformance/stack-encrypt/v1/eql-json-v1.cases.json b/conformance/stack-encrypt/v1/eql-json-v1.cases.json new file mode 100644 index 000000000..2a5280ee8 --- /dev/null +++ b/conformance/stack-encrypt/v1/eql-json-v1.cases.json @@ -0,0 +1,214 @@ +{ + "version": 1, + "profile": "eql-json-v1", + "documents": [ + "{\"name\":\"Alice\",\"age\":34,\"visits\":[{\"at\":3},{\"at\":1}]}", + "{\"name\":\"Alice\",\"age\":34,\"visits\":[{\"at\":3,\"where\":\"x\"},{\"at\":1}],\"tags\":[\"a\",\"b\"]}", + "{\"name\":\"Bob\",\"age\":7.5,\"nested\":{\"deep\":[[1,2],{}]},\"none\":null}", + "[1,\"two\",{\"three\":3}]", + "\"a scalar root\"", + "18446744073709551615", + "-9223372036854775808", + "-926809.5051008083", + "null", + "{}", + "[]" + ], + "invalid_documents": [ + { + "text": "{\"secret\":1,\"secret\":2}", + "reason": "json_duplicate_key" + }, + { + "text": "99999999999999999999", + "reason": "json_integer_range" + }, + { + "text": "{\"secret\":", + "reason": "json_syntax" + }, + { + "text": "", + "reason": "json_syntax" + } + ], + "predicate_refusals": [ + { + "id": "text-the-grammar-does-not-read", + "predicate": "$.extra.country = $v", + "vars": { + "v": "secret-nz", + "flag": true + }, + "reason": "json_predicate_syntax", + "construct": "text the grammar does not read", + "at": 16, + "at_text": "= $v" + }, + { + "id": "like-regex", + "predicate": "$.extra.country like_regex \"secret\"", + "vars": { + "v": "secret-nz", + "flag": true + }, + "reason": "json_predicate_unsupported", + "construct": "`like_regex`", + "at": 16, + "at_text": "like_regex \"secret\"" + }, + { + "id": "arithmetic", + "predicate": "$.weight + 1 == 2", + "vars": { + "v": "secret-nz", + "flag": true + }, + "reason": "json_predicate_unsupported", + "construct": "arithmetic (`+`, `-`, `*`, `/`, `%`)", + "at": 9, + "at_text": "+ 1 == 2" + }, + { + "id": "a-method", + "predicate": "$.weight.size() == 1", + "vars": { + "v": "secret-nz", + "flag": true + }, + "reason": "json_predicate_unsupported", + "construct": "a method (`.name()`)", + "at": 8, + "at_text": ".size() == 1" + }, + { + "id": "a-filter-outside-exists", + "predicate": "$.extra.preferred_languages[*] ? (@ == $v) == 1", + "vars": { + "v": "secret-nz", + "flag": true + }, + "reason": "json_predicate_unsupported", + "construct": "a filter outside `exists`", + "at": 31, + "at_text": "? (@ == $v) == 1" + }, + { + "id": "in-a-filter", + "predicate": "exists($.extra.emergency_contacts[*] ? (@.relationship == $v || @.age > 1))", + "vars": { + "v": "secret-nz", + "flag": true + }, + "reason": "json_predicate_unsupported", + "construct": "`||` in a filter", + "at": 61, + "at_text": "|| @.age > 1))" + }, + { + "id": "an-unknown-variable", + "predicate": "$.extra.country == $missing", + "vars": { + "v": "secret-nz", + "flag": true + }, + "reason": "json_predicate_unknown_variable", + "construct": "a variable", + "at": 19, + "at_text": "$missing" + }, + { + "id": "a-value-an-ordering-cannot-take", + "predicate": "$.weight < $flag", + "vars": { + "v": "secret-nz", + "flag": true + }, + "reason": "json_predicate_value_type", + "construct": "a value", + "at": 11, + "at_text": "$flag" + }, + { + "id": "an-object", + "predicate": "$.extra.country == $obj", + "vars": { + "obj": { + "secret-nz": 1 + } + }, + "reason": "json_predicate_value_type", + "construct": "a value", + "at": 19, + "at_text": "$obj" + }, + { + "id": "at-a-wildcard", + "predicate": "$.extra.preferred_languages[*] != $v", + "vars": { + "v": "secret-nz", + "flag": true + }, + "reason": "json_predicate_not_equal_at_wildcard", + "construct": "`!=` at a path with a wildcard", + "at": 31, + "at_text": "!= $v" + }, + { + "id": "a-position", + "predicate": "$.extra.preferred_languages[2] == $v", + "vars": { + "v": "secret-nz", + "flag": true + }, + "reason": "json_path_position", + "construct": "a position (`[2]`)", + "at": 27, + "at_text": "[2] == $v" + }, + { + "id": "a-filter-with-no-wildcard", + "predicate": "exists($.address ? (@.postcode == $v))", + "vars": { + "v": "secret-nz", + "flag": true + }, + "reason": "json_path_no_wildcard", + "construct": "a filter on a path with no wildcard", + "at": 7, + "at_text": "$.address ? (@.postcode == $v))" + }, + { + "id": "a-wildcard-in-a-member", + "predicate": "exists($.extra.emergency_contacts[*] ? (@.calls[*] == $v))", + "vars": { + "v": "secret-nz", + "flag": true + }, + "reason": "json_path_wildcard", + "construct": "a wildcard in a filter's member", + "at": 40, + "at_text": "@.calls[*] == $v))" + }, + { + "id": "a-number-json-cannot-hold", + "predicate": "$.weight == 1e999", + "vars": { + "v": "secret-nz", + "flag": true + }, + "reason": "json_syntax", + "construct": "a value", + "at": 12, + "at_text": "1e999" + }, + { + "id": "too-deep", + "predicate": "((((((((((((((((((((((((((((((((($.a == 1)))))))))))))))))))))))))))))))))", + "reason": "json_predicate_too_deep", + "construct": "nesting", + "at": 33, + "at_text": "$.a == 1)))))))))))))))))))))))))))))))))" + } + ] +} diff --git a/conformance/stack-encrypt/v1/eql-json-v1.enrollment.json b/conformance/stack-encrypt/v1/eql-json-v1.enrollment.json new file mode 100644 index 000000000..abf02b0a4 --- /dev/null +++ b/conformance/stack-encrypt/v1/eql-json-v1.enrollment.json @@ -0,0 +1,267 @@ +{ + "version": 1, + "profile": "eql-json-v1", + "suites": { + "engine-queries": { + "language": "rust", + "lane": "fast", + "workspace": ".", + "package": "stack-encrypt", + "binary": "json_operators", + "features": ["--all-features"] + }, + "engine-reads": { + "language": "rust", + "lane": "fast", + "workspace": ".", + "package": "stack-encrypt", + "binary": "json_read_checks", + "features": ["--all-features"] + }, + "engine-vectors": { + "language": "rust", + "lane": "fast", + "workspace": ".", + "package": "stack-encrypt", + "binary": "json_tree_vectors", + "features": ["--all-features"] + }, + "eql-targets": { + "language": "rust", + "lane": "fast", + "workspace": "packages/eql", + "package": "eql-encryption-tests", + "binary": "targets", + "features": [] + }, + "eql-record": { + "language": "rust", + "lane": "fast", + "workspace": "packages/eql", + "package": "eql-encryption-tests", + "binary": "json_tree_data_plan", + "features": [] + }, + "eql-typed": { + "language": "rust", + "lane": "fast", + "workspace": "packages/eql", + "package": "eql-encryption-tests", + "binary": "json_tree", + "features": [] + }, + "postgres": { + "language": "rust", + "lane": "postgres", + "workspace": "packages/eql", + "package": "eql-encryption-tests", + "binary": "json_tree", + "features": [] + }, + "go": { + "language": "go", + "lane": "fast" + }, + "go-postgres": { + "language": "go", + "lane": "postgres" + } + }, + "requirements": [ + { + "id": "eql-resolver-catalog-v1", + "evidence": { + "eql-targets": [ + "the_table::names_every_stored_catalog_domain_in_order", + "the_table::every_row_is_a_compiled_domain_type_and_so_is_its_query", + "the_table::the_producible_types_are_the_ones_with_a_rust_plaintext", + "given_an_unknown_name::every_entry_point_says_no_such_type", + "given_a_name_the_engine_cannot_produce::every_entry_point_refuses_it_with_the_table_reason" + ], + "eql-record": [ + "the_engine_and_the_table_agree_on_which_targets_take_json_options" + ], + "go": [ + "TestTheCatalogListsEachTypesOperators", + "TestTextEqIsRefusedByTheBuildWithoutEQLTypes", + "TestTargetWithoutEQLIsRefusedBeforeTheGuest" + ] + } + }, + { + "id": "json-write-v1", + "evidence": { + "eql-record": [ + "a_batch_of_documents_round_trips_in_one_request", + "the_data_plan_and_the_typed_layout_agree", + "text_that_is_not_a_document_is_refused_before_any_key", + "a_sql_null_stores_no_document", + "a_document_does_not_open_under_another_column" + ], + "eql-typed": ["a_document_round_trips_through_the_typed_target"], + "engine-vectors": [ + "the_derivations_are_pinned_under_the_deterministic_source" + ], + "go": [ + "TestJSONDocumentsRoundTripThroughTheGuest", + "TestJSONDocumentsDoNotOpenUnderAnotherColumn", + "TestJSONTextThatIsNotADocumentIsErrEncoding", + "TestACustomersDocumentsRoundTrip", + "TestASubscribersTypedDocumentsRoundTrip" + ], + "go-postgres": ["TestJSONTypedDocumentsInPostgres"] + } + }, + { + "id": "json-query-v1", + "evidence": { + "eql-record": [ + "a_document_field_answers_the_json_operators", + "every_operand_is_the_typed_chains", + "a_json_operator_refuses_what_it_cannot_answer", + "a_document_has_no_one_plaintext_query" + ], + "engine-queries": [ + "every_operator_derives_its_tagged_operand", + "the_orderings_follow_the_resolved_options", + "malformed_inputs_are_refused_with_their_reason" + ], + "eql-typed": [ + "the_query_operands_are_the_engines_terms_in_the_sql_contract" + ], + "go": [ + "TestJSONOperatorsDeriveTaggedTerms", + "TestACustomersFieldsDeriveEachQuery", + "TestASubscribersTypedNeedlesDeriveTheRawQuery" + ], + "postgres": [ + "postgres_containment_over_the_encrypted_column_is_jsonb_containment", + "generated::given_generated_documents_in_postgres::contains_selects_the_rows_jsonb_contains_on_both_plans", + "generated::given_generated_documents_in_postgres::contained_by_selects_the_rows_jsonb_does_except_a_root_scalar_in_a_root_array" + ], + "go-postgres": ["TestJSONDocumentsQueryAndReadInPostgres"] + } + }, + { + "id": "json-request-bound-reads-v1", + "evidence": { + "eql-record": ["reads_open_against_the_request_that_prepared_them"], + "engine-queries": [ + "reads_open_against_the_request_that_prepared_them", + "rows_that_are_not_the_reads_are_refused", + "a_malformed_read_is_refused_before_any_key" + ], + "engine-reads": [ + "given_the_entry_arrow_returned::another_paths_entry_is_refused_before_any_key_is_retrieved", + "given_the_rows_of_a_read::the_honest_rows_open_to_the_nodes_value_in_any_order", + "given_the_rows_of_a_read::another_position_of_the_array_is_refused_by_its_label", + "given_the_rows_of_a_read::a_truncated_subtree_is_refused_by_the_cascade", + "given_the_rows_of_a_read::rows_of_two_documents_are_refused", + "given_answers_the_check_cannot_tell_apart::rows_withheld_whole_read_as_no_node", + "given_answers_the_check_cannot_tell_apart::another_rows_node_at_the_same_path_opens" + ], + "go": [ + "TestJSONSelectorReadsOpenAgainstTheirRequest", + "TestJSONReadTypesPrintAndLogRedacted" + ], + "postgres": [ + "postgres_every_subtree_read_answers_as_jsonb_does", + "postgres_wildcard_reads_answer_as_jsonb_path_query", + "postgres_a_position_reaches_array_elements_only", + "postgres_hostile_answers_are_refused_by_the_client", + "postgres_reads_never_answer_wrongly_under_tampering", + "generated::given_generated_documents_in_postgres::arrow_opens_a_leaf_to_the_value_jsonb_holds" + ] + } + }, + { + "id": "json-element-match-v1", + "evidence": { + "engine-queries": [ + "element_match::given_documents::one_element_must_satisfy_every_member", + "element_match::given_documents::answers_as_the_jsonpath_over_the_plaintext", + "element_match::given_the_fields_type::does_not_answer_it_when_the_type_names_no_domain", + "element_match::given_the_fields_type::does_not_list_it_without_an_ordered_path" + ], + "go": [ + "TestElementMatchIsTheReadsAndTheOrderingsOfEachMember", + "TestElementMatchRefusesBeforeTheEngine", + "TestElementMatchEngineRefusalsAreTypedErrors", + "TestAtPathMakesThePredicatesOfItsMember" + ], + "postgres": [ + "elements::postgres_an_element_predicate_holds_of_one_element", + "elements::postgres_one_element_must_satisfy_every_member", + "elements::postgres_a_probe_hit_raises", + "elements::postgres_the_elements_operand_domain_checks_each_member" + ], + "go-postgres": ["TestJSONElementMatchInPostgres"] + } + }, + { + "id": "json-predicate-match-v1", + "evidence": { + "engine-queries": [ + "compiled_match::given_an_input_it_cannot_answer::shared_predicate_refusals", + "compiled_match::given_documents::unknown_null_and_absent_answer_as_jsonpath_does", + "compiled_match::given_the_fields_type::does_not_answer_it_when_the_type_names_no_domain", + "compiled_match::given_a_predicate::each_leaf_is_its_operators_operand" + ], + "go": [ + "TestMatchCompilesToTheOperandsOfTheOtherOperators", + "TestMatchTakesItsVarsAsAnyJSONObject", + "TestMatchRefusalsNameTheConstructAndItsOffset", + "TestMatchOperandPrintsAndLogsRedacted" + ], + "postgres": [ + "predicate::postgres_a_compiled_predicate_answers_as_jsonb_path_match", + "predicate::postgres_a_compiled_predicate_answers_as_the_operators_written_out", + "predicate::postgres_a_null_leaf_is_unknown_under_and_or_and_not", + "predicate::postgres_the_predicate_domain_checks_each_node", + "predicate::postgres_the_operator_takes_a_typed_predicate_and_jsonpath_stays_blocked" + ], + "go-postgres": ["TestJSONPredicateMatchInPostgres"] + } + }, + { + "id": "json-extraction-v1", + "evidence": { + "engine-reads": [ + "given_the_entry_arrow_returned::the_entry_at_the_path_opens_to_its_scalar" + ], + "go": ["TestJSONSelectorReadsOpenAgainstTheirRequest"], + "postgres": [ + "generated::given_generated_documents_in_postgres::arrow_opens_a_leaf_to_the_value_jsonb_holds" + ] + } + }, + { + "id": "json-subtree-v1", + "evidence": { + "eql-record": ["reads_open_against_the_request_that_prepared_them"], + "engine-reads": [ + "given_the_rows_of_a_read::the_honest_rows_open_to_the_nodes_value_in_any_order" + ], + "postgres": [ + "postgres_every_subtree_read_answers_as_jsonb_does", + "postgres_wildcard_reads_answer_as_jsonb_path_query" + ], + "go-postgres": ["TestJSONDocumentsQueryAndReadInPostgres"] + } + }, + { + "id": "json-tamper-v1", + "evidence": { + "engine-reads": [ + "given_the_rows_of_a_read::a_truncated_subtree_is_refused_by_the_cascade", + "given_the_rows_of_a_read::rows_of_two_documents_are_refused" + ], + "postgres": [ + "postgres_hostile_answers_are_refused_by_the_client", + "postgres_reads_never_answer_wrongly_under_tampering" + ], + "go": ["TestJSONDocumentsDoNotOpenUnderAnotherColumn"] + } + } + ] +} diff --git a/docs/plans/2026-10-31-stack-encrypt-conformance.md b/docs/plans/2026-10-31-stack-encrypt-conformance.md index 1ac4ad892..d5428bb22 100644 --- a/docs/plans/2026-10-31-stack-encrypt-conformance.md +++ b/docs/plans/2026-10-31-stack-encrypt-conformance.md @@ -224,3 +224,19 @@ binding/search-scope split. Next: the EQL/JSON operation profile. Python and TypeScript runner delivery and the wider fresh artifact exchange remain separate planned work. + +## EQL/JSON operation checkpoint + +Implemented the nine requirements in +`conformance/stack-encrypt/eql-json-contract.md`, enrolling the existing engine, +EQL binding, Go SDK and independent PostgreSQL suites. A shared corpus replaces +duplicate write/refusal tables; existing typed, generated, property and +adversarial assertions remain. Required fast and PostgreSQL commands check +compiled native inventory and exact Go pass events, refuse skipped evidence, +and report engine, lockfile, guest and installed SQL identities in CI artifacts. + +Next: backend lifecycle, cancellation, no-secret diagnostics, provider key-I/O +and batching requirements. Python/TypeScript enrollment and the broader fresh +artifact/version exchange remain separate delivery slices. The EQL/JSON profile +does not complete the refactoring track or introduce the future binding/search +scope split. diff --git a/languages/golang/README.md b/languages/golang/README.md index b5ffefe9a..6832d64ee 100644 --- a/languages/golang/README.md +++ b/languages/golang/README.md @@ -47,6 +47,9 @@ mise run go:test The `.wasm` files are build outputs and are not committed on development branches. `mise run go:test:postgres` also runs the PostgreSQL suite against -the EQL v4 bundle built from this tree; it needs the database named by -`EQL_TEST_DATABASE_URL`. See `mise.toml` and [AGENTS.md](../../AGENTS.md) -for the build and release details. +the EQL v4 bundle built from this tree, and the shared EQL/JSON conformance +lane before it. It needs the disposable database named by +`EQL_TEST_DATABASE_URL`, `psql`, and the Rust test tools +(`mise install --env test`), because the conformance lane runs Rust tests too. +See `mise.toml` and [AGENTS.md](../../AGENTS.md) for the build and release +details. diff --git a/languages/golang/encrypt/eql_test.go b/languages/golang/encrypt/eql_test.go index dc11ea565..10af03190 100644 --- a/languages/golang/encrypt/eql_test.go +++ b/languages/golang/encrypt/eql_test.go @@ -25,9 +25,7 @@ import ( func deterministicEQLClient(t *testing.T) *encrypt.Client { t.Helper() c, err := encrypt.NewDeterministicEQLClient(context.Background(), testSeed) - if errors.Is(err, encrypt.ErrDeterministicEQLGuestNotBuilt) || errors.Is(err, encrypt.ErrGuestNotBuilt) { - t.Skip(err) - } + skipMissingConformanceGuest(t, err) if err != nil { t.Fatal(err) } diff --git a/languages/golang/encrypt/fixture_conformance_test.go b/languages/golang/encrypt/fixture_conformance_test.go index eec80b529..0771bc8bb 100644 --- a/languages/golang/encrypt/fixture_conformance_test.go +++ b/languages/golang/encrypt/fixture_conformance_test.go @@ -18,7 +18,7 @@ import ( // remain failures in the caller, including errors wrapping neither sentinel. func skipMissingConformanceGuest(t *testing.T, err error) { t.Helper() - if (errors.Is(err, encrypt.ErrGuestNotBuilt) || errors.Is(err, encrypt.ErrDeterministicGuestNotBuilt)) && os.Getenv("STACK_CONFORMANCE_REQUIRED") != "1" { + if (errors.Is(err, encrypt.ErrGuestNotBuilt) || errors.Is(err, encrypt.ErrDeterministicGuestNotBuilt) || errors.Is(err, encrypt.ErrDeterministicEQLGuestNotBuilt)) && os.Getenv("STACK_CONFORMANCE_REQUIRED") != "1" { t.Skip(err) } } @@ -74,6 +74,8 @@ func TestFixtureConformanceRequiresGuest(t *testing.T) { } // These full-suite tests also share queryFixtureClient's required-mode policy; // they remain outside the shared fixture requirements and aggregate selection. + names["TestJSONDocumentsRoundTripThroughTheGuest"] = true + names["TestMatchRefusalsNameTheConstructAndItsOffset"] = true names["TestContextConformance"] = true names["TestContextBoundaries"] = true names["TestAQueryTheFieldDoesNotAnswerIsRefused"] = true diff --git a/languages/golang/encrypt/json_conformance_test.go b/languages/golang/encrypt/json_conformance_test.go new file mode 100644 index 000000000..f21720f03 --- /dev/null +++ b/languages/golang/encrypt/json_conformance_test.go @@ -0,0 +1,43 @@ +package encrypt_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +type jsonConformanceCorpus struct { + Version int `json:"version"` + Profile string `json:"profile"` + Documents []string `json:"documents"` + InvalidDocuments []struct { + Text string `json:"text"` + Reason string `json:"reason"` + } `json:"invalid_documents"` + PredicateRefusals []struct { + ID string `json:"id"` + Predicate string `json:"predicate"` + Vars json.RawMessage `json:"vars"` + Reason string `json:"reason"` + Construct string `json:"construct"` + At int `json:"at"` + AtText string `json:"at_text"` + } `json:"predicate_refusals"` +} + +func jsonConformanceCases(t *testing.T) jsonConformanceCorpus { + t.Helper() + data, err := os.ReadFile(filepath.Join("..", "..", "..", "conformance", "stack-encrypt", "v1", "eql-json-v1.cases.json")) + if err != nil { + t.Fatal(err) + } + var cases jsonConformanceCorpus + if err := json.Unmarshal(data, &cases); err != nil { + t.Fatal(err) + } + if cases.Version != 1 || cases.Profile != "eql-json-v1" || len(cases.Documents) == 0 || len(cases.InvalidDocuments) == 0 || len(cases.PredicateRefusals) == 0 { + t.Fatal("empty or unsupported JSON conformance corpus") + } + return cases +} diff --git a/languages/golang/encrypt/json_match_test.go b/languages/golang/encrypt/json_match_test.go index 6edc85ba3..8ae6d3bb8 100644 --- a/languages/golang/encrypt/json_match_test.go +++ b/languages/golang/encrypt/json_match_test.go @@ -274,46 +274,28 @@ func TestMatchRefusalsNameTheConstructAndItsOffset(t *testing.T) { ctx := context.Background() cipher := c.DefaultKeyset() f := testusers.EntityFields.CustomFieldValues - vars := map[string]any{"v": "secret-nz", "flag": true} - deep := strings.Repeat("(", 33) + "$.a == 1" + strings.Repeat(")", 33) - for _, tc := range []struct { - name, predicate string - vars any - reason string - construct string - at string // the text the offset points at - }{ - {"text the grammar does not read", `$.extra.country = $v`, vars, "json_predicate_syntax", "text the grammar does not read", "= $v"}, - {"like_regex", `$.extra.country like_regex "secret"`, vars, "json_predicate_unsupported", "`like_regex`", "like_regex"}, - {"arithmetic", `$.weight + 1 == 2`, vars, "json_predicate_unsupported", "arithmetic (`+`, `-`, `*`, `/`, `%`)", "+ 1"}, - {"a method", `$.weight.size() == 1`, vars, "json_predicate_unsupported", "a method (`.name()`)", ".size()"}, - {"a filter outside exists", `$.extra.preferred_languages[*] ? (@ == $v) == 1`, vars, "json_predicate_unsupported", "a filter outside `exists`", "? (@"}, - {"|| in a filter", `exists($.extra.emergency_contacts[*] ? (@.relationship == $v || @.age > 1))`, vars, "json_predicate_unsupported", "`||` in a filter", "|| @.age"}, - {"an unknown variable", `$.extra.country == $missing`, vars, "json_predicate_unknown_variable", "a variable", "$missing"}, - {"a value an ordering cannot take", `$.weight < $flag`, vars, "json_predicate_value_type", "a value", "$flag"}, - {"an object", `$.extra.country == $obj`, map[string]any{"obj": map[string]any{"secret-nz": 1}}, "json_predicate_value_type", "a value", "$obj"}, - {"!= at a wildcard", `$.extra.preferred_languages[*] != $v`, vars, "json_predicate_not_equal_at_wildcard", "`!=` at a path with a wildcard", "!= $v"}, - {"too deep", deep, nil, "json_predicate_too_deep", "nesting", "$.a"}, - {"a position", `$.extra.preferred_languages[2] == $v`, vars, "json_path_position", "a position (`[2]`)", "[2]"}, - {"a filter with no wildcard", `exists($.address ? (@.postcode == $v))`, vars, "json_path_no_wildcard", "a filter on a path with no wildcard", "$.address ?"}, - {"a wildcard in a member", `exists($.extra.emergency_contacts[*] ? (@.calls[*] == $v))`, vars, "json_path_wildcard", "a wildcard in a filter's member", "@.calls"}, - {"a number JSON cannot hold", `$.weight == 1e999`, vars, "json_syntax", "a value", "1e999"}, - } { - _, err := f.Match(ctx, cipher, tc.predicate, tc.vars, encrypt.JSONReadOptions{}) + for _, tc := range jsonConformanceCases(t).PredicateRefusals { + var vars any + if tc.Vars != nil { + vars = tc.Vars + } + if strings.Index(tc.Predicate, tc.AtText) != tc.At { + t.Fatalf("%s: offset does not name at_text", tc.ID) + } + _, err := f.Match(ctx, cipher, tc.Predicate, vars, encrypt.JSONReadOptions{}) var d *encrypt.Diagnostic if !errors.Is(err, encrypt.ErrEncoding) || !errors.As(err, &d) { - t.Errorf("%s: %v, want ErrEncoding with a Diagnostic", tc.name, err) + t.Errorf("%s: %v, want ErrEncoding with a Diagnostic", tc.ID, err) continue } - want := strings.Index(tc.predicate, tc.at) - if at, ok := d.At(); d.Reason() != tc.reason || d.Construct() != tc.construct || !ok || at != want { - t.Errorf("%s: reason %q, construct %q, at %d (%v); want %q, %q, %d: %v", tc.name, d.Reason(), d.Construct(), at, ok, tc.reason, tc.construct, want, err) + if at, ok := d.At(); d.Reason() != tc.Reason || d.Construct() != tc.Construct || !ok || at != tc.At { + t.Errorf("%s: reason %q, construct %q, at %d (%v); want %q, %q, %d: %v", tc.ID, d.Reason(), d.Construct(), at, ok, tc.Reason, tc.Construct, tc.At, err) } if d.Field() != "custom_field_values" || d.Code != "stack_encrypt::dynamic_predicate" { - t.Errorf("%s: field %q, code %q", tc.name, d.Field(), d.Code) + t.Errorf("%s: field %q, code %q", tc.ID, d.Field(), d.Code) } - if strings.Contains(err.Error(), "secret") || strings.Contains(err.Error(), tc.predicate) { - t.Errorf("%s: the error holds the caller's text: %v", tc.name, err) + if strings.Contains(err.Error(), "secret") || strings.Contains(err.Error(), tc.Predicate) { + t.Errorf("%s: the error holds the caller's text: %v", tc.ID, err) } } @@ -363,7 +345,7 @@ func TestMatchRefusalsNameTheConstructAndItsOffset(t *testing.T) { `$.score < 3`, `exists($.extra.emergency_contacts[*] ? (@.name == $v && @.age > 1))`, } { - _, err := f.Match(ctx, cipher, predicate, vars, encrypt.JSONReadOptions{}) + _, err := f.Match(ctx, cipher, predicate, map[string]any{"v": "secret-nz", "flag": true}, encrypt.JSONReadOptions{}) var d *encrypt.Diagnostic if !errors.Is(err, encrypt.ErrTerm) || !errors.As(err, &d) || d.Code != "stack_encrypt::json_path_not_ordered" { t.Errorf("%s: %v, want ErrTerm json_path_not_ordered", predicate, err) diff --git a/languages/golang/encrypt/json_tree_test.go b/languages/golang/encrypt/json_tree_test.go index 37023f35f..9fd49a503 100644 --- a/languages/golang/encrypt/json_tree_test.go +++ b/languages/golang/encrypt/json_tree_test.go @@ -26,18 +26,10 @@ func jsonPlan() *record.Plan { }}} } -var jsonDocuments = []string{ - `{"name":"Alice","age":34,"visits":[{"at":3},{"at":1}]}`, - `[1,"two",{"three":3}]`, - `"a scalar root"`, - `null`, - `18446744073709551615`, - `{}`, -} - func TestJSONDocumentsRoundTripThroughTheGuest(t *testing.T) { c := deterministicEQLClient(t) ctx := context.Background() + jsonDocuments := jsonConformanceCases(t).Documents rows := make([]record.Source, len(jsonDocuments)) for i, doc := range jsonDocuments { rows[i] = record.Source{"profile": []byte(doc)} @@ -93,7 +85,7 @@ func TestJSONDocumentsRoundTripThroughTheGuest(t *testing.T) { func TestJSONDocumentsDoNotOpenUnderAnotherColumn(t *testing.T) { c := deterministicEQLClient(t) ctx := context.Background() - sealed, err := c.DefaultKeyset().Seal(ctx, jsonPlan(), []record.Source{{"profile": []byte(jsonDocuments[0])}}) + sealed, err := c.DefaultKeyset().Seal(ctx, jsonPlan(), []record.Source{{"profile": []byte(jsonConformanceCases(t).Documents[0])}}) if err != nil { t.Fatal(err) } @@ -108,11 +100,8 @@ func TestJSONDocumentsDoNotOpenUnderAnotherColumn(t *testing.T) { // the engine's code and without the text. func TestJSONTextThatIsNotADocumentIsErrEncoding(t *testing.T) { c := deterministicEQLClient(t) - for text, code := range map[string]string{ - `{"secret":1,"secret":2}`: "json_duplicate_key", - `99999999999999999999`: "json_integer_range", - `{"secret":`: "json_syntax", - } { + for _, tc := range jsonConformanceCases(t).InvalidDocuments { + text, code := tc.Text, tc.Reason _, err := c.DefaultKeyset().Seal(context.Background(), jsonPlan(), []record.Source{{"profile": []byte(text)}}) if !errors.Is(err, encrypt.ErrEncoding) { t.Fatalf("%s: %v, want ErrEncoding", text, err) diff --git a/mise.test.toml b/mise.test.toml index 57e4acb9a..3a25a7589 100644 --- a/mise.test.toml +++ b/mise.test.toml @@ -11,6 +11,10 @@ # (cargo@0.96 needs rustc 1.93; the Ubuntu runners ship 1.92). Bump a pin # deliberately, after checking it still builds with the runners' rustc. [tools] +# EQL setup can leave a cargo-binstall shim in the shared mise cache. Select +# its prebuilt installer here too: an unselected shim is still found on PATH +# by the cargo backend, then fails before it can install any test tools. +"cargo-binstall" = "1.26.2" "cargo:cargo-nextest" = "0.9.146" # Coverage → CRAP (Change Risk Anti-Patterns) metric tooling. `cargo llvm-cov` # produces an LCOV file that `cargo crap` scores; see `mise run crap:stack-auth`. diff --git a/mise.toml b/mise.toml index 9e174844d..07a2603aa 100644 --- a/mise.toml +++ b/mise.toml @@ -382,8 +382,8 @@ run = """ #!/usr/bin/env bash set -euo pipefail : "${EQL_TEST_DATABASE_URL:?set EQL_TEST_DATABASE_URL to a libpq connection string for a disposable PostgreSQL}" -mise -C packages/eql run build -psql "$EQL_TEST_DATABASE_URL" -v ON_ERROR_STOP=1 -q -f packages/eql/release/cipherstash-encrypt-v4.sql +# The conformance task owns SQL setup and the JSON PostgreSQL tests. +mise run test:conformance:eql:postgres cd languages/golang # The PostgreSQL tests, named once: the -run pattern and the check below # are both derived from this list. @@ -391,10 +391,6 @@ TESTS=( TestEQLTypesRoundTripAndQueryInPostgres TestEQLOrderMatchesPlaintextInPostgres TestQueriesRunAgainstPostgres - TestJSONDocumentsQueryAndReadInPostgres - TestJSONTypedDocumentsInPostgres - TestJSONElementMatchInPostgres - TestJSONPredicateMatchInPostgres ) pattern="^($(IFS='|'; echo "${TESTS[*]}"))$" # The tests skip without DATABASE_URL and without their guest builds; -v and @@ -433,3 +429,18 @@ cd languages/golang # if set, else the developer profile. See encrypt/example/README.md. CGO_ENABLED=0 go run ./encrypt/example """ + +[tasks."test:conformance:eql"] +description = "Required EQL/JSON profiles over native Rust and Go (build both deterministic guests first)" +run = "node scripts/stack-eql-conformance.mjs fast" + +[tasks."test:conformance:eql:postgres"] +description = "Build and install EQL v4 into disposable EQL_TEST_DATABASE_URL, then run required Rust/Go JSON PostgreSQL oracles" +run = """ +#!/usr/bin/env bash +set -euo pipefail +: "${EQL_TEST_DATABASE_URL:?set a disposable PostgreSQL database connection}" +mise -C packages/eql run clean +mise -C packages/eql run build +node scripts/stack-eql-conformance.mjs postgres +""" diff --git a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs index 77fcb1265..a52ab565a 100644 --- a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs +++ b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs @@ -131,18 +131,12 @@ fn frame(value: T) -> ValueFrame { /// The documents a batch carries, as values. fn documents() -> Vec { - vec![ - json!({"name": "Alice", "age": 34, "visits": [{"at": 3, "where": "x"}, {"at": 1}], "tags": ["a", "b"]}), - json!({"name": "Bob", "age": 7.5, "nested": {"deep": [[1, 2], {}]}, "none": null}), - json!([1, "two", {"three": 3}]), - json!("a scalar root"), - json!(18446744073709551615u64), - json!(-9223372036854775808i64), - json!(-926_809.505_100_808_3), - Value::Null, - json!({}), - json!([]), - ] + conformance_cases()["documents"] + .as_array() + .unwrap() + .iter() + .map(|text| serde_json::from_str(text.as_str().unwrap()).unwrap()) + .collect() } /// The EQL value a sealed row stores for `profile`, parsed. @@ -443,12 +437,9 @@ async fn text_that_is_not_a_document_is_refused_before_any_key() { let (cipher, calls) = common::cipher().await; let keyset = cipher.default_keyset(); let plan = plan(); - for (text, code) in [ - (r#"{"secret": 1, "secret": 2}"#, "json_duplicate_key"), - ("99999999999999999999", "json_integer_range"), - (r#"{"secret": "#, "json_syntax"), - ("", "json_syntax"), - ] { + for case in conformance_cases()["invalid_documents"].as_array().unwrap() { + let text = case["text"].as_str().unwrap(); + let code = case["reason"].as_str().unwrap(); let source = BTreeMap::from([("profile", text)]); let error = match record::encrypt_with(&keyset, frame(&source), &plan, &Eql) { Err(error) => error, @@ -1154,3 +1145,13 @@ async fn the_engine_and_the_table_agree_on_which_targets_take_json_options() { assert_eq!(engine, !refused, "{}", target.name); } } + +fn conformance_cases() -> Value { + let cases: Value = serde_json::from_str(include_str!( + "../../../../../conformance/stack-encrypt/v1/eql-json-v1.cases.json" + )) + .unwrap(); + assert_eq!(cases["version"], 1); + assert_eq!(cases["profile"], "eql-json-v1"); + cases +} diff --git a/packages/stack-encrypt/tests/json_operators.rs b/packages/stack-encrypt/tests/json_operators.rs index 63e913121..52cbc540d 100644 --- a/packages/stack-encrypt/tests/json_operators.rs +++ b/packages/stack-encrypt/tests/json_operators.rs @@ -2183,6 +2183,45 @@ mod compiled_match { ) } + /// Shared native/Go predicate refusals preserve structured diagnostics. + #[test] + fn shared_predicate_refusals() { + let cases: Value = serde_json::from_str(include_str!( + "../../../conformance/stack-encrypt/v1/eql-json-v1.cases.json" + )) + .unwrap(); + assert_eq!(cases["version"], 1); + assert_eq!(cases["profile"], "eql-json-v1"); + for case in cases["predicate_refusals"].as_array().unwrap() { + let predicate = case["predicate"].as_str().unwrap(); + assert_eq!( + predicate.find(case["at_text"].as_str().unwrap()), + Some(case["at"].as_u64().unwrap() as usize) + ); + let request = match case.get("vars") { + Some(vars) => input(predicate, vars.clone()), + None => json!({"predicate": predicate}), + }; + let error = check(&request).unwrap_err(); + let payload = stack_encrypt::ErrorPayload::payload(&error); + assert_eq!( + error.reason().map(Reason::as_str), + case["reason"].as_str(), + "{}", + case["id"] + ); + assert_eq!(payload["at"], case["at"], "{}", case["id"]); + assert_eq!(payload["construct"], case["construct"], "{}", case["id"]); + assert_eq!(error.field(), Some("profile")); + let rendered = format!("{error} {payload:?}"); + assert!( + !rendered.contains("secret") && !rendered.contains(predicate), + "{}: the error holds the caller's text: {rendered}", + case["id"] + ); + } + } + /// A refusal names its reason, the construct and its offset in the /// caller's text, and the field; the input's own shape is refused /// as the other operators refuse theirs. diff --git a/packages/stack-encrypt/tests/json_tree_vectors.rs b/packages/stack-encrypt/tests/json_tree_vectors.rs index 2a3c5bfb4..467a973ec 100644 --- a/packages/stack-encrypt/tests/json_tree_vectors.rs +++ b/packages/stack-encrypt/tests/json_tree_vectors.rs @@ -171,6 +171,11 @@ async fn the_derivations_are_pinned_under_the_deterministic_source() { "sealed_root": vectors(&JsonOptions::default()).await, "open_root": vectors(&JsonOptions { sealed_root: false, ..JsonOptions::new() }).await, }); + assert!( + std::env::var("STACK_CONFORMANCE_REQUIRED").as_deref() != Ok("1") + || std::env::var_os("STACK_ENCRYPT_UPDATE_JSON_TREE_VECTORS").is_none(), + "required conformance must not regenerate JSON tree vectors" + ); if update_requested(std::env::var_os("STACK_ENCRYPT_UPDATE_JSON_TREE_VECTORS").as_deref()) { std::fs::write( fixture_path(), @@ -213,3 +218,22 @@ async fn the_deterministic_source_seals_the_same_tags_twice() { let two = vectors(&JsonOptions::default()).await; assert_eq!(one, two, "a pin needs a run that repeats"); } + +#[test] +fn required_checks_refuse_json_vector_updates() { + for value in ["", "0", "1"] { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "the_derivations_are_pinned_under_the_deterministic_source", + "--nocapture", + ]) + .env("STACK_CONFORMANCE_REQUIRED", "1") + .env("STACK_ENCRYPT_UPDATE_JSON_TREE_VECTORS", value) + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr) + .contains("required conformance must not regenerate JSON tree vectors")); + } +} diff --git a/scripts/__tests__/crates-ci.test.mjs b/scripts/__tests__/crates-ci.test.mjs index 4d33fe72d..72c504612 100644 --- a/scripts/__tests__/crates-ci.test.mjs +++ b/scripts/__tests__/crates-ci.test.mjs @@ -34,6 +34,33 @@ const readRepo = (relPath) => readFileSync(join(REPO_ROOT, relPath), 'utf8') const ROOT_MISE = 'mise.toml' const IMPORTED_WORKFLOW_DIR = '.github/imported-workflows' +describe('the test toolchain installs from a shared mise cache', () => { + it('runs the prebuilt CRAP executable on a compatible image before coverage', () => { + const workflow = readWorkflow('.github/workflows/crap-crates.yml') + for (const name of ['crap-stack-auth', 'crap-stack-encrypt']) { + const job = workflow.jobs[name] + expect(job['runs-on']).toMatch(/ubuntu-2404$/) + const probe = job.steps.findIndex((s) => s.run === 'cargo crap --version') + const gate = job.steps.findIndex((s) => + s.run?.startsWith('mise run crap:'), + ) + expect(probe).toBeGreaterThan(-1) + expect(gate).toBeGreaterThan(probe) + } + }) + + it('selects the prebuilt cargo-binstall before installing cargo tools', () => { + // EQL installs cargo-binstall. A restored shim with no selected version + // makes mise's cargo backend fail instead of falling back to cargo install. + const tools = readRepo('mise.test.toml') + .split(/^\[tools\]\s*$/m)[1] + .split(/^\[/m)[0] + expect(tools).toMatch(/^"cargo:cargo-nextest"\s*=/m) + expect(tools).toMatch(/^"cargo-binstall"\s*=\s*"\d+\.\d+\.\d+"\s*$/m) + expect(tools).not.toMatch(/^"cargo:cargo-binstall"\s*=/m) + }) +}) + // --------------------------------------------------------------------------- // The task graph // --------------------------------------------------------------------------- diff --git a/scripts/__tests__/eql-suite-ci.test.mjs b/scripts/__tests__/eql-suite-ci.test.mjs index 1c9dc60fe..bcd8ceebe 100644 --- a/scripts/__tests__/eql-suite-ci.test.mjs +++ b/scripts/__tests__/eql-suite-ci.test.mjs @@ -4,6 +4,7 @@ import { dirname, join, relative, sep } from 'node:path' import yaml from 'js-yaml' import { describe, expect, it } from 'vitest' import { REPO_ROOT } from './lib/repo-root.mjs' +import { includes } from './lib/rust-compiled-files.mjs' import { readWorkflow, WORKFLOW_DIR, workflowFiles } from './lib/workflows.mjs' /** @@ -400,16 +401,29 @@ describe('the relevance filter still selects the imported tree', () => { }) it('prefixes every source path with the subtree root', () => { - // Three legitimate exceptions, all outside the subtree by nature: the + // Legitimate exceptions, all outside the subtree by nature: the // workflow's own path, a composite action it `uses:`, and a crate the // workspace compiles by path (plus the root manifest it inherits from). const external = externalPathDependencies.dirs.map((dir) => `${dir}/**`) + // The EQL data-plan suite compiles the shared corpus from outside its + // subtree. Keep this exception tied to that actual dependency. + const sharedCorpus = 'conformance/stack-encrypt/v1/eql-json-v1.cases.json' + const usesSharedCorpus = includes( + readFileSync( + join( + REPO_ROOT, + 'packages/eql/tests/encryption/tests/json_tree_data_plan.rs', + ), + 'utf8', + ), + ).some((entry) => entry.literal?.endsWith(`/${sharedCorpus}`)) const offenders = filterPaths.filter( (path) => path !== EQL_WORKFLOW && !path.startsWith(EQL_PREFIX) && !localActionPrefixes.some((prefix) => path.startsWith(prefix)) && !external.includes(path) && + !(usesSharedCorpus && path === 'conformance/stack-encrypt/**') && !(externalPathDependencies.inherits && path === 'Cargo.toml'), ) @@ -535,7 +549,11 @@ describe('the shared Rust cache is pointed and saved correctly', () => { // Held non-empty, so a change to `isEqlJob` cannot empty the check. expect(EQL_RUST_CACHE_STEPS.length).toBeGreaterThan(0) const wrong = EQL_RUST_CACHE_STEPS.filter( - ({ step }) => step?.with?.workspaces !== CARGO_WORKSPACE, + ({ step }) => + !String(step?.with?.workspaces ?? '') + .split('\n') + .map((line) => line.split('->')[0].trim()) + .includes(CARGO_WORKSPACE), ).map((entry) => stepLabel(entry, `workspaces: ${entry.step?.with?.workspaces}`), ) diff --git a/scripts/__tests__/stack-conformance-ci.test.mjs b/scripts/__tests__/stack-conformance-ci.test.mjs index 8c8c9c34e..bae94cf5a 100644 --- a/scripts/__tests__/stack-conformance-ci.test.mjs +++ b/scripts/__tests__/stack-conformance-ci.test.mjs @@ -233,12 +233,13 @@ describe('shared Stack Encrypt conformance is required in CI', () => { * conformance/stack-encrypt/README.md § Files: every file in v1/ is * ..json, with the profile in its header, so a file's name * says which profile it belongs to and whether it holds cases, one - * producer's frozen outputs or one runner's extra inputs. + * producer's frozen outputs, one runner's extra inputs or the index of + * existing tests enrolled as its evidence. */ describe('conformance files are named after their profile and role', () => { const dir = 'conformance/stack-encrypt/v1' const NAME = - /^(?[a-z0-9-]+-v\d+)\.(?cases|expected\.(?[a-z0-9-]+)|inputs\.(?[a-z0-9-]+))\.json$/ + /^(?[a-z0-9-]+-v\d+)\.(?cases|enrollment|expected\.(?[a-z0-9-]+)|inputs\.(?[a-z0-9-]+))\.json$/ const files = readdirSync(join(REPO_ROOT, dir)).filter( (file) => file !== 'fixture-enrollment.json', ) @@ -254,6 +255,10 @@ describe('conformance files are named after their profile and role', () => { expect(files, `${file} has no cases file`).toContain( `${profile}.cases.json`, ) + } + if (role === 'enrollment') { + expect(header.requirements?.length).toBeGreaterThan(0) + } else if (role !== 'cases') { expect(Object.keys(header.cases ?? {}).length).toBeGreaterThan(0) } if (producer) expect(header.producer).toMatch(new RegExp(`^${producer}`)) diff --git a/scripts/__tests__/stack-eql-conformance.test.mjs b/scripts/__tests__/stack-eql-conformance.test.mjs new file mode 100644 index 000000000..bd85042ae --- /dev/null +++ b/scripts/__tests__/stack-eql-conformance.test.mjs @@ -0,0 +1,219 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + checkEnvironment, + checkGoResults, + checkRustInventory, + selections, +} from '../stack-eql-conformance.mjs' +import { REPO_ROOT } from './lib/repo-root.mjs' +import { readWorkflow } from './lib/workflows.mjs' + +const read = (path) => readFileSync(join(REPO_ROOT, path), 'utf8') +const index = JSON.parse( + read('conformance/stack-encrypt/v1/eql-json-v1.enrollment.json'), +) + +describe('required EQL/JSON conformance', () => { + it('enrolls every operation family and both language/database lanes', () => { + expect(index.requirements.map((r) => r.id)).toEqual([ + 'eql-resolver-catalog-v1', + 'json-write-v1', + 'json-query-v1', + 'json-request-bound-reads-v1', + 'json-element-match-v1', + 'json-predicate-match-v1', + 'json-extraction-v1', + 'json-subtree-v1', + 'json-tamper-v1', + ]) + const contract = read('conformance/stack-encrypt/eql-json-contract.md') + for (const requirement of index.requirements) + expect(contract).toContain(requirement.id) + const reached = new Set() + for (const lane of ['fast', 'postgres']) { + for (const language of ['rust', 'go']) { + for (const [id, suite] of selections(index, lane, language)) { + reached.add(id) + expect(suite.tests.size).toBeGreaterThan(0) + } + } + } + expect([...reached].sort()).toEqual(Object.keys(index.suites).sort()) + const renamed = structuredClone(index) + renamed.requirements[0].evidence.missing = ['TestMissing'] + expect(() => selections(renamed, 'fast', 'all')).toThrow('Invalid suite') + expect(() => selections(index, 'unknown', 'all')).toThrow() + }) + + it('refuses renamed, filtered or wrongly ignored compiled native evidence', () => { + const suite = { + package: 'engine', + binary: 'json', + lane: 'fast', + tests: new Set(['required']), + } + const inventory = { + 'rust-suites': { + 'engine::json': { + status: 'listed', + testcases: { + required: { ignored: false, 'filter-match': { status: 'matches' } }, + }, + }, + }, + } + expect(() => checkRustInventory(inventory, suite)).not.toThrow() + for (const replacement of [ + undefined, + { ignored: true, 'filter-match': { status: 'matches' } }, + { ignored: false, 'filter-match': { status: 'mismatch' } }, + ]) { + const changed = structuredClone(inventory) + changed['rust-suites']['engine::json'].testcases.required = replacement + expect(() => checkRustInventory(changed, suite)).toThrow( + 'Missing or incorrectly ignored', + ) + } + expect(() => + checkRustInventory(inventory, { ...suite, lane: 'postgres' }), + ).toThrow() + inventory['rust-suites']['engine::json'].testcases.required.ignored = true + expect(() => + checkRustInventory(inventory, { ...suite, lane: 'postgres' }), + ).not.toThrow() + }) + + it('requires a Go pass for every name and rejects nested skips', () => { + const passed = JSON.stringify({ Action: 'pass', Test: 'TestRequired' }) + expect(() => checkGoResults(passed, ['TestRequired'])).not.toThrow() + expect(() => checkGoResults('', ['TestRequired'])).toThrow('did not pass') + for (const action of ['skip', 'fail']) { + const output = `${passed}\n${JSON.stringify({ Action: action, Test: 'TestRequired/case' })}` + expect(() => checkGoResults(output, ['TestRequired'])).toThrow(action) + } + expect(() => checkGoResults(passed, ['TestRenamed'])).toThrow( + 'did not pass', + ) + }) + + it('refuses every update-variable spelling, including an empty or zero value', () => { + for (const name of [ + 'STACK_CONFORMANCE_UPDATE', + 'STACK_ENCRYPT_UPDATE_FIXTURES', + 'STACK_ENCRYPT_UPDATE_JSON_TREE_VECTORS', + ]) { + for (const value of ['', '0', '1']) + expect(() => checkEnvironment({ [name]: value })).toThrow(name) + } + expect(() => checkEnvironment({})).not.toThrow() + }) + + it('shares the original write and predicate tables without dropping boundary cases', () => { + const corpus = JSON.parse( + read('conformance/stack-encrypt/v1/eql-json-v1.cases.json'), + ) + expect(corpus.version).toBe(1) + expect(new Set(corpus.documents).size).toBe(corpus.documents.length) + for (const text of [ + 'null', + '{}', + '[]', + '18446744073709551615', + '-9223372036854775808', + '-926809.5051008083', + ]) + expect(corpus.documents).toContain(text) + expect(corpus.invalid_documents.map((c) => c.reason)).toEqual([ + 'json_duplicate_key', + 'json_integer_range', + 'json_syntax', + 'json_syntax', + ]) + expect( + corpus.predicate_refusals.find((c) => c.id === 'too-deep'), + ).not.toHaveProperty('vars') + const ids = corpus.predicate_refusals.map((c) => c.id) + expect(new Set(ids).size).toBe(15) + for (const c of corpus.predicate_refusals) { + expect(c.at).toBeGreaterThanOrEqual(0) + expect(c.at).toBeLessThan(Buffer.byteLength(c.predicate)) + expect(c.reason).toMatch(/^json_/) + expect(c.construct).toBeTruthy() + expect(Buffer.from(c.predicate).subarray(c.at).toString()).toBe(c.at_text) + } + for (const path of [ + 'packages/stack-encrypt/tests/json_operators.rs', + 'packages/eql/tests/encryption/tests/json_tree_data_plan.rs', + 'languages/golang/encrypt/json_conformance_test.go', + ]) + expect(read(path)).toContain('eql-json-v1.cases.json') + }) + + it('runs both required lanes in CI and retains reports with SQL and engine identity', () => { + const workflow = readWorkflow('.github/workflows/tests-golang.yml') + const eqlSetup = workflow.jobs['wasi-check'].steps.find( + (step) => + step.uses?.startsWith('jdx/mise-action@') && + step.with?.working_directory === 'packages/eql', + ) + // Installing the EQL tools must not export its local DATABASE_URL into + // the hermetic Go job, which has no PostgreSQL service. + expect(eqlSetup).toBeDefined() + expect(eqlSetup.with.env).toBe(false) + for (const task of ['test:conformance:eql', 'go:test:postgres']) { + const jobs = Object.values(workflow.jobs).filter((job) => + job.steps?.some((step) => step.run === `mise run ${task}`), + ) + expect(jobs).toHaveLength(1) + const step = jobs[0].steps.find((step) => step.run === `mise run ${task}`) + expect(step['continue-on-error']).toBeUndefined() + expect(jobs[0].env.MISE_ENV).toBe('test') + expect(step.env.STACK_CONFORMANCE_REPORT_DIR).toBeTruthy() + if (task.endsWith(':postgres')) { + const steps = jobs[0].steps + const repair = steps.findIndex((s) => + s.run?.includes('mise install --force rust'), + ) + const cache = steps.findIndex((s) => + s.uses?.startsWith('Swatinem/rust-cache@'), + ) + expect(repair).toBeGreaterThan(-1) + expect(cache).toBeGreaterThan(repair) + expect(steps[repair].run).toContain( + 'mise -C packages/eql install --force rust', + ) + expect(steps[repair].run).toContain('mise x -- cargo --version') + expect(steps[repair].run).toContain( + 'mise -C packages/eql x -- cargo --version', + ) + expect(jobs[0].services.postgres).toBeDefined() + expect(step.env.EQL_TEST_DATABASE_URL).toContain( + 'job.services.postgres.ports', + ) + } + expect( + jobs[0].steps.some( + (s) => + s.uses?.startsWith('actions/upload-artifact@') && + s.with?.path?.includes('eql-conformance') && + s.with['if-no-files-found'] === 'error', + ), + ).toBe(true) + } + // Parse YAML paths rather than counting source spellings or indentation. + for (const event of ['push', 'pull_request']) { + expect(workflow.on[event].paths).toContain( + 'scripts/stack-eql-conformance.mjs', + ) + } + const tasks = read('mise.toml') + const postgresTask = tasks + .split('[tasks."go:test:postgres"]')[1] + .split('[tasks.')[0] + expect(postgresTask).toContain('mise run test:conformance:eql:postgres') + expect(postgresTask).not.toMatch(/^psql\s/m) + expect(postgresTask).not.toContain('mise -C packages/eql run build') + }) +}) diff --git a/scripts/stack-eql-conformance.mjs b/scripts/stack-eql-conformance.mjs new file mode 100644 index 000000000..10012e74a --- /dev/null +++ b/scripts/stack-eql-conformance.mjs @@ -0,0 +1,296 @@ +#!/usr/bin/env node +// Named evidence over the existing suites. Missing, ignored and skipped tests +// are failures; a successful empty selection is not conformance. +import { spawnSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..') +const read = (path) => readFileSync(join(root, path), 'utf8') +const hash = (path) => + createHash('sha256') + .update(readFileSync(join(root, path))) + .digest('hex') + +export function selections(index, lane, language) { + if ( + index.version !== 1 || + index.profile !== 'eql-json-v1' || + !['fast', 'postgres'].includes(lane) || + !['rust', 'go', 'all'].includes(language) + ) { + throw new Error( + 'Unsupported EQL conformance version, profile, lane or language', + ) + } + const selected = new Map() + const ids = new Set() + for (const requirement of index.requirements) { + if (!/^[a-z][a-z0-9-]+-v1$/.test(requirement.id) || ids.has(requirement.id)) + throw new Error('Invalid or repeated requirement ID') + ids.add(requirement.id) + if (!Object.keys(requirement.evidence).length) + throw new Error(`Empty evidence: ${requirement.id}`) + for (const [id, tests] of Object.entries(requirement.evidence)) { + const suite = index.suites[id] + if ( + !suite || + !['rust', 'go'].includes(suite.language) || + !['fast', 'postgres'].includes(suite.lane) + ) + throw new Error(`Invalid suite: ${id}`) + if ( + !tests.length || + new Set(tests).size !== tests.length || + tests.some( + (name) => + !(suite.language === 'go' ? /^Test\w+$/ : /^[a-zA-Z0-9_:]+$/).test( + name, + ), + ) + ) + throw new Error(`Invalid tests: ${id}`) + if ( + suite.lane !== lane || + (language !== 'all' && language !== suite.language) + ) + continue + if (!selected.has(id)) selected.set(id, { ...suite, tests: new Set() }) + for (const name of tests) selected.get(id).tests.add(name) + } + } + if (!selected.size) throw new Error('No EQL conformance evidence selected') + return selected +} + +export function checkRustInventory(inventory, suite) { + const binary = inventory['rust-suites'][`${suite.package}::${suite.binary}`] + if (binary?.status !== 'listed') + throw new Error(`Missing compiled binary: ${suite.binary}`) + for (const name of suite.tests) { + const test = binary.testcases[name] + if ( + !test || + test.ignored !== (suite.lane === 'postgres') || + test['filter-match'].status !== 'matches' + ) { + throw new Error( + `Missing or incorrectly ignored compiled test: ${suite.binary}::${name}`, + ) + } + } +} + +export function checkGoResults(output, tests) { + const passed = new Set() + for (const line of output.trim().split('\n')) { + if (!line) continue + const event = JSON.parse(line) + if (event.Action === 'skip' || event.Action === 'fail') + throw new Error( + `Go conformance ${event.Action}: ${event.Test ?? event.Package}`, + ) + if (event.Action === 'pass' && event.Test) passed.add(event.Test) + } + for (const name of tests) + if (!passed.has(name)) throw new Error(`Go test did not pass: ${name}`) +} + +export function checkEnvironment(env) { + for (const name of [ + 'STACK_CONFORMANCE_UPDATE', + 'STACK_ENCRYPT_UPDATE_FIXTURES', + 'STACK_ENCRYPT_UPDATE_JSON_TREE_VECTORS', + ]) { + if (Object.hasOwn(env, name)) + throw new Error(`Conformance cannot update fixtures: ${name}`) + } +} + +function command(program, args, { capture = false, ...options } = {}) { + const result = spawnSync(program, args, { + cwd: root, + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + stdio: capture ? ['ignore', 'pipe', 'inherit'] : 'inherit', + ...options, + }) + if (result.error) throw result.error + if (result.status !== 0) { + if (capture && result.stdout) process.stderr.write(result.stdout) + throw new Error(`${program} failed (${result.status})`) + } + return result.stdout +} + +function cargo(args, capture = false, workspace = '.') { + return command('mise', ['x', '--env', 'test', '--', 'cargo', ...args], { + capture, + cwd: join(root, workspace), + }) +} + +function metadata(workspace) { + return JSON.parse( + cargo( + [ + 'metadata', + '--locked', + '--no-deps', + '--format-version', + '1', + '--manifest-path', + 'Cargo.toml', + ], + true, + workspace, + ), + ) +} + +export function main(args = process.argv.slice(2)) { + // Positional arguments intentionally have no implicit database fallback. + const [lane, language = 'all'] = args + if (args.length > 2) + throw new Error( + 'Usage: stack-eql-conformance.mjs [rust|go|all]', + ) + checkEnvironment(process.env) + const index = JSON.parse( + read('conformance/stack-encrypt/v1/eql-json-v1.enrollment.json'), + ) + const selected = selections(index, lane, language) + process.env.STACK_CONFORMANCE_REQUIRED = '1' + delete process.env.STACK_CONFORMANCE_INPUT + delete process.env.STACK_CONFORMANCE_OUTPUT + const report = { + profile: 'eql-json-v1', + lane, + language, + engine: metadata('.').packages.find((p) => p.name === 'stack-encrypt') + .version, + eql_bindings: metadata('packages/eql').packages.find( + (p) => p.name === 'eql-bindings', + ).version, + corpus_sha256: hash('conformance/stack-encrypt/v1/eql-json-v1.cases.json'), + enrollment_sha256: hash( + 'conformance/stack-encrypt/v1/eql-json-v1.enrollment.json', + ), + root_lock_sha256: hash('Cargo.lock'), + eql_lock_sha256: hash('packages/eql/Cargo.lock'), + evidence: {}, + } + if ([...selected.values()].some((s) => s.language === 'go')) { + // Both builds are required: capability refusal is tested with the plain guest. + report.guests = Object.fromEntries( + [ + 'stack_encrypt_guest_deterministic.wasm', + 'stack_encrypt_guest_eql_deterministic.wasm', + ].map((name) => [ + name, + hash(`languages/golang/encrypt/testdata/${name}`), + ]), + ) + } + if (lane === 'postgres') { + const url = process.env.EQL_TEST_DATABASE_URL + if (!url) + throw new Error( + 'Set EQL_TEST_DATABASE_URL to a disposable PostgreSQL database', + ) + const bundle = 'packages/eql/release/cipherstash-encrypt-v4.sql' + report.expected_schema = 'eql_v4' + report.sql_bundle_sha256 = hash(bundle) + // Install exactly the bytes whose identity is recorded, into the same + // database both runners use. Never report a merely adjacent bundle's hash. + command('psql', ['-X', url, '-v', 'ON_ERROR_STOP=1', '-q', '-f', bundle]) + report.database = JSON.parse( + command( + 'psql', + [ + '-X', + url, + '-At', + '-v', + 'ON_ERROR_STOP=1', + '-c', + "select json_build_object('postgres', current_setting('server_version'), 'eql', eql_v4.version(), 'schema', (select nspname from pg_namespace where oid = to_regnamespace('eql_v4')))", + ], + { capture: true }, + ), + ) + process.env.DATABASE_URL = url + } + for (const [id, suite] of selected) { + console.log(`EQL conformance: ${id} (${suite.tests.size} required tests)`) + if (suite.language === 'rust') { + const common = [ + '--locked', + '--manifest-path', + 'Cargo.toml', + '-p', + suite.package, + '--test', + suite.binary, + ...suite.features, + '--run-ignored', + lane === 'postgres' ? 'only' : 'default', + ] + const inventory = JSON.parse( + cargo( + ['nextest', 'list', ...common, '--message-format', 'json'], + true, + suite.workspace, + ), + ) + checkRustInventory(inventory, suite) + // Run the whole original suite, retaining unindexed regression and property + // tests. PostgreSQL tests are intentionally #[ignore] outside this lane. + cargo( + [ + 'nextest', + 'run', + ...common, + '--success-output', + lane === 'postgres' ? 'immediate' : 'never', + ], + false, + suite.workspace, + ) + } else { + const pattern = `^(${[...suite.tests].sort().join('|')})$` + const output = command( + 'go', + [ + 'test', + '-C', + 'languages/golang', + '-count=1', + '-json', + '-run', + pattern, + './encrypt', + ], + { capture: true, env: { ...process.env, CGO_ENABLED: '0' } }, + ) + checkGoResults(output, suite.tests) + console.log(`Passed ${suite.tests.size} required Go tests; no skips`) + } + report.evidence[id] = [...suite.tests].sort() + } + console.log(JSON.stringify(report, null, 2)) + if (process.env.STACK_CONFORMANCE_REPORT_DIR) { + mkdirSync(process.env.STACK_CONFORMANCE_REPORT_DIR, { recursive: true }) + writeFileSync( + join( + process.env.STACK_CONFORMANCE_REPORT_DIR, + `eql-json-${lane}-${language}.json`, + ), + `${JSON.stringify(report, null, 2)}\n`, + ) + } +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) main()