From 3dd39085f9e1bcf70b1b08ac867d87d2e0d46b9c Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Tue, 3 Nov 2026 02:07:35 +0900 Subject: [PATCH 1/5] feat(python): parity with the Go SDK's rows, and heap hygiene (#1393) Add a deterministic-kms test feature to the extension. In that build, _engine._use_deterministic_keys(seed) makes the process take its keys from stack-kms's DeterministicSource, as the Go SDK's deterministic guest does. A normal build has neither the feature nor the function. tests_deterministic/ runs against the EQL rows the Go SDK stored under that seed: the frozen stored_v4_rows_v1.json, and the ordered, smallint and ledger fixtures. Python opens Go's rows to the same values, and seals the same values to byte-identical equality, order and match terms. Its query terms equal Go's stored terms. Two ledger rows Python cannot hold are refused with decrypt_not_representable. mise run test:deterministic runs them in their own pytest process, and tests-python.yml runs that task with no credentials. Add a heap-hygiene test to the extension: a spy allocator fails when a block freed during encrypt or query still contains the plaintext. It found that the owned copy of a str, which pyo3 makes under abi3-py39, was freed unwiped. Read now wipes it with zeroize. The four native-plan fixtures and the shared conformance corpus are not part of this commit. The fixtures declare plans the Python SDK cannot express. The corpus needs the stack rebased onto next. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests-python.yml | 12 + AGENTS.md | 2 +- languages/python/Cargo.lock | 3 + languages/python/README.md | 7 +- languages/python/crates/core/Cargo.toml | 6 + languages/python/crates/core/src/client.rs | 54 ++++ languages/python/crates/core/src/decrypt.rs | 4 + languages/python/crates/core/src/encrypt.rs | 8 +- languages/python/crates/core/src/query.rs | 10 + languages/python/crates/engine/Cargo.toml | 15 + languages/python/crates/engine/src/encrypt.rs | 52 ++-- .../python/crates/engine/src/heap_hygiene.rs | 257 ++++++++++++++++++ languages/python/crates/engine/src/lib.rs | 16 ++ languages/python/crates/engine/src/values.rs | 13 + languages/python/mise.toml | 12 + .../python/tests_deterministic/conftest.py | 40 +++ .../tests_deterministic/test_go_rows.py | 251 +++++++++++++++++ 17 files changed, 735 insertions(+), 27 deletions(-) create mode 100644 languages/python/crates/engine/src/heap_hygiene.rs create mode 100644 languages/python/tests_deterministic/conftest.py create mode 100644 languages/python/tests_deterministic/test_go_rows.py diff --git a/.github/workflows/tests-python.yml b/.github/workflows/tests-python.yml index 7331960a7..eaa349014 100644 --- a/.github/workflows/tests-python.yml +++ b/.github/workflows/tests-python.yml @@ -48,6 +48,10 @@ on: # in place, so a change to either reruns this suite. - packages/eql/crates/eql-bindings/** - languages/golang/encrypt/guest/src/targets.rs + # The EQL rows the Go SDK stored under a deterministic key source. + # `mise run test:deterministic` opens them and seals the same values, + # so a change to them must rerun that check. + - packages/eql/tests/encryption/fixtures/** # The Rust version the workspace builds under. The workspace's own # mise.toml is under languages/python/** above. - mise.toml @@ -173,6 +177,14 @@ jobs: - name: pytest run: mise run test + # Rebuilds the extension with the deterministic-kms test feature, then + # runs tests_deterministic/: Python opens the EQL rows the Go SDK stored + # under a deterministic key source, the frozen ones among them, and + # seals the same values to the same terms. No credentials. It runs after + # `pytest` because it replaces the build in .venv with a test build. + - name: Python against the Go SDK's rows + run: mise run test:deterministic + # The committed stackencrypt/_eql.py is generated from the EQL catalog # by eql-codegen (#1385). `mise run types:check` in test-eql.yml diffs # it, but that workflow does not run on a change under languages/python, diff --git a/AGENTS.md b/AGENTS.md index a496a3a77..ae9a0c9e4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate included, needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. -- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and `query` against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. There is no Python lockfile yet. The package has no runtime dependency, so only the `test` extra could be locked: pytest, attrs, Pydantic, SQLAlchemy, and psycopg. The middle three are for the declaration tests, and psycopg is for the live tests. None has an upper bound, so the tests meet each new release when users do. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, the call shape of `query`), the spec in #1341. +- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt` or `query` still contains the plaintext. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and `query` against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. There is no Python lockfile yet. The package has no runtime dependency, so only the `test` extra could be locked: pytest, attrs, Pydantic, SQLAlchemy, and psycopg. The middle three are for the declaration tests, and psycopg is for the live tests. None has an upper bound, so the tests meet each new release when users do. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, the call shape of `query`), the spec in #1341. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) - `languages/typescript/examples/*`: Working apps (basic, prisma, supabase-worker) - `docs/plans/*`: Internal design plans. User-facing documentation lives at https://cipherstash.com/docs (not in this repo). diff --git a/languages/python/Cargo.lock b/languages/python/Cargo.lock index 5b021c441..c7660fc87 100644 --- a/languages/python/Cargo.lock +++ b/languages/python/Cargo.lock @@ -3115,8 +3115,11 @@ name = "stackencrypt-engine" version = "0.1.0" dependencies = [ "pyo3", + "stack-encrypt", + "stack-kms", "stackencrypt-core", "tokio", + "zeroize", ] [[package]] diff --git a/languages/python/README.md b/languages/python/README.md index 6a73e8f23..93f03dcd1 100644 --- a/languages/python/README.md +++ b/languages/python/README.md @@ -110,8 +110,11 @@ The query type is the marker's `query_sql_domain`. For `TextEq` it is `eql_v4.qu From `languages/python`: ```sh -mise run test # no credentials needed -mise run test:live # needs the four CS_* variables and EQL_TEST_DATABASE_URL +mise run test # no credentials needed +mise run test:deterministic # no credentials needed +mise run test:live # needs the four CS_* variables and EQL_TEST_DATABASE_URL ``` +`mise run test:deterministic` builds a test version of the extension. Its keys come from a fixed seed, not from CipherStash. The tests in `tests_deterministic/` then open the EQL rows the Go SDK stored under that seed. They also check that Python seals the same values to the same terms. Run `mise run build` afterwards to get a normal build back. + `mise run test:live` installs the EQL v4 bundle into the database that `EQL_TEST_DATABASE_URL` names. Use a database you can throw away. diff --git a/languages/python/crates/core/Cargo.toml b/languages/python/crates/core/Cargo.toml index 03d9c887b..efc4cd11a 100644 --- a/languages/python/crates/core/Cargo.toml +++ b/languages/python/crates/core/Cargo.toml @@ -17,6 +17,12 @@ default = ["eql"] # reads a feature of exactly this name. Every build of the SDK holds the EQL # types, so the crate refuses to compile without it. eql = [] +# A test build: the shared client takes its keys from stack-kms's +# `DeterministicSource` and a seed, not from ZeroKMS, once +# `client::use_deterministic` is called. The frozen-row tests read rows the Go +# SDK sealed under the same source (the Go guest's `deterministic-kms` build). +# Never in a wheel a user installs. +deterministic-kms = ["stack-kms/test-support"] [dependencies] # By path, like every other eql-bindings dependency in the repository: the diff --git a/languages/python/crates/core/src/client.rs b/languages/python/crates/core/src/client.rs index b3eedf07d..1e81f14dd 100644 --- a/languages/python/crates/core/src/client.rs +++ b/languages/python/crates/core/src/client.rs @@ -69,6 +69,60 @@ pub fn shared() -> Result<&'static Client, Refusal> { Ok(CLIENT.get_or_init(|| client)) } +/// The client of the `deterministic-kms` test build, with its own runtime. +#[cfg(feature = "deterministic-kms")] +pub type Deterministic = ( + StackCipher, + tokio::runtime::Runtime, +); + +#[cfg(feature = "deterministic-kms")] +static DETERMINISTIC: OnceLock = OnceLock::new(); + +/// Make every later operation in the process use a client whose keys come +/// from `seed` through `DeterministicSource`, not from ZeroKMS. For the +/// `deterministic-kms` test build only: rows sealed under the same seed open +/// in any process, the Go SDK's frozen rows among them. +/// +/// # Errors +/// +/// A refusal when the live client is already built, when a deterministic +/// client is already in place, or when the client does not build. +#[cfg(feature = "deterministic-kms")] +pub fn use_deterministic(seed: [u8; 32]) -> Result<(), Refusal> { + let refused = |message: String| Refusal { + code: "stack_test_build".to_owned(), + message, + field: None, + reason: None, + }; + if CLIENT.get().is_some() || DETERMINISTIC.get().is_some() { + return Err(refused( + "the process already has a client; call this before any operation".to_owned(), + )); + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|error| refused(format!("the runtime could not start: {error}")))?; + let cipher = runtime + .block_on( + StackCipher::builder() + .kms(stack_kms::DeterministicSource::new(seed)) + .init(), + ) + .map_err(|error| Refusal::of(&error))?; + DETERMINISTIC + .set((cipher, runtime)) + .map_err(|_| refused("a deterministic client is already in place".to_owned())) +} + +/// The deterministic client, once `use_deterministic` has made one. +#[cfg(feature = "deterministic-kms")] +pub fn deterministic() -> Option<&'static Deterministic> { + DETERMINISTIC.get() +} + /// How many times the shared client has been built in this process: 0 /// before the first operation, 1 after it. For the tests. pub fn build_count() -> u64 { diff --git a/languages/python/crates/core/src/decrypt.rs b/languages/python/crates/core/src/decrypt.rs index 9a15187f9..e8d0bfa39 100644 --- a/languages/python/crates/core/src/decrypt.rs +++ b/languages/python/crates/core/src/decrypt.rs @@ -177,6 +177,10 @@ fn refusal(code: &str, field: &Field, message: String) -> Refusal { /// `decrypt_kind_mismatch` for a value stored as another kind than the /// field's. pub fn decrypt(plan: &ModelPlan, rows: Rows) -> Result>, Refusal> { + #[cfg(feature = "deterministic-kms")] + if let Some((cipher, runtime)) = client::deterministic() { + return runtime.block_on(decrypt_with(cipher, plan, rows)); + } let client = client::shared()?; client.block_on(decrypt_with(client.cipher(), plan, rows)) } diff --git a/languages/python/crates/core/src/encrypt.rs b/languages/python/crates/core/src/encrypt.rs index d8bbae87b..66e8b3a31 100644 --- a/languages/python/crates/core/src/encrypt.rs +++ b/languages/python/crates/core/src/encrypt.rs @@ -11,7 +11,9 @@ //! out as a [`Column`], which the extension turns into the dict's value. use stack_encrypt::dynamic::record::{self, Plan, EQL_KEY}; -use stack_encrypt::dynamic::{self, Output, ValueFrame}; +/// The engine's input frame, which the extension writes the models into. +pub use stack_encrypt::dynamic::ValueFrame; +use stack_encrypt::dynamic::{self, Output}; use stack_encrypt::kms::{DataKeySource, IndexKeySource}; use stack_encrypt::{CipherText, Encrypt, KeysetCipher, StackCipherText}; @@ -104,6 +106,10 @@ pub fn encrypt( source: ValueFrame, rows: usize, ) -> Result>, Refusal> { + #[cfg(feature = "deterministic-kms")] + if let Some((cipher, runtime)) = client::deterministic() { + return runtime.block_on(encrypt_with(&cipher.default_keyset(), plan, source, rows)); + } let client = client::shared()?; client.block_on(encrypt_with(&client.keyset(), plan, source, rows)) } diff --git a/languages/python/crates/core/src/query.rs b/languages/python/crates/core/src/query.rs index 5df850c43..9d7b22df1 100644 --- a/languages/python/crates/core/src/query.rs +++ b/languages/python/crates/core/src/query.rs @@ -35,6 +35,16 @@ pub fn query( value: Value<'_>, ) -> Result { answers_equality(plan, field)?; + #[cfg(feature = "deterministic-kms")] + if let Some((cipher, runtime)) = client::deterministic() { + return runtime.block_on(query_with( + &cipher.default_keyset(), + plan, + field, + kind, + value, + )); + } let client = client::shared()?; client.block_on(query_with(&client.keyset(), plan, field, kind, value)) } diff --git a/languages/python/crates/engine/Cargo.toml b/languages/python/crates/engine/Cargo.toml index 8b86ba296..c04d1529a 100644 --- a/languages/python/crates/engine/Cargo.toml +++ b/languages/python/crates/engine/Cargo.toml @@ -28,6 +28,9 @@ stackencrypt-core = { path = "../core" } # binary has to link it or fails with undefined `_Py*` symbols. maturin turns # the feature on through `[tool.maturin].features` in pyproject.toml. pyo3 = { version = "0.29", features = ["abi3-py39"] } +# To wipe the owned copy of a `str` the extension reads (src/values.rs, +# `Read`). The version the workspace already resolves. +zeroize = "1" # Only for the `gate-probe` feature below. tokio = { version = "1", features = ["rt-multi-thread", "time"], optional = true } @@ -36,3 +39,15 @@ tokio = { version = "1", features = ["rt-multi-thread", "time"], optional = true # Python test can send SIGINT during the call. Off in every normal build; # see docs/gate-facts/keyboard-interrupt.md for how to run it. gate-probe = ["dep:tokio"] +# A test build whose client takes its keys from a seed, not from ZeroKMS, once +# `_use_deterministic_keys` is called. `mise run test:deterministic` builds it +# and runs the frozen-row tests in `tests_deterministic/`. Never in a wheel a +# user installs. +deterministic-kms = ["stackencrypt-core/deterministic-kms"] + +[dev-dependencies] +# For the heap-hygiene test (src/heap_hygiene.rs): a cipher over the +# in-memory key source, and a runtime to run the engine on. +stack-encrypt = { workspace = true } +stack-kms = { workspace = true, features = ["test-support"] } +tokio = { version = "1", features = ["rt"] } diff --git a/languages/python/crates/engine/src/encrypt.rs b/languages/python/crates/engine/src/encrypt.rs index 6a86bec11..601b491ae 100644 --- a/languages/python/crates/engine/src/encrypt.rs +++ b/languages/python/crates/engine/src/encrypt.rs @@ -13,7 +13,7 @@ use std::cell::RefCell; use pyo3::prelude::*; use pyo3::types::{PyDict, PyList, PyType}; use stackencrypt_core::encrypt::plaintext::{Encrypt, MapSink, SeqSink, Sink, SourceError}; -use stackencrypt_core::encrypt::{self as core, Column, Field, ModelPlan}; +use stackencrypt_core::encrypt::{self as core, Column, Field, ModelPlan, ValueFrame}; use stackencrypt_core::refusal::Refusal; use stackencrypt_core::values::{self, Kind}; @@ -107,28 +107,7 @@ pub fn encrypt<'py>( return Ok(PyList::empty(py)); } let rows = models.len(); - let failed = RefCell::new(None); - let source = Models { - plan, - models: &models, - failed: &failed, - }; - let frame = match core::frame_of(source) { - Ok(frame) => frame, - Err(error) => { - return Err(failed.into_inner().unwrap_or_else(|| { - raise( - py, - Refusal { - code: "stack_encrypt_unknown".to_owned(), - message: format!("the models could not be written as a frame: {error}"), - field: None, - reason: None, - }, - ) - })) - } - }; + let frame = input_frame(py, plan, &models)?; let encrypted = py .detach(|| core::encrypt(&plan.plan, frame, rows)) .map_err(|refused| raise(py, refused))?; @@ -147,6 +126,33 @@ pub fn encrypt<'py>( Ok(out) } +/// The models as the engine's input frame, every value read and checked. +pub(crate) fn input_frame( + py: Python<'_>, + plan: &Plan, + models: &[Bound<'_, PyAny>], +) -> PyResult { + let failed = RefCell::new(None); + let source = Models { + plan, + models, + failed: &failed, + }; + core::frame_of(source).map_err(|error| { + failed.into_inner().unwrap_or_else(|| { + raise( + py, + Refusal { + code: "stack_encrypt_unknown".to_owned(), + message: format!("the models could not be written as a frame: {error}"), + field: None, + reason: None, + }, + ) + }) + }) +} + /// How many times the shared engine client has been built in this process. /// For the tests: two `Cipher` instances share one. #[pyfunction] diff --git a/languages/python/crates/engine/src/heap_hygiene.rs b/languages/python/crates/engine/src/heap_hygiene.rs new file mode 100644 index 000000000..961f58cc9 --- /dev/null +++ b/languages/python/crates/engine/src/heap_hygiene.rs @@ -0,0 +1,257 @@ +//! No heap block the extension frees during `encrypt` or `query` still +//! contains the plaintext it was given. +//! +//! The scope is the extension's own copies: what this crate and the engine +//! allocate in Rust while they read a Python value, write the input frame, +//! seal it, and derive a query term. Python's own objects are not watched. +//! CPython allocates them with its own allocator, and the caller owns them, so +//! the SDK makes no promise about them. This includes the `bytes` object that +//! pyo3 makes on its way to a `str`'s UTF-8 under `abi3-py39`. +//! +//! The test binary installs a spy as its global allocator, as stack-encrypt's +//! `tests/heap_hygiene.rs` does. While a test watches its own thread, the spy +//! scans every block freed on that thread for a marker that the plaintext +//! contains. A block freed with the marker still in it is plaintext that a +//! later allocation can read back, and fails the test. The Go SDK's test reads +//! the guest's memory after the call instead, which allocator reuse has +//! mostly overwritten by then. This one sees each free as it happens. +//! +//! One free is excluded by name, as in stack-encrypt's test: vitaminc's +//! aws-lc backend frees an unwiped plaintext buffer on every seal, which is +//! vitaminc's to fix. The spy counts those, and recognises them by their +//! backtrace, so a build without symbols fails rather than passes. +//! +//! Unix only: the spy scans a freed block with libc's `memmem`. CI runs it on +//! Linux. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; +use std::ffi::c_void; +use std::sync::Mutex; + +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyString}; +use stack_encrypt::StackCipher; +use stack_kms::FakeDataKeySource; + +use crate::encrypt::{build_plan, field_value, input_frame, Plan}; + +/// What every plaintext under test contains. +const MARK: &[u8] = b"zq-heap-marker-7f3a9c"; + +thread_local! { + static WATCHING: Cell = const { Cell::new(false) }; + static BUSY: Cell = const { Cell::new(false) }; +} + +/// Held for the whole of a watch: the harness runs tests on several threads, +/// and the findings below are one set. +static GATE: Mutex<()> = Mutex::new(()); + +/// The marked frees seen, as backtraces; and how many were vitaminc's. +static FOUND: Mutex> = Mutex::new(Vec::new()); +static UPSTREAM: Mutex = Mutex::new(0); + +/// The vitaminc frame whose free is excluded (see the module docs). +const UPSTREAM_SEAL: &str = "vitaminc_encrypt::backend::aws_lc::CipherKey::seal"; + +/// Whether the `len` bytes at `ptr` contain [`MARK`]. A freed block can hold +/// bytes Rust never initialised, and reading those through a `&[u8]` is +/// undefined behaviour, so the scan is libc's `memmem`. +/// +/// # Safety +/// +/// `ptr` must point to `len` allocated bytes. +unsafe fn has_mark(ptr: *const u8, len: usize) -> bool { + extern "C" { + fn memmem( + haystack: *const c_void, + haystack_len: usize, + needle: *const c_void, + needle_len: usize, + ) -> *mut c_void; + } + !memmem(ptr.cast(), len, MARK.as_ptr().cast(), MARK.len()).is_null() +} + +struct Spy; + +unsafe impl GlobalAlloc for Spy { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + // Zeroed, so a block holds only what was written into it during its + // own lifetime, not what an earlier free left there. + System.alloc_zeroed(layout) + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + let watching = WATCHING.with(Cell::get) && !BUSY.with(Cell::get); + // SAFETY: `ptr` is a live block of `layout.size()` bytes until the + // `System.dealloc` below. + if watching && has_mark(ptr, layout.size()) { + BUSY.with(|busy| busy.set(true)); + let trace = std::backtrace::Backtrace::force_capture().to_string(); + // Linux writes `<…::CipherKey>::seal`, macOS `…::CipherKey::seal`. + if trace.replace(['<', '>'], "").contains(UPSTREAM_SEAL) { + *UPSTREAM.lock().unwrap() += 1; + } else { + FOUND + .lock() + .unwrap() + .push(format!("{} bytes freed unwiped:\n{trace}", layout.size())); + } + BUSY.with(|busy| busy.set(false)); + } + System.dealloc(ptr, layout) + } +} + +#[global_allocator] +static SPY: Spy = Spy; + +/// Zero the stack below the caller's frame, so that a future the engine +/// boxes copies zeros, not what earlier work left on the stack. +#[inline(never)] +fn scrub_stack() { + let mut pad = [0u8; 256 * 1024]; + for byte in pad.iter_mut() { + // SAFETY: a volatile write to a live local, so it is not elided. + unsafe { std::ptr::write_volatile(byte, 0) }; + } + std::hint::black_box(&pad); +} + +/// Run `f` with this thread watched, and fail if any block it freed still +/// contained the marker. +fn watched(what: &str, f: impl FnOnce() -> T) -> T { + let _gate = GATE.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + FOUND.lock().unwrap().clear(); + *UPSTREAM.lock().unwrap() = 0; + scrub_stack(); + WATCHING.with(|on| on.set(true)); + let out = f(); + WATCHING.with(|on| on.set(false)); + let found = std::mem::take(&mut *FOUND.lock().unwrap()); + let upstream = std::mem::take(&mut *UPSTREAM.lock().unwrap()); + if upstream > 0 { + eprintln!("{what}: skipped {upstream} unwiped frees in vitaminc's aws-lc seal"); + } + assert!( + found.is_empty(), + "{what}: {} heap blocks were freed with plaintext in them:\n{}", + found.len(), + found.join("\n---\n") + ); + out +} + +fn marked(tail: &str) -> String { + format!("{}-{tail}", std::str::from_utf8(MARK).unwrap()) +} + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() +} + +/// A plan for `users` with two sealed text fields: one with an equality +/// index, one with none. +fn plan(py: Python<'_>) -> Plan { + let text = || py.get_type::(); + build_plan( + py, + "users", + vec![ + ( + "email".into(), + Some("TextEq".into()), + Some("string".into()), + text(), + ), + ( + "note".into(), + Some("Text".into()), + Some("string".into()), + text(), + ), + ], + ) + .unwrap() +} + +/// A model as `encrypt` reads it: an object with one attribute per field. +fn model<'py>(py: Python<'py>, email: &str, note: &str) -> Bound<'py, PyAny> { + let fields = PyDict::new(py); + fields.set_item("email", email).unwrap(); + fields.set_item("note", note).unwrap(); + py.import("types") + .unwrap() + .getattr("SimpleNamespace") + .unwrap() + .call((), Some(&fields)) + .unwrap() +} + +#[test] +fn encrypt_frees_no_copy_of_the_plaintext_unwiped() { + Python::initialize(); + let runtime = runtime(); + let cipher = runtime + .block_on(StackCipher::builder().kms(FakeDataKeySource::new()).init()) + .unwrap(); + Python::attach(|py| { + let plan = plan(py); + // The Python objects are made before the watch: they are the + // caller's, not the extension's. + let models = vec![ + model(py, &marked("a@example.com"), &marked("a note")), + model(py, &marked("b@example.com"), &marked("another note")), + ]; + + let rows = watched("encrypt", || { + let frame = input_frame(py, &plan, &models).unwrap(); + runtime + .block_on(stackencrypt_core::encrypt::encrypt_with( + &cipher.default_keyset(), + &plan.plan, + frame, + models.len(), + )) + .unwrap() + }); + + assert_eq!(rows.len(), 2); + }); +} + +#[test] +fn query_frees_no_copy_of_the_plaintext_unwiped() { + Python::initialize(); + let runtime = runtime(); + let cipher = runtime + .block_on(StackCipher::builder().kms(FakeDataKeySource::new()).init()) + .unwrap(); + Python::attach(|py| { + let plan = plan(py); + let value = PyString::new(py, &marked("a@example.com")).into_any(); + let (field, kind) = (&plan.plan.fields()[0], plan.kinds[0]); + let Some((_, engine_kind)) = field.eql else { + panic!("email is sealed"); + }; + + let term = watched("query", || { + let read = field_value(&value, field, kind).unwrap().unwrap(); + runtime + .block_on(stackencrypt_core::query::query_with( + &cipher.default_keyset(), + &plan.plan, + "email", + engine_kind, + read.value(), + )) + .unwrap() + }); + + assert!(!term.as_bytes().windows(MARK.len()).any(|w| w == MARK)); + }); +} diff --git a/languages/python/crates/engine/src/lib.rs b/languages/python/crates/engine/src/lib.rs index 1447c254d..048cba2d9 100644 --- a/languages/python/crates/engine/src/lib.rs +++ b/languages/python/crates/engine/src/lib.rs @@ -6,6 +6,8 @@ mod decrypt; mod encrypt; mod errors; +#[cfg(all(test, unix))] +mod heap_hygiene; mod query; mod values; @@ -88,6 +90,18 @@ fn _probe_detached_sleep(py: Python<'_>, millis: u64) -> u64 { }) } +/// Make every later operation in this process use keys derived from `seed` +/// (32 bytes) by stack-kms's `DeterministicSource`, not ZeroKMS. Only in the +/// `deterministic-kms` test build. Call it before any operation. +#[cfg(feature = "deterministic-kms")] +#[pyfunction] +fn _use_deterministic_keys(py: Python<'_>, seed: &[u8]) -> PyResult<()> { + let seed: [u8; 32] = seed + .try_into() + .map_err(|_| pyo3::exceptions::PyValueError::new_err("the seed is 32 bytes"))?; + stackencrypt_core::client::use_deterministic(seed).map_err(|refused| errors::raise(py, refused)) +} + #[pymodule] fn _engine(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(build_info, module)?)?; @@ -102,6 +116,8 @@ fn _engine(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(encrypt::request_count, module)?)?; #[cfg(feature = "gate-probe")] module.add_function(wrap_pyfunction!(_probe_detached_sleep, module)?)?; + #[cfg(feature = "deterministic-kms")] + module.add_function(wrap_pyfunction!(_use_deterministic_keys, module)?)?; Ok(()) } diff --git a/languages/python/crates/engine/src/values.rs b/languages/python/crates/engine/src/values.rs index 0bfb462d1..3e5051d1c 100644 --- a/languages/python/crates/engine/src/values.rs +++ b/languages/python/crates/engine/src/values.rs @@ -7,6 +7,7 @@ //! never changes a value: nothing here converts, only reads. use std::borrow::Cow; +use zeroize::Zeroize; use pyo3::prelude::*; use pyo3::sync::PyOnceLock; @@ -118,12 +119,24 @@ impl PyKind { /// A field's value as read from its attribute. Text and bytes borrow from /// the attribute object where they can, so the plaintext is not copied /// before the engine writes it into the protected frame. +/// +/// Under `abi3-py39`, pyo3 cannot borrow a `str`'s UTF-8, so `Text` is an +/// owned copy on every Python version. That copy is the extension's own, and +/// is wiped when the `Read` is dropped (`src/heap_hygiene.rs` checks it). pub enum Read<'a> { Text(Cow<'a, str>), Bytes(&'a [u8]), Other(Value<'static>), } +impl Drop for Read<'_> { + fn drop(&mut self) { + if let Self::Text(Cow::Owned(text)) = self { + text.zeroize(); + } + } +} + impl Read<'_> { /// The value, as the core rules take it. pub fn value(&self) -> Value<'_> { diff --git a/languages/python/mise.toml b/languages/python/mise.toml index 191bea47d..7e6c999d2 100644 --- a/languages/python/mise.toml +++ b/languages/python/mise.toml @@ -53,6 +53,18 @@ description = "Run the pure-Python test suite against the package `build` instal depends = ["build"] run = ".venv/bin/python -m pytest" +[tasks."test:deterministic"] +description = "Build the extension with the deterministic-kms test feature into .venv and run tests_deterministic/: Python against the Go SDK's frozen EQL rows, with no credentials" +shell = "bash -c" +# The deterministic client replaces the live one for the whole process, so +# these tests run in their own pytest process, from their own folder. The +# build is a test build: run `mise run build` afterwards for a normal one. +run = """ +set -euo pipefail +MATURIN_PEP517_ARGS="--profile dev --features deterministic-kms" mise run build +.venv/bin/python -m pytest tests_deterministic +""" + [tasks."test:cargo"] description = "Run both crates' Rust tests with --locked; puts libpython on the loader path so the engine crate's tests link" shell = "bash -c" diff --git a/languages/python/tests_deterministic/conftest.py b/languages/python/tests_deterministic/conftest.py new file mode 100644 index 000000000..0c10c9000 --- /dev/null +++ b/languages/python/tests_deterministic/conftest.py @@ -0,0 +1,40 @@ +"""Every test here runs against the `deterministic-kms` test build. + +`mise run test:deterministic` builds the extension with that feature and runs +this folder in its own pytest process. The session switches the process to +the deterministic client before any test runs: its keys come from a seed +through stack-kms's `DeterministicSource`, not from ZeroKMS. So no test here +needs credentials or makes a request. + +The seed is the one the Go SDK's frozen rows were sealed under, so rows the +Go SDK stored open here. The switch is for the whole process, which is why +this folder is not under `tests/`. +""" + +import json +import os +from pathlib import Path + +import pytest + +from stackencrypt import _engine + +REPO = Path(__file__).resolve().parents[3] +# The Go SDK's EQL fixtures, all sealed under one seed. +FIXTURES = REPO / "packages/eql/tests/encryption/fixtures" + + +def pytest_sessionstart(session): + use = getattr(_engine, "_use_deterministic_keys", None) + if use is None: + pytest.exit( + "the extension was built without the deterministic-kms feature; " + "run `mise run test:deterministic`", + returncode=2, + ) + # Cipher checks that credentials are present before it builds a plan. + # These parse, and the deterministic client never uses them. + os.environ["CS_CLIENT_ACCESS_KEY"] = "CSAKtestKeyId.testKeySecret" + os.environ["CS_WORKSPACE_CRN"] = "crn:ap-southeast-2.aws:ZVATKW3VHMFG27DY" + frozen = json.loads((FIXTURES / "stored_v4_rows_v1.json").read_text()) + use(frozen["seed"].encode()) diff --git a/languages/python/tests_deterministic/test_go_rows.py b/languages/python/tests_deterministic/test_go_rows.py new file mode 100644 index 000000000..faa05869e --- /dev/null +++ b/languages/python/tests_deterministic/test_go_rows.py @@ -0,0 +1,251 @@ +"""Python against the EQL rows the Go SDK stored, under the same seed. + +Every fixture here is in `packages/eql/tests/encryption/fixtures/`, written by +the Go SDK's generated code over the deterministic EQL guest. Each model +declares the same table, columns and EQL types as its Go type in +`languages/golang/encrypt/internal/testusers/`. + +- `stored_v4_rows_v1.json` is FROZEN: nothing writes it again. Go + (`TestFrozenV4RowsStillOpen`) and Rust (`tests/v4_types.rs`) check it too. + A build that cannot open it is red. +- `v4_ordered_rows.json`, `v4_smallint_rows.json` and `v4_ledger_rows.json` + are regenerated when the stored format changes on purpose. They hold what + the Go SDK seals today, for the types the frozen file lacks: the order + terms, `Smallint`, dates, timestamps and decimals. + +For each, Python opens the rows Go stored, and seals the same plaintexts to +the same stored values apart from each ciphertext `c`. The equality, order +and match terms are deterministic, so they must be byte-identical. Python's +equality query term for each value must be the term Go stored. +""" + +import dataclasses +import datetime +import decimal +import json +import struct +from typing import Annotated + +import pytest + +import stackencrypt +from conftest import FIXTURES +from stackencrypt import ( + Bigint, + BigintEq, + BigintOrd, + BigintOrdOpe, + Boolean, + Cipher, + Date, + DateEq, + DecryptError, + Double, + DoubleEq, + DoubleOrd, + DoubleOrdOpe, + Integer, + IntegerEq, + IntegerOrd, + IntegerOrdOpe, + Numeric, + NumericEq, + Passthrough, + Real, + RealEq, + RealOrd, + RealOrdOpe, + Smallint, + SmallintEq, + Text, + TextEq, + TextMatch, + TextOrd, + TextOrdOpe, + Timestamp, + TimestampEq, +) + + +@stackencrypt.model("profiles") +@dataclasses.dataclass +class Profile: + id: Annotated[int, Passthrough] + visits: Annotated[int, Integer] + balance: Annotated[int, Bigint] + active: Annotated[bool, Boolean] + ratio: Annotated[float, Real] + score: Annotated[float, Double] + note: Annotated[str, Text] + name: Annotated[str, TextMatch] + age: Annotated[int, IntegerEq] + rank: Annotated[int, BigintEq] + weight: Annotated[float, RealEq] + price: Annotated[float, DoubleEq] + email: Annotated[str, TextEq] + + +@stackencrypt.model("rankings") +@dataclasses.dataclass +class Ranking: + id: Annotated[int, Passthrough] + level: Annotated[int, IntegerOrd] + floor: Annotated[int, IntegerOrdOpe] + points: Annotated[int, BigintOrd] + total: Annotated[int, BigintOrdOpe] + speed: Annotated[float, RealOrd] + pace: Annotated[float, RealOrdOpe] + score: Annotated[float, DoubleOrd] + mean: Annotated[float, DoubleOrdOpe] + title: Annotated[str, TextOrd] + handle: Annotated[str, TextOrdOpe] + + +@stackencrypt.model("profiles") +@dataclasses.dataclass +class Tally: + id: Annotated[int, Passthrough] + small: Annotated[int, Smallint] + tally: Annotated[int, SmallintEq] + wide: Annotated[int, SmallintEq] + + +@stackencrypt.model("profiles") +@dataclasses.dataclass +class Ledger: + id: Annotated[int, Passthrough] + opened: Annotated[datetime.date, Date] + due: Annotated[datetime.date, DateEq] + created: Annotated[datetime.datetime, Timestamp] + posted: Annotated[datetime.datetime, TimestampEq] + total: Annotated[decimal.Decimal, Numeric] + amount: Annotated[decimal.Decimal, NumericEq] + + +# (fixture, model, the row ids Python's types can hold). Ledger rows 1 and 2 +# hold years past Python's 1 to 9999 and nanosecond timestamps. +CASES = [ + ("stored_v4_rows_v1.json", Profile, [1, 2]), + ("v4_ordered_rows.json", Ranking, [1, 2, 3]), + ("v4_smallint_rows.json", Tally, [1, 2, 3]), + ("v4_ledger_rows.json", Ledger, [3]), +] +IDS = [fixture for fixture, _, _ in CASES] + + +def rows_of(fixture, ids): + rows = json.loads((FIXTURES / fixture).read_text())["rows"] + assert set(ids) <= {row["id"] for row in rows}, (fixture, ids) + return [row for row in rows if row["id"] in ids] + + +def sealed_fields(cls): + return [field for field in dataclasses.fields(cls) if field.name != "id"] + + +def value_of(python_type, plaintext): + """A Go fixture's JSON plaintext as the field's Python value.""" + if python_type is datetime.datetime: + return datetime.datetime.fromisoformat(plaintext.replace("Z", "+00:00")) + if python_type is datetime.date: + return datetime.date.fromisoformat(plaintext) + if python_type is decimal.Decimal: + return decimal.Decimal(plaintext) + # JSON gives 0 for a float field whose value is zero. + return python_type(plaintext) + + +def model_of(cls, row): + values = { + field.name: value_of(field.type.__origin__, row["plaintext"][field.name]) + for field in sealed_fields(cls) + } + return cls(id=row["id"], **values) + + +def stored_row(cls, row): + """The row as psycopg's dict_row gives it: EQL values as JSON text.""" + return { + "id": row["id"], + **{f.name: json.dumps(row["stored"][f.name]) for f in sealed_fields(cls)}, + } + + +def exact(value): + """A form whose == also compares a float's sign, a Decimal's scale, the + type (True is not 1), and an instant whatever its zone.""" + if isinstance(value, float): + return struct.pack(" Date: Tue, 3 Nov 2026 02:59:29 +0900 Subject: [PATCH 2/5] fix(python): address the review of parity and heap hygiene (#1393) Refuse an opened timestamp with nanoseconds as decrypt_not_representable. to_python rounded it to microseconds, so a Go-written value opened changed, and resealed to another equality term. The ledger test now checks nanoseconds and years on their own: the old test passed only because those rows also had a year out of range. Wipe decrypted plaintext. A Scalar now wipes its text and bytes when dropped, and to_python borrows it. Decimal digits are wiped on read and on to_python. The hygiene test now watches decrypt too, and a decimal by its digits. The hygiene test now checks that the spy fails on a leak of each needle. Each test takes one lock before it attaches to Python: one test could hold the watch lock and wait for the GIL while another held the GIL and waited for the watch lock. In the test build, client::shared() refuses after the switch to the deterministic client, so an operation that skips the check cannot reach ZeroKMS. use_deterministic takes the build lock. CI's clippy step lints every feature. test_go_rows.py no longer imports from conftest. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/tests-python.yml | 4 +- AGENTS.md | 2 +- languages/python/Cargo.lock | 1 + languages/python/crates/core/Cargo.toml | 3 + languages/python/crates/core/src/client.rs | 20 ++ languages/python/crates/core/src/encrypt.rs | 10 +- languages/python/crates/core/src/values.rs | 14 ++ languages/python/crates/engine/src/decrypt.rs | 4 +- .../python/crates/engine/src/heap_hygiene.rs | 202 ++++++++++++++---- languages/python/crates/engine/src/values.rs | 53 +++-- .../tests_deterministic/test_go_rows.py | 52 ++++- 11 files changed, 296 insertions(+), 69 deletions(-) diff --git a/.github/workflows/tests-python.yml b/.github/workflows/tests-python.yml index eaa349014..f0fec514d 100644 --- a/.github/workflows/tests-python.yml +++ b/.github/workflows/tests-python.yml @@ -163,8 +163,10 @@ jobs: # pyo3's build script asks the `python3` on PATH for its configuration. # That is why this runs after mise is installed, once per interpreter. + # With every feature, so the code behind the test features + # (deterministic-kms, gate-probe) is linted too. - name: clippy - run: cargo clippy --locked --no-deps --workspace --all-targets -- -D warnings + run: cargo clippy --locked --no-deps --workspace --all-targets --all-features -- -D warnings # `cargo test --locked` for both crates. The task puts the interpreter's # library directory on the loader path, because the engine crate links diff --git a/AGENTS.md b/AGENTS.md index ae9a0c9e4..2a6e685ee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate included, needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. -- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt` or `query` still contains the plaintext. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and `query` against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. There is no Python lockfile yet. The package has no runtime dependency, so only the `test` extra could be locked: pytest, attrs, Pydantic, SQLAlchemy, and psycopg. The middle three are for the declaration tests, and psycopg is for the live tests. None has an upper bound, so the tests meet each new release when users do. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, the call shape of `query`), the spec in #1341. +- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core/fuzz` is a detached cargo-fuzz crate over the value conversion. The root `mise.toml`'s `fuzz:python-values` task runs it, from `fuzz.yml`, so it is the one Python task outside `languages/python/mise.toml`. `crates/core/testdata/value-conformance.json` is a copy of vitaminc's shared value corpus; its `README.md` records the commit it came from. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. Its extension tests include `crates/engine/src/heap_hygiene.rs`: a spy allocator fails the test when a block the extension frees during `encrypt`, `query` or `decrypt` still contains text or decimal plaintext. Python's own objects are out of its scope. It runs on Unix, and CI runs it on Linux. `mise run test:deterministic` rebuilds the extension with the `deterministic-kms` test feature, whose keys come from a seed through stack-kms's `DeterministicSource`. It then runs `tests_deterministic/` in its own pytest process: Python opens the EQL rows the Go SDK stored under that seed in `packages/eql/tests/encryption/fixtures/`, the frozen `stored_v4_rows_v1.json` among them, and seals the same values to the same terms. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt`, `decrypt` and `query` against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into, searched in and read back from a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. There is no Python lockfile yet. The package has no runtime dependency, so only the `test` extra could be locked: pytest, attrs, Pydantic, SQLAlchemy, and psycopg. The middle three are for the declaration tests, and psycopg is for the live tests. None has an upper bound, so the tests meet each new release when users do. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: the Python ADRs under `languages/python/docs/adr/` (ADR-0001, the declaration; ADR-0002, the call shape of `query`), the spec in #1341. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) - `languages/typescript/examples/*`: Working apps (basic, prisma, supabase-worker) - `docs/plans/*`: Internal design plans. User-facing documentation lives at https://cipherstash.com/docs (not in this repo). diff --git a/languages/python/Cargo.lock b/languages/python/Cargo.lock index c7660fc87..f1746dd78 100644 --- a/languages/python/Cargo.lock +++ b/languages/python/Cargo.lock @@ -3107,6 +3107,7 @@ dependencies = [ "stack-encrypt", "stack-kms", "tokio", + "zeroize", "zerokms-protocol", ] diff --git a/languages/python/crates/core/Cargo.toml b/languages/python/crates/core/Cargo.toml index efc4cd11a..a7053ab67 100644 --- a/languages/python/crates/core/Cargo.toml +++ b/languages/python/crates/core/Cargo.toml @@ -41,6 +41,9 @@ stack-kms = { workspace = true } # One multi-thread runtime per process, beside the shared client: an # operation blocks its own thread on it with the GIL released. tokio = { version = "1", features = ["rt-multi-thread"] } +# To wipe the opened text and bytes a `Scalar` holds when it is dropped. +# The version the workspace already resolves. +zeroize = "1" # For the request-counting connection: the trait a ZeroKMS request # implements. The version stack-kms resolves. zerokms-protocol = "0.12.31" diff --git a/languages/python/crates/core/src/client.rs b/languages/python/crates/core/src/client.rs index 1e81f14dd..f99045a58 100644 --- a/languages/python/crates/core/src/client.rs +++ b/languages/python/crates/core/src/client.rs @@ -55,6 +55,21 @@ static REQUESTS: AtomicU64 = AtomicU64::new(0); /// client key or the ZeroKMS client (`stack_kms_*`), or the keyset load /// (`stack_encrypt_*`). Nothing is cached on failure. pub fn shared() -> Result<&'static Client, Refusal> { + // In the test build, an operation that reaches here after the switch has + // not checked for the deterministic client, and would send requests to + // ZeroKMS from a test process. Refuse it, so a new operation that skips + // the check fails its test instead. + #[cfg(feature = "deterministic-kms")] + if DETERMINISTIC.get().is_some() { + return Err(Refusal { + code: "stack_test_build".to_owned(), + message: "this operation asked for the live client after the switch to the \ + deterministic one; call client::deterministic() first" + .to_owned(), + field: None, + reason: None, + }); + } if let Some(client) = CLIENT.get() { return Ok(client); } @@ -96,6 +111,11 @@ pub fn use_deterministic(seed: [u8; 32]) -> Result<(), Refusal> { field: None, reason: None, }; + // The lock `shared` builds under: a live build in flight on another + // thread finishes before this checks, so the two never both exist. + let _building = BUILDING + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); if CLIENT.get().is_some() || DETERMINISTIC.get().is_some() { return Err(refused( "the process already has a client; call this before any operation".to_owned(), diff --git a/languages/python/crates/core/src/encrypt.rs b/languages/python/crates/core/src/encrypt.rs index 66e8b3a31..7ee652dc5 100644 --- a/languages/python/crates/core/src/encrypt.rs +++ b/languages/python/crates/core/src/encrypt.rs @@ -180,12 +180,16 @@ fn read_row(plan: &ModelPlan, row: StackCipherText) -> Result, Refus field: Some(field.name.clone()), ..refused })?; - columns.push(match (&field.eql, scalar) { + let mut scalar = scalar; + columns.push(match (&field.eql, &mut scalar) { + // The EQL JSON is ciphertext, not plaintext: taken out of the + // scalar, which wipes what is left when dropped. (Some(_), Scalar::Bytes(json)) => Column::Eql( - String::from_utf8(json).map_err(|_| engine_fault("an EQL value is not UTF-8"))?, + String::from_utf8(std::mem::take(json)) + .map_err(|_| engine_fault("an EQL value is not UTF-8"))?, ), (Some(_), _) => return Err(engine_fault("an EQL value is not bytes")), - (None, scalar) => Column::Passthrough(scalar), + (None, _) => Column::Passthrough(scalar), }); } Ok(columns) diff --git a/languages/python/crates/core/src/values.rs b/languages/python/crates/core/src/values.rs index 4efdde9c7..c7dd86297 100644 --- a/languages/python/crates/core/src/values.rs +++ b/languages/python/crates/core/src/values.rs @@ -17,6 +17,7 @@ use stack_encrypt::dynamic::plaintext::{ kinds, Error as SinkError, FixedType, Float, Int, Logical, MapSink, Physical, SeqSink, Sink, SourceError, TimeUnit, }; +use zeroize::Zeroize; use std::convert::Infallible; use std::marker::PhantomData; @@ -248,6 +249,19 @@ pub enum Scalar { }, } +/// An opened text or bytes value is plaintext, and the SDK's own copy of it: +/// it is wiped when dropped. The extension copies it into a Python object +/// first (`src/heap_hygiene.rs` in the engine crate checks the drop). +impl Drop for Scalar { + fn drop(&mut self) { + match self { + Self::Text(text) => text.zeroize(), + Self::Bytes(bytes) => bytes.zeroize(), + _ => {} + } + } +} + impl Scalar { /// The scalar as the value the SDK writes, or `None` for a null, which /// the SDK never writes. diff --git a/languages/python/crates/engine/src/decrypt.rs b/languages/python/crates/engine/src/decrypt.rs index 063d2ad40..7bcb262d8 100644 --- a/languages/python/crates/engine/src/decrypt.rs +++ b/languages/python/crates/engine/src/decrypt.rs @@ -213,8 +213,8 @@ fn python_value(py: Python<'_>, field: &Field, value: Scalar) -> PyResult = const { Cell::new(false) }; static BUSY: Cell = const { Cell::new(false) }; @@ -48,6 +67,17 @@ thread_local! { /// and the findings below are one set. static GATE: Mutex<()> = Mutex::new(()); +/// Held for the whole of each test, taken before the test attaches to +/// Python. Without it, one test can hold [`GATE`] and wait for the GIL while +/// another holds the GIL and waits for [`GATE`]. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> std::sync::MutexGuard<'static, ()> { + SERIAL + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + /// The marked frees seen, as backtraces; and how many were vitaminc's. static FOUND: Mutex> = Mutex::new(Vec::new()); static UPSTREAM: Mutex = Mutex::new(0); @@ -55,9 +85,9 @@ static UPSTREAM: Mutex = Mutex::new(0); /// The vitaminc frame whose free is excluded (see the module docs). const UPSTREAM_SEAL: &str = "vitaminc_encrypt::backend::aws_lc::CipherKey::seal"; -/// Whether the `len` bytes at `ptr` contain [`MARK`]. A freed block can hold -/// bytes Rust never initialised, and reading those through a `&[u8]` is -/// undefined behaviour, so the scan is libc's `memmem`. +/// Whether the `len` bytes at `ptr` contain one of [`NEEDLES`]. A freed +/// block can hold bytes Rust never initialised, and reading those through a +/// `&[u8]` is undefined behaviour, so the scan is libc's `memmem`. /// /// # Safety /// @@ -71,7 +101,9 @@ unsafe fn has_mark(ptr: *const u8, len: usize) -> bool { needle_len: usize, ) -> *mut c_void; } - !memmem(ptr.cast(), len, MARK.as_ptr().cast(), MARK.len()).is_null() + NEEDLES + .iter() + .any(|needle| !memmem(ptr.cast(), len, needle.as_ptr().cast(), needle.len()).is_null()) } struct Spy; @@ -154,10 +186,11 @@ fn runtime() -> tokio::runtime::Runtime { .unwrap() } -/// A plan for `users` with two sealed text fields: one with an equality -/// index, one with none. +/// A plan for `users`: a text field with an equality index, one with none, +/// and a decimal with an equality index. fn plan(py: Python<'_>) -> Plan { let text = || py.get_type::(); + let decimal = decimal_type(py).cast_into::().unwrap(); build_plan( py, "users", @@ -174,16 +207,29 @@ fn plan(py: Python<'_>) -> Plan { Some("string".into()), text(), ), + ( + "amount".into(), + Some("NumericEq".into()), + Some("decimal".into()), + decimal, + ), ], ) .unwrap() } +fn decimal_type(py: Python<'_>) -> Bound<'_, PyAny> { + py.import("decimal").unwrap().getattr("Decimal").unwrap() +} + /// A model as `encrypt` reads it: an object with one attribute per field. fn model<'py>(py: Python<'py>, email: &str, note: &str) -> Bound<'py, PyAny> { let fields = PyDict::new(py); fields.set_item("email", email).unwrap(); fields.set_item("note", note).unwrap(); + fields + .set_item("amount", decimal_type(py).call1((DECIMAL,)).unwrap()) + .unwrap(); py.import("types") .unwrap() .getattr("SimpleNamespace") @@ -192,13 +238,40 @@ fn model<'py>(py: Python<'py>, email: &str, note: &str) -> Bound<'py, PyAny> { .unwrap() } +fn cipher(runtime: &tokio::runtime::Runtime) -> StackCipher { + runtime + .block_on(StackCipher::builder().kms(FakeDataKeySource::new()).init()) + .unwrap() +} + +/// Two marked models, encrypted outside any watch. +fn stored( + py: Python<'_>, + runtime: &tokio::runtime::Runtime, + cipher: &StackCipher, + plan: &Plan, +) -> Vec> { + let models = vec![ + model(py, &marked("a@example.com"), &marked("a note")), + model(py, &marked("b@example.com"), &marked("another note")), + ]; + let frame = input_frame(py, plan, &models).unwrap(); + runtime + .block_on(encrypt_with( + &cipher.default_keyset(), + &plan.plan, + frame, + models.len(), + )) + .unwrap() +} + #[test] fn encrypt_frees_no_copy_of_the_plaintext_unwiped() { + let _serial = serial(); Python::initialize(); let runtime = runtime(); - let cipher = runtime - .block_on(StackCipher::builder().kms(FakeDataKeySource::new()).init()) - .unwrap(); + let cipher = cipher(&runtime); Python::attach(|py| { let plan = plan(py); // The Python objects are made before the watch: they are the @@ -211,7 +284,7 @@ fn encrypt_frees_no_copy_of_the_plaintext_unwiped() { let rows = watched("encrypt", || { let frame = input_frame(py, &plan, &models).unwrap(); runtime - .block_on(stackencrypt_core::encrypt::encrypt_with( + .block_on(encrypt_with( &cipher.default_keyset(), &plan.plan, frame, @@ -226,32 +299,89 @@ fn encrypt_frees_no_copy_of_the_plaintext_unwiped() { #[test] fn query_frees_no_copy_of_the_plaintext_unwiped() { + let _serial = serial(); Python::initialize(); let runtime = runtime(); - let cipher = runtime - .block_on(StackCipher::builder().kms(FakeDataKeySource::new()).init()) - .unwrap(); + let cipher = cipher(&runtime); Python::attach(|py| { let plan = plan(py); - let value = PyString::new(py, &marked("a@example.com")).into_any(); - let (field, kind) = (&plan.plan.fields()[0], plan.kinds[0]); - let Some((_, engine_kind)) = field.eql else { - panic!("email is sealed"); - }; - - let term = watched("query", || { - let read = field_value(&value, field, kind).unwrap().unwrap(); - runtime - .block_on(stackencrypt_core::query::query_with( - &cipher.default_keyset(), - &plan.plan, - "email", - engine_kind, - read.value(), - )) - .unwrap() + let text = PyString::new(py, &marked("a@example.com")).into_any(); + let decimal = decimal_type(py).call1((DECIMAL,)).unwrap(); + + for (at, value) in [(0, &text), (2, &decimal)] { + let (field, kind) = (&plan.plan.fields()[at], plan.kinds[at]); + let Some((_, engine_kind)) = field.eql else { + panic!("{} is sealed", field.name); + }; + let term = watched("query", || { + let read = field_value(value, field, kind).unwrap().unwrap(); + runtime + .block_on(query_with( + &cipher.default_keyset(), + &plan.plan, + &field.name, + engine_kind, + read.value(), + )) + .unwrap() + }); + assert!(!term.as_bytes().windows(MARK.len()).any(|w| w == MARK)); + } + }); +} + +#[test] +fn decrypt_frees_no_copy_of_the_plaintext_unwiped() { + let _serial = serial(); + Python::initialize(); + let runtime = runtime(); + let cipher = cipher(&runtime); + Python::attach(|py| { + let plan = plan(py); + let stored = stored(py, &runtime, &cipher, &plan); + + let opened = watched("decrypt", || { + let mut rows = Rows::with_capacity(stored.len()); + for columns in &stored { + let mut row = Row::new(); + for (field, column) in plan.plan.fields().iter().zip(columns) { + let Column::Eql(json) = column else { + panic!("{} is sealed", field.name); + }; + row.eql(&plan.plan, field, json.as_bytes()).unwrap(); + } + rows.push(row); + } + let opened = runtime + .block_on(decrypt_with(&cipher, &plan.plan, rows)) + .unwrap(); + // Each value as the Python object `decrypt` builds the instance + // from. The objects are the caller's once returned. + opened + .into_iter() + .flatten() + .map(|scalar| to_python(py, scalar).unwrap()) + .collect::>() }); - assert!(!term.as_bytes().windows(MARK.len()).any(|w| w == MARK)); + assert_eq!(opened.len(), 6); + assert_eq!( + opened[0].bind(py).extract::().unwrap(), + marked("a@example.com") + ); }); } + +/// The spy is not blind: a copy of each needle freed unwiped fails the +/// watch. If frees went unseen (another thread, an exclusion matching too +/// much), the three tests above would pass whatever the extension did. +#[test] +fn the_spy_fails_a_copy_freed_unwiped() { + let _serial = serial(); + for needle in NEEDLES { + let leaked = std::panic::catch_unwind(|| { + watched("a leak", || drop(std::hint::black_box(needle.to_vec()))); + }); + assert!(leaked.is_err(), "the spy missed {needle:?}"); + } +} diff --git a/languages/python/crates/engine/src/values.rs b/languages/python/crates/engine/src/values.rs index 3e5051d1c..dfb68f1e6 100644 --- a/languages/python/crates/engine/src/values.rs +++ b/languages/python/crates/engine/src/values.rs @@ -7,7 +7,7 @@ //! never changes a value: nothing here converts, only reads. use std::borrow::Cow; -use zeroize::Zeroize; +use zeroize::{Zeroize, Zeroizing}; use pyo3::prelude::*; use pyo3::sync::PyOnceLock; @@ -241,7 +241,8 @@ fn decimal( ) -> PyResult, Refusal>> { let parts = value.call_method0("as_tuple")?; let sign: u8 = parts.getattr("sign")?.extract()?; - let digits: Vec = parts.getattr("digits")?.extract()?; + // The digits are plaintext: wiped when dropped. + let digits = Zeroizing::new(parts.getattr("digits")?.extract::>()?); let Ok(exponent) = parts.getattr("exponent")?.extract::() else { return Ok(Err(refusal( "value_not_representable", @@ -257,7 +258,7 @@ fn decimal( ) }; let mut mantissa: i128 = 0; - for digit in digits { + for &digit in digits.iter() { mantissa = match mantissa .checked_mul(10) .and_then(|m| m.checked_add(i128::from(digit))) @@ -318,22 +319,33 @@ fn type_name(value: &Bound<'_, PyAny>) -> String { } /// An opened scalar as a Python object, of the type the field declares. +/// +/// The scalar's text and bytes are copied into the Python object, and the +/// scalar wipes its own copy when it is dropped. +/// +/// # Errors +/// +/// A `ValueError` for a date or timestamp outside `datetime`'s range, and +/// for a timestamp with nanoseconds: `datetime` keeps microseconds, and +/// rounding would change the value. The Go SDK can store one. pub fn to_python(py: Python<'_>, scalar: Scalar) -> PyResult> { - Ok(match scalar { + Ok(match &scalar { Scalar::Null => py.None(), - Scalar::Text(text) => PyString::new(py, &text).into_any().unbind(), + Scalar::Text(text) => PyString::new(py, text).into_any().unbind(), Scalar::Int(i) => i.into_pyobject(py)?.into_any().unbind(), - Scalar::Float(f) => PyFloat::new(py, f).into_any().unbind(), - Scalar::Bool(b) => PyBool::new(py, b).to_owned().into_any().unbind(), - Scalar::Bytes(bytes) => PyBytes::new(py, &bytes).into_any().unbind(), + Scalar::Float(f) => PyFloat::new(py, *f).into_any().unbind(), + Scalar::Bool(b) => PyBool::new(py, *b).to_owned().into_any().unbind(), + Scalar::Bytes(bytes) => PyBytes::new(py, bytes).into_any().unbind(), Scalar::Date(days) => PyKind::Date .type_object(py)? - .call_method1("fromordinal", (i64::from(days) + 719_163,))? + .call_method1("fromordinal", (i64::from(*days) + 719_163,))? .unbind(), - Scalar::Timestamp(nanos) => { - // Python's datetime keeps microseconds; the engine's timestamp - // is in nanoseconds, so the last three digits are dropped. An - // encrypted datetime had none: Python wrote it. + &Scalar::Timestamp(nanos) => { + if nanos.rem_euclid(1_000) != 0 { + return Err(pyo3::exceptions::PyValueError::new_err( + "the timestamp has nanoseconds, and datetime keeps microseconds", + )); + } let micros = nanos.div_euclid(1_000); let seconds = micros.div_euclid(1_000_000); let micros = micros.rem_euclid(1_000_000); @@ -343,19 +355,16 @@ pub fn to_python(py: Python<'_>, scalar: Scalar) -> PyResult> { .call1((0, seconds, micros))?; epoch(py)?.add(delta)?.unbind() } - Scalar::Decimal { mantissa, scale } => { + &Scalar::Decimal { mantissa, scale } => { // Decimal((sign, digits, exponent)), so that the scale is kept: - // Decimal("1.50") is not Decimal("1.5") to Python. + // Decimal("1.50") is not Decimal("1.5") to Python. The digits + // are plaintext, and both copies are wiped when dropped. let sign = u8::from(mantissa < 0); - let digits: Vec = mantissa - .unsigned_abs() - .to_string() - .bytes() - .map(|b| b - b'0') - .collect(); + let text = Zeroizing::new(mantissa.unsigned_abs().to_string()); + let digits = Zeroizing::new(text.bytes().map(|b| b - b'0').collect::>()); // A tuple of ints: pyo3 converts a Vec to bytes, which // Decimal refuses. - let digits = PyTuple::new(py, digits)?; + let digits = PyTuple::new(py, digits.iter())?; PyKind::Decimal .type_object(py)? .call1(((sign, digits, -i64::from(scale)),))? diff --git a/languages/python/tests_deterministic/test_go_rows.py b/languages/python/tests_deterministic/test_go_rows.py index faa05869e..b01e07d2a 100644 --- a/languages/python/tests_deterministic/test_go_rows.py +++ b/languages/python/tests_deterministic/test_go_rows.py @@ -24,12 +24,12 @@ import decimal import json import struct +from pathlib import Path from typing import Annotated import pytest import stackencrypt -from conftest import FIXTURES from stackencrypt import ( Bigint, BigintEq, @@ -122,8 +122,13 @@ class Ledger: amount: Annotated[decimal.Decimal, NumericEq] +# The Go SDK's EQL fixtures. Not imported from conftest.py: `from conftest` +# resolves to whichever conftest.py is first on sys.path. +FIXTURES = Path(__file__).resolve().parents[3] / "packages/eql/tests/encryption/fixtures" + # (fixture, model, the row ids Python's types can hold). Ledger rows 1 and 2 -# hold years past Python's 1 to 9999 and nanosecond timestamps. +# hold years past Python's 1 to 9999 and nanosecond timestamps: the tests +# at the end check each refusal on its own. CASES = [ ("stored_v4_rows_v1.json", Profile, [1, 2]), ("v4_ordered_rows.json", Ranking, [1, 2, 3]), @@ -241,11 +246,50 @@ def test_every_case_compares_terms(): assert terms == {"hm", "op", "bf"} +@stackencrypt.model("profiles") +@dataclasses.dataclass +class Opened: + id: Annotated[int, Passthrough] + opened: Annotated[datetime.date, Date] + + +@stackencrypt.model("profiles") +@dataclasses.dataclass +class Created: + id: Annotated[int, Passthrough] + created: Annotated[datetime.datetime, Timestamp] + + @pytest.mark.parametrize("row_id", [1, 2]) -def test_a_go_row_python_cannot_hold_is_refused_not_rounded(row_id): +def test_a_date_outside_pythons_years_is_refused(row_id): + # -262143-01-01 and +262142-12-31: datetime holds years 1 to 9999. [row] = rows_of("v4_ledger_rows.json", [row_id]) with pytest.raises(DecryptError) as raised: - Cipher(Ledger).decrypt([stored_row(Ledger, row)]) + Cipher(Opened).decrypt([stored_row(Opened, row)]) assert raised.value.code == "decrypt_not_representable" + assert raised.value.field == "opened" + + +@pytest.mark.parametrize("row_id", [1, 2]) +def test_a_timestamp_with_nanoseconds_is_refused_not_rounded(row_id): + # 1969-12-31T23:59:59.999999999Z and 2024-02-29T12:34:56.123456789Z: + # years datetime holds, with nanoseconds it does not. Rounded, a value + # would seal to another equality term than the one Go stored. + [row] = rows_of("v4_ledger_rows.json", [row_id]) + assert row["plaintext"]["created"].endswith("9Z") + + with pytest.raises(DecryptError) as raised: + Cipher(Created).decrypt([stored_row(Created, row)]) + + assert raised.value.code == "decrypt_not_representable" + assert raised.value.field == "created" + + +def test_a_timestamp_in_whole_microseconds_opens(): + [row] = rows_of("v4_ledger_rows.json", [3]) + + [created] = Cipher(Created).decrypt([stored_row(Created, row)]) + + assert created.created == datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) From 5f2c5029658c24658d61f87091634bf7b4a3ecc8 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Tue, 3 Nov 2026 17:34:01 +0900 Subject: [PATCH 3/5] fix(python): wipe opened numbers, and close the switch race (#1393) A Scalar now wipes every variant when dropped, not only text and bytes: integers, floats, dates, timestamps and decimal mantissas sit inline in a row's buffer. decrypt also stops moving each value out of its row. A moved value left its bytes in the old slot, and the buffer was freed with them. to_python now borrows the value, so each one is wiped where it lies. The hygiene test now watches a decimal's mantissa as the engine's i128, and fails without either fix. decrypt_not_representable now names the one cause that applies: a year outside 1 to 9999, or a timestamp with nanoseconds. In the test build, shared() checks for the deterministic client again after it takes the build lock. A thread that waited for the lock during the switch built a live client. A --features flag on maturin's command line replaces the features in pyproject.toml. test:deterministic now names pyo3/extension-module again. Maturin 1.10 and later set PYO3_BUILD_EXTENSION_MODULE, but pyproject.toml allows 1.9, where the test build linked libpython. Co-Authored-By: Claude Opus 5.5 --- languages/python/crates/core/src/client.rs | 38 ++++++++++++------- languages/python/crates/core/src/values.rs | 15 ++++++-- languages/python/crates/engine/src/decrypt.rs | 24 ++++++++---- languages/python/crates/engine/src/encrypt.rs | 2 +- .../python/crates/engine/src/heap_hygiene.rs | 21 +++++++--- languages/python/crates/engine/src/values.rs | 11 +++--- languages/python/mise.toml | 5 ++- languages/python/pyproject.toml | 5 ++- .../tests_deterministic/test_go_rows.py | 2 + 9 files changed, 85 insertions(+), 38 deletions(-) diff --git a/languages/python/crates/core/src/client.rs b/languages/python/crates/core/src/client.rs index f99045a58..d749cc6e6 100644 --- a/languages/python/crates/core/src/client.rs +++ b/languages/python/crates/core/src/client.rs @@ -55,27 +55,18 @@ static REQUESTS: AtomicU64 = AtomicU64::new(0); /// client key or the ZeroKMS client (`stack_kms_*`), or the keyset load /// (`stack_encrypt_*`). Nothing is cached on failure. pub fn shared() -> Result<&'static Client, Refusal> { - // In the test build, an operation that reaches here after the switch has - // not checked for the deterministic client, and would send requests to - // ZeroKMS from a test process. Refuse it, so a new operation that skips - // the check fails its test instead. #[cfg(feature = "deterministic-kms")] - if DETERMINISTIC.get().is_some() { - return Err(Refusal { - code: "stack_test_build".to_owned(), - message: "this operation asked for the live client after the switch to the \ - deterministic one; call client::deterministic() first" - .to_owned(), - field: None, - reason: None, - }); - } + refuse_after_switch()?; if let Some(client) = CLIENT.get() { return Ok(client); } let _building = BUILDING .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); + // Again under the lock: a switch can finish while this thread waits for + // it, and the live client must then not be built. + #[cfg(feature = "deterministic-kms")] + refuse_after_switch()?; if let Some(client) = CLIENT.get() { return Ok(client); } @@ -94,6 +85,25 @@ pub type Deterministic = ( #[cfg(feature = "deterministic-kms")] static DETERMINISTIC: OnceLock = OnceLock::new(); +/// In the test build, an operation that asks for the live client after the +/// switch has not checked for the deterministic client, and would send +/// requests to ZeroKMS from a test process. Refuse it, so a new operation +/// that skips the check fails its test instead. +#[cfg(feature = "deterministic-kms")] +fn refuse_after_switch() -> Result<(), Refusal> { + if DETERMINISTIC.get().is_none() { + return Ok(()); + } + Err(Refusal { + code: "stack_test_build".to_owned(), + message: "this operation asked for the live client after the switch to the \ + deterministic one; call client::deterministic() first" + .to_owned(), + field: None, + reason: None, + }) +} + /// Make every later operation in the process use a client whose keys come /// from `seed` through `DeterministicSource`, not from ZeroKMS. For the /// `deterministic-kms` test build only: rows sealed under the same seed open diff --git a/languages/python/crates/core/src/values.rs b/languages/python/crates/core/src/values.rs index c7dd86297..c00bc8f39 100644 --- a/languages/python/crates/core/src/values.rs +++ b/languages/python/crates/core/src/values.rs @@ -249,15 +249,24 @@ pub enum Scalar { }, } -/// An opened text or bytes value is plaintext, and the SDK's own copy of it: -/// it is wiped when dropped. The extension copies it into a Python object +/// An opened value is plaintext, and the SDK's own copy of it: it is wiped +/// when dropped, the inline numbers in a row's buffer as well as the text +/// and bytes on the heap. The extension copies it into a Python object /// first (`src/heap_hygiene.rs` in the engine crate checks the drop). impl Drop for Scalar { fn drop(&mut self) { match self { + Self::Null => {} Self::Text(text) => text.zeroize(), Self::Bytes(bytes) => bytes.zeroize(), - _ => {} + Self::Int(int) | Self::Timestamp(int) => int.zeroize(), + Self::Float(float) => float.zeroize(), + Self::Bool(bool) => bool.zeroize(), + Self::Date(days) => days.zeroize(), + Self::Decimal { mantissa, scale } => { + mantissa.zeroize(); + scale.zeroize(); + } } } } diff --git a/languages/python/crates/engine/src/decrypt.rs b/languages/python/crates/engine/src/decrypt.rs index 7bcb262d8..bd1fbc3c9 100644 --- a/languages/python/crates/engine/src/decrypt.rs +++ b/languages/python/crates/engine/src/decrypt.rs @@ -43,9 +43,10 @@ pub fn decrypt<'py>( .detach(|| core::decrypt(&plan.plan, stored)) .map_err(|refused| raise(py, refused))?; // Every value first, then every instance: a value Python cannot hold - // fails the call before any constructor runs. + // fails the call before any constructor runs. Borrowed, not moved: each + // value is wiped in its row's buffer when `opened` drops. let mut arguments = Vec::with_capacity(opened.len()); - for values in opened { + for values in &opened { let kwargs = PyDict::new(py); for (field, value) in plan.plan.fields().iter().zip(values) { kwargs.set_item(&field.name, python_value(py, field, value)?)?; @@ -202,10 +203,20 @@ fn passthrough( } /// An opened value as the Python object of its field. -fn python_value(py: Python<'_>, field: &Field, value: Scalar) -> PyResult> { - let ranged = matches!(value, Scalar::Date(_) | Scalar::Timestamp(_)); +fn python_value(py: Python<'_>, field: &Field, value: &Scalar) -> PyResult> { + // What a datetime refuses, decided here: `to_python` raises Python's own + // error, which names neither cause in a form the user can act on. + let cause = match value { + Scalar::Timestamp(nanos) if nanos.rem_euclid(1_000) != 0 => { + "it has nanoseconds, and Python's datetime keeps only microseconds" + } + Scalar::Date(_) | Scalar::Timestamp(_) => { + "its year is outside 1 to 9999, the years Python's datetime holds" + } + _ => "", + }; to_python(py, value).map_err(|error| { - if !ranged { + if cause.is_empty() { return error; } raise( @@ -213,8 +224,7 @@ fn python_value(py: Python<'_>, field: &Field, value: Scalar) -> PyResult( for (field, column) in plan.plan.fields().iter().zip(columns) { let value = match column { Column::Eql(json) => json.into_pyobject(py)?.into_any().unbind(), - Column::Passthrough(scalar) => to_python(py, scalar)?, + Column::Passthrough(scalar) => to_python(py, &scalar)?, }; row.set_item(&field.name, value)?; } diff --git a/languages/python/crates/engine/src/heap_hygiene.rs b/languages/python/crates/engine/src/heap_hygiene.rs index b03a146c8..f1b8a7189 100644 --- a/languages/python/crates/engine/src/heap_hygiene.rs +++ b/languages/python/crates/engine/src/heap_hygiene.rs @@ -6,7 +6,7 @@ //! seal it, derive a query term, open a row, and turn an opened value into a //! Python object. Text and decimals are watched: the text by a marker it //! contains, a decimal by its digits, as text and as the digit values a -//! `Decimal`'s tuple holds. Python's own objects are not watched. +//! `Decimal`'s tuple holds, and by its mantissa as the engine's `i128`. Python's own objects are not watched. //! CPython allocates them with its own allocator, and the caller owns them, so //! the SDK makes no promise about them. This includes the `bytes` object that //! pyo3 makes on its way to a `str`'s UTF-8 under `abi3-py39`. @@ -55,8 +55,17 @@ const DIGIT_VALUES: &[u8] = &[ 8, 6, 7, 5, 3, 0, 9, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9, 7, 9, ]; +/// The decimal's mantissa as the engine holds it: an `i128`, little-endian, +/// to its last byte that is not zero. +const MANTISSA: [u8; 9] = { + let bytes = 867_530_914_159_265_358_979_i128.to_le_bytes(); + [ + bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], bytes[8], + ] +}; + /// Every needle a freed block must not contain. -const NEEDLES: [&[u8]; 3] = [MARK, DIGITS, DIGIT_VALUES]; +const NEEDLES: [&[u8]; 4] = [MARK, DIGITS, DIGIT_VALUES, &MANTISSA]; thread_local! { static WATCHING: Cell = const { Cell::new(false) }; @@ -357,11 +366,13 @@ fn decrypt_frees_no_copy_of_the_plaintext_unwiped() { .unwrap(); // Each value as the Python object `decrypt` builds the instance // from. The objects are the caller's once returned. - opened - .into_iter() + let objects = opened + .iter() .flatten() .map(|scalar| to_python(py, scalar).unwrap()) - .collect::>() + .collect::>(); + drop(opened); + objects }); assert_eq!(opened.len(), 6); diff --git a/languages/python/crates/engine/src/values.rs b/languages/python/crates/engine/src/values.rs index dfb68f1e6..f99f56b81 100644 --- a/languages/python/crates/engine/src/values.rs +++ b/languages/python/crates/engine/src/values.rs @@ -320,16 +320,17 @@ fn type_name(value: &Bound<'_, PyAny>) -> String { /// An opened scalar as a Python object, of the type the field declares. /// -/// The scalar's text and bytes are copied into the Python object, and the -/// scalar wipes its own copy when it is dropped. +/// The scalar is borrowed and copied into the Python object. It wipes its +/// own copy when dropped, so the caller drops it where it lies: a scalar +/// moved out of a row's buffer leaves its bytes behind in the buffer. /// /// # Errors /// /// A `ValueError` for a date or timestamp outside `datetime`'s range, and /// for a timestamp with nanoseconds: `datetime` keeps microseconds, and /// rounding would change the value. The Go SDK can store one. -pub fn to_python(py: Python<'_>, scalar: Scalar) -> PyResult> { - Ok(match &scalar { +pub fn to_python(py: Python<'_>, scalar: &Scalar) -> PyResult> { + Ok(match scalar { Scalar::Null => py.None(), Scalar::Text(text) => PyString::new(py, text).into_any().unbind(), Scalar::Int(i) => i.into_pyobject(py)?.into_any().unbind(), @@ -384,7 +385,7 @@ mod tests { let decimal = py.import("decimal").unwrap().getattr("Decimal").unwrap(); for (mantissa, scale, text) in [(1_995, 2, "19.95"), (-150, 2, "-1.50"), (0, 2, "0.00")] { - let opened = to_python(py, Scalar::Decimal { mantissa, scale }).unwrap(); + let opened = to_python(py, &Scalar::Decimal { mantissa, scale }).unwrap(); let opened = opened.bind(py); assert!(opened.get_type().is(&decimal), "{text}"); assert_eq!(opened.str().unwrap().to_string(), text); diff --git a/languages/python/mise.toml b/languages/python/mise.toml index 7e6c999d2..457aad949 100644 --- a/languages/python/mise.toml +++ b/languages/python/mise.toml @@ -59,9 +59,12 @@ shell = "bash -c" # The deterministic client replaces the live one for the whole process, so # these tests run in their own pytest process, from their own folder. The # build is a test build: run `mise run build` afterwards for a normal one. +# `--features` replaces pyproject.toml's list, so it names +# pyo3/extension-module again. Maturin 1.10 and later also set +# PYO3_BUILD_EXTENSION_MODULE, but pyproject.toml allows 1.9. run = """ set -euo pipefail -MATURIN_PEP517_ARGS="--profile dev --features deterministic-kms" mise run build +MATURIN_PEP517_ARGS="--profile dev --features pyo3/extension-module,deterministic-kms" mise run build .venv/bin/python -m pytest tests_deterministic """ diff --git a/languages/python/pyproject.toml b/languages/python/pyproject.toml index b32553992..836e39bd5 100644 --- a/languages/python/pyproject.toml +++ b/languages/python/pyproject.toml @@ -28,8 +28,9 @@ module-name = "stackencrypt._engine" python-source = "python" # The engine crate, not the workspace root: maturin builds one cdylib. manifest-path = "crates/engine/Cargo.toml" -# The one place `extension-module` is set. See crates/engine/Cargo.toml for -# why it is not in the crate manifest. +# Never set in the crate manifest: see crates/engine/Cargo.toml for why. A +# build that passes `--features` replaces this list, and names it again: +# the `test:deterministic` task in mise.toml. features = ["pyo3/extension-module"] [tool.pytest.ini_options] diff --git a/languages/python/tests_deterministic/test_go_rows.py b/languages/python/tests_deterministic/test_go_rows.py index b01e07d2a..e6cd2d7d5 100644 --- a/languages/python/tests_deterministic/test_go_rows.py +++ b/languages/python/tests_deterministic/test_go_rows.py @@ -270,6 +270,7 @@ def test_a_date_outside_pythons_years_is_refused(row_id): assert raised.value.code == "decrypt_not_representable" assert raised.value.field == "opened" + assert "year" in raised.value.message @pytest.mark.parametrize("row_id", [1, 2]) @@ -285,6 +286,7 @@ def test_a_timestamp_with_nanoseconds_is_refused_not_rounded(row_id): assert raised.value.code == "decrypt_not_representable" assert raised.value.field == "created" + assert "nanoseconds" in raised.value.message def test_a_timestamp_in_whole_microseconds_opens(): From db760af94d01ba6c0c33c812899bc46219f75535 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Tue, 3 Nov 2026 17:52:08 +0900 Subject: [PATCH 4/5] fix(python): one client switch, and no plaintext moved out of a row (#1393) encrypt, decrypt and query each wrote their call twice: once for the deterministic test client, once for the live one. client::run! now takes the call once and compiles it for both, so the two cannot drift. The clients have different key source types, so a function that returned the current client would still need two calls. encrypt now borrows each column, as decrypt does. A moved passthrough value left its bytes in the row's buffer when the buffer was freed. A decrypted row is now written into one buffer and moved whole. It was a Vec> collected into a Vec, which left no plaintext only because rustc reused the buffer in place. Co-Authored-By: Claude Opus 5.5 --- languages/python/crates/core/src/client.rs | 29 ++++++++++++---- languages/python/crates/core/src/decrypt.rs | 33 ++++++++++--------- languages/python/crates/core/src/encrypt.rs | 7 +--- languages/python/crates/core/src/query.rs | 13 +------- languages/python/crates/engine/src/encrypt.rs | 10 +++--- 5 files changed, 48 insertions(+), 44 deletions(-) diff --git a/languages/python/crates/core/src/client.rs b/languages/python/crates/core/src/client.rs index d749cc6e6..921a3b121 100644 --- a/languages/python/crates/core/src/client.rs +++ b/languages/python/crates/core/src/client.rs @@ -23,7 +23,7 @@ use stack_encrypt::kms::{ HttpConnectionOpts, KeyProvider, KeyProviderError, ProfileStore, StackKms, StackKmsBuilderError, ZeroKMSConnection, ZeroKMSConnectionInit, ZeroKmsEndpoint, }; -use stack_encrypt::{KeysetCipher, StackCipher}; +use stack_encrypt::StackCipher; use zerokms_protocol::{ViturRequest, ViturRequestError}; use crate::credentials; @@ -75,6 +75,28 @@ pub fn shared() -> Result<&'static Client, Refusal> { Ok(CLIENT.get_or_init(|| client)) } +/// Run an operation's future on the process's client, and return its result. +/// +/// `run!(|cipher| future)` names the `StackCipher` the body builds its future +/// from. The body is written once and compiled for each client: the live +/// one, and in the test build the deterministic one, which takes its place +/// after the switch. The two have different key source types, so a function +/// that returned "the current client" would still leave each operation to +/// write a branch per client. +macro_rules! run { + (|$cipher:ident| $future:expr) => {{ + #[cfg(feature = "deterministic-kms")] + if let Some((cipher, runtime)) = $crate::client::deterministic() { + let $cipher = cipher; + return runtime.block_on($future); + } + let client = $crate::client::shared()?; + let $cipher = client.cipher(); + client.block_on($future) + }}; +} +pub(crate) use run; + /// The client of the `deterministic-kms` test build, with its own runtime. #[cfg(feature = "deterministic-kms")] pub type Deterministic = ( @@ -166,11 +188,6 @@ pub fn request_count() -> u64 { } impl Client { - /// The cipher bound to the client's default keyset. - pub fn keyset(&self) -> KeysetCipher<'_, Kms> { - self.cipher.default_keyset() - } - /// The cipher itself. pub fn cipher(&self) -> &StackCipher { &self.cipher diff --git a/languages/python/crates/core/src/decrypt.rs b/languages/python/crates/core/src/decrypt.rs index e8d0bfa39..cf7931c28 100644 --- a/languages/python/crates/core/src/decrypt.rs +++ b/languages/python/crates/core/src/decrypt.rs @@ -177,12 +177,7 @@ fn refusal(code: &str, field: &Field, message: String) -> Refusal { /// `decrypt_kind_mismatch` for a value stored as another kind than the /// field's. pub fn decrypt(plan: &ModelPlan, rows: Rows) -> Result>, Refusal> { - #[cfg(feature = "deterministic-kms")] - if let Some((cipher, runtime)) = client::deterministic() { - return runtime.block_on(decrypt_with(cipher, plan, rows)); - } - let client = client::shared()?; - client.block_on(decrypt_with(client.cipher(), plan, rows)) + client::run!(|cipher| decrypt_with(cipher, plan, rows)) } /// [`decrypt`] on `cipher`. Each value opens under the keyset that sealed @@ -348,7 +343,8 @@ impl<'a> Sink for OpenedRow<'a> { fn map(self, _: Option) -> Result, ReadError> { Ok(OpenedFields { - values: vec![None; self.fields.len()], + values: vec![Scalar::Null; self.fields.len()], + seen: vec![false; self.fields.len()], fields: self.fields, rows: self.rows, }) @@ -359,9 +355,13 @@ impl<'a> Sink for OpenedRow<'a> { } } +/// One row as it opens. Each value is written into its slot, and `end` +/// moves the whole buffer into the rows: no value is moved out of a buffer +/// that is then freed with its bytes still in it. struct OpenedFields<'a> { fields: &'a [Field], - values: Vec>, + values: Vec, + seen: Vec, rows: &'a mut Vec>, } @@ -381,30 +381,31 @@ impl MapSink for OpenedFields<'_> { .ok_or(ReadError("a field the plan does not have"))?; Ok(OpenedValue { slot: &mut self.values[at], + seen: &mut self.seen[at], }) } fn end(self) -> Result<(), ReadError> { - let row = self - .values - .into_iter() - .collect::>>() - .ok_or(ReadError("a row without one of the plan's fields"))?; - self.rows.push(row); + if !self.seen.iter().all(|&seen| seen) { + return Err(ReadError("a row without one of the plan's fields")); + } + self.rows.push(self.values); Ok(()) } } /// One opened field: one scalar. struct OpenedValue<'a> { - slot: &'a mut Option, + slot: &'a mut Scalar, + seen: &'a mut bool, } impl OpenedValue<'_> { fn put(self, scalar: Scalar) -> Result<(), ReadError> { - if self.slot.replace(scalar).is_some() { + if std::mem::replace(self.seen, true) { return not_expected("a field twice in one row"); } + *self.slot = scalar; Ok(()) } } diff --git a/languages/python/crates/core/src/encrypt.rs b/languages/python/crates/core/src/encrypt.rs index 7ee652dc5..7c51b70e0 100644 --- a/languages/python/crates/core/src/encrypt.rs +++ b/languages/python/crates/core/src/encrypt.rs @@ -106,12 +106,7 @@ pub fn encrypt( source: ValueFrame, rows: usize, ) -> Result>, Refusal> { - #[cfg(feature = "deterministic-kms")] - if let Some((cipher, runtime)) = client::deterministic() { - return runtime.block_on(encrypt_with(&cipher.default_keyset(), plan, source, rows)); - } - let client = client::shared()?; - client.block_on(encrypt_with(&client.keyset(), plan, source, rows)) + client::run!(|cipher| encrypt_with(&cipher.default_keyset(), plan, source, rows)) } /// [`encrypt`] under `keyset`. diff --git a/languages/python/crates/core/src/query.rs b/languages/python/crates/core/src/query.rs index 9d7b22df1..818596bf5 100644 --- a/languages/python/crates/core/src/query.rs +++ b/languages/python/crates/core/src/query.rs @@ -35,18 +35,7 @@ pub fn query( value: Value<'_>, ) -> Result { answers_equality(plan, field)?; - #[cfg(feature = "deterministic-kms")] - if let Some((cipher, runtime)) = client::deterministic() { - return runtime.block_on(query_with( - &cipher.default_keyset(), - plan, - field, - kind, - value, - )); - } - let client = client::shared()?; - client.block_on(query_with(&client.keyset(), plan, field, kind, value)) + client::run!(|cipher| query_with(&cipher.default_keyset(), plan, field, kind, value)) } /// [`query`] under `keyset`. diff --git a/languages/python/crates/engine/src/encrypt.rs b/languages/python/crates/engine/src/encrypt.rs index dc17f96e2..339d42a54 100644 --- a/languages/python/crates/engine/src/encrypt.rs +++ b/languages/python/crates/engine/src/encrypt.rs @@ -11,7 +11,7 @@ use std::cell::RefCell; use pyo3::prelude::*; -use pyo3::types::{PyDict, PyList, PyType}; +use pyo3::types::{PyDict, PyList, PyString, PyType}; use stackencrypt_core::encrypt::plaintext::{Encrypt, MapSink, SeqSink, Sink, SourceError}; use stackencrypt_core::encrypt::{self as core, Column, Field, ModelPlan, ValueFrame}; use stackencrypt_core::refusal::Refusal; @@ -112,12 +112,14 @@ pub fn encrypt<'py>( .detach(|| core::encrypt(&plan.plan, frame, rows)) .map_err(|refused| raise(py, refused))?; let out = PyList::empty(py); - for columns in encrypted { + // Borrowed, not moved: a passthrough value is wiped in its row's buffer + // when `encrypted` drops. + for columns in &encrypted { let row = PyDict::new(py); for (field, column) in plan.plan.fields().iter().zip(columns) { let value = match column { - Column::Eql(json) => json.into_pyobject(py)?.into_any().unbind(), - Column::Passthrough(scalar) => to_python(py, &scalar)?, + Column::Eql(json) => PyString::new(py, json).into_any().unbind(), + Column::Passthrough(scalar) => to_python(py, scalar)?, }; row.set_item(&field.name, value)?; } From 9ea5ac81d10315831df9febbc0a006b25d9eb1d9 Mon Sep 17 00:00:00 2001 From: Yuji Yokoo Date: Tue, 3 Nov 2026 19:08:15 +0900 Subject: [PATCH 5/5] fix(python): take the datetime refusal's cause from to_python (#1393) decrypt chose the cause of decrypt_not_representable from the value before to_python ran, with its own copy of the nanosecond rule. Any other Python error on a date or timestamp was reported as a year out of range, and the real error was lost. to_python now returns the cause itself, as Unfit::Nanoseconds or Unfit::Year. A year is reported only for datetime's own range errors: ValueError from fromordinal, OverflowError from timedelta or the addition. Every other error is Python's own and is raised as it is. The comment in crates/engine/Cargo.toml now names both places that turn on extension-module: pyproject.toml, and the test:deterministic task. Co-Authored-By: Claude Opus 5.5 --- languages/python/crates/engine/Cargo.toml | 4 +- languages/python/crates/engine/src/decrypt.rs | 21 +---- languages/python/crates/engine/src/encrypt.rs | 6 +- .../python/crates/engine/src/heap_hygiene.rs | 2 +- languages/python/crates/engine/src/values.rs | 83 ++++++++++++++----- 5 files changed, 77 insertions(+), 39 deletions(-) diff --git a/languages/python/crates/engine/Cargo.toml b/languages/python/crates/engine/Cargo.toml index c04d1529a..2548ad639 100644 --- a/languages/python/crates/engine/Cargo.toml +++ b/languages/python/crates/engine/Cargo.toml @@ -26,7 +26,9 @@ stackencrypt-core = { path = "../core" } # `extension-module` is deliberately NOT here. It tells pyo3 not to link # libpython, which is right for the wheel and wrong for `cargo test`: a test # binary has to link it or fails with undefined `_Py*` symbols. maturin turns -# the feature on through `[tool.maturin].features` in pyproject.toml. +# the feature on in two places. One is `[tool.maturin].features` in +# pyproject.toml. The other is the `test:deterministic` task in mise.toml: a +# `--features` flag replaces pyproject.toml's list, so the task names it again. pyo3 = { version = "0.29", features = ["abi3-py39"] } # To wipe the owned copy of a `str` the extension reads (src/values.rs, # `Read`). The version the workspace already resolves. diff --git a/languages/python/crates/engine/src/decrypt.rs b/languages/python/crates/engine/src/decrypt.rs index bd1fbc3c9..843b5f7f3 100644 --- a/languages/python/crates/engine/src/decrypt.rs +++ b/languages/python/crates/engine/src/decrypt.rs @@ -204,28 +204,15 @@ fn passthrough( /// An opened value as the Python object of its field. fn python_value(py: Python<'_>, field: &Field, value: &Scalar) -> PyResult> { - // What a datetime refuses, decided here: `to_python` raises Python's own - // error, which names neither cause in a form the user can act on. - let cause = match value { - Scalar::Timestamp(nanos) if nanos.rem_euclid(1_000) != 0 => { - "it has nanoseconds, and Python's datetime keeps only microseconds" - } - Scalar::Date(_) | Scalar::Timestamp(_) => { - "its year is outside 1 to 9999, the years Python's datetime holds" - } - _ => "", - }; - to_python(py, value).map_err(|error| { - if cause.is_empty() { - return error; - } + to_python(py, value)?.map_err(|unfit| { raise( py, Refusal { code: "decrypt_not_representable".to_owned(), message: format!( - "field {:?}: the opened value does not fit its Python type: {cause}.", - field.name + "field {:?}: the opened value does not fit its Python type: {}.", + field.name, + unfit.cause() ), field: Some(field.name.clone()), reason: None, diff --git a/languages/python/crates/engine/src/encrypt.rs b/languages/python/crates/engine/src/encrypt.rs index 339d42a54..d5571dda0 100644 --- a/languages/python/crates/engine/src/encrypt.rs +++ b/languages/python/crates/engine/src/encrypt.rs @@ -10,6 +10,7 @@ use std::cell::RefCell; +use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::{PyDict, PyList, PyString, PyType}; use stackencrypt_core::encrypt::plaintext::{Encrypt, MapSink, SeqSink, Sink, SourceError}; @@ -119,7 +120,10 @@ pub fn encrypt<'py>( for (field, column) in plan.plan.fields().iter().zip(columns) { let value = match column { Column::Eql(json) => PyString::new(py, json).into_any().unbind(), - Column::Passthrough(scalar) => to_python(py, scalar)?, + // A passthrough value came from Python, so it fits. + Column::Passthrough(scalar) => { + to_python(py, scalar)?.map_err(|unfit| PyValueError::new_err(unfit.cause()))? + } }; row.set_item(&field.name, value)?; } diff --git a/languages/python/crates/engine/src/heap_hygiene.rs b/languages/python/crates/engine/src/heap_hygiene.rs index f1b8a7189..b7bf4c5c6 100644 --- a/languages/python/crates/engine/src/heap_hygiene.rs +++ b/languages/python/crates/engine/src/heap_hygiene.rs @@ -369,7 +369,7 @@ fn decrypt_frees_no_copy_of_the_plaintext_unwiped() { let objects = opened .iter() .flatten() - .map(|scalar| to_python(py, scalar).unwrap()) + .map(|scalar| to_python(py, scalar).unwrap().unwrap()) .collect::>(); drop(opened); objects diff --git a/languages/python/crates/engine/src/values.rs b/languages/python/crates/engine/src/values.rs index f99f56b81..4d24fec7e 100644 --- a/languages/python/crates/engine/src/values.rs +++ b/languages/python/crates/engine/src/values.rs @@ -318,6 +318,44 @@ fn type_name(value: &Bound<'_, PyAny>) -> String { .map_or_else(|_| "an unknown type".to_owned(), |name| name.to_string()) } +/// Why an opened date or timestamp has no Python object. The Go SDK can +/// store both. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Unfit { + /// `datetime` keeps microseconds, and rounding would change the value. + Nanoseconds, + /// `datetime` holds the years 1 to 9999. + Year, +} + +impl Unfit { + /// The cause, for a message. + pub fn cause(self) -> &'static str { + match self { + Self::Nanoseconds => { + "it has nanoseconds, and Python's datetime keeps only microseconds" + } + Self::Year => "its year is outside 1 to 9999, the years Python's datetime holds", + } + } +} + +/// `datetime`'s refusal of a value out of its range as [`Unfit::Year`]. +/// `fromordinal` raises `ValueError` for it, and `timedelta` and the +/// addition raise `OverflowError`. Any other error is Python's own. +fn in_range(py: Python<'_>, result: PyResult) -> PyResult> { + match result { + Ok(value) => Ok(Ok(value)), + Err(error) + if error.is_instance_of::(py) + || error.is_instance_of::(py) => + { + Ok(Err(Unfit::Year)) + } + Err(error) => Err(error), + } +} + /// An opened scalar as a Python object, of the type the field declares. /// /// The scalar is borrowed and copied into the Python object. It wipes its @@ -326,35 +364,40 @@ fn type_name(value: &Bound<'_, PyAny>) -> String { /// /// # Errors /// -/// A `ValueError` for a date or timestamp outside `datetime`'s range, and -/// for a timestamp with nanoseconds: `datetime` keeps microseconds, and -/// rounding would change the value. The Go SDK can store one. -pub fn to_python(py: Python<'_>, scalar: &Scalar) -> PyResult> { - Ok(match scalar { +/// The outer error is Python's own. The inner one is a date or timestamp +/// that `datetime` cannot hold, with the reason. +pub fn to_python(py: Python<'_>, scalar: &Scalar) -> PyResult, Unfit>> { + Ok(Ok(match scalar { Scalar::Null => py.None(), Scalar::Text(text) => PyString::new(py, text).into_any().unbind(), Scalar::Int(i) => i.into_pyobject(py)?.into_any().unbind(), Scalar::Float(f) => PyFloat::new(py, *f).into_any().unbind(), Scalar::Bool(b) => PyBool::new(py, *b).to_owned().into_any().unbind(), Scalar::Bytes(bytes) => PyBytes::new(py, bytes).into_any().unbind(), - Scalar::Date(days) => PyKind::Date - .type_object(py)? - .call_method1("fromordinal", (i64::from(*days) + 719_163,))? - .unbind(), + Scalar::Date(days) => { + let date = PyKind::Date.type_object(py)?; + let opened = date.call_method1("fromordinal", (i64::from(*days) + 719_163,)); + match in_range(py, opened)? { + Ok(date) => date.unbind(), + Err(unfit) => return Ok(Err(unfit)), + } + } &Scalar::Timestamp(nanos) => { if nanos.rem_euclid(1_000) != 0 { - return Err(pyo3::exceptions::PyValueError::new_err( - "the timestamp has nanoseconds, and datetime keeps microseconds", - )); + return Ok(Err(Unfit::Nanoseconds)); } let micros = nanos.div_euclid(1_000); let seconds = micros.div_euclid(1_000_000); let micros = micros.rem_euclid(1_000_000); - let delta = py - .import("datetime")? - .getattr("timedelta")? - .call1((0, seconds, micros))?; - epoch(py)?.add(delta)?.unbind() + let timedelta = py.import("datetime")?.getattr("timedelta")?; + let epoch = epoch(py)?; + let opened = timedelta + .call1((0, seconds, micros)) + .and_then(|delta| epoch.add(delta)); + match in_range(py, opened)? { + Ok(timestamp) => timestamp.unbind(), + Err(unfit) => return Ok(Err(unfit)), + } } &Scalar::Decimal { mantissa, scale } => { // Decimal((sign, digits, exponent)), so that the scale is kept: @@ -371,7 +414,7 @@ pub fn to_python(py: Python<'_>, scalar: &Scalar) -> PyResult> { .call1(((sign, digits, -i64::from(scale)),))? .unbind() } - }) + })) } #[cfg(test)] @@ -385,7 +428,9 @@ mod tests { let decimal = py.import("decimal").unwrap().getattr("Decimal").unwrap(); for (mantissa, scale, text) in [(1_995, 2, "19.95"), (-150, 2, "-1.50"), (0, 2, "0.00")] { - let opened = to_python(py, &Scalar::Decimal { mantissa, scale }).unwrap(); + let opened = to_python(py, &Scalar::Decimal { mantissa, scale }) + .unwrap() + .unwrap(); let opened = opened.bind(py); assert!(opened.get_type().is(&decimal), "{text}"); assert_eq!(opened.str().unwrap().to_string(), text);