diff --git a/.github/workflows/tests-python.yml b/.github/workflows/tests-python.yml index 54d4ad4da..7331960a7 100644 --- a/.github/workflows/tests-python.yml +++ b/.github/workflows/tests-python.yml @@ -18,8 +18,10 @@ name: Tests (Python) # is released. The two jobs are independent, so a pytest failure does not # hide whether the wheels build. # -# No job here holds a CipherStash credential. The live test job (credentials, -# Postgres) arrives with the first `encrypt` ticket, #1389, as its own job. +# A third job, `live`, runs the tests that need CipherStash credentials and a +# PostgreSQL with the EQL v4 bundle (#1389). It is the only job with a +# credential. It runs on pushes and on pull requests from this repository, +# and skips on a fork's pull request, which GitHub gives no secrets. on: pull_request: @@ -49,6 +51,12 @@ on: # The Rust version the workspace builds under. The workspace's own # mise.toml is under languages/python/** above. - mise.toml + # The `live` job installs the EQL v4 bundle built from this SQL, as the + # Go workflow's eql-postgres job does, and runs the secrets pre-flight. + - packages/eql/src/** + - packages/eql/mise.toml + - packages/eql/tasks/** + - .github/actions/require-cs-secrets/** - .github/workflows/tests-python.yml # Unfiltered, like tests-rust.yml. Every push to main runs the suite, so # main runs a superset of what the filtered `pull_request` trigger above @@ -177,6 +185,92 @@ jobs: - name: stackencrypt._eql matches the EQL catalog run: cargo test --locked --manifest-path ../../packages/eql/Cargo.toml -p eql-codegen --test py_eql_parity + # `encrypt` against real ZeroKMS, and the row it returns inserted into a + # PostgreSQL with the EQL v4 bundle built from the tree. The live tests skip + # without credentials or a database, so the pre-flight below turns a missing + # secret into a failure, and `mise run test:live` fails on any skip. + live: + name: "Python live: encrypt into PostgreSQL with EQL v4" + # A fork's pull request has no secrets: skip it rather than fail it, as + # the integration workflows do. + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 45 + env: + CS_WORKSPACE_CRN: ${{ vars.CS_WORKSPACE_CRN }} + CS_CLIENT_ID: ${{ vars.CS_CLIENT_ID }} + CS_CLIENT_KEY: ${{ secrets.CS_CLIENT_KEY }} + CS_CLIENT_ACCESS_KEY: ${{ secrets.CS_CLIENT_ACCESS_KEY }} + services: + postgres: + image: postgres:17 + env: + POSTGRES_PASSWORD: postgres + # Container port only: Docker picks the host port, and the test step + # reads it from `job.services`. + ports: + - 5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + defaults: + run: + working-directory: languages/python + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + # Fast pre-flight: fail in seconds if a secret was rotated or cleared. + # Ordering is asserted by scripts/__tests__/ffi-binding-step-order.test.mjs. + - uses: ./.github/actions/require-cs-secrets + with: + workspace-crn: ${{ vars.CS_WORKSPACE_CRN }} + client-id: ${{ vars.CS_CLIENT_ID }} + client-key: ${{ secrets.CS_CLIENT_KEY }} + client-access-key: ${{ secrets.CS_CLIENT_ACCESS_KEY }} + + # This workspace's tools (python, uv, rust), as the `python` job. + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + install_args: python uv rust + working_directory: languages/python + cache: true + # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs + cache_key_prefix: mise-v1-ubuntu-2404 + + # EQL's tools, for the build of its bundle. EQL's mise.toml names its + # own Rust. + - uses: jdx/mise-action@1648a7812b9aeae629881980618f079932869151 # v4 + with: + version: 2026.4.0 + install: true + working_directory: packages/eql + cache: true + # Must name runs-on's image; see scripts/__tests__/workflow-mise-cache-image.test.mjs + cache_key_prefix: mise-v1-ubuntu-2404 + + # One cache per workspace: this one, and EQL's, whose eql-codegen + # writes the bundle. + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: languages/python + + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: packages/eql + + # Builds the wheel and the bundle, installs the bundle, and runs the + # live tests; a skip fails the task. + - name: Live tests + env: + EQL_TEST_DATABASE_URL: "host=localhost port=${{ job.services.postgres.ports['5432'] }} user=postgres password=postgres dbname=postgres sslmode=disable" + run: mise run test:live + # One abi3-py39 wheel per platform the FFI ships for, by the same Rust # triples. python-ci.test.mjs holds the list to the protect-ffi platform # packages. The wheels are uploaded as artifacts. Nothing publishes them: diff --git a/AGENTS.md b/AGENTS.md index cbf61e4e7..e4ef21c32 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -93,7 +93,7 @@ Every npm package except EQL lives under `languages/typescript/`: packages in `l - `packages/stack-auth`, `packages/stack-profile`, `packages/stack-kms`, `packages/stack-encrypt`, `packages/stack-encrypt-derive`, `packages/stack-guest-abi`: The Rust crates imported from `cipherstash/cipherstash-suite` with their history — `stack-auth` and `stack-profile` (published to crates.io, one version group), `stack-kms` (published to crates.io from 0.1.0, its own version group, re-exported by `stack-encrypt` as `stack_encrypt::kms`), `stack-encrypt` and `stack-encrypt-derive` (published to crates.io from 0.1.0, one version group; `eql-bindings`' `stack-encrypt` feature depends on them from the registry), and `stack-guest-abi` (`publish = false`). They are the members of the **root Cargo workspace**, with the three node binding crates below. See "Working on the Rust crates". - `languages/typescript/packages/auth`, `languages/typescript/packages/profile`, `languages/typescript/packages/stack-auth-wasm`: The node bindings of those crates. `@cipherstash/auth` (napi-rs v2) and its six `platforms/*` packages are published to npm from this repository by `release.yml` (`auth-artifacts`, `publish-auth`); a change to what it ships, the `stack-auth` crate included, needs an `@cipherstash/auth` changeset (`require-auth-npm-changeset.yml`). `@cipherstash/profile` and its platforms are private and never published; `@cipherstash/stack-auth-wasm` is private and builds the wasm that `@cipherstash/auth` ships. Their `build` and `test` scripts never invoke cargo; `build:native`, `build:debug` and `test:cargo` do. - `languages/golang`: The Go module — the SDK `encrypt` with its generated-code support `encrypt/gensupport` and the policy packages `encrypt/policy` and `encrypt/policy/protosource`; the credential package `auth`; the generator `stashgen` and its command `cmd/stashgen`; and `internal` (the shared guest plumbing and the `record` wire model). A wazero host with no cgo. Its two WASI guests (`encrypt/guest`, `auth/guest`) are detached Cargo workspaces built by `mise run wasm:guest:build` and `mise run wasm:auth-guest:build`; `mise run wasm:guest:build:deterministic` builds the seeded TEST build the hermetic Go tests use; the `.wasm` files they embed are gitignored. The stack-encrypt guest has a second build with the EQL types (`mise run wasm:guest:build:eql`, cargo feature `eql`, linking `eql-bindings` by path; `wasm:guest:build:eql:deterministic` is its test build), embedded by `encrypt/eql` and registered on import, so a program whose generated code names an EQL type runs it (ADR-0007, amended). `encrypt/eql`'s types and `Types` table are generated from the EQL catalog by `eql-codegen` (`eql_gen.go`, written by `mise run types:generate` in `packages/eql` and drift-gated by `types:check` and `cargo test -p eql-codegen`). Generated `*_stash.go` files are committed and CI fails when `go generate ./...` changes one. Releases are cut by hand with `.github/workflows/release-golang.yml` (`gh workflow run release-golang.yml -f version=X.Y.Z[-pre] --ref `): because the guests are gitignored on main and Go fetches a version as the files committed at its tag, the workflow builds them, commits them on a release-only commit no branch points at, tags it `languages/golang/vX.Y.Z` and pushes the tag alone, then attaches the EQL v4 SQL bundle to the GitHub release. The ref must carry the EQL v4 bundle, or the build stops at "Build the EQL SQL bundles". `scripts/go-release-version.mjs` refuses, before anything is built, every version Go would refuse (non-canonical semver, `+build` metadata, a v2+ major on a module path with no `/vN` suffix, a version shaped like a pseudo-version), a final version dispatched from anywhere but `main`, and a final version while `languages/golang/go.mod` pins a `github.com/cipherstash/vitaminc/…` module at a pseudo-version (vitaminc's Go bindings have no tags, and a TEMPORARY pin on an unreleased commit must be gone before the module is tagged); a pre-release may come from a branch and may carry such a pin. The build job also refuses a version the Go proxy already lists in `@v/list`, which catches a tag deleted after the proxy fetched it; it never asks the proxy for the version itself before the tag exists, because the proxy would cache "not found" for up to 30 minutes and `verify` would fail after the push. The build checks the `Cargo.lock` of the EQL workspace and both guest workspaces with `--locked` first, since none of those builds passes it. It runs in a job with no write token and hands the guests to a `publish` job that holds `contents: write` and runs only git and gh. `publish` fetches the commit with plain unauthenticated git rather than `actions/checkout`, whose `token` input defaults to the job's writable token, and refuses a release commit whose tree is not the one the build tested. `publish` takes the paths it commits from the workflow's fixed `GUESTS` list, never from the artifact, and refuses a `guests.sha256` that is not exactly one SHA-256 line for each of those three guests: the build job runs third-party code after writing that manifest, so a path read from it could put any file on the tag (`scripts/__tests__/go-release-artifact.test.mjs`). A version is immutable once the Go proxy has fetched it, so a tag is never moved or deleted: if `publish` or `verify` fails after the tag is pushed, use "Re-run failed jobs" on the same run, which keeps a tag that run pushed and creates or completes the release; a new dispatch of the same version is refused. Never tag a main commit by hand: `go get` would fetch a module with no guests. -- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. There is no Python lockfile yet. The package has no runtime dependency, so only the `test` extra could be locked: pytest, plus attrs, Pydantic and SQLAlchemy for the declaration tests. Those three have no upper bounds, so the tests meet each new release when users do. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: Python ADR-0001 under `languages/python/docs/adr/`, the spec in #1341. +- `languages/python`: The Python SDK, `stackencrypt`, a pyo3 extension over `stack-encrypt`, built by maturin. **The fourth Cargo workspace**, excluded from the root one. It has two crates. `crates/core` has no pyo3: the engine-side logic, linked and tested without a Python interpreter. `crates/engine` is the `_engine` cdylib, `abi3-py39`, one wheel per platform for Python 3.9 to 3.14. `crates/core` compiles the Go guest's EQL resolver, `languages/golang/encrypt/guest/src/targets.rs`, in place through `#[path]`. So both SDKs run one resolver, and a change to that file is a change to both. The `extension-module` feature is set only by maturin in `pyproject.toml`, never in a crate manifest, so `cargo test` links libpython. Its tasks live in its own `mise.toml` and run from that folder. `mise run build` builds the wheel into `.venv`. `mise run test` runs pytest. `mise run test:cargo` runs plain `cargo test --locked` for both crates, with libpython on the loader path. It does not use nextest, and its CI job sets no `MISE_ENV`. The nextest rule under "Working on the Rust crates" is for the root workspace, whose `stack-kms` tests set environment variables. No test in this workspace does, and nextest comes from `mise.test.toml`, which this workspace's CI does not load. The wheel's version is `crates/engine/Cargo.toml`'s. `stackencrypt.__engine__` is the commit of this checkout, recorded by `crates/core/build.rs`. Its `Cargo.lock` is seeded from the root one, so every crate it shares with the root workspace resolves to the version the root tests. After a root dependency moves, re-seed it the same way: copy the root `Cargo.lock` over it and run `cargo metadata --offline` there. That command prunes the copy to what the workspace uses. Until then, `tests-python.yml` fails on `--locked`. Nothing is published. CI is `tests-python.yml`. It runs the Rust checks and pytest on Python 3.9 and 3.14, where `MISE_PYTHON_VERSION` moves a leg off the local version. A `live` job runs `mise run test:live`: `encrypt` against ZeroKMS with the repository's `CS_*` secrets, and the rows inserted into a `postgres:17` service with the EQL v4 bundle built from the tree. It skips on a fork's pull request, and the task fails on any skipped test. It then builds one abi3 wheel per FFI platform, uploads them as artifacts, and writes each build time to the job summary. `scripts/__tests__/python-ci.test.mjs` compares its path filter with the workspace's `Cargo.lock` and its wheel matrix with the protect-ffi platform packages. There is no Python lockfile yet. The package has no runtime dependency, so only the `test` extra could be locked: pytest, attrs, Pydantic, SQLAlchemy, and psycopg. The middle three are for the declaration tests, and psycopg is for the live tests. None has an upper bound, so the tests meet each new release when users do. `python/stackencrypt/_eql.py`, the EQL type markers, is generated from the EQL catalog by `eql-codegen py-eql`. To regenerate it, run `mise run types:generate` in `packages/eql`. Never edit it by hand: `tests-python.yml` runs its parity tests and fails on a hand edit. Design: Python ADR-0001 under `languages/python/docs/adr/`, the spec in #1341. - `e2e/*`: Cross-package end-to-end tests (package managers, supply chain, Prisma example README) - `languages/typescript/examples/*`: Working apps (basic, prisma, supabase-worker) - `docs/plans/*`: Internal design plans. User-facing documentation lives at https://cipherstash.com/docs (not in this repo). diff --git a/languages/python/Cargo.lock b/languages/python/Cargo.lock index 08f59ff48..5b021c441 100644 --- a/languages/python/Cargo.lock +++ b/languages/python/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aead" version = "0.5.2" @@ -60,6 +66,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -105,6 +126,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "async-compression" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68650b7df54f0293fd061972a0fb05aaf4fc0879d3b3d21a638a182c5c543b9f" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "atomic" version = "0.6.1" @@ -114,6 +158,12 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" @@ -127,7 +177,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" dependencies = [ "aws-lc-sys", - "untrusted", + "untrusted 0.7.1", "zeroize", ] @@ -162,6 +212,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -231,6 +287,27 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e2211b0817f061502a8dd9f11a37e879e79763e3c698d2418cf824d8cb2f21e" +[[package]] +name = "brotli" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.19.1" @@ -303,12 +380,24 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chacha20" version = "0.10.2" @@ -396,6 +485,34 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00828ba6fd27b45a448e57dbfe84f1029d4c9f26b368157e9a448a5f49a2ec2a" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + [[package]] name = "const-hex" version = "1.17.0" @@ -429,6 +546,26 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -453,6 +590,45 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crypto-common" version = "0.1.7" @@ -551,6 +727,12 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "data-encoding" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" + [[package]] name = "dbl" version = "0.3.2" @@ -690,7 +872,7 @@ dependencies = [ name = "eql-bindings" version = "3.0.6" dependencies = [ - "base64", + "base64 0.22.1", "base85-simd", "hex", "schemars", @@ -736,6 +918,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -978,6 +1170,76 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" +[[package]] +name = "hickory-net" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c480823ed7c2c5d0f09c41020cb6b7c28029ce60ec42dc942158dcf22f8e0a4d" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni 0.22.4", + "rand 0.10.1", + "thiserror 2.0.18", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12b92608f679a6fa515dd1d15c1ff89443026e391200a2c840c7afcba482893d" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni 0.22.4", + "once_cell", + "prefix-trie", + "rand 0.10.1", + "ring", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3da5255c95d5a716857d54b5b8f4e8d67c3484d3beaaaae2ce25063b3ba981" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-net", + "hickory-proto", + "ipconfig", + "ipnet", + "jni 0.22.4", + "moka", + "ndk-context", + "once_cell", + "parking_lot", + "rand 0.10.1", + "resolv-conf", + "smallvec", + "system-configuration", + "thiserror 2.0.18", + "tokio", + "tracing", +] + [[package]] name = "hmac" version = "0.13.0" @@ -987,6 +1249,45 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "hybrid-array" version = "0.4.10" @@ -996,6 +1297,66 @@ dependencies = [ "typenum", ] +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1174,6 +1535,28 @@ dependencies = [ "generic-array", ] +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +dependencies = [ + "serde", +] + [[package]] name = "is-docker" version = "0.2.0" @@ -1208,6 +1591,77 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.0", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.114", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.114", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -1235,7 +1689,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" dependencies = [ "aws-lc-rs", - "base64", + "base64 0.22.1", "getrandom 0.2.17", "js-sys", "pem", @@ -1301,6 +1755,12 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "md-5" version = "0.10.6" @@ -1339,6 +1799,16 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.1.1" @@ -1350,12 +1820,35 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moka" +version = "0.12.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac832c50ced444ef6be0767a008b02c106a909ba79d1d830501e94b96f6b7e" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + [[package]] name = "mutants" version = "0.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "add0ac067452ff1aca8c5002111bd6b1c895baee6e45fcbc44e0193aea17be56" +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "nom" version = "8.0.0" @@ -1448,6 +1941,10 @@ name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "opaque-debug" @@ -1465,6 +1962,12 @@ dependencies = [ "libc", ] +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "ore-rs" version = "0.10.0" @@ -1517,7 +2020,7 @@ version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -1587,6 +2090,17 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prefix-trie" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -1709,6 +2223,63 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.2", + "lru-slab", + "rand 0.10.1", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + [[package]] name = "quote" version = "1.0.47" @@ -1812,6 +2383,15 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.0", +] + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -1912,44 +2492,188 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c" [[package]] -name = "rmp" -version = "0.8.15" +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-core", + "futures-util", + "hickory-resolver", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "once_cell", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted 0.9.0", + "windows-sys 0.52.0", +] + +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ - "num-traits", + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", ] [[package]] -name = "rmp-serde" -version = "1.3.1" +name = "rustls-pki-types" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ - "rmp", - "serde", + "web-time", + "zeroize", ] [[package]] -name = "rustc_version" -version = "0.4.1" +name = "rustls-platform-verifier" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" dependencies = [ - "semver", + "core-foundation 0.10.1", + "core-foundation-sys", + "jni 0.21.1", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", ] [[package]] -name = "rustix" -version = "0.38.44" +name = "rustls-platform-verifier-android" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.52.0", + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted 0.9.0", ] [[package]] @@ -1964,6 +2688,24 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "schemars" version = "1.2.2" @@ -1995,6 +2737,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.27" @@ -2151,6 +2916,28 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "simple_asn1" version = "0.6.4" @@ -2196,11 +2983,16 @@ name = "stack-auth" version = "0.43.0" dependencies = [ "aquamarine", - "base64", + "base64 0.22.1", + "bytes", "cts-common", "jsonwebtoken", "miette", "open", + "reqwest", + "rustls", + "rustls-platform-verifier", + "rustls-webpki", "serde", "serde_json", "serde_urlencoded", @@ -2233,6 +3025,7 @@ dependencies = [ "serde_json", "serdect", "sha2 0.11.0", + "stack-auth", "stack-encrypt-derive", "stack-kms", "thiserror 1.0.69", @@ -2266,9 +3059,12 @@ dependencies = [ "base64ct", "blake3", "futures", + "lazy_static", "miette", "opaque-debug", "recipher", + "reqwest", + "rustls", "serde", "serde_cbor", "serde_json", @@ -2309,6 +3105,9 @@ dependencies = [ "serde_json", "stack-auth", "stack-encrypt", + "stack-kms", + "tokio", + "zerokms-protocol", ] [[package]] @@ -2366,6 +3165,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -2377,6 +3185,33 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tap" version = "1.0.1" @@ -2522,6 +3357,29 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml_datetime" version = "1.1.2+spec-1.1.0" @@ -2552,6 +3410,56 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.44" @@ -2584,6 +3492,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "ts-rs" version = "10.1.0" @@ -2674,6 +3588,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -3108,6 +4028,25 @@ dependencies = [ "zeroize", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -3145,6 +4084,20 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +dependencies = [ + "cfg-if", + "futures-util", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.108" @@ -3199,6 +4152,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -3211,6 +4177,16 @@ dependencies = [ "semver", ] +[[package]] +name = "web-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web-time" version = "1.1.0" @@ -3221,6 +4197,21 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi" version = "0.3.9" @@ -3293,6 +4284,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -3311,6 +4313,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -3338,6 +4349,21 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -3371,6 +4397,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" @@ -3383,6 +4415,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -3395,6 +4433,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -3419,6 +4463,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -3431,6 +4481,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -3443,6 +4499,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -3455,6 +4517,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -3669,7 +4737,7 @@ version = "0.12.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c28e88315a5109d0a1e7ee4b7b4b8776a0bff5f5b139ae83960a3debe84e92e" dependencies = [ - "base64", + "base64 0.22.1", "cipherstash-config", "const-hex", "cts-common", diff --git a/languages/python/Cargo.toml b/languages/python/Cargo.toml index 19c20c95e..ffecbe78f 100644 --- a/languages/python/Cargo.toml +++ b/languages/python/Cargo.toml @@ -29,11 +29,14 @@ repository = "https://github.com/cipherstash/stack" [workspace.dependencies] # `dynamic` is the runtime value model every binding speaks (frames, plans -# read at run time). `http`, the HTTP client and TLS stack, is off: nothing -# makes a request yet. `Cipher` finds the credentials over a transport that -# sends nothing (`crates/core/src/credentials.rs`). The ticket that makes -# the first request turns it on, here and on stack-auth below. -stack-encrypt = { path = "../../packages/stack-encrypt", default-features = false, features = ["dynamic"] } +# read at run time). `http` is the HTTP client and TLS stack: `encrypt` +# makes ZeroKMS requests through the shared client in +# `crates/core/src/client.rs` (#1389). Named, not left to the default, so a +# later `default-features = false` cannot remove it. +stack-encrypt = { path = "../../packages/stack-encrypt", default-features = false, features = ["dynamic", "http"] } # The credentials, as every SDK finds them. By path, the same crate -# stack-encrypt depends on. Its `http` feature is off, as above. -stack-auth = { path = "../../packages/stack-auth", default-features = false } +# stack-encrypt depends on. `http` for the transport the shared client uses. +stack-auth = { path = "../../packages/stack-auth", default-features = false, features = ["http"] } +# The ZeroKMS client the shared client is built over, and its HTTP +# connection, which `client.rs` wraps to count requests for the tests. +stack-kms = { path = "../../packages/stack-kms", default-features = false, features = ["http", "profile"] } diff --git a/languages/python/README.md b/languages/python/README.md new file mode 100644 index 000000000..4edb2d0ba --- /dev/null +++ b/languages/python/README.md @@ -0,0 +1,77 @@ +# stackencrypt + +`stackencrypt` is the Python SDK for CipherStash Stack. It encrypts the fields of a Python model into values you store in PostgreSQL with EQL v4. EQL is CipherStash's set of PostgreSQL types for encrypted values. + +This is a stub. The package is not released. Today it can encrypt a model. Decrypting and querying come in later releases. + +## Before you start + +You need three things: + +- CipherStash credentials. Set `CS_CLIENT_ACCESS_KEY`, `CS_WORKSPACE_CRN`, `CS_CLIENT_ID`, and `CS_CLIENT_KEY`. Or log in with `stash auth login`. +- A PostgreSQL database. +- The EQL v4 bundle installed in that database. + +To build the EQL v4 bundle from this repository and install it, run these commands from the repository root: + +```sh +mise -C packages/eql run build +psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -f packages/eql/release/cipherstash-encrypt-v4.sql +``` + +## Create the table + +Each encrypted column has an EQL v4 type. A model field marked `TextEq` is stored in a `public.eql_v4_text_eq` column. The index on `eql_v4.eq_term(email)` lets equality queries use the index instead of reading every row. + +```sql +CREATE TABLE users ( + email public.eql_v4_text_eq, + age int +); +CREATE INDEX users_email_eq ON users USING hash (eql_v4.eq_term(email)); +``` + +## Encrypt a row + +Declare the model with `stackencrypt.model`, then encrypt it with a `Cipher`: + +```python +import dataclasses +from typing import Annotated + +import psycopg +import stackencrypt +from stackencrypt import Passthrough, TextEq + + +@stackencrypt.model("users") +@dataclasses.dataclass +class User: + email: Annotated[str, TextEq] + age: Annotated[int, Passthrough] + + +cipher = stackencrypt.Cipher(User) +rows = cipher.encrypt([User(email="a@example.com", age=3)]) + +with psycopg.connect(DATABASE_URL) as conn: + conn.execute( + "INSERT INTO users (email, age) VALUES (%s, %s)", + (rows[0]["email"], rows[0]["age"]), + ) +``` + +`encrypt` returns one dict per model. A sealed field such as `email` is a `str` of EQL JSON. psycopg sends it with a plain `%s`. PostgreSQL reads it as the column's type, so the insert needs no cast. A `Passthrough` field such as `age` is the value itself. + +`encrypt` checks every value before it sends anything. A value of the wrong type, such as `True` for an `int` field, raises `stackencrypt.InvalidValueError`. The error names the field. It never contains the value. + +## Run the tests + +From `languages/python`: + +```sh +mise run test # no credentials needed +mise run test:live # needs the four CS_* variables and EQL_TEST_DATABASE_URL +``` + +`mise run test:live` installs the EQL v4 bundle into the database that `EQL_TEST_DATABASE_URL` names. Use a database you can throw away. diff --git a/languages/python/crates/core/Cargo.toml b/languages/python/crates/core/Cargo.toml index 0bf3e0132..a4f475ac0 100644 --- a/languages/python/crates/core/Cargo.toml +++ b/languages/python/crates/core/Cargo.toml @@ -29,6 +29,13 @@ miette = "7.5.0" serde = "1" stack-auth = { workspace = true } stack-encrypt = { workspace = true } +stack-kms = { workspace = true } +# One multi-thread runtime per process, beside the shared client: an +# operation blocks its own thread on it with the GIL released. +tokio = { version = "1", features = ["rt-multi-thread"] } +# For the request-counting connection: the trait a ZeroKMS request +# implements. The version stack-kms resolves. +zerokms-protocol = "0.12.31" [dev-dependencies] # The shared resolver file's own tests use it. diff --git a/languages/python/crates/core/src/client.rs b/languages/python/crates/core/src/client.rs new file mode 100644 index 000000000..6a774bfec --- /dev/null +++ b/languages/python/crates/core/src/client.rs @@ -0,0 +1,205 @@ +//! The one engine client a process shares, and the runtime it runs on. +//! +//! Every `Cipher` in the process uses the same `StackCipher`. It is built on +//! the first operation that needs it, by whichever thread gets there first, +//! and never rebuilt: credentials are read once, and the default keyset is +//! loaded once (the spec in #1341, "Operations"). Beside it is one +//! multi-thread tokio runtime. An operation hands its future to that runtime +//! and blocks its own thread, with the GIL released, so calls from several +//! Python threads run inside the engine at the same time. +//! +//! The client is built over a ZeroKMS connection that counts every request +//! it sends. The count is for the tests: the spec's request rules (one key +//! request per 500 values, one keyset load per process) are observed +//! through it, not assumed. + +use std::future::Future; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Mutex, OnceLock}; + +use stack_auth::AutoStrategy; +use stack_encrypt::kms::{ + ClientOpts, ConnectionInitError, EnvKeyProvider, FallbackKeyProvider, HttpConnection, + HttpConnectionOpts, KeyProvider, KeyProviderError, ProfileStore, StackKms, + StackKmsBuilderError, ZeroKMSConnection, ZeroKMSConnectionInit, ZeroKmsEndpoint, +}; +use stack_encrypt::{KeysetCipher, StackCipher}; +use zerokms_protocol::{ViturRequest, ViturRequestError}; + +use crate::credentials; +use crate::refusal::Refusal; + +/// The ZeroKMS client the shared cipher is built over. +pub type Kms = StackKms; + +/// The shared engine client with its runtime. +pub struct Client { + cipher: StackCipher, + runtime: tokio::runtime::Runtime, +} + +static CLIENT: OnceLock = OnceLock::new(); +// Serialises the build, so two threads making their first call at once build +// one client and load the keyset once. `OnceLock::get_or_init` would do the +// same, but it cannot fail, and a failed build must not be cached: the user +// fixes the credentials and calls again. +static BUILDING: Mutex<()> = Mutex::new(()); +static BUILDS: AtomicU64 = AtomicU64::new(0); +static REQUESTS: AtomicU64 = AtomicU64::new(0); + +/// The shared client, built on the first call. +/// +/// # Errors +/// +/// The refusal of whatever failed: the credentials (`stack_auth_*`), the +/// client key or the ZeroKMS client (`stack_kms_*`), or the keyset load +/// (`stack_encrypt_*`). Nothing is cached on failure. +pub fn shared() -> Result<&'static Client, Refusal> { + if let Some(client) = CLIENT.get() { + return Ok(client); + } + let _building = BUILDING + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(client) = CLIENT.get() { + return Ok(client); + } + let client = build()?; + BUILDS.fetch_add(1, Ordering::Relaxed); + Ok(CLIENT.get_or_init(|| client)) +} + +/// How many times the shared client has been built in this process: 0 +/// before the first operation, 1 after it. For the tests. +pub fn build_count() -> u64 { + BUILDS.load(Ordering::Relaxed) +} + +/// How many requests the shared client has sent to ZeroKMS in this process, +/// the keyset load included. For the tests. +pub fn request_count() -> u64 { + REQUESTS.load(Ordering::Relaxed) +} + +impl Client { + /// The cipher bound to the client's default keyset. + pub fn keyset(&self) -> KeysetCipher<'_, Kms> { + self.cipher.default_keyset() + } + + /// Run `future` to completion on the shared runtime, blocking this + /// thread. The caller releases the GIL around this. + pub fn block_on(&self, future: F) -> F::Output { + self.runtime.block_on(future) + } +} + +fn build() -> Result { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(|error| Refusal { + code: "stack_runtime".to_owned(), + message: format!("the engine's runtime could not start: {error}"), + field: None, + reason: None, + })?; + let cipher = runtime.block_on(init())?; + Ok(Client { cipher, runtime }) +} + +/// What `StackCipher::builder().init()` does, with the counting connection +/// in place of the plain one: the credentials from `AutoStrategy`, the +/// client key from the environment or the profile, one ZeroKMS client, and +/// the default keyset loaded. +async fn init() -> Result, Refusal> { + // First: it reads one variable and sends nothing, so a bad value is + // refused before the credentials are read. + let endpoint = endpoint_from_env()?; + let credentials = AutoStrategy::detect().map_err(|error| credentials::refusal(&error))?; + let client_key = FallbackKeyProvider::new( + EnvKeyProvider, + ProfileClientKey(ProfileStore::resolve(None).ok()), + ) + .client_key() + .await + .map_err(|error| Refusal::of(&error))?; + // Requests go out one at a time, as the spec states the request count: + // ceil(values / 500) key requests "in sequence". The Go guest sets the + // same. + let options = ClientOpts::new(HttpConnectionOpts::new(endpoint)) + .with_max_concurrent_reqs(1) + .map_err(|error| Refusal::of(&error))?; + let kms = + Kms::connect(options, credentials, client_key).map_err(|error| Refusal::of(&error))?; + StackCipher::builder() + .kms(kms) + .init() + .await + .map_err(|error| Refusal::of(&error)) +} + +/// The ZeroKMS endpoint override, as `StackKmsBuilder` reads it: the first +/// of `CS_ZEROKMS_HOST` and the legacy `CS_VITUR_HOST` that is set. `None` +/// leaves the endpoint to the token's services claim, as every other SDK +/// does. `init` builds the client itself to count its requests, so it reads +/// the variable here; the builder's own reader is private. +fn endpoint_from_env() -> Result, Refusal> { + for env_var in stack_encrypt::kms::vars::CS_ZEROKMS_HOST { + if let Ok(value) = std::env::var(env_var) { + return value.parse().map(Some).map_err(|source| { + Refusal::of(&StackKmsBuilderError::InvalidEndpoint { env_var, source }) + }); + } + } + Ok(None) +} + +/// The CLI's profile as a key provider, where it resolves. Where it does +/// not, the error says what to do, as stack-encrypt's own provider does. +struct ProfileClientKey(Option); + +impl KeyProvider for ProfileClientKey { + async fn client_key(&self) -> Result { + match &self.0 { + Some(store) => store.client_key().await, + None => Err(KeyProviderError::NotConfigured( + "no client key: set CS_CLIENT_ID and CS_CLIENT_KEY, or log in with `stash auth login`" + .into(), + )), + } + } +} + +/// The HTTP connection, counting every request it sends. +pub struct Counting(HttpConnection); + +impl ZeroKMSConnectionInit for Counting { + type ConnectionOpts = HttpConnectionOpts; + type Error = ConnectionInitError; + + fn init(opts: Self::ConnectionOpts) -> Result { + HttpConnection::init(opts).map(Self) + } +} + +impl ZeroKMSConnection for Counting { + async fn send( + &self, + request: Request, + access_token: &str, + ) -> Result { + REQUESTS.fetch_add(1, Ordering::Relaxed); + self.0.send(request, access_token).await + } + + // Both forwarded: the trait's defaults would tell the client the base + // URL is set when the connection underneath has none. + fn ensure_base_url(&self, url: ZeroKmsEndpoint) { + self.0.ensure_base_url(url); + } + + fn has_base_url(&self) -> bool { + self.0.has_base_url() + } +} diff --git a/languages/python/crates/core/src/credentials.rs b/languages/python/crates/core/src/credentials.rs index b17774ce3..a0d496a4c 100644 --- a/languages/python/crates/core/src/credentials.rs +++ b/languages/python/crates/core/src/credentials.rs @@ -31,13 +31,18 @@ pub fn detect() -> Result<(), Refusal> { .transport(NoRequest) .detect() .map(drop) - .map_err(|error| { - let refused = Refusal::of(&error); - Refusal { - message: message(&error).unwrap_or(refused.message.clone()), - ..refused - } - }) + .map_err(|error| refusal(&error)) +} + +/// An auth error as the SDK's refusal, with the message for a Python user. +/// The shared client ([`crate::client`]) reports its own credential errors +/// through this, so the two paths agree. +pub fn refusal(error: &AuthError) -> Refusal { + let refused = Refusal::of(error); + match message(error) { + Some(message) => Refusal { message, ..refused }, + None => refused, + } } /// The message for a Python user, where the engine's names a Rust API or diff --git a/languages/python/crates/core/src/encrypt.rs b/languages/python/crates/core/src/encrypt.rs new file mode 100644 index 000000000..ac40cab34 --- /dev/null +++ b/languages/python/crates/core/src/encrypt.rs @@ -0,0 +1,180 @@ +//! `encrypt`: the engine's record encryption under a model's plan, and the +//! walk of its output into one row of column values per model. +//! +//! The extension writes the models into one input frame, a sequence of +//! maps, straight from their attributes. This module runs the engine over +//! it on the shared client, then reads the ciphertext tree the engine +//! returns: per row a map of `field → { output-key → node }` +//! (`stack_encrypt::dynamic::record`, "The stored record is wire format"). +//! An EQL field's JSON rides under `"eql"` as bytes; a passthrough field's +//! value rides under `"passthrough"` as the frame it went in as. Both come +//! out as a [`Column`], which the extension turns into the dict's value. + +use stack_encrypt::dynamic::record::{self, Plan, EQL_KEY}; +use stack_encrypt::dynamic::{self, Output, ValueFrame}; +use stack_encrypt::{CipherText, Encrypt, StackCipherText}; + +use crate::client; +use crate::plan; +use crate::refusal::Refusal; +use crate::targets; +use crate::values::{self, Kind, Scalar}; + +/// One field of a model's plan, as `encrypt` needs it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Field { + /// The field, which is the column. + pub name: String, + /// The EQL type and its kind, or `None` for a passthrough. + pub eql: Option<(String, Kind)>, +} + +/// A model's plan, built once per `Cipher` and read on every call. +pub struct ModelPlan { + plan: Plan, + fields: Vec, +} + +impl ModelPlan { + /// Build the plan the engine runs for a model stored in `table`. + /// + /// # Errors + /// + /// The engine's refusal, as [`plan::check`] gives it. + pub fn new(table: &str, fields: Vec) -> Result { + let columns: Vec<(&str, plan::Column<'_>)> = fields + .iter() + .map(|field| { + let column = field + .eql + .as_ref() + .map_or(plan::Column::Passthrough, |(name, _)| { + plan::Column::Eql(name) + }); + (field.name.as_str(), column) + }) + .collect(); + let plan = plan::build(table, &columns)?; + Ok(Self { plan, fields }) + } + + /// The fields, in plan order. The extension writes each model's values + /// in this order and reads each row's columns back in it. + pub fn fields(&self) -> &[Field] { + &self.fields + } +} + +/// One column of an encrypted row. +#[derive(Clone, Debug, PartialEq)] +pub enum Column { + /// The EQL JSON of a sealed field, as text. + Eql(String), + /// The echoed value of a passthrough field. + Passthrough(Scalar), +} + +/// Encrypt `rows` models, written as one frame, under `plan`. Returns one +/// row per model, its columns in [`ModelPlan::fields`] order. +/// +/// Builds the shared client on the first call in the process. The caller +/// has released the GIL. +/// +/// # Errors +/// +/// The client's refusal, or the engine's for the encryption. +pub fn encrypt( + plan: &ModelPlan, + source: ValueFrame, + rows: usize, +) -> Result>, Refusal> { + let client = client::shared()?; + let tree = client.block_on(async { + let keyset = client.keyset(); + let pending = record::encrypt_with(&keyset, source, &plan.plan, &targets::resolver()) + .map_err(|error| plan::refusal(&error))?; + pending.await.map_err(|error| Refusal::of(&error)) + })?; + read_rows(plan, tree, rows) +} + +fn read_rows( + plan: &ModelPlan, + tree: StackCipherText, + rows: usize, +) -> Result>, Refusal> { + let CipherText::Sequence(trees) = tree else { + return Err(engine_fault("the output is not a sequence of rows")); + }; + if trees.len() != rows { + return Err(engine_fault( + "the output has another number of rows than the input", + )); + } + trees.into_iter().map(|row| read_row(plan, row)).collect() +} + +fn read_row(plan: &ModelPlan, row: StackCipherText) -> Result, Refusal> { + let CipherText::Map(entries) = row else { + return Err(engine_fault("a row is not a map of fields")); + }; + let mut columns = Vec::with_capacity(plan.fields.len()); + for field in &plan.fields { + let (_, outputs) = entries + .iter() + .find(|(name, _)| name == &field.name) + .ok_or_else(|| engine_fault("a row lacks a declared field"))?; + let CipherText::Map(outputs) = outputs else { + return Err(engine_fault("a field is not a map of outputs")); + }; + let key = if field.eql.is_some() { + EQL_KEY + } else { + Output::Passthrough.key() + }; + let (_, node) = outputs + .iter() + .find(|(output, _)| output == key) + .ok_or_else(|| engine_fault("a field lacks its output"))?; + let CipherText::Passthrough(frame) = node else { + return Err(engine_fault("an output is not a passthrough node")); + }; + let scalar = values::read(&frame.0).map_err(|refused| Refusal { + field: Some(field.name.clone()), + ..refused + })?; + columns.push(match (&field.eql, scalar) { + (Some(_), Scalar::Bytes(json)) => Column::Eql( + String::from_utf8(json).map_err(|_| engine_fault("an EQL value is not UTF-8"))?, + ), + (Some(_), _) => return Err(engine_fault("an EQL value is not bytes")), + (None, scalar) => Column::Passthrough(scalar), + }); + } + Ok(columns) +} + +fn engine_fault(what: &str) -> Refusal { + Refusal { + code: "stack_encrypt_unknown".to_owned(), + message: format!( + "the engine's output does not have the shape the SDK reads: {what}. \ + Report this message as a stackencrypt bug." + ), + field: None, + reason: None, + } +} + +/// A source the engine can read: anything that writes the rows. Re-exported +/// so the extension names one trait. +pub use dynamic::plaintext; + +/// Write `source` as the one input frame `encrypt` takes. +/// +/// # Errors +/// +/// The error the source raised while writing, as the engine's decode error. +pub fn frame_of(source: P) -> Result { + ValueFrame::of(source) +} diff --git a/languages/python/crates/core/src/lib.rs b/languages/python/crates/core/src/lib.rs index f6437adb5..31a17d9de 100644 --- a/languages/python/crates/core/src/lib.rs +++ b/languages/python/crates/core/src/lib.rs @@ -2,13 +2,18 @@ //! linked and tested without a Python interpreter. //! //! It holds the engine commit the build recorded (`build.rs`), the engine's -//! check of a model's plan ([`plan`]), and the credential lookup -//! ([`credentials`]). Both report an engine error as a [`refusal::Refusal`]. -//! The Python SDK spec (#1341, "Layout") says what else belongs here. +//! check of a model's plan ([`plan`]), the credential lookup +//! ([`credentials`]), the one engine client a process shares ([`client`]), +//! the value rules ([`values`]) and `encrypt` itself ([`encrypt`]). Each +//! reports an engine error as a [`refusal::Refusal`]. The Python SDK spec +//! (#1341, "Layout") says what else belongs here. +pub mod client; pub mod credentials; +pub mod encrypt; pub mod plan; pub mod refusal; +pub mod values; // The resolver file below holds the EQL types only under this feature. #[cfg(not(feature = "eql"))] diff --git a/languages/python/crates/core/src/plan.rs b/languages/python/crates/core/src/plan.rs index 222f43c13..6687af6a5 100644 --- a/languages/python/crates/core/src/plan.rs +++ b/languages/python/crates/core/src/plan.rs @@ -29,15 +29,23 @@ pub enum Column<'a> { /// /// The engine's refusal, as a [`Refusal`]. pub fn check(table: &str, fields: &[(&str, Column<'_>)]) -> Result<(), Refusal> { + build(table, fields).map(drop) +} + +/// The engine's plan for a model stored in `table`, with `fields` in +/// declaration order: what [`check`] checks, kept for `encrypt`. +/// +/// # Errors +/// +/// As [`check`]. +pub fn build(table: &str, fields: &[(&str, Column<'_>)]) -> Result { // A frame of strings and lists of strings always encodes. let frame = dynamic::frame::serde::to_vec(&Wire { table, fields }) .expect("a plan of strings encodes as a frame"); - record::plan_with(&frame, &targets::resolver()) - .map(drop) - .map_err(|error| refusal(&error)) + record::plan_with(&frame, &targets::resolver()).map_err(|error| refusal(&error)) } -fn refusal(error: &dynamic::Error) -> Refusal { +pub fn refusal(error: &dynamic::Error) -> Refusal { let payload = error.payload(); Refusal { field: payload diff --git a/languages/python/crates/core/src/values.rs b/languages/python/crates/core/src/values.rs new file mode 100644 index 000000000..b4727d483 --- /dev/null +++ b/languages/python/crates/core/src/values.rs @@ -0,0 +1,476 @@ +//! The value rules: a host value into the engine's scalar, and an opened +//! scalar back out. +//! +//! The kind comes from the field's marker, never from the value. The +//! extension reads a Python value by exact type and hands it over as a +//! [`Value`]; this module writes it as the engine's physical value under the +//! kind's logical type, and refuses what does not fit (the spec in #1341, +//! "Values"). A passthrough has no kind: it is carried as it is. An opened +//! passthrough comes back as a [`Scalar`], read from its frame here, so the +//! extension never walks a frame itself. +//! +//! The exact-type check (`bool` is not an `int`) is the extension's: it is +//! a check on Python types. What is here is everything after it. + +use stack_encrypt::dynamic::frame::Frame; +use stack_encrypt::dynamic::plaintext::{ + kinds, Error as SinkError, FixedType, Float, Int, Logical, MapSink, Physical, SeqSink, Sink, + SourceError, TimeUnit, +}; + +use stack_encrypt::Encrypt; + +use crate::refusal::Refusal; + +/// A marker's kind, as the catalog names it: the `plaintext` attribute of +/// the generated marker. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Kind { + String, + Int16, + Int32, + Int64, + Float32, + Float64, + Bool, + Decimal, + Date, + Timestamp, +} + +impl Kind { + /// The kind of this name, or `None` for a kind no Python type exists + /// for, such as `json`. The decorator refuses those first. + pub fn parse(name: &str) -> Option { + Some(match name { + "string" => Self::String, + "int16" => Self::Int16, + "int32" => Self::Int32, + "int64" => Self::Int64, + "float32" => Self::Float32, + "float64" => Self::Float64, + "bool" => Self::Bool, + "decimal" => Self::Decimal, + "date" => Self::Date, + "timestamp" => Self::Timestamp, + _ => return None, + }) + } + + /// The catalog's name, the `reason` of a value refusal. + pub fn name(self) -> &'static str { + match self { + Self::String => "string", + Self::Int16 => "int16", + Self::Int32 => "int32", + Self::Int64 => "int64", + Self::Float32 => "float32", + Self::Float64 => "float64", + Self::Bool => "bool", + Self::Decimal => "decimal", + Self::Date => "date", + Self::Timestamp => "timestamp", + } + } +} + +/// A host value, already read by exact type, as the engine takes it. +#[derive(Clone, Copy, Debug)] +pub enum Value<'a> { + Text(&'a str), + /// Any Python `int` that fits; one that does not is refused by the + /// extension as out of range before it gets here. + Int(i128), + Float(f64), + Bool(bool), + Bytes(&'a [u8]), + /// Days since 1970-01-01. + Date(i64), + /// Nanoseconds since the Unix epoch, UTC. A naive datetime never gets + /// here: the extension refuses it. + Timestamp(i128), + /// `mantissa × 10^-scale`. + Decimal { + mantissa: i128, + scale: u32, + }, +} + +/// Why a value does not fit its kind. The extension turns it into the +/// `InvalidValueError` with the field's name. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Misfit { + /// An integer outside the kind's width, or a date or timestamp outside + /// the engine's range: `value_out_of_range`. + OutOfRange, + /// A float the kind's width does not round-trip, or a decimal outside + /// the engine's bounds: `value_not_representable`. + NotRepresentable, +} + +impl Misfit { + /// The SDK's error code. + pub fn code(self) -> &'static str { + match self { + Self::OutOfRange => "value_out_of_range", + Self::NotRepresentable => "value_not_representable", + } + } +} + +/// Write `value` into `sink` as the scalar of `kind`, or as it is for a +/// passthrough (`None`). +/// +/// # Errors +/// +/// [`Misfit`] when the value does not fit the kind. A value of the wrong +/// shape for the kind, such as text for an integer kind, is a bug in the +/// extension's type check and is reported as out of range. +pub fn write(sink: S, kind: Option, value: Value<'_>) -> Result { + let (logical, physical) = + scalar(kind, value).map_err(|misfit| S::Error::source(SourceError::new(misfit)))?; + sink.scalar(logical, physical) +} + +/// The engine's scalar for `value` under `kind`. +/// +/// # Errors +/// +/// As [`write`]. +pub fn scalar( + kind: Option, + value: Value<'_>, +) -> Result<(Option, Physical<'_>), Misfit> { + let int = |i: i128, width: fn(i128) -> Option| { + width(i).map(Physical::Int).ok_or(Misfit::OutOfRange) + }; + Ok(match (kind, value) { + (Some(Kind::String), Value::Text(s)) | (None, Value::Text(s)) => (None, Physical::Text(s)), + (Some(Kind::Int16), Value::Int(i)) => { + (None, int(i, |i| i16::try_from(i).ok().map(Int::I16))?) + } + (Some(Kind::Int32), Value::Int(i)) => { + (None, int(i, |i| i32::try_from(i).ok().map(Int::I32))?) + } + (Some(Kind::Int64), Value::Int(i)) => { + (None, int(i, |i| i64::try_from(i).ok().map(Int::I64))?) + } + // A passthrough integer keeps the narrowest signed width that holds + // it, as a Go `int` would. + (None, Value::Int(i)) => ( + None, + Physical::Int(i64::try_from(i).map(Int::I64).unwrap_or(Int::I128(i))), + ), + (Some(Kind::Float32), Value::Float(f)) => { + let narrowed = f as f32; + if f64::from(narrowed) != f && !f.is_nan() { + return Err(Misfit::NotRepresentable); + } + (None, Physical::Float(Float::F32(narrowed))) + } + (Some(Kind::Float64), Value::Float(f)) | (None, Value::Float(f)) => { + (None, Physical::Float(Float::F64(f))) + } + (Some(Kind::Bool), Value::Bool(b)) | (None, Value::Bool(b)) => (None, Physical::Bool(b)), + (None, Value::Bytes(b)) => (None, Physical::Bytes(b)), + (Some(Kind::Date), Value::Date(days)) | (None, Value::Date(days)) => { + let days = i32::try_from(days).map_err(|_| Misfit::OutOfRange)?; + let physical = Physical::Int(Int::I32(days)); + if !Logical::Date.admits(&physical) { + return Err(Misfit::OutOfRange); + } + (Some(Logical::Date), physical) + } + (Some(Kind::Timestamp), Value::Timestamp(nanos)) | (None, Value::Timestamp(nanos)) => { + let physical = Physical::Int(Int::I128(nanos)); + if !Logical::Timestamp(TimeUnit::Nanos).admits(&physical) { + return Err(Misfit::OutOfRange); + } + (Some(Logical::Timestamp(TimeUnit::Nanos)), physical) + } + (Some(Kind::Decimal), Value::Decimal { mantissa, scale }) + | (None, Value::Decimal { mantissa, scale }) => { + let scale = u8::try_from(scale).map_err(|_| Misfit::NotRepresentable)?; + let logical = Logical::Decimal { scale }; + let physical = Physical::Int(Int::I128(mantissa)); + if !logical.admits(&physical) { + return Err(Misfit::NotRepresentable); + } + (Some(logical), physical) + } + // The extension checks the Python type before this, so a value of + // another shape than its kind is its bug, not the user's. + _ => return Err(Misfit::OutOfRange), + }) +} + +impl std::fmt::Display for Misfit { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.code()) + } +} + +impl std::error::Error for Misfit {} + +/// An opened scalar, as a host value. What a passthrough comes back as. +#[derive(Clone, Debug, PartialEq)] +pub enum Scalar { + Null, + Text(String), + Int(i128), + Float(f64), + Bool(bool), + Bytes(Vec), + /// Days since 1970-01-01. + Date(i32), + /// Nanoseconds since the Unix epoch, UTC. + Timestamp(i128), + /// `mantissa × 10^-scale`. + Decimal { + mantissa: i128, + scale: u8, + }, +} + +/// Read the one scalar a passthrough frame holds. +/// +/// # Errors +/// +/// A refusal when the bytes are not a frame of one scalar in the value +/// model. The engine wrote them, so that is an engine fault. +pub fn read(bytes: &[u8]) -> Result { + let frame = Frame::parse(bytes).map_err(|_| malformed("not a frame"))?; + frame + .visit(Reader) + .map_err(|ReadError(what)| malformed(what)) +} + +fn malformed(what: &str) -> Refusal { + Refusal { + code: "stack_encrypt_unknown".to_owned(), + message: format!("the engine returned a passthrough that is not one scalar: {what}"), + field: None, + reason: None, + } +} + +#[derive(Debug)] +struct ReadError(&'static str); + +impl SinkError for ReadError { + fn source(_: SourceError) -> Self { + ReadError("the frame could not be walked") + } +} + +struct Reader; + +impl Sink for Reader { + type Ok = Scalar; + type Error = ReadError; + type Seq = Never; + type Map = Never; + + fn scalar(self, ty: Option, value: Physical<'_>) -> Result { + Ok(match (ty, value) { + (None, Physical::Null) => Scalar::Null, + (None, Physical::Bool(b)) => Scalar::Bool(b), + (None, Physical::Int(i)) => { + Scalar::Int(i.to_i128().ok_or(ReadError("an integer too wide"))?) + } + (None, Physical::Float(Float::F32(f))) => Scalar::Float(f64::from(f)), + (None, Physical::Float(Float::F64(f))) => Scalar::Float(f), + (None, Physical::Text(s)) => Scalar::Text(s.to_owned()), + (None, Physical::Bytes(b)) => Scalar::Bytes(b.to_vec()), + (Some(Logical::Date), Physical::Int(days)) => Scalar::Date( + days.to_i128() + .and_then(|d| i32::try_from(d).ok()) + .ok_or(ReadError("a date outside the range"))?, + ), + (Some(Logical::Timestamp(unit)), Physical::Int(count)) => Scalar::Timestamp( + kinds::timestamp_nanos(unit, count) + .ok_or(ReadError("a timestamp outside the range"))?, + ), + (Some(Logical::Decimal { scale }), Physical::Int(mantissa)) => Scalar::Decimal { + mantissa: mantissa.to_i128().ok_or(ReadError("a decimal too wide"))?, + scale, + }, + _ => return Err(ReadError("a logical type the SDK does not read")), + }) + } + + fn typed_null(self, _: FixedType) -> Result { + Ok(Scalar::Null) + } + + fn seq(self, _: Option) -> Result { + Err(ReadError("a sequence")) + } + + fn map(self, _: Option) -> Result { + Err(ReadError("a map")) + } + + fn passthrough(self, _: P) -> Result { + Err(ReadError("a nested passthrough")) + } +} + +/// A container sink that is never built: a passthrough is one scalar. +enum Never {} + +impl SeqSink for Never { + type Ok = Scalar; + type Error = ReadError; + type Element<'s> = Reader; + + fn element(&mut self) -> Result { + match *self {} + } + + fn end(self) -> Result { + match self {} + } +} + +impl MapSink for Never { + type Ok = Scalar; + type Error = ReadError; + type Entry<'s> = Reader; + + fn entry(&mut self, _: &str) -> Result { + match *self {} + } + + fn end(self) -> Result { + match self {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + use stack_encrypt::dynamic::ValueFrame; + + #[test] + fn every_kind_has_one_name() { + for name in [ + "string", + "int16", + "int32", + "int64", + "float32", + "float64", + "bool", + "decimal", + "date", + "timestamp", + ] { + assert_eq!(Kind::parse(name).map(Kind::name), Some(name)); + } + assert_eq!(Kind::parse("json"), None); + } + + #[test] + fn an_integer_is_refused_outside_its_kinds_width() { + assert!(scalar(Some(Kind::Int16), Value::Int(32_768)).is_err()); + assert!(scalar(Some(Kind::Int16), Value::Int(32_767)).is_ok()); + assert!(scalar(Some(Kind::Int32), Value::Int(i128::from(i32::MIN) - 1)).is_err()); + assert!(scalar(Some(Kind::Int64), Value::Int(i128::from(i64::MAX))).is_ok()); + assert_eq!( + scalar(Some(Kind::Int64), Value::Int(i128::from(i64::MAX) + 1)), + Err(Misfit::OutOfRange) + ); + } + + #[test] + fn a_float_that_does_not_round_trip_through_float32_is_refused() { + assert!(scalar(Some(Kind::Float32), Value::Float(0.5)).is_ok()); + assert_eq!( + scalar(Some(Kind::Float32), Value::Float(0.1)), + Err(Misfit::NotRepresentable) + ); + assert!(scalar(Some(Kind::Float64), Value::Float(0.1)).is_ok()); + } + + #[test] + fn a_decimal_is_refused_outside_the_engines_bounds() { + assert!(scalar( + Some(Kind::Decimal), + Value::Decimal { + mantissa: 12_345, + scale: 2 + } + ) + .is_ok()); + assert_eq!( + scalar( + Some(Kind::Decimal), + Value::Decimal { + mantissa: 1, + scale: 29 + } + ), + Err(Misfit::NotRepresentable) + ); + assert_eq!( + scalar( + Some(Kind::Decimal), + Value::Decimal { + mantissa: 1 << 96, + scale: 0 + } + ), + Err(Misfit::NotRepresentable) + ); + } + + #[test] + fn a_scalar_written_as_a_passthrough_reads_back() { + for (kind, value, expected) in [ + (None, Value::Text("ada"), Scalar::Text("ada".to_owned())), + (None, Value::Int(42), Scalar::Int(42)), + (None, Value::Float(1.5), Scalar::Float(1.5)), + (None, Value::Bool(true), Scalar::Bool(true)), + (None, Value::Bytes(b"\x00\x01"), Scalar::Bytes(vec![0, 1])), + (None, Value::Date(-25_567), Scalar::Date(-25_567)), + ( + None, + Value::Timestamp(1_000_000_000), + Scalar::Timestamp(1_000_000_000), + ), + ( + None, + Value::Decimal { + mantissa: 1_995, + scale: 2, + }, + Scalar::Decimal { + mantissa: 1_995, + scale: 2, + }, + ), + ] { + let frame = ValueFrame::of(Written { kind, value }).unwrap(); + assert_eq!(read(frame.risky_ref()).unwrap(), expected, "{value:?}"); + } + } + + #[test] + fn a_frame_that_is_not_one_scalar_is_refused() { + assert!(read(b"not a frame").is_err()); + let seq = ValueFrame::of(vec!["a", "b"]).unwrap(); + assert!(read(seq.risky_ref()).is_err()); + } + + struct Written<'a> { + kind: Option, + value: Value<'a>, + } + + impl Encrypt for Written<'_> { + fn visit(self, sink: S) -> Result { + write(sink, self.kind, self.value) + } + } +} diff --git a/languages/python/crates/engine/src/encrypt.rs b/languages/python/crates/engine/src/encrypt.rs new file mode 100644 index 000000000..fc30750c5 --- /dev/null +++ b/languages/python/crates/engine/src/encrypt.rs @@ -0,0 +1,251 @@ +//! `encrypt` at the boundary: the models' attributes into the engine's +//! input frame, and the engine's rows into Python dicts. +//! +//! The frame is written straight from the attributes, with the GIL held, +//! and every value is checked while it is written: the whole list is +//! refused before any request (the spec in #1341, "Errors"). Then the GIL +//! is released and the engine runs on the shared client. The dicts are +//! built from the engine's output; no plaintext passes through a Python +//! dict on the way in. + +use std::cell::RefCell; + +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyList, PyType}; +use stackencrypt_core::encrypt::plaintext::{Encrypt, MapSink, SeqSink, Sink, SourceError}; +use stackencrypt_core::encrypt::{self as core, Column, Field, ModelPlan}; +use stackencrypt_core::refusal::Refusal; +use stackencrypt_core::values::{self, Kind}; + +use crate::errors::raise; +use crate::values::{read, to_python, PyKind}; + +/// One plan field as the decorator hands it over: `(name, eql_type, kind, +/// value_type)`. +type FieldSpec<'py> = (String, Option, Option, Bound<'py, PyType>); + +/// A model's plan, built once by `Cipher` and passed to every call. +#[pyclass(frozen, module = "stackencrypt._engine")] +pub struct Plan { + plan: ModelPlan, + /// The Python type each field's value must have, in plan order. + kinds: Vec, +} + +/// Build the plan for a model stored in `table`. +/// +/// `fields` is `(name, eql_type, kind, value_type)` per plan field in +/// declaration order: the EQL type's name and its kind for a sealed field, +/// both `None` for a passthrough, and the Python type the field's +/// annotation declares. The decorator checked that the type matches the +/// kind, and that a passthrough type is one the engine carries. +#[pyfunction] +pub fn build_plan(py: Python<'_>, table: &str, fields: Vec>) -> PyResult { + let mut kinds = Vec::with_capacity(fields.len()); + let mut specs = Vec::with_capacity(fields.len()); + for (name, eql_type, kind, value_type) in fields { + let eql = match (eql_type, kind) { + (Some(eql_type), Some(kind)) => { + let kind = Kind::parse(&kind).ok_or_else(|| { + raise( + py, + declaration_fault(&name, &format!("kind {kind:?} has no Python type")), + ) + })?; + kinds.push(PyKind::of_kind(kind)); + Some((eql_type, kind)) + } + (None, None) => { + let kind = PyKind::of_type(py, &value_type)?.ok_or_else(|| { + raise( + py, + declaration_fault(&name, "its type is not one a passthrough stores"), + ) + })?; + kinds.push(kind); + None + } + _ => { + return Err(raise( + py, + declaration_fault(&name, "an EQL type without a kind, or a kind without one"), + )) + } + }; + specs.push(Field { name, eql }); + } + let plan = ModelPlan::new(table, specs).map_err(|refused| raise(py, refused))?; + Ok(Plan { plan, kinds }) +} + +fn declaration_fault(field: &str, what: &str) -> Refusal { + Refusal { + code: "declaration_type_mismatch".to_owned(), + message: format!( + "field {field:?} reached the engine with {what}. The decorator should have \ + refused it. Report this message as a stackencrypt bug." + ), + field: Some(field.to_owned()), + reason: None, + } +} + +/// Encrypt `models` under `plan`: one dict per model, with a `str` of EQL +/// JSON for each sealed field and the value itself for each passthrough. +/// +/// An empty list returns an empty list and touches nothing. Otherwise the +/// values are read and checked first, then the shared client is built on +/// the first call in the process, and the engine runs with the GIL +/// released. +#[pyfunction] +pub fn encrypt<'py>( + py: Python<'py>, + plan: &Plan, + models: Vec>, +) -> PyResult> { + if models.is_empty() { + return Ok(PyList::empty(py)); + } + let rows = models.len(); + let failed = RefCell::new(None); + let source = Models { + plan, + models: &models, + failed: &failed, + }; + let frame = match core::frame_of(source) { + Ok(frame) => frame, + Err(error) => { + return Err(failed.into_inner().unwrap_or_else(|| { + raise( + py, + Refusal { + code: "stack_encrypt_unknown".to_owned(), + message: format!("the models could not be written as a frame: {error}"), + field: None, + reason: None, + }, + ) + })) + } + }; + let encrypted = py + .detach(|| core::encrypt(&plan.plan, frame, rows)) + .map_err(|refused| raise(py, refused))?; + let out = PyList::empty(py); + for columns in encrypted { + let row = PyDict::new(py); + for (field, column) in plan.plan.fields().iter().zip(columns) { + let value = match column { + Column::Eql(json) => json.into_pyobject(py)?.into_any().unbind(), + Column::Passthrough(scalar) => to_python(py, scalar)?, + }; + row.set_item(&field.name, value)?; + } + out.append(row)?; + } + Ok(out) +} + +/// How many times the shared engine client has been built in this process. +/// For the tests: two `Cipher` instances share one. +#[pyfunction] +pub fn client_builds() -> u64 { + stackencrypt_core::client::build_count() +} + +/// How many requests the shared client has sent to ZeroKMS in this process, +/// the keyset load included. For the tests. +#[pyfunction] +pub fn request_count() -> u64 { + stackencrypt_core::client::request_count() +} + +/// The models as the engine's source: a sequence of maps, one entry per +/// plan field, read from each model's attribute of that name. +/// +/// A value the rules refuse, or an attribute read that raises, stops the +/// walk. The error is kept in `failed` and raised after, because the sink's +/// own error type cannot carry it: the spec wants a property that raises to +/// propagate unchanged, and a refused value to name its field. +struct Models<'a, 'py> { + plan: &'a Plan, + models: &'a [Bound<'py, PyAny>], + failed: &'a RefCell>, +} + +impl Models<'_, '_> { + /// Keep `error` to raise after the walk, and stop the walk. + fn stop(&self, error: PyErr) -> E { + *self.failed.borrow_mut() = Some(error); + E::source(SourceError::msg("a value was refused")) + } + + fn write_field( + &self, + sink: S, + model: &Bound<'_, PyAny>, + field: &Field, + kind: PyKind, + ) -> Result { + let py = model.py(); + // The attribute, as the user's class gives it: a property that + // raises propagates unchanged. + let attribute = model + .getattr(&field.name) + .map_err(|error| self.stop(error))?; + let engine_kind = field.eql.as_ref().map(|(_, kind)| *kind); + let reason = engine_kind.map_or("passthrough", Kind::name); + let refuse = |refused: Refusal| { + self.stop::(raise( + py, + Refusal { + message: format!("field {:?}: {}", field.name, refused.message), + field: Some(field.name.clone()), + reason: Some(reason.to_owned()), + ..refused + }, + )) + }; + let read = match read(&attribute, kind, reason) { + Ok(Ok(read)) => read, + Ok(Err(refused)) => return Err(refuse(refused)), + Err(error) => return Err(self.stop(error)), + }; + let value = read.value(); + // The range rules first, so a misfit is refused with its own code + // and the field's name, not as the sink's error. + if let Err(misfit) = values::scalar(engine_kind, value) { + let what = match engine_kind { + Some(kind) => format!("the value does not fit {}.", kind.name()), + None => "the value is outside what the engine stores.".to_owned(), + }; + return Err(refuse(Refusal { + code: misfit.code().to_owned(), + message: what, + field: None, + reason: None, + })); + } + // A passthrough is written as a plain scalar: the plan's + // `"passthrough"` output carries the field's frame as it is, so a + // passthrough node here would come back nested in one. + values::write(sink, engine_kind, value) + } +} + +impl Encrypt for Models<'_, '_> { + fn visit(self, sink: S) -> Result { + let fields = self.plan.plan.fields(); + let mut rows = sink.seq(Some(self.models.len()))?; + for model in self.models { + let mut row = rows.element()?.map(Some(fields.len()))?; + for (field, kind) in fields.iter().zip(&self.plan.kinds) { + let entry = row.entry(&field.name)?; + self.write_field(entry, model, field, *kind)?; + } + row.end()?; + } + rows.end() + } +} diff --git a/languages/python/crates/engine/src/errors.rs b/languages/python/crates/engine/src/errors.rs new file mode 100644 index 000000000..58f120786 --- /dev/null +++ b/languages/python/crates/engine/src/errors.rs @@ -0,0 +1,42 @@ +//! A refusal as the Python exception the SDK defines for it. +//! +//! The exception classes live in `stackencrypt._errors`, in Python, so that +//! the pure-Python checks and the extension raise the same ones. The class +//! is chosen by the code's prefix, as the spec's "Errors" section assigns +//! them: `value_*` is `InvalidValueError`, `stack_auth_*` is +//! `AuthenticationError`, and anything else is the base `StashError`. + +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyTuple}; +use stackencrypt_core::refusal::Refusal; + +/// The exception for `refused`, ready to raise. +pub fn raise(py: Python<'_>, refused: Refusal) -> PyErr { + match exception(py, &refused) { + Ok(error) => error, + // The SDK's own module failed to import or to build the exception. + // Nothing to do but say so, with the refusal in the message. + Err(error) => PyErr::new::(format!( + "stackencrypt could not raise {}: {} ({error})", + refused.code, refused.message + )), + } +} + +fn exception(py: Python<'_>, refused: &Refusal) -> PyResult { + let class = if refused.code.starts_with("value_") { + "InvalidValueError" + } else if refused.code.starts_with("stack_auth_") { + "AuthenticationError" + } else { + "StashError" + }; + let errors = py.import("stackencrypt._errors")?; + let class = errors.getattr(class)?; + let args = PyTuple::new(py, [&refused.code, &refused.message])?; + let kwargs = PyDict::new(py); + kwargs.set_item("field", refused.field.as_deref())?; + kwargs.set_item("reason", refused.reason.as_deref())?; + let instance = class.call(args, Some(&kwargs))?; + Ok(PyErr::from_value(instance)) +} diff --git a/languages/python/crates/engine/src/lib.rs b/languages/python/crates/engine/src/lib.rs index 16d075db9..e9d2d008f 100644 --- a/languages/python/crates/engine/src/lib.rs +++ b/languages/python/crates/engine/src/lib.rs @@ -3,6 +3,10 @@ //! Deliberately thin: this crate only converts between Python and Rust at //! the boundary. Logic belongs in `stackencrypt-core` and `stack-encrypt`. +mod encrypt; +mod errors; +mod values; + use pyo3::prelude::*; use stackencrypt_core::credentials; use stackencrypt_core::plan::{self, Column}; @@ -87,6 +91,11 @@ fn _engine(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(build_info, module)?)?; module.add_function(wrap_pyfunction!(check_plan, module)?)?; module.add_function(wrap_pyfunction!(detect_credentials, module)?)?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(encrypt::build_plan, module)?)?; + module.add_function(wrap_pyfunction!(encrypt::encrypt, module)?)?; + module.add_function(wrap_pyfunction!(encrypt::client_builds, module)?)?; + module.add_function(wrap_pyfunction!(encrypt::request_count, module)?)?; #[cfg(feature = "gate-probe")] module.add_function(wrap_pyfunction!(_probe_detached_sleep, module)?)?; Ok(()) diff --git a/languages/python/crates/engine/src/values.rs b/languages/python/crates/engine/src/values.rs new file mode 100644 index 000000000..0bfb462d1 --- /dev/null +++ b/languages/python/crates/engine/src/values.rs @@ -0,0 +1,405 @@ +//! Python values at the boundary: a model's attribute read by exact type +//! into a core [`Value`], and an opened core [`Scalar`] back into a Python +//! object. +//! +//! The type check is exact, `type(v) is T`: a `bool` is not an `int`, and a +//! `datetime` is not a `date` (the spec in #1341, "Values"). The extension +//! never changes a value: nothing here converts, only reads. + +use std::borrow::Cow; + +use pyo3::prelude::*; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyBool, PyBytes, PyFloat, PyInt, PyString, PyTuple, PyType}; +use stackencrypt_core::refusal::Refusal; +use stackencrypt_core::values::{Kind, Scalar, Value}; + +/// The Python type a field's value must have, exactly. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PyKind { + Str, + Int, + Float, + Bool, + Bytes, + Date, + DateTime, + Decimal, +} + +impl PyKind { + /// The Python type of a marker's kind: the `_KIND_TYPES` table in + /// `_declaration.py`, checked there at the class statement. + pub fn of_kind(kind: Kind) -> Self { + match kind { + Kind::String => Self::Str, + Kind::Int16 | Kind::Int32 | Kind::Int64 => Self::Int, + Kind::Float32 | Kind::Float64 => Self::Float, + Kind::Bool => Self::Bool, + Kind::Decimal => Self::Decimal, + Kind::Date => Self::Date, + Kind::Timestamp => Self::DateTime, + } + } + + /// The kind of a declared type object, or `None` for a type the SDK + /// does not carry. The decorator refuses those at the class statement. + pub fn of_type(py: Python<'_>, declared: &Bound<'_, PyType>) -> PyResult> { + for kind in [ + Self::Str, + Self::Int, + Self::Float, + Self::Bool, + Self::Bytes, + Self::Date, + Self::DateTime, + Self::Decimal, + ] { + if kind.type_object(py)?.is(declared) { + return Ok(Some(kind)); + } + } + Ok(None) + } + + fn type_object<'py>(self, py: Python<'py>) -> PyResult<&'py Bound<'py, PyType>> { + static DATE: PyOnceLock> = PyOnceLock::new(); + static DATETIME: PyOnceLock> = PyOnceLock::new(); + static DECIMAL: PyOnceLock> = PyOnceLock::new(); + static STR: PyOnceLock> = PyOnceLock::new(); + static INT: PyOnceLock> = PyOnceLock::new(); + static FLOAT: PyOnceLock> = PyOnceLock::new(); + static BOOL: PyOnceLock> = PyOnceLock::new(); + static BYTES: PyOnceLock> = PyOnceLock::new(); + let imported = |module: &str, name: &str| -> PyResult> { + Ok(py + .import(module)? + .getattr(name)? + .cast_into::()? + .unbind()) + }; + let cell = match self { + Self::Str => { + STR.get_or_try_init(py, || PyResult::Ok(py.get_type::().unbind()))? + } + Self::Int => { + INT.get_or_try_init(py, || PyResult::Ok(py.get_type::().unbind()))? + } + Self::Float => { + FLOAT.get_or_try_init(py, || PyResult::Ok(py.get_type::().unbind()))? + } + Self::Bool => { + BOOL.get_or_try_init(py, || PyResult::Ok(py.get_type::().unbind()))? + } + Self::Bytes => { + BYTES.get_or_try_init(py, || PyResult::Ok(py.get_type::().unbind()))? + } + Self::Date => DATE.get_or_try_init(py, || imported("datetime", "date"))?, + Self::DateTime => DATETIME.get_or_try_init(py, || imported("datetime", "datetime"))?, + Self::Decimal => DECIMAL.get_or_try_init(py, || imported("decimal", "Decimal"))?, + }; + Ok(cell.bind(py)) + } + + fn name(self) -> &'static str { + match self { + Self::Str => "str", + Self::Int => "int", + Self::Float => "float", + Self::Bool => "bool", + Self::Bytes => "bytes", + Self::Date => "date", + Self::DateTime => "datetime", + Self::Decimal => "Decimal", + } + } +} + +/// A field's value as read from its attribute. Text and bytes borrow from +/// the attribute object where they can, so the plaintext is not copied +/// before the engine writes it into the protected frame. +pub enum Read<'a> { + Text(Cow<'a, str>), + Bytes(&'a [u8]), + Other(Value<'static>), +} + +impl Read<'_> { + /// The value, as the core rules take it. + pub fn value(&self) -> Value<'_> { + match self { + Self::Text(text) => Value::Text(text), + Self::Bytes(bytes) => Value::Bytes(bytes), + Self::Other(value) => *value, + } + } +} + +/// Read `value` as a field of `kind`, by exact type. +/// +/// # Errors +/// +/// `value_none` for `None`, `value_wrong_type` for another type, +/// `value_naive_datetime` for a datetime with no `tzinfo`, and the range +/// refusals for an `int` or `Decimal` the engine cannot hold. `field` is +/// not set: the caller names it. +pub fn read<'a>( + value: &'a Bound<'_, PyAny>, + kind: PyKind, + reason: &'static str, +) -> PyResult, Refusal>> { + let py = value.py(); + if value.is_none() { + return Ok(Err(refusal("value_none", "is None", reason))); + } + if !value.get_type().is(kind.type_object(py)?) { + return Ok(Err(refusal( + "value_wrong_type", + &format!( + "is {} and the field takes {}", + type_name(value), + kind.name() + ), + reason, + ))); + } + let other = |value: Value<'static>| Ok(Ok(Read::Other(value))); + match kind { + PyKind::Str => Ok(Ok(Read::Text(value.cast::()?.to_cow()?))), + PyKind::Bytes => Ok(Ok(Read::Bytes(value.cast::()?.as_bytes()))), + PyKind::Int => match value.extract::() { + Ok(i) => other(Value::Int(i)), + Err(_) => Ok(Err(refusal( + "value_out_of_range", + "does not fit 128 bits", + reason, + ))), + }, + PyKind::Float => other(Value::Float(value.extract::()?)), + PyKind::Bool => other(Value::Bool(value.extract::()?)), + PyKind::Date => { + // Python's ordinal counts from 0001-01-01; the engine's day + // count from 1970-01-01, ordinal 719163. + let ordinal: i64 = value.call_method0("toordinal")?.extract()?; + other(Value::Date(ordinal - 719_163)) + } + PyKind::DateTime => { + if value.getattr("tzinfo")?.is_none() { + return Ok(Err(refusal( + "value_naive_datetime", + "is a naive datetime. Give it a tzinfo, such as timezone.utc", + reason, + ))); + } + // Exact arithmetic in Python: the timedelta since the epoch, in + // days, seconds and microseconds. `timestamp()` is a float and + // would round. + let delta = value.sub(epoch(py)?)?; + let days: i128 = delta.getattr("days")?.extract()?; + let seconds: i128 = delta.getattr("seconds")?.extract()?; + let micros: i128 = delta.getattr("microseconds")?.extract()?; + other(Value::Timestamp( + ((days * 86_400 + seconds) * 1_000_000 + micros) * 1_000, + )) + } + PyKind::Decimal => Ok(decimal(value, reason)?.map(Read::Other)), + } +} + +fn epoch(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { + static EPOCH: PyOnceLock> = PyOnceLock::new(); + let epoch = EPOCH.get_or_try_init(py, || -> PyResult> { + let datetime = py.import("datetime")?; + let utc = datetime.getattr("timezone")?.getattr("utc")?; + Ok(datetime + .getattr("datetime")? + .call1((1970, 1, 1, 0, 0, 0, 0, utc))? + .unbind()) + })?; + Ok(epoch.bind(py)) +} + +/// A `Decimal` as its mantissa and scale, from `as_tuple()`: the sign, the +/// digits, and the exponent. A NaN or an infinity has a letter for an +/// exponent, and no mantissa. +fn decimal( + value: &Bound<'_, PyAny>, + reason: &'static str, +) -> PyResult, Refusal>> { + let parts = value.call_method0("as_tuple")?; + let sign: u8 = parts.getattr("sign")?.extract()?; + let digits: Vec = parts.getattr("digits")?.extract()?; + let Ok(exponent) = parts.getattr("exponent")?.extract::() else { + return Ok(Err(refusal( + "value_not_representable", + "is not a finite number", + reason, + ))); + }; + let not_representable = || { + refusal( + "value_not_representable", + "has more than 96 bits of digits", + reason, + ) + }; + let mut mantissa: i128 = 0; + for digit in digits { + mantissa = match mantissa + .checked_mul(10) + .and_then(|m| m.checked_add(i128::from(digit))) + { + Some(m) => m, + None => return Ok(Err(not_representable())), + }; + } + // A zero with a positive exponent is zero at scale 0, as a nonzero one + // is multiplied out below. The loop would otherwise run once per unit of + // a huge exponent with the GIL held: Decimal("0E+999999999999999999"). + // A negative exponent keeps its scale, so Decimal("0.00") stays 0.00. + if mantissa == 0 && exponent > 0 { + return Ok(Ok(Value::Decimal { + mantissa: 0, + scale: 0, + })); + } + // Past 38 the mantissa overflows i128 whatever its digits, so refuse it + // before the loop rather than after a long one. + if exponent > 38 { + return Ok(Err(not_representable())); + } + let scale = if exponent < 0 { + match u32::try_from(-exponent) { + Ok(scale) => scale, + Err(_) => return Ok(Err(not_representable())), + } + } else { + for _ in 0..exponent { + mantissa = match mantissa.checked_mul(10) { + Some(m) => m, + None => return Ok(Err(not_representable())), + }; + } + 0 + }; + if sign == 1 { + mantissa = -mantissa; + } + Ok(Ok(Value::Decimal { mantissa, scale })) +} + +fn refusal(code: &str, what: &str, reason: &'static str) -> Refusal { + Refusal { + code: code.to_owned(), + message: format!("the value {what}."), + field: None, + reason: Some(reason.to_owned()), + } +} + +fn type_name(value: &Bound<'_, PyAny>) -> String { + value + .get_type() + .name() + .map_or_else(|_| "an unknown type".to_owned(), |name| name.to_string()) +} + +/// An opened scalar as a Python object, of the type the field declares. +pub fn to_python(py: Python<'_>, scalar: Scalar) -> PyResult> { + Ok(match scalar { + Scalar::Null => py.None(), + Scalar::Text(text) => PyString::new(py, &text).into_any().unbind(), + Scalar::Int(i) => i.into_pyobject(py)?.into_any().unbind(), + Scalar::Float(f) => PyFloat::new(py, f).into_any().unbind(), + Scalar::Bool(b) => PyBool::new(py, b).to_owned().into_any().unbind(), + Scalar::Bytes(bytes) => PyBytes::new(py, &bytes).into_any().unbind(), + Scalar::Date(days) => PyKind::Date + .type_object(py)? + .call_method1("fromordinal", (i64::from(days) + 719_163,))? + .unbind(), + Scalar::Timestamp(nanos) => { + // Python's datetime keeps microseconds; the engine's timestamp + // is in nanoseconds, so the last three digits are dropped. An + // encrypted datetime had none: Python wrote it. + let micros = nanos.div_euclid(1_000); + let seconds = micros.div_euclid(1_000_000); + let micros = micros.rem_euclid(1_000_000); + let delta = py + .import("datetime")? + .getattr("timedelta")? + .call1((0, seconds, micros))?; + epoch(py)?.add(delta)?.unbind() + } + Scalar::Decimal { mantissa, scale } => { + // Decimal((sign, digits, exponent)), so that the scale is kept: + // Decimal("1.50") is not Decimal("1.5") to Python. + let sign = u8::from(mantissa < 0); + let digits: Vec = mantissa + .unsigned_abs() + .to_string() + .bytes() + .map(|b| b - b'0') + .collect(); + // A tuple of ints: pyo3 converts a Vec to bytes, which + // Decimal refuses. + let digits = PyTuple::new(py, digits)?; + PyKind::Decimal + .type_object(py)? + .call1(((sign, digits, -i64::from(scale)),))? + .unbind() + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_opened_decimal_is_a_decimal_with_its_scale() { + Python::initialize(); + Python::attach(|py| { + let decimal = py.import("decimal").unwrap().getattr("Decimal").unwrap(); + for (mantissa, scale, text) in [(1_995, 2, "19.95"), (-150, 2, "-1.50"), (0, 2, "0.00")] + { + let opened = to_python(py, Scalar::Decimal { mantissa, scale }).unwrap(); + let opened = opened.bind(py); + assert!(opened.get_type().is(&decimal), "{text}"); + assert_eq!(opened.str().unwrap().to_string(), text); + } + }); + } + + #[test] + fn a_zero_with_a_huge_exponent_reads_at_once() { + Python::initialize(); + Python::attach(|py| { + let decimal = py.import("decimal").unwrap().getattr("Decimal").unwrap(); + let zero = decimal.call1(("0E+999999999999999999",)).unwrap(); + assert!(matches!( + decimal_of(&zero), + Ok(Value::Decimal { + mantissa: 0, + scale: 0 + }) + )); + let small_zero = decimal.call1(("0.00",)).unwrap(); + assert!(matches!( + decimal_of(&small_zero), + Ok(Value::Decimal { + mantissa: 0, + scale: 2 + }) + )); + let big = decimal.call1(("1E+39",)).unwrap(); + assert_eq!( + decimal_of(&big).unwrap_err().code, + "value_not_representable" + ); + }); + } + + fn decimal_of(value: &Bound<'_, PyAny>) -> Result, Refusal> { + decimal(value, "decimal").unwrap() + } +} diff --git a/languages/python/mise.toml b/languages/python/mise.toml index 8b2948f7b..191bea47d 100644 --- a/languages/python/mise.toml +++ b/languages/python/mise.toml @@ -69,3 +69,25 @@ export DYLD_FALLBACK_LIBRARY_PATH="${libdir}${DYLD_FALLBACK_LIBRARY_PATH:+:$DYLD export LD_LIBRARY_PATH="${libdir}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" cargo test --locked --workspace """ + +[tasks."test:live"] +description = "Install the EQL v4 bundle built from the tree into the database EQL_TEST_DATABASE_URL names, and run the live tests against it and ZeroKMS; needs the four CS_* credentials" +depends = ["build"] +shell = "bash -c" +# The bundle is built from packages/eql, as the Go task go:test:postgres +# builds it: the copy under crates/eql-bindings/sql is the released bundle +# and moves only at the lockstep bump. The tests skip without credentials or +# a database, so a skip here is a failure: this task is where both are +# given. +run = """ +set -euo pipefail +: "${EQL_TEST_DATABASE_URL:?set EQL_TEST_DATABASE_URL to a libpq connection string for a disposable PostgreSQL}" +mise -C ../../packages/eql run build +psql "$EQL_TEST_DATABASE_URL" -v ON_ERROR_STOP=1 -q -f ../../packages/eql/release/cipherstash-encrypt-v4.sql +log="${TMPDIR:-/tmp}/stackencrypt-live.log" +.venv/bin/python -m pytest -m live -v -rs | tee "$log" +if grep -q ' SKIPPED' "$log"; then + echo "error: a live test skipped; set the four CS_* credentials and EQL_TEST_DATABASE_URL" >&2 + exit 1 +fi +""" diff --git a/languages/python/pyproject.toml b/languages/python/pyproject.toml index 948311cab..b32553992 100644 --- a/languages/python/pyproject.toml +++ b/languages/python/pyproject.toml @@ -16,7 +16,10 @@ classifiers = ["Private :: Do Not Upload"] # and no upper bounds on purpose. They are the libraries users build their # classes with, so the tests meet each new release when users do. A new # release that fails a test is a finding about users' classes. -test = ["pytest>=8", "attrs>=23", "pydantic>=2", "sqlalchemy>=2"] +# +# psycopg for the live tests, which insert an encrypted row (#1389). The +# binary extra ships libpq, so the tests need no system Postgres client. +test = ["pytest>=8", "attrs>=23", "pydantic>=2", "sqlalchemy>=2", "psycopg[binary]>=3.1"] [tool.maturin] # The pure-Python package is the distribution; the Rust extension is its @@ -31,3 +34,7 @@ features = ["pyo3/extension-module"] [tool.pytest.ini_options] testpaths = ["tests"] +# `live` marks the tests that need credentials and a database. They skip +# without them; `mise run test:live` runs only them and fails on a skip. +markers = ["live: needs CipherStash credentials, and some a PostgreSQL with EQL v4"] +addopts = "--strict-markers" diff --git a/languages/python/python/stackencrypt/__init__.py b/languages/python/python/stackencrypt/__init__.py index 0369ef1de..b306957ed 100644 --- a/languages/python/python/stackencrypt/__init__.py +++ b/languages/python/python/stackencrypt/__init__.py @@ -3,7 +3,7 @@ Walking skeleton. The package loads its native extension, ``_engine``, exposes two facts about the build, exports the markers a model's fields are declared with, reads a model's declaration with ``model``, and builds a ``Cipher`` for -a declared class. Nothing encrypts yet. +a declared class that encrypts its instances. """ from stackencrypt import _engine, _eql @@ -12,7 +12,12 @@ # The markers for the EQL types the engine produces today. `_eql.__all__` # lists them; a marker the engine cannot produce yet stays in `_eql` only. from stackencrypt._eql import * # noqa: F403 -from stackencrypt._errors import AuthenticationError, DeclarationError, StashError +from stackencrypt._errors import ( + AuthenticationError, + DeclarationError, + InvalidValueError, + StashError, +) from stackencrypt._markers import Ignore, Passthrough # The distribution's version, and the commit of the engine it was built from. @@ -25,6 +30,7 @@ "Cipher", "DeclarationError", "Ignore", + "InvalidValueError", "Passthrough", "StashError", "__engine__", diff --git a/languages/python/python/stackencrypt/_cipher.py b/languages/python/python/stackencrypt/_cipher.py index 158595a5a..76f2b1b69 100644 --- a/languages/python/python/stackencrypt/_cipher.py +++ b/languages/python/python/stackencrypt/_cipher.py @@ -1,10 +1,13 @@ """``Cipher``: encrypts and decrypts the instances of one declared class. -Today it checks the class and the credentials, and stores the declaration. -``encrypt``, ``decrypt`` and ``query`` come in later tickets. +It checks the class and the credentials, and builds the model's plan once. +``encrypt`` is here. ``decrypt`` and ``query`` come in later tickets. """ +from typing import Dict, List, Sequence + from stackencrypt import _engine +from stackencrypt._markers import Passthrough from stackencrypt._declaration import Declaration, check_unchanged, declaration_of from stackencrypt._errors import AuthenticationError, DeclarationError @@ -18,6 +21,9 @@ class Cipher: The class must also be as it was when declared. The credentials come from ``CS_CLIENT_ACCESS_KEY`` and ``CS_WORKSPACE_CRN``, or from the profile that ``stash auth login`` writes. Building a cipher makes no request. + + Every cipher in the process shares one engine client. It is built on the + first ``encrypt`` of any cipher, which loads the default keyset once. """ def __init__(self, cls: type) -> None: @@ -33,6 +39,32 @@ def __init__(self, cls: type) -> None: raise AuthenticationError(code, message, field=field, reason=reason) self._cls = cls self._declaration = declaration + self._plan = _engine.build_plan( + declaration.table, + [ + ( + field.name, + None if field.marker is Passthrough else field.marker.name, + None if field.marker is Passthrough else field.marker.plaintext, + field.value_type, + ) + for field in declaration.fields + ], + ) + + def encrypt(self, models: Sequence[object]) -> List[Dict[str, object]]: + """Encrypt ``models``, instances of the declared class. + + Returns one dict per model, from column name to column value: the + EQL JSON as a ``str`` for a sealed field, which psycopg inserts with + a plain ``%s``, and the value itself for a passthrough field. An + ``Ignore`` field is left out. + + Every value is checked first, by exact type and range. A value that + does not fit raises ``InvalidValueError`` naming the field, and no + request is made. An empty list makes no request at all. + """ + return _engine.encrypt(self._plan, list(models)) @property def declaration(self) -> Declaration: diff --git a/languages/python/python/stackencrypt/_declaration.py b/languages/python/python/stackencrypt/_declaration.py index 10d5b4d9d..6282ec412 100644 --- a/languages/python/python/stackencrypt/_declaration.py +++ b/languages/python/python/stackencrypt/_declaration.py @@ -93,10 +93,12 @@ class around ``Annotated`` is read by structure, so ``Mapped[Annotated[str, @dataclass(frozen=True) class DeclaredField: - """One field in the plan: its name, which is the column, and its marker.""" + """One field in the plan: its name, which is the column, its marker, and + the Python type its annotation declares.""" name: str marker: Type[Marker] + value_type: type = dataclasses.field(default=object, repr=False, compare=False) @dataclass(frozen=True) @@ -173,7 +175,10 @@ def decorate(cls: T) -> T: ) declaration = Declaration( table=table, - fields=tuple(DeclaredField(name=f.name, marker=f.marker) for f in planned), + fields=tuple( + DeclaredField(name=f.name, marker=f.marker, value_type=f.value_type) + for f in planned + ), hints=hints, constructor=_constructor(cls), methods=methods, diff --git a/languages/python/python/stackencrypt/_engine.pyi b/languages/python/python/stackencrypt/_engine.pyi index 641c2ac5d..eeff8de00 100644 --- a/languages/python/python/stackencrypt/_engine.pyi +++ b/languages/python/python/stackencrypt/_engine.pyi @@ -1,6 +1,6 @@ """Type stub for the native extension. Kept by hand, beside the Rust source.""" -from typing import List, Optional, Tuple +from typing import Any, Dict, List, Optional, Sequence, Tuple, final def build_info() -> tuple[str, str]: """``(version, engine_commit)`` as recorded at build time.""" @@ -14,3 +14,24 @@ def check_plan( def detect_credentials() -> Optional[Tuple[str, str, Optional[str], Optional[str]]]: """Find the credentials, with no request: ``None``, or the refusal as ``(code, message, field, reason)``.""" + +@final +class Plan: + """A model's plan, built once by ``Cipher``. Opaque.""" + +def build_plan( + table: str, fields: List[Tuple[str, Optional[str], Optional[str], type]] +) -> Plan: + """The plan for a model stored in ``table``: ``(name, eql_type, kind, + value_type)`` per field, ``eql_type`` and ``kind`` ``None`` for a + passthrough.""" + +def encrypt(plan: Plan, models: Sequence[Any]) -> List[Dict[str, object]]: + """One dict per model: EQL JSON as ``str`` for a sealed field, the value + for a passthrough. Raises ``InvalidValueError`` before any request.""" + +def client_builds() -> int: + """How many times the shared engine client has been built. For tests.""" + +def request_count() -> int: + """How many requests the shared client has sent to ZeroKMS. For tests.""" diff --git a/languages/python/python/stackencrypt/_errors.py b/languages/python/python/stackencrypt/_errors.py index 076876ad3..1b1e33022 100644 --- a/languages/python/python/stackencrypt/_errors.py +++ b/languages/python/python/stackencrypt/_errors.py @@ -43,3 +43,14 @@ class AuthenticationError(StashError): The code is the engine's, such as ``stack_auth_not_authenticated``. """ + + +class InvalidValueError(StashError, ValueError): + """A model's value that does not fit its field, refused before any request. + + The code is one of ``value_none``, ``value_wrong_type``, + ``value_out_of_range``, ``value_not_representable`` and + ``value_naive_datetime``. ``field`` names the field. ``reason`` is the + field's kind, such as ``int32``, or ``passthrough``. The message never + contains the value. + """ diff --git a/languages/python/tests/test_encrypt.py b/languages/python/tests/test_encrypt.py new file mode 100644 index 000000000..3364979a3 --- /dev/null +++ b/languages/python/tests/test_encrypt.py @@ -0,0 +1,209 @@ +"""``Cipher.encrypt`` before any request: the value rules and the empty list. + +Every value is checked while the models are written into the engine's input +frame, before the shared client is built. So these tests need fake +credentials only, and make no request. The live round trip is in +``test_encrypt_live.py``. +""" + +import dataclasses +import datetime +import decimal +import math +from typing import Annotated + +import pytest + +import stackencrypt +from stackencrypt import ( + Bigint, + Cipher, + Date, + Double, + InvalidValueError, + Numeric, + Passthrough, + Real, + Smallint, + StashError, + TextEq, + Timestamp, +) +from stackencrypt import _engine + +SECRET = "secret@example.com" + + +@pytest.fixture(autouse=True) +def credentials(monkeypatch, tmp_path): + """Credentials that parse, so ``Cipher`` builds. Nothing here sends them.""" + for name in ("CS_CLIENT_ID", "CS_CLIENT_KEY", "CS_CTS_HOST", "CS_ZEROKMS_HOST", "CS_VITUR_HOST"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("CS_CLIENT_ACCESS_KEY", "CSAKtestKeyId.testKeySecret") + monkeypatch.setenv("CS_WORKSPACE_CRN", "crn:ap-southeast-2.aws:ZVATKW3VHMFG27DY") + monkeypatch.setenv("CS_CONFIG_PATH", str(tmp_path / "no-profile")) + + +@pytest.fixture +def untouched(): + """Fails the test if the shared client was built or a request was sent.""" + builds, requests = _engine.client_builds(), _engine.request_count() + yield + assert _engine.client_builds() == builds + assert _engine.request_count() == requests + + +def cipher_for(**fields): + """A Cipher for a dataclass with ``email: TextEq`` and ``fields``.""" + annotations = {"email": Annotated[str, TextEq], **fields} + cls = dataclasses.make_dataclass("User", annotations.items()) + return Cipher(stackencrypt.model("users")(cls)), cls + + +def refused(cipher, model): + with pytest.raises(InvalidValueError) as raised: + cipher.encrypt([model]) + error = raised.value + assert SECRET not in str(error) + return error + + +def test_an_empty_list_is_an_empty_list_with_no_request(untouched): + cipher, _ = cipher_for() + assert cipher.encrypt([]) == [] + + +def test_an_invalid_value_error_is_a_stash_error_and_a_value_error(): + assert issubclass(InvalidValueError, StashError) + assert issubclass(InvalidValueError, ValueError) + + +def test_a_bool_on_an_int_field_is_the_wrong_type(untouched): + cipher, User = cipher_for(age=Annotated[int, Bigint]) + error = refused(cipher, User(email=SECRET, age=True)) + assert error.code == "value_wrong_type" + assert error.field == "age" + assert error.reason == "int64" + assert "bool" in error.message + + +def test_a_datetime_on_a_date_field_is_the_wrong_type(untouched): + cipher, User = cipher_for(born=Annotated[datetime.date, Date]) + now = datetime.datetime.now(datetime.timezone.utc) + error = refused(cipher, User(email=SECRET, born=now)) + assert error.code == "value_wrong_type" + assert error.field == "born" + + +def test_none_on_a_sealed_field_is_refused(untouched): + cipher, User = cipher_for() + error = refused(cipher, User(email=None)) + assert error.code == "value_none" + assert error.field == "email" + assert error.reason == "string" + + +def test_none_on_a_passthrough_field_is_refused(untouched): + cipher, User = cipher_for(age=Annotated[int, Passthrough]) + error = refused(cipher, User(email=SECRET, age=None)) + assert error.code == "value_none" + assert error.reason == "passthrough" + + +def test_a_passthrough_value_of_another_type_is_refused(untouched): + cipher, User = cipher_for(age=Annotated[int, Passthrough]) + error = refused(cipher, User(email=SECRET, age="3")) + assert error.code == "value_wrong_type" + assert error.field == "age" + + +def test_an_int_outside_its_kind_is_out_of_range(untouched): + cipher, User = cipher_for(rank=Annotated[int, Smallint]) + error = refused(cipher, User(email=SECRET, rank=40_000)) + assert error.code == "value_out_of_range" + assert error.field == "rank" + assert error.reason == "int16" + + +def test_an_int_wider_than_128_bits_is_out_of_range(untouched): + cipher, User = cipher_for(age=Annotated[int, Passthrough]) + error = refused(cipher, User(email=SECRET, age=2**200)) + assert error.code == "value_out_of_range" + + +def test_a_float_that_float32_cannot_hold_is_not_representable(untouched): + cipher, User = cipher_for(score=Annotated[float, Real]) + error = refused(cipher, User(email=SECRET, score=0.1)) + assert error.code == "value_not_representable" + assert error.reason == "float32" + + +def test_a_float64_takes_any_float(untouched): + # Refused later by the engine for want of credentials, not by the value + # rules: the value check passed. + cipher, User = cipher_for(score=Annotated[float, Double]) + with pytest.raises(StashError) as raised: + cipher.encrypt([User(email=SECRET, score=0.1)]) + assert not isinstance(raised.value, InvalidValueError) + + +@pytest.mark.parametrize( + "value", + [decimal.Decimal("NaN"), decimal.Decimal("Infinity"), decimal.Decimal("1e-29")], + ids=["nan", "infinity", "scale-29"], +) +def test_a_decimal_outside_the_engines_bounds_is_not_representable(untouched, value): + cipher, User = cipher_for(price=Annotated[decimal.Decimal, Numeric]) + error = refused(cipher, User(email=SECRET, price=value)) + assert error.code == "value_not_representable" + assert error.reason == "decimal" + + +def test_a_naive_datetime_is_refused(untouched): + cipher, User = cipher_for(seen=Annotated[datetime.datetime, Timestamp]) + error = refused(cipher, User(email=SECRET, seen=datetime.datetime(2026, 1, 1))) + assert error.code == "value_naive_datetime" + assert "tzinfo" in error.message + + +def test_every_model_is_checked_before_any_request(untouched): + # The bad value is in the last model of many. + cipher, User = cipher_for(age=Annotated[int, Bigint]) + models = [User(email=SECRET, age=i) for i in range(100)] + [User(email=SECRET, age=math.pi)] + with pytest.raises(InvalidValueError) as raised: + cipher.encrypt(models) + assert raised.value.field == "age" + + +def test_a_property_that_raises_propagates_unchanged(untouched): + @stackencrypt.model("users") + class User: + email: Annotated[str, TextEq] + + def __init__(self, email): + self._email = email + + @property + def email(self): + raise LookupError("the user's own error") + + with pytest.raises(LookupError, match="the user's own error"): + Cipher(User).encrypt([User(email=SECRET)]) + + +def test_a_zero_decimal_with_a_huge_exponent_passes_the_value_check(): + # The read is constant time: a loop per unit of exponent would hang here. + # The call then fails building the client, with no client key set. + cipher, User = cipher_for(price=Annotated[decimal.Decimal, Numeric]) + with pytest.raises(StashError) as raised: + cipher.encrypt([User(email=SECRET, price=decimal.Decimal("0E+999999999999999999"))]) + assert not isinstance(raised.value, InvalidValueError) + + +def test_a_bad_zerokms_host_is_refused_naming_it(monkeypatch, untouched): + monkeypatch.setenv("CS_ZEROKMS_HOST", "not a url") + cipher, User = cipher_for() + with pytest.raises(StashError) as raised: + cipher.encrypt([User(email=SECRET)]) + assert raised.value.code == "stack_kms_invalid_endpoint" + assert "CS_ZEROKMS_HOST" in str(raised.value) diff --git a/languages/python/tests/test_encrypt_live.py b/languages/python/tests/test_encrypt_live.py new file mode 100644 index 000000000..b6a54e547 --- /dev/null +++ b/languages/python/tests/test_encrypt_live.py @@ -0,0 +1,106 @@ +"""``Cipher.encrypt`` against ZeroKMS and a PostgreSQL with the EQL v4 bundle. + +These tests need CipherStash credentials (``CS_CLIENT_ACCESS_KEY``, +``CS_WORKSPACE_CRN``, ``CS_CLIENT_ID``, ``CS_CLIENT_KEY``) and a database +URL in ``EQL_TEST_DATABASE_URL`` whose database has the EQL v4 bundle +installed. Without them every test here skips. ``mise run test:live`` +installs the bundle and runs them, and fails if any skips. +""" + +import dataclasses +import decimal +import os +from typing import Annotated + +import pytest + +import stackencrypt +from stackencrypt import Cipher, Passthrough, TextEq +from stackencrypt import _engine + +CREDENTIALS = ("CS_CLIENT_ACCESS_KEY", "CS_WORKSPACE_CRN", "CS_CLIENT_ID", "CS_CLIENT_KEY") +DATABASE_URL = os.environ.get("EQL_TEST_DATABASE_URL", "") + +pytestmark = [ + pytest.mark.live, + pytest.mark.skipif( + not all(os.environ.get(name) for name in CREDENTIALS), + reason="needs CS_CLIENT_ACCESS_KEY, CS_WORKSPACE_CRN, CS_CLIENT_ID and CS_CLIENT_KEY", + ), +] + +needs_database = pytest.mark.skipif(not DATABASE_URL, reason="needs EQL_TEST_DATABASE_URL") + + +@stackencrypt.model("users") +@dataclasses.dataclass +class User: + email: Annotated[str, TextEq] + age: Annotated[int, Passthrough] + + +@stackencrypt.model("contacts") +@dataclasses.dataclass +class Contact: + email: Annotated[str, TextEq] + + +@needs_database +def test_an_encrypted_row_inserts_with_a_plain_parameter(): + psycopg = pytest.importorskip("psycopg") + rows = Cipher(User).encrypt([User(email="a@example.com", age=3)]) + + assert set(rows[0]) == {"email", "age"} + assert isinstance(rows[0]["email"], str) + assert rows[0]["age"] == 3 + with psycopg.connect(DATABASE_URL) as conn, conn.cursor() as cur: + cur.execute( + "CREATE TEMPORARY TABLE users (email public.eql_v4_text_eq, age int)" + ) + # The README's insert: a plain %s, no cast. Postgres infers the + # domain from the column for a str parameter. + cur.execute( + "INSERT INTO users (email, age) VALUES (%s, %s)", + (rows[0]["email"], rows[0]["age"]), + ) + cur.execute("SELECT email IS NOT NULL, pg_typeof(email)::text, age FROM users") + assert cur.fetchone() == (True, "eql_v4_text_eq", 3) + + +def test_the_plaintext_is_not_in_the_encrypted_column(): + rows = Cipher(User).encrypt([User(email="hidden@example.com", age=3)]) + assert "hidden@example.com" not in rows[0]["email"] + + +def test_1001_one_field_models_make_three_key_requests(): + cipher = Cipher(Contact) + # The first call in the process may load the keyset. Warm the client up + # first, so the count below is the key requests alone. + cipher.encrypt([Contact(email="warm@example.com")]) + before = _engine.request_count() + + cipher.encrypt([Contact(email=f"user{i}@example.com") for i in range(1001)]) + + # ceil(1001 / 500): stack-kms's 500 keys per request. + assert _engine.request_count() - before == 3 + + +def test_two_ciphers_share_one_engine_client(): + Cipher(User).encrypt([User(email="a@example.com", age=1)]) + Cipher(Contact).encrypt([Contact(email="b@example.com")]) + assert _engine.client_builds() == 1 + + +@stackencrypt.model("prices") +@dataclasses.dataclass +class Price: + sku: Annotated[str, TextEq] + amount: Annotated[decimal.Decimal, Passthrough] + + +def test_a_decimal_passthrough_comes_back_as_the_same_decimal(): + rows = Cipher(Price).encrypt([Price(sku="a", amount=decimal.Decimal("19.50"))]) + amount = rows[0]["amount"] + assert type(amount) is decimal.Decimal + # The scale is kept: 19.50, not 19.5. + assert str(amount) == "19.50" diff --git a/scripts/__tests__/eql-suite-ci.test.mjs b/scripts/__tests__/eql-suite-ci.test.mjs index c0e14ad30..14f2ba582 100644 --- a/scripts/__tests__/eql-suite-ci.test.mjs +++ b/scripts/__tests__/eql-suite-ci.test.mjs @@ -577,8 +577,30 @@ const RUST_CACHE_STEPS = workflowFiles().flatMap((relPath) => { ) }) -/** The rust-cache steps in EQL's jobs. */ -const EQL_RUST_CACHE_STEPS = RUST_CACHE_STEPS.filter(({ eql }) => eql) +/** + * Whether a rust-cache step names the root of another Cargo workspace: a + * directory outside the EQL tree with its own `Cargo.lock`. A job that builds + * both EQL and that workspace (tests-python.yml's `live` builds the Python + * wheel and the EQL bundle) caches each in its own step, and the other + * workspace's step is right where it is. A member of the EQL workspace, or + * the monorepo root, is not another workspace, so this check still catches + * both. + */ +const namesAnotherWorkspace = ({ step }) => { + const dir = step?.with?.workspaces + return ( + typeof dir === 'string' && + dir !== '.' && + dir !== CARGO_WORKSPACE && + !dir.startsWith(`${CARGO_WORKSPACE}/`) && + existsSync(join(REPO_ROOT, dir, 'Cargo.lock')) + ) +} + +/** The rust-cache steps in EQL's jobs that cache the EQL workspace. */ +const EQL_RUST_CACHE_STEPS = RUST_CACHE_STEPS.filter( + (entry) => entry.eql && !namesAnotherWorkspace(entry), +) const stepLabel = ({ relPath, jobName }, detail) => `${relPath} (${jobName}): ${detail}` diff --git a/scripts/__tests__/ffi-binding-step-order.test.mjs b/scripts/__tests__/ffi-binding-step-order.test.mjs index 517ccf9a1..406411400 100644 --- a/scripts/__tests__/ffi-binding-step-order.test.mjs +++ b/scripts/__tests__/ffi-binding-step-order.test.mjs @@ -229,6 +229,10 @@ const BINDING_EXEMPT_JOBS = new Map([ '.github/workflows/test-eql.yml / e2e', 'Rust: the proptest-e2e oracle encrypts through cipherstash-client, and never loads index.node or dist/wasm.', ], + [ + '.github/workflows/tests-python.yml / live', + 'Rust and Python: the Python live tests encrypt through the stackencrypt extension, built from languages/python by maturin, which links stack-encrypt directly. It never loads index.node or dist/wasm.', + ], [ '.github/workflows/tests-golang.yml / live', 'Go and Rust: the Go live tests encrypt through the stack-encrypt WASI guest, and the stack-encrypt examples link the crate directly. Neither loads index.node or dist/wasm.', diff --git a/scripts/__tests__/python-ci.test.mjs b/scripts/__tests__/python-ci.test.mjs index f27814809..cd3784079 100644 --- a/scripts/__tests__/python-ci.test.mjs +++ b/scripts/__tests__/python-ci.test.mjs @@ -317,11 +317,15 @@ describe('tests-python.yml', () => { } }) - it('gives no job a CipherStash credential', () => { - // The live job (credentials, Postgres) is #1389's and lands as its own - // job. Until then nothing in this file may read a CS_* variable or - // secret: a credentialed job is one ffi-binding-step-order.test.mjs - // holds to a pre-flight and a binding build this workflow has no use for. - expect(read(WORKFLOW)).not.toMatch(/(vars|secrets)\.CS_/) + it('gives a CipherStash credential to the live job only, which skips on a fork', () => { + // A credentialed job is one ffi-binding-step-order.test.mjs holds to the + // secrets pre-flight. Only `live` needs one (#1389). A fork's pull + // request has no secrets, so the job skips there rather than fails. + const { jobs } = readWorkflow(WORKFLOW) + const credentialed = Object.entries(jobs) + .filter(([, job]) => /(vars|secrets)\.CS_/.test(JSON.stringify(job))) + .map(([id]) => id) + expect(credentialed).toEqual(['live']) + expect(jobs.live.if).toMatch(/head\.repo\.full_name == github\.repository/) }) }) diff --git a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs index 33c6854ac..e0f7ebd14 100644 --- a/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs +++ b/scripts/__tests__/workflow-dispatch-job-conditions.test.mjs @@ -128,6 +128,8 @@ const EXPECTED_FORK_GUARDED_JOBS = [ '.github/workflows/prisma-next-e2e.yml / e2e', '.github/workflows/test-eql.yml / build-archive', '.github/workflows/test-eql.yml / e2e', + // The CS_* secrets, which forks are not given (#1389). + '.github/workflows/tests-python.yml / live', ] /** The context path every fork guard turns on. */