diff --git a/.agents/skills/diffractscout-agent-pipeline/SKILL.md b/.agents/skills/diffractscout-agent-pipeline/SKILL.md new file mode 100644 index 0000000..6b1893d --- /dev/null +++ b/.agents/skills/diffractscout-agent-pipeline/SKILL.md @@ -0,0 +1,49 @@ +--- +name: diffractscout-agent-pipeline +description: Execute DiffractScout scientific jobs through its durable CLI, Python or local MCP interface, including planning, candidate discovery, CIF preparation, theoretical diffraction, recovery and verified exports. Use for research execution requests, not general repository coding or experimental-pattern fitting. +--- + +# DiffractScout scientific jobs + +Use the shared workflow service to turn the user's scientific request into a +recorded calculation with verified exports. Read the repository's +[agent pipeline guide](../../../docs/AGENT_PIPELINE.md) for execution and +recovery, and [scientific contracts](../../../docs/SCIENTIFIC_CONTRACTS.md) +when interpreting numerical fields. Concrete requests are in +[examples/agent](../../../examples/agent/README.md). + +Obtain `diffractscout agent context` and `diffractscout agent schema request` +(or their MCP/Python equivalents) for current capabilities and fields. Choose +the existing workflow for the requested outcome; use `prepare_analyze` when +starting CIF preparation and subsequent theoretical analysis are both requested. +Do not build another numerical implementation or add a language model to the +calculation path. + +Preserve the requested composition, local source identity, radiation and scan +range. Check available inputs before asking for missing information. Synchrotron +peak positions require an explicit wavelength or energy; ask for it if unavailable. +Nominal chemistry and prototype cell parameters are starting assumptions. Cited +lattice/coordinate edits must come from supplied evidence. Do not infer phase +fractions, invent elastic constants or claim experimental phase identification. + +Normalize relative paths from the request file, select a workspace that remains +stable across sessions, and omit `output_dir` for a unique service-managed output +or select a fresh target outside the input tree. +Plan the request and resolve actionable diagnostics before execution. Keep +Materials Project credentials in the runtime environment (`MP_API_KEY`) or +runtime API argument, never in the request or outputs. Respect the user's +database scope and download authorization; do not widen limits to clear a failure. + +Use `submit` for work that may outlive the current tool call. Record the returned +`run_id` and workspace, poll `status`, and consume new event pages using +`next_cursor`. A successful submission is not completion. Before resubmitting +after a lost response, inspect existing jobs. Resume an interrupted or blocked +job only after addressing its diagnostic; preserve checkpoint and source checks. +Cancellation is cooperative and retains completed evidence. Batch independent +requests serially unless finite concurrency is explicitly appropriate. + +After a completed or partial run, obtain `result` with integrity verification. +Request table pages only as needed. Report artifact paths, radiation/range, +source/prototype assumptions, diagnostics and missing values with the result. +If verification fails or work remains blocked, identify the affected stage and +required input; do not present an unverified bundle or partial task as complete. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6589c48..6cb9961 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,6 +97,25 @@ jobs: path: ci_reference_engines if-no-files-found: warn + mcp-stdio: + name: MCP stdio / Python ${{ matrix.python-version }} / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.10", "3.13"] + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Install optional official MCP SDK and test tools + run: python -m pip install -e ".[mcp,test]" + - name: Exercise JSON CLI and real MCP stdio client + run: python -m pytest -q tests/test_agent_cli.py tests/test_mcp_server.py + cross-platform-smoke: name: Smoke / ${{ matrix.os }} runs-on: ${{ matrix.os }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 11112c6..02edc1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ All notable changes are recorded here. The project follows semantic versioning a ### Added +- Durable scientific jobs shared by the JSON `agent` CLI, Python workflow + service, optional local stdio MCP tools and bilingual desktop Tasks page. + Supports existing discovery, CIF preparation/adaptation, diffraction, + elasticity and exports with planning, progress events, bounded batches, + source/checkpoint integrity checks, recovery and cooperative cancellation. + Includes offline request examples and a project scientific-workflow skill. + - Contextual help throughout the main desktop, initial CIF preparation, results and retained workbenches. Delayed hover, keyboard focus and F1 explain actions, drop-down choices, units and table headings in plain diff --git a/README.md b/README.md index 3d30ac3..81e53bb 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ **Turn local CIF structures or a chemical-system query into theoretical diffraction references with verifiable provenance.** -[Install and GUI](#graphical-interface) · [CLI guide](docs/CLI.md) · [Scientific contracts](docs/SCIENTIFIC_CONTRACTS.md) · [中文说明](README.zh-CN.md) +[Install and GUI](#graphical-interface) · [CLI guide](docs/CLI.md) · [External agents](docs/AGENT_PIPELINE.md) · [Scientific contracts](docs/SCIENTIFIC_CONTRACTS.md) · [中文说明](README.zh-CN.md) [![CI](https://github.com/D-sudoasd/DiffractScout/actions/workflows/ci.yml/badge.svg)](https://github.com/D-sudoasd/DiffractScout/actions/workflows/ci.yml) [![MIT](https://img.shields.io/badge/License-MIT-455A64)](LICENSE) [![Python 3.10–3.13](https://img.shields.io/badge/Python-3.10--3.13-3776AB)](pyproject.toml) @@ -96,6 +96,30 @@ illustrative and may show English even when a fresh launch starts in Chinese. On Windows, double-click `启动DiffractScout.bat` after an editable install, or drag CIF files onto `quick_export_diffractscout.bat` for a one-shot lab export. Both repository launchers are source-checkout convenience entry points: they run the checkout source through `scripts/diffractscout_entry.py` and prefer the repository `.venv\Scripts\python.exe`, then the current/active `python`, then `py -3`. The quick-export launcher also uses an installed `diffractscout-quick-export` command when `.venv` is absent. Installed `diffractscout-gui` / `diffractscout gui` / `diffractscout-quick-export` do not require the repository launchers. +## External-agent workflows + +The JSON CLI, Python workflow service and optional local stdio MCP adapter share +durable jobs with the desktop **Tasks / 任务** page. An external agent can plan +existing discovery, CIF preparation/adaptation, theoretical diffraction, +elasticity and export workflows, submit long work, inspect progress, resume +interrupted items and read verified results. + +```bash +diffractscout agent context +diffractscout agent plan examples/agent/analyze-local.json +diffractscout agent run examples/agent/analyze-local.json +diffractscout agent list +``` + +These examples work offline after the base installation. For a long job, use +`submit` and follow its `run_id` with `status`, `events` and `result`. See the +[agent pipeline guide](docs/AGENT_PIPELINE.md), +[runnable requests](examples/agent/README.md) and +[project workflow skill](.agents/skills/diffractscout-agent-pipeline/SKILL.md). +Install `.[mcp]` only when connecting an MCP client. Database credentials remain +runtime inputs; the service does not contain an internal LLM or experimental +pattern-fitting engine. + ## CIF2Peaks parity features DiffractScout reimplements the CIF2Peaks desktop workflow inside a provenance-first package (Gemmi engine; not bit-identical intensities). Practical parity includes: @@ -250,6 +274,7 @@ in the [CLI guide](docs/CLI.md). |---|---|---| | base | `python -m pip install -e .` | Offline local CIF analysis, CLI/API, synthetic demo and verification | | `mp` | `python -m pip install -e ".[mp]"` | Materials Project discovery/download and optional provider metadata; requires your own API key | +| `mcp` | `python -m pip install -e ".[mcp]"` | Optional local stdio MCP tools for the shared durable scientific workflow service | | `figures` | `python -m pip install -e ".[figures]"` | Optional matplotlib rendering path and paper-figure tooling | | `gui-dnd` | `python -m pip install -e ".[gui-dnd]"` | Optional `tkinterdnd2` file/folder drag-and-drop; button-based GUI use remains available without it | | `test` | `python -m pip install -e ".[test]"` | Pytest, coverage, Ruff, YAML support, and development checks | diff --git a/README.zh-CN.md b/README.zh-CN.md index 2dd19bf..449dfde 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -6,7 +6,7 @@ **从本地 CIF 或化学体系检索出发，生成带来源、诊断和校验清单的理论衍射参考。** -[图形界面与安装](#图形界面) · [CLI](docs/CLI.md) · [科研约定](docs/SCIENTIFIC_CONTRACTS.md) · [英文主页](README.md) +[图形界面与安装](#图形界面) · [CLI](docs/CLI.md) · [外部 agent](docs/AGENT_PIPELINE.md) · [科研约定](docs/SCIENTIFIC_CONTRACTS.md) · [英文主页](README.md) [![CI](https://github.com/D-sudoasd/DiffractScout/actions/workflows/ci.yml/badge.svg)](https://github.com/D-sudoasd/DiffractScout/actions/workflows/ci.yml) [![MIT](https://img.shields.io/badge/License-MIT-455A64)](LICENSE) [![Python 3.10–3.13](https://img.shields.io/badge/Python-3.10--3.13-3776AB)](pyproject.toml) @@ -51,6 +51,27 @@ macOS 桌面不作为支持目标。命令行输出统一为 UTF-8，重定向 [查看完整示意图 / View full-size schematic](assets/readme/principle.png) +## 外部 agent 科研工作流 + +JSON CLI、Python 工作流服务和可选的本地 stdio MCP 共用持久任务记录， +桌面“任务”页可查看同一工作区。外部 agent 提供科研参数后，可规划已有的候选 +检索、CIF 准备与显式修改、理论衍射、弹性和导出流程，再提交长任务、查看进度、 +恢复中断项目并读取经过完整性校验的结果。 + +```bash +diffractscout agent context +diffractscout agent plan examples/agent/analyze-local.json +diffractscout agent run examples/agent/analyze-local.json +diffractscout agent list +``` + +基础安装即可离线运行以上示例。长任务使用 `submit`，保留返回的 `run_id`，再用 +`status`、`events` 和 `result` 查看状态与结果。参见 +[完整说明](docs/AGENT_PIPELINE.md)、[可运行请求](examples/agent/README.md) 和 +[项目工作流技能](.agents/skills/diffractscout-agent-pipeline/SKILL.md)。需要连接 MCP +客户端时安装 `.[mcp]`；数据库密钥只在运行时提供。此入口不包含内部大模型或实验 +谱线拟合；理论计算和原型起始模型的科研边界沿用现有约定。 + ## 主要工作流 使用与开发入口：[文档索引](docs/README.md) · [贡献指南](CONTRIBUTING.md) · @@ -192,6 +213,7 @@ Remove-Item Env:MP_API_KEY |---|---|---| | base | `python -m pip install -e .` | 离线本地 CIF、CLI/API、合成演示和校验 | | `mp` | `python -m pip install -e ".[mp]"` | Materials Project 检索/下载和可选 provider 元数据；需要自己的 API 密钥 | +| `mcp` | `python -m pip install -e ".[mcp]"` | 通过本地 stdio MCP 连接共用的持久科研任务服务 | | `figures` | `python -m pip install -e ".[figures]"` | 可选 matplotlib 渲染路径和论文制图工具 | | `gui-dnd` | `python -m pip install -e ".[gui-dnd]"` | 可选 `tkinterdnd2` 文件/文件夹拖放；没有它仍可使用按钮式 GUI | | `test` | `python -m pip install -e ".[test]"` | pytest、覆盖率、Ruff、YAML 支持和开发检查 | diff --git a/docs/AGENT_PIPELINE.md b/docs/AGENT_PIPELINE.md new file mode 100644 index 0000000..df8680e --- /dev/null +++ b/docs/AGENT_PIPELINE.md @@ -0,0 +1,203 @@ +# External-agent scientific workflows + +DiffractScout exposes its existing scientific workflows through one durable +job service, shared by the Python API, JSON CLI, optional local MCP server and +desktop Tasks page. An external agent supplies scientific intent and parameters; +DiffractScout performs the calculations and records their provenance. There is +no internal language model or experimental-pattern fitting in this interface. + +## Choose the workflow + +| `workflow` | Input and outcome | +| --- | --- | +| `analyze` | Local CIF files/folders → structure checks, theoretical diffraction, optional compatible elasticity, exports | +| `discover` | Composition/system/record IDs → candidate catalogue without structure downloads | +| `run` | Database discovery → downloads → analysis and exports | +| `prepare` | Grade or explicit chemistry, optional templates/cited parameters → checked starting CIFs and preparation report | +| `prepare_analyze` | Preparation → analysis of the prepared CIFs and exports | +| `fetch` | Chemical system and phase selection/templates → symmetry prototypes | +| `adapt` | One source CIF and explicit chemistry/cited cell or coordinates → a new CIF with provenance | + +Discovery is a database candidate search, not identification of phases in a +sample. Preparation can retain prototype parameters and nominal chemistry; +it does not predict phase fractions, phase partitioning or phase equilibrium. +Read [scientific contracts](SCIENTIFIC_CONTRACTS.md) and the +[initial CIF guide](INITIAL_CIFS.md) when interpreting these results. + +## Discover, plan, execute, inspect + +Install the current checkout with `python -m pip install -e .`. Use one explicit +workspace for a continuing task, especially when the caller can change its +working directory. The default is `outputs/agent-workspace` relative to the +process working directory. The examples below run from the repository root. + +```bash +diffractscout agent context +diffractscout agent schema request +diffractscout agent plan examples/agent/analyze-local.json +diffractscout agent run examples/agent/analyze-local.json +``` + +`context` reports workflows, optional dependency availability and whether runtime +credentials are configured. `schema request` is the authoritative field/default +reference for `diffractscout_workflow_v1`; obtain it rather than copying a schema +from prose. Unknown fields and credentials in a request are rejected. +`plan` checks local inputs, settings, dependencies and required information without +network access or output writes. Inspect its `ready` and `diagnostics` fields: +a successful CLI invocation alone does not establish readiness. Planning cannot +guarantee later database availability or scientific suitability of a candidate. + +For long jobs, submit and retain the returned identity. In subsequent commands, +replace `RUN_ID` with the returned `run_id` and keep the same workspace. + +```bash +diffractscout agent --workspace outputs/agent-workspace submit examples/agent/prepare-analyze-tc4.json +diffractscout agent --workspace outputs/agent-workspace status RUN_ID +diffractscout agent --workspace outputs/agent-workspace events RUN_ID --after 0 --limit 50 +diffractscout agent --workspace outputs/agent-workspace result RUN_ID +``` + +`submit` returns after starting the background worker. The client may disconnect; +the durable record remains available to a later CLI, MCP or desktop session. +Poll `status` at an interval appropriate to the task, then read new event pages +using the returned `next_cursor` as the next `--after` value. `has_more` indicates +that more events are available now. Do not resubmit the same calculation merely +because the client lost a response; inspect `list` and existing job records first. + +`run` waits in the foreground and is useful for short/offline work. JSON stdout +contains one result or error object; diagnostics go to stderr. CLI exit codes +are `0` for a successful operation, `3` for partial work, and `2` for errors or +failed, blocked, interrupted or cancelled work. Always inspect the returned state +and diagnostics as well as the exit code. `pending`/`running` are not completion. + +## Requests and paths + +See [runnable offline examples](../examples/agent/README.md) for local analysis, +explicit synchrotron radiation, TC4 preparation plus analysis and serial batches. +Request sections select `analysis`, `discovery`, `preparation`, `adaptation`, +`exports`, and `execution` options as applicable. `analysis` and `discovery` +reuse the ordinary scientific settings. The supported workbook templates are +`full` and `peak_table`; CSV exports remain available with Excel disabled. + +For CLI JSON files and desktop imports, paths are relative to the request file, +including CIF inputs, preparation templates, phase-parameter files and output +targets. Inline JSON, stdin, MCP and ordinary Python mappings use the calling +process directory unless paths have already been normalized. Python callers can +use `WorkflowRequest.from_dict(payload, base_dir=...)` to set the base explicitly. +Workspace selection is independent of the request file's base directory. + +Omit `output_dir` for a unique service-managed output in the workspace, or supply +a fresh output target outside the input tree. `execution.overwrite` does +not grant permission to delete arbitrary directories or bypass bundle checks. +Mutable job records and checkpoints belong outside committed result bundles; +editing a file in a verified bundle invalidates its integrity record. + +Specify radiation and range for the scientific question. Synchrotron requests +require an explicit `energy_keV` or `wavelength_A`; a missing value must not be +replaced by the laboratory default. Units are 2θ in degrees, wavelength and d +in Å, q = 2π/d and g = 1/d in Å⁻¹, and stiffness in GPa. +Cell or fractional-coordinate adaptations require a citation supplied by the +caller. Missing elastic data remain missing, and directional moduli require a +compatible tensor coordinate frame. + +## Recovery and cancellation + +```bash +diffractscout agent list +diffractscout agent status RUN_ID +diffractscout agent resume RUN_ID +diffractscout agent resume RUN_ID --foreground +diffractscout agent cancel RUN_ID +``` + +The service persists state JSON and event JSONL, source snapshots, and checkpoints +for completed discovery queries, downloads and phase analyses. Resume verifies +saved source snapshots and checkpoint hashes before reuse. Completed work can be +reused after interruption; an in-flight operation may need to run again. Changed +artifacts must be resolved from the reported diagnostic, not by removing locks +or modifying the saved request/checkpoint files. A submitted job uses its frozen +sources; edits to an original input require a new task to analyze the new data. + +Status distinguishes `completed`, `partial`, `failed`, `blocked`, `interrupted` +and `cancelled`. A partial result needs its per-item diagnostics inspected before +use. A blocked job needs its stated missing dependency, credentials, parameter or +authorization resolved before resuming. Runtime credentials must be supplied +again if the worker process/environment no longer has them. Cancellation is +cooperative: it takes effect at a safe operation boundary and preserves completed +evidence. Restarting the desktop does not erase task history; OS termination +may interrupt the worker and requires a later explicit resume. + +## Read verified results + +`result` verifies the published artifact set by default and reports artifact +paths and available tables. Use a table name returned by that response to request +a page rather than loading all rows into an agent conversation: + +```bash +diffractscout agent result RUN_ID +diffractscout agent result RUN_ID --table TABLE_NAME --offset 0 --limit 50 +``` + +Verification failures must be reported as failures; `--no-verify` is a diagnostic +option and does not establish that modified output is valid. File integrity +demonstrates preservation of the recorded computation, not agreement with an +experiment. Report the radiation, range, source identity, warnings, missing +values and relevant prototype assumptions alongside scientific tables. + +## Batches and database access + +The `diffractscout_batch_v1` manifest contains `requests` and `max_workers`. +`run` or `submit` accepts this wrapper; plan individual requests before execution. +Execution is serial by default. Increase concurrency explicitly and finitely +only for independent requests with separate output targets; batch children have +their own task identities and status records. See the +[offline batch example](../examples/agent/batch-offline.json). + +Database workflows need `.[mp]` and runtime `MP_API_KEY`. Do not place the key in +requests, saved configuration, task names, examples or result metadata. Optional +Python `api_key` arguments and desktop key entry are runtime-only inputs. Keep +subsystem/candidate limits within the intended search and inspect query estimates +before execution. Large downloads require `execution.authorize_large_download` +when exceeding `execution.confirm_above`; repeated network attempts are bounded +by `execution.max_attempts`. A provider failure is a diagnostic, not an empty +candidate list. + +## Python and local MCP + +The [API guide](API.md#durable-workflow-service) shows Python calls. The optional +MCP adapter uses the same service and requires a separate extra: + +```bash +python -m pip install -e ".[mcp]" +python -m diffractscout.mcp_server +``` + +Configure a local MCP client to launch the interpreter with module arguments, +for example: + +```json +{ + "mcpServers": { + "diffractscout": { + "command": "python", + "args": ["-m", "diffractscout.mcp_server"] + } + } +} +``` + +Use the interpreter containing the installed package. Provide `MP_API_KEY` through +the server's runtime environment only for database work. MCP transport is stdio; +the process is not an HTTP service. Tools are `context`, `schema`, `plan`, `run`, +`submit`, `list_jobs`, `status`, `events`, `result`, `resume`, and `cancel`. +Read calls return bounded pages where applicable. A long `run` remains a blocking +tool call; `submit` followed by `status`/`events` is the durable interaction. + +In the desktop, open **Tasks / 任务** and choose the same workspace to inspect +these jobs. See [desktop task history](GUI.md#task-history-and-recovery). +The project skill +[$diffractscout-agent-pipeline](../.agents/skills/diffractscout-agent-pipeline/SKILL.md) +guides an external agent through this scientific execution workflow. + +Implementation checks, recovery operation counts and installation evidence are +recorded in the [2026-10-22 acceptance report](evidence/agent-pipeline-20261022.md). diff --git a/docs/AGENT_WORKFLOW.md b/docs/AGENT_WORKFLOW.md index 2051f9d..c0c1409 100644 --- a/docs/AGENT_WORKFLOW.md +++ b/docs/AGENT_WORKFLOW.md @@ -45,11 +45,11 @@ python -m diffractscout benchmark -o "outputs/dev-benchmark-$runId" ## 项目技能的入口 -本仓库提供一个具体工作流：[$diffractscout-cif-reference](../.agents/skills/diffractscout-cif-reference/SKILL.md)。例如：“用这个 CIF 生成 Cu Kα、5–120° 的理论峰表，导出 Excel，并核验来源与单位。” +本仓库提供两个科研执行技能：[$diffractscout-cif-reference](../.agents/skills/diffractscout-cif-reference/SKILL.md) 用于本地 CIF 的直接峰表与 Excel 导出，例如：“用这个 CIF 生成 Cu Kα、5–120° 的理论峰表，导出 Excel，并核验来源与单位。”[$diffractscout-agent-pipeline](../.agents/skills/diffractscout-agent-pipeline/SKILL.md) 用于通过共用持久任务服务执行检索、CIF 准备、理论分析、恢复与结果核验；外部调用说明见 [agent pipeline](AGENT_PIPELINE.md)。 技能位于 Codex 支持的 `.agents/skills/` 仓库目录。其他 agent 可直接读取对应 `SKILL.md`。无需复制到全局技能库或修改全局模型设置；如果当前会话尚未显示新技能，可直接指定文件，或重新打开项目会话检查发现结果。 -技能复用现有 CLI 和校验工具；详细公式链接到科学约定。只在本地 CIF 导出与核验时触发。新增技能应解决一个真实重复流程，保持名称和描述短而明确，避免把通用编码任务吸入科研导出流程。 +技能复用现有 CLI、工作流服务和校验工具；详细公式链接到科学约定。按科研请求选择直接导出或持久任务流程，不对普通代码修改触发。新增技能应解决一个真实重复流程，保持名称和描述短而明确，避免把通用编码任务吸入科研导出流程。 ## 完成标准与规则回归 diff --git a/docs/API.md b/docs/API.md index 28a1838..0ce9d49 100644 --- a/docs/API.md +++ b/docs/API.md @@ -5,6 +5,62 @@ The public result schemas and scientific meanings are defined in validation workflow are documented in the [GUI guide](GUI.md), [README](../README.md), and [release procedure](RELEASE.md). +## Durable workflow service + +`diffractscout.workflow` provides the same durable jobs used by the JSON CLI, +local MCP adapter and desktop Tasks page. It wraps existing scientific routines; +the ordinary functions below remain available for direct synchronous use. +See the [agent pipeline guide](AGENT_PIPELINE.md) for request semantics, +recovery and scientific interpretation. + +```python +import json +from pathlib import Path +from diffractscout import workflow + +request_file = Path("examples/agent/analyze-local.json").resolve() +request = workflow.WorkflowRequest.from_dict( + json.loads(request_file.read_text(encoding="utf-8")), + base_dir=request_file.parent, +) +plan = workflow.plan_workflow(request) +if not plan["ready"]: + raise RuntimeError(plan["diagnostics"]) + +workspace = Path("outputs/agent-workspace").resolve() +job = workflow.submit_job(request, workspace=workspace) +run_id = job["run_id"] +print(run_id, workflow.status_job(run_id, workspace=workspace)) +``` + +Poll `status_job` until work reaches a terminal state; then inspect diagnostics +and call `get_result(run_id, workspace=workspace)` for verified artifact metadata. +For a short foreground call, use `run_job(request, workspace=workspace)` instead +of `submit_job`. Runtime credentials can be passed as `api_key=` or supplied +through `MP_API_KEY`; requests must not contain credentials. + +| Function | Purpose | +| --- | --- | +| `context()`, `schema("request")` | Discover capabilities/dependencies and current request fields/defaults | +| `plan_workflow(request)` | Check local readiness without execution, network access or writes | +| `run_job(request, workspace=...)`, `submit_job(request, workspace=...)` | Execute in the foreground or submit background work | +| `list_jobs(workspace=...)`, `status_job(run_id, workspace=...)` | Inspect durable history and progress | +| `read_events(run_id, workspace=..., after=0, limit=100)` | Read event pages; continue from `next_cursor` | +| `resume_job(run_id, workspace=..., background=True)`, `cancel_job(run_id, workspace=...)` | Resume with integrity checks or request cooperative cancellation | +| `get_result(run_id, workspace=..., table=None, offset=0, limit=100, verify=True)` | Verify artifacts and optionally read a named table page | +| `load_result_object(run_id, workspace=...)` | Verify and reconstruct the published result object for Python consumers | +| `run_batch(requests, workspace=..., max_workers=1, background=False)` | Run independent requests serially by default, with explicit finite concurrency | + +Pass a `WorkflowRequest` or request mapping. Use `WorkflowRequest.from_dict` +with `base_dir` when the mapping came from a file; ordinary mappings resolve +relative paths from the calling process directory. Omitted `output_dir` selects +a unique service-managed output. Workspace paths are independent of that base. +`schema()` and the service validate the contract; duplicating its full field +list in a caller is unnecessary. Batch requests use separate output targets and +retain individual job identities. `load_result_object` returns the result type +for that workflow; `get_result` provides a common artifact/table interface +across all workflow types, including `fetch` and `adapt`. + ## Local CIF analysis ```python diff --git a/docs/CLI.md b/docs/CLI.md index 98cf8fd..cedcecb 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -4,6 +4,53 @@ The command-line interface uses the same calculation and bundle-verification code as the Python API and desktop application. Run `diffractscout --help` for the command list or add `--help` to a command for its options. +## JSON interface for external agents + +`diffractscout agent` (also `diffractscout-agent`) returns JSON on stdout for +scientific execution and durable job control. Ordinary commands below remain +available for interactive shell use. + +```bash +diffractscout agent context +diffractscout agent schema request +diffractscout agent plan examples/agent/analyze-local.json +diffractscout agent run examples/agent/analyze-local.json +diffractscout agent submit examples/agent/prepare-analyze-tc4.json +diffractscout agent list +``` + +Replace `RUN_ID` with the identity returned by `run`, `submit` or `list`: + +```bash +diffractscout agent status RUN_ID +diffractscout agent events RUN_ID --after 0 --limit 50 +diffractscout agent result RUN_ID +diffractscout agent resume RUN_ID +diffractscout agent cancel RUN_ID +``` + +Use `--workspace PATH` before or after the subcommand to select durable history. +The default is `outputs/agent-workspace` relative to the current directory. +Requests can be a JSON filename, inline JSON object, or `-` for stdin. File +paths resolve from the request file; inline/stdin paths resolve from the current +directory. Missing `output_dir` gets a unique output in the workspace. +`plan` is local and read-only: check `ready` and `diagnostics` before execution. +For long work, use `submit`, retain `run_id`, then poll status and incremental +events. Submission alone is not completion. `resume --foreground` waits for +recovery instead of starting it in the background. + +`result` verifies artifacts by default; use `--table TABLE_NAME --offset 0 +--limit 50` for a table page after discovering available names. `events` returns +`next_cursor` for the next `--after` and `has_more` for further pages. Errors are +JSON objects; progress messages use stderr. Exit `0` means the operation +succeeded, `3` indicates partial work, and `2` indicates an error or unsuccessful +terminal state. Inspect returned states even when exit `0` is returned. + +The [agent pipeline guide](AGENT_PIPELINE.md) covers cancellation, source and +checkpoint integrity, runtime credentials, bounded batches, MCP and Python +equivalents. [Offline requests](../examples/agent/README.md) are runnable from +the checkout without a database key. + ## Start with a complete offline run ```bash diff --git a/docs/GUI.md b/docs/GUI.md index 6a1b954..641373f 100644 --- a/docs/GUI.md +++ b/docs/GUI.md @@ -39,8 +39,8 @@ to Chinese. ## Layout and scrolling -The workbench separates **Local CIF analysis**, **Materials Project**, and -**Analysis results**. Common radiation, scan-range and export controls appear +The workbench separates **Local CIF analysis**, **Materials Project**, +**Tasks**, and **Analysis results**. Common radiation, scan-range and export controls appear first. **Profile and d-spacing**, **Resource limits**, and **Cij** settings expand on demand; collapsing them retains their values. Chinese and English choices map to the same canonical settings used by the CLI and API. @@ -90,6 +90,39 @@ Language changes retain the input selection and all calculation settings. Duplicate input paths are removed. Only readable files with the `.cif` extension enter the calculation. Selecting `Replace an existing verified DiffractScout bundle` authorizes replacement only when the existing directory contains a supported manifest and currently passes the bundle-integrity check. +## Task history and recovery + +![Chinese Tasks page at the minimum client size](assets/gui-tasks-zh.png) + +The screenshot uses synthetic history to illustrate the task controls; an +[English capture](assets/gui-tasks-en.png) is also available. + +Open **Tasks / 任务** to see scientific jobs saved by the desktop, JSON CLI, +Python service or local MCP server. Choose the same **Workspace / 工作区** used +by the caller; the default is `outputs/agent-workspace` relative to the launch +directory. Changing the workspace changes the displayed history. + +- **Import request / 导入请求** selects a workflow JSON file and submits it. + Paths in that file resolve from its own directory. The + [offline requests](../examples/agent/README.md) provide working examples. +- **Refresh / 刷新** rereads saved jobs. The page also refreshes while open; + selecting a row shows its state, stage, item counts, diagnostics and events. +- **Resume / 继续** continues an eligible unfinished job after its reported + problem has been resolved. The service checks source/checkpoint integrity + before reusing completed work. Supply a runtime database key again if needed. +- **Cancel / 取消** requests a stop at a safe operation boundary. Completed + evidence is retained; the current network or calculation operation may finish + before the state changes. +- **Open result / 打开结果** verifies a completed or partial output, then opens + its result view when available. Prototype-fetch and CIF-adaptation jobs show + their artifact summary. Read partial-job diagnostics before scientific use. + +Closing and reopening the application retains history in the workspace. If the +worker is interrupted, select its recorded job and resume it; do not remove lock +files or edit checkpoints. A submission message means the job has been saved, +and the state must reach completion before the result is ready. Details of +states and the shared service are in the [agent pipeline guide](AGENT_PIPELINE.md). + ## Reuse analysis settings Use **Save parameters** to store the current analysis and export options in a diff --git a/docs/README.md b/docs/README.md index 03d854c..e0f77ae 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,7 @@ and [Chinese README](../README.zh-CN.md) cover installation and first use. | Use the desktop application | [GUI](GUI.md) | | Read or save Excel results | [Excel](EXCEL.md) | | Call the Python API | [API](API.md) | +| Execute scientific jobs from an external agent, resume work or connect MCP | [Agent pipeline](AGENT_PIPELINE.md), [offline requests](../examples/agent/README.md) | | Interpret equations, units, assumptions and limits | [Scientific contracts](SCIENTIFIC_CONTRACTS.md) | | Understand field aliases and engine differences | [Schema aliases](SCHEMA_ALIASES.md), [engine parity](ENGINE_PARITY.md) | | Make a code or documentation change | [Contributing](../CONTRIBUTING.md), [agent workflow](AGENT_WORKFLOW.md), [project instructions](../AGENTS.md) | diff --git a/docs/assets/gui-tasks-en.png b/docs/assets/gui-tasks-en.png new file mode 100644 index 0000000..9e074c3 Binary files /dev/null and b/docs/assets/gui-tasks-en.png differ diff --git a/docs/assets/gui-tasks-zh.png b/docs/assets/gui-tasks-zh.png new file mode 100644 index 0000000..cf1588d Binary files /dev/null and b/docs/assets/gui-tasks-zh.png differ diff --git a/docs/evidence/agent-pipeline-20261022.md b/docs/evidence/agent-pipeline-20261022.md new file mode 100644 index 0000000..b8614b1 --- /dev/null +++ b/docs/evidence/agent-pipeline-20261022.md @@ -0,0 +1,105 @@ +# Agent pipeline acceptance — 2026-10-22 + +Local acceptance on Windows 11, Python 3.13.0. This record concerns software +execution, scientific-contract preservation and recovery. It does not establish +experimental phase identification or quantitative phase analysis. + +## Delivered behavior + +- Shared versioned request/planning service for Python, JSON CLI, optional local + MCP and the desktop Tasks page. +- Frozen source snapshots; SHA-256-verified per-query, per-material and per-phase + checkpoints; independent final export, verification and atomic publication. +- Foreground/background runs, bounded batches, idempotent submission identifiers, + cancellation, interrupted-run recovery and verified paginated result access. +- Local CIFs, database discovery/analysis, initial CIF preparation plus analysis, + explicit prototype retrieval and explicit CIF adaptation. +- Per-field parameter origins, runtime-only credentials, original input preservation, + result coverage, scientific warnings and portable replay requests. + +The numerical definitions and ordinary CLI result contracts are retained. +New workflow metadata is added only by the new execution service. + +## Verification + +| Check | Observed result | +| --- | --- | +| Offline suite, excluding main GUI and Tasks GUI modules | 851 passed, 6 skipped, 168.24 s | +| Final affected core/delivery/CLI regressions after background-process corrections | 70 passed, 32.06 s | +| Analytic scientific benchmark | 45/45 checks passed | +| Six-module GUI suite | 115 passed | +| Final Tasks GUI checks | 8 passed; one Tk initialization skip passed in a fresh isolated invocation | +| Optional MCP protocol pytest | 2 passed, including an actual stdio subprocess | +| Clean installed-wheel CLI checks | 9 commands passed, outside the checkout and without PYTHONPATH | +| Clean installed-wheel MCP checks | 18 official SDK 2.3.0 calls passed, including background completion and pagination | +| Installed dependency consistency | pip check passed | +| Wheel/source comparison | All 87 packaged Python files byte-match the accepted source | +| Ruff, documentation checker, whitespace checks | Passed | + +The offline-suite skips comprise five unavailable inherited external CIF fixtures +and the optional MCP module in the base interpreter; MCP was checked separately. +One repeated-root Tk initialization transiently reported a missing Tcl script; +the skipped import/relative-path test passed alone. The actual local-job test +closed the GUI during work, observed independent completion, reopened history and +loaded the verified result. Offline preparation and 900×640 Chinese/English +interaction checks also passed. The accompanying screenshots illustrate synthetic +task history, not experimental results. + +Full offline command (source checkout): + +```powershell +$env:PYTHONPATH = Join-Path (Get-Location) 'src' +py -3 -m pytest -q --ignore=tests/test_gui.py --ignore=tests/test_gui_tasks.py +``` + +Focused final command: + +```powershell +py -3 -m pytest -q tests/test_workflow.py tests/test_workflow_delivery.py tests/test_agent_cli.py +``` + +The independent review found two concrete recovery/delivery defects: a valid +published bundle could become blocked if its task acknowledgement failed, and +discovery replay requests were incorrectly converted to analysis requests. +Both have regression coverage. Additional regressions cover retained Windows +process handles, duplicate batch submissions, relative tensor identities and +portable replay of manually supplied elastic tensors. + +## Measured recovery work + +The reproducible [measurement script](../../scripts/measure_workflow_reuse.py) +uses two synthetic CIFs and injects one export failure after both calculations. +The [recorded measurement](agent-reuse-20261022.json) contains the complete counts +and timings. Both final bundles passed integrity verification. + +| Work during retry | Ordinary pipeline restart | Workflow resume | +| --- | ---: | ---: | +| Provider queries | 1 | 0 | +| Material downloads | 2 | 0 | +| Phase calculations | 2 | 0 | +| Wall time in this small local fixture | 0.264 s | 0.298 s | + +The evidence demonstrates elimination of repeated acquisition/calculation after +the injected failure. It does not demonstrate a wall-time speedup; persistent +checks and export overhead dominate this small fixture. + +## Installation artifact and limits + +Accepted wheel: +`outputs/agent-wheel-20261022-accepted/diffractscout-0.4.0-py3-none-any.whl` + +SHA-256: +`27503be08b499a74ca6dd7279941d6c2049ecd0006e637e8a87e11390f8be7bb` + +The isolated installation initially had no MCP, mp-api or pymatgen and completed +base analysis/background flows. Optional MCP installation was then verified +separately. Detailed installed-CLI/MCP/source-comparison JSON evidence is stored +beside the wheel. Live Materials Project access was not exercised; provider +failure, empty-query, retry and resume semantics use deterministic offline +providers. Hosted CI has not run. The optional MCP CI matrix now covers +Windows/Ubuntu and Python 3.10/3.13 when CI executes. + +No new portable Windows executable or formal release was produced. Frozen-worker +dispatch has a targeted regression and follows the documented +[PyInstaller subprocess lifetime behavior](https://pyinstaller.org/en/stable/common-issues-and-pitfalls.html#using-sys-executable-to-spawn-subprocesses-that-outlive-the-application-process-implementing-application-restart). +The source desktop and installed Python package were exercised directly. diff --git a/docs/evidence/agent-reuse-20261022.json b/docs/evidence/agent-reuse-20261022.json new file mode 100644 index 0000000..1233093 --- /dev/null +++ b/docs/evidence/agent-reuse-20261022.json @@ -0,0 +1,38 @@ +{ + "scenario": "two synthetic CIFs; export fails once after both calculations", + "runs": { + "legacy_restart": { + "initial": { + "queries": 1, + "material_downloads": 2, + "phase_calculations": 2, + "seconds": 0.070319099992048 + }, + "retry": { + "queries": 1, + "material_downloads": 2, + "phase_calculations": 2, + "seconds": 0.264390199969057 + }, + "verified": true, + "bundle": "legacy" + }, + "workflow_resume": { + "initial": { + "queries": 1, + "material_downloads": 2, + "phase_calculations": 2, + "seconds": 0.22331999999005347 + }, + "retry": { + "queries": 0, + "material_downloads": 0, + "phase_calculations": 0, + "seconds": 0.29836600000271574 + }, + "verified": true, + "bundle": "workspace\\jobs\\fb7df5c69565486fb9d4cb2ea06e8f92\\bundle" + } + }, + "all_passed": true +} diff --git a/examples/agent/README.md b/examples/agent/README.md new file mode 100644 index 0000000..ed3dbb7 --- /dev/null +++ b/examples/agent/README.md @@ -0,0 +1,42 @@ +# Offline workflow requests + +Install the checkout with `python -m pip install -e .`. Run these commands from +the repository root; `python -m diffractscout` can replace `diffractscout`. +No Materials Project credentials or MCP installation are needed. + +```bash +diffractscout agent plan examples/agent/analyze-local.json +diffractscout agent run examples/agent/analyze-local.json +diffractscout agent run examples/agent/analyze-synchrotron.json +diffractscout agent run examples/agent/prepare-analyze-tc4.json +diffractscout agent run examples/agent/batch-offline.json +diffractscout agent list +``` + +The examples omit `output_dir` so the service chooses a distinct output for each +run within the workspace. Relative input paths resolve from the JSON file's +directory. If moving a JSON file, adjust its relative input paths too. For a +specific destination, add a fresh `output_dir`; relative output paths also +resolve from the request file. Do not delete prior output or enable overwrite +to make an example pass. A workspace can be selected with +`agent --workspace /absolute/path/to/workspace ...`. + +| File | Purpose | +| --- | --- | +| [analyze-local.json](analyze-local.json) | Synthetic FCC Al, Cu Kα, 5–100°, synthetic paired stiffness tensor, full Excel | +| [analyze-synchrotron.json](analyze-synchrotron.json) | Same synthetic CIF, explicit 83 keV, 0.5–15°, peak-table workbook | +| [prepare-analyze-tc4.json](prepare-analyze-tc4.json) | Offline alpha, beta and alpha-double-prime starting models, then theoretical diffraction | +| [batch-offline.json](batch-offline.json) | Two independent requests, serial execution and CSV output | + +The FCC fixture and its tensor are synthetic software examples. TC4 uses nominal +grade chemistry with packaged prototype lattice and internal coordinates; these +are starting assumptions, not measured TC4 phase parameters or an equilibrium +phase prediction. Read the preparation report and provenance before scientific +use. The serial batch manifest is accepted by `run` and `submit`; use `plan` on +each individual request when checking a batch before execution. + +For background submission, replace `run` with `submit`, retain each returned +`run_id`, then use `status`, `events`, and `result` as described in the +[agent pipeline guide](../../docs/AGENT_PIPELINE.md). A saved request is an +execution description; completion and integrity must be read from the job and +its verified result. diff --git a/examples/agent/analyze-local.json b/examples/agent/analyze-local.json new file mode 100644 index 0000000..5a5e988 --- /dev/null +++ b/examples/agent/analyze-local.json @@ -0,0 +1,15 @@ +{ + "schema": "diffractscout_workflow_v1", + "workflow": "analyze", + "name": "Offline synthetic FCC reference", + "inputs": ["../demo_cifs/synthetic_fcc_al.cif"], + "experiment": "laboratory", + "analysis": { + "input_mode": "source", + "source_preset": "Cu Ka", + "two_theta_min_deg": 5.0, + "two_theta_max_deg": 100.0, + "include_elasticity": true + }, + "exports": {"include_excel": true, "template": "full"} +} diff --git a/examples/agent/analyze-synchrotron.json b/examples/agent/analyze-synchrotron.json new file mode 100644 index 0000000..8afa93c --- /dev/null +++ b/examples/agent/analyze-synchrotron.json @@ -0,0 +1,17 @@ +{ + "schema": "diffractscout_workflow_v1", + "workflow": "analyze", + "name": "Offline synthetic reference at 83 keV", + "inputs": ["../demo_cifs/synthetic_fcc_al.cif"], + "experiment": "synchrotron", + "analysis": { + "input_mode": "energy", + "energy_keV": 83.0, + "two_theta_min_deg": 0.5, + "two_theta_max_deg": 15.0, + "step_deg": 0.005, + "fwhm_deg": 0.03, + "include_elasticity": false + }, + "exports": {"include_excel": true, "template": "peak_table"} +} diff --git a/examples/agent/batch-offline.json b/examples/agent/batch-offline.json new file mode 100644 index 0000000..31d4a38 --- /dev/null +++ b/examples/agent/batch-offline.json @@ -0,0 +1,35 @@ +{ + "schema": "diffractscout_batch_v1", + "max_workers": 1, + "requests": [ + { + "schema": "diffractscout_workflow_v1", + "workflow": "analyze", + "name": "Batch synthetic laboratory reference", + "inputs": ["../demo_cifs/synthetic_fcc_al.cif"], + "analysis": { + "input_mode": "source", + "source_preset": "Cu Ka", + "two_theta_min_deg": 5.0, + "two_theta_max_deg": 100.0, + "include_elasticity": false + }, + "exports": {"include_excel": false} + }, + { + "schema": "diffractscout_workflow_v1", + "workflow": "prepare_analyze", + "name": "Batch offline TC4 alpha reference", + "composition": "TC4", + "preparation": {"offline": true, "phases": ["alpha"]}, + "analysis": { + "input_mode": "source", + "source_preset": "Cu Ka", + "two_theta_min_deg": 5.0, + "two_theta_max_deg": 100.0, + "include_elasticity": false + }, + "exports": {"include_excel": false} + } + ] +} diff --git a/examples/agent/prepare-analyze-tc4.json b/examples/agent/prepare-analyze-tc4.json new file mode 100644 index 0000000..cafebc9 --- /dev/null +++ b/examples/agent/prepare-analyze-tc4.json @@ -0,0 +1,18 @@ +{ + "schema": "diffractscout_workflow_v1", + "workflow": "prepare_analyze", + "name": "Offline TC4 starting models and theoretical references", + "composition": "TC4", + "preparation": { + "offline": true, + "phases": ["alpha", "beta", "alpha-double-prime"] + }, + "analysis": { + "input_mode": "source", + "source_preset": "Cu Ka", + "two_theta_min_deg": 5.0, + "two_theta_max_deg": 100.0, + "include_elasticity": false + }, + "exports": {"include_excel": true, "template": "full"} +} diff --git a/pyproject.toml b/pyproject.toml index 7f3581d..e3bb6bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ dependencies = [ ] [project.optional-dependencies] +mcp = ["mcp>=2.3,<3"] complete = [ "mp-api>=0.43,<0.46; python_version < '3.11'", "mp-api>=0.43; python_version >= '3.11'", @@ -85,6 +86,8 @@ paper = [ [project.scripts] diffractscout = "diffractscout.cli:main" +diffractscout-agent = "diffractscout.agent_cli:main" +diffractscout-mcp = "diffractscout.mcp_server:main" diffractscout-quick-export = "diffractscout.quick_export:main" diffractscout-compat = "diffractscout.compat.launcher:main" diff --git a/scripts/measure_workflow_reuse.py b/scripts/measure_workflow_reuse.py new file mode 100644 index 0000000..14eca25 --- /dev/null +++ b/scripts/measure_workflow_reuse.py @@ -0,0 +1,99 @@ +"""Compare real operation counts after one injected export failure, offline. + +Writes only a new user-selected output directory. This is an engineering +recovery benchmark, not validation against experimental diffraction data. +""" +from __future__ import annotations + +import argparse +import json +import sys +import time +from dataclasses import asdict +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from diffractscout import pipeline # noqa: E402 +from diffractscout.demo import SYNTHETIC_FCC_CIF # noqa: E402 +from diffractscout.models import AnalysisSettings, CandidateRecord, DownloadArtifact # noqa: E402 +from diffractscout.validation import verify_bundle # noqa: E402 +from diffractscout.workflow import resume_job, run_job # noqa: E402 + + +class OfflineProvider: + name = "recovery-measurement" + + def __init__(self): + self.queries = self.material_downloads = 0 + + def metadata(self): + return {"provider": self.name, "data_nature": "synthetic fixture"} + + def search_subsystem(self, _system, **_options): + self.queries += 1 + return [CandidateRecord(f"fixture-{index}", formula="Al") for index in (1, 2)] + + def download_candidates(self, candidates, output_dir, **_options): + output_dir.mkdir(parents=True, exist_ok=True) + results = [] + for candidate in candidates: + self.material_downloads += 1 + path = output_dir / f"{candidate.material_id}.cif" + path.write_text(SYNTHETIC_FCC_CIF, encoding="utf-8") + results.append(DownloadArtifact(candidate, path)) + return results + + +def measure(output: Path) -> dict: + output = output.resolve() + output.mkdir(parents=True, exist_ok=False) + settings = AnalysisSettings(include_elasticity=False, include_patterns=False) + real_calculate = pipeline.simulate_powder_pattern + evidence = {"scenario": "two synthetic CIFs; export fails once after both calculations", "runs": {}} + for mode in ("legacy_restart", "workflow_resume"): + provider = OfflineProvider() + started = time.perf_counter() + with patch.object(pipeline, "simulate_powder_pattern", wraps=real_calculate) as calculate: + with patch.object(pipeline, "export_result_bundle", side_effect=OSError("injected export failure")): + if mode == "legacy_restart": + try: + pipeline.run_pipeline("Al", provider, output / "legacy", analysis_settings=settings) + except OSError: + pass + else: + first = run_job({"workflow": "run", "composition": "Al", "analysis": asdict(settings)}, + workspace=output / "workspace", provider=provider) + if first["state"] != "failed": + raise RuntimeError("Expected the injected export failure to fail the first attempt.") + initial = {"queries": provider.queries, "material_downloads": provider.material_downloads, + "phase_calculations": calculate.call_count, "seconds": time.perf_counter()-started} + retried = time.perf_counter() + if mode == "legacy_restart": + result = pipeline.run_pipeline("Al", provider, output / "legacy", analysis_settings=settings) + bundle = result.output_dir + else: + state = resume_job(first["run_id"], workspace=output / "workspace", provider=provider, background=False) + if state["state"] != "completed": + raise RuntimeError(str(state["diagnostics"])) + bundle = Path(state["output_dir"]) + retry = {"queries": provider.queries-initial["queries"], + "material_downloads": provider.material_downloads-initial["material_downloads"], + "phase_calculations": calculate.call_count-initial["phase_calculations"], + "seconds": time.perf_counter()-retried} + evidence["runs"][mode] = {"initial": initial, "retry": retry, "verified": verify_bundle(bundle)["ok"], + "bundle": str(bundle.relative_to(output))} + resumed = evidence["runs"]["workflow_resume"]["retry"] + if any(resumed[key] != 0 for key in ("queries", "material_downloads", "phase_calculations")): + raise RuntimeError("Resume repeated a successful operation.") + evidence["all_passed"] = all(run["verified"] for run in evidence["runs"].values()) + (output / "measurement.json").write_text(json.dumps(evidence, indent=2)+"\n", encoding="utf-8") + return evidence + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + print(json.dumps(measure(args.output), indent=2)) diff --git a/src/diffractscout/__init__.py b/src/diffractscout/__init__.py index f7c9fef..ecaf109 100644 --- a/src/diffractscout/__init__.py +++ b/src/diffractscout/__init__.py @@ -19,4 +19,21 @@ "__version__", ] +_WORKFLOW_EXPORTS = { + "WorkflowRequest", "context", "schema", "plan_workflow", "run_job", "submit_job", + "resume_job", "cancel_job", "status_job", "list_jobs", "read_events", "get_result", + "run_batch", +} +__all__ += sorted(_WORKFLOW_EXPORTS) + + +def __getattr__(name: str): + if name in _WORKFLOW_EXPORTS: + from . import workflow + + value = getattr(workflow, name) + globals()[name] = value + return value + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + __version__ = "0.4.0" diff --git a/src/diffractscout/agent_cli.py b/src/diffractscout/agent_cli.py new file mode 100644 index 0000000..135042d --- /dev/null +++ b/src/diffractscout/agent_cli.py @@ -0,0 +1,159 @@ +"""JSON-only command line access to the shared workflow service.""" +from __future__ import annotations + +import argparse +from contextlib import redirect_stdout +import json +import os +from pathlib import Path +import sys +from typing import Any, Sequence + + +class _Parser(argparse.ArgumentParser): + def error(self, message: str) -> None: + raise ValueError(message) + + def print_help(self, file=None) -> None: + super().print_help(file=file or sys.stderr) + + +def error_payload(exc: Exception) -> dict[str, Any]: + """Keep failures machine readable without echoing environment credentials.""" + message = str(exc) + for key in ("MP_API_KEY", "MATERIALS_PROJECT_API_KEY"): + secret = os.environ.get(key) + if secret: + message = message.replace(secret, "[redacted]") + return {"ok": False, "error": {"type": type(exc).__name__, "message": message}} + + +def build_parser() -> argparse.ArgumentParser: + parser = _Parser(prog="diffractscout agent", description=__doc__) + parser.add_argument("--workspace", help="Directory containing durable job records.") + commands = parser.add_subparsers(dest="command", required=True) + for name in ("context", "schema", "plan", "run", "submit", "list", "status", + "events", "resume", "cancel", "result"): + command = commands.add_parser(name) + command.add_argument("--workspace", default=argparse.SUPPRESS) + if name == "schema": + command.add_argument("name", nargs="?", default="request") + if name in {"plan", "run", "submit"}: + command.add_argument("request", help="Request JSON file, JSON object, or '-' for stdin.") + if name in {"status", "events", "resume", "cancel", "result"}: + command.add_argument("run_id") + if name == "resume": + command.add_argument("--foreground", action="store_true") + if name == "events": + command.add_argument("--after", type=int, default=0) + command.add_argument("--limit", type=int, default=100) + if name == "result": + command.add_argument("--table") + command.add_argument("--offset", type=int, default=0) + command.add_argument("--limit", type=int, default=100) + command.add_argument("--no-verify", action="store_true") + return parser + + +def load_request(source: str) -> tuple[dict[str, Any], Path]: + """Read one object; relative paths in a file are relative to that file.""" + base_dir = Path.cwd() + if source == "-": + payload = json.load(sys.stdin) + elif source.lstrip().startswith("{"): + payload = json.loads(source) + else: + path = Path(source).expanduser().resolve() + base_dir = path.parent + payload = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(payload, dict): + raise ValueError("A workflow request or batch manifest must be a JSON object.") + return payload, base_dir + + +def execute_request( + payload: dict[str, Any], *, base_dir: Path | None = None, + workspace: str | None = None, background: bool = False, +) -> dict[str, Any]: + """Normalize requests once, then invoke the same service as GUI and API.""" + from . import workflow + + if payload.get("schema") == "diffractscout_batch_v1": + unknown = set(payload) - {"schema", "requests", "max_workers"} + if unknown: + raise ValueError("Unknown batch manifest fields: " + ", ".join(sorted(unknown))) + requests = payload.get("requests") + if not isinstance(requests, list) or not requests: + raise ValueError("Batch requests must be a nonempty array of workflow requests.") + normalized = [workflow.WorkflowRequest.from_dict(item, base_dir=base_dir) + for item in requests] + return workflow.run_batch(normalized, workspace=workspace, + max_workers=payload.get("max_workers", 1), + background=background) + request = workflow.WorkflowRequest.from_dict(payload, base_dir=base_dir) + function = workflow.submit_job if background else workflow.run_job + return function(request, workspace=workspace) + + +def _dispatch(args: argparse.Namespace) -> Any: + from . import workflow + + command, workspace = args.command, args.workspace + if command == "context": + return workflow.context() + if command == "schema": + return workflow.schema(args.name) + if command in {"plan", "run", "submit"}: + payload, base_dir = load_request(args.request) + if command == "plan": + request = workflow.WorkflowRequest.from_dict(payload, base_dir=base_dir) + return workflow.plan_workflow(request) + return execute_request(payload, base_dir=base_dir, workspace=workspace, + background=command == "submit") + if command == "list": + return {"jobs": workflow.list_jobs(workspace=workspace)} + if command == "events": + return workflow.read_events(args.run_id, workspace=workspace, + after=args.after, limit=args.limit) + if command == "result": + return workflow.get_result(args.run_id, workspace=workspace, table=args.table, + offset=args.offset, limit=args.limit, + verify=not args.no_verify) + if command == "resume": + return workflow.resume_job(args.run_id, workspace=workspace, + background=not args.foreground) + function = workflow.status_job if command == "status" else workflow.cancel_job + return function(args.run_id, workspace=workspace) + + +def exit_code(payload: Any) -> int: + if not isinstance(payload, dict): + return 0 + state = payload.get("state") + if state == "partial": + return 3 + if state in {"cancelled", "interrupted", "blocked", "failed"} or payload.get("ok") is False: + return 2 + return 0 + + +def main(argv: Sequence[str] | None = None) -> int: + from .console import configure_cli_output + + configure_cli_output() + try: + args = build_parser().parse_args(argv) + # Legacy provider progress is diagnostic output, never part of JSON stdout. + with redirect_stdout(sys.stderr): + payload = _dispatch(args) + serialized = json.dumps(payload, ensure_ascii=False, allow_nan=False) + code = exit_code(payload) + except Exception as exc: + serialized = json.dumps(error_payload(exc), ensure_ascii=False) + code = 2 + print(serialized) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/diffractscout/cli.py b/src/diffractscout/cli.py index c14e844..ce4693e 100644 --- a/src/diffractscout/cli.py +++ b/src/diffractscout/cli.py @@ -208,6 +208,9 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--version", action="version", version=f"diffractscout {__version__}") subparsers = parser.add_subparsers(dest="command", required=True) + agent = subparsers.add_parser("agent", help="JSON workflow and durable job commands.") + agent.add_argument("agent_args", nargs=argparse.REMAINDER) + analyze = subparsers.add_parser("analyze", help="Analyze local CIF files or directories.") analyze.add_argument("inputs", nargs="+", help="CIF files or directories.") analyze.add_argument("-o", "--output", required=True) @@ -377,6 +380,14 @@ def build_parser() -> argparse.ArgumentParser: def main(argv: Sequence[str] | None = None) -> int: configure_cli_output() effective = list(sys.argv[1:] if argv is None else argv) + if effective and effective[0] == "_agent-worker": + from .workflow.worker import main as worker_main + + return worker_main(effective[1:]) + if effective and effective[0] == "agent": + from .agent_cli import main as agent_main + + return agent_main(effective[1:]) if effective and effective[0] == "compat": from .compat.launcher import main as compat_main diff --git a/src/diffractscout/gui.py b/src/diffractscout/gui.py index fea4d71..69da8df 100644 --- a/src/diffractscout/gui.py +++ b/src/diffractscout/gui.py @@ -11,8 +11,6 @@ import subprocess import sys import tempfile -import threading -import traceback from datetime import datetime from pathlib import Path from typing import Any, Callable, Mapping @@ -27,10 +25,8 @@ TEAL, TEAL_DARK, TEXT, configure_styles, ) from .models import AnalysisSettings, DiscoverySettings, ElasticTensor, PipelineResult -from .pipeline import analyze_cifs, run_pipeline -from .providers.materials_project import MaterialsProjectProvider from .selection import validate_discovery_settings -from .utils import sha256_file +from .utils import sha256_file, to_jsonable try: # Tk remains optional on minimal/headless Python installations. import tkinter as tk @@ -267,7 +263,6 @@ def __init__(self) -> None: self.lang = DEFAULT_LANG self.events: queue.Queue[tuple[str, object]] = queue.Queue() self.running = False - self._worker_thread: threading.Thread | None = None # Each preview records its directory, workbook path, and SHA-256 # at creation. Changed previews are retained when the GUI closes. self._preview_dirs: list[tuple[Path, Path, str]] = [] @@ -381,7 +376,7 @@ def visit(widget: Any) -> None: ): self._add_hover_help(widget, key) self.help.add_notebook(self.notebook, [lambda k=k: self._t(k) for k in - ("help_tab_local", "help_tab_mp", "help_tab_results")]) + ("help_tab_local", "help_tab_mp", "help_tab_results", "help_tab_tasks")]) self.help.add(self._main_paned, lambda: self._t("help_resize_log"), resolve=lambda event: self._t("help_resize_log") if event is None or self._main_paned.identify(event.x, event.y) != "" else "") @@ -529,6 +524,7 @@ def _open_initial_cif_dialog(self) -> None: self, language=self.lang, on_load=self._load_prepared_inputs, + workspace=self.tasks_view.workspace_path(), ) def _load_prepared_inputs(self, paths: list[Path]) -> None: @@ -608,7 +604,14 @@ def _build_main_split(self) -> None: self.results_view = ResultView(notebook, translate=self._t) notebook.add(self.results_view, text=self._t("tab_results")) - self._notebook_tabs = [(0, "tab_local"), (1, "tab_mp"), (2, "tab_results")] + from .gui_tasks import TasksView + + self.tasks_view = TasksView( + notebook, translate=self._t, on_result=self._show_task_result, + api_key=lambda: self.mp_key.get().strip(), help_view=self.help, + ) + notebook.add(self.tasks_view, text=self._t("tab_tasks")) + self._notebook_tabs = [(0, "tab_local"), (1, "tab_mp"), (2, "tab_results"), (3, "tab_tasks")] self._build_local_tab(local) self._build_mp_tab(mp) self._build_activity_panel(activity_host) @@ -1645,6 +1648,7 @@ def _apply_language(self) -> None: for section in self._disclosures: self._label_disclosure(section) self.results_view.refresh_language() + self.tasks_view.refresh_language() self._refresh_status_text() self._refresh_inputs() self._refresh_cij_status() @@ -2299,18 +2303,13 @@ def _run_local(self) -> None: include_excel = bool(self.include_excel.get()) overwrite = bool(self.overwrite.get()) label = self._t("analyze_local") if hasattr(self, "_t") else "Analyze selected CIFs" - self._start_task( - label, - lambda: analyze_cifs( - inputs, - output, - settings=settings, - recursive=recursive, - include_excel=include_excel, - overwrite=overwrite, - elastic_overrides=overrides, - ), - ) + self._submit_workflow({ + "schema": "diffractscout_workflow_v1", "workflow": "analyze", "name": label, + "inputs": inputs, "output_dir": str(Path(output).expanduser().resolve()), + "analysis": to_jsonable(settings), "elastic_overrides": to_jsonable(overrides or {}), + "exports": {"include_excel": include_excel, "template": "full"}, + "execution": {"recursive": recursive, "overwrite": overwrite}, + }) def _run_mp(self) -> None: if self.running: @@ -2346,44 +2345,30 @@ def _run_mp(self) -> None: include_excel = bool(self.include_excel.get()) overwrite = bool(self.overwrite.get()) - def run() -> PipelineResult: - provider = MaterialsProjectProvider(api_key) - return run_pipeline( - composition, - provider, - output, - discovery_settings=discovery, - analysis_settings=analysis, - conventional_unit_cell=conventional, - include_elasticity=analysis.include_elasticity, - include_excel=include_excel, - overwrite=overwrite, - confirm_above=limit, - authorize_large_download=True, - ) - - self._start_task(self._t("run_mp"), run) - - def _start_task(self, label: str, function: Callable[[], PipelineResult]) -> None: - self.running = True - self._set_status("status_busy", label=label) - self.progress.pack(side="right", before=self._status_actions, padx=(6, 12), pady=8) - self.progress.start(12) - DiffractScoutApp._update_open_button_state(self) - for button in self._run_buttons: - button.configure(state="disabled") - self._refresh_readiness() - self._log(self._t("log_started", label=label), "info") - for button in self._result_buttons + self._preset_buttons: - button.configure(state="disabled") - self._worker_thread = threading.Thread(target=self._worker, args=(function,), daemon=False) - self._worker_thread.start() + self._submit_workflow({ + "schema": "diffractscout_workflow_v1", "workflow": "run", "name": self._t("run_mp"), + "composition": composition, "output_dir": str(Path(output).expanduser().resolve()), + "analysis": to_jsonable(analysis), "discovery": to_jsonable(discovery), + "exports": {"include_excel": include_excel, "template": "full"}, + "execution": {"conventional_unit_cell": conventional, "overwrite": overwrite, + "confirm_above": limit, "authorize_large_download": True}, + }, api_key=api_key) + + def _submit_workflow(self, request: dict, *, api_key: str | None = None) -> None: + self.tasks_view.submit(request, api_key=api_key) + self.notebook.select(self.tasks_view) + self._log(self._t("tasks_submitting")) + + def _show_task_result(self, result: Any) -> None: + if isinstance(result, PipelineResult): + # Reuse the established result rendering and diagnostic summary. + self.events.put(("done", result)) + else: + from .gui_cifs import InitialCifDialog - def _worker(self, function: Callable[[], PipelineResult]) -> None: - try: - self.events.put(("done", function())) - except Exception as exc: # pragma: no cover - thread/UI path - self.events.put(("error", (exc, traceback.format_exc()))) + dialog = InitialCifDialog(self, language=self.lang, on_load=self._load_prepared_inputs, + workspace=self.tasks_view.workspace_path()) + dialog._show_result(result) def _finish_task(self) -> None: self.running = False @@ -2644,15 +2629,6 @@ def _save_excel_copy(self) -> None: messagebox.showerror(self._t("err_save_excel"), str(exc)) def _on_close(self) -> None: - if self.running or (self._worker_thread is not None and self._worker_thread.is_alive()): - # Do not destroy a live controller or rely on daemon-thread - # truncation. The worker owns a transactional pipeline and - # will publish a safe boundary through the event queue. - messagebox.showinfo( - self._t("msg_close_title"), - self._t("msg_close_running"), - ) - return retained = self._cleanup_preview_dirs() if retained: paths = "\n".join(str(path) for path in retained) diff --git a/src/diffractscout/gui_cifs.py b/src/diffractscout/gui_cifs.py index 790cb48..9abbf14 100644 --- a/src/diffractscout/gui_cifs.py +++ b/src/diffractscout/gui_cifs.py @@ -6,6 +6,7 @@ import queue import re import threading +import time import webbrowser from datetime import datetime from pathlib import Path @@ -112,11 +113,30 @@ def prepare_kwargs_from_form( def _prepare_cifs(**kwargs: Any) -> Any: - """Import the optional workflow only when a user starts the operation.""" - - from .initial_cifs import prepare_cifs - - return prepare_cifs(**kwargs) + """Submit a durable job; this thread only observes the independent worker.""" + + from .gui_tasks import workflow_api, ACTIVE_STATES + from .utils import to_jsonable + + options = dict(kwargs) + workspace = options.pop("_workspace", None) + api_key = options.pop("api_key", None) + composition = options.pop("composition") + output_dir = options.pop("output_dir") + api = workflow_api() + request = api.WorkflowRequest.from_dict({ + "schema": "diffractscout_workflow_v1", "workflow": "prepare", + "composition": composition, "output_dir": str(output_dir), + "preparation": to_jsonable(options), + }) + status = api.submit_job(request, workspace=workspace, api_key=api_key) + run_id = status["run_id"] + while status["state"] in ACTIVE_STATES: + time.sleep(0.5) + status = api.status_job(run_id, workspace=workspace) + if status["state"] not in {"completed", "partial"}: + raise RuntimeError(f"{run_id}: {status['state']}\n{status.get('diagnostics', [])}") + return api.load_result_object(run_id, workspace=workspace) def _worker_run( @@ -153,6 +173,7 @@ def __init__( language: str = "zh", on_load: Callable[[list[Path]], Any] | None = None, on_close: Callable[[], Any] | None = None, + workspace: str | Path | None = None, ) -> None: super().__init__(master) self.ui_font = configure_styles(self) @@ -160,6 +181,7 @@ def __init__( self.lang = "en" if str(language).lower() == "en" else "zh" self._on_load = on_load self._on_close = on_close + self._workspace = workspace self._closed = False self._events: queue.Queue[tuple[str, object]] = queue.Queue() self._after_id: str | None = None @@ -669,7 +691,8 @@ def _start(self) -> None: self.progress.start(12) self._set_status(self._t("running")) thread = threading.Thread( - target=_worker_run, args=(kwargs, self._events), daemon=True, name="initial-cif-worker" + target=_worker_run, args=({**kwargs, "_workspace": self._workspace}, self._events), + daemon=True, name="initial-cif-monitor" ) thread.start() self._schedule_poll() diff --git a/src/diffractscout/gui_help_text.py b/src/diffractscout/gui_help_text.py index 7776dbf..ee48e60 100644 --- a/src/diffractscout/gui_help_text.py +++ b/src/diffractscout/gui_help_text.py @@ -151,6 +151,19 @@ ("help_cifs_details", "选中上方一行后，这里显示文件位置和说明。可滚动查看，也可选中文字后复制。", "Select a row above to view its file path and notes here. Scroll to read more, or select text to copy it."), ) +_MESSAGES += ( + ("help_tab_tasks", "查看保存的任务、当前步骤和实际完成数量。选择任务后可恢复、取消或核验并打开结果；关闭窗口后任务继续运行。", "View saved tasks, current stages and actual item counts. Select a task to resume, cancel, or verify and open results. Tasks continue after closing the window."), + ("help_tasks_workspace", "选择保存任务记录的文件夹。同一文件夹中的历史任务会在重启后重新显示；切换文件夹不会取消任务。", "Choose the folder holding task records. Its history reappears after restart. Changing folders does not cancel tasks."), + ("help_tasks_import", "选择工作流请求 JSON 文件并提交。文件中的相对路径以该文件所在文件夹为准；联网任务使用主窗口填写的密钥。", "Choose and submit a workflow request JSON file. Relative paths use the file's folder. Online tasks use the key entered in the main window."), + ("help_tasks_refresh", "重新读取当前任务文件夹的记录。运行中的任务也会自动更新，数量表示已实际完成的项目。", "Reload task records in the current folder. Running tasks also update automatically; counts represent actual completed items."), + ("help_tasks_resume", "选择中断、受阻或未全部完成的任务后恢复。先根据详情补齐缺失条件；已完成并通过核验的项目可以复用。", "Resume an interrupted, blocked or unfinished task. Resolve missing requirements in its details first. Completed verified items can be reused."), + ("help_tasks_cancel", "请求取消选中的活动任务。当前操作可能需要先完成安全写入，状态会由取消中变为已取消。", "Request cancellation of the selected active task. The current operation may need to finish a safe write before the state changes from cancelling to cancelled."), + ("help_tasks_open", "选择已完成或部分完成的任务，核验保存文件后打开结果。文件缺失或内容变化时会显示错误。", "Select a completed or partial task to verify its saved files and open results. Missing or changed files produce an error."), + ("help_tasks_folder", "核验所选任务的结果文件后，在文件管理器中打开保存结果的文件夹。核验失败时不会打开。", "Verify the selected task's result files, then open their folder in the file manager. Verification failure keeps it closed."), + ("help_tasks_list", "选择一行查看任务状态、当前步骤和实际完成数量。运行中不显示估算百分比；双击可核验并打开已完成结果。", "Select a row for its state, stage and actual completed item count. Double-click to verify and open completed results."), + ("help_tasks_details", "显示所选任务的记录，包括诊断、缺失条件、文件位置、完整性和完成范围。可以选中文字复制。", "Read the selected task's diagnostics, missing requirements, file locations, integrity and coverage. Select text to copy it."), +) + HELP_TEXT = { "zh": {key: zh for key, zh, _en in _MESSAGES}, "en": {key: en for key, _zh, en in _MESSAGES}, diff --git a/src/diffractscout/gui_i18n.py b/src/diffractscout/gui_i18n.py index 7198ba8..6ca9d93 100644 --- a/src/diffractscout/gui_i18n.py +++ b/src/diffractscout/gui_i18n.py @@ -749,6 +749,57 @@ def normalize_lang(lang: str | None) -> str: return DEFAULT_LANG +_TASK_STRINGS = ( + ("tab_tasks", "任务", "Tasks"), + ("tasks_workspace", "任务文件夹", "Task workspace"), + ("tasks_choose", "选择…", "Choose…"), + ("tasks_import", "导入任务 JSON", "Import job JSON"), + ("tasks_refresh", "刷新", "Refresh"), + ("tasks_resume", "恢复", "Resume"), + ("tasks_cancel", "取消任务", "Cancel task"), + ("tasks_open", "核验并打开", "Verify and open"), + ("tasks_folder", "结果文件夹", "Open folder"), + ("tasks_name", "名称 / 任务 ID", "Name / run ID"), + ("tasks_state", "状态", "State"), + ("tasks_stage", "当前步骤", "Stage"), + ("tasks_counts", "完成 / 总数", "Done / total"), + ("tasks_missing", "需要补齐", "Missing requirements"), + ("tasks_output_dir", "结果位置", "Result location"), + ("tasks_integrity", "完整性", "Integrity"), + ("tasks_coverage", "完成范围", "Coverage"), + ("tasks_counters", "实际操作数量", "Operation counts"), + ("tasks_elapsed_seconds", "用时（秒）", "Elapsed seconds"), + ("tasks_tables", "结果表格", "Result tables"), + ("task_stage_planning", "检查请求", "Planning"), + ("task_stage_snapshot", "保存输入副本", "Snapshot inputs"), + ("task_stage_discovery", "查询结构", "Discover structures"), + ("task_stage_download", "下载结构", "Download structures"), + ("task_stage_preparation", "准备初始 CIF", "Prepare initial CIFs"), + ("task_stage_analysis", "计算衍射", "Analyze diffraction"), + ("task_stage_export", "导出结果", "Export results"), + ("task_stage_verification", "核验结果", "Verify results"), + ("task_stage_fetch", "获取原型", "Fetch prototypes"), + ("task_stage_adapt", "调整结构", "Adapt structure"), + ("tasks_hint", "关闭窗口后任务继续运行。重启后选择同一任务文件夹即可查看记录。", "Tasks continue after closing. Choose this workspace after restart to view history."), + ("tasks_submitting", "正在提交任务…", "Submitting task…"), + ("tasks_saved", "已保存任务：{run_id}", "Task saved: {run_id}"), + ("tasks_verified", "结果已核验：{run_id}", "Results verified: {run_id}"), + ("tasks_error", "任务操作失败：{error}", "Task operation failed: {error}"), + ("task_state_pending", "等待运行", "Pending"), + ("task_state_running", "运行中", "Running"), + ("task_state_cancelling", "取消中", "Cancelling"), + ("task_state_cancelled", "已取消", "Cancelled"), + ("task_state_interrupted", "已中断", "Interrupted"), + ("task_state_blocked", "等待补齐条件", "Blocked"), + ("task_state_completed", "已完成", "Completed"), + ("task_state_partial", "部分完成", "Partial"), + ("task_state_failed", "失败", "Failed"), +) +for _key, _zh, _en in _TASK_STRINGS: + STRINGS["zh"][_key] = _zh + STRINGS["en"][_key] = _en +REQUIRED_KEYS = REQUIRED_KEYS | frozenset(key for key, _zh, _en in _TASK_STRINGS) + for _language, _help_messages in HELP_TEXT.items(): STRINGS[_language].update(_help_messages) REQUIRED_KEYS = REQUIRED_KEYS | frozenset(HELP_TEXT["zh"]) diff --git a/src/diffractscout/gui_tasks.py b/src/diffractscout/gui_tasks.py new file mode 100644 index 0000000..d831c75 --- /dev/null +++ b/src/diffractscout/gui_tasks.py @@ -0,0 +1,383 @@ +"""Tk task history backed by the shared durable workflow service.""" + +from __future__ import annotations + +import json +import queue +import threading +from pathlib import Path +from typing import Any, Callable + +try: + import tkinter as tk + from tkinter import filedialog, ttk +except ImportError: # pragma: no cover - optional Tk + tk = ttk = filedialog = None + + +def workflow_api() -> Any: + # Loading the desktop does not require starting the workflow service. + from . import workflow + + return workflow + + +ACTIVE_STATES = {"pending", "running", "cancelling"} +RESUMABLE_STATES = {"interrupted", "blocked", "partial", "failed", "cancelled"} + + +if ttk is not None: + class TasksView(ttk.Frame): + """Only queue messages cross threads; Tk updates stay in the event loop.""" + + def __init__(self, master: Any, *, translate: Callable[..., str], + on_result: Callable[[Any], None], api_key: Callable[[], str], + help_view: Any = None) -> None: + super().__init__(master, padding=8) + self._t = translate + self._on_result = on_result + self._api_key = api_key + self._messages: queue.Queue = queue.Queue() + self._closed = False + self._refreshing = False + self._generation = 0 + self._workspace_identity: Path | None = None + self._jobs: dict[str, dict] = {} + self._follow: set[str] = set() + self._opening: set[str] = set() + self._summaries: dict[str, dict] = {} + self._events: dict[str, dict] = {} + self._event_cursors: dict[str, int] = {} + self._targets: list[tuple[Any, str]] = [] + self.workspace = tk.StringVar(value=str(Path.cwd() / "outputs" / "agent-workspace")) + self.message = tk.StringVar() + self.columnconfigure(0, weight=1) + self.rowconfigure(2, weight=1) + location = ttk.Frame(self) + location.grid(row=0, column=0, sticky="ew") + location.columnconfigure(1, weight=1) + label = ttk.Label(location) + label.grid(row=0, column=0, padx=(0, 6)) + self._targets.append((label, "tasks_workspace")) + self.workspace_entry = ttk.Entry(location, textvariable=self.workspace) + self.workspace_entry.grid(row=0, column=1, sticky="ew") + self.workspace_entry.bind("", lambda _e: self.refresh()) + self.browse_button = self._button(location, "tasks_choose", self.choose_workspace) + self.browse_button.grid(row=0, column=2, padx=(6, 0)) + actions = ttk.Frame(self) + actions.grid(row=1, column=0, sticky="ew", pady=6) + self.import_button = self._button(actions, "tasks_import", self.import_request) + self.refresh_button = self._button(actions, "tasks_refresh", self.refresh) + self.resume_button = self._button(actions, "tasks_resume", self.resume) + self.cancel_button = self._button(actions, "tasks_cancel", self.cancel) + self.open_button = self._button(actions, "tasks_open", self.open_selected) + self.folder_button = self._button(actions, "tasks_folder", self.open_folder) + for widget in (self.import_button, self.refresh_button, self.resume_button, + self.cancel_button, self.open_button, self.folder_button): + widget.pack(side="left", padx=(0, 5)) + split = ttk.Panedwindow(self, orient="vertical") + split.grid(row=2, column=0, sticky="nsew") + table = ttk.Frame(split) + table.columnconfigure(0, weight=1) + table.rowconfigure(0, weight=1) + self.tree = ttk.Treeview(table, columns=("name", "state", "stage", "counts"), + show="headings", height=4, selectmode="browse") + for key, width in (("name", 260), ("state", 100), ("stage", 150), ("counts", 95)): + self.tree.column(key, width=width, minwidth=60, stretch=True) + self.tree.grid(row=0, column=0, sticky="nsew") + scroll = ttk.Scrollbar(table, orient="vertical", command=self.tree.yview) + scroll.grid(row=0, column=1, sticky="ns") + self.tree.configure(yscrollcommand=scroll.set) + self.tree.bind("<>", self._select) + self.tree.bind("", lambda _e: self.open_selected()) + split.add(table, weight=2) + details_host = ttk.Frame(split) + details_host.columnconfigure(0, weight=1) + details_host.rowconfigure(0, weight=1) + self.details = tk.Text(details_host, height=4, wrap="word", state="disabled") + self.details.grid(row=0, column=0, sticky="nsew") + details_scroll = ttk.Scrollbar(details_host, command=self.details.yview) + details_scroll.grid(row=0, column=1, sticky="ns") + self.details.configure(yscrollcommand=details_scroll.set) + split.add(details_host, weight=1) + self.hint = ttk.Label(self, textvariable=self.message, wraplength=760) + self.hint.grid(row=3, column=0, sticky="ew", pady=(5, 0)) + self.bind("", lambda e: self.hint.configure(wraplength=max(180, e.width - 20))) + self.bind("", self._destroyed, add="+") + if help_view is not None: + for widget, key in ((self.workspace_entry, "help_tasks_workspace"), + (self.browse_button, "help_tasks_workspace"), + (self.import_button, "help_tasks_import"), + (self.refresh_button, "help_tasks_refresh"), + (self.resume_button, "help_tasks_resume"), + (self.cancel_button, "help_tasks_cancel"), + (self.open_button, "help_tasks_open"), + (self.folder_button, "help_tasks_folder"), + (self.tree, "help_tasks_list"), + (self.details, "help_tasks_details")): + help_view.add(widget, lambda k=key: self._t(k)) + help_view.add_tree(self.tree, lambda: self._t("help_tasks_list"), { + "name": lambda: self._t("help_tasks_list"), + "state": lambda: self._t("help_tasks_list"), + "stage": lambda: self._t("help_tasks_list"), + "counts": lambda: self._t("help_tasks_refresh"), + }) + self.refresh_language() + self._after_id = self.after(100, self._poll) + self.refresh() + + def _button(self, parent: Any, key: str, command: Callable) -> Any: + button = ttk.Button(parent, command=command) + self._targets.append((button, key)) + return button + + def _background(self, kind: str, function: Callable) -> None: + messages, generation = self._messages, self._generation + + def work() -> None: + try: + messages.put((generation, kind, function())) + except Exception as exc: + messages.put((generation, "error", str(exc))) + + threading.Thread(target=work, daemon=True, name="workflow-desktop-monitor").start() + + def workspace_path(self) -> Path: + path = Path(self.workspace.get()).expanduser().absolute() + if path != self._workspace_identity: + self._workspace_identity = path + self._generation += 1 + self._refreshing = False + self._follow.clear() + self._opening.clear() + self._summaries.clear() + self._events.clear() + self._event_cursors.clear() + self._jobs.clear() + return path + + def choose_workspace(self) -> None: + value = filedialog.askdirectory(parent=self, title=self._t("tasks_workspace")) + if value: + self.workspace.set(value) + self.refresh() + + def submit(self, request: dict, *, api_key: str | None = None) -> None: + workspace = self.workspace_path() + self.message.set(self._t("tasks_submitting")) + + def work() -> dict: + api = workflow_api() + parsed = api.WorkflowRequest.from_dict(request) + return api.submit_job(parsed, workspace=workspace, api_key=api_key) + + self._background("submitted", work) + + def import_request(self) -> None: + name = filedialog.askopenfilename(parent=self, filetypes=(("JSON", "*.json"),)) + if not name: + return + workspace, api_key = self.workspace_path(), self._api_key() or None + + def work() -> dict: + path = Path(name) + api = workflow_api() + request = api.WorkflowRequest.from_dict(json.loads(path.read_text(encoding="utf-8-sig")), + base_dir=path.parent) + return api.submit_job(request, workspace=workspace, api_key=api_key) + + self._background("submitted", work) + + def refresh(self) -> None: + workspace = self.workspace_path() + if self._refreshing or self._closed: + return + self._refreshing = True + self._background("jobs", lambda: workflow_api().list_jobs(workspace=workspace)) + + def selected(self) -> str | None: + return next(iter(self.tree.selection()), None) + + def resume(self) -> None: + run_id = self.selected() + if run_id: + workspace, api_key = self.workspace_path(), self._api_key() or None + self._background("submitted", lambda: workflow_api().resume_job( + run_id, workspace=workspace, background=True, api_key=api_key)) + + def cancel(self) -> None: + run_id = self.selected() + if run_id: + workspace = self.workspace_path() + self._background("cancelled", lambda: workflow_api().cancel_job(run_id, workspace=workspace)) + + def open_selected(self) -> None: + run_id = self.selected() + if run_id: + self._open(run_id) + + def open_folder(self) -> None: + run_id = self.selected() + if run_id: + workspace = self.workspace_path() + self._background("folder", lambda: workflow_api().get_result( + run_id, workspace=workspace, verify=True)) + + def _open(self, run_id: str) -> None: + if run_id in self._opening: + return + self._opening.add(run_id) + workspace = self.workspace_path() + + def work() -> tuple: + api = workflow_api() + # Verify even when a workflow produces tables rather than a PipelineResult. + summary = api.get_result(run_id, workspace=workspace, verify=True) + job = api.status_job(run_id, workspace=workspace) + result = (api.load_result_object(run_id, workspace=workspace) + if job.get("workflow") not in {"fetch", "adapt"} else None) + return run_id, result, summary + + self._background("result", work) + + def _select(self, _event: Any = None) -> None: + run_id = self.selected() + job = self._jobs.get(run_id or "", {}) + state = job.get("state") + self.resume_button.configure(state="normal" if state in RESUMABLE_STATES else "disabled") + self.cancel_button.configure(state="normal" if state in ACTIVE_STATES else "disabled") + self.open_button.configure(state="normal" if state in {"completed", "partial"} else "disabled") + self.folder_button.configure(state="normal" if state in {"completed", "partial"} else "disabled") + self.details.configure(state="normal") + self.details.delete("1.0", "end") + if job: + lines = [f"{job.get('name') or run_id} [{run_id}]", + f"{self._t('tasks_state')}: {self._t('task_state_' + str(state))} " + f"{self._t('tasks_stage')}: {self._stage_text(job.get('stage', ''))}"] + for diagnostic in job.get("diagnostics", []): + fields = diagnostic.get("missing_fields") or [] + lines.append(f"{diagnostic.get('code', '')}: {diagnostic.get('message', '')}") + if fields: + lines.append(f"{self._t('tasks_missing')}: {', '.join(fields)}") + for key in ("output_dir", "integrity", "coverage", "counters", "elapsed_seconds"): + if key in job: + value = job[key] + if isinstance(value, (dict, list)): + value = json.dumps(value, ensure_ascii=False) + lines.append(f"{self._t('tasks_' + key)}: {value}") + summary = self._summaries.get(run_id, {}) + for artifact in summary.get("artifacts", job.get("artifacts", [])): + lines.append(f"{artifact.get('kind', '')}: {artifact.get('path', '')}") + if summary.get("tables"): + lines.append(f"{self._t('tasks_tables')}: {', '.join(summary['tables'])}") + for event in self._events.get(run_id, {}).get("events", []): + lines.append(" | ".join(str(event[key]) for key in + ("time", "event", "stage", "item", "message") if key in event)) + self.details.insert("1.0", "\n".join(lines)) + self.details.configure(state="disabled") + + def _stage_text(self, stage: str) -> str: + key = "task_stage_" + stage + text = self._t(key) + return stage if text == key else text + + def _read_events(self, run_id: str) -> None: + workspace = self.workspace_path() + cursor = self._event_cursors.get(run_id, 0) + self._background("events", lambda: (run_id, workflow_api().read_events( + run_id, workspace=workspace, after=cursor, limit=100))) + + def _render(self) -> None: + selection = self.selected() + present = set(self.tree.get_children()) + for run_id, job in self._jobs.items(): + state = str(job.get("state", "")) + counts = f"{job.get('completed_items', 0)} / {job.get('total_items', '—')}" + values = (job.get("name") or run_id, self._t("task_state_" + state), + self._stage_text(job.get("stage", "")), counts) + if run_id in present: + self.tree.item(run_id, values=values) + present.remove(run_id) + else: + self.tree.insert("", "end", iid=run_id, values=values) + for run_id in present: + self.tree.delete(run_id) + if selection in self._jobs: + self.tree.selection_set(selection) + self._select() + + def refresh_language(self) -> None: + for widget, key in self._targets: + widget.configure(text=self._t(key)) + for column in ("name", "state", "stage", "counts"): + self.tree.heading(column, text=self._t("tasks_" + column)) + self.message.set(self._t("tasks_hint")) + self._render() + + def _poll(self) -> None: + if self._closed: + return + while True: + try: + generation, kind, payload = self._messages.get_nowait() + except queue.Empty: + break + if generation != self._generation: + continue + if kind == "error": + self._refreshing = False + self._opening.clear() + self.message.set(self._t("tasks_error", error=payload)) + elif kind == "jobs": + self._refreshing = False + self._jobs = {job["run_id"]: job for job in payload} + self._render() + if self.selected(): + self._read_events(self.selected()) + for run_id in self._follow.copy(): + state = self._jobs.get(run_id, {}).get("state") + if state in {"completed", "partial"}: + self._follow.discard(run_id) + self._open(run_id) + elif state and state not in ACTIVE_STATES: + self._follow.discard(run_id) + elif kind in {"submitted", "cancelled"}: + run_id = payload["run_id"] + self._jobs[run_id] = payload + if kind == "submitted": + self._follow.add(run_id) + self._render() + self.tree.selection_set(run_id) + self.tree.see(run_id) + self._select() + self.message.set(self._t("tasks_saved", run_id=run_id)) + elif kind == "result": + run_id, result, summary = payload + self._opening.discard(run_id) + self._summaries[run_id] = summary + self.message.set(self._t("tasks_verified", run_id=run_id)) + if result is not None: + self._on_result(result) + else: + self._select() + elif kind == "folder": + from .gui import open_path + + try: + open_path(payload["output_dir"]) + except OSError as exc: + self.message.set(self._t("tasks_error", error=exc)) + elif kind == "events": + run_id, events = payload + self._event_cursors[run_id] = events.get("next_cursor", 0) + previous = self._events.get(run_id, {}).get("events", []) + self._events[run_id] = {"events": (previous + events.get("events", []))[-100:]} + if run_id == self.selected(): + self._select() + self.refresh() + self._after_id = self.after(1000, self._poll) + + def _destroyed(self, event: Any) -> None: + if event.widget is self: + self._closed = True + self.after_cancel(self._after_id) diff --git a/src/diffractscout/mcp_server.py b/src/diffractscout/mcp_server.py new file mode 100644 index 0000000..5096ee0 --- /dev/null +++ b/src/diffractscout/mcp_server.py @@ -0,0 +1,136 @@ +"""Optional local stdio MCP adapter for the shared workflow service. + +Install ``diffractscout[mcp]`` to run this module. Importing DiffractScout does +not import the MCP SDK or start a server. +""" +from __future__ import annotations + +import json +import sys +from typing import Any, Callable + +from .agent_cli import error_payload, execute_request + + +def create_server(): + """Create an official SDK v2 server; only this entry needs the optional SDK.""" + try: + from mcp.server import MCPServer + from mcp.types import CallToolResult, TextContent, ToolAnnotations + from pydantic import Field + from typing import Annotated + except ImportError as exc: + raise ImportError("Install the optional MCP adapter with: pip install 'diffractscout[mcp]'") from exc + + from . import __version__, workflow + + server = MCPServer( + "DiffractScout", version=__version__, + instructions=("Use context and schema to discover workflow requests. Plan before execution. " + "Use submit for long jobs, then status/events/result. Credentials are read " + "from the server environment; never include them in requests. Results are " + "theoretical calculations with provenance, not experimental phase identification."), + ) + read = ToolAnnotations(read_only_hint=True, destructive_hint=False, + idempotent_hint=True, open_world_hint=False) + write = ToolAnnotations(read_only_hint=False, destructive_hint=False, + idempotent_hint=False, open_world_hint=True) + + def invoke(function: Callable, *args, **kwargs): + try: + return function(*args, **kwargs) + except Exception as exc: + payload = error_payload(exc) + return CallToolResult(is_error=True, structured_content=payload, + content=[TextContent(type="text", text=json.dumps(payload))]) + + # Resolve locally imported Annotated/Field before SDK signature inspection. + page_limit = Annotated[int, Field(ge=1, le=1000, description="Maximum rows returned.")] + nonnegative = Annotated[int, Field(ge=0)] + + @server.tool(annotations=read) + def context() -> dict[str, Any]: + """Describe supported workflows, units, capabilities, and execution boundaries.""" + return invoke(workflow.context) + + @server.tool(annotations=read) + def schema(name: str = "request") -> dict[str, Any]: + """Return the named workflow JSON Schema, including the request contract.""" + return invoke(workflow.schema, name) + + @server.tool(annotations=read) + def plan(request: dict[str, Any]) -> dict[str, Any]: + """Validate a workflow request and describe its stages without executing it.""" + return invoke(workflow.plan_workflow, request) + + @server.tool(annotations=write) + def run(request: dict[str, Any], workspace: str | None = None) -> dict[str, Any]: + """Run one request or diffractscout_batch_v1 manifest to completion.""" + return invoke(execute_request, request, workspace=workspace) + + @server.tool(annotations=write) + def submit(request: dict[str, Any], workspace: str | None = None) -> dict[str, Any]: + """Submit durable background work and return immediately with its job identity.""" + return invoke(execute_request, request, workspace=workspace, background=True) + + def list_jobs(workspace: str | None = None, offset: int = 0, + limit: int = 100) -> dict[str, Any]: + """List durable jobs with bounded pagination; offsets start at zero.""" + def get_page(): + if offset < 0 or not 1 <= limit <= 1000: + raise ValueError("offset must be nonnegative and limit must be between 1 and 1000.") + jobs = workflow.list_jobs(workspace=workspace) + end = offset + limit + return {"jobs": jobs[offset:end], "offset": offset, "limit": limit, + "total": len(jobs), "next_offset": end if end < len(jobs) else None} + return invoke(get_page) + + @server.tool(annotations=read) + def status(run_id: str, workspace: str | None = None) -> dict[str, Any]: + """Read a job's state, progress, counters, diagnostics, and artifact paths.""" + return invoke(workflow.status_job, run_id, workspace=workspace) + + def events(run_id: str, workspace: str | None = None, after: int = 0, + limit: int = 100) -> dict[str, Any]: + """Read a bounded event page after the given sequence cursor.""" + return invoke(workflow.read_events, run_id, workspace=workspace, after=after, limit=limit) + + def result(run_id: str, workspace: str | None = None, table: str | None = None, + offset: int = 0, limit: int = 100, verify: bool = True) -> dict[str, Any]: + """Read verified artifacts or a bounded table page; preserve scientific missing values.""" + return invoke(workflow.get_result, run_id, workspace=workspace, table=table, + offset=offset, limit=limit, verify=verify) + + # Numeric constraints are part of the published MCP input schemas. + for function, fields in ((list_jobs, {"offset": nonnegative, "limit": page_limit}), + (events, {"after": nonnegative, "limit": page_limit}), + (result, {"offset": nonnegative, "limit": page_limit})): + function.__annotations__.update(fields) + server.tool(annotations=read)(function) + + @server.tool(annotations=write) + def resume(run_id: str, workspace: str | None = None, + background: bool = True) -> dict[str, Any]: + """Resume unfinished work with the service's input and checkpoint integrity checks.""" + return invoke(workflow.resume_job, run_id, workspace=workspace, background=background) + + @server.tool(annotations=ToolAnnotations(read_only_hint=False, destructive_hint=False, + idempotent_hint=True, open_world_hint=False)) + def cancel(run_id: str, workspace: str | None = None) -> dict[str, Any]: + """Request cooperative cancellation; retain completed items and recorded evidence.""" + return invoke(workflow.cancel_job, run_id, workspace=workspace) + + return server + + +def main() -> int: + try: + create_server().run(transport="stdio") + except ImportError as exc: + print(json.dumps(error_payload(exc)), file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/diffractscout/network.py b/src/diffractscout/network.py new file mode 100644 index 0000000..9c051d9 --- /dev/null +++ b/src/diffractscout/network.py @@ -0,0 +1,14 @@ +"""Classify transient transport failures without depending on an HTTP client.""" +from __future__ import annotations + + +def transient_error(error: BaseException) -> bool: + status = getattr(error, "status_code", None) + if status is None: + status = getattr(getattr(error, "response", None), "status_code", None) + if status is not None: + return status in {408, 429, 500, 502, 503, 504} + return isinstance(error, (TimeoutError, ConnectionError)) or any( + cls.__name__ in {"Timeout", "ConnectTimeout", "ReadTimeout", "ConnectError", "ConnectionError"} + for cls in type(error).__mro__ + ) diff --git a/src/diffractscout/pipeline.py b/src/diffractscout/pipeline.py index 9f08215..590ca59 100644 --- a/src/diffractscout/pipeline.py +++ b/src/diffractscout/pipeline.py @@ -2191,6 +2191,7 @@ def analyze_cifs( include_excel: bool = True, overwrite: bool = False, elastic_overrides: Mapping[str, ElasticTensor] | None = None, + _execution: Any = None, ) -> PipelineResult: settings = settings or AnalysisSettings() validate_analysis_settings(settings) @@ -2217,7 +2218,10 @@ def analyze_cifs( ) for path in paths ] - analyses, diagnostics = _analyze_paths(copied, settings) + runner = _execution.analyze_paths if _execution is not None else _analyze_paths + analyses, diagnostics = runner(copied, settings) + if _execution is not None: + _execution.before_export(diagnostics) export_result_bundle( staging, analyses=analyses, @@ -2226,6 +2230,8 @@ def analyze_cifs( diagnostics=diagnostics, include_excel=include_excel, ) + if _execution is not None: + _execution.before_verify(staging) manifest = _verify_and_commit( target, staging, @@ -2234,7 +2240,7 @@ def analyze_cifs( expected_state=expected_target_state, diagnostics=diagnostics, ) - except Exception: + except BaseException: shutil.rmtree(staging, ignore_errors=True) raise _rebind_committed_results(analyses, [], staging, target) @@ -2267,6 +2273,7 @@ def export_discovery( discovery_settings: DiscoverySettings | None = None, include_excel: bool = True, overwrite: bool = False, + _execution: Any = None, ) -> PipelineResult: target = _validate_output_target(output_dir, overwrite=overwrite) expected_target_state = _capture_target_state(target) @@ -2281,6 +2288,8 @@ def export_discovery( for warning in discovery.warnings ] try: + if _execution is not None: + _execution.before_export(diagnostics) export_result_bundle( staging, analyses=[], @@ -2290,6 +2299,8 @@ def export_discovery( diagnostics=diagnostics, include_excel=include_excel, ) + if _execution is not None: + _execution.before_verify(staging) manifest = _verify_and_commit( target, staging, @@ -2297,7 +2308,7 @@ def export_discovery( expected_state=expected_target_state, diagnostics=diagnostics, ) - except Exception: + except BaseException: shutil.rmtree(staging, ignore_errors=True) raise _rebind_committed_results([], [], staging, target, discovery=discovery) @@ -2325,6 +2336,7 @@ def run_pipeline( overwrite: bool = False, confirm_above: int = 200, authorize_large_download: bool = False, + _execution: Any = None, ) -> PipelineResult: if analysis_settings is None: requested_analysis_settings = AnalysisSettings( @@ -2453,8 +2465,11 @@ def run_pipeline( ) path_pairs.append((item.cif_path, tensor)) - analyses, analysis_diagnostics = _analyze_paths(path_pairs, settings) + runner = _execution.analyze_paths if _execution is not None else _analyze_paths + analyses, analysis_diagnostics = runner(path_pairs, settings) diagnostics.extend(analysis_diagnostics) + if _execution is not None: + _execution.before_export(diagnostics) export_result_bundle( staging, analyses=analyses, @@ -2464,6 +2479,8 @@ def run_pipeline( diagnostics=diagnostics, include_excel=include_excel, ) + if _execution is not None: + _execution.before_verify(staging) manifest = _verify_and_commit( target, staging, @@ -2472,7 +2489,7 @@ def run_pipeline( expected_state=expected_target_state, diagnostics=diagnostics, ) - except Exception: + except BaseException: shutil.rmtree(staging, ignore_errors=True) raise diff --git a/src/diffractscout/providers/materials_project.py b/src/diffractscout/providers/materials_project.py index 2a865b5..debdda9 100644 --- a/src/diffractscout/providers/materials_project.py +++ b/src/diffractscout/providers/materials_project.py @@ -485,13 +485,15 @@ def download_candidates( ) ) except Exception as exc: + from ..network import transient_error + results.append( DownloadArtifact( candidate=candidate, cif_path=None, status="failed", error=str(exc), - provider_metadata=self.metadata(), + provider_metadata={**self.metadata(), "retryable": transient_error(exc)}, ) ) diff --git a/src/diffractscout/workflow/__init__.py b/src/diffractscout/workflow/__init__.py new file mode 100644 index 0000000..3058ee6 --- /dev/null +++ b/src/diffractscout/workflow/__init__.py @@ -0,0 +1,16 @@ +"""Versioned, resumable workflows for Python, CLI, GUI and external agents.""" +from .contracts import ( + ActionableDiagnostic, WorkflowRequest, + context, plan_workflow, schema, +) +from .records import ArtifactRef, RunStatus, WorkflowPlan +from .results import get_result, load_result_object +from .service import ( + cancel_job, list_jobs, read_events, resume_job, run_batch, run_job, status_job, submit_job, +) + +__all__ = [ + "ActionableDiagnostic", "ArtifactRef", "RunStatus", "WorkflowPlan", "WorkflowRequest", + "context", "plan_workflow", "schema", "get_result", "load_result_object", + "cancel_job", "list_jobs", "read_events", "resume_job", "run_batch", "run_job", "status_job", "submit_job", +] diff --git a/src/diffractscout/workflow/checkpoints.py b/src/diffractscout/workflow/checkpoints.py new file mode 100644 index 0000000..fe24216 --- /dev/null +++ b/src/diffractscout/workflow/checkpoints.py @@ -0,0 +1,130 @@ +"""Content-verified JSON/NPZ checkpoints, without executable serialization.""" +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import tempfile +from pathlib import Path +from typing import Any + +import numpy as np + +from ..models import ElasticTensor, PhaseAnalysis, ReflectionRecord, StructureRecord +from ..pipeline import _reject_reparse_components, _rename_directory_noreplace +from ..utils import package_versions, sha256_file, to_jsonable, write_json +from .store import read_json + + +def fingerprint(value: Any) -> str: + return hashlib.sha256(json.dumps(to_jsonable(value), sort_keys=True, separators=(",", ":"), + allow_nan=False).encode("utf-8")).hexdigest() + + +def environment() -> dict[str, Any]: + # Version alone does not identify an editable checkout. Hash the relevant + # Python sources, including this serializer, so changed code cannot silently + # consume a prior numerical checkpoint under the same package version. + package = Path(__file__).resolve().parents[1] + names = {"diffraction.py", "structure.py", "elasticity.py", "elasticity_input.py", "models.py", + "phase_cif.py", "cif_quality.py", "initial_cifs.py", "selection.py", "composition.py", "pipeline.py"} + sources = {p.relative_to(package).as_posix(): sha256_file(p) + for p in sorted(package.rglob("*.py")) + if (p.parent == package and p.name in names) or p.parent.name in {"workflow", "providers"}} + source_digest = sha256_file(os.sys.executable) if getattr(os.sys, "frozen", False) else fingerprint(sources) + return {"versions": package_versions(), "source_sha256": source_digest, + "python": os.sys.version.split()[0], "checkpoint_schema": 1} + + +def seal(directory: Path) -> None: + members = [] + for path in sorted(directory.rglob("*")): + _reject_reparse_components(path, label="checkpoint member") + if path.is_file() and path.name != "checkpoint.json": + members.append({"path": path.relative_to(directory).as_posix(), + "sha256": sha256_file(path), "size": path.stat().st_size}) + write_json(directory / "checkpoint.json", {"schema": "diffractscout_checkpoint_v1", "files": members}) + + +def verify_checkpoint(directory: Path) -> None: + _reject_reparse_components(directory, label="checkpoint") + manifest = read_json(directory / "checkpoint.json") + if manifest.get("schema") != "diffractscout_checkpoint_v1": + raise ValueError(f"Unknown checkpoint schema: {directory}") + seen = set() + for entry in manifest["files"]: + name = entry["path"] + relative = Path(name) + if relative.is_absolute() or ".." in relative.parts or name in seen or name == "checkpoint.json": + raise ValueError(f"Unsafe checkpoint member: {name}") + seen.add(name) + path = directory / relative + _reject_reparse_components(path, label="checkpoint member") + if not path.is_file() or path.stat().st_size != entry["size"] or sha256_file(path) != entry["sha256"]: + raise ValueError(f"Checkpoint integrity failed: {path}") + actual = set() + for p in directory.rglob("*"): + _reject_reparse_components(p, label="checkpoint member") + if p.is_file() and p != directory / "checkpoint.json": + actual.add(p.relative_to(directory).as_posix()) + if actual != seen: + raise ValueError(f"Checkpoint members differ from manifest: {directory}") + + +def publish_checkpoint(destination: Path, writer) -> Path: + destination.parent.mkdir(parents=True, exist_ok=True) + staging = Path(tempfile.mkdtemp(prefix=".checkpoint-", dir=destination.parent)) + try: + writer(staging) + seal(staging) + verify_checkpoint(staging) + _rename_directory_noreplace(staging, destination) + finally: + if staging.exists(): + shutil.rmtree(staging) + return destination + + +def save_analysis(directory: Path, analysis: PhaseAnalysis) -> None: + payload = to_jsonable(analysis) + payload.pop("two_theta_grid") + payload.pop("intensity_profile") + write_json(directory / "analysis.json", payload) + np.savez(directory / "arrays.npz", two_theta_grid=analysis.two_theta_grid, + intensity_profile=analysis.intensity_profile) + + +def tensor_from_dict(payload: dict[str, Any] | None) -> ElasticTensor | None: + if payload is None: + return None + values = dict(payload) + values["stiffness_GPa"] = np.asarray(values["stiffness_GPa"], dtype=float) + if values.get("raw_payload_path"): + values["raw_payload_path"] = Path(values["raw_payload_path"]) + return ElasticTensor(**values) + + +def load_analysis(directory: Path, *, cif_path: Path | None = None) -> PhaseAnalysis: + verify_checkpoint(directory) + data = read_json(directory / "analysis.json") + structure = data.pop("structure") + old_path = Path(structure["cif_path"]) + structure["cif_path"] = cif_path or old_path + structure["cell_parameters"] = tuple(structure["cell_parameters"]) + result = PhaseAnalysis( + structure=StructureRecord(**structure), + reflections=[ReflectionRecord(**row) for row in data.pop("reflections")], + elastic_tensor=tensor_from_dict(data.pop("elastic_tensor")), + two_theta_grid=np.array([]), intensity_profile=np.array([]), **data, + ) + with np.load(directory / "arrays.npz", allow_pickle=False) as arrays: + result.two_theta_grid = arrays["two_theta_grid"] + result.intensity_profile = arrays["intensity_profile"] + if cif_path is not None: + from ..pipeline import _rebind_metadata_paths + result.structure.source_metadata = _rebind_metadata_paths(result.structure.source_metadata, old_path.parent, cif_path.parent) + result.metadata = _rebind_metadata_paths(result.metadata, old_path.parent, cif_path.parent) + if result.elastic_tensor and result.elastic_tensor.raw_payload_path: + result.elastic_tensor.raw_payload_path = cif_path.parent / result.elastic_tensor.raw_payload_path.name + return result diff --git a/src/diffractscout/workflow/contracts.py b/src/diffractscout/workflow/contracts.py new file mode 100644 index 0000000..a0e8b81 --- /dev/null +++ b/src/diffractscout/workflow/contracts.py @@ -0,0 +1,630 @@ +"""Strict workflow requests and local, side-effect-free execution planning. + +Planning deliberately never constructs a provider or calls an execution helper. +The existing scientific validators remain the authority for numerical settings. +""" + +from __future__ import annotations + +import copy +import importlib.util +import math +import os +import types +from dataclasses import asdict, dataclass, field, fields +from pathlib import Path +from typing import Any, Literal, Mapping, Union, get_args, get_origin, get_type_hints + +from ..models import AnalysisSettings, DiscoverySettings, ElasticTensor + +REQUEST_SCHEMA = "diffractscout_workflow_v1" +WORKFLOWS = ("analyze", "discover", "run", "prepare", "prepare_analyze", "fetch", "adapt") +EXPORT_DEFAULTS = {"include_excel": True, "template": "full"} +EXECUTION_DEFAULTS = { + "recursive": True, + "overwrite": False, + "conventional_unit_cell": True, + "confirm_above": 200, + "authorize_large_download": False, + "max_attempts": 3, +} +_CHEMISTRY = ("nominal", "weight_percent", "atomic_percent") +_PREPARATION_DEFAULTS = { + "nominal": "", "weight_percent": "", "atomic_percent": "", "host": "", + "phases": None, "templates": {}, "parameter_file": None, "phase_parameters": None, + "offline": False, "max_subsystems": 64, "max_prototype_attempts": 8, + "preview_wavelength_A": 1.5406, +} +_ADAPTATION_DEFAULTS = { + "nominal": "", "weight_percent": "", "atomic_percent": "", "citation": "", + "a": None, "b": None, "c": None, "alpha": None, "beta": None, "gamma": None, + "fract": {}, +} +_SECRET_KEYS = {"api_key", "mp_api_key", "apikey", "password", "token", "access_token", + "secret", "credentials", "authorization"} + + +def _reject_credentials(value: Any) -> None: + if isinstance(value, Mapping): + for key, item in value.items(): + if isinstance(key, str) and key.lower().replace("-", "_") in _SECRET_KEYS: + raise ValueError("Credentials must be supplied at runtime, outside the request.") + _reject_credentials(item) + elif isinstance(value, (list, tuple)): + for item in value: + _reject_credentials(item) + + +def _object(value: Any, name: str, allowed: Any) -> dict[str, Any]: + if not isinstance(value, Mapping) or any(not isinstance(key, str) for key in value): + raise ValueError(f"{name} must be an object with string keys.") + unknown = set(value) - set(allowed) + if unknown: + raise ValueError(f"Unknown {name} option(s): {', '.join(sorted(unknown))}.") + return copy.deepcopy(dict(value)) + + +def _text(value: Any, name: str, *, nullable: bool = False) -> None: + if value is None and nullable: + return + if not isinstance(value, str): + raise ValueError(f"{name} must be text.") + + +def _number(value: Any, name: str, *, positive: bool = False, integer: bool = False) -> None: + expected = int if integer else (int, float) + if isinstance(value, bool) or not isinstance(value, expected) or not math.isfinite(value): + raise ValueError(f"{name} must be a finite {'integer' if integer else 'number'}.") + if positive and value <= 0: + raise ValueError(f"{name} must be positive.") + + +def _matches(value: Any, annotation: Any) -> bool: + origin = get_origin(annotation) + if origin in (Union, types.UnionType): + return any(_matches(value, item) for item in get_args(annotation)) + if origin is Literal: + return value in get_args(annotation) and isinstance(value, str) + if annotation is type(None): + return value is None + if annotation is float: + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) + return type(value) is annotation + + +def _settings(value: Any, cls: type, name: str) -> dict[str, Any]: + payload = _object(value, name, (item.name for item in fields(cls))) + if cls is AnalysisSettings and "input_mode" not in payload: + numeric = [key for key in ("energy_keV", "wavelength_A") if payload.get(key) is not None] + if len(numeric) > 1: + raise ValueError("Supply only one of energy_keV and wavelength_A.") + if numeric and "source_preset" not in payload: + payload["input_mode"] = "energy" if numeric[0] == "energy_keV" else "wavelength" + for key, annotation in get_type_hints(cls).items(): + if key in payload and not _matches(payload[key], annotation): + raise ValueError(f"Invalid type or value for {name}.{key}.") + settings = cls(**payload) + if cls is AnalysisSettings: + from ..diffraction import validate_analysis_settings + validate_analysis_settings(settings) + else: + from ..selection import validate_discovery_settings + validate_discovery_settings(settings) + return asdict(settings) + + +def _path(value: str, base_dir: str | Path | None) -> str: + path = Path(value).expanduser() + if not path.is_absolute() and base_dir is not None: + path = Path(base_dir).expanduser() / path + return str(path.resolve()) + + +def _preparation(value: Any, base_dir: str | Path | None) -> dict[str, Any]: + from ..initial_cifs import _validate_phase_parameters + from ..phase_cif import canonical_phase, canonicalize_phases + + payload = _object(value, "preparation", _PREPARATION_DEFAULTS) + for key in (*_CHEMISTRY, "host"): + if key in payload: + _text(payload[key], f"preparation.{key}") + if "offline" in payload and type(payload["offline"]) is not bool: + raise ValueError("preparation.offline must be boolean.") + for key in ("max_subsystems", "max_prototype_attempts"): + if key in payload: + _number(payload[key], f"preparation.{key}", integer=True, positive=True) + if "preview_wavelength_A" in payload: + _number(payload["preview_wavelength_A"], "preparation.preview_wavelength_A", positive=True) + if payload.get("phases") is not None: + if not isinstance(payload["phases"], list) or any(not isinstance(x, str) for x in payload["phases"]): + raise ValueError("preparation.phases must be an array of phase names.") + payload["phases"] = list(canonicalize_phases(payload["phases"])) + if "templates" in payload: + templates = payload["templates"] + if not isinstance(templates, Mapping): + raise ValueError("preparation.templates must map phase names to paths.") + normalized = {} + for key, path in templates.items(): + _text(key, "template phase") + _text(path, "template path") + if not path.strip(): + raise ValueError("Template path must not be empty.") + phase = canonical_phase(key) + if phase in normalized: + raise ValueError(f"Duplicate template for {phase}.") + normalized[phase] = _path(path, base_dir) + payload["templates"] = normalized + if payload.get("parameter_file") is not None: + _text(payload["parameter_file"], "preparation.parameter_file") + if not payload["parameter_file"].strip(): + raise ValueError("parameter_file must not be empty.") + payload["parameter_file"] = _path(payload["parameter_file"], base_dir) + if payload.get("phase_parameters") is not None: + payload["phase_parameters"] = _validate_phase_parameters(payload["phase_parameters"]) + if payload.get("parameter_file") is not None and payload.get("phase_parameters") is not None: + raise ValueError("Use parameter_file or phase_parameters, not both.") + return payload + + +def _adaptation(value: Any) -> dict[str, Any]: + from ..phase_cif import _validate_requested_cell, _validate_requested_coordinates, resolve_occupancies + + payload = _object(value, "adaptation", _ADAPTATION_DEFAULTS) + for key in (*_CHEMISTRY, "citation"): + if key in payload: + _text(payload[key], f"adaptation.{key}") + lattice = {key: payload[key] for key in ("a", "b", "c", "alpha", "beta", "gamma") + if payload.get(key) is not None} + for key, item in lattice.items(): + _number(item, f"adaptation.{key}") + coordinates = _object(payload.get("fract", {}), "adaptation.fract", ("x", "y", "z")) + for key, item in coordinates.items(): + _number(item, f"adaptation.fract.{key}") + _validate_requested_cell(lattice) + _validate_requested_coordinates(coordinates) + if (lattice or coordinates) and not payload.get("citation", "").strip(): + raise ValueError("Lattice or fractional coordinate changes require a citation.") + chemistry = {key: payload.get(key, "") for key in _CHEMISTRY} + if any(chemistry.values()): + from ..cif_quality import INTERSTITIALS + _, _, occupancies = resolve_occupancies(**chemistry) + if set(occupancies) & INTERSTITIALS: + raise ValueError("Adaptation cannot substitute interstitial percentages onto metal sites.") + return payload + + +def _elastic_overrides(value: Any, base_dir: str | Path | None) -> dict[str, Any]: + from ..elasticity import validate_elastic_tensor + from ..utils import to_jsonable + + if not isinstance(value, Mapping): + raise ValueError("elastic_overrides must be an object.") + output = {} + tensor_fields = {item.name for item in fields(ElasticTensor)} + for key, raw in value.items(): + _text(key, "elastic override key") + payload = asdict(raw) if isinstance(raw, ElasticTensor) else _object(raw, "elastic tensor", tensor_fields) + if "stiffness_GPa" not in payload: + raise ValueError("Elastic tensor requires stiffness_GPa.") + if payload.get("unit", "GPa") != "GPa": + raise ValueError("stiffness_GPa must use GPa units.") + for name in tensor_fields - {"stiffness_GPa", "warnings", "raw_payload_path"}: + if name in payload: + _text(payload[name], f"elastic tensor.{name}") + if "warnings" in payload and (not isinstance(payload["warnings"], list) or any( + not isinstance(item, str) for item in payload["warnings"] + )): + raise ValueError("Elastic tensor warnings must be an array of strings.") + metadata = {name: payload[name] for name in ( + "source_provider", "source_record_id", "source_url", "methodology_url", + "nature_of_data", "coordinate_frame", + ) if name in payload} + if payload.get("raw_payload_path") is not None: + raw_path = payload["raw_payload_path"] + if not isinstance(raw_path, (str, Path)): + raise ValueError("raw_payload_path must be a path.") + metadata["raw_payload_path"] = Path(_path(str(raw_path), base_dir)) + tensor = validate_elastic_tensor(payload["stiffness_GPa"], **metadata) + if tensor.status == "invalid": + raise ValueError("Invalid elastic tensor: " + "; ".join(tensor.warnings)) + normalized_key = _path(key, base_dir) if Path(key).is_absolute() or "/" in key or "\\" in key else key + output[normalized_key] = to_jsonable(asdict(tensor)) + return output + + +@dataclass +class WorkflowRequest: + workflow: str + schema: str = REQUEST_SCHEMA + name: str = "" + inputs: list[str] = field(default_factory=list) + composition: str = "" + analysis: dict[str, Any] = field(default_factory=dict) + discovery: dict[str, Any] = field(default_factory=dict) + preparation: dict[str, Any] = field(default_factory=dict) + adaptation: dict[str, Any] = field(default_factory=dict) + exports: dict[str, Any] = field(default_factory=lambda: dict(EXPORT_DEFAULTS)) + execution: dict[str, Any] = field(default_factory=lambda: dict(EXECUTION_DEFAULTS)) + output_dir: str | None = None + experiment: str = "laboratory" + submission_id: str | None = None + parent_run_id: str | None = None + elastic_overrides: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_dict(cls, mapping: Mapping[str, Any], base_dir: str | Path | None = None) -> WorkflowRequest: + _reject_credentials(mapping) + payload = _object(mapping, "request", (item.name for item in fields(cls))) + if payload.get("schema", REQUEST_SCHEMA) != REQUEST_SCHEMA: + raise ValueError(f"Request schema must be {REQUEST_SCHEMA!r}.") + if payload.get("workflow") not in WORKFLOWS: + raise ValueError(f"workflow must be one of {', '.join(WORKFLOWS)}.") + for key in ("name", "composition"): + if key in payload: + _text(payload[key], key) + for key in ("submission_id", "parent_run_id", "output_dir"): + if key in payload: + _text(payload[key], key, nullable=True) + if payload.get("experiment", "laboratory") not in ("laboratory", "synchrotron"): + raise ValueError("experiment must be laboratory or synchrotron.") + inputs = payload.get("inputs", []) + if not isinstance(inputs, list) or any(not isinstance(item, str) or not item.strip() for item in inputs): + raise ValueError("inputs must be an array of non-empty paths.") + payload["inputs"] = [_path(item, base_dir) for item in inputs] + if payload.get("output_dir") is not None: + if not payload["output_dir"].strip(): + raise ValueError("output_dir must not be empty.") + target = Path(payload["output_dir"]).expanduser() + if not target.is_absolute() and base_dir is not None: + target = Path(base_dir) / target + # Preserve reparse/symlink components for the output validator. + payload["output_dir"] = os.path.abspath(target) + payload["analysis"] = _settings(payload.get("analysis", {}), AnalysisSettings, "analysis") + payload["discovery"] = _settings(payload.get("discovery", {}), DiscoverySettings, "discovery") + payload["preparation"] = _preparation(payload.get("preparation", {}), base_dir) + payload["adaptation"] = _adaptation(payload.get("adaptation", {})) + exports = {**EXPORT_DEFAULTS, **_object(payload.get("exports", {}), "exports", EXPORT_DEFAULTS)} + if type(exports["include_excel"]) is not bool: + raise ValueError("exports.include_excel must be boolean.") + if exports["template"] not in ("full", "peak_table"): + raise ValueError("exports.template must be full or peak_table.") + payload["exports"] = exports + execution = {**EXECUTION_DEFAULTS, **_object(payload.get("execution", {}), "execution", EXECUTION_DEFAULTS)} + for key, default in EXECUTION_DEFAULTS.items(): + if isinstance(default, bool): + if type(execution[key]) is not bool: + raise ValueError(f"execution.{key} must be boolean.") + else: + _number(execution[key], f"execution.{key}", integer=True, positive=True) + payload["execution"] = execution + if execution["max_attempts"] > 3: + raise ValueError("execution.max_attempts must be between 1 and 3.") + payload["elastic_overrides"] = _elastic_overrides(payload.get("elastic_overrides", {}), base_dir) + result = cls(**payload) + result._provided_fields = { + section: set(mapping.get(section, {})) for section in ("analysis", "discovery", "preparation", "exports", "execution") + } + return result + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class ActionableDiagnostic: + code: str + message: str + stage: str = "planning" + item: str = "" + retryable: bool = False + missing_fields: list[str] = field(default_factory=list) + level: str = "error" + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def _available(module: str) -> bool: + # Inspect the top-level spec only: dotted find_spec imports parent packages. + try: + return importlib.util.find_spec(module) is not None + except (ImportError, ValueError, AttributeError): + return False + + +def _validate_preparation_template(path: str, phase: str, host: str) -> None: + """Check either an explicit phase model or the supported P1 restoration case.""" + from ..cif_quality import _check_topology, _dataset, _spglib_cell, validate_phase_model + from ..structure import load_structure + + loaded = load_structure(path) + if loaded.space_group_number != 1: + validate_phase_model(path, phase, host=host) + return + cell, fingerprints = _spglib_cell(loaded.small_structure) + dataset = _dataset(cell) + _check_topology(dataset, phase) + _check_topology(_dataset((dataset.std_lattice, dataset.std_positions, dataset.std_types)), phase) + present = {element for fingerprint in fingerprints.values() for element, _ in fingerprint} + if host and host not in present: + raise ValueError(f"The prototype does not contain host {host}.") + + +def context() -> dict[str, Any]: + """Describe local capabilities without importing optional packages or networking.""" + return { + "schema": REQUEST_SCHEMA, + "workflows": list(WORKFLOWS), + "request_schema": "request", + "credentials": {"environment": "MP_API_KEY", "configured": bool(os.environ.get("MP_API_KEY", "").strip()), + "request_storage": False}, + "dependencies": {name: _available(module) for name, module in { + "gemmi": "gemmi", "numpy": "numpy", "pandas": "pandas", "openpyxl": "openpyxl", + "spglib": "spglib", "mp-api": "mp_api", "pymatgen": "pymatgen", "matplotlib": "matplotlib", + }.items()}, + "planning": {"network": False, "writes": False}, + } + + +def plan_workflow(request: WorkflowRequest | Mapping[str, Any], *, provider: Any = None, + api_key: str | None = None) -> dict[str, Any]: + """Validate readiness using local metadata only; injected providers are never called.""" + if api_key is not None and not isinstance(api_key, str): + raise ValueError("Runtime api_key must be text.") + provided = getattr(request, "_provided_fields", None) + request = WorkflowRequest.from_dict(request.to_dict() if isinstance(request, WorkflowRequest) else request) + provided = provided or request._provided_fields + diagnostics: list[dict[str, Any]] = [] + + def diagnostic(code: str, message: str, stage: str = "planning", item: str = "", + missing_fields: list[str] | None = None) -> None: + diagnostics.append(ActionableDiagnostic(code, message, stage, item, + missing_fields=missing_fields or []).to_dict()) + + stages = { + "analyze": ["analyze", "export", "verify"], + "discover": ["discover", "export", "verify"], + "run": ["discover", "download", "analyze", "export", "verify"], + "prepare": ["prepare", "verify"], + "prepare_analyze": ["prepare", "analyze", "export", "verify"], + "fetch": ["fetch", "verify"], + "adapt": ["adapt", "verify"], + }[request.workflow] + needs_analysis = "analyze" in stages + if needs_analysis and request.experiment == "synchrotron" and not ( + request.analysis["wavelength_A"] is not None or request.analysis["energy_keV"] is not None + ): + diagnostic("missing_radiation", "Synchrotron analysis requires an explicit wavelength_A or energy_keV.", + "analyze", missing_fields=["analysis.wavelength_A", "analysis.energy_keV"]) + sources = list(request.inputs) + local_cifs: list[str] = [] + if request.workflow in ("analyze", "adapt"): + if not sources: + diagnostic("missing_inputs", "Provide local CIF input paths.", request.workflow, missing_fields=["inputs"]) + else: + from ..pipeline import collect_cif_paths + if request.workflow == "adapt" and (len(sources) != 1 or not Path(sources[0]).is_file()): + diagnostic("invalid_inputs", "Adapt requires exactly one local CIF file.", "adapt", "inputs") + try: + local_cifs = [str(path) for path in collect_cif_paths(sources, recursive=request.execution["recursive"])] + if not local_cifs: + diagnostic("no_cifs", "No CIF files were found in the supplied inputs.", request.workflow, "inputs") + except (OSError, ValueError) as exc: + diagnostic("invalid_inputs", str(exc), request.workflow, "inputs") + options = {**_PREPARATION_DEFAULTS, **request.preparation} + templates = options["templates"] + sources.extend(templates.values()) + if options["parameter_file"]: + sources.append(options["parameter_file"]) + if request.output_dir: + from ..pipeline import _reject_reparse_components, _validate_input_output_separation + try: + _reject_reparse_components(Path(request.output_dir), label="workflow output") + _validate_input_output_separation(sources, request.output_dir) + target = Path(request.output_dir) + if target.exists() and (not target.is_dir() or ( + any(target.iterdir()) and not request.execution["overwrite"] + )): + diagnostic("output_exists", "Choose a new output directory; the target is already occupied.", item="output_dir") + if target.exists() and request.workflow in ("prepare", "prepare_analyze"): + diagnostic("output_exists", "Preparation requires a new output directory.", "prepare", "output_dir") + if (target.is_dir() and any(target.iterdir()) and request.execution["overwrite"] + and request.workflow in ("analyze", "discover", "run")): + from ..validation import verify_bundle + verification = verify_bundle(target) + if not verification["ok"]: + diagnostic("unsafe_overwrite", "The existing output is not an intact DiffractScout bundle.", item="output_dir") + except (OSError, ValueError) as exc: + diagnostic("unsafe_output", str(exc), item="output_dir") + estimated_queries = 0 + requires_provider = request.workflow in ("discover", "run") + parameter_sources: dict[str, Any] = { + "analysis": {key: "request" if key in provided["analysis"] else "default" for key in request.analysis}, + "discovery": {key: "request" if key in provided["discovery"] else "default" for key in request.discovery}, + "radiation": request.analysis["input_mode"], + "output": "request.output_dir" if request.output_dir else "managed per-run bundle", + } + if "input_mode" not in provided["analysis"]: + for field_name in ("energy_keV", "wavelength_A"): + if field_name in provided["analysis"] and request.analysis[field_name] is not None: + parameter_sources["analysis"]["input_mode"] = "derived_from_" + field_name + parsed = None + if request.workflow in ("discover", "run", "prepare", "prepare_analyze", "fetch"): + from ..composition import parse_composition_text + try: + parsed = parse_composition_text(request.composition) + if not parsed.elements and not parsed.material_ids: + diagnostic("missing_composition", "Provide a chemical composition or material IDs.", + request.workflow, missing_fields=["composition"]) + except ValueError as exc: + diagnostic("invalid_composition", str(exc), request.workflow, "composition") + if parsed is not None and request.workflow in ("discover", "run"): + mode = request.discovery["mode"] + if mode == "mpids_only" or (parsed.material_ids and not parsed.elements and mode == "possible_phases"): + estimated_queries = int(bool(parsed.material_ids)) + if not parsed.material_ids: + diagnostic("missing_material_ids", "mpids_only requires material IDs.", "discover", "composition") + elif not parsed.elements: + diagnostic("missing_elements", "Element-based discovery requires parsed elements.", "discover", "composition") + else: + count = len(parsed.elements) + upper = min(count, request.discovery["max_subsystem_order"] or count) + estimated_queries = (1 if mode == "single_chemsys" else sum(math.comb(count, order) for order in range(1, upper + 1))) + if mode != "single_chemsys" and estimated_queries > request.discovery["max_subsystems"]: + diagnostic("query_limit", "Chemical-subsystem expansion exceeds discovery.max_subsystems.", "discover", "discovery.max_subsystems") + estimated_queries += int(bool(parsed.material_ids)) + if request.workflow in ("prepare", "prepare_analyze", "fetch"): + from ..initial_cifs import _resolve_request, read_phase_parameters + from ..phase_cif import canonicalize_phases + selected = canonicalize_phases(options["phases"]) + missing = set(selected) - set(templates) + parameters = options["phase_parameters"] or {} + try: + if options["parameter_file"]: + parameters = read_phase_parameters(options["parameter_file"]) + if set(parameters) - set(selected) or set(templates) - set(selected): + raise ValueError("Templates and phase parameters must use selected phases.") + for path in templates.values(): + if not Path(path).is_file(): + raise FileNotFoundError(f"Template CIF not found: {path}") + if request.workflow != "fetch": + elements, host, chemistry = _resolve_request( + request.composition, {key: options[key] for key in _CHEMISTRY}, options["host"], parameters, + ) + parameter_sources["preparation"] = { + "host": host, "chemistry": chemistry, + "phase_parameters": options["parameter_file"] or ("request" if parameters else "prototype"), + "templates": dict(templates), + } + # Packaged fallback is defined by the current preparation implementation. + # Availability is not a claim that arbitrary chemistry has a valid prototype. + requires_provider = bool(missing and host != "Ti" and not options["offline"]) + if missing and options["offline"] and host != "Ti": + diagnostic("missing_templates", "Offline preparation requires caller templates for this host.", "prepare", "preparation.templates") + will_query = bool(missing and not options["offline"] and ( + provider is not None or api_key or os.environ.get("MP_API_KEY") or requires_provider + )) + else: + elements = parsed.elements if parsed else () + if not elements: + raise ValueError("fetch requires a composition containing elements.") + host = options["host"] or elements[0] + if host not in elements: + raise ValueError("Host must be an element in the requested composition.") + requires_provider = bool(missing - {"alpha-double-prime"}) + will_query = bool(missing and (requires_provider or provider is not None or api_key or os.environ.get("MP_API_KEY"))) + if will_query: + estimated_queries = (1 << len(elements)) - 1 + if estimated_queries > options["max_subsystems"]: + diagnostic("query_limit", "Chemical-subsystem expansion exceeds preparation.max_subsystems.", request.workflow, "preparation.max_subsystems") + # Supplied templates can be checked locally, with no normalization writes. + for phase, path in templates.items(): + if request.workflow == "fetch": + from ..phase_cif import _read_cif_identity + _read_cif_identity(Path(path)) + else: + _validate_preparation_template(path, phase, host) + except (OSError, ValueError) as exc: + diagnostic("invalid_preparation", str(exc), request.workflow, "preparation") + if request.workflow == "adapt": + from ..phase_cif import resolve_occupancies + try: + resolve_occupancies(**{key: request.adaptation.get(key, "") for key in _CHEMISTRY}) + except ValueError as exc: + diagnostic("invalid_adaptation", str(exc), "adapt", "adaptation") + dependencies = context()["dependencies"] + required = ["gemmi", "numpy", "spglib"] + if needs_analysis or request.workflow == "discover": + required.append("pandas") + if request.exports["include_excel"]: + required.append("openpyxl") + if needs_analysis and request.analysis["include_figures"]: + required.append("matplotlib") + # Optional provider dependencies are required only for live database work. + if requires_provider and provider is None: + required.extend(("mp-api", "pymatgen")) + if not (api_key or os.environ.get("MP_API_KEY", "")).strip(): + diagnostic("missing_credentials", "Materials Project credentials are required at runtime.", request.workflow, + missing_fields=["MP_API_KEY"]) + for dependency in required: + if not dependencies[dependency]: + diagnostic("missing_dependency", f"Required dependency is unavailable: {dependency}.", item=dependency) + return { + "ready": not diagnostics, "request": request.to_dict(), "stages": stages, + "diagnostics": diagnostics, "estimated_queries": estimated_queries, + "required_dependencies": required, "parameter_sources": parameter_sources, + "local_cifs": local_cifs, + } + + +def _annotation_schema(annotation: Any) -> dict[str, Any]: + origin = get_origin(annotation) + if origin in (Union, types.UnionType): + return {"anyOf": [_annotation_schema(item) for item in get_args(annotation)]} + if origin is Literal: + return {"type": "string", "enum": list(get_args(annotation))} + return {"type": {str: "string", float: "number", int: "integer", bool: "boolean", type(None): "null"}[annotation]} + + +def _settings_schema(cls: type) -> dict[str, Any]: + defaults = asdict(cls()) + hints = get_type_hints(cls) + properties = {key: {**_annotation_schema(hints[key]), "default": default} for key, default in defaults.items()} + return {"type": "object", "additionalProperties": False, "properties": properties, "default": {}} + + +def schema(name: str = "request") -> dict[str, Any]: + """Return the JSON Schema for the supported request contract.""" + if name not in ("request", REQUEST_SCHEMA): + raise ValueError(f"Unknown schema {name!r}; use 'request'.") + string = {"type": "string"} + nullable_string = {"type": ["string", "null"], "default": None} + number = {"type": "number"} + phase_properties = { + "lattice": {"type": "object", "additionalProperties": False, "properties": {key: number for key in ("a", "b", "c", "alpha", "beta", "gamma")}}, + "fract": {"type": "object", "additionalProperties": False, "properties": {key: number for key in ("x", "y", "z")}}, + **{key: string for key in (*_CHEMISTRY, "citation", "conditions")}, + } + preparation_properties = { + **{key: {"type": "string", "default": ""} for key in (*_CHEMISTRY, "host")}, + "phases": {"type": ["array", "null"], "items": string, "default": None}, + "templates": {"type": "object", "additionalProperties": {"type": "string", "minLength": 1}, "default": {}}, + "parameter_file": nullable_string, + "phase_parameters": {"type": ["object", "null"], "additionalProperties": {"type": "object", "additionalProperties": False, "properties": phase_properties}, "default": None}, + "offline": {"type": "boolean", "default": False}, + "max_subsystems": {"type": "integer", "minimum": 1, "default": 64}, + "max_prototype_attempts": {"type": "integer", "minimum": 1, "default": 8}, + "preview_wavelength_A": {"type": "number", "exclusiveMinimum": 0, "default": 1.5406}, + } + adaptation_properties = { + **{key: {"type": "string", "default": ""} for key in (*_CHEMISTRY, "citation")}, + **{key: {"type": ["number", "null"], "default": None} for key in ("a", "b", "c", "alpha", "beta", "gamma")}, + "fract": phase_properties["fract"], + } + tensor_properties = {key: string for key in ("source_provider", "source_record_id", "source_url", "methodology_url", "nature_of_data", "coordinate_frame", "unit", "status")} + tensor_properties.update({ + "stiffness_GPa": {"type": "array", "minItems": 6, "maxItems": 6, "items": {"type": "array", "minItems": 6, "maxItems": 6, "items": number}}, + "warnings": {"type": "array", "items": string}, "raw_payload_path": nullable_string, + }) + properties = { + "schema": {"type": "string", "const": REQUEST_SCHEMA, "default": REQUEST_SCHEMA}, + "workflow": {"type": "string", "enum": list(WORKFLOWS)}, + "name": {"type": "string", "default": ""}, "composition": {"type": "string", "default": ""}, + "inputs": {"type": "array", "items": {"type": "string", "minLength": 1}, "default": []}, + "output_dir": nullable_string, "submission_id": nullable_string, "parent_run_id": nullable_string, + "experiment": {"type": "string", "enum": ["laboratory", "synchrotron"], "default": "laboratory"}, + "analysis": _settings_schema(AnalysisSettings), "discovery": _settings_schema(DiscoverySettings), + "preparation": {"type": "object", "additionalProperties": False, "properties": preparation_properties, "default": {}}, + "adaptation": {"type": "object", "additionalProperties": False, "properties": adaptation_properties, "default": {}}, + "exports": {"type": "object", "additionalProperties": False, "properties": { + "include_excel": {"type": "boolean", "default": True}, "template": {"type": "string", "enum": ["full", "peak_table"], "default": "full"}, + }, "default": dict(EXPORT_DEFAULTS)}, + "execution": {"type": "object", "additionalProperties": False, "properties": { + key: {"type": "boolean" if isinstance(default, bool) else "integer", "default": default, + **({} if isinstance(default, bool) else {"minimum": 1}), + **({"maximum": 3} if key == "max_attempts" else {})} for key, default in EXECUTION_DEFAULTS.items() + }, "default": dict(EXECUTION_DEFAULTS)}, + "elastic_overrides": {"type": "object", "additionalProperties": {"type": "object", "additionalProperties": False, "required": ["stiffness_GPa"], "properties": tensor_properties}, "default": {}}, + } + return {"$schema": "https://json-schema.org/draft/2020-12/schema", "title": REQUEST_SCHEMA, + "type": "object", "additionalProperties": False, "required": ["workflow"], "properties": properties} diff --git a/src/diffractscout/workflow/execution.py b/src/diffractscout/workflow/execution.py new file mode 100644 index 0000000..d9fdcea --- /dev/null +++ b/src/diffractscout/workflow/execution.py @@ -0,0 +1,182 @@ +"""Execution callbacks shared by the existing numerical pipeline.""" +from __future__ import annotations + +import json +import os +import shutil +from pathlib import Path +from typing import Any + +from ..models import DiagnosticRecord +from ..pipeline import _analyze_paths +from ..utils import sha256_file, to_jsonable, write_json +from .checkpoints import fingerprint, load_analysis, publish_checkpoint, save_analysis, verify_checkpoint + + +class WorkflowCancelled(BaseException): + """Bypass provider/per-phase Exception handlers at a safe boundary.""" + + +class WorkflowIntegrityError(BaseException): + """An invalid saved checkpoint must not be converted to an empty query.""" + + +class Execution: + def __init__(self, store, request, *, api_key: str | None = None): + self.store, self.root, self.request = store, store.root, request + self.max_attempts = request.execution["max_attempts"] + self.secrets = [key for key in (api_key, os.environ.get("MP_API_KEY")) if key] + self.extra_diagnostics: list[DiagnosticRecord] = [] + self.provider = None + self.analysis_refs: list[dict[str, str]] = [] + + def scrub(self, value: Any) -> Any: + if isinstance(value, str): + for key in self.secrets: + value = value.replace(key, "[redacted]") + return value + if isinstance(value, dict): + return {key: self.scrub(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [self.scrub(item) for item in value] + return value + + def diagnostic(self, code: str, error: Any, *, stage: str = "execution", item: str = "", + retryable: bool = False, missing_fields=None) -> dict[str, Any]: + return {"code": code, "message": self.scrub(str(error)), "stage": stage, "item": item, + "retryable": retryable, "missing_fields": missing_fields or []} + + def check_cancel(self) -> None: + if (self.root / "cancel.json").exists(): + raise WorkflowCancelled() + + def analyze_paths(self, paths_and_tensors, settings): + items = list(paths_and_tensors) + analyses, diagnostics = [], [] + self.store.stage("analysis", total=len(items)) + self.analysis_refs = [] + for index, (path, tensor) in enumerate(items): + self.check_cancel() + # The sidecar path contains a random staging directory. Its content + # and scientific metadata, not that temporary path, identify reuse. + tensor_data = to_jsonable(tensor) + if tensor_data: + tensor_data.pop("raw_payload_path", None) + key = fingerprint({"cif": sha256_file(path), "name": path.name, + "settings": settings, "tensor": tensor_data}) + checkpoint = self.root / "analyses" / key + if not checkpoint.exists() and self.request.parent_run_id: + from .store import job_path, read_json + parent = job_path(self.root.parent.parent, self.request.parent_run_id) + previous = parent / "analyses" / key + if previous.exists() and read_json(parent / "environment.json") == read_json(self.root / "environment.json"): + verify_checkpoint(previous) + publish_checkpoint(checkpoint, lambda directory: shutil.copytree(previous, directory, dirs_exist_ok=True)) + self.store.event("parent_checkpoint_reused", item=path.name, parent_run_id=self.request.parent_run_id) + if checkpoint.exists(): + analysis = load_analysis(checkpoint, cif_path=path) + analysis.elastic_tensor = tensor + self.store.count("reused") + item_diagnostics = [DiagnosticRecord("analysis", path.name, "warning", value) + for value in analysis.warnings] + results = [analysis] + else: + self.store.count("analyses") + results, item_diagnostics = _analyze_paths([(path, tensor)], settings) + if results: + publish_checkpoint(checkpoint, lambda directory: save_analysis(directory, results[0])) + if results: + self.analysis_refs.append({"checkpoint": str(checkpoint.relative_to(self.root)), "cif_name": path.name}) + analyses.extend(results) + diagnostics.extend(item_diagnostics) + self.store.update(completed_items=index+1, total_items=len(items)) + self.store.event("phase_completed", item=path.name, success=bool(results)) + return analyses, diagnostics + + def before_export(self, diagnostics) -> None: + self.check_cancel() + diagnostics.extend(self.extra_diagnostics) + if self.provider: + diagnostics.extend(DiagnosticRecord(row["stage"], row["item"], "error", row["message"]) + for row in self.provider.failures) + diagnostics[:] = [DiagnosticRecord(row.stage, self.scrub(row.item), row.level, self.scrub(row.message)) + for row in diagnostics] + self.store.stage("export") + + def before_verify(self, staging: Path) -> None: + self.check_cancel() + self.store.stage("verification") + if self.request.exports["template"] == "peak_table": + self._compact_workbook(staging) + self.decorate_bundle(staging, analysis=self.request.workflow != "discover") + + def decorate_bundle(self, staging: Path, *, analysis: bool) -> None: + """Add reproducibility records before the original atomic publication.""" + from .preparation import manifest + from .store import read_json + request = read_json(self.root / "request.json") + write_json(staging / "workflow_request.json", request) + write_json(staging / "workflow_environment.json", read_json(self.root / "environment.json")) + write_json(staging / "workflow_summary.json", { + "run_id": self.store.state["run_id"], "workflow": self.request.workflow, + "parameter_sources": read_json(self.root / "plan.json").get("parameter_sources", {}), + "counters": self.store.state["counters"], "scientific_scope": "theoretical_reference", + }) + sources = self.root / "sources" + if sources.exists(): + shutil.copytree(sources, staging / "workflow_sources") + replay = dict(request) + replay.update(output_dir=None, submission_id=None, parent_run_id=None) + if analysis: + replay.update(workflow="analyze", inputs=[f"inputs/{path.name}" for path in sorted((staging / "inputs").glob("*.cif"))], + composition="", preparation={}, adaptation={}, elastic_overrides={}) + for key, value in self.request.elastic_overrides.items(): + payload = dict(value) + if payload.get("raw_payload_path"): + payload["raw_payload_path"] = "workflow_sources/" + Path(payload["raw_payload_path"]).relative_to(sources).as_posix() + replay["elastic_overrides"][Path(key).name] = payload + elif sources.exists(): + mapping = read_json(sources / "source-map.json") + replay["inputs"] = ["workflow_sources/"+name for name in mapping["inputs"]] + replay["preparation"] = dict(replay["preparation"]) + if mapping["templates"]: + replay["preparation"]["templates"] = {phase: "workflow_sources/"+name for phase, name in mapping["templates"].items()} + if mapping["parameter_file"]: + replay["preparation"]["parameter_file"] = "workflow_sources/"+mapping["parameter_file"] + replay["execution"] = {**replay["execution"], "overwrite": False} + write_json(staging / "replay_request.json", replay) + manifest(staging) + + def _compact_workbook(self, staging: Path) -> None: + if not self.request.exports["include_excel"]: + return + import csv + from openpyxl import Workbook + from openpyxl.styles import Font, PatternFill + workbook = Workbook() + sheet = workbook.active + sheet.title = "Peak reference" + sheet.append(["Phase", "hkl", "2theta (deg)", "d (Angstrom)"]) + with (staging / "peak_reference.csv").open(encoding="utf-8-sig", newline="") as stream: + for row in csv.DictReader(stream): + phase = row["phase_name"] + if phase.startswith(("=", "+", "-", "@")): + phase = "'" + phase + sheet.append([phase, row.get("family_label") or f"({row['h']} {row['k']} {row['l']})", + float(row["two_theta_deg"]), float(row["d_spacing_A"])]) + sheet.freeze_panes = "A2" + sheet.auto_filter.ref = sheet.dimensions + for cell in sheet[1]: + cell.font = Font(bold=True) + cell.fill = PatternFill("solid", fgColor="E8E8E8") + for column, width in (("A", 28), ("B", 18), ("C", 20), ("D", 20)): + sheet.column_dimensions[column].width = width + for row in sheet.iter_rows(min_row=2, min_col=3, max_col=4): + for cell in row: + cell.number_format = "0.000000" + path = staging / "peak_table.xlsx" + workbook.save(path) + manifest = json.loads((staging / "manifest.json").read_text(encoding="utf-8")) + manifest["files"].append({"path": path.name, "sha256": sha256_file(path), + "size_bytes": path.stat().st_size, "role": "tabular_result"}) + write_json(staging / "manifest.json", manifest) diff --git a/src/diffractscout/workflow/preparation.py b/src/diffractscout/workflow/preparation.py new file mode 100644 index 0000000..ad18e1d --- /dev/null +++ b/src/diffractscout/workflow/preparation.py @@ -0,0 +1,117 @@ +"""Per-phase preparation checkpoints using the established CIF workflows.""" +from __future__ import annotations + +import shutil +from dataclasses import replace +from pathlib import Path + +from ..initial_cifs import InitialCifRecord, PrepareCifsResult, _write_csv, prepare_cifs, read_phase_parameters +from ..phase_cif import adapt_cif, canonicalize_phases, fetch_prototypes +from ..pipeline import _capture_target_state, _create_staging_output, _validate_output_target, _verify_and_commit +from ..utils import sha256_file, utc_now_iso, write_json +from .checkpoints import publish_checkpoint, verify_checkpoint +from .store import read_json + + +def manifest(directory: Path) -> None: + write_json(directory / "manifest.json", {"schema": "diffractscout_bundle_manifest_v1", + "generated_at_utc": utc_now_iso(), "files": [ + {"path": path.relative_to(directory).as_posix(), "sha256": sha256_file(path), "size_bytes": path.stat().st_size} + for path in sorted(directory.rglob("*")) if path.is_file() and path != directory / "manifest.json"]}) + + +def prepare(execution, target: Path, *, provider=None, api_key=None) -> PrepareCifsResult: + request = execution.request + options = dict(request.preparation) + phases = canonicalize_phases(options.pop("phases", None)) + parameters = options.pop("phase_parameters", None) or {} + parameter_file = options.pop("parameter_file", None) + if parameter_file: + parameters = read_phase_parameters(parameter_file) + templates = options.pop("templates", {}) + bundles, records = [], [] + execution.store.stage("preparation", total=len(phases)) + for index, phase in enumerate(phases): + execution.check_cancel() + checkpoint = execution.root / "prepared" / phase + if checkpoint.exists(): + verify_checkpoint(checkpoint) + execution.store.count("reused") + else: + def write(directory: Path) -> None: + result = prepare_cifs(request.composition, directory / "bundle", phases=[phase], + phase_parameters={phase: parameters[phase]} if phase in parameters else {}, + templates={phase: templates[phase]} if phase in templates else {}, + provider=provider, api_key=api_key, **options) + if result.exit_code != 0: + raise ValueError("; ".join(record.note for record in result.records)) + try: + execution.store.count("preparations") + publish_checkpoint(checkpoint, write) + except Exception as exc: + records.append({"phase": phase, "status": "failed", "note": execution.scrub(str(exc))}) + execution.store.event("preparation_failed", item=phase, message=execution.scrub(str(exc))) + continue + bundle = checkpoint / "bundle" + payload = read_json(bundle / "preparation.json") + for row in payload["records"]: + row = dict(row) + if row.get("cif"): + row["cif"] = f"phases/{phase}/{row['cif']}" + records.append(row) + bundles.append((phase, bundle, payload)) + execution.store.update(completed_items=index+1, total_items=len(phases)) + execution.check_cancel() + target.parent.mkdir(parents=True, exist_ok=True) + target = _validate_output_target(target, overwrite=request.execution["overwrite"]) + expected = _capture_target_state(target) + staging = _create_staging_output(target) + try: + for phase, bundle, _payload in bundles: + shutil.copytree(bundle, staging / "phases" / phase) + payload = dict(bundles[0][2]) if bundles else {"schema": "diffractscout_initial_cifs_v1"} + payload.update(composition=request.composition, records=records, + phase_bundle_paths=[f"phases/{phase}" for phase, _, _ in bundles]) + write_json(staging / "preparation.json", execution.scrub(payload)) + _write_csv(staging / "initial_cifs.csv", ("phase", "status", "cif", "space_group", "lattice_basis", "composition_role", "note"), records) + report = ["# Initial CIF preparation", "", request.composition, ""] + for row in records: + report.append(f"- {row['phase']}: {row['status']} — {row.get('note', '')}") + report.extend(["", "Each phases/ directory retains the complete source/provenance report.", + "These are starting models, not experimentally refined structures."]) + (staging / "report.md").write_text("\n".join(report)+"\n", encoding="utf-8") + manifest(staging) + execution.decorate_bundle(staging, analysis=False) + _verify_and_commit(target, staging, overwrite=request.execution["overwrite"], expected_state=expected) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + return PrepareCifsResult(target, target / "report.md", target / "initial_cifs.csv", target / "manifest.json", + tuple(InitialCifRecord(row["phase"], row["status"], target / row["cif"] if row.get("cif") else None, row.get("note", "")) for row in records)) + + +def explicit_derivation(execution, target: Path, *, provider=None, api_key=None): + request = execution.request + target.parent.mkdir(parents=True, exist_ok=True) + target = _validate_output_target(target, overwrite=request.execution["overwrite"]) + expected = _capture_target_state(target) + staging = _create_staging_output(target) + try: + execution.check_cancel() + execution.store.stage(request.workflow) + if request.workflow == "fetch": + options = {key: value for key, value in request.preparation.items() + if key in {"phases", "templates", "host", "max_subsystems"}} + result = fetch_prototypes(request.composition, staging / "prototypes", provider=provider, api_key=api_key, **options) + else: + result = adapt_cif(request.inputs[0], staging / "derived.cif", **request.adaptation) + manifest(staging) + execution.decorate_bundle(staging, analysis=False) + _verify_and_commit(target, staging, overwrite=request.execution["overwrite"], expected_state=expected) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + if request.workflow == "fetch": + return replace(result, output_dir=target, index_path=target / result.index_path.relative_to(staging), + records=tuple(replace(record, cif_path=target / record.cif_path.relative_to(staging) if record.cif_path else None) for record in result.records)) + return replace(result, cif_path=target / result.cif_path.relative_to(staging), sidecar_path=target / result.sidecar_path.relative_to(staging)) diff --git a/src/diffractscout/workflow/provider.py b/src/diffractscout/workflow/provider.py new file mode 100644 index 0000000..3732f88 --- /dev/null +++ b/src/diffractscout/workflow/provider.py @@ -0,0 +1,141 @@ +"""Checkpoint individual successful provider operations, including empty queries.""" +from __future__ import annotations + +import shutil +import time +from dataclasses import replace +from pathlib import Path +from typing import Any + +from ..models import CandidateRecord, DownloadArtifact +from ..network import transient_error +from ..pipeline import _normalize_download_artifacts, _reconcile_download_coverage +from ..utils import to_jsonable, write_json +from .checkpoints import fingerprint, publish_checkpoint, verify_checkpoint +from .store import read_json + + +def artifact_from_dict(row: dict[str, Any], root: Path) -> DownloadArtifact: + values = dict(row) + values["candidate"] = CandidateRecord(**values["candidate"]) + for key in ("cif_path", "elasticity_path"): + values[key] = root / values[key] if values.get(key) else None + return DownloadArtifact(**values) + + +class CheckpointProvider: + def __init__(self, provider: Any, execution: Any): + self.provider = provider + self.execution = execution + self.name = provider.name + self.failures: list[dict[str, Any]] = [] + + def metadata(self) -> dict[str, Any]: + return self.execution.scrub(self.provider.metadata()) + + def _query(self, operation: str, argument: Any, options: dict) -> list[CandidateRecord]: + execution = self.execution + execution.check_cancel() + execution.store.stage("discovery") + key = fingerprint([self.name, operation, argument, options]) + target = execution.root / "queries" / key + if target.exists(): + try: + verify_checkpoint(target) + except Exception as exc: + from .execution import WorkflowIntegrityError + raise WorkflowIntegrityError(str(exc)) from exc + execution.store.count("reused") + return [CandidateRecord(**row) for row in read_json(target / "records.json")] + for attempt in range(execution.max_attempts): + execution.check_cancel() + execution.store.count("queries") + try: + function = getattr(self.provider, operation) + rows = function(argument, **options) + rows = [replace(row, source_url=execution.scrub(row.source_url)) for row in rows] + publish_checkpoint(target, lambda directory: write_json(directory / "records.json", to_jsonable(rows))) + execution.store.event("query_completed", item=str(argument), count=len(rows), outcome="records" if rows else "empty") + return rows + except Exception as exc: + retryable = transient_error(exc) + if retryable and attempt + 1 < execution.max_attempts: + execution.store.event("retry", stage="discovery", item=str(argument), attempt=attempt+1) + time.sleep(0.25 * 2**attempt) + continue + failure = execution.diagnostic("QUERY_FAILED", exc, stage="discovery", item=str(argument), retryable=retryable) + self.failures.append(failure) + raise RuntimeError(failure["message"]) from None + return [] # all branches above return or raise + + def search_subsystem(self, chemsys: str, **kwargs: Any) -> list[CandidateRecord]: + return self._query("search_subsystem", chemsys, kwargs) + + def search_material_ids(self, material_ids, **kwargs: Any) -> list[CandidateRecord]: + return self._query("search_material_ids", list(material_ids), kwargs) + + def download_candidates(self, candidates, output_dir: Path, **kwargs: Any) -> list[DownloadArtifact]: + execution = self.execution + candidates = list(candidates) + results = [] + output_dir.mkdir(parents=True, exist_ok=True) + execution.store.stage("download", total=len(candidates)) + for index, candidate in enumerate(candidates): + execution.check_cancel() + key = fingerprint([to_jsonable(candidate), kwargs]) + target = execution.root / "downloads" / key + if target.exists(): + verify_checkpoint(target) + row = artifact_from_dict(read_json(target / "artifact.json"), target) + execution.store.count("reused") + else: + row = None + for attempt in range(execution.max_attempts): + execution.check_cancel() + temporary = execution.root / "downloads" / ("attempt-" + key) + # Only this worker-owned uncommitted attempt is disposable. + if temporary.exists(): + shutil.rmtree(temporary) + temporary.mkdir(parents=True) + execution.store.count("downloads") + try: + raw = self.provider.download_candidates([candidate], temporary, **kwargs) + raw, coverage = _reconcile_download_coverage([candidate], raw) + normalized, diagnostics = _normalize_download_artifacts( + raw, temporary, include_elasticity=kwargs.get("include_elasticity", True)) + for issue in [*coverage, *diagnostics]: + execution.extra_diagnostics.append(issue) + if not normalized: + raise RuntimeError("Provider returned no artifact for requested material.") + row = normalized[0] + row = replace(row, error=execution.scrub(row.error), elasticity_error=execution.scrub(row.elasticity_error), + provider_metadata=execution.scrub(row.provider_metadata)) + retryable = bool(row.provider_metadata.get("retryable", False)) + except Exception as exc: + retryable = transient_error(exc) + row = DownloadArtifact(candidate, None, status="failed", error=execution.scrub(str(exc)), + provider_metadata={"retryable": retryable}) + if row.status == "ok" or not retryable or attempt + 1 == execution.max_attempts: + break + execution.store.event("retry", stage="download", item=candidate.material_id, attempt=attempt+1) + time.sleep(0.25 * 2**attempt) + if row is None: + raise RuntimeError("Download attempts produced no result.") + if row.status == "ok": + def save(directory: Path) -> None: + # Keep all normalized metadata sidecars; their contents + # and every referenced source are verified on reuse. + shutil.copytree(temporary, directory, dirs_exist_ok=True) + payload = execution.scrub(to_jsonable(row)) + for name in ("cif_path", "elasticity_path"): + value = getattr(row, name) + payload[name] = value.relative_to(temporary).as_posix() if value else None + write_json(directory / "artifact.json", payload) + publish_checkpoint(target, save) + row = artifact_from_dict(read_json(target / "artifact.json"), target) + else: + self.failures.append(execution.diagnostic("DOWNLOAD_FAILED", row.error, stage="download", + item=candidate.material_id, retryable=retryable)) + results.append(row) + execution.store.update(completed_items=index+1, total_items=len(candidates)) + return results diff --git a/src/diffractscout/workflow/records.py b/src/diffractscout/workflow/records.py new file mode 100644 index 0000000..18632db --- /dev/null +++ b/src/diffractscout/workflow/records.py @@ -0,0 +1,38 @@ +"""Public machine-readable response types; runtime values are plain JSON objects.""" +from typing import Any, TypedDict + + +class ArtifactRef(TypedDict, total=False): + kind: str + path: str + sha256: str + + +class WorkflowPlan(TypedDict): + ready: bool + request: dict[str, Any] + stages: list[str] + diagnostics: list[dict[str, Any]] + estimated_queries: int + required_dependencies: list[str] + parameter_sources: dict[str, Any] + + +class RunStatus(TypedDict, total=False): + schema: str + run_id: str + name: str + workflow: str + state: str + stage: str + completed_items: int + total_items: int + diagnostics: list[dict[str, Any]] + artifacts: list[ArtifactRef] + integrity: str + coverage: dict[str, int] + counters: dict[str, int] + elapsed_seconds: float + output_dir: str + workspace: str + active: bool diff --git a/src/diffractscout/workflow/results.py b/src/diffractscout/workflow/results.py new file mode 100644 index 0000000..e4d3c63 --- /dev/null +++ b/src/diffractscout/workflow/results.py @@ -0,0 +1,112 @@ +"""Verified result loading and bounded, machine-readable table inspection.""" +from __future__ import annotations + +import csv +from pathlib import Path +from typing import Any + +from ..initial_cifs import InitialCifRecord, PrepareCifsResult +from ..models import CandidateRecord, DiagnosticRecord, DiscoveryResult, DiscoverySettings, ParsedComposition, PipelineResult +from ..phase_cif import AdaptResult, FetchPrototypesResult, PrototypeRecord +from ..utils import sha256_file, to_jsonable, write_json +from ..validation import verify_bundle +from .checkpoints import load_analysis, publish_checkpoint, verify_checkpoint +from .provider import artifact_from_dict +from .store import job_path, read_json + + +def result_record(root: Path, result: Any, execution) -> Path: + attempt = execution.store.state["attempt"] + destination = root / "results" / str(attempt) + if isinstance(result, PipelineResult): + data = {"kind": "pipeline", "output_dir": str(result.output_dir), + "discovery": to_jsonable(result.discovery), "downloads": to_jsonable(result.downloads), + "diagnostics": to_jsonable(result.diagnostics), "warnings": result.warnings, + "analyses": execution.analysis_refs} + else: + data = {"kind": "prepare" if isinstance(result, PrepareCifsResult) else + "fetch" if isinstance(result, FetchPrototypesResult) else "adapt", + "result": to_jsonable(result), "output_dir": str(getattr(result, "output_dir", result.cif_path.parent if isinstance(result, AdaptResult) else ""))} + output = Path(data["output_dir"]) + data["manifest_sha256"] = sha256_file(output / "manifest.json") + publish_checkpoint(destination, lambda directory: write_json(directory / "result.json", execution.scrub(data))) + return destination + + +def _record(root: Path, verify: bool = True) -> tuple[dict, Path]: + status = read_json(root / "state.json") + path = root / status.get("result_record", "results/missing") + verify_checkpoint(path) + data = read_json(path / "result.json") + output = Path(data["output_dir"]) + if verify: + report = verify_bundle(output) + if not report["ok"] or sha256_file(output / "manifest.json") != data["manifest_sha256"]: + raise ValueError("Result bundle integrity failed: " + "; ".join(report["errors"])) + return data, output + + +def load_result_object(run_id: str, *, workspace=None): + root = job_path(workspace, run_id) + data, output = _record(root) + kind = data["kind"] + if kind == "pipeline": + analyses = [load_analysis(root / entry["checkpoint"], cif_path=output / "inputs" / entry["cif_name"]) + for entry in data["analyses"]] + discovery = data["discovery"] + if discovery: + parsed = discovery["parsed"] + for key in ("elements", "labels", "material_ids", "notes"): + parsed[key] = tuple(parsed[key]) + discovery = DiscoveryResult(parsed=ParsedComposition(**parsed), settings=DiscoverySettings(**discovery["settings"]), + candidates=[CandidateRecord(**row) for row in discovery["candidates"]], + subsystems=discovery["subsystems"], subsystem_counts=discovery["subsystem_counts"], + provider_metadata=discovery["provider_metadata"], warnings=discovery["warnings"]) + return PipelineResult(output, discovery, [artifact_from_dict(row, output) for row in data["downloads"]], + analyses, output / "manifest.json", data["warnings"], + [DiagnosticRecord(**row) for row in data["diagnostics"]]) + values = data["result"] + if kind == "prepare": + return PrepareCifsResult(output, Path(values["report_path"]), Path(values["index_path"]), Path(values["manifest_path"]), + tuple(InitialCifRecord(row["phase"], row["status"], Path(row["cif_path"]) if row["cif_path"] else None, row["note"]) + for row in values["records"])) + if kind == "fetch": + records = [] + for row in values["records"]: + row["cif_path"] = Path(row["cif_path"]) if row["cif_path"] else None + records.append(PrototypeRecord(**row)) + return FetchPrototypesResult(output, Path(values["index_path"]), values["host"], tuple(values["elements"]), tuple(records)) + values["cif_path"], values["sidecar_path"] = Path(values["cif_path"]), Path(values["sidecar_path"]) + values["warnings"] = tuple(values["warnings"]) + return AdaptResult(**values) + + +def get_result(run_id: str, *, workspace=None, table=None, offset=0, limit=100, verify=True) -> dict: + if type(offset) is not int or offset < 0 or type(limit) is not int or not 1 <= limit <= 1000: + raise ValueError("offset must be nonnegative; limit must be between 1 and 1000.") + root = job_path(workspace, run_id) + data, output = _record(root, verify=verify) + status = read_json(root / "state.json") + response = {"run_id": run_id, "state": status["state"], "integrity": "passed" if verify else "not_checked", + "coverage": status.get("coverage", {}), "diagnostics": status.get("diagnostics", []), + "needs_review": status.get("needs_review", False), + "artifacts": status.get("artifacts", []), "output_dir": str(output), "kind": data["kind"]} + if table is None: + response["tables"] = sorted(path.stem for path in output.glob("*.csv")) + return response + tables = {path.stem: path for path in output.glob("*.csv")} + if table not in tables: + raise ValueError(f"Unknown table {table!r}; choose one of {', '.join(sorted(tables))}.") + rows = [] + with tables[table].open(encoding="utf-8-sig", newline="") as stream: + reader = csv.DictReader(stream) + fields = reader.fieldnames + for index, row in enumerate(reader): + if index < offset: + continue + rows.append(row) + if len(rows) > limit: + break + response.update(table=table, columns=fields, rows=rows[:limit], offset=offset, + next_offset=offset+min(len(rows), limit), has_more=len(rows) > limit) + return response diff --git a/src/diffractscout/workflow/runner.py b/src/diffractscout/workflow/runner.py new file mode 100644 index 0000000..6cd1e52 --- /dev/null +++ b/src/diffractscout/workflow/runner.py @@ -0,0 +1,197 @@ +"""One worker, one task; all numerical work goes through existing workflows.""" +from __future__ import annotations + +import os +import time +from pathlib import Path + +from ..models import AnalysisSettings, DiscoverySettings, PipelineResult +from ..pipeline import analyze_cifs, export_discovery, run_pipeline +from ..utils import sha256_file +from ..validation import verify_bundle +from .checkpoints import environment, tensor_from_dict +from .contracts import WorkflowRequest, plan_workflow +from .execution import Execution, WorkflowCancelled, WorkflowIntegrityError +from .preparation import explicit_derivation, prepare +from .provider import CheckpointProvider +from .results import result_record +from .snapshots import snapshot_request +from .store import RunStore, file_lock, read_json + + +def _owned_published_output(target: Path, root: Path, request_hash: str) -> bool: + """Recognize publication whose acknowledgement was interrupted. + + Preserve the verified original and finish in a new revision using saved + calculations. Never infer ownership merely from a directory's name. + """ + try: + from .checkpoints import fingerprint + return (verify_bundle(target)["ok"] + and read_json(target / "workflow_summary.json").get("run_id") == root.name + and fingerprint(read_json(target / "workflow_request.json")) == request_hash) + except (ValueError, OSError, KeyError): + return False + + +def execute(root: Path, *, provider=None, api_key=None, expected_attempt=None) -> dict: + try: + with file_lock(root / "worker.lock"): + if expected_attempt is not None and read_json(root / "state.json")["attempt"] != expected_attempt: + return read_json(root / "state.json") + return _execute_locked(root, provider=provider, api_key=api_key) + except BlockingIOError: + return read_json(root / "state.json") + + +def _execute_locked(root: Path, *, provider=None, api_key=None) -> dict: + store = RunStore(root) + request = WorkflowRequest.from_dict(read_json(root / "request.json")) + execution = Execution(store, request, api_key=api_key) + if provider is not None and isinstance(getattr(provider, "api_key", None), str): + execution.secrets.append(provider.api_key) + started = time.monotonic() + previous_elapsed = store.state.get("elapsed_seconds", 0) + store.update(state="running", worker_pid=os.getpid(), attempt=store.state.get("attempt", 0)+1, + diagnostics=[], integrity="not_checked") + store.event("started", attempt=store.state["attempt"]) + try: + execution.check_cancel() + from .checkpoints import fingerprint + if fingerprint(request.to_dict()) != store.state["request_hash"]: + raise ValueError("Saved request changed; submit a new task instead of editing a task record.") + saved_environment = read_json(root / "environment.json") + if saved_environment != environment(): + raise ValueError("Software or dependency versions changed; submit a new task. Existing checkpoints were preserved.") + store.stage("snapshot") + request = snapshot_request(root, request) + execution.request = request + target = Path(request.output_dir) if request.output_dir else root / "bundle" + if target.exists() and (store.state.get("result_record") or _owned_published_output(target, root, store.state["request_hash"])): + if not store.state.get("result_record"): + store.event("publication_recovered", preserved_bundle=str(target)) + store.update(recovered_bundles=[*store.state.get("recovered_bundles", []), str(target)]) + target = target.with_name(f"{target.name}-retry-{store.state['attempt']}") + request.output_dir = str(target) + plan = plan_workflow(request, provider=provider, api_key=api_key) + if not plan["ready"]: + store.update(state="blocked", stage="planning", diagnostics=plan["diagnostics"]) + return store.state + needs_provider = request.workflow in {"run", "discover"} or ( + request.workflow == "fetch" and ("mp-api" in plan["required_dependencies"] or bool(api_key or os.environ.get("MP_API_KEY")))) or ( + request.workflow in {"prepare", "prepare_analyze"} and not request.preparation.get("offline", False) + and bool(api_key or os.environ.get("MP_API_KEY") or provider)) + if needs_provider and provider is None: + from ..providers.materials_project import MaterialsProjectProvider + provider = MaterialsProjectProvider(api_key or os.environ.get("MP_API_KEY", "")) + if provider is not None: + provider = CheckpointProvider(provider, execution) + execution.provider = provider + target.parent.mkdir(parents=True, exist_ok=True) + store.update(output_dir=str(target)) + settings = AnalysisSettings(**request.analysis) + options = request.execution + common = {"include_excel": request.exports["include_excel"], "overwrite": options["overwrite"], "_execution": execution} + if request.workflow == "analyze": + result = analyze_cifs(request.inputs, target, settings=settings, recursive=options["recursive"], + elastic_overrides={key: tensor_from_dict(value) for key, value in request.elastic_overrides.items()}, **common) + elif request.workflow in {"run", "discover"}: + discovery = DiscoverySettings(**request.discovery) + if request.workflow == "discover": + result = export_discovery(request.composition, provider, target, discovery_settings=discovery, **common) + else: + result = run_pipeline(request.composition, provider, target, discovery_settings=discovery, + analysis_settings=settings, conventional_unit_cell=options["conventional_unit_cell"], + confirm_above=options["confirm_above"], authorize_large_download=options["authorize_large_download"], **common) + elif request.workflow in {"prepare", "prepare_analyze"}: + prepared_target = target if request.workflow == "prepare" else root / f"preparation-{store.state['attempt']}" + prepared = prepare(execution, prepared_target, provider=provider, api_key=api_key) + if request.workflow == "prepare": + result = prepared + else: + paths = [str(row.cif_path) for row in prepared.records if row.cif_path] + if not paths: + result = prepared + else: + from ..models import DiagnosticRecord + execution.extra_diagnostics.extend(DiagnosticRecord("preparation", row.phase, "error", row.note) + for row in prepared.records if not row.cif_path) + result = analyze_cifs(paths, target, settings=settings, **common) + store.update(preparation_bundle=str(prepared_target)) + else: + result = explicit_derivation(execution, target, provider=provider, api_key=api_key) + output = Path(getattr(result, "output_dir", result.cif_path.parent if request.workflow == "adapt" else target)) + report = verify_bundle(output) + if not report["ok"]: + raise ValueError("Published result failed verification: " + "; ".join(report["errors"])) + saved_result = result_record(root, result, execution) + diagnostics = [] + if isinstance(result, PipelineResult): + diagnostics = [execution.diagnostic("ITEM_FAILED" if row.level == "error" else "SCIENTIFIC_WARNING", row.message, + stage=row.stage, item=row.item) | {"level": row.level} for row in result.diagnostics] + successes = len(result.analyses) + total = len(result.downloads) if result.discovery is not None and request.workflow != "discover" else len(request.inputs) + if request.workflow == "prepare_analyze": + total = len(prepared.records) + if request.workflow == "discover": + successes = len(result.discovery.candidates) + total = successes + errors = sum(row.level == "error" for row in result.diagnostics) + if request.workflow == "discover": + state = "partial" if errors and successes else "failed" if errors else "completed" + else: + state = "failed" if not successes else "partial" if errors else "completed" + empty_phases = [analysis.phase_name for analysis in result.analyses if not analysis.reflections] + for phase in empty_phases: + diagnostics.append(execution.diagnostic("EMPTY_REFLECTION_TABLE", "No reflections in the requested window; the reference table is empty.", + stage="analysis", item=phase) | {"level": "warning"}) + if empty_phases and state == "completed": + state = "partial" + if not successes and request.workflow != "discover": + diagnostics.append(execution.diagnostic("NO_ANALYZABLE_PHASES", "No phase could be analyzed; inspect source and provider diagnostics.")) + else: + rows = getattr(result, "records", None) + successes = sum(row.cif_path is not None for row in rows) if rows is not None else 1 + total = len(rows) if rows is not None else 1 + state = "completed" if successes == total else "partial" if successes else "failed" + if rows: + diagnostics = [execution.diagnostic("PREPARATION_FAILED" if not row.cif_path else "INITIAL_MODEL", row.note, + stage="preparation", item=row.phase) for row in rows if row.note] + if provider is not None: + diagnostics.extend(provider.failures) + if provider.failures and state == "completed": + state = "partial" if successes else "failed" + artifacts = [{"kind": "bundle", "path": str(output)}, + {"kind": "manifest", "path": str(output / "manifest.json"), "sha256": sha256_file(output / "manifest.json")}] + artifacts.extend({"kind": "workbook", "path": str(path)} for path in sorted(output.glob("*.xlsx"))) + if store.state.get("preparation_bundle"): + artifacts.append({"kind": "preparation", "path": store.state["preparation_bundle"]}) + store.update(state=state, stage="done", result_record=str(saved_result.relative_to(root)), + completed_items=successes, total_items=total, + artifacts=artifacts, integrity="passed", coverage={"requested": total, "successful": successes, "failed": max(0, total-successes)}, + diagnostics=diagnostics, output_dir=str(output), needs_review=bool(diagnostics)) + store.event("finished", state=state, integrity="passed") + except WorkflowCancelled: + store.update(state="cancelled", stage=store.state["stage"]) + store.event("cancelled") + except KeyboardInterrupt: + store.update(state="interrupted") + store.event("interrupted") + except WorkflowIntegrityError as exc: + store.update(state="blocked", diagnostics=[execution.diagnostic("CHECKPOINT_INVALID", exc, stage=store.state["stage"])]) + store.event("failed", code="CHECKPOINT_INVALID", message=execution.scrub(str(exc))) + except Exception as exc: + code = "OUTPUT_CONFLICT" if isinstance(exc, FileExistsError) else "WORKFLOW_FAILED" + if "Checkpoint" in str(exc) or "checkpoint" in str(exc): + code = "CHECKPOINT_INVALID" + elif "versions changed" in str(exc): + code = "ENVIRONMENT_CHANGED" + diagnostics = [execution.diagnostic(code, exc, stage=store.state["stage"])] + if execution.provider: + diagnostics.extend(execution.provider.failures) + store.update(state="blocked" if code in {"OUTPUT_CONFLICT", "ENVIRONMENT_CHANGED", "CHECKPOINT_INVALID"} else "failed", + diagnostics=diagnostics) + store.event("failed", code=code, message=execution.scrub(str(exc))) + finally: + store.update(elapsed_seconds=previous_elapsed + time.monotonic()-started) + return store.state diff --git a/src/diffractscout/workflow/service.py b/src/diffractscout/workflow/service.py new file mode 100644 index 0000000..16b1860 --- /dev/null +++ b/src/diffractscout/workflow/service.py @@ -0,0 +1,201 @@ +"""Public workflow API used unchanged by CLI, desktop and MCP clients.""" +from __future__ import annotations + +import os +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from uuid import NAMESPACE_URL, uuid4, uuid5 + +from ..pipeline import _validate_input_output_separation +from ..utils import utc_now_iso, write_json +from .checkpoints import environment, fingerprint +from .contracts import WorkflowRequest, plan_workflow +from .results import get_result +from .snapshots import snapshot_request +from .store import TERMINAL, events_page, file_lock, is_locked, job_path, process_alive, read_json, workspace_path + + +def _create(request, *, workspace=None, provider=None, api_key=None): + provided = getattr(request, "_provided_fields", None) + request = WorkflowRequest.from_dict(request.to_dict() if isinstance(request, WorkflowRequest) else request) + if provided is not None: + request._provided_fields = provided + workspace = workspace_path(workspace) + run_id = uuid5(NAMESPACE_URL, request.submission_id).hex if request.submission_id else uuid4().hex + root = job_path(workspace, run_id) + _validate_input_output_separation(request.inputs, root) + digest = fingerprint(request.to_dict()) + workspace.mkdir(parents=True, exist_ok=True) + with file_lock(workspace / "submissions.lock", blocking=True): + if root.exists(): + state = read_json(root / "state.json") + if state["request_hash"] != digest: + raise ValueError("submission_id already identifies a different request; use a new submission_id.") + return root, False + plan = plan_workflow(request, provider=provider, api_key=api_key) + root.mkdir(parents=True) + write_json(root / "request.json", request.to_dict()) + write_json(root / "plan.json", plan) + write_json(root / "environment.json", environment()) + now = utc_now_iso() + state = {"schema": "diffractscout_run_v1", "run_id": run_id, "name": request.name or request.workflow, + "workflow": request.workflow, "workspace": str(workspace), "request_hash": digest, + "state": "pending" if plan["ready"] else "blocked", "stage": "planning", "attempt": 0, + "created_at": now, "updated_at": now, "completed_items": 0, "total_items": 0, + "diagnostics": plan["diagnostics"], "artifacts": [], "integrity": "not_checked", "coverage": {}, + "counters": {"queries": 0, "downloads": 0, "analyses": 0, "preparations": 0, "reused": 0}, + "elapsed_seconds": 0.0, "output_dir": request.output_dir or str(root / "bundle")} + write_json(root / "state.json", state) + if plan["ready"]: + try: + snapshot_request(root, request) + except Exception as exc: + state.update(state="blocked", diagnostics=[{"code": "SNAPSHOT_FAILED", "message": str(exc), + "stage": "snapshot", "item": "", "retryable": False, "missing_fields": []}]) + write_json(root / "state.json", state) + return root, True + + +def _spawn(arguments: list[str], *, api_key=None) -> subprocess.Popen: + env = os.environ.copy() + source = str(Path(__file__).resolve().parents[2]) + env["PYTHONPATH"] = source + os.pathsep + env.get("PYTHONPATH", "") + env["PYTHONUTF8"] = "1" + if api_key: + env["MP_API_KEY"] = api_key + kwargs = {"env": env, "stdin": subprocess.DEVNULL, "stdout": subprocess.DEVNULL, "stderr": subprocess.DEVNULL} + if os.name == "nt": + kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW | subprocess.CREATE_NEW_PROCESS_GROUP + else: + kwargs["start_new_session"] = True + if getattr(sys, "frozen", False): + env["PYINSTALLER_RESET_ENVIRONMENT"] = "1" + command = [sys.executable, "_agent-worker", *arguments] + else: + command = [sys.executable, "-m", "diffractscout.workflow.worker", *arguments] + return subprocess.Popen(command, **kwargs) + + +def _launch(root: Path, *, api_key=None) -> None: + attempt = read_json(root / "state.json")["attempt"] + process = _spawn(["--workspace", str(root.parent.parent), "--run-id", root.name, "--expected-attempt", str(attempt)], api_key=api_key) + write_json(root / "launch.json", {"pid": process.pid, "created_at": utc_now_iso()}) + + +def run_job(request, *, workspace=None, provider=None, api_key=None) -> dict: + root, created = _create(request, workspace=workspace, provider=provider, api_key=api_key) + if not created or read_json(root / "state.json")["state"] == "blocked": + return status_job(root.name, workspace=root.parent.parent) + from .runner import execute + return execute(root, provider=provider, api_key=api_key, expected_attempt=0) + + +def submit_job(request, *, workspace=None, api_key=None) -> dict: + root, created = _create(request, workspace=workspace, api_key=api_key) + if created and read_json(root / "state.json")["state"] == "pending": + _launch(root, api_key=api_key) + return status_job(root.name, workspace=root.parent.parent) + + +def status_job(run_id: str, *, workspace=None) -> dict: + root = job_path(workspace, run_id) + state = read_json(root / "state.json") + active = is_locked(root / "worker.lock") + if state["state"] in {"running", "cancelling"} and not active: + state["state"] = "interrupted" + elif state["state"] == "pending" and (root / "launch.json").exists(): + if not process_alive(read_json(root / "launch.json")["pid"]): + state["state"] = "interrupted" + if (root / "cancel.json").exists() and state["state"] not in {"completed", "partial", "failed"}: + state["state"] = "cancelling" if active else "cancelled" + state["active"] = active + return state + + +def list_jobs(*, workspace=None) -> list[dict]: + base = workspace_path(workspace) + if not (base / "jobs").exists(): + return [] + rows = [status_job(root.name, workspace=base) for root in (base / "jobs").iterdir() + if root.is_dir() and (root / "state.json").exists()] + return sorted(rows, key=lambda row: (row["created_at"], row["run_id"]), reverse=True) + + +def read_events(run_id: str, *, workspace=None, after=0, limit=100) -> dict: + root = job_path(workspace, run_id) + if not (root / "state.json").is_file(): + raise FileNotFoundError(f"Task not found: {run_id}") + return {"run_id": run_id, **events_page(root, after=after, limit=limit)} + + +def cancel_job(run_id: str, *, workspace=None) -> dict: + root = job_path(workspace, run_id) + state = status_job(run_id, workspace=workspace) + if state["state"] not in TERMINAL or state["state"] in {"blocked", "interrupted"}: + write_json(root / "cancel.json", {"requested_at": utc_now_iso()}) + return status_job(run_id, workspace=workspace) + + +def resume_job(run_id: str, *, workspace=None, background=True, provider=None, api_key=None) -> dict: + root = job_path(workspace, run_id) + with file_lock(root / "control.lock", blocking=True): + state = status_job(run_id, workspace=workspace) + if state["active"]: + return state + if state["state"] == "completed": + get_result(run_id, workspace=workspace) + return state + if background and provider is not None: + raise ValueError("Injected Python providers require background=False.") + launch = root / "launch.json" + if launch.exists() and process_alive(read_json(launch)["pid"]): + return state + (root / "cancel.json").unlink(missing_ok=True) + if background: + _launch(root, api_key=api_key) + else: + from .runner import execute + return execute(root, provider=provider, api_key=api_key, expected_attempt=state["attempt"]) + return status_job(run_id, workspace=workspace) + + +def run_batch(requests, *, workspace=None, max_workers=1, background=False) -> dict: + if not isinstance(requests, (list, tuple)) or not requests: + raise ValueError("A batch must contain at least one request.") + if type(max_workers) is not int or max_workers < 1: + raise ValueError("max_workers must be a positive integer.") + max_workers = min(max_workers, len(requests)) + base = workspace_path(workspace) + if background: + roots, entries = [], [] + for request in requests: + try: + root = _create(request, workspace=base)[0] + roots.append(root) + entries.append(root) + except (ValueError, OSError, RuntimeError) as exc: + entries.append({"state": "failed", "diagnostics": [{"code": "INVALID_REQUEST", "message": str(exc)}]}) + if not roots: + return {"state": "failed", "jobs": entries} + # Repeated submission IDs are references to one task, not invitations + # for the batch scheduler to execute the same task a second time. + roots = list(dict.fromkeys(roots)) + batch_id = uuid4().hex + batch_path = base / "batches" / (batch_id + ".json") + write_json(batch_path, {"run_ids": [root.name for root in roots], "max_workers": max_workers}) + process = _spawn(["--workspace", str(base), "--batch-id", batch_id]) + for root in roots: + write_json(root / "launch.json", {"pid": process.pid, "created_at": utc_now_iso()}) + return {"batch_id": batch_id, "state": "pending", "jobs": [status_job(entry.name, workspace=base) if isinstance(entry, Path) else entry for entry in entries]} + def run(request): + try: + return run_job(request, workspace=base) + except (ValueError, OSError, RuntimeError) as exc: + return {"state": "failed", "diagnostics": [{"code": "INVALID_REQUEST", "message": str(exc)}]} + with ThreadPoolExecutor(max_workers=max_workers) as pool: + jobs = list(pool.map(run, requests)) + successes = sum(row["state"] == "completed" for row in jobs) + state = "completed" if successes == len(jobs) else "partial" if any(row["state"] in {"completed", "partial"} for row in jobs) else "failed" + return {"state": state, "jobs": jobs, "completed": successes, "total": len(jobs)} diff --git a/src/diffractscout/workflow/snapshots.py b/src/diffractscout/workflow/snapshots.py new file mode 100644 index 0000000..22e123e --- /dev/null +++ b/src/diffractscout/workflow/snapshots.py @@ -0,0 +1,59 @@ +"""Freeze local sources once, before a resumable workflow begins.""" +from __future__ import annotations + +from pathlib import Path + +from ..pipeline import _copy_local_input, _copy_local_snapshot, _lookup_elastic_override, collect_cif_paths +from ..utils import sha256_file, to_jsonable, write_json +from .checkpoints import publish_checkpoint, tensor_from_dict, verify_checkpoint +from .contracts import WorkflowRequest +from .store import read_json + + +def snapshot_request(root: Path, request: WorkflowRequest) -> WorkflowRequest: + target = root / "sources" + if not target.exists(): + def write(directory: Path) -> None: + mapping = {"inputs": [], "templates": {}, "parameter_file": None, "elastic_overrides": {}} + inputs_dir = directory / "inputs" + inputs_dir.mkdir() + overrides = {key: tensor_from_dict(value) for key, value in request.elastic_overrides.items()} + for source in collect_cif_paths(request.inputs, recursive=request.execution["recursive"]) if request.inputs else []: + copied, tensor = _copy_local_input( + source, inputs_dir, include_elasticity=request.analysis["include_elasticity"], + elastic_override=_lookup_elastic_override(source, overrides) if overrides else None) + relative = copied.relative_to(directory).as_posix() + mapping["inputs"].append(relative) + if tensor is not None and overrides: + payload = to_jsonable(tensor) + if tensor.raw_payload_path: + payload["raw_payload_path"] = tensor.raw_payload_path.relative_to(directory).as_posix() + mapping["elastic_overrides"][relative] = payload + template_dir = directory / "templates" + template_dir.mkdir() + for phase, value in request.preparation.get("templates", {}).items(): + source = Path(value) + copied, _ = _copy_local_input(source, template_dir, include_elasticity=False) + mapping["templates"][phase] = copied.relative_to(directory).as_posix() + if request.preparation.get("parameter_file"): + source = Path(request.preparation["parameter_file"]) + copied = directory / "phase_parameters.json" + _copy_local_snapshot(source, copied, expected_digest=sha256_file(source), expected_size=source.stat().st_size) + mapping["parameter_file"] = copied.name + write_json(directory / "source-map.json", mapping) + publish_checkpoint(target, write) + verify_checkpoint(target) + mapping = read_json(target / "source-map.json") + payload = request.to_dict() + payload["inputs"] = [str(target / name) for name in mapping["inputs"]] + if mapping["templates"]: + payload["preparation"]["templates"] = {phase: str(target / name) for phase, name in mapping["templates"].items()} + if mapping["parameter_file"]: + payload["preparation"]["parameter_file"] = str(target / mapping["parameter_file"]) + payload["elastic_overrides"] = {} + for relative, tensor in mapping["elastic_overrides"].items(): + tensor = dict(tensor) + if tensor.get("raw_payload_path"): + tensor["raw_payload_path"] = str(target / tensor["raw_payload_path"]) + payload["elastic_overrides"][str(target / relative)] = tensor + return WorkflowRequest.from_dict(payload) diff --git a/src/diffractscout/workflow/store.py b/src/diffractscout/workflow/store.py new file mode 100644 index 0000000..aa8ad67 --- /dev/null +++ b/src/diffractscout/workflow/store.py @@ -0,0 +1,169 @@ +"""Portable task records and operating-system owned, crash-released locks.""" +from __future__ import annotations + +import contextlib +import json +import os +import re +from pathlib import Path +from typing import Any, Iterator + +from ..pipeline import _reject_reparse_components +from ..utils import utc_now_iso, write_json + +TERMINAL = {"completed", "partial", "failed", "blocked", "cancelled", "interrupted"} + + +def process_alive(pid: int) -> bool: + """Distinguish an exited worker from a retained Windows process handle.""" + if pid <= 0: + return False + if os.name == "nt": + import ctypes + from ctypes import wintypes + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + kernel.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + kernel.OpenProcess.restype = wintypes.HANDLE + kernel.WaitForSingleObject.argtypes = [wintypes.HANDLE, wintypes.DWORD] + kernel.WaitForSingleObject.restype = wintypes.DWORD + kernel.CloseHandle.argtypes = [wintypes.HANDLE] + kernel.CloseHandle.restype = wintypes.BOOL + handle = kernel.OpenProcess(0x00100000, False, pid) # SYNCHRONIZE + if not handle: + return ctypes.get_last_error() != 87 # ERROR_INVALID_PARAMETER + try: + return kernel.WaitForSingleObject(handle, 0) != 0 # WAIT_OBJECT_0 means exited + finally: + kernel.CloseHandle(handle) + # Linux zombie processes still answer kill(pid, 0), but cannot own a task. + stat_file = Path(f"/proc/{pid}/stat") + try: + if stat_file.exists() and stat_file.read_text().rsplit(")", 1)[1].split()[0] == "Z": + return False + except (OSError, IndexError): + pass + from ..pipeline import _process_is_alive + return _process_is_alive(pid) + + +def workspace_path(workspace: str | Path | None = None) -> Path: + path = Path(workspace or Path.cwd() / "outputs" / "agent-workspace").expanduser().absolute() + _reject_reparse_components(path, label="workflow workspace") + return path + + +def job_path(workspace: str | Path | None, run_id: str) -> Path: + if not re.fullmatch(r"[a-f0-9]{32}", run_id): + raise ValueError("Invalid run_id: expected a 32-character hexadecimal task identifier.") + path = workspace_path(workspace) / "jobs" / run_id + _reject_reparse_components(path, label="workflow task") + return path + + +def read_json(path: Path) -> Any: + _reject_reparse_components(path, label="workflow record") + return json.loads(path.read_text(encoding="utf-8")) + + +@contextlib.contextmanager +def file_lock(path: Path, *, blocking: bool = False) -> Iterator[None]: + """An open file owns the lock; no PID guessing or lock-file deletion.""" + _reject_reparse_components(path, label="workflow lock") + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a+b") as handle: + handle.seek(0, os.SEEK_END) + if not handle.tell(): + handle.write(b"\0") + handle.flush() + handle.seek(0) + try: + if os.name == "nt": + import msvcrt + msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK if blocking else msvcrt.LK_NBLCK, 1) + else: + import fcntl + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | (0 if blocking else fcntl.LOCK_NB)) + except OSError as exc: + raise BlockingIOError(f"Task is already owned by another worker: {path.parent.name}") from exc + try: + yield + finally: + handle.seek(0) + if os.name == "nt": + import msvcrt + msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1) + else: + import fcntl + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + + +def is_locked(path: Path) -> bool: + if not path.exists(): + return False + try: + with file_lock(path): + return False + except BlockingIOError: + return True + + +class RunStore: + """Only the worker writes state/events; clients write a cancellation marker.""" + + def __init__(self, root: Path): + self.root = root + self.state = read_json(root / "state.json") + self._sequence = 0 + event_file = root / "events.jsonl" + if event_file.exists(): + # A killed writer may leave an incomplete final line. Discard only + # that line before resuming; valid prior events remain untouched. + raw = event_file.read_bytes() + end = raw.rfind(b"\n") + 1 + if end != len(raw): + with event_file.open("r+b") as stream: + stream.truncate(end) + for line in raw[:end].splitlines(): + self._sequence = max(self._sequence, json.loads(line)["sequence"]) + + def update(self, **values: Any) -> dict[str, Any]: + self.state.update(values) + self.state["updated_at"] = utc_now_iso() + write_json(self.root / "state.json", self.state) + return self.state + + def event(self, event: str, **values: Any) -> None: + self._sequence += 1 + record = {"sequence": self._sequence, "time": utc_now_iso(), "event": event, **values} + with (self.root / "events.jsonl").open("a", encoding="utf-8", newline="\n") as stream: + stream.write(json.dumps(record, ensure_ascii=False, allow_nan=False) + "\n") + stream.flush() + os.fsync(stream.fileno()) + + def count(self, name: str, amount: int = 1) -> None: + counters = self.state.setdefault("counters", {}) + counters[name] = counters.get(name, 0) + amount + self.update(counters=counters) + + def stage(self, name: str, *, completed: int = 0, total: int = 0) -> None: + self.update(stage=name, completed_items=completed, total_items=total) + self.event("stage", stage=name, completed_items=completed, total_items=total) + + +def events_page(root: Path, *, after: int, limit: int) -> dict[str, Any]: + if after < 0 or not 1 <= limit <= 1000: + raise ValueError("after must be nonnegative; limit must be between 1 and 1000.") + rows: list[dict[str, Any]] = [] + path = root / "events.jsonl" + if path.exists(): + with path.open(encoding="utf-8") as stream: + for line in stream: + if not line.endswith("\n"): + break + row = json.loads(line) + if row["sequence"] > after: + rows.append(row) + if len(rows) > limit: + break + return {"events": rows[:limit], "next_cursor": rows[min(len(rows), limit)-1]["sequence"] if rows else after, + "has_more": len(rows) > limit} diff --git a/src/diffractscout/workflow/worker.py b/src/diffractscout/workflow/worker.py new file mode 100644 index 0000000..4c7395d --- /dev/null +++ b/src/diffractscout/workflow/worker.py @@ -0,0 +1,34 @@ +"""Detached local worker entry point; no network service or global daemon.""" +from __future__ import annotations + +import argparse +import re +from concurrent.futures import ThreadPoolExecutor + +from .runner import execute +from .store import job_path, read_json, workspace_path + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--workspace", required=True) + parser.add_argument("--expected-attempt", type=int) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--run-id") + group.add_argument("--batch-id") + args = parser.parse_args(argv) + if args.run_id: + execute(job_path(args.workspace, args.run_id), expected_attempt=args.expected_attempt) + else: + if not re.fullmatch(r"[a-f0-9]{32}", args.batch_id): + raise ValueError("Invalid batch_id.") + batch = read_json(workspace_path(args.workspace) / "batches" / (args.batch_id + ".json")) + roots = [job_path(args.workspace, run_id) for run_id in batch["run_ids"]] + with ThreadPoolExecutor(max_workers=batch["max_workers"]) as pool: + list(pool.map(lambda root: execute(root, expected_attempt=0), + [root for root in roots if read_json(root / "state.json")["state"] == "pending"])) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_agent_cli.py b/tests/test_agent_cli.py new file mode 100644 index 0000000..daa0d3a --- /dev/null +++ b/tests/test_agent_cli.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace +import sys + +import pytest + +import diffractscout +from diffractscout import agent_cli +from diffractscout.cli import main as cli_main + + +@pytest.fixture +def service(monkeypatch): + calls = [] + + class Request: + @classmethod + def from_dict(cls, payload, base_dir=None): + if not isinstance(payload, dict): + raise ValueError("Request must be an object") + return {"payload": payload, "base_dir": base_dir} + + def record(name, result): + def call(*args, **kwargs): + calls.append((name, args, kwargs)) + return result + return call + + api = SimpleNamespace(WorkflowRequest=Request) + results = {"context": {"workflows": ["analyze"]}, "schema": {"type": "object"}, + "plan_workflow": {"stages": ["analyze"]}, + "run_job": {"run_id": "job-1", "state": "completed"}, + "submit_job": {"run_id": "job-1", "state": "pending"}, + "resume_job": {"run_id": "job-1", "state": "pending"}, + "cancel_job": {"run_id": "job-1", "state": "cancelling"}, + "status_job": {"run_id": "job-1", "state": "running"}, + "read_events": {"events": [], "next_after": 0}, + "get_result": {"artifacts": [], "rows": []}, + "list_jobs": [{"run_id": "job-1"}], + "run_batch": {"state": "completed", "jobs": []}} + for name, result in results.items(): + setattr(api, name, record(name, result)) + monkeypatch.setattr(diffractscout, "workflow", api, raising=False) + return api, calls + + +def test_agent_context_routes_through_existing_cli(service, capsys): + assert cli_main(["agent", "context"]) == 0 + assert json.loads(capsys.readouterr().out) == {"workflows": ["analyze"]} + + +@pytest.mark.parametrize("arguments", [[], ["unknown"], ["status"], ["events", "job", "--limit", "no"]]) +def test_argument_errors_are_json(arguments, capsys): + assert agent_cli.main(arguments) == 2 + output = capsys.readouterr() + assert json.loads(output.out)["error"]["type"] == "ValueError" + + +@pytest.mark.parametrize("source", ["[]", "{broken", "missing-request.json"]) +def test_request_read_errors_are_json(service, source, capsys): + assert agent_cli.main(["run", source]) == 2 + assert json.loads(capsys.readouterr().out)["ok"] is False + + +def test_request_file_parent_is_resolution_base(service, tmp_path, capsys): + path = tmp_path / "request.json" + path.write_text(json.dumps({"workflow": "analyze", "inputs": ["sample.cif"]}), encoding="utf-8") + assert agent_cli.main(["run", str(path), "--workspace", str(tmp_path / "jobs")]) == 0 + call = service[1][-1] + assert call[0] == "run_job" + assert call[1][0]["base_dir"] == tmp_path + assert call[2]["workspace"] == str(tmp_path / "jobs") + assert json.loads(capsys.readouterr().out)["state"] == "completed" + + +def test_submit_uses_background_service(service, capsys): + assert agent_cli.main(["--workspace", "jobs", "submit", '{"workflow":"analyze"}']) == 0 + assert service[1][-1][0] == "submit_job" + assert service[1][-1][2] == {"workspace": "jobs"} + assert json.loads(capsys.readouterr().out)["state"] == "pending" + + +def test_stdin_and_provider_progress_do_not_contaminate_json(service, monkeypatch, capsys): + import io + + monkeypatch.setattr(sys, "stdin", io.StringIO('{"workflow":"analyze"}')) + + def run(*args, **kwargs): + print("provider progress") + return {"state": "partial"} + + service[0].run_job = run + assert agent_cli.main(["run", "-"]) == 3 + output = capsys.readouterr() + assert json.loads(output.out) == {"state": "partial"} + assert "provider progress" in output.err + + +@pytest.mark.parametrize("state,code", [("completed", 0), ("partial", 3), ("failed", 2), + ("blocked", 2), ("cancelled", 2), ("interrupted", 2), + ("pending", 0), ("running", 0)]) +def test_state_exit_codes(service, capsys, state, code): + service[0].status_job = lambda *args, **kwargs: {"state": state} + assert agent_cli.main(["status", "job-1"]) == code + assert json.loads(capsys.readouterr().out)["state"] == state + + +def test_batch_file_normalizes_each_request(service, tmp_path, capsys): + manifest = {"schema": "diffractscout_batch_v1", "requests": [ + {"workflow": "analyze", "inputs": ["one.cif"]}, + {"workflow": "analyze", "inputs": ["two.cif"]}], "max_workers": 2} + path = tmp_path / "batch.json" + path.write_text(json.dumps(manifest), encoding="utf-8") + assert agent_cli.main(["run", str(path)]) == 0 + name, args, kwargs = service[1][-1] + assert name == "run_batch" + assert len(args[0]) == 2 + assert all(item["base_dir"] == tmp_path for item in args[0]) + assert kwargs == {"workspace": None, "max_workers": 2, "background": False} + json.loads(capsys.readouterr().out) + + +@pytest.mark.parametrize("payload", [ + {"schema": "diffractscout_batch_v1", "requests": []}, + {"schema": "diffractscout_batch_v1", "requests": {}, "max_workers": 1}, + {"schema": "diffractscout_batch_v1", "requests": [{}], "api_key": "forbidden"}, +]) +def test_invalid_batch_is_rejected(service, payload, capsys): + assert agent_cli.main(["run", json.dumps(payload)]) == 2 + assert json.loads(capsys.readouterr().out)["ok"] is False + assert not service[1] + + +@pytest.mark.parametrize("argv,name,options", [ + (["schema", "request"], "schema", {}), + (["plan", '{}'], "plan_workflow", {}), + (["list"], "list_jobs", {"workspace": None}), + (["cancel", "job-1"], "cancel_job", {"workspace": None}), + (["resume", "job-1"], "resume_job", {"workspace": None, "background": True}), + (["resume", "job-1", "--foreground"], "resume_job", {"workspace": None, "background": False}), + (["events", "job-1", "--after", "5", "--limit", "3"], "read_events", + {"workspace": None, "after": 5, "limit": 3}), + (["result", "job-1", "--table", "reflections", "--offset", "4", "--limit", "2"], + "get_result", {"workspace": None, "table": "reflections", "offset": 4, "limit": 2, "verify": True}), +]) +def test_service_dispatch(service, capsys, argv, name, options): + assert agent_cli.main(argv) == 0 + assert service[1][-1][0] == name + assert service[1][-1][2] == options + json.loads(capsys.readouterr().out) + + +def test_error_does_not_echo_environment_key(service, monkeypatch, capsys): + monkeypatch.setenv("MP_API_KEY", "sensitive-value") + + def fail(*args, **kwargs): + raise ValueError("Provider rejected sensitive-value") + + service[0].status_job = fail + assert agent_cli.main(["status", "job-1"]) == 2 + output = capsys.readouterr().out + assert "sensitive-value" not in output + assert "[redacted]" in output + + +def test_package_exports_do_not_import_optional_mcp(): + import subprocess + + process = subprocess.run([sys.executable, "-c", "import sys, diffractscout; assert 'mcp' not in sys.modules"], + env={**__import__("os").environ, "PYTHONPATH": str(Path(__file__).parents[1] / "src")}, + capture_output=True, text=True) + assert process.returncode == 0, process.stderr + + +def test_real_workflow_cli_plan_run_progress_and_result(demo_inputs, tmp_path, capsys): + request = tmp_path / "request.json" + request.write_text(json.dumps({ + "schema": "diffractscout_workflow_v1", "workflow": "analyze", + "inputs": [demo_inputs.name], "output_dir": "bundle", + "analysis": {"include_elasticity": False, "include_patterns": False}, + "exports": {"include_excel": False}, + }), encoding="utf-8") + workspace = str(tmp_path / "jobs") + assert cli_main(["agent", "plan", str(request)]) == 0 + assert isinstance(json.loads(capsys.readouterr().out), dict) + assert not (tmp_path / "bundle").exists() + assert cli_main(["agent", "run", str(request), "--workspace", workspace]) == 0 + completed = json.loads(capsys.readouterr().out) + assert completed["state"] == "completed" + assert (tmp_path / "bundle").is_dir() + run_id = completed["run_id"] + for operation in ("status", "events", "result"): + assert cli_main(["agent", operation, run_id, "--workspace", workspace]) == 0 + assert isinstance(json.loads(capsys.readouterr().out), dict) + + +def test_real_workflow_partial_exit_code(demo_inputs, tmp_path, capsys): + (demo_inputs / "empty.cif").write_text("", encoding="utf-8") + request = {"workflow": "analyze", "inputs": [str(demo_inputs)], + "output_dir": str(tmp_path / "partial"), + "analysis": {"include_elasticity": False, "include_patterns": False}, + "exports": {"include_excel": False}} + assert cli_main(["agent", "run", json.dumps(request), + "--workspace", str(tmp_path / "jobs")]) == 3 + assert json.loads(capsys.readouterr().out)["state"] == "partial" diff --git a/tests/test_gui.py b/tests/test_gui.py index d2b1b62..62a8a2b 100644 --- a/tests/test_gui.py +++ b/tests/test_gui.py @@ -528,7 +528,7 @@ def set(self, value): scheduled = [] errors = [] warnings = [] - controller._start_task = lambda label, function: scheduled.append((label, function)) + controller._submit_workflow = lambda request: scheduled.append(request) controller._format_input_problems = ( gui_module.DiffractScoutApp._format_input_problems.__get__(controller) ) @@ -1328,7 +1328,7 @@ def test_local_left_form_controls_are_reachable_after_scroll(geometry: str) -> N app.destroy() -def test_local_worker_uses_ui_state_snapshot(tmp_path, monkeypatch) -> None: +def test_local_worker_uses_ui_state_snapshot(tmp_path) -> None: """Worker options are frozen without needing a live Tk display.""" if not hasattr(gui_module.DiffractScoutApp, "_run_local"): @@ -1348,40 +1348,34 @@ class Controller: pass controller = Controller() - sentinel = object() - observed: dict[str, object] = {} scheduled: dict[str, object] = {} - - def fake_analyze(inputs, output, **kwargs): - observed.update({"inputs": inputs, "output": output, **kwargs}) - return sentinel - - monkeypatch.setattr(gui_module, "analyze_cifs", fake_analyze) controller.running = False input_cif = tmp_path / "input.cif" input_cif.write_text("data_input\n", encoding="utf-8") controller.local_inputs = [input_cif] controller.local_output = Variable(str(tmp_path / "bundle")) - controller.elastic_overrides = {} + controller.elastic_overrides = { + str(input_cif.resolve()): gui_module.parse_cubic_cij(160, 90, 45, source="manual") + } controller.local_recursive = Variable(True) controller.include_excel = Variable(False) controller.overwrite = Variable(False) controller._form_analysis_settings = lambda: AnalysisSettings() - controller._start_task = lambda label, function: scheduled.update( - {"label": label, "function": function} - ) + controller._submit_workflow = lambda request: scheduled.update(request) gui_module.DiffractScoutApp._run_local(controller) - function = scheduled["function"] controller.local_recursive.set(False) controller.include_excel.set(True) controller.overwrite.set(True) - assert callable(function) - assert function() is sentinel - assert observed["recursive"] is True - assert observed["include_excel"] is False - assert observed["overwrite"] is False + assert scheduled["workflow"] == "analyze" + assert scheduled["inputs"] == [str(input_cif)] + assert scheduled["execution"]["recursive"] is True + assert scheduled["exports"]["include_excel"] is False + assert scheduled["execution"]["overwrite"] is False + tensor = scheduled["elastic_overrides"][str(input_cif.resolve())] + assert tensor["stiffness_GPa"][0][0] == 160 + assert tensor["source_record_id"] == "manual" def test_readiness_tracks_required_inputs_and_busy_state(tmp_path) -> None: diff --git a/tests/test_gui_cifs.py b/tests/test_gui_cifs.py index 3efd335..3feda39 100644 --- a/tests/test_gui_cifs.py +++ b/tests/test_gui_cifs.py @@ -402,6 +402,7 @@ def test_offline_tc4_runs_real_service_through_gui_and_loads_cifs( loaded: list[tuple[list[Path], int]] = [] root, dialog = _make_hidden_dialog( language="zh", + workspace=tmp_path / "workspace", on_load=lambda paths: loaded.append((paths, threading.get_ident())), ) try: diff --git a/tests/test_gui_tasks.py b/tests/test_gui_tasks.py new file mode 100644 index 0000000..a18e4ef --- /dev/null +++ b/tests/test_gui_tasks.py @@ -0,0 +1,352 @@ +"""Task controls exercised through real Tk widgets with offline workflow doubles.""" + +from __future__ import annotations + +import json +import threading +import time +from types import SimpleNamespace + +import pytest + +from diffractscout import gui, gui_cifs, gui_tasks +from diffractscout.gui_i18n import t +from diffractscout.workflow.contracts import WorkflowRequest +from diffractscout.models import AnalysisSettings, DiscoverySettings + + +def pump(root, predicate, timeout=5): + end = time.monotonic() + timeout + while not predicate() and time.monotonic() < end: + root.update() + time.sleep(0.01) + assert predicate() + + +def make_app(): + try: + return gui.create_app() + except RuntimeError as exc: + if str(exc) == "Tkinter is unavailable in this Python installation.": + pytest.skip(str(exc)) + raise + except Exception as exc: + if gui._is_known_gui_startup_tcl_error(exc): + pytest.skip(str(exc)) + raise + + +class FakeWorkflow: + WorkflowRequest = WorkflowRequest + + def __init__(self): + self.jobs = {"old-run": dict(run_id="old-run", name="Saved analysis", workflow="analyze", + state="interrupted", stage="analysis", completed_items=2, + total_items=3, diagnostics=[{"code": "missing_input", "message": "missing.cif"}])} + self.calls = [] + self.result = object() + self.bad_integrity = False + + def record(self, action, **kwargs): + self.calls.append((action, kwargs, threading.get_ident())) + + def list_jobs(self, **kwargs): + self.record("list", **kwargs) + return [dict(job) for job in self.jobs.values()] + + def submit_job(self, request, **kwargs): + self.record("submit", request=request.to_dict(), **kwargs) + job = dict(run_id="new-run", workflow=request.workflow, name=request.name, + state="running", stage="analysis", completed_items=0, total_items=3) + self.jobs[job["run_id"]] = job + return dict(job) + + def status_job(self, run_id, **kwargs): + return dict(self.jobs[run_id]) + + def resume_job(self, run_id, **kwargs): + self.record("resume", run_id=run_id, **kwargs) + self.jobs[run_id]["state"] = "running" + return dict(self.jobs[run_id]) + + def cancel_job(self, run_id, **kwargs): + self.record("cancel", run_id=run_id, **kwargs) + self.jobs[run_id]["state"] = "cancelled" + return dict(self.jobs[run_id]) + + def get_result(self, run_id, **kwargs): + self.record("verify", run_id=run_id, **kwargs) + if self.bad_integrity: + raise ValueError("Result hash changed") + return {"run_id": run_id, "artifacts": []} + + def load_result_object(self, run_id, **kwargs): + self.record("load", run_id=run_id, **kwargs) + return self.result + + def read_events(self, run_id, **kwargs): + return {"events": [{"message": "checkpoint written"}]} + + +@pytest.fixture +def task_window(monkeypatch, tmp_path): + if gui_tasks.tk is None: + pytest.skip("Tk is unavailable") + try: + root = gui_tasks.tk.Tk() + except gui_tasks.tk.TclError as exc: + pytest.skip(str(exc)) + root.withdraw() + api = FakeWorkflow() + monkeypatch.setattr(gui_tasks, "workflow_api", lambda: api) + lang = ["zh"] + delivered = [] + view = gui_tasks.TasksView(root, translate=lambda key, **fmt: t(lang[0], key, **fmt), + on_result=lambda result: delivered.append((result, threading.get_ident())), + api_key=lambda: "runtime-key") + view.workspace.set(str(tmp_path / "workspace")) + view.pack(fill="both", expand=True) + view.refresh() + pump(root, lambda: bool(view.tree.get_children())) + yield root, view, api, lang, delivered + root.destroy() + + +def test_history_resume_cancel_counts_and_thread_boundary(task_window): + root, view, api, lang, delivered = task_window + view.tree.selection_set("old-run") + view.tree.event_generate("<>") + root.update() + assert view.tree.set("old-run", "counts") == "2 / 3" + assert "missing.cif" in view.details.get("1.0", "end") + assert view.resume_button.instate(("!disabled",)) + view.resume_button.invoke() + pump(root, lambda: api.jobs["old-run"]["state"] == "running") + pump(root, lambda: view.cancel_button.instate(("!disabled",))) + view.cancel_button.invoke() + pump(root, lambda: api.jobs["old-run"]["state"] == "cancelled") + assert all(thread != threading.get_ident() for _, _, thread in api.calls) + assert not delivered + lang[0] = "en" + view.refresh_language() + assert view.resume_button.cget("text") == "Resume" + assert view.tree.heading("counts", "text") == "Done / total" + + +def test_import_resolves_relative_paths_and_keeps_key_out_of_request(task_window, tmp_path, monkeypatch): + root, view, api, lang, delivered = task_window + request_file = tmp_path / "request.json" + request_file.write_text(json.dumps({"schema": "diffractscout_workflow_v1", "workflow": "analyze", + "inputs": ["structure.cif"], "output_dir": "results"}), encoding="utf-8") + monkeypatch.setattr(gui_tasks.filedialog, "askopenfilename", lambda **_: str(request_file)) + view.import_button.invoke() + pump(root, lambda: any(action == "submit" for action, _, _ in api.calls)) + call = next(data for action, data, _ in api.calls if action == "submit") + assert call["request"]["inputs"] == [str((tmp_path / "structure.cif").resolve())] + assert call["request"]["output_dir"] == str((tmp_path / "results").resolve()) + assert call["api_key"] == "runtime-key" + assert "runtime-key" not in json.dumps(call["request"]) + + +def test_verified_open_and_corruption_stays_out_of_results(task_window): + root, view, api, lang, delivered = task_window + api.jobs["old-run"]["state"] = "completed" + view.refresh() + pump(root, lambda: view._jobs["old-run"]["state"] == "completed") + view.tree.selection_set("old-run") + view._select() + api.bad_integrity = True + view.open_button.invoke() + pump(root, lambda: "hash changed" in view.message.get()) + assert not delivered + assert not any(action == "load" for action, _, _ in api.calls) + api.bad_integrity = False + view.open_button.invoke() + pump(root, lambda: bool(delivered)) + assert delivered == [(api.result, threading.get_ident())] + assert next(data for action, data, _ in api.calls if action == "verify")["verify"] is True + + +def test_verified_folder_never_opens_changed_result(task_window, monkeypatch, tmp_path): + root, view, api, lang, delivered = task_window + api.jobs["old-run"]["state"] = "completed" + view.refresh() + pump(root, lambda: view._jobs["old-run"]["state"] == "completed") + view.tree.selection_set("old-run") + view._select() + opened = [] + monkeypatch.setattr(gui, "open_path", lambda path: opened.append(path)) + api.bad_integrity = True + view.folder_button.invoke() + pump(root, lambda: "hash changed" in view.message.get()) + assert not opened + api.bad_integrity = False + monkeypatch.setattr(api, "get_result", lambda *a, **kw: {"output_dir": str(tmp_path)}) + view.folder_button.invoke() + pump(root, lambda: bool(opened)) + assert opened == [str(tmp_path)] + + +def test_workspace_switch_discards_old_monitor_messages_and_history_returns(task_window, tmp_path): + root, view, api, lang, delivered = task_window + old_generation = view._generation + view.workspace.set(str(tmp_path / "another-workspace")) + view.refresh() + view._messages.put((old_generation, "submitted", {"run_id": "stale", "state": "running"})) + pump(root, lambda: view._jobs.get("old-run") is not None) + assert "stale" not in view._jobs + # A newly constructed view loads saved history without submitting anything. + restored = gui_tasks.TasksView(root, translate=lambda key, **fmt: t("en", key, **fmt), + on_result=lambda _: None, api_key=lambda: "") + restored.pack() + pump(root, lambda: "old-run" in restored.tree.get_children()) + assert not any(action == "submit" for action, _, _ in api.calls) + restored.destroy() + + +def test_preparation_uses_durable_job_without_saving_credentials(monkeypatch, tmp_path): + api = FakeWorkflow() + monkeypatch.setattr(gui_tasks, "workflow_api", lambda: api) + submitted = api.submit_job + + def completed(request, **kwargs): + job = submitted(request, **kwargs) + job["state"] = "completed" + return job + + monkeypatch.setattr(api, "submit_job", completed) + result = gui_cifs._prepare_cifs(composition="Ti-Al-V", output_dir=tmp_path / "prepared", + nominal="TC4", offline=True, phases=["alpha"], + api_key="private-key", _workspace=tmp_path / "workspace") + assert result is api.result + data = next(data for action, data, _ in api.calls if action == "submit") + assert data["request"]["workflow"] == "prepare" + assert data["request"]["preparation"]["nominal"] == "TC4" + assert data["workspace"] == tmp_path / "workspace" + assert data["api_key"] == "private-key" + assert "private-key" not in json.dumps(data["request"]) + + +def test_mp_form_keeps_analysis_download_authorization_and_runtime_key(tmp_path): + def variable(value): + return SimpleNamespace(get=lambda: value) + + submitted = [] + controller = SimpleNamespace( + running=False, mp_composition=variable("Ti-Al-V"), mp_key=variable("private-mp-key"), + mp_output=variable(str(tmp_path / "results")), mp_conventional=variable(False), + include_excel=variable(False), overwrite=variable(False), + _form_analysis_settings=lambda: AnalysisSettings(input_mode="energy", energy_keV=83, + include_elasticity=False, include_patterns=False), + _form_discovery_settings=lambda: DiscoverySettings(max_total=20), + _t=lambda key, **fmt: t("en", key, **fmt), + _submit_workflow=lambda request, **kwargs: submitted.append((request, kwargs)), + ) + gui.DiffractScoutApp._run_mp(controller) + request, runtime = submitted[0] + parsed = WorkflowRequest.from_dict(request) + assert parsed.workflow == "run" + assert parsed.analysis["energy_keV"] == 83 + assert parsed.analysis["include_patterns"] is False + assert parsed.analysis["include_elasticity"] is False + assert parsed.execution["conventional_unit_cell"] is False + assert parsed.execution["confirm_above"] == 20 + assert parsed.execution["authorize_large_download"] is True + assert runtime == {"api_key": "private-mp-key"} + assert "private-mp-key" not in json.dumps(parsed.to_dict()) + + +def test_main_window_tasks_at_minimum_size_and_language_toggle(monkeypatch): + api = FakeWorkflow() + monkeypatch.setattr(gui_tasks, "workflow_api", lambda: api) + app = make_app() + closed = False + try: + # Map only this test window off screen; do not interact with other apps. + app.geometry("900x640+20000+20000") + app.notebook.select(app.tasks_view) + app.update() + assert app.winfo_width() == 900 and app.winfo_height() == 640 + view = app.tasks_view + for language in ("en", "zh"): + app._set_language(language) + app.update() + assert app.notebook.tab(view, "text") == t(language, "tab_tasks") + assert view.open_button.cget("text") == t(language, "tasks_open") + for button in (view.import_button, view.refresh_button, view.resume_button, + view.cancel_button, view.open_button, view.folder_button): + assert button.winfo_ismapped() + assert button.winfo_rootx() >= view.winfo_rootx() + assert button.winfo_rootx() + button.winfo_width() <= view.winfo_rootx() + view.winfo_width() + assert view.details.winfo_height() > 30 + assert view.tree.winfo_height() > 30 + app.running = True + app._on_close() + closed = True + assert view._closed + finally: + if not closed: + app.destroy() + + +def test_real_local_job_survives_close_and_reopens_verified_result(tmp_path): + from pathlib import Path + from diffractscout import workflow + + source = Path(gui.__file__).parent / "benchmark_data" / "bcc_fe.cif" + workspace = tmp_path / "workspace" + output = tmp_path / "results" + app = make_app() + closed = False + try: + app.withdraw() + app.tasks_view.workspace.set(str(workspace)) + app._add_input_paths([source]) + app.local_output.set(str(output)) + app.energy_shortcut.set("83 keV") + app.include_excel.set(False) + app.include_patterns.set(False) + app.elastic_overrides[str(source.resolve())] = gui.parse_cubic_cij(230, 135, 116, "GUI override") + app._run_local() + pump(app, lambda: bool(app.tasks_view._jobs), timeout=15) + run_id = next(iter(app.tasks_view._jobs)) + at_close = workflow.status_job(run_id, workspace=workspace) + assert at_close["state"] in gui_tasks.ACTIVE_STATES + app._on_close() + closed = True + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + status = workflow.status_job(run_id, workspace=workspace) + if status["state"] not in gui_tasks.ACTIVE_STATES: + break + time.sleep(0.1) + assert status["state"] == "completed", status + result = workflow.load_result_object(run_id, workspace=workspace) + assert result.output_dir == output + assert len(result.analyses) == 1 + analysis = result.analyses[0] + assert analysis.energy_keV == 83 + assert analysis.elastic_tensor.stiffness_GPa[0, 0] == 230 + assert analysis.elastic_tensor.source_record_id == "GUI override" + assert not (output / "results.xlsx").exists() + assert workflow.get_result(run_id, workspace=workspace)["integrity"] == "passed" + # Restart the real task monitor against the same saved workspace. + root = gui_tasks.tk.Tk() + root.withdraw() + loaded = [] + try: + view = gui_tasks.TasksView(root, translate=lambda key, **fmt: t("en", key, **fmt), + on_result=loaded.append, api_key=lambda: "") + view.workspace.set(str(workspace)) + view.refresh() + pump(root, lambda: run_id in view.tree.get_children()) + view.tree.selection_set(run_id) + view._select() + view.open_button.invoke() + pump(root, lambda: bool(loaded)) + assert loaded[0].output_dir == output + finally: + root.destroy() + finally: + if not closed: + app.destroy() diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py new file mode 100644 index 0000000..8a56ba4 --- /dev/null +++ b/tests/test_mcp_server.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +import os +import sys +from types import SimpleNamespace + +import pytest + +import diffractscout +from diffractscout.mcp_server import create_server + +mcp = pytest.importorskip("mcp", reason="Install diffractscout[mcp] for protocol tests") +Client, StdioServerParameters = mcp.Client, mcp.StdioServerParameters + + +@pytest.fixture +def service(monkeypatch): + class Request: + @classmethod + def from_dict(cls, data, base_dir=None): + return data + + calls = [] + + def submit(request, **kwargs): + calls.append((request, kwargs)) + return {"run_id": "background-job", "state": "pending"} + + def missing(*args, **kwargs): + raise FileNotFoundError("No such job") + + api = SimpleNamespace(WorkflowRequest=Request, + context=lambda: {"workflows": ["analyze"]}, + schema=lambda name: {"type": "object"}, + plan_workflow=lambda request: {"stages": ["analyze"]}, + run_job=lambda request, **kw: {"state": "completed"}, + submit_job=submit, status_job=missing, + list_jobs=lambda **kw: [{"run_id": f"job-{i}"} for i in range(4)], + read_events=lambda *args, **kw: {"events": [], **kw}, + get_result=lambda *args, **kw: {"rows": [], **kw}, + resume_job=lambda *args, **kw: {"state": "pending"}, + cancel_job=lambda *args, **kw: {"state": "cancelling"}) + monkeypatch.setattr(diffractscout, "workflow", api, raising=False) + return api, calls + + +def test_sdk_tools_metadata_structured_results_and_errors(service): + async def exercise(): + async with Client(create_server()) as client: + tools = {tool.name: tool for tool in (await client.list_tools()).tools} + assert set(tools) == {"context", "schema", "plan", "run", "submit", "list_jobs", + "status", "events", "resume", "cancel", "result"} + assert tools["context"].annotations.read_only_hint is True + assert tools["submit"].annotations.read_only_hint is False + assert tools["result"].input_schema["properties"]["limit"]["maximum"] == 1000 + assert tools["list_jobs"].input_schema["properties"]["offset"]["minimum"] == 0 + assert tools["context"].output_schema["type"] == "object" + context = await client.call_tool("context") + assert context.structured_content == {"workflows": ["analyze"]} + submitted = await client.call_tool("submit", {"request": {"workflow": "analyze"}}) + assert submitted.structured_content["state"] == "pending" + assert len(service[1]) == 1 + page = await client.call_tool("list_jobs", {"offset": 1, "limit": 2}) + assert page.structured_content["jobs"] == [{"run_id": "job-1"}, {"run_id": "job-2"}] + assert page.structured_content["next_offset"] == 3 + missing = await client.call_tool("status", {"run_id": "absent"}) + assert missing.is_error + assert missing.structured_content["error"]["type"] == "FileNotFoundError" + invalid = await client.call_tool("events", {"run_id": "x", "limit": 0}) + assert invalid.is_error + asyncio.run(exercise()) + + +def test_real_stdio_client_context_schema_submit_and_missing_job(tmp_path, demo_inputs): + async def exercise(): + source = Path(__file__).parents[1] / "src" + parameters = StdioServerParameters( + command=sys.executable, args=["-m", "diffractscout.mcp_server"], + env={**os.environ, "PYTHONPATH": str(source)}, cwd=str(tmp_path), + ) + async with Client(parameters, read_timeout_seconds=20) as client: + tools = {tool.name for tool in (await client.list_tools()).tools} + assert {"context", "schema", "submit", "result"} <= tools + context = await client.call_tool("context") + assert not context.is_error + assert isinstance(context.structured_content, dict) + schema = await client.call_tool("schema") + assert not schema.is_error + assert isinstance(schema.structured_content, dict) + missing = await client.call_tool("status", {"run_id": "missing", + "workspace": str(tmp_path / "jobs")}) + assert missing.is_error + assert "error" in missing.structured_content + json.loads(missing.content[0].text) + submitted = await client.call_tool("submit", { + "workspace": str(tmp_path / "jobs"), "request": { + "schema": "diffractscout_workflow_v1", "workflow": "analyze", + "inputs": [str(demo_inputs)], "output_dir": str(tmp_path / "bundle"), + "analysis": {"include_elasticity": False, "include_patterns": False}, + "exports": {"include_excel": False}, + }}) + assert not submitted.is_error, submitted.content + run_id = submitted.structured_content["run_id"] + for _ in range(200): + status = await client.call_tool("status", {"run_id": run_id, + "workspace": str(tmp_path / "jobs")}) + assert not status.is_error + if status.structured_content["state"] not in {"pending", "running"}: + break + await asyncio.sleep(0.05) + assert status.structured_content["state"] == "completed", status.structured_content + result = await client.call_tool("result", {"run_id": run_id, + "workspace": str(tmp_path / "jobs")}) + assert not result.is_error, result.content + assert isinstance(result.structured_content, dict) + asyncio.run(exercise()) diff --git a/tests/test_workflow.py b/tests/test_workflow.py new file mode 100644 index 0000000..b43925d --- /dev/null +++ b/tests/test_workflow.py @@ -0,0 +1,393 @@ +"""Offline integration checks for durable workflow execution and recovery.""" + +from __future__ import annotations + +import json +import shutil +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +from dataclasses import asdict +from pathlib import Path +from threading import Event + +import numpy as np +import pytest + +from diffractscout import pipeline +from diffractscout.demo import SYNTHETIC_FCC_CIF +from diffractscout.models import AnalysisSettings, CandidateRecord, DownloadArtifact +from diffractscout.utils import write_json +from diffractscout.validation import verify_bundle +from diffractscout.workflow import ( + cancel_job, + get_result, + list_jobs, + load_result_object, + read_events, + resume_job, + run_batch, + run_job, + status_job, + submit_job, +) +from diffractscout.workflow import service + + +class CountingProvider: + """An offline provider that records actual external-operation boundaries.""" + + name = "Offline counting provider" + + def __init__(self, count=1, *, query_error=None): + self.candidates = [CandidateRecord(f"mp-{index + 1}", formula="Al") for index in range(count)] + self.queries = Counter() + self.downloads = Counter() + self.query_error = query_error + + def metadata(self): + return {"provider": self.name, "offline_fixture": True} + + def search_subsystem(self, chemsys, **kwargs): + self.queries[chemsys] += 1 + if self.query_error: + raise self.query_error + return list(self.candidates) + + def search_material_ids(self, material_ids, **kwargs): + self.queries[tuple(material_ids)] += 1 + return [item for item in self.candidates if item.material_id in material_ids] + + def download_candidates(self, candidates, output_dir, **kwargs): + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + artifacts = [] + for candidate in candidates: + self.downloads[candidate.material_id] += 1 + path = output_dir / f"{candidate.material_id}.cif" + path.write_text(SYNTHETIC_FCC_CIF, encoding="utf-8") + artifacts.append(DownloadArtifact(candidate, path)) + return artifacts + + +@pytest.fixture +def workspace(tmp_path, monkeypatch): + monkeypatch.delenv("MP_API_KEY", raising=False) + return tmp_path / "workspace" + + +def request_for(inputs, **changes): + request = { + "workflow": "analyze", "inputs": [str(path) for path in inputs], + "analysis": {"step_deg": 0.2, "two_theta_max_deg": 80, "include_elasticity": False}, + "exports": {"include_excel": False}, + } + request.update(changes) + return request + + +def root_for(workspace, status): + return workspace / "jobs" / status["run_id"] + + +def instrument_calculations(monkeypatch): + original = pipeline.simulate_powder_pattern + calls = Counter() + + def counted(structure, *args, **kwargs): + calls[structure.cif_path.name] += 1 + return original(structure, *args, **kwargs) + + monkeypatch.setattr(pipeline, "simulate_powder_pattern", counted) + return calls + + +def fail_export_once(monkeypatch): + original = pipeline.export_result_bundle + attempts = [] + + def export(*args, **kwargs): + attempts.append(1) + if len(attempts) == 1: + raise OSError("Injected export failure after numerical work") + return original(*args, **kwargs) + + monkeypatch.setattr(pipeline, "export_result_bundle", export) + return attempts + + +@pytest.mark.parametrize("count", [1, 2]) +def test_local_results_match_existing_pipeline_and_verify(demo_inputs, tmp_path, workspace, count): + inputs = [demo_inputs / "synthetic_fcc_al.cif"] + if count == 2: + second = demo_inputs / "second.cif" + second.write_text(SYNTHETIC_FCC_CIF.replace("4.000000", "4.100000"), encoding="utf-8") + inputs.append(second) + raw = request_for(inputs) + settings = AnalysisSettings(**raw["analysis"]) + baseline = pipeline.analyze_cifs(inputs, tmp_path / "baseline", settings=settings, include_excel=False) + status = run_job(raw, workspace=workspace) + assert status["state"] == "completed", status["diagnostics"] + result = load_result_object(status["run_id"], workspace=workspace) + assert len(result.analyses) == count + for expected, actual in zip(baseline.analyses, result.analyses, strict=True): + assert expected.phase_name == actual.phase_name + assert [asdict(row) for row in expected.reflections] == [asdict(row) for row in actual.reflections] + np.testing.assert_array_equal(expected.two_theta_grid, actual.two_theta_grid) + np.testing.assert_allclose(expected.intensity_profile, actual.intensity_profile, rtol=0, atol=0) + assert actual.structure.cif_path.is_file() + assert verify_bundle(status["output_dir"])["ok"] + response = get_result(status["run_id"], workspace=workspace, table="peak_reference", limit=1) + assert len(response["rows"]) == 1 + assert response["integrity"] == "passed" + + +def test_export_failure_resume_reuses_queries_downloads_and_calculations(workspace, monkeypatch): + provider = CountingProvider(count=2) + calculations = instrument_calculations(monkeypatch) + exports = fail_export_once(monkeypatch) + status = run_job(request_for([], workflow="run", composition="Al"), workspace=workspace, provider=provider) + assert status["state"] == "failed", status["diagnostics"] + assert sum(calculations.values()) == 2 + assert provider.queries == {"Al": 1} + assert provider.downloads == {"mp-1": 1, "mp-2": 1} + resumed = resume_job(status["run_id"], workspace=workspace, background=False, provider=provider) + assert resumed["state"] == "completed", resumed["diagnostics"] + assert len(exports) == 2 + assert sum(calculations.values()) == 2 + assert provider.queries == {"Al": 1} + assert provider.downloads == {"mp-1": 1, "mp-2": 1} + assert resumed["counters"]["reused"] >= 5 + assert verify_bundle(resumed["output_dir"])["ok"] + + +@pytest.mark.parametrize("explicit_output", [False, True]) +def test_partial_phase_resume_recomputes_only_failure(workspace, tmp_path, monkeypatch, explicit_output): + provider = CountingProvider(count=2) + original = pipeline.simulate_powder_pattern + calls = Counter() + + def fail_second_once(structure, *args, **kwargs): + name = structure.cif_path.name + calls[name] += 1 + if "mp-2" in name and calls[name] == 1: + raise ValueError("Injected one-phase calculation failure") + return original(structure, *args, **kwargs) + + monkeypatch.setattr(pipeline, "simulate_powder_pattern", fail_second_once) + raw = request_for([], workflow="run", composition="Al") + if explicit_output: + raw["output_dir"] = str(tmp_path / "external-result") + status = run_job(raw, workspace=workspace, provider=provider) + assert status["state"] == "partial", status["diagnostics"] + first_output = Path(status["output_dir"]) + assert verify_bundle(first_output)["ok"] + resumed = resume_job(status["run_id"], workspace=workspace, background=False, provider=provider) + assert resumed["state"] == "completed", resumed["diagnostics"] + assert calls["mp-1.cif"] == 1 + assert calls["mp-2.cif"] == 2 + assert provider.queries == {"Al": 1} + assert provider.downloads == {"mp-1": 1, "mp-2": 1} + assert Path(resumed["output_dir"]) != first_output + assert verify_bundle(first_output)["ok"] + assert verify_bundle(resumed["output_dir"])["ok"] + + +@pytest.mark.parametrize("mutation", ["change", "remove"]) +def test_resume_uses_frozen_sources_after_original_changes(demo_inputs, workspace, monkeypatch, mutation): + source = demo_inputs / "synthetic_fcc_al.cif" + original_bytes = source.read_bytes() + fail_export_once(monkeypatch) + calculations = instrument_calculations(monkeypatch) + status = run_job(request_for([source]), workspace=workspace) + assert status["state"] == "failed", status["diagnostics"] + if mutation == "change": + source.write_text("no longer a valid CIF", encoding="utf-8") + else: + shutil.rmtree(demo_inputs) + resumed = resume_job(status["run_id"], workspace=workspace, background=False) + assert resumed["state"] == "completed", resumed["diagnostics"] + assert sum(calculations.values()) == 1 + assert (Path(resumed["output_dir"]) / "inputs" / source.name).read_bytes() == original_bytes + + +@pytest.mark.parametrize("checkpoint_kind", ["analyses", "queries", "downloads", "sources"]) +def test_corrupt_checkpoint_is_never_reused(workspace, monkeypatch, checkpoint_kind): + provider = CountingProvider() + calculations = instrument_calculations(monkeypatch) + fail_export_once(monkeypatch) + status = run_job(request_for([], workflow="run", composition="Al"), workspace=workspace, provider=provider) + assert status["state"] == "failed", status["diagnostics"] + directory = root_for(workspace, status) / checkpoint_kind + sealed = next(directory.rglob("checkpoint.json")).parent + manifest = json.loads((sealed / "checkpoint.json").read_text(encoding="utf-8")) + member = sealed / manifest["files"][0]["path"] + member.write_bytes(member.read_bytes() + b"corruption") + before = dict(calculations) + resumed = resume_job(status["run_id"], workspace=workspace, background=False, provider=provider) + assert resumed["state"] == "blocked", resumed["diagnostics"] + assert any(item["code"] == "CHECKPOINT_INVALID" for item in resumed["diagnostics"]) + assert calculations == before + assert not Path(resumed["output_dir"]).exists() + + +def test_duplicate_submission_is_idempotent_and_rejects_changed_request(demo_inputs, workspace, monkeypatch): + calculations = instrument_calculations(monkeypatch) + raw = request_for([demo_inputs / "synthetic_fcc_al.cif"], submission_id="identical-offline-submission") + first = run_job(raw, workspace=workspace) + duplicate = run_job(json.loads(json.dumps(raw)), workspace=workspace) + assert first["state"] == duplicate["state"] == "completed" + assert duplicate["run_id"] == first["run_id"] + assert sum(calculations.values()) == 1 + assert len(list_jobs(workspace=workspace)) == 1 + with pytest.raises(ValueError, match="different request"): + run_job({**raw, "name": "changed request"}, workspace=workspace) + assert sum(calculations.values()) == 1 + + +def test_parent_run_reuses_only_matching_analysis_inputs(demo_inputs, workspace, monkeypatch): + calculations = instrument_calculations(monkeypatch) + raw = request_for([demo_inputs / "synthetic_fcc_al.cif"]) + parent = run_job(raw, workspace=workspace) + child = run_job({**raw, "parent_run_id": parent["run_id"]}, workspace=workspace) + assert parent["state"] == child["state"] == "completed" + assert parent["run_id"] != child["run_id"] + assert sum(calculations.values()) == 1 + assert child["counters"]["analyses"] == 0 + assert verify_bundle(child["output_dir"])["ok"] + changed = {**raw, "parent_run_id": parent["run_id"], "analysis": {**raw["analysis"], "step_deg": 0.1}} + changed_child = run_job(changed, workspace=workspace) + assert changed_child["state"] == "completed", changed_child["diagnostics"] + assert changed_child["counters"]["analyses"] == 1 + assert sum(calculations.values()) == 2 + + +def test_cancellation_resumes_from_last_successful_phase(demo_inputs, workspace, monkeypatch): + inputs = [demo_inputs / "synthetic_fcc_al.cif", demo_inputs / "second.cif"] + inputs[1].write_text(SYNTHETIC_FCC_CIF, encoding="utf-8") + original = pipeline.simulate_powder_pattern + calls = Counter() + + def cancel_after_first(structure, *args, **kwargs): + calls[structure.cif_path.name] += 1 + result = original(structure, *args, **kwargs) + if sum(calls.values()) == 1: + current = list_jobs(workspace=workspace)[0] + cancel_job(current["run_id"], workspace=workspace) + return result + + monkeypatch.setattr(pipeline, "simulate_powder_pattern", cancel_after_first) + status = run_job(request_for(inputs), workspace=workspace) + assert status["state"] == "cancelled", status["diagnostics"] + assert sum(calls.values()) == 1 + resumed = resume_job(status["run_id"], workspace=workspace, background=False) + assert resumed["state"] == "completed", resumed["diagnostics"] + assert sum(calls.values()) == 2 + events = read_events(status["run_id"], workspace=workspace)["events"] + assert "cancelled" in {row["event"] for row in events} + assert [row["sequence"] for row in events] == sorted({row["sequence"] for row in events}) + + +def test_keyboard_interruption_releases_lock_and_can_resume(demo_inputs, workspace, monkeypatch): + original = pipeline.simulate_powder_pattern + attempts = [] + + def interrupt_once(*args, **kwargs): + attempts.append(1) + if len(attempts) == 1: + raise KeyboardInterrupt() + return original(*args, **kwargs) + + monkeypatch.setattr(pipeline, "simulate_powder_pattern", interrupt_once) + status = run_job(request_for([demo_inputs / "synthetic_fcc_al.cif"]), workspace=workspace) + assert status["state"] == "interrupted" + assert status_job(status["run_id"], workspace=workspace)["active"] is False + resumed = resume_job(status["run_id"], workspace=workspace, background=False) + assert resumed["state"] == "completed", resumed["diagnostics"] + assert len(attempts) == 2 + + +def test_parallel_resume_callers_have_one_calculation_owner(demo_inputs, workspace, monkeypatch): + monkeypatch.setattr(service, "_launch", lambda *args, **kwargs: None) + status = submit_job(request_for([demo_inputs / "synthetic_fcc_al.cif"]), workspace=workspace) + entered, release, second_entered = Event(), Event(), Event() + original = pipeline.simulate_powder_pattern + attempts = [] + + def hold_calculation(*args, **kwargs): + attempts.append(1) + entered.set() + assert release.wait(10), "test did not release the active worker" + return original(*args, **kwargs) + + monkeypatch.setattr(pipeline, "simulate_powder_pattern", hold_calculation) + + def competing_resume(): + second_entered.set() + return resume_job(status["run_id"], workspace=workspace, background=False) + + with ThreadPoolExecutor(max_workers=2) as pool: + owner = pool.submit(resume_job, status["run_id"], workspace=workspace, background=False) + try: + assert entered.wait(10), "worker did not reach a calculation" + other = pool.submit(competing_resume) + assert second_entered.wait(5) + assert status_job(status["run_id"], workspace=workspace)["active"] is True + finally: + release.set() + finished = owner.result(timeout=10) + second_status = other.result(timeout=10) + assert finished["state"] == "completed", finished["diagnostics"] + assert second_status["state"] in {"running", "completed"} + assert len(attempts) == 1 + + +def test_stale_running_state_is_reported_interrupted(demo_inputs, workspace, monkeypatch): + monkeypatch.setattr(service, "_launch", lambda *args, **kwargs: None) + status = submit_job(request_for([demo_inputs / "synthetic_fcc_al.cif"]), workspace=workspace) + state_path = root_for(workspace, status) / "state.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + state.update(state="running", stage="analysis", worker_pid=2147483647) + write_json(state_path, state) + assert status_job(status["run_id"], workspace=workspace)["state"] == "interrupted" + resumed = resume_job(status["run_id"], workspace=workspace, background=False) + assert resumed["state"] == "completed", resumed["diagnostics"] + + +def test_batch_mixed_success_is_reported_without_losing_success(demo_inputs, workspace): + response = run_batch([ + request_for([demo_inputs / "synthetic_fcc_al.cif"]), + request_for([demo_inputs / "missing.cif"]), + ], workspace=workspace, max_workers=2) + assert response["state"] == "partial" + assert response["completed"] == 1 + assert response["total"] == 2 + assert response["jobs"][0]["state"] == "completed" + assert response["jobs"][1]["state"] in {"blocked", "failed"} + assert verify_bundle(response["jobs"][0]["output_dir"])["ok"] + + +def test_empty_discovery_is_distinct_from_query_failure(workspace): + raw = request_for([], workflow="discover", composition="Al") + empty = run_job(raw, workspace=workspace, provider=CountingProvider(count=0)) + failed = run_job(raw, workspace=workspace, provider=CountingProvider(count=0, query_error=ValueError("offline query rejected"))) + assert empty["state"] == "completed", empty["diagnostics"] + assert empty["coverage"]["successful"] == 0 + assert failed["state"] == "failed", failed["diagnostics"] + assert any(row["code"] == "QUERY_FAILED" for row in failed["diagnostics"]) + assert not any(row["code"] == "QUERY_FAILED" for row in empty["diagnostics"]) + + +def test_offline_preparation_and_analysis_produce_verified_models(workspace): + status = run_job(request_for([], workflow="prepare_analyze", composition="Ti", preparation={ + "offline": True, "phases": ["beta"], + }), workspace=workspace) + assert status["state"] == "completed", status["diagnostics"] + assert status["counters"]["preparations"] == 1 + assert status["counters"]["analyses"] == 1 + assert status["coverage"] == {"requested": 1, "successful": 1, "failed": 0} + assert status["counters"]["queries"] == status["counters"]["downloads"] == 0 + assert verify_bundle(status["output_dir"])["ok"] + assert verify_bundle(status["preparation_bundle"])["ok"] + result = load_result_object(status["run_id"], workspace=workspace) + assert len(result.analyses) == 1 + assert result.analyses[0].structure.space_group_number == 229 diff --git a/tests/test_workflow_contracts.py b/tests/test_workflow_contracts.py new file mode 100644 index 0000000..fdd8929 --- /dev/null +++ b/tests/test_workflow_contracts.py @@ -0,0 +1,257 @@ +"""Offline request validation and genuinely read-only workflow planning.""" + +from __future__ import annotations + +import json +from dataclasses import asdict +from pathlib import Path + +import numpy as np +import pytest + +from diffractscout.models import AnalysisSettings, DiscoverySettings, ElasticTensor +from diffractscout.workflow import contracts +from diffractscout.workflow.contracts import WorkflowRequest, context, plan_workflow, schema + + +@pytest.fixture +def local_request(tmp_path): + source = tmp_path / "inputs" / "source.CIF" + source.parent.mkdir() + source.write_text("data_read_only", encoding="utf-8") + return {"workflow": "analyze", "inputs": [str(source)], "output_dir": str(tmp_path / "results")} + + +def codes(plan): + return {item["code"] for item in plan["diagnostics"]} + + +def test_defaults_normalize_and_round_trip_without_aliasing(): + raw = {"workflow": "analyze"} + request = WorkflowRequest.from_dict(raw) + assert request.analysis == asdict(AnalysisSettings()) + assert request.discovery == asdict(DiscoverySettings()) + assert request.exports == {"include_excel": True, "template": "full"} + assert request.execution["max_attempts"] == 3 + assert request.execution["confirm_above"] == 200 + assert WorkflowRequest.from_dict(request.to_dict()).to_dict() == request.to_dict() + serialized = request.to_dict() + serialized["analysis"]["step_deg"] = 2 + assert request.analysis["step_deg"] == 0.02 + assert raw == {"workflow": "analyze"} + + +@pytest.mark.parametrize("patch", [ + {"unexpected": True}, {"schema": "wrong"}, {"workflow": "unknown"}, + {"analysis": {"step_deg": True}}, {"analysis": {"include_patterns": 1}}, + {"analysis": {"step_deg": float("nan")}}, {"analysis": {"step_deg": "0.1"}}, + {"analysis": {"step_deg": 0}}, {"analysis": {"d_min_A": 5, "d_max_A": 1}}, + {"analysis": {"two_theta_max_deg": 181}}, {"analysis": {"unknown": 1}}, + {"analysis": {"profile_model": "unknown"}}, {"discovery": {"max_total": False}}, + {"discovery": {"exclude_deprecated": "false"}}, {"discovery": {"e_hull_max_eV_atom": -1}}, + {"execution": {"max_attempts": 0}}, {"execution": {"confirm_above": 1.1}}, + {"execution": {"overwrite": 0}}, {"execution": {"unknown": True}}, + {"exports": {"include_excel": "true"}}, {"exports": {"template": "unknown"}}, + {"inputs": "source.cif"}, {"inputs": [""]}, {"output_dir": ""}, + {"composition": 123}, {"submission_id": 1}, {"experiment": "unknown"}, + {"api_key": "do-not-store"}, {"preparation": {"credentials": "do-not-store"}}, + {"preparation": {"max_subsystems": 0}}, {"preparation": {"preview_wavelength_A": -1}}, + {"preparation": {"offline": "yes"}}, {"adaptation": {"a": -1, "citation": "source"}}, + {"adaptation": {"fract": {"q": 1}, "citation": "source"}}, + {"adaptation": {"fract": {"x": True}, "citation": "source"}}, + {"adaptation": {"a": 3}}, {"adaptation": {"atomic_percent": "Ti=90,O=10"}}, +]) +def test_invalid_requests_fail_before_output_creation(patch, tmp_path): + with pytest.raises(ValueError): + WorkflowRequest.from_dict({"workflow": "analyze", "output_dir": str(tmp_path / "new"), **patch}) + assert not (tmp_path / "new").exists() + + +@pytest.mark.parametrize(("analysis", "mode"), [ + ({"energy_keV": 100}, "energy"), ({"wavelength_A": 0.12398}, "wavelength"), + ({"source_preset": "Custom", "wavelength_A": 1.2}, "source"), +]) +def test_numeric_radiation_mode_inference(analysis, mode): + request = WorkflowRequest.from_dict({"workflow": "analyze", "analysis": analysis}) + assert request.analysis["input_mode"] == mode + + +@pytest.mark.parametrize("analysis", [ + {"energy_keV": 100, "wavelength_A": 1}, + {"source_preset": "Cu Ka", "energy_keV": 100}, + {"input_mode": "source", "energy_keV": 100}, + {"input_mode": "energy", "wavelength_A": 1}, + {"input_mode": "wavelength", "energy_keV": 1}, + {"energy_keV": 0}, {"source_preset": "Custom"}, +]) +def test_contradictory_radiation_is_rejected(analysis): + with pytest.raises(ValueError): + WorkflowRequest.from_dict({"workflow": "analyze", "analysis": analysis}) + + +def test_request_file_relative_paths_resolve_without_reads(tmp_path): + request = WorkflowRequest.from_dict({ + "workflow": "prepare", "inputs": ["source.cif"], "output_dir": "derived", + "preparation": {"templates": {"alpha": "template.cif"}, "parameter_file": "parameters.json"}, + }, base_dir=tmp_path) + assert request.inputs == [str(tmp_path / "source.cif")] + assert request.output_dir == str(tmp_path / "derived") + assert request.preparation["templates"]["alpha"] == str(tmp_path / "template.cif") + assert request.preparation["parameter_file"] == str(tmp_path / "parameters.json") + assert list(tmp_path.iterdir()) == [] + + +def test_elastic_tensor_serialization_revalidates_scientific_payload(): + tensor = ElasticTensor(np.eye(6) * 100, source_provider="caller") + request = WorkflowRequest.from_dict({"workflow": "analyze", "elastic_overrides": {"phase": tensor}}) + assert json.loads(json.dumps(request.to_dict()))["elastic_overrides"]["phase"]["stiffness_GPa"][0][0] == 100 + request = WorkflowRequest.from_dict(request.to_dict()) + assert request.elastic_overrides["phase"]["source_provider"] == "caller" + for bad in ({"stiffness_GPa": [[1]]}, {"stiffness_GPa": np.zeros((6, 6)).tolist()}, + {"stiffness_GPa": np.eye(6).tolist(), "unit": "Pa"}): + with pytest.raises(ValueError): + WorkflowRequest.from_dict({"workflow": "analyze", "elastic_overrides": {"phase": bad}}) + + +def test_local_plan_never_writes_or_requires_mp_credentials(local_request, monkeypatch): + monkeypatch.delenv("MP_API_KEY", raising=False) + monkeypatch.setattr(contracts, "_available", lambda name: name not in ("mp_api", "pymatgen")) + source = Path(local_request["inputs"][0]) + before = source.read_bytes() + plan = plan_workflow(local_request) + assert plan["ready"], plan["diagnostics"] + assert plan["local_cifs"] == [str(source)] + assert plan["estimated_queries"] == 0 + assert "mp-api" not in plan["required_dependencies"] + assert source.read_bytes() == before + assert not Path(local_request["output_dir"]).exists() + + +def test_missing_fields_and_paths_are_actionable(tmp_path): + plan = plan_workflow({"workflow": "analyze", "inputs": [str(tmp_path / "absent.cif")]}) + assert not plan["ready"] + assert "invalid_inputs" in codes(plan) + for item in plan["diagnostics"]: + assert {"code", "message", "stage", "item", "retryable", "missing_fields"} <= set(item) + assert plan_workflow({"workflow": "analyze"})["diagnostics"][0]["missing_fields"] == ["inputs"] + + +def test_optional_output_uses_managed_bundle_without_creating_it(local_request): + local_request.pop("output_dir") + plan = plan_workflow(local_request) + assert plan["ready"] + assert plan["request"]["output_dir"] is None + assert plan["parameter_sources"]["output"] == "managed per-run bundle" + + +def test_preparation_preserves_supported_p1_template_restoration(tmp_path): + import gemmi + from diffractscout.cif_quality import write_clean_model + + small = gemmi.SmallStructure() + small.cell = gemmi.UnitCell(3.2, 3.2, 3.2, 90, 90, 90) + small.spacegroup_hm = "P 1" + for index, position in enumerate(((0, 0, 0), (0.5, 0.5, 0.5))): + site = gemmi.SmallStructure.Site() + site.element = gemmi.Element("Ti") + site.type_symbol = "Ti" + site.label = f"Ti{index}" + site.fract = gemmi.Fractional(*position) + site.occ = 1 + small.add_site(site) + template = tmp_path / "p1.cif" + write_clean_model(small, template, details="Synthetic test") + before = template.read_bytes() + plan = plan_workflow({"workflow": "prepare", "composition": "Ti", "preparation": { + "phases": ["beta"], "templates": {"beta": str(template)}, "offline": True, + }}) + assert plan["ready"], plan["diagnostics"] + assert template.read_bytes() == before + assert list(tmp_path.iterdir()) == [template] + + +def test_unsafe_existing_output_cannot_be_planned_for_overwrite(local_request): + output = Path(local_request["output_dir"]) + output.mkdir() + unrelated = output / "unrelated.txt" + unrelated.write_text("preserve", encoding="utf-8") + local_request["execution"] = {"overwrite": True} + assert "unsafe_overwrite" in codes(plan_workflow(local_request)) + assert unrelated.read_text() == "preserve" + + +def test_input_output_separation_is_checked_without_creating_target(local_request): + source = Path(local_request["inputs"][0]) + local_request["output_dir"] = str(source.parent) + plan = plan_workflow(local_request) + assert "unsafe_output" in codes(plan) + assert source.read_text() == "data_read_only" + + +def test_synchrotron_never_silently_uses_laboratory_defaults(local_request): + local_request["experiment"] = "synchrotron" + assert "missing_radiation" in codes(plan_workflow(local_request)) + local_request["analysis"] = {"energy_keV": 100} + assert plan_workflow(local_request)["ready"] + + +def test_provider_injection_and_query_estimate_are_pure(tmp_path, monkeypatch): + class NoCallProvider: + def __getattr__(self, name): + raise AssertionError(f"Planner accessed provider.{name}") + + monkeypatch.delenv("MP_API_KEY", raising=False) + monkeypatch.setattr(contracts, "_available", lambda name: name not in ("mp_api", "pymatgen")) + raw = {"workflow": "run", "composition": "Ti-Nb-Zr", "output_dir": str(tmp_path / "new")} + blocked = plan_workflow(raw) + assert {"missing_credentials", "missing_dependency"} <= codes(blocked) + plan = plan_workflow(raw, provider=NoCallProvider()) + assert plan["ready"], plan["diagnostics"] + assert plan["estimated_queries"] == 7 + assert not (tmp_path / "new").exists() + raw["discovery"] = {"max_subsystems": 3} + assert "query_limit" in codes(plan_workflow(raw, provider=NoCallProvider())) + + +def test_preparation_reuses_phase_parameter_validation(tmp_path): + raw = {"workflow": "prepare", "composition": "Ti", "output_dir": str(tmp_path / "new"), + "preparation": {"offline": True, "phases": ["beta"]}} + assert plan_workflow(raw)["ready"] + raw["preparation"]["phase_parameters"] = {"beta": {"lattice": {"a": 3.3}}} + with pytest.raises(ValueError, match="citation"): + WorkflowRequest.from_dict(raw) + raw["preparation"]["phase_parameters"]["beta"]["citation"] = "caller supplied value" + assert plan_workflow(raw)["ready"] + assert not (tmp_path / "new").exists() + + +def test_missing_preparation_files_block_without_writes(tmp_path): + raw = {"workflow": "prepare", "composition": "Ti", "output_dir": str(tmp_path / "new"), + "preparation": {"templates": {"beta": str(tmp_path / "missing.cif")}}} + assert "invalid_preparation" in codes(plan_workflow(raw)) + raw["preparation"] = {"parameter_file": str(tmp_path / "missing.json")} + assert "invalid_preparation" in codes(plan_workflow(raw)) + assert not (tmp_path / "new").exists() + + +def test_schema_has_precise_nested_types_and_defaults(): + document = schema() + assert document["additionalProperties"] is False + properties = document["properties"] + assert properties["schema"]["const"] == "diffractscout_workflow_v1" + assert properties["execution"]["properties"]["overwrite"]["type"] == "boolean" + assert properties["execution"]["properties"]["max_attempts"]["default"] == 3 + assert properties["analysis"]["properties"]["energy_keV"]["anyOf"] == [{"type": "number"}, {"type": "null"}] + assert properties["preparation"]["properties"]["templates"]["additionalProperties"]["type"] == "string" + assert properties["exports"]["properties"]["template"]["enum"] == ["full", "peak_table"] + json.dumps(document, allow_nan=False) + with pytest.raises(ValueError): + schema("unknown") + + +def test_context_contains_only_credential_presence(monkeypatch): + monkeypatch.setenv("MP_API_KEY", "sentinel-sensitive-value") + capability = context() + assert capability["credentials"]["configured"] is True + assert "sentinel-sensitive-value" not in json.dumps(capability) + assert capability["planning"] == {"network": False, "writes": False} diff --git a/tests/test_workflow_delivery.py b/tests/test_workflow_delivery.py new file mode 100644 index 0000000..63e0e47 --- /dev/null +++ b/tests/test_workflow_delivery.py @@ -0,0 +1,212 @@ +"""Delivery, retry, and provenance contracts beyond ordinary happy paths.""" +from __future__ import annotations + +import json +import time +from pathlib import Path + +import numpy as np +import pytest + +from diffractscout.demo import SYNTHETIC_FCC_CIF +from diffractscout.models import CandidateRecord, DownloadArtifact +from diffractscout.workflow import WorkflowRequest, get_result, load_result_object, plan_workflow, run_batch, run_job, status_job + + +class RetryProvider: + name = "retry-fixture" + + def __init__(self, error=TimeoutError, failures=2): + self.error, self.failures = error, failures + self.queries = self.downloads = 0 + + def metadata(self): + return {"provider": self.name} + + def search_subsystem(self, _system, **_options): + self.queries += 1 + if self.queries <= self.failures: + raise self.error("transient query failure") + return [CandidateRecord("fixture-1", formula="Al")] + + def download_candidates(self, candidates, output_dir, **_options): + self.downloads += 1 + if self.downloads <= self.failures: + raise self.error("transient download failure") + path = output_dir / "fixture.cif" + path.write_text(SYNTHETIC_FCC_CIF, encoding="utf-8") + return [DownloadArtifact(candidates[0], path)] + + +@pytest.mark.parametrize("error,expected_state,attempts", [(TimeoutError, "completed", 3), (ValueError, "failed", 1)]) +def test_only_transient_errors_receive_bounded_retries(tmp_path, error, expected_state, attempts): + provider = RetryProvider(error=error) + result = run_job({"workflow": "run", "composition": "Al", "analysis": {"include_elasticity": False}}, + workspace=tmp_path / "jobs", provider=provider) + assert result["state"] == expected_state, result["diagnostics"] + assert provider.queries == attempts + assert provider.downloads == (3 if expected_state == "completed" else 0) + + +def test_manual_elastic_tensor_survives_portable_replay(tmp_path): + source = tmp_path / "al.cif" + source.write_text(SYNTHETIC_FCC_CIF, encoding="utf-8") + workspace = tmp_path / "jobs" + first = run_job({"workflow": "analyze", "inputs": [str(source)], + "elastic_overrides": {str(source): {"stiffness_GPa": (np.eye(6)*100).tolist(), "source_provider": "manual fixture"}}}, + workspace=workspace) + assert first["state"] == "completed", first["diagnostics"] + assert first["completed_items"] == first["total_items"] == 1 + replay_file = Path(first["output_dir"]) / "replay_request.json" + replay = WorkflowRequest.from_dict(json.loads(replay_file.read_text()), base_dir=replay_file.parent) + second = run_job(replay, workspace=workspace) + assert second["state"] == "completed", second["diagnostics"] + expected = load_result_object(first["run_id"], workspace=workspace) + actual = load_result_object(second["run_id"], workspace=workspace) + assert actual.analyses[0].reflections[0].young_modulus_hkl_normal_GPa == expected.analyses[0].reflections[0].young_modulus_hkl_normal_GPa + assert actual.analyses[0].elastic_tensor.source_provider == "manual fixture" + + +def test_relative_tensor_identity_resolves_against_request_file(tmp_path): + request = WorkflowRequest.from_dict({"workflow": "analyze", "inputs": ["inputs/al.cif"], + "elastic_overrides": {"inputs/al.cif": {"stiffness_GPa": (np.eye(6)*100).tolist()}}}, base_dir=tmp_path) + assert request.inputs[0] in request.elastic_overrides + + +def test_plan_preserves_user_values_and_declared_defaults(tmp_path): + source = tmp_path / "al.cif" + source.write_text(SYNTHETIC_FCC_CIF) + request = WorkflowRequest.from_dict({"workflow": "analyze", "inputs": [str(source)], "analysis": {"energy_keV": 100}}) + plan = plan_workflow(request) + assert plan["parameter_sources"]["analysis"]["energy_keV"] == "request" + assert plan["parameter_sources"]["analysis"]["step_deg"] == "default" + assert plan["parameter_sources"]["analysis"]["input_mode"] == "derived_from_energy_keV" + + +def test_empty_peak_table_is_not_reported_as_complete_reference(tmp_path): + source = tmp_path / "al.cif" + source.write_text(SYNTHETIC_FCC_CIF) + status = run_job({"workflow": "analyze", "inputs": [str(source)], + "analysis": {"two_theta_min_deg": 0, "two_theta_max_deg": 1, "include_elasticity": False}}, workspace=tmp_path / "jobs") + assert status["state"] == "partial" + assert any(item["code"] == "EMPTY_REFLECTION_TABLE" for item in status["diagnostics"]) + assert get_result(status["run_id"], workspace=tmp_path / "jobs")["integrity"] == "passed" + + +def test_background_batch_keeps_valid_jobs_when_one_request_is_invalid(tmp_path): + source = tmp_path / "al.cif" + source.write_text(SYNTHETIC_FCC_CIF) + workspace = tmp_path / "jobs" + response = run_batch([{"workflow": "analyze", "inputs": [str(source)]}, {"workflow": "invalid"}], + workspace=workspace, background=True) + assert response["jobs"][1]["state"] == "failed" + run_id = response["jobs"][0]["run_id"] + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + status = status_job(run_id, workspace=workspace) + if status["state"] not in {"pending", "running"}: + break + time.sleep(0.1) + assert status["state"] == "completed", status + + +def test_background_batch_preserves_order_and_schedules_duplicate_id_once(tmp_path): + from diffractscout.workflow.store import process_alive + source = tmp_path / "al.cif" + source.write_text(SYNTHETIC_FCC_CIF) + workspace = tmp_path / "jobs" + request = {"workflow": "analyze", "inputs": [str(source)], "submission_id": "duplicate-in-batch"} + response = run_batch([{"workflow": "invalid"}, request, request], workspace=workspace, background=True) + assert response["jobs"][0]["state"] == "failed" + run_id = response["jobs"][1]["run_id"] + assert response["jobs"][2]["run_id"] == run_id + root = workspace / "jobs" / run_id + launch = json.loads((root / "launch.json").read_text()) + deadline = time.monotonic() + 30 + while process_alive(launch["pid"]) and time.monotonic() < deadline: + time.sleep(0.1) + status = status_job(run_id, workspace=workspace) + assert status["state"] == "completed", status + assert status["attempt"] == 1 + + +def test_exited_worker_is_not_kept_alive_by_a_process_handle(): + import os + import subprocess + import sys + from diffractscout.workflow.store import process_alive + kwargs = {"creationflags": subprocess.CREATE_NO_WINDOW} if os.name == "nt" else {} + with subprocess.Popen([sys.executable, "-c", "pass"], **kwargs) as child: + child.wait(timeout=10) + assert process_alive(child.pid) is False + assert process_alive(os.getpid()) is True + + +def test_retry_limit_cannot_exceed_three(): + with pytest.raises(ValueError, match="between 1 and 3"): + WorkflowRequest.from_dict({"workflow": "discover", "composition": "Al", "execution": {"max_attempts": 4}}) + + +def test_published_bundle_is_recoverable_after_acknowledgement_failure(tmp_path, monkeypatch): + from diffractscout.workflow import runner, resume_job + from diffractscout.validation import verify_bundle + source = tmp_path / "al.cif" + source.write_text(SYNTHETIC_FCC_CIF) + workspace = tmp_path / "jobs" + original = runner.result_record + monkeypatch.setattr(runner, "result_record", lambda *args: (_ for _ in ()).throw(OSError("record write failed"))) + first = run_job({"workflow": "analyze", "inputs": [str(source)]}, workspace=workspace) + assert first["state"] == "failed" + assert verify_bundle(first["output_dir"])["ok"] + monkeypatch.setattr(runner, "result_record", original) + recovered = resume_job(first["run_id"], workspace=workspace, background=False) + assert recovered["state"] == "completed", recovered["diagnostics"] + assert recovered["counters"]["analyses"] == 1 + assert first["output_dir"] in recovered["recovered_bundles"] + assert verify_bundle(first["output_dir"])["ok"] + assert verify_bundle(recovered["output_dir"])["ok"] + + +def test_discovery_replay_retains_discovery_request(tmp_path): + provider = RetryProvider(failures=0) + result = run_job({"workflow": "discover", "composition": "Al"}, workspace=tmp_path / "jobs", provider=provider) + path = Path(result["output_dir"]) / "replay_request.json" + request = WorkflowRequest.from_dict(json.loads(path.read_text()), base_dir=path.parent) + assert request.workflow == "discover" + assert request.composition == "Al" + assert plan_workflow(request, provider=provider)["ready"] + + +def test_frozen_worker_launch_uses_app_dispatch(monkeypatch): + from diffractscout.workflow import service + captured = {} + monkeypatch.setattr(service.sys, "frozen", True, raising=False) + monkeypatch.setattr(service.subprocess, "Popen", lambda command, **kwargs: captured.update(command=command, **kwargs)) + service._spawn(["--run-id", "a"*32, "--workspace", "example"]) + assert captured["command"][1] == "_agent-worker" + assert captured["env"]["PYINSTALLER_RESET_ENVIRONMENT"] == "1" + + +@pytest.mark.parametrize("workflow", ["fetch", "adapt"]) +def test_explicit_derivations_remain_offline_and_replayable(tmp_path, workflow): + import diffractscout + source = Path(diffractscout.__file__).parent / "prototype_data" / "cod_9008554_beta_ti.cif" + request = {"workflow": workflow, "composition": "Ti"} + if workflow == "fetch": + request["preparation"] = {"phases": ["beta"], "templates": {"beta": str(source)}} + else: + request["inputs"] = [str(source)] + request["adaptation"] = {"atomic_percent": "Ti=100", "a": 3.3, "citation": "synthetic test value"} + workspace = tmp_path / "jobs" + result = run_job(request, workspace=workspace) + assert result["state"] == "completed", result["diagnostics"] + assert result["counters"]["queries"] == result["counters"]["downloads"] == 0 + assert get_result(result["run_id"], workspace=workspace)["integrity"] == "passed" + restored = load_result_object(result["run_id"], workspace=workspace) + if workflow == "fetch": + assert restored.records[0].cif_path.is_file() + else: + assert restored.cif_path.is_file() and restored.sidecar_path.is_file() + path = Path(result["output_dir"]) / "replay_request.json" + replay = WorkflowRequest.from_dict(json.loads(path.read_text()), base_dir=path.parent) + assert plan_workflow(replay)["ready"]