From ff7fe62e9cc34d6036c454bc752fd6787fc7406b Mon Sep 17 00:00:00 2001 From: Lan_zhijiang Date: Wed, 21 Oct 2026 17:41:18 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat(capture):=20=E5=8E=9F=E5=AD=90?= =?UTF-8?q?=E4=BF=9D=E5=AD=98=E5=85=B1=E4=BA=AB=E9=87=87=E9=9B=86=E5=9B=BE?= =?UTF-8?q?=E4=B8=8E=E5=B9=82=E7=AD=89=E5=9B=9E=E6=89=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增受 Peer JWT 保护的 Capture HTTP 分发及原始图文解释 - 通过组合事务提交图、标准二进制存储和材料指纹回执 - 在当前迁移头后追加 capture_receipts，Host SDK 增至 0.3.4 - 验证并发重放、冲突、删除后重放及失败回滚 --- .changes/capture.added.md | 3 + app/database_contract/constants.py | 3 +- app/persistence/capture/__init__.py | 3 + app/persistence/capture/repository.py | 40 +++ app/persistence/capture/uow.py | 29 ++ app/schemas/__init__.py | 2 + app/schemas/capture.py | 51 ++++ app/version.py | 2 +- docs/30-unit-tdd/README.md | 1 + docs/30-unit-tdd/capture-extension.md | 39 +++ .../first-party-extension-distribution.md | 5 + extensions/capture/.changes/capture.added.md | 1 + extensions/capture/CHANGELOG.md | 5 + extensions/capture/__init__.py | 29 ++ extensions/capture/api.py | 57 ++++ extensions/capture/pyproject.toml | 30 ++ extensions/capture/resolver.py | 142 ++++++++++ extensions/capture/schema.py | 78 ++++++ extensions/capture/submit.py | 174 ++++++++++++ migrations/revision-integrity.json | 1 + .../dcf44bc9af73_add_capture_receipts.py | 40 +++ .../capture/fixtures/capture-v1.json | 44 +++ tests/extensions/capture/test_contract.py | 30 ++ tests/extensions/capture/test_submission.py | 260 ++++++++++++++++++ 24 files changed, 1067 insertions(+), 2 deletions(-) create mode 100644 .changes/capture.added.md create mode 100644 app/persistence/capture/__init__.py create mode 100644 app/persistence/capture/repository.py create mode 100644 app/persistence/capture/uow.py create mode 100644 app/schemas/capture.py create mode 100644 docs/30-unit-tdd/capture-extension.md create mode 100644 extensions/capture/.changes/capture.added.md create mode 100644 extensions/capture/CHANGELOG.md create mode 100644 extensions/capture/__init__.py create mode 100644 extensions/capture/api.py create mode 100644 extensions/capture/pyproject.toml create mode 100644 extensions/capture/resolver.py create mode 100644 extensions/capture/schema.py create mode 100644 extensions/capture/submit.py create mode 100644 migrations/versions/dcf44bc9af73_add_capture_receipts.py create mode 100644 tests/extensions/capture/fixtures/capture-v1.json create mode 100644 tests/extensions/capture/test_contract.py create mode 100644 tests/extensions/capture/test_submission.py diff --git a/.changes/capture.added.md b/.changes/capture.added.md new file mode 100644 index 0000000..3fc77ac --- /dev/null +++ b/.changes/capture.added.md @@ -0,0 +1,3 @@ +新增采集提交回执及组合事务，使图片、图和回执一起持久化，同次重试不会重复创建或恢复已删除内容。 + +Host SDK 升至 0.3.4，新增 Capture persistence API；旧扩展兼容范围保持不变。 diff --git a/app/database_contract/constants.py b/app/database_contract/constants.py index 984ed84..0b9a9ff 100644 --- a/app/database_contract/constants.py +++ b/app/database_contract/constants.py @@ -4,7 +4,7 @@ CONTRACT_FORMAT = 1 -CONTRACT_REVISION = "peer-jwt-signing-defaults-v1" +CONTRACT_REVISION = "capture-receipts-v1" PROTOCOL_SCHEMA = "inkcre" INTERNAL_SCHEMA = "inkcre_internal" @@ -36,6 +36,7 @@ "block_embeddings", "block_lexical_records", "blocks", + "capture_receipts", "peers", "configs", "extensions", diff --git a/app/persistence/capture/__init__.py b/app/persistence/capture/__init__.py new file mode 100644 index 0000000..0e4bb18 --- /dev/null +++ b/app/persistence/capture/__init__.py @@ -0,0 +1,3 @@ +from .uow import CaptureUnitOfWork, capture_uow + +__all__ = ["CaptureUnitOfWork", "capture_uow"] diff --git a/app/persistence/capture/repository.py b/app/persistence/capture/repository.py new file mode 100644 index 0000000..6dcb775 --- /dev/null +++ b/app/persistence/capture/repository.py @@ -0,0 +1,40 @@ +"""Session-bound receipt serialization; lifecycle belongs to the capture UoW.""" + +from hashlib import sha256 +from uuid import UUID + +import sqlalchemy +from sqlalchemy.ext.asyncio import AsyncSession + +from app.schemas.capture import CaptureReceipt, CaptureReceiptModel + + +class CaptureReceiptRepository: + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def lock(self, submission_id: UUID) -> None: + key = int.from_bytes( + sha256(b"inkcre.capture.submit.v1:" + submission_id.bytes).digest()[:8], + "big", + signed=True, + ) + await self._session.execute( + sqlalchemy.text("SELECT pg_advisory_xact_lock(:key)"), {"key": key} + ) + + async def get(self, submission_id: UUID) -> CaptureReceipt | None: + row = await self._session.get(CaptureReceiptModel, submission_id) + return CaptureReceipt.model_validate(row) if row is not None else None + + async def create( + self, submission_id: UUID, payload_digest: str, root_block_id: int + ) -> CaptureReceipt: + row = CaptureReceiptModel( + submission_id=submission_id, + payload_digest=payload_digest, + root_block_id=root_block_id, + ) + self._session.add(row) + await self._session.flush() + return CaptureReceipt.model_validate(row) diff --git a/app/persistence/capture/uow.py b/app/persistence/capture/uow.py new file mode 100644 index 0000000..c7f6913 --- /dev/null +++ b/app/persistence/capture/uow.py @@ -0,0 +1,29 @@ +"""Capture bytes, graph and receipt commit together on one private session.""" + +from collections.abc import AsyncGenerator +from contextlib import asynccontextmanager +from dataclasses import dataclass + +from app.engine import AsyncSessionFactory +from app.persistence.info_base.repository import BlockRepository, RelationRepository +from app.persistence.info_base.storage import StorageRepository +from app.persistence.info_base.uow import GraphUnitOfWork + +from .repository import CaptureReceiptRepository + + +@dataclass(frozen=True) +class CaptureUnitOfWork: + graph: GraphUnitOfWork + receipts: CaptureReceiptRepository + + +@asynccontextmanager +async def capture_uow() -> AsyncGenerator[CaptureUnitOfWork, None]: + async with AsyncSessionFactory.begin() as session: + yield CaptureUnitOfWork( + GraphUnitOfWork( + BlockRepository(session), RelationRepository(session), StorageRepository(session) + ), + CaptureReceiptRepository(session), + ) diff --git a/app/schemas/__init__.py b/app/schemas/__init__.py index 0e79a00..e33bc41 100644 --- a/app/schemas/__init__.py +++ b/app/schemas/__init__.py @@ -23,6 +23,7 @@ "DeploymentConfigModel", "SinkModel", "SinkTypeModel", + "CaptureReceiptModel", ] import sqlalchemy.orm @@ -54,3 +55,4 @@ from .deployment_config import DeploymentConfigModel from .lexical_retrieval import BlockLexicalRecordModel from .sink import SinkModel, SinkTypeModel +from .capture import CaptureReceiptModel diff --git a/app/schemas/capture.py b/app/schemas/capture.py new file mode 100644 index 0000000..ff37f54 --- /dev/null +++ b/app/schemas/capture.py @@ -0,0 +1,51 @@ +"""Shared receipts for committed capture commands; graph content remains authoritative.""" + +from datetime import datetime +from typing import Literal +from uuid import UUID + +import pydantic +import sqlalchemy +import sqlmodel + + +class CaptureReceiptModel(sqlmodel.SQLModel, table=True): + __tablename__ = "capture_receipts" # type: ignore + __table_args__ = ( + sqlalchemy.CheckConstraint( + "payload_digest ~ '^[0-9a-f]{64}$'", name="capture_receipt_digest" + ), + ) + + submission_id: UUID = sqlmodel.Field(primary_key=True) + payload_digest: str = sqlmodel.Field( + sa_column=sqlalchemy.Column(sqlalchemy.Text, nullable=False) + ) + # A historical reference must survive deletion, or delayed retries resurrect content. + root_block_id: int = sqlmodel.Field(nullable=False) + committed_at: datetime = sqlmodel.Field( + default=None, + sa_column=sqlalchemy.Column( + sqlalchemy.TIMESTAMP(timezone=True), + nullable=False, + server_default=sqlalchemy.text("CURRENT_TIMESTAMP"), + ), + ) + + +class CaptureReceipt(pydantic.BaseModel): + model_config = pydantic.ConfigDict(frozen=True, from_attributes=True) + + submission_id: UUID + payload_digest: str + root_block_id: int + committed_at: datetime + + +class CaptureResult(pydantic.BaseModel): + model_config = pydantic.ConfigDict(frozen=True) + + submission_id: UUID + root_block_id: int + committed_at: datetime + availability: Literal["available", "deleted"] diff --git a/app/version.py b/app/version.py index 20eef61..9bae26f 100644 --- a/app/version.py +++ b/app/version.py @@ -3,7 +3,7 @@ from pathlib import Path import tomllib -CORE_VERSION = "0.3.3" +CORE_VERSION = "0.3.4" APPLICATION_VERSION = tomllib.loads( (Path(__file__).parents[1] / "pyproject.toml").read_text(encoding="utf-8") )["project"]["version"] diff --git a/docs/30-unit-tdd/README.md b/docs/30-unit-tdd/README.md index 8a5ef9c..a688b95 100644 --- a/docs/30-unit-tdd/README.md +++ b/docs/30-unit-tdd/README.md @@ -12,6 +12,7 @@ This directory owns expensive internal design truth for logical units delivered | [agent-query-sink.md](agent-query-sink.md) | Asynchronous Agent-composed query lifecycle, result delivery, and setup contract | | [mail-extension.md](mail-extension.md) | Mail identity, MIME materialization, collection, graph, and failure boundaries | | [memos-extension.md](memos-extension.md) | Memos adapter identity, graph grammar, persistence, and failure boundaries | +| [capture-extension.md](capture-extension.md) | Capture submission identity, original materials, shared graph, and atomic recovery | | [rss-extension.md](rss-extension.md) | RSS adapter identity, collection lifecycle, reconciliation, and materialization | | [twitter-extension.md](twitter-extension.md) | Twitter 共同身份、Bookmark 过渡、事务合并和按需媒体读取 | | [security-model.md](security-model.md) | Core-py realization of the shared security boundary model | diff --git a/docs/30-unit-tdd/capture-extension.md b/docs/30-unit-tdd/capture-extension.md new file mode 100644 index 0000000..15f6db6 --- /dev/null +++ b/docs/30-unit-tdd/capture-extension.md @@ -0,0 +1,39 @@ +# Capture + +`inkcre/capture` 保存一次主动采集的原材料、真实可得的出处与用户附注。原生 Mac Peer 负责系统交互和本机待办，Python 分发执行提交并解释图，TypeScript 分发在 Web Peer 中解释同一份持久内容。Mac 不是扩展宿主；停止 Mac 或原执行 Peer 不影响其他已安装读取分发的 Peer 读取共享图和原始图片。 + +此文拥有 Capture 这个具体扩展的业务合同及 Python 实现边界；请求字段由 `extensions/capture/schema.py` 定义，Swift 与 Python 共用 `tests/extensions/capture/fixtures/capture-v1.json` 指纹样例。数据库结构和迁移由本仓库拥有。Mac 的 SQLite、权限与安装说明归 `InKCre/impl-swift/README.md`，Web 分发归 `InKCre/impl-ts/extensions/capture`。 + +## 提交与恢复 + +已启用的 Python 分发公告精确能力 `extensions.capture.submit.v1`，通过既有 Peer HTTP 协议接收 `POST /capture/submit`。调用者使用部署 Peer JWT；这不是匿名写入入口。执行分发要求 Python Host SDK `>=0.3.4 <0.4.0`，读取分发要求 TypeScript Host SDK `>=0.4.0 <0.5.0`。两者独立安装和启用。 + +请求包含 UUID `submission_id`、带时区的 `captured_at`、入口 `selection | paste | screenshot | manual`、可缺失的 `origin`、有序 `materials` 和可缺失的 `annotation`。出处的 `application_id`、`title`、`url` 都允许缺失；显式粘贴不能推断原始来源。文本材料使用 `kind: text` 和 `text`，图片使用 `kind: image`、`mime_type`、`data_base64` 和可缺失的 `filename`。材料与附注不能同时为空。 + +单次最多 128 份材料，文本和附注的 UTF-8 字节数加解码图片累计不得超过 32 MiB，HTTP JSON 请求不得超过 48 MiB。支持 PNG、JPEG、GIF、WebP；执行方验证 Base64、实际图片格式与声明的 MIME，并拒绝解码尺寸过大的图片。原始字节不经过缩放或重新编码。代理也必须允许相同请求体上限；这两个上限是协议边界，不是所有设备与代理都已完成性能校准的承诺。 + +调用方先冻结提交 UUID、材料和目标信息库，再发送。网络超时不证明提交失败。恢复先查询同一信息库的 `capture_receipts`，核对指纹，再确认本机结果；没有回执时只可使用原 UUID 和原材料重试。切换当前连接不能重定向已有待办。再次主动采集相同内容使用新 UUID，不进行内容去重。 + +指纹为规范 JSON 的 SHA-256：对象键排序、紧凑分隔符、UTF-8、缺失的可选字段显式为 `null`，时间转 UTC 并保留六位小数，图片的 `data_base64` 替换为原始字节的 `data_sha256`。UUID 使用标准小写形式；数组顺序、文本换行及 Unicode 组合形式保持原样。Swift 对冻结载荷的网络时间表示再规范化，避免本机亚毫秒时间参与指纹却未被发送。 + +相同 UUID、相同指纹返回原回执；相同 UUID、不同指纹返回 409，客户端保留材料并要求处理。成功结果包含 `submission_id`、`root_block_id`、`committed_at` 和 `availability: available | deleted`。根 Block 删除后回执仍保留，重试返回 `deleted`，不能复活内容。 + +## 持久图 + +| 对象或关系 | 合同 | +| --- | --- | +| 根 Block | `extensions.capture.capture.v1`，内容为时间、入口、出处的 JSON | +| 根 → 材料 | `material:0` 起连续递增的关系，序号不可重复 | +| 文本材料 | 标准 `core.text.v1`，原文保持不变 | +| 图片材料 | `extensions.capture.attachment.v1`，内容为 MIME 与可选文件名 | +| 图片材料 → 原始内容 | 唯一 `content` 关系，目标为 `core.image.v1` | +| 原始图片 | 标准 PostgreSQL binary storage（`storage = -4`）保存原始字节 | +| 根 → 用户附注 | 至多一条 `annotation` 关系，目标为 `core.text.v1` | + +Resolver 按关系解释材料顺序，不把附注混入原文；缺失或畸形关系显式报错。Web 来源链接仅允许 HTTP(S) 点击，其他来源位置仍可作为文本查看。图片通过标准 storage/resolver 读取，不依赖 Mac 本地文件、Python 内存或派生 OCR。OCR、通用文件、整页抓取和知识查询不属于此分发的提交合同。 + +## 原子性与验证 + +`capture_uow` 将现有图持久化、binary storage bytes 与回执放入同一个 PostgreSQL 事务。UUID 派生的事务 advisory lock 串行化同一提交的竞争者；回执主键是最后一道唯一性约束。回执中的根 ID 不设删除级联外键，防止根删除后丢失已处理身份。通用 `submit_graph` 仍保持每次新建图的原有语义。 + +验证代码覆盖同身份并发、不同载荷冲突、根删除后的重放、新 UUID 的相同内容、真实数据库约束失败后的完整回滚、请求验证不回显私密材料，以及 Swift/Python 规范指纹一致性。数据库测试只在显式指定的、已迁移的可丢弃 PostgreSQL 上运行。真实 Mac 队列还以提交后丢响应代理验证了重开 SQLite 后通过共享回执恢复结果。 diff --git a/docs/40-deployment/first-party-extension-distribution.md b/docs/40-deployment/first-party-extension-distribution.md index 6e06224..b6a2300 100644 --- a/docs/40-deployment/first-party-extension-distribution.md +++ b/docs/40-deployment/first-party-extension-distribution.md @@ -152,6 +152,11 @@ Core Host SDK 0.3 adds `app.http.get_public_http_base_url()` without changing th or persistence API. Its authority is `app/version.py`, independently of the Core service package version in `pyproject.toml`; preparing a service Release PR does not change the Host SDK version. +Host SDK 0.3.4 adds `app.persistence.capture.capture_uow` and the typed Capture receipt/result +models. Capture requires `>=0.3.4 <0.4.0` because it commits the graph, image bytes, and +idempotency receipt in one transaction. Existing lifecycle and graph interfaces are unchanged. +Database admission must also include the `capture_receipts` migration before enabling Capture. + Core Host SDK 0.3.1 adds batched JSON identity lookup and transaction advisory locking to the existing Graph repository. Twitter archive reconciliation requires `>=0.3.1 <0.4.0`. diff --git a/extensions/capture/.changes/capture.added.md b/extensions/capture/.changes/capture.added.md new file mode 100644 index 0000000..bd76cf5 --- /dev/null +++ b/extensions/capture/.changes/capture.added.md @@ -0,0 +1 @@ +提供采集 v1 的原子提交、幂等回执和共享内容解释，保留材料、出处与用户附注的区别。 diff --git a/extensions/capture/CHANGELOG.md b/extensions/capture/CHANGELOG.md new file mode 100644 index 0000000..5a759af --- /dev/null +++ b/extensions/capture/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +此文件记录 Capture Python 分发的独立版本。发布条目由项目内变更片段生成。 + + diff --git a/extensions/capture/__init__.py b/extensions/capture/__init__.py new file mode 100644 index 0000000..69ebe07 --- /dev/null +++ b/extensions/capture/__init__.py @@ -0,0 +1,29 @@ +"""Capture contract execution and interpretation for an admitted Python Peer.""" + +import fastapi +import pydantic + +from app.business.extension import ExtensionBase +from app.business.peer import PeerHTTPInbound + +from .schema import CAPTURE_CAPABILITY + + +class CaptureConfig(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") + + +class Extension(ExtensionBase[CaptureConfig], ext_id="capture", config_cls=CaptureConfig): + @classmethod + def _init_resolvers(cls) -> None: + from . import resolver as resolver_module # noqa: F401 + + @classmethod + def peer_inbounds(cls): + return (PeerHTTPInbound(CAPTURE_CAPABILITY, "POST", "/capture/submit"),) + + @classmethod + def _register_apis(cls, router: fastapi.APIRouter) -> None: + from .api import router as capture_router + + router.include_router(capture_router) diff --git a/extensions/capture/api.py b/extensions/capture/api.py new file mode 100644 index 0000000..e5201f6 --- /dev/null +++ b/extensions/capture/api.py @@ -0,0 +1,57 @@ +"""Bounded, authenticated HTTP boundary for the exact capture submission capability.""" + +from fastapi import APIRouter, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from fastapi.routing import APIRoute + +from app.schemas.capture import CaptureResult + +from .schema import MAX_REQUEST_BYTES, CaptureRequest +from .submit import CaptureConflict, InvalidCapture, submit_capture + + +class CaptureRoute(APIRoute): + def get_route_handler(self): + handler = super().get_route_handler() + + async def bounded(request: Request): + received = 0 + + async def receive(): + nonlocal received + message = await request.receive() + if message["type"] == "http.request": + received += len(message.get("body", b"")) + if received > MAX_REQUEST_BYTES: + raise HTTPException(status_code=413, detail="采集请求超过 48 MiB") + return message + + try: + return await handler(Request(request.scope, receive)) + except RequestValidationError as error: + # FastAPI's default input echo would include whole images and captured private text. + return JSONResponse( + status_code=422, + content={ + "detail": [ + {"loc": list(item["loc"]), "msg": f"字段格式不正确（{item['type']}）"} + for item in error.errors() + ] + }, + ) + + return bounded + + +router = APIRouter(route_class=CaptureRoute) + + +@router.post("/submit") +async def capture(body: CaptureRequest) -> CaptureResult: + try: + return await submit_capture(body) + except CaptureConflict as error: + raise HTTPException(status_code=409, detail=str(error)) from error + except InvalidCapture as error: + raise HTTPException(status_code=422, detail=str(error)) from error diff --git a/extensions/capture/pyproject.toml b/extensions/capture/pyproject.toml new file mode 100644 index 0000000..f6e934d --- /dev/null +++ b/extensions/capture/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=80,<81"] +build-backend = "setuptools.build_meta" + +[project] +name = "inkcre-ext-capture" +version = "0.1.0" +description = "Reliable manual capture for InKCre peers" +requires-python = ">=3.12,<3.13" +license = "MIT" +dependencies = [ + "fastapi>=0.139.2,<0.140.0", + "pydantic>=2.11.7,<3.0.0", + "pillow>=12.3.0,<13.0.0", +] + +[project.entry-points."inkcre.core.extensions"] +capture = "extensions.capture:Extension" + +[tool.inkcre-extension] +name = "inkcre/capture" +nickname = "随手收集" +host-sdk = "core-py" +host-sdk-version = ">=0.3.4 <0.4.0" + +[tool.setuptools] +packages = ["extensions.capture"] + +[tool.setuptools.package-dir] +"extensions.capture" = "." diff --git a/extensions/capture/resolver.py b/extensions/capture/resolver.py new file mode 100644 index 0000000..c7dae35 --- /dev/null +++ b/extensions/capture/resolver.py @@ -0,0 +1,142 @@ +"""Interpret source roles from the shared graph without depending on the capture Peer.""" + +import re + +from app.business.info_base.resolver import Resolver, ResolverManager, TextProjectionContext +from app.business.info_base.resolver.label import format_label +from app.business.info_base.services import BlockService + +from .schema import ( + ATTACHMENT_RESOLVER, + CAPTURE_RESOLVER, + AttachmentMetadata, + CaptureMetadata, + SolvedAttachment, + SolvedCapture, + SolvedMaterial, +) + + +class AttachmentResolver(Resolver[SolvedAttachment, str], rso_type=ATTACHMENT_RESOLVER): + async def _get_solved_content( + self, *, refresh: bool = False, materialize_missing: bool = True + ) -> SolvedAttachment: + metadata = AttachmentMetadata.model_validate_json( + await self.get_raw_content(refresh=refresh) + ) + relations = await self.get_relations( + include_in=False, include_out=True, refresh=refresh + ) + links = [relation for relation in relations if relation.content == "content"] + if len(links) != 1: + raise ValueError("采集附件必须包含一条 content 关系") + block = await BlockService.get(links[0].to_) + if block is None or block.resolver != "core.image.v1": + raise ValueError("采集附件的图片内容不存在或类型不符") + return SolvedAttachment(metadata=metadata, content_block_id=links[0].to_) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) + block = await BlockService.get(solved.content_block_id) + if block is None: + raise ValueError("采集图片已不存在") + content = await ResolverManager.get(block).get_text( + context=context, refresh=refresh, materialize_missing=materialize_missing + ) + return "\n".join(filter(None, (solved.metadata.filename, content))) + + async def get_label(self, *, refresh: bool = False) -> str: + solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) + return format_label("采集图片", solved.metadata.filename or solved.metadata.mime_type) + + +class CaptureResolver(Resolver[SolvedCapture, str], rso_type=CAPTURE_RESOLVER): + async def _get_solved_content( + self, *, refresh: bool = False, materialize_missing: bool = True + ) -> SolvedCapture: + metadata = CaptureMetadata.model_validate_json( + await self.get_raw_content(refresh=refresh) + ) + relations = await self.get_relations( + include_in=False, include_out=True, refresh=refresh + ) + material_ids: dict[int, int] = {} + annotations: list[int] = [] + for relation in relations: + match = re.fullmatch(r"material:(0|[1-9][0-9]*)", relation.content) + if match: + index = int(match[1]) + if index in material_ids: + raise ValueError("采集材料存在重复序号") + material_ids[index] = relation.to_ + elif relation.content.startswith("material:"): + raise ValueError("采集材料序号无效") + elif relation.content == "annotation": + annotations.append(relation.to_) + if sorted(material_ids) != list(range(len(material_ids))) or len(annotations) > 1: + raise ValueError("采集材料顺序或附注关系无效") + materials = [] + for index in range(len(material_ids)): + block = await BlockService.get(material_ids[index]) + if block is None or block.resolver not in ("core.text.v1", ATTACHMENT_RESOLVER): + raise ValueError("采集材料已不存在或类型不符") + materials.append( + SolvedMaterial(block_id=material_ids[index], resolver=block.resolver) + ) + if annotations: + block = await BlockService.get(annotations[0]) + if block is None or block.resolver != "core.text.v1": + raise ValueError("采集附注已不存在或类型不符") + return SolvedCapture( + metadata=metadata, + materials=tuple(materials), + annotation_block_id=annotations[0] if annotations else None, + ) + + async def get_text( + self, + *, + context: TextProjectionContext = "default", + refresh: bool = False, + materialize_missing: bool = True, + ) -> str: + solved = await self.get_solved_content(refresh=refresh, materialize_missing=False) + origin = solved.metadata.origin + parts = [f"采集时间：{solved.metadata.captured_at.isoformat()}"] + if origin: + parts.extend( + f"{label}：{value}" + for label, value in ( + ("来源应用", origin.application_id), + ("来源标题", origin.title), + ("来源地址", origin.url), + ) + if value + ) + items = [(item.block_id, "材料") for item in solved.materials] + if solved.annotation_block_id is not None: + items.append((solved.annotation_block_id, "用户附注")) + for block_id, label in items: + block = await BlockService.get(block_id) + if block is None: + raise ValueError("采集内容已不存在") + text = await ResolverManager.get(block).get_text( + context=context, refresh=refresh, materialize_missing=materialize_missing + ) + parts.append(f"{label}：\n{text}") + return "\n\n".join(parts) + + async def get_label(self, *, refresh: bool = False) -> str: + metadata = CaptureMetadata.model_validate_json( + await self.get_raw_content(refresh=refresh) + ) + return format_label( + "随手收集", + (metadata.origin.title if metadata.origin else None) or metadata.entry, + ) diff --git a/extensions/capture/schema.py b/extensions/capture/schema.py new file mode 100644 index 0000000..a43ca1f --- /dev/null +++ b/extensions/capture/schema.py @@ -0,0 +1,78 @@ +"""Capture v1 wire and graph contracts shared with native and web Peers.""" + +from datetime import UTC +from typing import Annotated, Literal +from uuid import UUID + +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, field_validator + + +MAX_MATERIAL_BYTES = 32 * 1024 * 1024 +MAX_REQUEST_BYTES = 48 * 1024 * 1024 +CAPTURE_RESOLVER = "extensions.capture.capture.v1" +ATTACHMENT_RESOLVER = "extensions.capture.attachment.v1" +CAPTURE_CAPABILITY = "extensions.capture.submit.v1" +ImageMIME = Literal["image/png", "image/jpeg", "image/gif", "image/webp"] + + +class ContractModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class CaptureOrigin(ContractModel): + application_id: str | None = Field(default=None, max_length=1024) + title: str | None = Field(default=None, max_length=4096) + url: str | None = Field(default=None, max_length=8192) + + +class CaptureMetadata(ContractModel): + captured_at: AwareDatetime + entry: Literal["selection", "paste", "screenshot", "manual"] + origin: CaptureOrigin | None = None + + @field_validator("captured_at") + @classmethod + def utc_time(cls, value: AwareDatetime) -> AwareDatetime: + return value.astimezone(UTC) + + +class TextMaterial(ContractModel): + kind: Literal["text"] + text: str = Field(min_length=1, max_length=MAX_MATERIAL_BYTES) + + +class ImageMaterial(ContractModel): + kind: Literal["image"] + mime_type: ImageMIME + data_base64: str = Field(min_length=1, max_length=MAX_REQUEST_BYTES) + filename: str | None = Field(default=None, max_length=512) + + +Material = Annotated[TextMaterial | ImageMaterial, Field(discriminator="kind")] + + +class CaptureRequest(CaptureMetadata): + submission_id: UUID + materials: tuple[Material, ...] = Field(default=(), max_length=128) + annotation: str | None = Field(default=None, max_length=MAX_MATERIAL_BYTES) + + +class AttachmentMetadata(ContractModel): + mime_type: ImageMIME + filename: str | None = Field(default=None, max_length=512) + + +class SolvedMaterial(ContractModel): + block_id: int + resolver: str + + +class SolvedAttachment(ContractModel): + metadata: AttachmentMetadata + content_block_id: int + + +class SolvedCapture(ContractModel): + metadata: CaptureMetadata + materials: tuple[SolvedMaterial, ...] + annotation_block_id: int | None diff --git a/extensions/capture/submit.py b/extensions/capture/submit.py new file mode 100644 index 0000000..19eb2de --- /dev/null +++ b/extensions/capture/submit.py @@ -0,0 +1,174 @@ +"""Validate one capture intention and atomically persist its result exactly once.""" + +import asyncio +import base64 +import binascii +from dataclasses import dataclass +from hashlib import sha256 +from io import BytesIO +import json +import warnings + +from PIL import Image, UnidentifiedImageError + +from app.business.info_base.commands import persist_graph +from app.business.info_base.storage import StorageManager, WritableStorage +from app.persistence.capture import capture_uow +from app.schemas.capture import CaptureResult +from app.schemas.info_base.main import GraphBlockForm, GraphForm, GraphRelationForm + +from .schema import ( + ATTACHMENT_RESOLVER, + CAPTURE_RESOLVER, + MAX_MATERIAL_BYTES, + AttachmentMetadata, + CaptureMetadata, + CaptureRequest, + ImageMaterial, + TextMaterial, +) + + +class CaptureConflict(ValueError): + pass + + +class InvalidCapture(ValueError): + pass + + +@dataclass(frozen=True) +class PreparedCapture: + digest: str + images: dict[int, bytes] + + +def prepare_capture(request: CaptureRequest) -> PreparedCapture: + """Normalize only protocol representations, never authored text or material order.""" + if not request.materials and not request.annotation: + raise InvalidCapture("至少需要一份材料或附注") + document = request.model_dump(mode="json") + document["captured_at"] = request.captured_at.isoformat(timespec="microseconds") + materials = [] + images: dict[int, bytes] = {} + size = len((request.annotation or "").encode("utf-8")) + for index, material in enumerate(request.materials): + item = material.model_dump(mode="json") + if isinstance(material, TextMaterial): + size += len(material.text.encode("utf-8")) + else: + try: + content = base64.b64decode(material.data_base64, validate=True) + except (ValueError, binascii.Error) as error: + raise InvalidCapture("图片必须使用有效的 Base64 编码") from error + size += len(content) + if size > MAX_MATERIAL_BYTES: + raise InvalidCapture("材料累计超过 32 MiB") + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(BytesIO(content)) as image: + if Image.MIME.get(image.format or "") != material.mime_type: + raise InvalidCapture("图片内容与声明的 MIME 不一致") + image.verify() + except ( + UnidentifiedImageError, + OSError, + SyntaxError, + Image.DecompressionBombError, + Image.DecompressionBombWarning, + ) as error: + raise InvalidCapture("图片格式无效或解码尺寸过大") from error + images[index] = content + del item["data_base64"] + item["data_sha256"] = sha256(content).hexdigest() + if size > MAX_MATERIAL_BYTES: + raise InvalidCapture("材料累计超过 32 MiB") + materials.append(item) + if size > MAX_MATERIAL_BYTES: + raise InvalidCapture("材料累计超过 32 MiB") + document["materials"] = materials + digest = sha256( + json.dumps(document, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode( + "utf-8" + ) + ).hexdigest() + return PreparedCapture(digest, images) + + +async def submit_capture(request: CaptureRequest) -> CaptureResult: + prepared = await asyncio.to_thread(prepare_capture, request) + async with capture_uow() as uow: + await uow.receipts.lock(request.submission_id) + receipt = await uow.receipts.get(request.submission_id) + if receipt is not None: + if receipt.payload_digest != prepared.digest: + raise CaptureConflict("提交身份已用于不同内容；请保留原提交并检查待办") + exists = await uow.graph.blocks.get(receipt.root_block_id) + return CaptureResult( + submission_id=receipt.submission_id, + root_block_id=receipt.root_block_id, + committed_at=receipt.committed_at, + availability="available" if exists is not None else "deleted", + ) + + metadata = CaptureMetadata( + captured_at=request.captured_at, entry=request.entry, origin=request.origin + ) + blocks = [ + GraphBlockForm(id=-1, resolver=CAPTURE_RESOLVER, content=metadata.model_dump_json()) + ] + relations: list[GraphRelationForm] = [] + for index, material in enumerate(request.materials): + local_id = -(len(blocks) + 1) + if isinstance(material, ImageMaterial): + storage = StorageManager.from_record(await uow.graph.storage.get(-4)) + if not isinstance(storage, WritableStorage): + raise TypeError("Capture requires writable PostgreSQL binary storage") + pointer = await storage.create_content(prepared.images[index], uow.graph.storage) + attachment = AttachmentMetadata( + mime_type=material.mime_type, filename=material.filename + ) + blocks.extend( + ( + GraphBlockForm( + id=local_id, + resolver=ATTACHMENT_RESOLVER, + content=attachment.model_dump_json(), + ), + GraphBlockForm( + id=local_id - 1, + resolver="core.image.v1", + content=pointer, + storage=-4, + ), + ) + ) + relations.append( + GraphRelationForm(from_=local_id, to_=local_id - 1, content="content") + ) + else: + blocks.append( + GraphBlockForm(id=local_id, resolver="core.text.v1", content=material.text) + ) + relations.append( + GraphRelationForm(from_=-1, to_=local_id, content=f"material:{index}") + ) + if request.annotation: + local_id = -(len(blocks) + 1) + blocks.append( + GraphBlockForm(id=local_id, resolver="core.text.v1", content=request.annotation) + ) + relations.append(GraphRelationForm(from_=-1, to_=local_id, content="annotation")) + result = await persist_graph( + GraphForm(blocks=tuple(blocks), relations=tuple(relations)), uow.graph + ) + root_id = next(item.id for item in result.blocks if item.local_id == -1) + receipt = await uow.receipts.create(request.submission_id, prepared.digest, root_id) + response = CaptureResult( + submission_id=receipt.submission_id, + root_block_id=receipt.root_block_id, + committed_at=receipt.committed_at, + availability="available", + ) + return response diff --git a/migrations/revision-integrity.json b/migrations/revision-integrity.json index 181e3f6..fa1bc07 100644 --- a/migrations/revision-integrity.json +++ b/migrations/revision-integrity.json @@ -24,6 +24,7 @@ "d41cc84db0c5_add_job_abort_request.py": "fc1fe04942e95c7b8c5f43dbcf369bb88479cb83f8e7dc58528f08caf7a536d2", "d4e6f8a0b2c3_merge_extension_setup_main.py": "2073996504ae5d5cb5d8cc0d468d3c9e93870a44bd5f454986662c22da48fba8", "d9f4e2a1b7c3_adopt_peer_database_protocol.py": "9a163533b5a0619e51bfb94ba7ad0c5c3d168fba5808b08f67531fd8f7e5f263", + "dcf44bc9af73_add_capture_receipts.py": "8bf0ce2993af0d11ce56b7c981c8b749622de45f447f5450378c898537b3c2e2", "e1f4a5b6c7d8_migrate_memos_attachment_v2.py": "5f3691149e06b28378f79eed7eaacd1c415ee6c92fb83b3b5745a429346d5e01", "e5a01f9e69ef_init.py": "a10de6c0818abc593951a34e51a974ffcd999f60ba5afc441f87ac6cef8d869e", "e6a1b2c3d4f5_extension_peer_participation_v2.py": "bf71d955b1e3e27015f412815d020666157eda6af78ed0472ca2b7d38ada872f", diff --git a/migrations/versions/dcf44bc9af73_add_capture_receipts.py b/migrations/versions/dcf44bc9af73_add_capture_receipts.py new file mode 100644 index 0000000..2fb7d19 --- /dev/null +++ b/migrations/versions/dcf44bc9af73_add_capture_receipts.py @@ -0,0 +1,40 @@ +"""add_capture_receipts + +Revision ID: dcf44bc9af73 +Revises: 09a921a95314 +Create Date: 2026-10-21 01:38:42.379663 + +""" +from typing import Sequence + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'dcf44bc9af73' +down_revision: str | Sequence[str] | None = '09a921a95314' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('capture_receipts', + sa.Column('submission_id', sa.Uuid(), nullable=False), + sa.Column('payload_digest', sa.Text(), nullable=False), + sa.Column('root_block_id', sa.Integer(), nullable=False), + sa.Column('committed_at', sa.TIMESTAMP(timezone=True), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False), + sa.CheckConstraint("payload_digest ~ '^[0-9a-f]{64}$'", name='capture_receipt_digest'), + sa.PrimaryKeyConstraint('submission_id'), + schema='inkcre' + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('capture_receipts', schema='inkcre') + # ### end Alembic commands ### diff --git a/tests/extensions/capture/fixtures/capture-v1.json b/tests/extensions/capture/fixtures/capture-v1.json new file mode 100644 index 0000000..d23e627 --- /dev/null +++ b/tests/extensions/capture/fixtures/capture-v1.json @@ -0,0 +1,44 @@ +[ + { + "payload": { + "submission_id": "00000000-0000-4000-8000-000000000001", + "captured_at": "2026-10-21T01:02:03.123Z", + "entry": "paste", + "origin": null, + "materials": [ + { + "kind": "text", + "text": "中文 👩🏽‍💻 é\n/