mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-08-29 04:26:38 +00:00
feat(attachment): add ordered image batch admission
This commit is contained in:
Tianyi Cui
+2
-2
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.md
|
||||
2026-07-22-web-multimodal-image-input-and-durable-attachments.md: 8639a1ab638c85fc01a29083a1b81eacdc2152d4
|
||||
2026-07-22-web-multimodal-image-input-and-durable-attachments.zh.md: 1090edd06a62d70a736c85eb1c7d9e6edba187c8
|
||||
2026-07-22-web-multimodal-image-input-and-durable-attachments.md: c12821f9d01be12117e987a2612f3953a8eaae20
|
||||
2026-07-22-web-multimodal-image-input-and-durable-attachments.zh.md: 17626bc7696e8e3b6cad01c7fec3b5f0a718921a
|
||||
|
||||
+22
-7
@@ -16,7 +16,7 @@ Peer products converge on an attachment rail above the editor, but their storage
|
||||
|
||||
## Decision
|
||||
|
||||
Pasted or dropped raster images are the Web composer's first consumer of a durable attachment capability. Unsent files remain temporary client-owned draft state. The host validates and durably commits every accepted user image before appending its message event. A provider adapter that produces structured image output must durably commit the output before appending its assistant block. Canonical user and assistant content contains only role-neutral `ImageBlock` references.
|
||||
Pasted or dropped raster images are the Web composer's first consumer of a durable attachment capability. Unsent files remain temporary client-owned draft state. Every rich-content intake adapter decodes its wire blocks, proves route capability, and delegates the complete image batch to the attachment service before appending its message event. A provider adapter that produces structured image output must durably commit the output before appending its assistant block. Canonical user and assistant content contains only role-neutral `ImageBlock` references.
|
||||
|
||||
Version one supports PNG, JPEG, WebP, and GIF paste and drag-and-drop, image-only or mixed prompts, historical user and assistant image rendering, and original-image preview on a single click (display and interaction specifics superseded in part by the [attachment-display alignment note](2026-08-11-web-attachment-display-alignment.md)). File picking, generic files, PDF, audio, video, image copying, and a custom context menu remain separate follow-ups.
|
||||
|
||||
@@ -114,7 +114,7 @@ type PromptInputPart =
|
||||
}
|
||||
```
|
||||
|
||||
Base64 crosses JSON-RPC once and is discarded after persistence. The host validates canonical base64, image count, aggregate bytes, individual bytes, the declared MIME against a fully decoded raster, intrinsic dimensions, and decoded-pixel count. It awaits the seam's storage-free `validateImage` for every batch member before saving any member, so one malformed image cannot strand the batch's valid members as unreferenced objects. Storage commits then run in submission order to bound full-raster decoder memory. If a later storage I/O operation fails, the host appends no user event, but an earlier immutable content-addressed object may remain unreferenced; version one leaves cleanup to future reference-aware garbage collection instead of adding destructive rollback to the deduplicated store. Only after every image succeeds does it call the agent with normalized text and durable image blocks in the submitted order. A failure exposes no attachment path or raw bytes.
|
||||
Base64 crosses a wire boundary once and is discarded after persistence. Each front door validates canonical base64 and declared MIME shape, then calls `AttachmentStore.saveImages()` with the whole decoded batch. The service owns image count, aggregate bytes, individual bytes, fully decoded raster/MIME agreement, intrinsic dimensions, and decoded-pixel count; it validates every batch member before saving any member, so one malformed image cannot strand the batch's valid members as unreferenced objects. Storage commits then run in submission order to bound full-raster decoder memory. If a later storage I/O operation fails, the caller appends no model-visible event and receives no partial references, but an earlier immutable content-addressed object may remain unreferenced; version one leaves cleanup to future reference-aware garbage collection instead of adding destructive rollback to the deduplicated store. Only after every image succeeds does the front door call the agent with normalized text and durable image blocks in wire order. A failure exposes no attachment path or raw bytes.
|
||||
|
||||
`session.attachment` is a read-only, session-scoped endpoint. The host serves bytes only when a durable event in that session references the requested attachment identifier. The client deduplicates loads by session and attachment identifier while that session is rendered, revokes resolved URLs on rendered-session disposal, and rejects invalidated late loads before allocating an object URL so an unmounted session or disposed service cannot repopulate the cache.
|
||||
|
||||
@@ -128,7 +128,7 @@ The Pi-AI adapter is the first visual-input route: it resolves `ctx.attachments`
|
||||
|
||||
Core supports structured assistant image blocks, but no current production provider route is certified for image output. Any future output-capable adapter must retrieve provider bytes under bounded size and time policy, validate them through the same attachment service, persist them, and only then publish the atomic `ImageBlock`. A URL in assistant Markdown remains text and is never downloaded automatically.
|
||||
|
||||
Provider-neutral token estimation does not guess visual pricing from image dimensions; provider-reported usage remains authoritative. ACP renders an explicit image marker until that protocol API gains native image support rather than silently omitting the block.
|
||||
Provider-neutral token estimation does not guess visual pricing from image dimensions; provider-reported usage remains authoritative. ACP advertises image prompts only when its configured exact route and attachment deployment can accept them, persists inline input before publishing the user event, and re-reads committed assistant image references for native ACP image updates. MCP keeps canonical raw blocks for programmatic callers while projecting admitted images to durable core blocks; Code Mode carries any settled image-bearing sub-result through the outer result as logged source-attributed context.
|
||||
|
||||
Compaction replays the selected conversation prefix, including image references, into the configured summarization route. A visual-capable route resolves those references through its adapter; a text-only route fails explicitly instead of silently dropping the visual context. The synthesized checkpoint remains text-only, and `compact-basic` rejects image summary output with `UNSUPPORTED_CONTENT`.
|
||||
|
||||
@@ -148,22 +148,24 @@ Malformed base64, unsupported or mismatched media, truncated image payloads, exc
|
||||
|
||||
| Surface | Responsibility |
|
||||
| --- | --- |
|
||||
| `packages/attachment/attachment` | Opaque attachment identifier, image reference, limits, failures, and `ctx.attachments` service. |
|
||||
| `packages/attachment/attachment` | Opaque attachment identifier, image reference, limits, failures, and single/batch admission through `ctx.attachments`. |
|
||||
| `packages/attachment/attachment-local` | Private content-addressed storage, complete raster decoding, integrity verification, and configuration. |
|
||||
| `packages/llm/llm` | Role-neutral `ImageBlock` and input-modality metadata. |
|
||||
| `packages/llm/llm-pi-ai` | Resolve durable supported image input into native provider content. |
|
||||
| `packages/llm/llm-deepseek` | Reject image content explicitly. |
|
||||
| `packages/compact/compact-basic` | Preserve images in summary input and reject non-text checkpoint output explicitly. |
|
||||
| `packages/host/apiproxy` and `packages/bundle/base` | Narrow upload wire, persist-before-event ordering, session-authorized reads, limits and model preflight, plus default profile composition. |
|
||||
| `packages/host/apiproxy` and `packages/bundle/base` | Narrow upload wire, routed-model preflight, delegation to shared batch admission, persist-before-event ordering, session-authorized reads, and default profile composition. |
|
||||
| `packages/client/connection` and `packages/client/runtime` | Bounded request buffering, wire types, fixture images, prompt uploads, attachment reads, and durable-reference folding. |
|
||||
| `packages/client/ui-conversation` | Per-session draft images, attachment rail, user and assistant image controls, and original preview. |
|
||||
| `packages/acp/acp` | Explicit fallback rendering for image blocks. |
|
||||
| `packages/acp/acp` | Conditional native image capability, atomic inline-image admission, and verified assistant-image delivery. |
|
||||
| `packages/mcp/mcp-client` | Lossless canonical MCP results plus capability-gated durable image projection and explicit diagnostics for unsupported rich blocks. |
|
||||
| `packages/core/tools` | Generic Code Mode forwarding of settled image-bearing sub-results after the outer result. |
|
||||
|
||||
The attachment packages form the interface/implementation side of one capability seam. Composer behavior stays in the conversation object layer, provider conversion stays in adapters, and no change is required in `agent-loop`.
|
||||
|
||||
### Implementation
|
||||
|
||||
The implemented slice includes the attachment seam, role-neutral image block, Pi-AI input conversion, DeepSeek rejection, durable host ordering, Web upload/read protocol, current image-limit enforcement, bounded Web request bodies, in-memory draft images, paste/drop rail, user and assistant history rendering, single-click preview, compaction handling, and keyless assembled Web coverage.
|
||||
The implemented slice includes the attachment seam and shared batch admission, role-neutral image block, Pi-AI input conversion, DeepSeek rejection, durable Web/ACP/MCP ordering, Web upload/read protocol, conditional ACP image wire support, lossless MCP canonical results with durable image projection, generic Code Mode rich-result forwarding, current image-limit enforcement, bounded Web request bodies, in-memory draft images, paste/drop rail, user and assistant history rendering, single-click preview, compaction handling, and keyless assembled Web and ACP coverage.
|
||||
|
||||
No compatibility shim is required for the pre-release prompt wire; all call sites and fixtures change with the introducing slice.
|
||||
|
||||
@@ -193,12 +195,25 @@ Composer presentation can use a generic attachment rail, but provider semantics
|
||||
|
||||
UI state can be stale and does not protect direct SDK, ACP, replay, or uncatalogued model paths. Silent filtering changes user intent. Provider enforcement remains mandatory, while UI checks are optional earlier feedback.
|
||||
|
||||
### Add a generic RichContent service above the core content vocabulary
|
||||
|
||||
Rejected because the core already has the role-neutral `ContentBlock` vocabulary and attachment references. A second generic service would duplicate ordering, capability, logging, and lifetime semantics while still requiring each wire adapter to parse its own protocol. Narrow image adapters around the existing core preserve ownership and leave audio/resources to earn their own lifecycle contracts.
|
||||
|
||||
### Normalize MCP results into core content as the canonical tool value
|
||||
|
||||
Rejected because Code Mode and programmatic callers need the complete MCP JSON blocks and optional `structuredContent`; replacing that value with a Native projection would make the bridge lossy. MCP retains the protocol value and prepares a separate model projection, with final post-execute policy remaining authoritative.
|
||||
|
||||
### Perform attachment reads and writes inside synchronous output renderers
|
||||
|
||||
Rejected because tool renderers are pure, synchronous, and replayable. MCP prepares image projection during async execution and installs it only at the registry's finalization boundary; ACP performs async admission and output conversion in its transport lifecycle. Code Mode forwarding observes the already settled final content instead of giving individual image tools private parent-token behavior.
|
||||
|
||||
## Testing
|
||||
|
||||
- Storage tests cover content-addressed deduplication, private permissions, admission failures, corruption/missing-object failures, and reading history after deployment limits are lowered.
|
||||
- Host and protocol tests cover persist-before-event ordering, absence of base64 in logs, session-scoped authorization, capability rejection, upload limits, bounded HTTP request bodies, image-admission/model-selection races (queued and steering placements), pending publication, idle release without publication, text-only queue edits, and selection against current derived history after compaction.
|
||||
- Client unit tests cover paste and drop, mixed clipboard text, image-only send, draft restoration, ordering, draft/session-scope/application object-URL cleanup, and a deferred historical read that completes after disposal; the keyless assembled built-client lane (`apps/web/tests/image-display.snapshot.ts`, `DSH_EXAMPLE_MODE=lib pnpm run test:snapshot`) covers the historical user and assistant galleries over the authorized attachment route, the original-size lightbox, and the composer paste rail.
|
||||
- Adapter and compaction tests cover native Pi-AI image conversion, late attachment-service composition, text-only rejection, recursively nested tool-result images, preserved summary input, and explicit image-output rejection.
|
||||
- Attachment, MCP, ACP, and Code Mode tests cover all-member validation before writes, mixed text/image ordering, no inline base64 in durable events, exact route-capability gates, explicit unsupported-content diagnostics, post-execute replacement/block precedence, cancellation during admission, verified assistant-image delivery, and generic nested-image forwarding. A keyless assembled ACP snapshot sends a real inline PNG and pins only its durable reference in the session log.
|
||||
- A credentialed real-API test sends a PNG through the Anthropic `claude-opus-4-8` route and requires the model to identify its QR code.
|
||||
- The current production adapter set has no certified image-output route; output-provider certification remains outside version one.
|
||||
|
||||
|
||||
+22
-7
@@ -16,7 +16,7 @@ Status: implemented
|
||||
|
||||
## 决策
|
||||
|
||||
粘贴或拖放的光栅图片是 Web 输入区对持久附件能力的首个应用场景。未发送文件仍是由客户端持有的临时草稿状态。宿主在追加相应消息事件前，校验并持久提交每张已接受的用户图片。生成结构化图片输出的提供方适配器在追加相应助手块前，也必须持久提交输出。规范用户内容与助手内容只包含角色无关的 `ImageBlock` 引用。
|
||||
粘贴或拖放的光栅图片是 Web 输入区对持久附件能力的首个应用场景。未发送文件仍是由客户端持有的临时草稿状态。每个丰富内容接入适配器都会解码自身协议块、证明路由能力，并在追加消息事件前把完整图片批次委托给附件服务。生成结构化图片输出的提供方适配器在追加相应助手块前，也必须持久提交输出。规范用户内容与助手内容只包含角色无关的 `ImageBlock` 引用。
|
||||
|
||||
第一版支持粘贴和拖放 PNG、JPEG、WebP 与 GIF，支持仅图片或混合提示词，支持渲染历史用户图片与助手图片，并支持单击预览原图（展示与交互细节部分由[附件展示对齐 Note](2026-08-11-web-attachment-display-alignment.md)取代）。文件选择、通用文件、PDF、音频、视频、图片复制和自定义上下文菜单仍分别作为后续工作。
|
||||
|
||||
@@ -114,7 +114,7 @@ type PromptInputPart =
|
||||
}
|
||||
```
|
||||
|
||||
Base64 只跨越一次 JSON-RPC，并在持久化后丢弃。宿主会校验规范 base64、图片数量、总字节数、单张图片字节数、声明的 MIME 与完整解码后的光栅图片是否一致、固有尺寸和解码像素数。它会在保存任何成员之前，等待服务边界上不触碰存储的 `validateImage` 完成对每个批次成员的校验，因此一张畸形图片不会把批次中的有效成员留成无引用对象。随后按提交顺序执行存储提交，以限制完整光栅解码器的内存占用。如果后续存储 I/O 操作失败，宿主不会追加用户事件，但先前的不可变内容寻址对象可能保持无引用状态；第一版将清理留给未来按引用感知的垃圾回收，而不向去重存储添加破坏性回滚。只有每张图片都成功后，宿主才会用规范化文本和按提交顺序排列的持久图片块调用 agent。失败时不公开任何附件路径或原始字节。
|
||||
Base64 只跨越一次协议边界，并在持久化后丢弃。每个入口都会校验规范 base64 与声明的 MIME 形状，再用完整解码批次调用 `AttachmentStore.saveImages()`。服务负责图片数量、总字节数、单张图片字节数、声明 MIME 与完整解码后的光栅图片是否一致、固有尺寸和解码像素数；它会在保存任何成员之前校验每个批次成员，因此一张畸形图片不会把批次中的有效成员留成无引用对象。随后按提交顺序执行存储提交，以限制完整光栅解码器的内存占用。如果后续存储 I/O 操作失败，调用方不会追加模型可见事件，也不会收到部分引用，但先前的不可变内容寻址对象可能保持无引用状态；第一版将清理留给未来按引用感知的垃圾回收，而不向去重存储添加破坏性回滚。只有每张图片都成功后，入口才会用规范化文本和按协议顺序排列的持久图片块调用 agent。失败时不公开任何附件路径或原始字节。
|
||||
|
||||
`session.attachment` 是只读且限定于会话作用域的端点。只有该会话中的持久事件引用了所请求的附件标识符，宿主才提供字节。会话处于渲染状态时，客户端会按会话和附件标识符对加载操作去重；已渲染会话释放时会撤销已解析的 URL，并在分配对象 URL 前拒绝已失效的延迟加载，以免已卸载的会话或已释放的服务重新写入缓存。
|
||||
|
||||
@@ -128,7 +128,7 @@ Pi-AI 适配器是首条视觉输入路径：它在请求时解析 `ctx.attachme
|
||||
|
||||
核心层支持结构化助手图片块，但当前没有任何生产提供方路径通过图片输出认证。未来任何支持输出的适配器都必须在有界的大小和时间策略下获取提供方字节，通过同一个附件服务校验并持久化字节，之后才能以原子方式发布 `ImageBlock`。助手 Markdown 中的 URL 仍是文本，绝不自动下载。
|
||||
|
||||
提供方无关的 token 估算不会根据图片尺寸猜测视觉定价；提供方返回的用量仍是权威值。在 ACP（Agent Client Protocol）接口原生支持图片前，ACP 会渲染明确的图片标记，而不是静默省略该块。
|
||||
提供方无关的 token 估算不会根据图片尺寸猜测视觉定价；提供方返回的用量仍是权威值。只有配置的确切路由与附件部署可以接受图片时，ACP（Agent Client Protocol）才公布图片提示词能力；它会在发布用户事件前持久化内联输入，并重新读取已提交的助手图片引用来发送原生 ACP 图片更新。MCP 为程序化调用方保留规范原始块，同时把已准入图片投影为持久核心块；Code Mode 会把任何已经结算且含图片的子结果经外层结果转运为带来源归属且写入日志的上下文。
|
||||
|
||||
压缩会把选定的会话前缀（包含图片引用）回放到已配置的摘要生成路径中。支持视觉的路径会通过适配器解析这些引用；仅文本路径会明确失败，而不是静默丢弃视觉上下文。合成的检查点仍仅包含文本，`compact-basic` 会以 `UNSUPPORTED_CONTENT` 拒绝包含图片的摘要输出。
|
||||
|
||||
@@ -148,22 +148,24 @@ Pi-AI 适配器是首条视觉输入路径：它在请求时解析 `ctx.attachme
|
||||
|
||||
| 接口 | 职责 |
|
||||
| --- | --- |
|
||||
| `packages/attachment/attachment` | 不透明附件标识符、图片引用、限制、错误和 `ctx.attachments` 服务。 |
|
||||
| `packages/attachment/attachment` | 不透明附件标识符、图片引用、限制、错误，以及通过 `ctx.attachments` 提供的单张／批量准入。 |
|
||||
| `packages/attachment/attachment-local` | 私有内容寻址存储、完整光栅解码、完整性校验和配置。 |
|
||||
| `packages/llm/llm` | 角色无关的 `ImageBlock` 和输入模态元数据。 |
|
||||
| `packages/llm/llm-pi-ai` | 将持久且受支持的图片输入解析为提供方原生内容。 |
|
||||
| `packages/llm/llm-deepseek` | 明确拒绝图片内容。 |
|
||||
| `packages/compact/compact-basic` | 在摘要输入中保留图片，并明确拒绝非文本检查点输出。 |
|
||||
| `packages/host/apiproxy` 和 `packages/bundle/base` | 范围狭窄的上传协议、先持久化再追加事件的顺序、会话授权读取、限制和模型前置检查，以及默认 profile 组合。 |
|
||||
| `packages/host/apiproxy` 和 `packages/bundle/base` | 范围狭窄的上传协议、路由模型前置检查、委托共享批量准入、先持久化再追加事件的顺序、会话授权读取，以及默认 profile 组合。 |
|
||||
| `packages/client/connection` 和 `packages/client/runtime` | 有界请求缓冲、协议类型、fixture（测试前置数据）图片、提示词上传、附件读取和持久引用折叠。 |
|
||||
| `packages/client/ui-conversation` | 每个会话的草稿图片、附件栏、用户与助手图片控件和原图预览。 |
|
||||
| `packages/acp/acp` | 图片块的明确兜底渲染。 |
|
||||
| `packages/acp/acp` | 条件式原生图片能力、原子内联图片准入，以及经过校验的助手图片交付。 |
|
||||
| `packages/mcp/mcp-client` | 无损规范 MCP 结果、经能力门禁的持久图片投影，以及针对不受支持丰富块的明确诊断。 |
|
||||
| `packages/core/tools` | 在外层结果之后通用转发已经结算且含图片的 Code Mode 子结果。 |
|
||||
|
||||
附件包（package）构成一个能力服务边界的接口与实现侧。输入区行为留在会话对象层，提供方转换留在适配器中，无需修改 `agent-loop`。
|
||||
|
||||
### 实现
|
||||
|
||||
已实现的范围包括附件服务边界、角色无关的图片块、Pi-AI 输入转换、DeepSeek 拒绝、宿主持久化顺序、Web 上传与读取协议、当前图片限制执行、大小受限的 Web 请求体、内存草稿图片、粘贴与拖放附件栏、用户与助手历史图片渲染、单击预览、压缩处理，以及组装后无需密钥的 Web 覆盖。
|
||||
已实现的范围包括附件服务边界与共享批量准入、角色无关的图片块、Pi-AI 输入转换、DeepSeek 拒绝、Web／ACP／MCP 的持久化顺序、Web 上传与读取协议、条件式 ACP 图片协议支持、无损 MCP 规范结果与持久图片投影、通用 Code Mode 丰富结果转发、当前图片限制执行、大小受限的 Web 请求体、内存草稿图片、粘贴与拖放附件栏、用户与助手历史图片渲染、单击预览、压缩处理，以及组装后无需密钥的 Web 与 ACP 覆盖。
|
||||
|
||||
预发布提示词协议不需要兼容包装层；引入相应切片时会同时修改所有调用点和 fixture。
|
||||
|
||||
@@ -193,12 +195,25 @@ Pi-AI 适配器是首条视觉输入路径：它在请求时解析 `ctx.attachme
|
||||
|
||||
UI 状态可能陈旧，也无法保护直接 SDK、ACP、回放或未收录模型的路径。静默过滤会改变用户意图。提供方强制检查仍是必需项，UI 检查则是可选的提前反馈。
|
||||
|
||||
### 在核心内容词汇之上添加通用 RichContent 服务
|
||||
|
||||
不予采用，因为核心已经拥有角色无关的 `ContentBlock` 词汇与附件引用。第二套通用服务会重复顺序、能力、日志和生命周期语义，同时每个协议适配器仍需解析自身协议。围绕现有核心构建范围狭窄的图片适配器，可以保持归属清晰，并让音频／资源在确有需要时建立自己的生命周期契约。
|
||||
|
||||
### 把 MCP 结果规范化为核心内容，并将其作为规范工具值
|
||||
|
||||
不予采用，因为 Code Mode 和程序化调用方需要完整 MCP JSON 块及可选 `structuredContent`；用 Native 投影替换该值会让桥接有损。MCP 保留协议值，并另行准备模型投影；最终 post-execute 策略仍具有权威性。
|
||||
|
||||
### 在同步输出渲染器中执行附件读写
|
||||
|
||||
不予采用，因为工具渲染器必须纯净、同步且可回放。MCP 在异步执行期间准备图片投影，只在注册表最终化边界安装；ACP 在自己的传输生命周期中执行异步准入和输出转换。Code Mode 转发观察已经结算的最终内容，而不是让各图片工具各自处理私有父 token 行为。
|
||||
|
||||
## 测试
|
||||
|
||||
- 存储测试覆盖内容寻址去重、私有权限、准入失败、对象损坏或缺失时的失败，以及收紧部署限制后读取历史数据。
|
||||
- 宿主与协议测试覆盖先持久化再追加事件的顺序、日志中不含 base64、会话作用域授权、能力拒绝、上传限制、大小受限的 HTTP 请求体、图片准入与模型选择的竞态（排队与 steering 两种放置）、待发布状态、未发布即空闲时的门槛释放、仅文本的队列编辑，以及压缩后依据当前派生历史进行的选择。
|
||||
- 客户端单元测试覆盖粘贴与拖放、混合剪贴板文本、仅图片发送、草稿恢复、顺序、草稿、会话作用域和应用层级的对象 URL 清理，以及一项在释放后才完成的延迟历史读取；keyless 的组装后构建产物通道（`apps/web/tests/image-display.snapshot.ts`，`DSH_EXAMPLE_MODE=lib pnpm run test:snapshot`）覆盖经授权附件路由渲染的历史用户与助手图片画廊、原图 lightbox，以及 composer 粘贴缩略图条。
|
||||
- 适配器与压缩测试覆盖 Pi-AI 原生图片转换、后置附件服务组合、仅文本拒绝、递归嵌套在工具结果中的图片、保留摘要输入，以及明确拒绝图片输出。
|
||||
- 附件、MCP、ACP 与 Code Mode 测试覆盖写入前校验全部成员、图文混合顺序、持久事件不含内联 base64、确切路由能力门禁、明确的不支持内容诊断、post-execute 替换／阻止优先级、准入期间取消、经过校验的助手图片交付，以及通用嵌套图片转发。组装后的无密钥 ACP 快照发送真实内联 PNG，并在会话日志中只固定其持久引用。
|
||||
- 需要凭据的实际 API 测试会通过 Anthropic `claude-opus-4-8` 路径发送一张 PNG，并要求模型识别其中的二维码。
|
||||
- 当前生产适配器集合没有经过认证的图片输出路由；输出提供方认证仍不在第一版范围内。
|
||||
|
||||
|
||||
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write docs/subsystems/attachment.md
|
||||
attachment.md: ff7f14ceae8d4f8055d5cfd4367373729dc5ecbc
|
||||
attachment.zh.md: d7a9527788588d5504fdeffd8ae7849b0f8b1378
|
||||
attachment.md: c769d9e608b9e1ab12a5960ca2629a297853bf26
|
||||
attachment.zh.md: d07ea722656fafd93793850b8dd268cb14e6856b
|
||||
|
||||
@@ -94,6 +94,16 @@ Immutable binary attachment service. Implementations validate bytes before publi
|
||||
*/
|
||||
abstract validateImage(input: SaveImageAttachment): Promise<void>
|
||||
|
||||
/**
|
||||
* Validate one ordered image batch before committing any member.
|
||||
* Validation failures start no writes; storage failures return no partial
|
||||
* references, although already published content-addressed objects may stay
|
||||
* unreachable until a future retention policy collects them.
|
||||
* @param inputs - encoded images in their owning message order.
|
||||
* @returns durable references in the exact input order.
|
||||
*/
|
||||
async saveImages(inputs: readonly SaveImageAttachment[]): Promise<readonly ImageAttachmentRef[]>
|
||||
|
||||
/**
|
||||
* Validate and durably commit one image before its owning session event is appended.
|
||||
* @param input - encoded bytes, declared media type, and optional display name.
|
||||
@@ -111,5 +121,5 @@ abstract saveImage(input: SaveImageAttachment): Promise<ImageAttachmentRef>
|
||||
abstract readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise<StoredImageAttachment>
|
||||
```
|
||||
|
||||
Source: [`packages/attachment/attachment/src/index.ts:29`](../../packages/attachment/attachment/src/index.ts)
|
||||
Source: [`packages/attachment/attachment/src/index.ts:30`](../../packages/attachment/attachment/src/index.ts)
|
||||
<!-- END GENERATED cordis-surface -->
|
||||
|
||||
@@ -94,6 +94,16 @@ Immutable binary attachment service. Implementations validate bytes before publi
|
||||
*/
|
||||
abstract validateImage(input: SaveImageAttachment): Promise<void>
|
||||
|
||||
/**
|
||||
* Validate one ordered image batch before committing any member.
|
||||
* Validation failures start no writes; storage failures return no partial
|
||||
* references, although already published content-addressed objects may stay
|
||||
* unreachable until a future retention policy collects them.
|
||||
* @param inputs - encoded images in their owning message order.
|
||||
* @returns durable references in the exact input order.
|
||||
*/
|
||||
async saveImages(inputs: readonly SaveImageAttachment[]): Promise<readonly ImageAttachmentRef[]>
|
||||
|
||||
/**
|
||||
* Validate and durably commit one image before its owning session event is appended.
|
||||
* @param input - encoded bytes, declared media type, and optional display name.
|
||||
@@ -111,5 +121,5 @@ abstract saveImage(input: SaveImageAttachment): Promise<ImageAttachmentRef>
|
||||
abstract readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise<StoredImageAttachment>
|
||||
```
|
||||
|
||||
Source: [`packages/attachment/attachment/src/index.ts:29`](../../packages/attachment/attachment/src/index.ts)
|
||||
Source: [`packages/attachment/attachment/src/index.ts:30`](../../packages/attachment/attachment/src/index.ts)
|
||||
<!-- END GENERATED cordis-surface -->
|
||||
|
||||
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write packages/attachment/attachment/README.md
|
||||
README.md: baeeca0cf939f1a3d4608769b362d532507b90f5
|
||||
README.zh.md: 238b90794c510e71fffe34d62b044a5c2ece8a6e
|
||||
README.md: c0a86d324da8c27ec386103f40ac50534c2483d7
|
||||
README.zh.md: 562c8af0df20634ac2072c4a8d422b4e0b4b47dd
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
English | [中文](README.zh.md)
|
||||
|
||||
The durable attachment seam. `ctx.attachments` validates and atomically commits immutable image bytes, then returns a serializable `ImageAttachmentRef`; consumers never persist browser paths, object URLs, provider URLs, or base64 in session events.
|
||||
The durable attachment seam. `ctx.attachments` validates and durably commits immutable image bytes, then returns a serializable `ImageAttachmentRef`; consumers never persist browser paths, object URLs, provider URLs, or base64 in session events.
|
||||
|
||||
Unsent composer images remain browser-owned temporary drafts. `validateImage` runs the same admission policy without persisting; batch writers validate every member first so a malformed member cannot strand earlier members as unreferenced objects. `saveImage` commits each accepted image before any model-visible session event is published, and `readImage` verifies the content-addressed object against its logged metadata. Callers may cancel `readImage`; implementations observe cancellation around backend and verification work and preserve it instead of translating it into a storage failure.
|
||||
Unsent composer images remain browser-owned temporary drafts. `validateImage` runs the same admission policy without persisting. `saveImages` owns batch count and aggregate-byte limits, validates every member before writing any member, then commits in order and returns references only after the complete batch succeeds. A later storage failure returns no partial references, although an earlier immutable content-addressed object may remain unreachable until reference-aware garbage collection exists. `saveImage` commits one accepted image before any model-visible session event is published, and `readImage` verifies the content-addressed object against its logged metadata. Callers may cancel `readImage`; implementations observe cancellation around backend and verification work and preserve it instead of translating it into a storage failure.
|
||||
|
||||
## Model Experience
|
||||
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
[English](README.md) | 中文
|
||||
|
||||
持久附件服务边界。`ctx.attachments` 校验并以原子方式提交不可变图片字节，随后返回可序列化的 `ImageAttachmentRef`；消费方绝不会在会话事件中持久保存浏览器路径、对象 URL、提供方 URL 或 base64。
|
||||
持久附件服务边界。`ctx.attachments` 校验并持久提交不可变图片字节，随后返回可序列化的 `ImageAttachmentRef`；消费方绝不会在会话事件中持久保存浏览器路径、对象 URL、提供方 URL 或 base64。
|
||||
|
||||
未发送的输入区图片仍是由浏览器持有的临时草稿。`validateImage` 运行相同的准入策略，但不执行持久化；批量写入方会先校验每个成员，避免某个格式错误的成员使较早的成员成为无引用对象。`saveImage` 会在发布任何模型可见的会话事件前提交每张已接受的图片，`readImage` 则根据已记录的元数据校验内容寻址对象。调用方可以取消 `readImage`；实现会在后端读取与校验工作的边界观察取消，并保留取消语义，而不会将其转换为存储失败。
|
||||
未发送的输入区图片仍是由浏览器持有的临时草稿。`validateImage` 运行相同的准入策略，但不执行持久化。`saveImages` 负责批次图片数量和总字节限制，先校验全部成员，再按顺序提交，并且只在完整批次成功后返回引用。后续存储失败不会返回部分引用，但较早写入的不可变内容寻址对象可能保持不可达，直至具备按引用感知的垃圾回收。`saveImage` 会在发布任何模型可见的会话事件前提交一张已接受的图片，`readImage` 则根据已记录的元数据校验内容寻址对象。调用方可以取消 `readImage`；实现会在后端读取与校验工作的边界观察取消，并保留取消语义，而不会将其转换为存储失败。
|
||||
|
||||
## 模型体验
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
/** Durable attachment storage seam (`ctx.attachments`). @module @deepseek-ai/dsh-attachment */
|
||||
|
||||
import { Context, Service } from '@deepseek-ai/cordis'
|
||||
import { AttachmentError } from './error.ts'
|
||||
import type {
|
||||
ImageAttachmentLimits,
|
||||
ImageAttachmentRef,
|
||||
@@ -42,6 +43,35 @@ export abstract class AttachmentStore extends Service {
|
||||
*/
|
||||
abstract validateImage(input: SaveImageAttachment): Promise<void>
|
||||
|
||||
/**
|
||||
* Validate one ordered image batch before committing any member.
|
||||
* Validation failures start no writes; storage failures return no partial
|
||||
* references, although already published content-addressed objects may stay
|
||||
* unreachable until a future retention policy collects them.
|
||||
* @param inputs - encoded images in their owning message order.
|
||||
* @returns durable references in the exact input order.
|
||||
*/
|
||||
async saveImages(inputs: readonly SaveImageAttachment[]): Promise<readonly ImageAttachmentRef[]> {
|
||||
const { maxImagesPerMessage, maxMessageImageBytes, mediaTypes } = this.imageLimits
|
||||
if (inputs.length > maxImagesPerMessage) {
|
||||
throw new AttachmentError('Image batch exceeds the configured image-count limit.', 'TOO_MANY_IMAGES')
|
||||
}
|
||||
const totalBytes = inputs.reduce((sum, input) => sum + input.data.byteLength, 0)
|
||||
if (totalBytes > maxMessageImageBytes) {
|
||||
throw new AttachmentError('Image batch exceeds the configured aggregate image-byte limit.', 'IMAGES_TOO_LARGE')
|
||||
}
|
||||
for (const input of inputs) {
|
||||
if (!mediaTypes.includes(input.mediaType)) {
|
||||
throw new AttachmentError(`Image type ${input.mediaType} is not accepted by this deployment.`, 'UNSUPPORTED_IMAGE_TYPE')
|
||||
}
|
||||
}
|
||||
for (const input of inputs) await this.validateImage(input)
|
||||
|
||||
const refs: ImageAttachmentRef[] = []
|
||||
for (const input of inputs) refs.push(await this.saveImage(input))
|
||||
return refs
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate and durably commit one image before its owning session event is appended.
|
||||
* @param input - encoded bytes, declared media type, and optional display name.
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import { Context } from '@deepseek-ai/cordis'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import AttachmentStore, {
|
||||
AttachmentId,
|
||||
type ImageAttachmentRef,
|
||||
type ImageMediaType,
|
||||
type SaveImageAttachment,
|
||||
type StoredImageAttachment,
|
||||
} from '../src/index.ts'
|
||||
|
||||
const LIMITS = {
|
||||
maxImageBytes: 4,
|
||||
maxImagesPerMessage: 2,
|
||||
maxMessageImageBytes: 5,
|
||||
maxImagePixels: 4,
|
||||
mediaTypes: ['image/png'] as const,
|
||||
}
|
||||
|
||||
class RecordingStore extends AttachmentStore {
|
||||
readonly imageLimits = LIMITS
|
||||
readonly calls: string[] = []
|
||||
rejectValidationAt: number | undefined
|
||||
rejectSaveAt: number | undefined
|
||||
|
||||
async validateImage(input: SaveImageAttachment): Promise<void> {
|
||||
const value = input.data[0] ?? 0
|
||||
this.calls.push(`validate:${value}`)
|
||||
if (value === this.rejectValidationAt) throw new Error(`invalid:${value}`)
|
||||
}
|
||||
|
||||
async saveImage(input: SaveImageAttachment): Promise<ImageAttachmentRef> {
|
||||
const value = input.data[0] ?? 0
|
||||
this.calls.push(`save:${value}`)
|
||||
if (value === this.rejectSaveAt) throw new Error(`write:${value}`)
|
||||
return {
|
||||
attachmentId: AttachmentId(`sha256:${String(value).padStart(64, '0')}`),
|
||||
mediaType: input.mediaType,
|
||||
bytes: input.data.byteLength,
|
||||
width: 1,
|
||||
height: 1,
|
||||
...input.name === undefined ? {} : { name: input.name },
|
||||
}
|
||||
}
|
||||
|
||||
readImage(_ref: ImageAttachmentRef): Promise<StoredImageAttachment> {
|
||||
throw new Error('not used')
|
||||
}
|
||||
}
|
||||
|
||||
function image(value: number, mediaType: ImageMediaType = 'image/png'): SaveImageAttachment {
|
||||
return { data: Uint8Array.of(value), mediaType, name: `${value}.png` }
|
||||
}
|
||||
|
||||
describe('AttachmentStore.saveImages', () => {
|
||||
it('validates the complete batch before saving in input order', async () => {
|
||||
const store = new RecordingStore(new Context())
|
||||
|
||||
const refs = await store.saveImages([image(1), image(2)])
|
||||
|
||||
expect(store.calls).toEqual(['validate:1', 'validate:2', 'save:1', 'save:2'])
|
||||
expect(refs.map(ref => ref.name)).toEqual(['1.png', '2.png'])
|
||||
})
|
||||
|
||||
it('rejects count, aggregate bytes, and deployment media types before validation', async () => {
|
||||
const store = new RecordingStore(new Context())
|
||||
|
||||
await expect(store.saveImages([image(1), image(2), image(3)]))
|
||||
.rejects.toMatchObject({ code: 'TOO_MANY_IMAGES' })
|
||||
await expect(store.saveImages([
|
||||
{ data: Uint8Array.of(1, 2, 3), mediaType: 'image/png' },
|
||||
{ data: Uint8Array.of(4, 5, 6), mediaType: 'image/png' },
|
||||
])).rejects.toMatchObject({ code: 'IMAGES_TOO_LARGE' })
|
||||
await expect(store.saveImages([image(1, 'image/jpeg')]))
|
||||
.rejects.toMatchObject({ code: 'UNSUPPORTED_IMAGE_TYPE' })
|
||||
expect(store.calls).toEqual([])
|
||||
})
|
||||
|
||||
it('starts no writes when any member fails validation', async () => {
|
||||
const store = new RecordingStore(new Context())
|
||||
store.rejectValidationAt = 2
|
||||
|
||||
await expect(store.saveImages([image(1), image(2)]))
|
||||
.rejects.toThrow('invalid:2')
|
||||
expect(store.calls).toEqual(['validate:1', 'validate:2'])
|
||||
})
|
||||
|
||||
it('returns no partial references when storage fails after an earlier commit', async () => {
|
||||
const store = new RecordingStore(new Context())
|
||||
store.rejectSaveAt = 2
|
||||
|
||||
await expect(store.saveImages([image(1), image(2)]))
|
||||
.rejects.toThrow('write:2')
|
||||
expect(store.calls).toEqual(['validate:1', 'validate:2', 'save:1', 'save:2'])
|
||||
})
|
||||
})
|
||||
@@ -145,36 +145,25 @@ async function durablePromptContent(ctx: Context, content: readonly PromptConten
|
||||
if (content.every(part => part.type === 'text')) {
|
||||
return content.map(part => ({ type: 'text', text: part.text }))
|
||||
}
|
||||
const limits = ctx.attachments.imageLimits
|
||||
if (content.filter(part => part.type === 'image').length > limits.maxImagesPerMessage) {
|
||||
throw new AttachmentError('Prompt exceeds the configured image-count limit.', 'TOO_MANY_IMAGES')
|
||||
}
|
||||
const prepared = content.map(part => part.type === 'text'
|
||||
? part
|
||||
: { part, data: decodeBase64(part.data) })
|
||||
const images = prepared.filter((part): part is Extract<typeof part, { data: Uint8Array }> => 'data' in part)
|
||||
const totalBytes = images.reduce((sum, image) => sum + image.data.byteLength, 0)
|
||||
if (totalBytes > limits.maxMessageImageBytes) {
|
||||
throw new AttachmentError('Prompt exceeds the configured aggregate image-byte limit.', 'IMAGES_TOO_LARGE')
|
||||
}
|
||||
for (const image of images) {
|
||||
await ctx.attachments.validateImage({
|
||||
data: image.data,
|
||||
mediaType: image.part.mediaType,
|
||||
...image.part.name === undefined ? {} : { name: image.part.name },
|
||||
})
|
||||
}
|
||||
const refs = await ctx.attachments.saveImages(images.map(image => ({
|
||||
data: image.data,
|
||||
mediaType: image.part.mediaType,
|
||||
...image.part.name === undefined ? {} : { name: image.part.name },
|
||||
})))
|
||||
const blocks: ContentBlock[] = []
|
||||
let imageIndex = 0
|
||||
for (const item of prepared) {
|
||||
if (!('data' in item)) {
|
||||
blocks.push({ type: 'text', text: item.text })
|
||||
continue
|
||||
}
|
||||
const attachment = await ctx.attachments.saveImage({
|
||||
data: item.data,
|
||||
mediaType: item.part.mediaType,
|
||||
...item.part.name === undefined ? {} : { name: item.part.name },
|
||||
})
|
||||
const attachment = refs[imageIndex++]
|
||||
/* v8 ignore next -- each prepared image supplied exactly one saveImages input and therefore one ordered ref. */
|
||||
if (attachment === undefined) throw new Error('attachment batch result did not preserve input cardinality')
|
||||
blocks.push({ type: 'image', attachment })
|
||||
}
|
||||
return blocks
|
||||
|
||||
@@ -9,6 +9,7 @@ import { describe, expect, it, vi } from 'vitest'
|
||||
import { Context } from '@deepseek-ai/cordis'
|
||||
import AgentRegistry, { agentEvents } from '@deepseek-ai/dsh-agent'
|
||||
import type { Agent } from '@deepseek-ai/dsh-agent'
|
||||
import AttachmentStore from '@deepseek-ai/dsh-attachment'
|
||||
import LlmService, { LlmAdapter, ReasoningEffortId } from '@deepseek-ai/dsh-llm'
|
||||
import type {
|
||||
GenerateOptions, LlmCallConfig, LlmModelInfo, LlmModelReasoningInfo, LlmProviderInfo,
|
||||
@@ -140,7 +141,7 @@ describe('Web session model selection', () => {
|
||||
height: 1,
|
||||
...input.name === undefined ? {} : { name: input.name },
|
||||
}))
|
||||
ctx.provide('attachments', {
|
||||
const attachments = {
|
||||
imageLimits: {
|
||||
maxImageBytes: 4,
|
||||
maxImagesPerMessage: 2,
|
||||
@@ -150,6 +151,12 @@ describe('Web session model selection', () => {
|
||||
},
|
||||
validateImage,
|
||||
saveImage,
|
||||
}
|
||||
ctx.provide('attachments', {
|
||||
...attachments,
|
||||
saveImages(inputs: readonly Parameters<typeof saveImage>[0][]) {
|
||||
return AttachmentStore.prototype.saveImages.call(attachments, inputs)
|
||||
},
|
||||
} as never)
|
||||
const followup = vi.fn()
|
||||
Object.assign(agent, { followup })
|
||||
|
||||
@@ -232,6 +232,10 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [
|
||||
signature: 'abstract validateImage(input: SaveImageAttachment): Promise<void>',
|
||||
jsDoc: '/**\n * Validate one image without persisting it.\n * Batch callers validate every member before saving any member.\n * @param input - encoded bytes, declared media type, and optional display name.\n * @returns completion after the encoded raster has been fully decoded.\n */',
|
||||
},
|
||||
{
|
||||
signature: 'async saveImages(inputs: readonly SaveImageAttachment[]): Promise<readonly ImageAttachmentRef[]>',
|
||||
jsDoc: '/**\n * Validate one ordered image batch before committing any member.\n * Validation failures start no writes; storage failures return no partial\n * references, although already published content-addressed objects may stay\n * unreachable until a future retention policy collects them.\n * @param inputs - encoded images in their owning message order.\n * @returns durable references in the exact input order.\n */',
|
||||
},
|
||||
{
|
||||
signature: 'abstract saveImage(input: SaveImageAttachment): Promise<ImageAttachmentRef>',
|
||||
jsDoc: '/**\n * Validate and durably commit one image before its owning session event is appended.\n * @param input - encoded bytes, declared media type, and optional display name.\n * @returns a durable content-addressed reference.\n */',
|
||||
|
||||
Reference in New Issue
Block a user