From 92a9741050a8f9b4dcfc753263befa04bc029abb Mon Sep 17 00:00:00 2001 From: creatixchu Date: Tue, 18 Aug 2026 14:04:50 +0800 Subject: [PATCH 01/28] docs(llm): anchor unified request-image management design PR From 8f83853b601b29286e10c7668dc19b8230e30463 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 10:56:49 +0800 Subject: [PATCH 02/28] refactor(attachment): saveImage returns the canonical ref beside source facts AttachmentStore.saveImage now resolves SavedImageAttachment: the durable reference paired with the submitted raster's intrinsic facts, so a store may persist a canonical re-encoding while callers keep the source dimensions for coordinate mapping. saveImages keeps returning refs; every fake store and the cordis API catalog follow the new signature. --- docs/subsystems/attachment.i18n.yaml | 4 ++-- docs/subsystems/attachment.md | 8 ++++++-- docs/subsystems/attachment.zh.md | 8 ++++++-- packages/acp/acp/tests/dispose.spec.ts | 2 +- packages/acp/acp/tests/harness.ts | 9 ++++++--- packages/acp/acp/tests/turns.spec.ts | 6 +++--- .../attachment/attachment-local/src/index.ts | 4 ++-- .../attachment/attachment-local/src/store.ts | 18 ++++++++++++----- packages/attachment/attachment/src/index.ts | 13 +++++++++--- packages/attachment/attachment/src/types.ts | 20 +++++++++++++++++++ .../attachment/attachment/tests/index.spec.ts | 18 ++++++++++------- .../extensions/tool-cordis/src/api-catalog.ts | 14 ++++++++++--- packages/fs/tool-fs/src/read-image.ts | 2 +- packages/fs/tool-fs/tests/read-image.spec.ts | 13 +++++++----- .../command-goal/tests/command-goal.spec.ts | 9 ++++++--- .../apiproxy/tests/api-proxy-models.spec.ts | 15 ++++++++------ .../commands/tests/commands.spec.ts | 12 ++++++++--- .../llm-deepseek/tests/dynamic-config.spec.ts | 8 ++++++-- packages/llm/llm-pi-ai/tests/adapter.spec.ts | 3 ++- .../llm/llm-pi-ai/tests/provider-apis.e2e.ts | 3 ++- .../mcp/mcp-client/tests/mcp-client.spec.ts | 10 +++++++--- .../plan/plan-mode/tests/plan-mode.spec.ts | 5 +++-- scripts/gen-cordis-catalog.ts | 2 ++ scripts/gen-tool-catalog.ts | 4 ++-- scripts/test-invariants.ts | 3 ++- 25 files changed, 150 insertions(+), 63 deletions(-) diff --git a/docs/subsystems/attachment.i18n.yaml b/docs/subsystems/attachment.i18n.yaml index 11f7369862..f904af9a27 100644 --- a/docs/subsystems/attachment.i18n.yaml +++ b/docs/subsystems/attachment.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/subsystems/attachment.md -attachment.md: 180b7e06f0461dd4136779917e0732921704803b -attachment.zh.md: 35aa24ec5957b41e12a543fd76ea20604894cd18 +attachment.md: 780d4744dc7ca8cada6209476cd208cf8ef95bc2 +attachment.zh.md: 843eca1c4deda9d3499207a2d0f163e401a50c9b diff --git a/docs/subsystems/attachment.md b/docs/subsystems/attachment.md index 180b7e06f0..780d4744dc 100644 --- a/docs/subsystems/attachment.md +++ b/docs/subsystems/attachment.md @@ -120,10 +120,14 @@ async saveImages(inputs: readonly SaveImageAttachment[]): Promise +abstract saveImage(input: SaveImageAttachment): Promise /** * Read one image and verify that bytes still match the recorded reference. diff --git a/docs/subsystems/attachment.zh.md b/docs/subsystems/attachment.zh.md index 35aa24ec59..843eca1c4d 100644 --- a/docs/subsystems/attachment.zh.md +++ b/docs/subsystems/attachment.zh.md @@ -120,10 +120,14 @@ async saveImages(inputs: readonly SaveImageAttachment[]): Promise +abstract saveImage(input: SaveImageAttachment): Promise /** * Read one image and verify that bytes still match the recorded reference. diff --git a/packages/acp/acp/tests/dispose.spec.ts b/packages/acp/acp/tests/dispose.spec.ts index 4aa32f078c..e5a4a66a3b 100644 --- a/packages/acp/acp/tests/dispose.spec.ts +++ b/packages/acp/acp/tests/dispose.spec.ts @@ -30,7 +30,7 @@ describe('ACP connection ownership', () => { it('disposal drains asynchronous assistant image delivery before releasing sessions', async () => { const script: StreamChunk[][] = [] harness = await makeBridgeHarness({ script }) - const ref = await harness.attachments!.saveImage({ data: Uint8Array.of(4), mediaType: 'image/png' }) + const { ref } = await harness.attachments!.saveImage({ data: Uint8Array.of(4), mediaType: 'image/png' }) script.push([ { type: 'block-start', index: 0, blockType: 'image' }, { type: 'block-end', index: 0, block: { type: 'image', attachment: ref } }, diff --git a/packages/acp/acp/tests/harness.ts b/packages/acp/acp/tests/harness.ts index ce6e93794f..7c0532e92d 100644 --- a/packages/acp/acp/tests/harness.ts +++ b/packages/acp/acp/tests/harness.ts @@ -13,7 +13,7 @@ import { type Stream, } from '@agentclientprotocol/sdk' import AttachmentStore, { AttachmentError, AttachmentId } from '@deepseek-ai/dsh-attachment' -import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' +import type { ImageAttachmentLimits, ImageAttachmentRef, SavedImageAttachment, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' import { type GenerateOptions, LlmAdapter, type LlmResolvedModelInfo, type StreamChunk } from '@deepseek-ai/dsh-llm' import AgentLoop from '@deepseek-ai/dsh-agent-loop' import { mountAgentLoopTestDependencies } from '@deepseek-ai/dsh-agent-loop-testkit' @@ -99,7 +99,7 @@ class MemoryAttachmentStore extends AttachmentStore { if (input.data.byteLength === 0) throw new AttachmentError('Image is empty.', 'INVALID_IMAGE') } - saveImage(input: SaveImageAttachment): Promise { + saveImage(input: SaveImageAttachment): Promise { this.saved.push(input) const digest = createHash('sha256').update(input.data).digest('hex') const ref: ImageAttachmentRef = { @@ -110,7 +110,10 @@ class MemoryAttachmentStore extends AttachmentStore { height: 1, } this.objects.set(ref.attachmentId, { ref, data: Uint8Array.from(input.data) }) - return Promise.resolve(ref) + return Promise.resolve({ + ref, + source: { mediaType: ref.mediaType, bytes: ref.bytes, width: ref.width, height: ref.height }, + }) } async readImage(ref: ImageAttachmentRef): Promise { diff --git a/packages/acp/acp/tests/turns.spec.ts b/packages/acp/acp/tests/turns.spec.ts index 71e2a21e43..c9b229caf1 100644 --- a/packages/acp/acp/tests/turns.spec.ts +++ b/packages/acp/acp/tests/turns.spec.ts @@ -44,7 +44,7 @@ describe('ACP prompt lifecycle', () => { it('delivers a committed assistant image as verified ACP base64', async () => { const script: StreamChunk[][] = [] harness = await makeBridgeHarness({ script }) - const ref = await harness.attachments!.saveImage({ data: Uint8Array.of(1), mediaType: 'image/png' }) + const { ref } = await harness.attachments!.saveImage({ data: Uint8Array.of(1), mediaType: 'image/png' }) script.push([ { type: 'block-start', index: 0, blockType: 'image' }, { @@ -68,7 +68,7 @@ describe('ACP prompt lifecycle', () => { it('preserves committed text/image/text order on the ACP wire', async () => { const script: StreamChunk[][] = [] harness = await makeBridgeHarness({ script }) - const ref = await harness.attachments!.saveImage({ data: Uint8Array.of(2), mediaType: 'image/jpeg' }) + const { ref } = await harness.attachments!.saveImage({ data: Uint8Array.of(2), mediaType: 'image/jpeg' }) script.push([ { type: 'block-start', index: 0, blockType: 'text' }, { type: 'block-end', index: 0, block: { type: 'text', text: 'before' } }, @@ -92,7 +92,7 @@ describe('ACP prompt lifecycle', () => { it('does not settle a prompt before ordered output delivery drains', async () => { const script: StreamChunk[][] = [] harness = await makeBridgeHarness({ script }) - const ref = await harness.attachments!.saveImage({ data: Uint8Array.of(3), mediaType: 'image/png' }) + const { ref } = await harness.attachments!.saveImage({ data: Uint8Array.of(3), mediaType: 'image/png' }) script.push([ { type: 'block-start', index: 0, blockType: 'image' }, { type: 'block-end', index: 0, block: { type: 'image', attachment: ref } }, diff --git a/packages/attachment/attachment-local/src/index.ts b/packages/attachment/attachment-local/src/index.ts index a4047da1f1..b529270c31 100644 --- a/packages/attachment/attachment-local/src/index.ts +++ b/packages/attachment/attachment-local/src/index.ts @@ -4,7 +4,7 @@ import { join, resolve } from 'node:path' import { Context } from '@deepseek-ai/cordis' import z from '@deepseek-ai/schemastery' import { AttachmentStore } from '@deepseek-ai/dsh-attachment' -import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' +import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, SavedImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' import { resolveDshHome } from '@deepseek-ai/dsh-home-paths' import { readImageFile, saveImageFile, validateImageFile } from './store.ts' @@ -75,7 +75,7 @@ export class LocalAttachmentStore extends AttachmentStore { await validateImageFile(input, this.imageLimits) } - async saveImage(input: SaveImageAttachment): Promise { + async saveImage(input: SaveImageAttachment): Promise { return saveImageFile(this.root, input, this.imageLimits) } diff --git a/packages/attachment/attachment-local/src/store.ts b/packages/attachment/attachment-local/src/store.ts index 723df98720..f98dbf0765 100644 --- a/packages/attachment/attachment-local/src/store.ts +++ b/packages/attachment/attachment-local/src/store.ts @@ -12,6 +12,7 @@ import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, + SavedImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' import { detectImage, probeImage } from './image.ts' @@ -131,9 +132,13 @@ async function ensureDurableHome(path: string): Promise { * @param root - absolute `DSH_HOME/attachments/v1` root. * @param input - encoded bytes and declared metadata. * @param limits - resolved storage policy. - * @returns durable content-addressed reference. + * @returns durable content-addressed reference beside the submitted source facts. */ -export async function saveImageFile(root: string, input: SaveImageAttachment, limits: ImageAttachmentLimits): Promise { +export async function saveImageFile( + root: string, + input: SaveImageAttachment, + limits: ImageAttachmentLimits, +): Promise { if (input.data.byteLength > limits.maxImageBytes) throw new AttachmentError('Image exceeds the configured byte limit.', 'IMAGE_TOO_LARGE') const metadata = await inspectMetadata(input.data, input.mediaType, limits) const sha256 = digest(input.data) @@ -187,9 +192,12 @@ export async function saveImageFile(root: string, input: SaveImageAttachment, li } const name = displayName(input.name) return { - attachmentId: AttachmentId(`sha256:${sha256}`), - ...metadata, - ...(name !== undefined ? { name } : {}), + ref: { + attachmentId: AttachmentId(`sha256:${sha256}`), + ...metadata, + ...(name !== undefined ? { name } : {}), + }, + source: metadata, } } diff --git a/packages/attachment/attachment/src/index.ts b/packages/attachment/attachment/src/index.ts index 1480751c15..8b3f81a98f 100644 --- a/packages/attachment/attachment/src/index.ts +++ b/packages/attachment/attachment/src/index.ts @@ -6,6 +6,7 @@ import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, + SavedImageAttachment, StoredImageAttachment, } from './types.ts' @@ -20,6 +21,8 @@ export type { ImageAttachmentRef, ImageMediaType, SaveImageAttachment, + SavedImageAttachment, + SourceImageInfo, StoredImageAttachment, } from './types.ts' @@ -71,16 +74,20 @@ export abstract class AttachmentStore extends Service { for (const input of inputs) await this.validateImage(input) const refs: ImageAttachmentRef[] = [] - for (const input of inputs) refs.push(await this.saveImage(input)) + for (const input of inputs) refs.push((await this.saveImage(input)).ref) return refs } /** * Validate and durably commit one image before its owning session event is appended. + * Implementations may store a canonical re-encoding of the submitted raster; + * the returned reference always describes the stored bytes, while `source` + * preserves the submitted raster's intrinsic facts for callers that report + * or map coordinates against the original. * @param input - encoded bytes, declared media type, and optional display name. - * @returns a durable content-addressed reference. + * @returns the durable content-addressed reference beside the submitted source facts. */ - abstract saveImage(input: SaveImageAttachment): Promise + abstract saveImage(input: SaveImageAttachment): Promise /** * Read one image and verify that bytes still match the recorded reference. diff --git a/packages/attachment/attachment/src/types.ts b/packages/attachment/attachment/src/types.ts index 7c29231172..93cbf6a3db 100644 --- a/packages/attachment/attachment/src/types.ts +++ b/packages/attachment/attachment/src/types.ts @@ -58,3 +58,23 @@ export interface StoredImageAttachment { ref: ImageAttachmentRef data: Uint8Array } + +/** Intrinsic facts of the submitted source raster, before any canonical re-encoding. */ +export interface SourceImageInfo { + /** Media type verified from the submitted bytes. */ + mediaType: ImageMediaType + /** Exact submitted encoded byte length. */ + bytes: number + /** Intrinsic width of the submitted raster in pixels. */ + width: number + /** Intrinsic height of the submitted raster in pixels. */ + height: number +} + +/** Commit result pairing the durable reference with the submitted source raster it was derived from. */ +export interface SavedImageAttachment { + /** Durable reference describing the stored bytes. */ + ref: ImageAttachmentRef + /** Submitted source raster facts; equals the `ref` fields when the store kept the submitted bytes. */ + source: SourceImageInfo +} diff --git a/packages/attachment/attachment/tests/index.spec.ts b/packages/attachment/attachment/tests/index.spec.ts index 622b797ce2..b3460a77ab 100644 --- a/packages/attachment/attachment/tests/index.spec.ts +++ b/packages/attachment/attachment/tests/index.spec.ts @@ -7,6 +7,7 @@ import AttachmentStore, { type ImageAttachmentRef, type ImageMediaType, type SaveImageAttachment, + type SavedImageAttachment, type StoredImageAttachment, } from '../src/index.ts' @@ -31,17 +32,20 @@ class RecordingStore extends AttachmentStore { if (value === this.rejectValidationAt) throw new Error(`invalid:${value}`) } - async saveImage(input: SaveImageAttachment): Promise { + async saveImage(input: SaveImageAttachment): Promise { const value = input.data[0] ?? 0 this.calls.push(`save:${value}`) if (value === this.rejectSaveAt) throw new Error(`write:${value}`) return { - attachmentId: AttachmentId(`sha256:${String(value).padStart(64, '0')}`), - mediaType: input.mediaType, - bytes: input.data.byteLength, - width: 1, - height: 1, - ...input.name === undefined ? {} : { name: input.name }, + ref: { + attachmentId: AttachmentId(`sha256:${String(value).padStart(64, '0')}`), + mediaType: input.mediaType, + bytes: input.data.byteLength, + width: 1, + height: 1, + ...input.name === undefined ? {} : { name: input.name }, + }, + source: { mediaType: input.mediaType, bytes: input.data.byteLength, width: 1, height: 1 }, } } diff --git a/packages/extensions/tool-cordis/src/api-catalog.ts b/packages/extensions/tool-cordis/src/api-catalog.ts index c821662e85..60f8ac66f3 100644 --- a/packages/extensions/tool-cordis/src/api-catalog.ts +++ b/packages/extensions/tool-cordis/src/api-catalog.ts @@ -443,10 +443,10 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ returns: 'durable references in the exact input order.', }, { - signature: 'abstract saveImage(input: SaveImageAttachment): Promise', - description: 'Validate and durably commit one image before its owning session event is appended.', + signature: 'abstract saveImage(input: SaveImageAttachment): Promise', + description: 'Validate and durably commit one image before its owning session event is appended. Implementations may store a canonical re-encoding of the submitted raster; the returned reference always describes the stored bytes, while `source` preserves the submitted raster\'s intrinsic facts for callers that report or map coordinates against the original.', parameters: [{ name: 'input', description: 'encoded bytes, declared media type, and optional display name.' }], - returns: 'a durable content-addressed reference.', + returns: 'the durable content-addressed reference beside the submitted source facts.', }, { signature: 'abstract readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise', @@ -4000,6 +4000,10 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'SandboxPolicyRequest', declaration: 'export interface SandboxPolicyRequest {\n session?: Session;\n mode?: SandboxMode;\n}', }, + { + name: 'SavedImageAttachment', + declaration: 'export interface SavedImageAttachment {\n ref: ImageAttachmentRef;\n source: SourceImageInfo;\n}', + }, { name: 'SaveImageAttachment', declaration: 'export interface SaveImageAttachment {\n data: Uint8Array;\n mediaType: ImageMediaType;\n name?: string;\n}', @@ -4404,6 +4408,10 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'SkillViewOptions', declaration: 'export interface SkillViewOptions extends SkillLookupOptions {\n readonly scope?: ScopeKey | undefined;\n}', }, + { + name: 'SourceImageInfo', + declaration: 'export interface SourceImageInfo {\n mediaType: ImageMediaType;\n bytes: number;\n width: number;\n height: number;\n}', + }, { name: 'SpawnTeammateRequest', declaration: 'export interface SpawnTeammateRequest {\n readonly name: string;\n readonly description: string;\n readonly prompt: ContentBlock[];\n readonly context: \'fresh\' | \'fork\';\n readonly provider: string;\n readonly signal: AbortSignal;\n}', diff --git a/packages/fs/tool-fs/src/read-image.ts b/packages/fs/tool-fs/src/read-image.ts index 92684971a7..074f816991 100644 --- a/packages/fs/tool-fs/src/read-image.ts +++ b/packages/fs/tool-fs/src/read-image.ts @@ -187,7 +187,7 @@ export function applyReadImageTool(ctx: Context): void { // committed object by the time the tool/result event is appended. let ref: ImageAttachmentRef try { - ref = await attachments.saveImage({ data, mediaType, name: basename(target.displayPath) }) + ref = (await attachments.saveImage({ data, mediaType, name: basename(target.displayPath) })).ref } catch (error: unknown) { if (!(error instanceof AttachmentError)) throw error // Dimension refusals stay recoverable tool errors: an oversized image diff --git a/packages/fs/tool-fs/tests/read-image.spec.ts b/packages/fs/tool-fs/tests/read-image.spec.ts index 7c52db5a8f..ca79c86315 100644 --- a/packages/fs/tool-fs/tests/read-image.spec.ts +++ b/packages/fs/tool-fs/tests/read-image.spec.ts @@ -21,7 +21,7 @@ import LocalFileSystem from '@deepseek-ai/dsh-fs-local' import * as FsPolicy from '@deepseek-ai/dsh-fs-observation-policy' import LocalAttachmentStore from '@deepseek-ai/dsh-attachment-local' import { AttachmentError, AttachmentId, AttachmentStore } from '@deepseek-ai/dsh-attachment' -import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' +import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, SavedImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' import * as ToolFs from '@deepseek-ai/dsh-tool-fs' import { applyReadImageTool, @@ -344,7 +344,7 @@ describe('argument and service preconditions', () => { throw new Error('unreachable: admission refuses before validation') } - saveImage(_input: SaveImageAttachment): Promise { + saveImage(_input: SaveImageAttachment): Promise { throw new Error('unreachable: admission refuses before save') } @@ -421,7 +421,7 @@ describe('image admission failures', () => { return Promise.resolve() } - async saveImage(_input: SaveImageAttachment): Promise { + async saveImage(_input: SaveImageAttachment): Promise { throw FailingStore.failure } @@ -475,8 +475,11 @@ describe('image admission failures', () => { return Promise.resolve() } - async saveImage(input: SaveImageAttachment): Promise { - return { attachmentId: AttachmentId('sha256:feed'), mediaType: input.mediaType, bytes: input.data.length, width: 1, height: 1 } + async saveImage(input: SaveImageAttachment): Promise { + return { + ref: { attachmentId: AttachmentId('sha256:feed'), mediaType: input.mediaType, bytes: input.data.length, width: 1, height: 1 }, + source: { mediaType: input.mediaType, bytes: input.data.length, width: 1, height: 1 }, + } } readImage(_ref: ImageAttachmentRef): Promise { diff --git a/packages/goal/command-goal/tests/command-goal.spec.ts b/packages/goal/command-goal/tests/command-goal.spec.ts index 2127844646..aa163784df 100644 --- a/packages/goal/command-goal/tests/command-goal.spec.ts +++ b/packages/goal/command-goal/tests/command-goal.spec.ts @@ -243,8 +243,11 @@ describe('/goal image attachments', () => { const saveImage = (input: { mediaType: string; name?: string }) => { saved += 1 return Promise.resolve({ - attachmentId: `att-${saved}`, mediaType: input.mediaType, bytes: 3, width: 1, height: 1, - ...input.name === undefined ? {} : { name: input.name }, + ref: { + attachmentId: `att-${saved}`, mediaType: input.mediaType, bytes: 3, width: 1, height: 1, + ...input.name === undefined ? {} : { name: input.name }, + }, + source: { mediaType: input.mediaType, bytes: 3, width: 1, height: 1 }, }) } test.ctx.provide('attachments', { @@ -256,7 +259,7 @@ describe('/goal image attachments', () => { saveImage, async saveImages(inputs: readonly { mediaType: string; name?: string }[]) { const refs = [] - for (const input of inputs) refs.push(await saveImage(input)) + for (const input of inputs) refs.push((await saveImage(input)).ref) return refs }, }) diff --git a/packages/host/apiproxy/tests/api-proxy-models.spec.ts b/packages/host/apiproxy/tests/api-proxy-models.spec.ts index d353ad0e62..55cb15ca9f 100644 --- a/packages/host/apiproxy/tests/api-proxy-models.spec.ts +++ b/packages/host/apiproxy/tests/api-proxy-models.spec.ts @@ -134,12 +134,15 @@ describe('Web session model selection', () => { const { ctx, agent, sessionId } = await harness() const validateImage = vi.fn((_input: { data: Uint8Array }) => Promise.resolve()) const saveImage = vi.fn((input: { data: Uint8Array; mediaType: 'image/png'; name?: string }) => Promise.resolve({ - attachmentId: `att-${String(input.data[0])}`, - mediaType: input.mediaType, - bytes: input.data.byteLength, - width: 1, - height: 1, - ...input.name === undefined ? {} : { name: input.name }, + ref: { + attachmentId: `att-${String(input.data[0])}`, + mediaType: input.mediaType, + bytes: input.data.byteLength, + width: 1, + height: 1, + ...input.name === undefined ? {} : { name: input.name }, + }, + source: { mediaType: input.mediaType, bytes: input.data.byteLength, width: 1, height: 1 }, })) const attachments = { imageLimits: { diff --git a/packages/interaction/commands/tests/commands.spec.ts b/packages/interaction/commands/tests/commands.spec.ts index 755bb0ab2f..5c80748227 100644 --- a/packages/interaction/commands/tests/commands.spec.ts +++ b/packages/interaction/commands/tests/commands.spec.ts @@ -479,8 +479,11 @@ describe('image attachments', () => { saveImage: vi.fn((input: { mediaType: string; name?: string }) => { saved += 1 return Promise.resolve({ - attachmentId: `att-${saved}`, mediaType: input.mediaType, bytes: 3, width: 1, height: 1, - ...input.name === undefined ? {} : { name: input.name }, + ref: { + attachmentId: `att-${saved}`, mediaType: input.mediaType, bytes: 3, width: 1, height: 1, + ...input.name === undefined ? {} : { name: input.name }, + }, + source: { mediaType: input.mediaType, bytes: 3, width: 1, height: 1 }, }) }), // The real base-class batch method over this double's limits and members. @@ -585,7 +588,10 @@ describe('image attachments', () => { const store = storeOf() store.saveImage.mockImplementationOnce((input: { mediaType: string }) => { controller.abort('operator cancelled during admission') - return Promise.resolve({ attachmentId: 'att-late', mediaType: input.mediaType, bytes: 3, width: 1, height: 1 }) + return Promise.resolve({ + ref: { attachmentId: 'att-late', mediaType: input.mediaType, bytes: 3, width: 1, height: 1 }, + source: { mediaType: input.mediaType, bytes: 3, width: 1, height: 1 }, + }) }) ctx.provide('attachments', store) const { agent } = await mintAgentScope(ctx, 'a') diff --git a/packages/llm/llm-deepseek/tests/dynamic-config.spec.ts b/packages/llm/llm-deepseek/tests/dynamic-config.spec.ts index ac2043170a..c048f68920 100644 --- a/packages/llm/llm-deepseek/tests/dynamic-config.spec.ts +++ b/packages/llm/llm-deepseek/tests/dynamic-config.spec.ts @@ -8,6 +8,7 @@ import AttachmentStore, { AttachmentId } from '@deepseek-ai/dsh-attachment' import type { ImageAttachmentLimits, ImageAttachmentRef, + SavedImageAttachment, SaveImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' @@ -43,8 +44,11 @@ class StaticAttachmentStore extends AttachmentStore { return Promise.resolve() } - saveImage(_input: SaveImageAttachment): Promise { - return Promise.resolve(IMAGE_REF) + saveImage(_input: SaveImageAttachment): Promise { + return Promise.resolve({ + ref: IMAGE_REF, + source: { mediaType: IMAGE_REF.mediaType, bytes: IMAGE_REF.bytes, width: IMAGE_REF.width, height: IMAGE_REF.height }, + }) } readImage(ref: ImageAttachmentRef, _signal?: AbortSignal): Promise { diff --git a/packages/llm/llm-pi-ai/tests/adapter.spec.ts b/packages/llm/llm-pi-ai/tests/adapter.spec.ts index a36a180f7d..c7336cb8d7 100644 --- a/packages/llm/llm-pi-ai/tests/adapter.spec.ts +++ b/packages/llm/llm-pi-ai/tests/adapter.spec.ts @@ -4,6 +4,7 @@ import { AttachmentId, AttachmentStore } from '@deepseek-ai/dsh-attachment' import type { ImageAttachmentLimits, ImageAttachmentRef, + SavedImageAttachment, SaveImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' @@ -227,7 +228,7 @@ describe('PiAiAdapter provider routing', () => { return Promise.reject(new Error('not used')) } - saveImage(_input: SaveImageAttachment): Promise { + saveImage(_input: SaveImageAttachment): Promise { return Promise.reject(new Error('not used')) } diff --git a/packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts b/packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts index 1fe529336f..b0b1dbba9a 100644 --- a/packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts +++ b/packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts @@ -5,6 +5,7 @@ import { AttachmentId, AttachmentStore } from '@deepseek-ai/dsh-attachment' import type { ImageAttachmentLimits, ImageAttachmentRef, + SavedImageAttachment, SaveImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' @@ -78,7 +79,7 @@ async function harness(image?: StoredImageAttachment): Promise { return Promise.reject(new Error('e2e attachment fixture is read-only')) } - saveImage(_input: SaveImageAttachment): Promise { + saveImage(_input: SaveImageAttachment): Promise { return Promise.reject(new Error('e2e attachment fixture is read-only')) } diff --git a/packages/mcp/mcp-client/tests/mcp-client.spec.ts b/packages/mcp/mcp-client/tests/mcp-client.spec.ts index 164c230c56..9f4854e2d8 100644 --- a/packages/mcp/mcp-client/tests/mcp-client.spec.ts +++ b/packages/mcp/mcp-client/tests/mcp-client.spec.ts @@ -3,7 +3,7 @@ import { Client } from '@modelcontextprotocol/sdk/client/index.js' import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js' import { Context } from '@deepseek-ai/cordis' import AttachmentStore, { AttachmentError, AttachmentId } from '@deepseek-ai/dsh-attachment' -import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' +import type { ImageAttachmentLimits, ImageAttachmentRef, SavedImageAttachment, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' import { CallId, LlmAdapter, LlmRuntime } from '@deepseek-ai/dsh-llm' import type { ContentBlock } from '@deepseek-ai/dsh-llm' import type { GenerateOptions, LlmResolvedModelInfo, StreamChunk } from '@deepseek-ai/dsh-llm' @@ -86,15 +86,19 @@ class RecordingAttachmentStore extends AttachmentStore { return Promise.resolve() } - saveImage(input: SaveImageAttachment): Promise { + saveImage(input: SaveImageAttachment): Promise { this.saved.push(input) const marker = input.data[0] ?? 0 - return Promise.resolve({ + const ref: ImageAttachmentRef = { attachmentId: AttachmentId(`sha256:${marker.toString(16).padStart(64, '0')}`), mediaType: input.mediaType, bytes: input.data.byteLength, width: 1, height: 1, + } + return Promise.resolve({ + ref, + source: { mediaType: ref.mediaType, bytes: ref.bytes, width: ref.width, height: ref.height }, }) } diff --git a/packages/plan/plan-mode/tests/plan-mode.spec.ts b/packages/plan/plan-mode/tests/plan-mode.spec.ts index 8285147953..d1b4be3058 100644 --- a/packages/plan/plan-mode/tests/plan-mode.spec.ts +++ b/packages/plan/plan-mode/tests/plan-mode.spec.ts @@ -653,7 +653,8 @@ describe('/plan', () => { const saveImage = (input: { mediaType: string }) => { saved += 1 return Promise.resolve({ - attachmentId: `att-${saved}`, mediaType: input.mediaType, bytes: 3, width: 1, height: 1, + ref: { attachmentId: `att-${saved}`, mediaType: input.mediaType, bytes: 3, width: 1, height: 1 }, + source: { mediaType: input.mediaType, bytes: 3, width: 1, height: 1 }, }) } ctx.provide('attachments', { @@ -665,7 +666,7 @@ describe('/plan', () => { saveImage, async saveImages(inputs: readonly { mediaType: string }[]) { const refs = [] - for (const input of inputs) refs.push(await saveImage(input)) + for (const input of inputs) refs.push((await saveImage(input)).ref) return refs }, }) diff --git a/scripts/gen-cordis-catalog.ts b/scripts/gen-cordis-catalog.ts index 136d9301b3..255ff45001 100644 --- a/scripts/gen-cordis-catalog.ts +++ b/scripts/gen-cordis-catalog.ts @@ -294,6 +294,8 @@ export const LINK_MAP: Readonly> = { EncodedImageAttachment: 'attachment.md', ImageAttachmentRef: 'attachment.md', SaveImageAttachment: 'attachment.md', + SavedImageAttachment: 'attachment.md', + SourceImageInfo: 'attachment.md', StoredImageAttachment: 'attachment.md', ShellExecRequest: 'shell.md', ShellExecSpec: 'shell.md', diff --git a/scripts/gen-tool-catalog.ts b/scripts/gen-tool-catalog.ts index 8475fd585e..87eee0a7e4 100644 --- a/scripts/gen-tool-catalog.ts +++ b/scripts/gen-tool-catalog.ts @@ -25,7 +25,7 @@ import { PwshLocalExecutor } from '@deepseek-ai/dsh-pwsh-local' import LocalSubprocessRuntime from '@deepseek-ai/dsh-subprocess-local' import LocalFileSystem from '@deepseek-ai/dsh-fs-local' import { AttachmentStore } from '@deepseek-ai/dsh-attachment' -import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' +import type { ImageAttachmentLimits, ImageAttachmentRef, SavedImageAttachment, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' import UserQuestionService from '@deepseek-ai/dsh-user-questions' import PlanModeController from '@deepseek-ai/dsh-plan-mode' import WebRuntime from '@deepseek-ai/dsh-web' @@ -83,7 +83,7 @@ class CatalogAttachmentStore extends AttachmentStore { return Promise.reject(new Error('gen-tool-catalog: attachment validation is unreachable during schema harvest')) } - override saveImage(_input: SaveImageAttachment): Promise { + override saveImage(_input: SaveImageAttachment): Promise { return Promise.reject(new Error('gen-tool-catalog: attachment writes are unreachable during schema harvest')) } diff --git a/scripts/test-invariants.ts b/scripts/test-invariants.ts index a3b96a90a3..4f57edc2f8 100644 --- a/scripts/test-invariants.ts +++ b/scripts/test-invariants.ts @@ -12,6 +12,7 @@ import { AttachmentStore } from '@deepseek-ai/dsh-attachment' import type { ImageAttachmentLimits, ImageAttachmentRef, + SavedImageAttachment, SaveImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' @@ -125,7 +126,7 @@ class TestAttachmentStore extends AttachmentStore { return Promise.reject(new Error('test invariant attachment store does not validate images')) } - saveImage(_input: SaveImageAttachment): Promise { + saveImage(_input: SaveImageAttachment): Promise { return Promise.reject(new Error('test invariant attachment store does not save images')) } From 83a526eea1342f3be36c54554b43f8b98ca6c87d Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 10:57:39 +0800 Subject: [PATCH 03/28] feat(attachment-local): store a deterministic canonical image encoding Admission now validates a wide source envelope (32MiB, 100MP, 16384px per side) and persists a canonical encoding instead of refusing large sources: EXIF orientation baked in, metadata stripped, long edge downscaled to the configured canonical target (default 2048px), PNG palette for alpha/PNG/GIF sources and a fixed JPEG quality ladder (85/75/60/45) until the canonical byte target holds (default 1MiB). In-budget PNG/JPEG/WebP passes through byte-identically so equal sources keep deduplicating to the same content address; GIF always re-encodes to the PNG of its first frame, pinning the first-frame meaning providers apply. Encoder parameters are fixed by design; only the canonical budget is deployment configuration. --- .../attachment-local/src/canonical.ts | 103 +++++++++++++ .../attachment/attachment-local/src/index.ts | 45 ++++-- .../attachment/attachment-local/src/store.ts | 21 ++- .../attachment-local/tests/canonical.spec.ts | 139 ++++++++++++++++++ .../attachment-local/tests/index.spec.ts | 10 +- .../attachment-local/tests/store.spec.ts | 64 +++++--- 6 files changed, 340 insertions(+), 42 deletions(-) create mode 100644 packages/attachment/attachment-local/src/canonical.ts create mode 100644 packages/attachment/attachment-local/tests/canonical.spec.ts diff --git a/packages/attachment/attachment-local/src/canonical.ts b/packages/attachment/attachment-local/src/canonical.ts new file mode 100644 index 0000000000..ada2164566 --- /dev/null +++ b/packages/attachment/attachment-local/src/canonical.ts @@ -0,0 +1,103 @@ +/** + * Deterministic canonical image encoding. Admission stores this encoding, so + * the same source bytes always publish the same content address on one + * runtime: encoder parameters are fixed here, never configurable, because a + * parameter change would silently split the content-addressed space. The + * deployment chooses only the canonical budget (long edge and byte target). + */ + +import sharp, { type Sharp } from 'sharp' +import { AttachmentError } from '@deepseek-ai/dsh-attachment' +import type { ImageMediaType } from '@deepseek-ai/dsh-attachment' +import type { DetectedImage } from './image.ts' + +/** Deployment-resolved canonical encoding budget. */ +export interface CanonicalImagePolicy { + /** Long-edge target in pixels; a larger source is downscaled proportionally. */ + maxDimension: number + /** Encoded-byte target; a larger encoding falls down the fixed quality ladder. */ + maxBytes: number +} + +/** Canonical bytes beside the facts a durable reference records about them. */ +export interface CanonicalImage { + data: Uint8Array + mediaType: ImageMediaType + width: number + height: number +} + +/** JPEG quality ladder tried in order once the preferred encoding exceeds the byte target. */ +const JPEG_QUALITIES = [85, 75, 60, 45] as const + +/** Encode one prepared pipeline and report the exact output facts. */ +async function encode(pipeline: Sharp, mediaType: 'image/png' | 'image/jpeg'): Promise { + const { data, info } = await pipeline.toBuffer({ resolveWithObject: true }) + return { data: new Uint8Array(data), mediaType, width: info.width, height: info.height } +} + +/** + * Whether stored bytes may be the submitted bytes unchanged. Byte-identical + * passthrough is preferred whenever the source already fits the budget: it + * keeps re-submissions of the same original deduplicating to the same object + * and never re-encodes what no policy requires changing. GIF is excluded — + * only its first frame is model-visible, so admission pins that meaning into + * the stored object instead of letting each provider drop frames differently. + * @param detected - verified source format and dimensions. + * @param bytes - submitted encoded byte length. + * @param policy - resolved canonical budget. + * @returns whether the submitted encoding already is canonical. + */ +export function isCanonical(detected: DetectedImage, bytes: number, policy: CanonicalImagePolicy): boolean { + return detected.mediaType !== 'image/gif' + && bytes <= policy.maxBytes + && Math.max(detected.width, detected.height) <= policy.maxDimension +} + +/** + * Produce the canonical encoding of one fully validated source raster. + * Passthrough returns the submitted array; every re-encode bakes EXIF + * orientation into pixels, strips metadata, downscales to the policy's long + * edge, and encodes with fixed parameters: PNG (palette) for sources that + * carry alpha or were PNG/GIF, JPEG for photographic sources, falling down + * one fixed JPEG quality ladder until the byte target holds. + * @param data - submitted encoded bytes, already fully decoded by admission. + * @param detected - verified source format and dimensions. + * @param policy - resolved canonical budget. + * @returns canonical bytes and their reference facts. + * @throws AttachmentError `IMAGE_TOO_LARGE` when the smallest ladder step still exceeds the byte target. + */ +export async function canonicalizeImage( + data: Uint8Array, + detected: DetectedImage, + policy: CanonicalImagePolicy, +): Promise { + if (isCanonical(detected, data.byteLength, policy)) { + return { data, mediaType: detected.mediaType, width: detected.width, height: detected.height } + } + try { + const source = sharp(data, { failOn: 'error', limitInputPixels: false }) + const { hasAlpha } = await source.metadata() + const prepared = source.rotate().resize({ + width: policy.maxDimension, + height: policy.maxDimension, + fit: 'inside', + withoutEnlargement: true, + }) + const preferPng = hasAlpha || detected.mediaType === 'image/png' || detected.mediaType === 'image/gif' + if (preferPng) { + const png = await encode(prepared.clone().png({ compressionLevel: 9, palette: true }), 'image/png') + if (png.data.byteLength <= policy.maxBytes) return png + } + for (const quality of JPEG_QUALITIES) { + const jpeg = await encode( + prepared.clone().flatten({ background: '#ffffff' }).jpeg({ quality }), + 'image/jpeg', + ) + if (jpeg.data.byteLength <= policy.maxBytes) return jpeg + } + } catch (error) { + throw new AttachmentError('Unable to canonicalize image attachment.', 'ATTACHMENT_WRITE_FAILED', { cause: error }) + } + throw new AttachmentError('Image cannot be encoded within the configured canonical byte target.', 'IMAGE_TOO_LARGE') +} diff --git a/packages/attachment/attachment-local/src/index.ts b/packages/attachment/attachment-local/src/index.ts index b529270c31..cbd702c2bf 100644 --- a/packages/attachment/attachment-local/src/index.ts +++ b/packages/attachment/attachment-local/src/index.ts @@ -6,41 +6,50 @@ import z from '@deepseek-ai/schemastery' import { AttachmentStore } from '@deepseek-ai/dsh-attachment' import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, SavedImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' import { resolveDshHome } from '@deepseek-ai/dsh-home-paths' +import type { CanonicalImagePolicy } from './canonical.ts' import { readImageFile, saveImageFile, validateImageFile } from './store.ts' +export { canonicalizeImage, isCanonical } from './canonical.ts' +export type { CanonicalImage, CanonicalImagePolicy } from './canonical.ts' export { readImageFile, saveImageFile, validateImageFile } from './store.ts' -/** Default maximum encoded bytes for one image. */ -export const DEFAULT_MAX_IMAGE_BYTES = 3.5 * 1024 * 1024 +/** Default maximum encoded bytes for one submitted image; oversized sources are refused, not shrunk. */ +export const DEFAULT_MAX_IMAGE_BYTES = 32 * 1024 * 1024 /** Default maximum images in one prompt. */ export const DEFAULT_MAX_IMAGES_PER_MESSAGE = 20 /** Default maximum aggregate image bytes in one prompt. */ export const DEFAULT_MAX_MESSAGE_IMAGE_BYTES = 100 * 1024 * 1024 -/** Default maximum intrinsic pixels for one image. */ -export const DEFAULT_MAX_IMAGE_PIXELS = 40_000_000 +/** Default maximum intrinsic pixels for one submitted image. */ +export const DEFAULT_MAX_IMAGE_PIXELS = 100_000_000 +/** Default per-side pixel cap for one submitted image. */ +export const DEFAULT_MAX_IMAGE_DIMENSION = 16384 /** - * Default maximum intrinsic width and height for one image. Deployed model - * routes reject any request whose history carries an image with a side above - * 2000px once the request holds many images, and an admitted image rides - * every later request of its session, so admission refuses at the same line - * to keep the durable history streamable. + * Default long-edge target of the stored canonical encoding. A larger source + * is admitted and downscaled to this edge, so admission bounds what rides + * every later model request without refusing ordinary large sources. */ -export const DEFAULT_MAX_IMAGE_DIMENSION = 2000 +export const DEFAULT_CANONICAL_MAX_DIMENSION = 2048 +/** Default byte target of the stored canonical encoding. */ +export const DEFAULT_CANONICAL_MAX_BYTES = 1024 * 1024 /** Local attachment backend configuration. */ export interface Config { /** Explicit harness home; omitted follows `DSH_HOME`, then `~/.dsh`. */ dshHome?: string - /** Maximum encoded bytes accepted for one image. */ + /** Maximum encoded bytes accepted for one submitted image. */ maxImageBytes?: number /** Maximum image count accepted in one submitted message. */ maxImagesPerMessage?: number /** Maximum aggregate encoded image bytes accepted in one submitted message. */ maxMessageImageBytes?: number - /** Maximum intrinsic width multiplied by height accepted for one image. */ + /** Maximum intrinsic width multiplied by height accepted for one submitted image. */ maxImagePixels?: number - /** Maximum intrinsic width and maximum intrinsic height accepted for one image. */ + /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. */ maxImageDimension?: number + /** Long-edge pixel target of the stored canonical encoding. */ + canonicalMaxDimension?: number + /** Encoded-byte target of the stored canonical encoding. */ + canonicalMaxBytes?: number } /** Persistent content-addressed local attachment store. */ @@ -52,11 +61,15 @@ export class LocalAttachmentStore extends AttachmentStore { maxMessageImageBytes: z.number().step(1).min(1).default(DEFAULT_MAX_MESSAGE_IMAGE_BYTES), maxImagePixels: z.number().step(1).min(1).default(DEFAULT_MAX_IMAGE_PIXELS), maxImageDimension: z.number().step(1).min(1).default(DEFAULT_MAX_IMAGE_DIMENSION), + canonicalMaxDimension: z.number().step(1).min(1).default(DEFAULT_CANONICAL_MAX_DIMENSION), + canonicalMaxBytes: z.number().step(1).min(1).default(DEFAULT_CANONICAL_MAX_BYTES), }) /** Absolute versioned storage root. */ readonly root: string readonly imageLimits: ImageAttachmentLimits + /** Resolved canonical encoding budget applied by every save. */ + readonly canonicalPolicy: Readonly constructor(ctx: Context, config: Config) { super(ctx) @@ -69,6 +82,10 @@ export class LocalAttachmentStore extends AttachmentStore { maxImageDimension: config.maxImageDimension ?? DEFAULT_MAX_IMAGE_DIMENSION, mediaTypes: Object.freeze(['image/png', 'image/jpeg', 'image/webp', 'image/gif'] as const), }) + this.canonicalPolicy = Object.freeze({ + maxDimension: config.canonicalMaxDimension ?? DEFAULT_CANONICAL_MAX_DIMENSION, + maxBytes: config.canonicalMaxBytes ?? DEFAULT_CANONICAL_MAX_BYTES, + }) } async validateImage(input: SaveImageAttachment): Promise { @@ -76,7 +93,7 @@ export class LocalAttachmentStore extends AttachmentStore { } async saveImage(input: SaveImageAttachment): Promise { - return saveImageFile(this.root, input, this.imageLimits) + return saveImageFile(this.root, input, this.imageLimits, this.canonicalPolicy) } async readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise { diff --git a/packages/attachment/attachment-local/src/store.ts b/packages/attachment/attachment-local/src/store.ts index f98dbf0765..9da83e30a0 100644 --- a/packages/attachment/attachment-local/src/store.ts +++ b/packages/attachment/attachment-local/src/store.ts @@ -15,6 +15,8 @@ import type { SavedImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' +import { canonicalizeImage } from './canonical.ts' +import type { CanonicalImagePolicy } from './canonical.ts' import { detectImage, probeImage } from './image.ts' const ID_PATTERN = /^sha256:([a-f0-9]{64})$/ @@ -128,20 +130,26 @@ async function ensureDurableHome(path: string): Promise { } /** - * Save and verify immutable image bytes below a versioned attachment root. + * Save and verify one image below a versioned attachment root. Admission + * validates the submitted source, then stores its deterministic canonical + * encoding; the returned reference describes the stored canonical bytes while + * `source` preserves the submitted raster's facts. * @param root - absolute `DSH_HOME/attachments/v1` root. * @param input - encoded bytes and declared metadata. - * @param limits - resolved storage policy. + * @param limits - resolved source admission policy. + * @param policy - resolved canonical encoding budget. * @returns durable content-addressed reference beside the submitted source facts. */ export async function saveImageFile( root: string, input: SaveImageAttachment, limits: ImageAttachmentLimits, + policy: CanonicalImagePolicy, ): Promise { if (input.data.byteLength > limits.maxImageBytes) throw new AttachmentError('Image exceeds the configured byte limit.', 'IMAGE_TOO_LARGE') const metadata = await inspectMetadata(input.data, input.mediaType, limits) - const sha256 = digest(input.data) + const canonical = await canonicalizeImage(input.data, metadata, policy) + const sha256 = digest(canonical.data) const bucket = join(root, 'objects', sha256.slice(0, 2)) const staging = join(root, 'tmp') // Establish DSH_HOME itself against the filesystem root once per process. @@ -155,7 +163,7 @@ export async function saveImageFile( let handle try { handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600) - await handle.writeFile(input.data) + await handle.writeFile(canonical.data) await handle.sync() await handle.close() handle = undefined @@ -194,7 +202,10 @@ export async function saveImageFile( return { ref: { attachmentId: AttachmentId(`sha256:${sha256}`), - ...metadata, + mediaType: canonical.mediaType, + bytes: canonical.data.byteLength, + width: canonical.width, + height: canonical.height, ...(name !== undefined ? { name } : {}), }, source: metadata, diff --git a/packages/attachment/attachment-local/tests/canonical.spec.ts b/packages/attachment/attachment-local/tests/canonical.spec.ts new file mode 100644 index 0000000000..0441fbc462 --- /dev/null +++ b/packages/attachment/attachment-local/tests/canonical.spec.ts @@ -0,0 +1,139 @@ +import { describe, expect, it } from 'vitest' +import sharp from 'sharp' +import { canonicalizeImage, isCanonical } from '../src/canonical.ts' +import type { CanonicalImagePolicy } from '../src/canonical.ts' +import { detectImage } from '../src/image.ts' + +const POLICY: CanonicalImagePolicy = { maxDimension: 2048, maxBytes: 1024 * 1024 } + +/** Deterministic pseudo-random RGB noise; PNG cannot compress it below raw size. */ +function noisePixels(width: number, height: number): Uint8Array { + const pixels = new Uint8Array(width * height * 3) + let state = 0x2545f491 + for (let index = 0; index < pixels.length; index += 1) { + state = (state * 1103515245 + 12345) & 0x7fffffff + pixels[index] = state & 0xff + } + return pixels +} + +async function noiseImage(width: number, height: number, format: 'png' | 'jpeg' | 'webp' | 'gif'): Promise { + const image = sharp(noisePixels(width, height), { raw: { width, height, channels: 3 } }) + return new Uint8Array(await image.toFormat(format).toBuffer()) +} + +async function flatImage(width: number, height: number, format: 'png' | 'jpeg' | 'webp' | 'gif', alpha = false): Promise { + const image = sharp({ + create: { width, height, channels: alpha ? 4 : 3, background: { r: 12, g: 200, b: 64, alpha: alpha ? 0.5 : 1 } }, + }) + return new Uint8Array(await image.toFormat(format, format === 'webp' && alpha ? { lossless: true } : {}).toBuffer()) +} + +describe('isCanonical', () => { + it('accepts an in-budget PNG/JPEG/WebP and refuses GIF, oversized edges, and oversized bytes', () => { + expect(isCanonical({ mediaType: 'image/png', width: 2048, height: 4 }, 100, POLICY)).toBe(true) + expect(isCanonical({ mediaType: 'image/gif', width: 4, height: 4 }, 100, POLICY)).toBe(false) + expect(isCanonical({ mediaType: 'image/jpeg', width: 2049, height: 4 }, 100, POLICY)).toBe(false) + expect(isCanonical({ mediaType: 'image/webp', width: 4, height: 4 }, POLICY.maxBytes + 1, POLICY)).toBe(false) + }) +}) + +describe('canonicalizeImage', () => { + it('passes an already-canonical source through byte-identically', async () => { + const data = await flatImage(6, 4, 'webp') + const detected = await detectImage(data) + + const canonical = await canonicalizeImage(data, detected, POLICY) + + expect(canonical.data).toBe(data) + expect(canonical).toMatchObject({ mediaType: 'image/webp', width: 6, height: 4 }) + }) + + it('downscales an oversized PNG to the long-edge target and stays PNG', async () => { + const data = await flatImage(10, 6, 'png') + const detected = await detectImage(data) + + const canonical = await canonicalizeImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes }) + + expect(canonical).toMatchObject({ mediaType: 'image/png', width: 5, height: 3 }) + await expect(detectImage(canonical.data)).resolves.toEqual({ mediaType: 'image/png', width: 5, height: 3 }) + const again = await canonicalizeImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes }) + expect(again.data).toEqual(canonical.data) + }) + + it('re-encodes the canonical output of a resize into itself (idempotence)', async () => { + const data = await flatImage(10, 6, 'png') + const first = await canonicalizeImage(data, await detectImage(data), { maxDimension: 5, maxBytes: POLICY.maxBytes }) + + const second = await canonicalizeImage(first.data, await detectImage(first.data), { maxDimension: 5, maxBytes: POLICY.maxBytes }) + + expect(second.data).toBe(first.data) + }) + + it('always re-encodes GIF to the PNG of its first frame', async () => { + const data = await flatImage(6, 4, 'gif') + const detected = await detectImage(data) + + const canonical = await canonicalizeImage(data, detected, POLICY) + + expect(canonical.mediaType).toBe('image/png') + await expect(detectImage(canonical.data)).resolves.toEqual({ mediaType: 'image/png', width: 6, height: 4 }) + }) + + it('keeps alpha sources on PNG when the budget holds', async () => { + const data = await flatImage(9, 5, 'webp', true) + const detected = await detectImage(data) + + const canonical = await canonicalizeImage(data, detected, { maxDimension: 4, maxBytes: POLICY.maxBytes }) + + expect(canonical).toMatchObject({ mediaType: 'image/png', width: 4, height: 2 }) + }) + + it('re-encodes an oversized photographic JPEG as JPEG', async () => { + const data = await noiseImage(64, 32, 'jpeg') + const detected = await detectImage(data) + + const canonical = await canonicalizeImage(data, detected, { maxDimension: 32, maxBytes: POLICY.maxBytes }) + + expect(canonical).toMatchObject({ mediaType: 'image/jpeg', width: 32, height: 16 }) + }) + + it('falls from PNG to the JPEG ladder when palette PNG exceeds the byte target', async () => { + // A smooth gradient: palette quantization dithers it into a sizable PNG + // while JPEG at quality 85 stays far smaller, so the budget between the + // two forces exactly one ladder hop. + const side = 256 + const pixels = new Uint8Array(side * side * 3) + for (let y = 0; y < side; y += 1) { + for (let x = 0; x < side; x += 1) { + const index = (y * side + x) * 3 + pixels[index] = x & 0xff + pixels[index + 1] = y & 0xff + pixels[index + 2] = (x + y) >> 1 & 0xff + } + } + const data = new Uint8Array(await sharp(pixels, { raw: { width: side, height: side, channels: 3 } }).png().toBuffer()) + const detected = await detectImage(data) + const paletteSize = (await sharp(data).png({ compressionLevel: 9, palette: true }).toBuffer()).byteLength + const jpegSize = (await sharp(data).flatten({ background: '#ffffff' }).jpeg({ quality: 85 }).toBuffer()).byteLength + expect(jpegSize).toBeLessThan(paletteSize) + const budget = { maxDimension: 2048, maxBytes: paletteSize - 1 } + + const canonical = await canonicalizeImage(data, detected, budget) + + expect(canonical.mediaType).toBe('image/jpeg') + expect(canonical.data.byteLength).toBeLessThanOrEqual(budget.maxBytes) + }) + + it('refuses a source that no ladder step fits into the byte target', async () => { + const data = await noiseImage(64, 64, 'png') + + await expect(canonicalizeImage(data, await detectImage(data), { maxDimension: 2048, maxBytes: 10 })) + .rejects.toMatchObject({ code: 'IMAGE_TOO_LARGE' }) + }) + + it('maps an encoder fault on undecodable bytes to a storage failure', async () => { + await expect(canonicalizeImage(Uint8Array.of(1, 2, 3), { mediaType: 'image/png', width: 5000, height: 5000 }, POLICY)) + .rejects.toMatchObject({ code: 'ATTACHMENT_WRITE_FAILED' }) + }) +}) diff --git a/packages/attachment/attachment-local/tests/index.spec.ts b/packages/attachment/attachment-local/tests/index.spec.ts index 92bbe3c0aa..0e86957f82 100644 --- a/packages/attachment/attachment-local/tests/index.spec.ts +++ b/packages/attachment/attachment-local/tests/index.spec.ts @@ -5,6 +5,8 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { describe, expect, it } from 'vitest' import LocalAttachmentStore, { + DEFAULT_CANONICAL_MAX_BYTES, + DEFAULT_CANONICAL_MAX_DIMENSION, DEFAULT_MAX_IMAGE_BYTES, DEFAULT_MAX_IMAGE_DIMENSION, DEFAULT_MAX_IMAGE_PIXELS, @@ -15,7 +17,7 @@ import LocalAttachmentStore, { describe('local attachment service', () => { it('resolves every omitted admission limit explicitly', () => { const service = new LocalAttachmentStore(new Context(), {}) - expect(DEFAULT_MAX_IMAGE_BYTES).toBe(3.5 * 1024 * 1024) + expect(DEFAULT_MAX_IMAGE_BYTES).toBe(32 * 1024 * 1024) expect(service.imageLimits).toEqual({ maxImageBytes: DEFAULT_MAX_IMAGE_BYTES, maxImagesPerMessage: DEFAULT_MAX_IMAGES_PER_MESSAGE, @@ -24,6 +26,10 @@ describe('local attachment service', () => { maxImageDimension: DEFAULT_MAX_IMAGE_DIMENSION, mediaTypes: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'], }) + expect(service.canonicalPolicy).toEqual({ + maxDimension: DEFAULT_CANONICAL_MAX_DIMENSION, + maxBytes: DEFAULT_CANONICAL_MAX_BYTES, + }) }) it('saves and reads through the service boundary', async () => { @@ -34,7 +40,7 @@ describe('local attachment service', () => { 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=', 'base64', )) - const ref = await service.saveImage({ data, mediaType: 'image/png' }) + const { ref } = await service.saveImage({ data, mediaType: 'image/png' }) await expect(service.readImage(ref)).resolves.toEqual({ ref, data }) } finally { await rm(dshHome, { recursive: true, force: true }) diff --git a/packages/attachment/attachment-local/tests/store.spec.ts b/packages/attachment/attachment-local/tests/store.spec.ts index a5b831e933..8fdd076f6e 100644 --- a/packages/attachment/attachment-local/tests/store.spec.ts +++ b/packages/attachment/attachment-local/tests/store.spec.ts @@ -7,6 +7,7 @@ import { mkdtemp, rm } from 'node:fs/promises' import { afterEach, describe, expect, it, vi } from 'vitest' import sharp from 'sharp' import type { ImageAttachmentLimits } from '@deepseek-ai/dsh-attachment' +import type { CanonicalImagePolicy } from '../src/canonical.ts' import { readImageFile, saveImageFile } from '../src/store.ts' const fsControl = vi.hoisted(() => ({ @@ -38,6 +39,8 @@ const PNG = Uint8Array.from(Buffer.from( 'base64', )) +const POLICY: CanonicalImagePolicy = { maxDimension: 2048, maxBytes: 1024 * 1024 } + const LIMITS: ImageAttachmentLimits = { maxImageBytes: 1024, maxImagesPerMessage: 2, @@ -79,7 +82,7 @@ describe('local attachment store', () => { const bucket = join(objects, sha256.slice(0, 2)) fsControl.syncedDirectories.length = 0 - await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS) + await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) // Each process first proves DSH_HOME durable all the way to the filesystem // root; existence alone cannot vouch for a concurrent creator's fsync. @@ -104,7 +107,7 @@ describe('local attachment store', () => { it('creates and persists a missing nested home directory against the filesystem root', async () => { const storageRoot = join(await root(), 'home', 'attachments', 'v1') - const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS) + const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) await expect(readImageFile(storageRoot, ref)).resolves.toEqual({ ref, data: PNG }) }) @@ -113,12 +116,12 @@ describe('local attachment store', () => { const storageRoot = await root() const first = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png', name: '/private/tmp/pixel.png', - }, LIMITS) - const second = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS) + }, LIMITS, POLICY) + const second = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) const sha256 = createHash('sha256').update(PNG).digest('hex') const object = join(storageRoot, 'objects', sha256.slice(0, 2), sha256) - expect(first).toEqual({ + expect(first.ref).toEqual({ attachmentId: `sha256:${sha256}`, mediaType: 'image/png', bytes: PNG.byteLength, @@ -126,25 +129,44 @@ describe('local attachment store', () => { height: 1, name: 'pixel.png', }) - expect(second.attachmentId).toBe(first.attachmentId) + expect(first.source).toEqual({ mediaType: 'image/png', bytes: PNG.byteLength, width: 1, height: 1 }) + expect(second.ref.attachmentId).toBe(first.ref.attachmentId) expect(new Uint8Array(await readFile(object))).toEqual(PNG) if (process.platform !== 'win32') { expect((await stat(object)).mode & 0o777).toBe(0o600) expect((await stat(join(storageRoot, 'objects', sha256.slice(0, 2)))).mode & 0o777).toBe(0o700) } - await expect(readImageFile(storageRoot, first)).resolves.toEqual({ ref: first, data: PNG }) + await expect(readImageFile(storageRoot, first.ref)).resolves.toEqual({ ref: first.ref, data: PNG }) + }) + + it('stores the canonical encoding of an oversized source and reads it back verified', async () => { + const storageRoot = await root() + const oversized = new Uint8Array(await sharp({ + create: { width: 4, height: 4, channels: 3, background: { r: 9, g: 9, b: 9 } }, + }).png().toBuffer()) + + const saved = await saveImageFile(storageRoot, { + data: oversized, mediaType: 'image/png', name: 'big.png', + }, { ...LIMITS, maxImagePixels: 64 }, { maxDimension: 2, maxBytes: 1024 * 1024 }) + + expect(saved.source).toEqual({ mediaType: 'image/png', bytes: oversized.byteLength, width: 4, height: 4 }) + expect(saved.ref).toMatchObject({ mediaType: 'image/png', width: 2, height: 2, name: 'big.png' }) + expect(saved.ref.bytes).not.toBe(oversized.byteLength) + const read = await readImageFile(storageRoot, saved.ref) + expect(read.data.byteLength).toBe(saved.ref.bytes) + expect(String(saved.ref.attachmentId)).toBe(`sha256:${createHash('sha256').update(read.data).digest('hex')}`) }) it('keeps admitted history readable after deployment limits become stricter', async () => { const storageRoot = await root() - const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS) + const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) await expect(readImageFile(storageRoot, ref)).resolves.toEqual({ ref, data: PNG }) }) it('forwards read cancellation to the filesystem and preserves its reason', async () => { const storageRoot = await root() - const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS) + const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) const controller = new AbortController() fsControl.readSignals.length = 0 @@ -160,35 +182,35 @@ describe('local attachment store', () => { const storageRoot = await root() await expect(saveImageFile(storageRoot, { data: new Uint8Array(0), mediaType: 'image/png', - }, LIMITS)).rejects.toMatchObject({ code: 'INVALID_IMAGE' }) + }, LIMITS, POLICY)).rejects.toMatchObject({ code: 'INVALID_IMAGE' }) await expect(saveImageFile(storageRoot, { data: Uint8Array.of(1, 2, 3), mediaType: 'image/png', - }, LIMITS)).rejects.toMatchObject({ code: 'INVALID_IMAGE' }) + }, LIMITS, POLICY)).rejects.toMatchObject({ code: 'INVALID_IMAGE' }) await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/jpeg', - }, LIMITS)).rejects.toMatchObject({ code: 'IMAGE_TYPE_MISMATCH' }) + }, LIMITS, POLICY)).rejects.toMatchObject({ code: 'IMAGE_TYPE_MISMATCH' }) await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png', - }, { ...LIMITS, maxImageBytes: 1 })).rejects.toMatchObject({ code: 'IMAGE_TOO_LARGE' }) + }, { ...LIMITS, maxImageBytes: 1 }, POLICY)).rejects.toMatchObject({ code: 'IMAGE_TOO_LARGE' }) const wide = new Uint8Array(await sharp({ create: { width: 5, height: 5, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 1 } }, }).png().toBuffer()) await expect(saveImageFile(storageRoot, { data: wide, mediaType: 'image/png', - }, LIMITS)).rejects.toMatchObject({ code: 'IMAGE_TOO_MANY_PIXELS' }) + }, LIMITS, POLICY)).rejects.toMatchObject({ code: 'IMAGE_TOO_MANY_PIXELS' }) await expect(saveImageFile(storageRoot, { data: wide, mediaType: 'image/png', - }, { ...LIMITS, maxImagePixels: 25, maxImageDimension: 4 })).rejects.toMatchObject({ code: 'IMAGE_DIMENSION_TOO_LARGE' }) + }, { ...LIMITS, maxImagePixels: 25, maxImageDimension: 4 }, POLICY)).rejects.toMatchObject({ code: 'IMAGE_DIMENSION_TOO_LARGE' }) const unnamed = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png', name: '\u0000', - }, LIMITS) - expect(unnamed).not.toHaveProperty('name') + }, LIMITS, POLICY) + expect(unnamed.ref).not.toHaveProperty('name') }) it('fails closed when an object is missing, corrupted, or addressed by an invalid reference', async () => { const storageRoot = await root() - const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS) + const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) const sha256 = String(ref.attachmentId).slice('sha256:'.length) const object = join(storageRoot, 'objects', sha256.slice(0, 2), sha256) await chmod(object, 0o600) @@ -216,11 +238,11 @@ describe('local attachment store', () => { const target = join(storageRoot, 'objects', sha256.slice(0, 2), sha256) await mkdir(join(storageRoot, 'objects', sha256.slice(0, 2)), { recursive: true }) await writeFile(target, Uint8Array.of(1, 2, 3)) - await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS)) + await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY)) .rejects.toMatchObject({ code: 'ATTACHMENT_CORRUPT' }) await writeFile(target, PNG) - const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS) + const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) await expect(readImageFile(storageRoot, { ...ref, width: ref.width + 1 })) .rejects.toMatchObject({ code: 'ATTACHMENT_CORRUPT' }) }) @@ -231,7 +253,7 @@ describe('local attachment store', () => { const target = join(storageRoot, 'objects', sha256.slice(0, 2), sha256) await mkdir(target, { recursive: true }) - await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS)) + await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY)) .rejects.toMatchObject({ code: 'ATTACHMENT_WRITE_FAILED' }) }) }) From 6e17c20804cd5c0c59d0f9658f81febbaceb4077 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 10:59:14 +0800 Subject: [PATCH 04/28] feat(tool-fs): read_image reports downscaled dimensions and coordinate scale When the attachment store's canonical encoding shrinks the file on disk, the read_image envelope names the original dimensions and the multiplier that maps coordinates measured on the attached image back onto the file, and the output schema carries sourceWidth/sourceHeight for programmatic callers. --- packages/fs/tool-fs/src/read-image.ts | 20 +++++++++-- packages/fs/tool-fs/tests/read-image.spec.ts | 35 ++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/packages/fs/tool-fs/src/read-image.ts b/packages/fs/tool-fs/src/read-image.ts index 074f816991..0cfaa93903 100644 --- a/packages/fs/tool-fs/src/read-image.ts +++ b/packages/fs/tool-fs/src/read-image.ts @@ -40,6 +40,10 @@ export interface ImageReadValue { width: number height: number name?: string + /** Intrinsic width of the file on disk; present only when storage downscaled it. */ + sourceWidth?: number + /** Intrinsic height of the file on disk; present only when storage downscaled it. */ + sourceHeight?: number } } @@ -93,15 +97,20 @@ export function imageRefFromValue(image: ImageReadValue['image']): ImageAttachme /** * Format an image read as the model-facing envelope beside its image block. + * A downscaled read names the on-disk dimensions and the multiplier that maps + * coordinates measured on the attached image back onto the original file. * @param displayPath - the backend-resolved path rendered in the envelope's `` element. * @param image - the canonical image metadata to summarize. * @returns the model-facing envelope; the image itself rides the adjacent image block. */ export function formatImageReadOutput(displayPath: string, image: ImageReadValue['image']): string { + const scaled = image.sourceWidth !== undefined && image.sourceHeight !== undefined + ? ` (downscaled from ${image.sourceWidth}x${image.sourceHeight} px; multiply coordinates by ${(image.sourceWidth / image.width).toFixed(2)} to locate features in the original file)` + : '' return `${displayPath} image -${image.mediaType} image, ${image.width}x${image.height} px, ${image.bytes} bytes +${image.mediaType} image, ${image.width}x${image.height} px, ${image.bytes} bytes${scaled} ` } @@ -150,6 +159,8 @@ export function applyReadImageTool(ctx: Context): void { width: { type: 'integer', required: true }, height: { type: 'integer', required: true }, name: { type: 'string' }, + sourceWidth: { type: 'integer' }, + sourceHeight: { type: 'integer' }, }, }, }, @@ -186,8 +197,11 @@ export function applyReadImageTool(ctx: Context): void { // Persist before returning: the image block must reference a durably // committed object by the time the tool/result event is appended. let ref: ImageAttachmentRef + let source: { width: number; height: number } try { - ref = (await attachments.saveImage({ data, mediaType, name: basename(target.displayPath) })).ref + const saved = await attachments.saveImage({ data, mediaType, name: basename(target.displayPath) }) + ref = saved.ref + source = saved.source } catch (error: unknown) { if (!(error instanceof AttachmentError)) throw error // Dimension refusals stay recoverable tool errors: an oversized image @@ -213,6 +227,7 @@ export function applyReadImageTool(ctx: Context): void { ) } ctx.emit('fs/observed', target, { kind: 'present', version: info.version }, exec) + const downscaled = source.width !== ref.width || source.height !== ref.height const value: ImageReadValue = { path: target.displayPath, image: { @@ -222,6 +237,7 @@ export function applyReadImageTool(ctx: Context): void { width: ref.width, height: ref.height, ...ref.name === undefined ? {} : { name: ref.name }, + ...downscaled ? { sourceWidth: source.width, sourceHeight: source.height } : {}, }, } return value diff --git a/packages/fs/tool-fs/tests/read-image.spec.ts b/packages/fs/tool-fs/tests/read-image.spec.ts index ca79c86315..6cea2cf18f 100644 --- a/packages/fs/tool-fs/tests/read-image.spec.ts +++ b/packages/fs/tool-fs/tests/read-image.spec.ts @@ -494,6 +494,41 @@ describe('image admission failures', () => { const image = result.content[1] as { attachment: ImageAttachmentRef } expect(image.attachment.name).toBeUndefined() }) + + it('names the on-disk dimensions and coordinate multiplier when storage downscales', async () => { + /** Store whose canonical encoding halves the source on both sides. */ + class DownscalingStore extends AttachmentStore { + readonly imageLimits: ImageAttachmentLimits = Object.freeze({ + maxImageBytes: 1024, + maxImagesPerMessage: 1, + maxMessageImageBytes: 1024, + maxImagePixels: 100, + maxImageDimension: 2000, + mediaTypes: Object.freeze(['image/png'] as const), + }) + + validateImage(_input: SaveImageAttachment): Promise { + return Promise.resolve() + } + + async saveImage(input: SaveImageAttachment): Promise { + return { + ref: { attachmentId: AttachmentId('sha256:feed'), mediaType: input.mediaType, bytes: 7, width: 2, height: 1 }, + source: { mediaType: input.mediaType, bytes: input.data.length, width: 4, height: 2 }, + } + } + + readImage(_ref: ImageAttachmentRef): Promise { + throw new Error('unreachable in this test') + } + } + await writeFile(join(dir, 'red.png'), PNG_1X1) + const ctx = await setup({ attachments: false }) + await ctx.plugin(DownscalingStore) + const result = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model')) + expect(result.isError).toBe(false) + expect(text(result)).toContain('image/png image, 2x1 px, 7 bytes (downscaled from 4x2 px; multiply coordinates by 2.00 to locate features in the original file)') + }) }) describe('registration surface', () => { From c6fa512e1581913ca1c4b3c2ed2364d3210176a9 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 11:22:00 +0800 Subject: [PATCH 05/28] fix(attachment-local): keep reference field order stable for logged fixtures The canonical ref serializes mediaType, width, height, bytes in the order the pre-canonicalization store used, so existing session-log fixtures and logged histories keep byte-identical reference JSON. --- packages/attachment/attachment-local/src/store.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/attachment/attachment-local/src/store.ts b/packages/attachment/attachment-local/src/store.ts index 9da83e30a0..27152d89cc 100644 --- a/packages/attachment/attachment-local/src/store.ts +++ b/packages/attachment/attachment-local/src/store.ts @@ -203,9 +203,9 @@ export async function saveImageFile( ref: { attachmentId: AttachmentId(`sha256:${sha256}`), mediaType: canonical.mediaType, - bytes: canonical.data.byteLength, width: canonical.width, height: canonical.height, + bytes: canonical.data.byteLength, ...(name !== undefined ? { name } : {}), }, source: metadata, From fec8aa62dfccfbb6bf9ba607e77672b36c80fce5 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 11:22:01 +0800 Subject: [PATCH 06/28] docs(attachment): document canonical admission; pin wide-image acceptance snapshot READMEs (both languages) describe the wide source envelope, the canonical encoding and its fixed encoder parameters, and read_image's downscale envelope; tool/config catalogs regenerate for the new schema and Config fields. The read-image-dimension scenario now pins the acceptance the old 2000px admission cap refused: the 2001x1 source is admitted and stored byte-identically, so the fixture stays platform-independent. --- docs/config-catalog.md | 12 ++++++++---- examples/acp-agent/tests/acp.snapshot.ts | 8 ++++---- .../tests/snapshots/read-image-dimension/input.json | 2 +- .../snapshots/read-image-dimension/session.jsonl | 10 +++++----- .../read-image-dimension/stdout.expected.jsonl | 2 +- .../attachment/attachment-local/README.i18n.yaml | 4 ++-- packages/attachment/attachment-local/README.md | 7 ++++--- packages/attachment/attachment-local/README.zh.md | 7 ++++--- packages/attachment/attachment/README.i18n.yaml | 4 ++-- packages/attachment/attachment/README.md | 2 +- packages/attachment/attachment/README.zh.md | 2 +- packages/fs/tool-fs/README.i18n.yaml | 4 ++-- packages/fs/tool-fs/README.md | 2 +- packages/fs/tool-fs/README.zh.md | 2 +- 14 files changed, 37 insertions(+), 31 deletions(-) diff --git a/docs/config-catalog.md b/docs/config-catalog.md index fa0e4caa36..c3ae3421d5 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -327,20 +327,24 @@ Source: [`packages/core/agent-tool-presentation/src/index.ts:38`](../packages/co export interface Config { /** Explicit harness home; omitted follows `DSH_HOME`, then `~/.dsh`. */ dshHome?: string - /** Maximum encoded bytes accepted for one image. */ + /** Maximum encoded bytes accepted for one submitted image. */ maxImageBytes?: number /** Maximum image count accepted in one submitted message. */ maxImagesPerMessage?: number /** Maximum aggregate encoded image bytes accepted in one submitted message. */ maxMessageImageBytes?: number - /** Maximum intrinsic width multiplied by height accepted for one image. */ + /** Maximum intrinsic width multiplied by height accepted for one submitted image. */ maxImagePixels?: number - /** Maximum intrinsic width and maximum intrinsic height accepted for one image. */ + /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. */ maxImageDimension?: number + /** Long-edge pixel target of the stored canonical encoding. */ + canonicalMaxDimension?: number + /** Encoded-byte target of the stored canonical encoding. */ + canonicalMaxBytes?: number } ``` -Source: [`packages/attachment/attachment-local/src/index.ts:31`](../packages/attachment/attachment-local/src/index.ts) +Source: [`packages/attachment/attachment-local/src/index.ts:36`](../packages/attachment/attachment-local/src/index.ts) diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index c2d7e44403..a6f12fdad5 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -250,10 +250,10 @@ const SCENARIOS: Scenario[] = [ toolSchemasSource: 'read-image', configPath: IMAGE_TEXT_ROUTE_CONFIG, }, - // Authored keyless replay of the oversized-image refusal: admission rejects - // the 2001x1 fixture at the default 2000px per-side limit, the model sees a - // recoverable tool error, and the turn still completes — the image never - // enters durable history. + // Authored keyless replay of wide-image admission: the 2001x1 fixture sits + // inside the wide source envelope and the canonical budget, so read_image + // succeeds and the attachment keeps the source bytes byte-identically — + // the same read the pre-canonicalization 2000px admission cap refused. { name: 'read-image-dimension', hasModelTurn: true, diff --git a/examples/acp-agent/tests/snapshots/read-image-dimension/input.json b/examples/acp-agent/tests/snapshots/read-image-dimension/input.json index 43e6299ef8..ff366b1109 100644 --- a/examples/acp-agent/tests/snapshots/read-image-dimension/input.json +++ b/examples/acp-agent/tests/snapshots/read-image-dimension/input.json @@ -8,7 +8,7 @@ }, { "op": "prompt", - "text": "Use read_image on wide.png in the current directory. If the tool refuses because the image is too large, reply with exactly the single word TOOLARGE." + "text": "Use read_image on wide.png in the current directory, then reply with exactly the single word WIDE." } ] } diff --git a/examples/acp-agent/tests/snapshots/read-image-dimension/session.jsonl b/examples/acp-agent/tests/snapshots/read-image-dimension/session.jsonl index db755998fb..9bd9a47fb2 100644 --- a/examples/acp-agent/tests/snapshots/read-image-dimension/session.jsonl +++ b/examples/acp-agent/tests/snapshots/read-image-dimension/session.jsonl @@ -1,9 +1,9 @@ {"type":"session","version":0,"id":"33333333-3333-4333-8333-333333333333","createdAt":1783951000000,"cwd":"{{cwd}}","delegationDepth":0} -{"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"inserted":[{"content":[{"type":"text","text":"Use read_image on wide.png in the current directory. If the tool refuses because the image is too large, reply with exactly the single word TOOLARGE."}],"source":{"kind":"user"},"role":"user","id":"0a0a0a0a-0000-4000-8000-000000000001"}]}} +{"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"inserted":[{"content":[{"type":"text","text":"Use read_image on wide.png in the current directory, then reply with exactly the single word WIDE."}],"source":{"kind":"user"},"role":"user","id":"0a0a0a0a-0000-4000-8000-000000000001"}]}} {"type":"turn/start","data":{"turn":1}} {"type":"agent/inbox/spliced","data":{"target":"next-turn","start":0,"removedCount":1,"inserted":[]}} {"type":"step/start","data":{"turn":1,"step":1}} -{"type":"user/message","data":{"content":[{"type":"text","text":"Use read_image on wide.png in the current directory. If the tool refuses because the image is too large, reply with exactly the single word TOOLARGE."}],"source":{"kind":"user"},"role":"user","id":"0a0a0a0a-0000-4000-8000-000000000001"},"surfaceOp":"append"} +{"type":"user/message","data":{"content":[{"type":"text","text":"Use read_image on wide.png in the current directory, then reply with exactly the single word WIDE."}],"source":{"kind":"user"},"role":"user","id":"0a0a0a0a-0000-4000-8000-000000000001"},"surfaceOp":"append"} {"type":"user/message","data":{"content":[{"type":"text","text":"Current runtime context. This snapshot supersedes earlier runtime-context snapshots.\n\nCurrent DSH file policy: danger-full-access. The DSH file sandbox does not restrict file modifications by available operations.\n\nApproval prompts are disabled in this session: actions that require approval are rejected automatically — do not request sandbox escalation (do not set `sandbox_permissions`)."}],"source":{"kind":"plugin","plugin":"@deepseek-ai/dsh-system-prompt","form":"snapshot","sections":[{"name":"sandbox:policy","text":"Current DSH file policy: danger-full-access. The DSH file sandbox does not restrict file modifications by available operations."},{"name":"approval:policy","text":"Approval prompts are disabled in this session: actions that require approval are rejected automatically — do not request sandbox escalation (do not set `sandbox_permissions`)."}]},"role":"user","id":"11a08f07-014a-408b-bfc5-634770ce7179"},"surfaceOp":"append"} {"type":"session/title","data":{"title":"Use read_image on wide.png in","messageSeqs":[4],"source":{"kind":"fallback"}}} {"type":"request/header","data":{"header":{"config":{"provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"system":"{{system}}","tools":"{{tools}}"},"reason":"initial"}} @@ -14,13 +14,13 @@ {"type":"assistant/chunk","data":{"turn":1,"step":1,"chunk":{"type":"finish","reason":{"kind":"tool-calls"}}}} {"type":"assistant/message","data":{"turn":1,"step":1,"message":{"role":"assistant","content":[{"type":"tool-call","id":"read-image-dimension","name":"read_image","arguments":"{\"file_path\":\"wide.png\"}"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"a25d70ac-2bd6-4e44-9121-ed74975ee229"},"usage":{"inputTokens":3,"outputTokens":3}},"sourceEventSeqs":[9,10,11,12],"surfaceOp":"append"} {"type":"tool/call","data":{"turn":1,"step":1,"callId":"read-image-dimension","name":"read_image","arguments":"{\"file_path\":\"wide.png\"}"}} -{"type":"tool/result","data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"read-image-dimension"},"content":[{"type":"tool-result","toolCallId":"read-image-dimension","content":[{"type":"text","text":"Error: cannot read \"{{cwd}}/wide.png\": at least one image side exceeds the 2000px limit; downscale the image and read the smaller copy"}],"isError":true}],"role":"user","id":"ee31751e-df5a-458e-8497-8113cf6107ef"}},"sourceEventSeqs":[14],"surfaceOp":"append"} +{"type":"tool/result","data":{"turn":1,"step":1,"message":{"source":{"kind":"tool","callId":"read-image-dimension"},"content":[{"type":"tool-result","toolCallId":"read-image-dimension","content":[{"type":"text","text":"{{cwd}}/wide.png\nimage\n\nimage/png image, 2001x1 px, 133 bytes\n"},{"type":"image","attachment":{"attachmentId":"sha256:0333f95051f5c038cab720d90112f1775e9ff1f8f7dddc86653e80ff241c5720","mediaType":"image/png","bytes":133,"width":2001,"height":1,"name":"wide.png"}}],"isError":false}],"role":"user","id":"ee31751e-df5a-458e-8497-8113cf6107ef"}},"sourceEventSeqs":[14],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":1}} {"type":"step/start","data":{"turn":1,"step":2}} {"type":"assistant/chunk","data":{"turn":1,"step":2,"chunk":{"type":"block-start","index":0,"blockType":"text"}}} -{"type":"assistant/chunk","data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"TOOLARGE"}}}} +{"type":"assistant/chunk","data":{"turn":1,"step":2,"chunk":{"type":"block-end","index":0,"block":{"type":"text","text":"WIDE"}}}} {"type":"assistant/chunk","data":{"turn":1,"step":2,"chunk":{"type":"usage","usage":{"inputTokens":3,"outputTokens":3}}}} {"type":"assistant/chunk","data":{"turn":1,"step":2,"chunk":{"type":"finish","reason":{"kind":"stop"}}}} -{"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"text","text":"TOOLARGE"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"3a95dd83-34f7-4bc0-afb6-7ba3c9b483be"},"usage":{"inputTokens":3,"outputTokens":3}},"sourceEventSeqs":[18,19,20,21],"surfaceOp":"append"} +{"type":"assistant/message","data":{"turn":1,"step":2,"message":{"role":"assistant","content":[{"type":"text","text":"WIDE"}],"source":{"kind":"model","provider":"deepseek-official","model":"deepseek-v4-flash-vision-exp"},"id":"3a95dd83-34f7-4bc0-afb6-7ba3c9b483be"},"usage":{"inputTokens":3,"outputTokens":3}},"sourceEventSeqs":[18,19,20,21],"surfaceOp":"append"} {"type":"step/end","data":{"turn":1,"step":2}} {"type":"turn/end","data":{"turn":1,"reason":{"kind":"completed"}}} diff --git a/examples/acp-agent/tests/snapshots/read-image-dimension/stdout.expected.jsonl b/examples/acp-agent/tests/snapshots/read-image-dimension/stdout.expected.jsonl index 7dbc881712..80d27b8114 100644 --- a/examples/acp-agent/tests/snapshots/read-image-dimension/stdout.expected.jsonl +++ b/examples/acp-agent/tests/snapshots/read-image-dimension/stdout.expected.jsonl @@ -1,4 +1,4 @@ {"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"promptCapabilities":{"image":true,"audio":false,"embeddedContext":false}},"authMethods":[]}} {"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}} -{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"TOOLARGE"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"WIDE"}}}} {"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}} diff --git a/packages/attachment/attachment-local/README.i18n.yaml b/packages/attachment/attachment-local/README.i18n.yaml index 1d7c63c469..11216d1aa1 100644 --- a/packages/attachment/attachment-local/README.i18n.yaml +++ b/packages/attachment/attachment-local/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/attachment/attachment-local/README.md -README.md: e4f2d5748768a1dc2a6b79c3ed9e364c56a67248 -README.zh.md: 6b548fb993faef996f1508ba9f9efc31b20fea64 +README.md: e8f89906f7bedb20b80a04ab6d2aa4b80c6d746f +README.zh.md: 61e1dde94751436bb4d8ff4dde1b68b1b10fc005 diff --git a/packages/attachment/attachment-local/README.md b/packages/attachment/attachment-local/README.md index e4f2d57487..e8f89906f7 100644 --- a/packages/attachment/attachment-local/README.md +++ b/packages/attachment/attachment-local/README.md @@ -2,7 +2,7 @@ English | [中文](README.zh.md) -The private local implementation of [`@deepseek-ai/dsh-attachment`](../attachment). Objects land at `/attachments/v1/objects//` and are addressed by an opaque `sha256:` id. Each process proves a home durable once by syncing every ancestor entry to the filesystem root, so a directory another process created but has not yet synced is never mistaken for a safe boundary. Writes then use a private staging directory, owner-only files, a synced temporary file, an atomic exclusive hard-link publish, and directory syncs on the publication path (POSIX; Windows relies on filesystem metadata journaling) so the reported reference survives a crash. Write admission and reads fully decode the raster before accepting its format and dimensions; reads also re-check the digest and logged metadata. Byte, total-pixel, and per-side dimension limits are write-time admission policy, so a later policy reduction does not make already-admitted history unreadable. The per-side default (2000px) stays below the strictest dimension bound deployed model routes enforce on requests carrying many images: an admitted image rides every later request of its session, so admission is the last point where a provider-rejected image can be kept out of durable history. +The private local implementation of [`@deepseek-ai/dsh-attachment`](../attachment). Objects land at `/attachments/v1/objects//` and are addressed by an opaque `sha256:` id. Each process proves a home durable once by syncing every ancestor entry to the filesystem root, so a directory another process created but has not yet synced is never mistaken for a safe boundary. Writes then use a private staging directory, owner-only files, a synced temporary file, an atomic exclusive hard-link publish, and directory syncs on the publication path (POSIX; Windows relies on filesystem metadata journaling) so the reported reference survives a crash. Write admission fully decodes the raster against a wide source envelope — byte, total-pixel, and per-side caps (defaults 32MiB, 100MP, 16384px) — and then persists a deterministic canonical encoding instead of the submitted bytes: EXIF orientation is baked into pixels, metadata is stripped, the long edge is downscaled to the configured canonical target (default 2048px), sources with alpha or PNG/GIF lineage encode as palette PNG and photographic sources as JPEG, stepping down a fixed quality ladder (85/75/60/45) until the configured canonical byte target holds (default 1MiB). A PNG/JPEG/WebP source already inside the canonical budget is stored byte-identically, so equal originals keep deduplicating to one content address; GIF always re-encodes to the PNG of its first frame, pinning at admission the first-frame meaning providers apply. Encoder parameters are deliberately fixed rather than configurable, because a parameter change would silently split the content-addressed space; the deployment chooses only the source envelope and the canonical budget. An admitted image rides every later request of its session, so canonicalizing at admission is what bounds durable history without refusing ordinary large sources. Reads re-check the digest and logged metadata, and a later policy reduction does not make already-admitted history unreadable. `DSH_HOME` resolves through the shared path policy: explicit config, `$DSH_HOME`, then `~/.dsh`. Session logs contain only the reference and verified metadata, never this host path. `readImage` forwards optional cancellation into the filesystem read, observes it around verification, and preserves it instead of wrapping it as `ATTACHMENT_READ_FAILED`. @@ -12,10 +12,11 @@ Indirectly, through durable replay of historical user images and structured mode #### KV Cache effect -None beyond the image block owned by the requesting adapter. +Canonicalization happens once at admission and is deterministic, so a stored image contributes identical request bytes on every later turn; nothing here re-encodes per request. ## Known Limitations and Deferred Work - Objects are retained indefinitely; reference-aware garbage collection is deferred. - The local backend assumes the host and provider adapter share this filesystem service. -- Animated GIF metadata is validated from the logical screen; frame-level decoding policy is provider-owned. +- Animated GIF sources keep only their first frame; animation is outside the version-one image contract. +- The canonical encoder is pinned by the installed sharp/libvips build; an encoder upgrade re-addresses future saves of the same source while already-stored objects stay valid. diff --git a/packages/attachment/attachment-local/README.zh.md b/packages/attachment/attachment-local/README.zh.md index 6b548fb993..61e1dde947 100644 --- a/packages/attachment/attachment-local/README.zh.md +++ b/packages/attachment/attachment-local/README.zh.md @@ -2,7 +2,7 @@ [English](README.md) | 中文 -这是 [`@deepseek-ai/dsh-attachment`](../attachment) 的私有本地实现。对象存放在 `/attachments/v1/objects//`,并通过不透明的 `sha256:` 标识符寻址。每个进程都会通过将每个祖先目录项逐级同步到文件系统根目录,为某个 home 一次性证明其持久性,因此绝不会把另一个进程已经创建但尚未同步的目录误认为安全边界。随后,写入过程使用私有暂存目录、仅所有者可访问的文件、经过同步的临时文件、原子且排他的硬链接发布,并对发布路径执行目录同步(适用于 POSIX;Windows 依赖文件系统元数据日志),确保已报告的引用能够在崩溃后继续存在。写入准入与读取都会完整解码光栅图片,之后才接受其格式和尺寸;读取还会重新校验摘要和已记录的元数据。字节、总像素和单边尺寸限制属于写入时的准入策略,因此后续收紧限制不会导致已经接纳的历史记录变得不可读。单边默认值(2000px)低于已部署模型路由对携带多张图片的请求所强制执行的最严格尺寸上限:一张已接纳的图片会随会话之后的每次请求发送,准入是把必然被上游拒绝的图片挡在持久历史之外的最后一道关口。 +这是 [`@deepseek-ai/dsh-attachment`](../attachment) 的私有本地实现。对象存放在 `/attachments/v1/objects//`,并通过不透明的 `sha256:` 标识符寻址。每个进程都会通过将每个祖先目录项逐级同步到文件系统根目录,为某个 home 一次性证明其持久性,因此绝不会把另一个进程已经创建但尚未同步的目录误认为安全边界。随后,写入过程使用私有暂存目录、仅所有者可访问的文件、经过同步的临时文件、原子且排他的硬链接发布,并对发布路径执行目录同步(适用于 POSIX;Windows 依赖文件系统元数据日志),确保已报告的引用能够在崩溃后继续存在。写入准入会按宽松的源图上限(字节、总像素、单边,默认 32MiB、1 亿像素、16384px)完整解码光栅图片,然后持久保存确定性的规范编码而不是提交的原始字节:EXIF 方向落实到像素并剥离元数据,长边等比缩放到配置的规范目标(默认 2048px),带透明通道或源自 PNG/GIF 的图片编码为 palette PNG,摄影类图片编码为 JPEG,并沿固定的质量阶梯(85/75/60/45)递降,直到满足配置的规范字节目标(默认 1MiB)。已在规范预算内的 PNG/JPEG/WebP 源图按字节原样存储,因此相同原图始终去重到同一个内容地址;GIF 一律重编码为其首帧的 PNG,在准入时就固化提供方实际采用的首帧语义。编码器参数刻意固定而不可配置,因为参数变化会悄悄割裂内容寻址空间;部署只选择源图上限与规范预算。一张已接纳的图片会随会话之后的每次请求发送,所以在准入时规范化才能在不拒绝普通大图的前提下约束持久历史。读取会重新校验摘要和已记录的元数据,后续收紧限制不会导致已经接纳的历史记录变得不可读。 `DSH_HOME` 按共享路径策略解析:显式配置、`$DSH_HOME`,最后是 `~/.dsh`。会话日志只包含引用和经过校验的元数据,绝不包含这个宿主路径。`readImage` 会把可选取消信号传入文件系统读取、在校验前后观察该信号,并保留取消语义,而不会将其包装成 `ATTACHMENT_READ_FAILED`。 @@ -12,10 +12,11 @@ #### KV 缓存影响 -除发起请求的适配器所持有的图片块外,不产生其他影响。 +规范化只在准入时发生一次且是确定性的,因此一张已存储的图片在之后每一轮贡献完全相同的请求字节;这里没有任何按请求重编码的环节。 ## 已知限制与待完成工作 - 对象会无限期保留;基于引用的垃圾回收尚未实现。 - 本地后端假定宿主与提供方适配器共享同一个文件系统服务。 -- 动态 GIF 的元数据根据逻辑屏幕进行校验;逐帧解码策略由提供方持有。 +- 动态 GIF 源图只保留首帧;动画在版本一图片契约之外。 +- 规范编码器由安装的 sharp/libvips 构建钉定;编码器升级会让同一源图之后的保存得到新地址,已存储对象保持有效。 diff --git a/packages/attachment/attachment/README.i18n.yaml b/packages/attachment/attachment/README.i18n.yaml index fd02d455a4..97ec722871 100644 --- a/packages/attachment/attachment/README.i18n.yaml +++ b/packages/attachment/attachment/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/attachment/attachment/README.md -README.md: 19232bd4bb86ed33e56fcdca93999967822422ab -README.zh.md: e5e7aab7c1af30b2b101bdcd218044cd1095ae0d +README.md: 89bc3ca3a288450c43fefb5dde38da7f65218f43 +README.zh.md: ca13c9e66234b280f7fa9d01fc80bd026604831f diff --git a/packages/attachment/attachment/README.md b/packages/attachment/attachment/README.md index 19232bd4bb..89bc3ca3a2 100644 --- a/packages/attachment/attachment/README.md +++ b/packages/attachment/attachment/README.md @@ -4,7 +4,7 @@ English | [中文](README.zh.md) The durable attachment seam. `ctx.attachments` validates and durably commits immutable image bytes, then returns a serializable `ImageAttachmentRef`; consumers never persist browser paths, object URLs, provider URLs, or base64 in session events. -Unsent composer images remain browser-owned temporary drafts. `validateImage` runs the same admission policy without persisting. `saveImages` owns batch count and aggregate-byte limits, validates every member before writing any member, then commits in order and returns references only after the complete batch succeeds. A later storage failure returns no partial references, although an earlier immutable content-addressed object may remain unreachable until reference-aware garbage collection exists. `AttachmentError.code` uses the closed `AttachmentErrorCode` string union. Its `ImageAdmissionErrorCode` subset marks caller-correctable image-input failures; `isImageAdmissionError` recognizes that subset at runtime so each protocol adapter can map its own error vocabulary. `saveImage` commits one accepted image before any model-visible session event is published, and `readImage` verifies the content-addressed object against its logged metadata. Callers may cancel `readImage`; implementations observe cancellation around backend and verification work and preserve it instead of translating it into a storage failure. +Unsent composer images remain browser-owned temporary drafts. `validateImage` runs the same admission policy without persisting. `saveImages` owns batch count and aggregate-byte limits, validates every member before writing any member, then commits in order and returns references only after the complete batch succeeds. A later storage failure returns no partial references, although an earlier immutable content-addressed object may remain unreachable until reference-aware garbage collection exists. `AttachmentError.code` uses the closed `AttachmentErrorCode` string union. Its `ImageAdmissionErrorCode` subset marks caller-correctable image-input failures; `isImageAdmissionError` recognizes that subset at runtime so each protocol adapter can map its own error vocabulary. `saveImage` commits one accepted image before any model-visible session event is published and resolves `SavedImageAttachment`: an implementation may persist a canonical re-encoding of the submitted raster, so the returned `ref` always describes the stored bytes while `source` (`SourceImageInfo`) preserves the submitted raster's media type, byte length, and dimensions for callers that report or map coordinates against the original. `readImage` verifies the content-addressed object against its logged metadata. Callers may cancel `readImage`; implementations observe cancellation around backend and verification work and preserve it instead of translating it into a storage failure. `admitEncodedImages(attachments, images)` is the shared wire entry used by every RPC endpoint that accepts browser uploads (the session prompt endpoint and the command executor): it enforces canonical base64 on every member, then delegates batch admission — limits, validation, ordered commit — to `saveImages`. The base64 upload form is `EncodedImageAttachment`, exported from `@deepseek-ai/dsh-attachment/types` so wire contracts can reference it. diff --git a/packages/attachment/attachment/README.zh.md b/packages/attachment/attachment/README.zh.md index e5e7aab7c1..ca13c9e662 100644 --- a/packages/attachment/attachment/README.zh.md +++ b/packages/attachment/attachment/README.zh.md @@ -4,7 +4,7 @@ 持久附件服务边界。`ctx.attachments` 校验并持久提交不可变图片字节,随后返回可序列化的 `ImageAttachmentRef`;消费方绝不会在会话事件中持久保存浏览器路径、对象 URL、提供方 URL 或 base64。 -未发送的输入区图片仍是由浏览器持有的临时草稿。`validateImage` 运行相同的准入策略,但不执行持久化。`saveImages` 负责批次图片数量和总字节限制,先校验全部成员,再按顺序提交,并且只在完整批次成功后返回引用。后续存储失败不会返回部分引用,但较早写入的不可变内容寻址对象可能保持不可达,直至具备按引用感知的垃圾回收。`AttachmentError.code` 使用封闭的 `AttachmentErrorCode` 字符串联合类型。其 `ImageAdmissionErrorCode` 子集标记可由调用方修正的图片输入失败;`isImageAdmissionError` 在运行时识别该子集,使每个协议适配器可以映射自己的错误词汇。`saveImage` 会在发布任何模型可见的会话事件前提交一张已接受的图片,`readImage` 则根据已记录的元数据校验内容寻址对象。调用方可以取消 `readImage`;实现会在后端读取与校验工作的边界观察取消,并保留取消语义,而不会将其转换为存储失败。 +未发送的输入区图片仍是由浏览器持有的临时草稿。`validateImage` 运行相同的准入策略,但不执行持久化。`saveImages` 负责批次图片数量和总字节限制,先校验全部成员,再按顺序提交,并且只在完整批次成功后返回引用。后续存储失败不会返回部分引用,但较早写入的不可变内容寻址对象可能保持不可达,直至具备按引用感知的垃圾回收。`AttachmentError.code` 使用封闭的 `AttachmentErrorCode` 字符串联合类型。其 `ImageAdmissionErrorCode` 子集标记可由调用方修正的图片输入失败;`isImageAdmissionError` 在运行时识别该子集,使每个协议适配器可以映射自己的错误词汇。`saveImage` 会在发布任何模型可见的会话事件前提交一张已接受的图片,并解析为 `SavedImageAttachment`:实现可以持久保存所提交光栅的规范重编码,因此返回的 `ref` 始终描述实际存储的字节,而 `source`(`SourceImageInfo`)保留所提交光栅的媒体类型、字节长度和尺寸,供需要对照原图汇报或换算坐标的调用方使用。`readImage` 则根据已记录的元数据校验内容寻址对象。调用方可以取消 `readImage`;实现会在后端读取与校验工作的边界观察取消,并保留取消语义,而不会将其转换为存储失败。 `admitEncodedImages(attachments, images)` 是每个接受浏览器上传的 RPC 端点(会话 prompt 端点与命令执行器)共用的 wire 入口:它对每个成员强制执行规范 base64,随后把批量准入——限额、校验、有序提交——委托给 `saveImages`。base64 上传形式为 `EncodedImageAttachment`,从 `@deepseek-ai/dsh-attachment/types` 导出,供 wire 契约引用。 diff --git a/packages/fs/tool-fs/README.i18n.yaml b/packages/fs/tool-fs/README.i18n.yaml index 068531d79a..3d9c4606c4 100644 --- a/packages/fs/tool-fs/README.i18n.yaml +++ b/packages/fs/tool-fs/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/fs/tool-fs/README.md -README.md: ce7c0ea9070e30c1e6b538933ff5c4605b8d59cc -README.zh.md: 88d27289a7ef087aa8ad2791fc9b9e0d3e1fba2e +README.md: 22384ddb18f2b36e9b8a177ee62eed9424ddcd6a +README.zh.md: 74c41f4f25d19089c40a52cff4e3dfa654630b1a diff --git a/packages/fs/tool-fs/README.md b/packages/fs/tool-fs/README.md index ce7c0ea907..22384ddb18 100644 --- a/packages/fs/tool-fs/README.md +++ b/packages/fs/tool-fs/README.md @@ -38,7 +38,7 @@ All keys are optional; the defaults are the shipped read caps. Field names are snake_case to match Claude Code and existing harness tool schemas. -Canonical successes are `read` → `{ path, offset, lines: [{ number, text }], totalLines }`, `read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name? } }`, `write` → `{ path, operation: 'create' | 'update', before: string | null, after }`, and `edit` → `{ path, before, after }`. Native renderers preserve the line-numbered read and mutation acknowledgements below. `write`/`edit` derive replayable diff-card metadata, and `read` derives a replayable read-card window `{ path, offset, lines, totalLines, lang? }`, from these canonical values; the canonical values themselves are execution-local and are not added to `tool/result`, only the derived presentation metadata is persisted. +Canonical successes are `read` → `{ path, offset, lines: [{ number, text }], totalLines }`, `read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name?, sourceWidth?, sourceHeight? } }` (the source fields appear only when the attachment store's canonical encoding downscaled the file, and the envelope then names the coordinate multiplier back to the original), `write` → `{ path, operation: 'create' | 'update', before: string | null, after }`, and `edit` → `{ path, before, after }`. Native renderers preserve the line-numbered read and mutation acknowledgements below. `write`/`edit` derive replayable diff-card metadata, and `read` derives a replayable read-card window `{ path, offset, lines, totalLines, lang? }`, from these canonical values; the canonical values themselves are execution-local and are not added to `tool/result`, only the derived presentation metadata is persisted. ## The tool is the executor; policy is an event gate diff --git a/packages/fs/tool-fs/README.zh.md b/packages/fs/tool-fs/README.zh.md index 88d27289a7..74c41f4f25 100644 --- a/packages/fs/tool-fs/README.zh.md +++ b/packages/fs/tool-fs/README.zh.md @@ -38,7 +38,7 @@ await ctx.plugin(ToolFs) // this package — re 字段名使用 snake_case,与 Claude Code 和现有 harness 工具 schema 一致。 -规范成功值分别为:`read` → `{ path, offset, lines: [{ number, text }], totalLines }`,`read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name? } }`,`write` → `{ path, operation: 'create' | 'update', before: string | null, after }`,`edit` → `{ path, before, after }`。原生渲染器会保留下方带行号的读取结果和变更确认。`write`/`edit` 从这些规范值派生可回放的 diff 卡片元数据,`read` 派生可回放的读取卡片窗口 `{ path, offset, lines, totalLines, lang? }`;规范值本身仅限于本次执行,不会添加到 `tool/result`,只有派生出的呈现元数据会被持久化。 +规范成功值分别为:`read` → `{ path, offset, lines: [{ number, text }], totalLines }`,`read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name?, sourceWidth?, sourceHeight? } }`(source 两个字段仅在附件存储的规范编码缩小了该文件时出现,此时信封会写明换算回原图的坐标倍率),`write` → `{ path, operation: 'create' | 'update', before: string | null, after }`,`edit` → `{ path, before, after }`。原生渲染器会保留下方带行号的读取结果和变更确认。`write`/`edit` 从这些规范值派生可回放的 diff 卡片元数据,`read` 派生可回放的读取卡片窗口 `{ path, offset, lines, totalLines, lang? }`;规范值本身仅限于本次执行,不会添加到 `tool/result`,只有派生出的呈现元数据会被持久化。 ## 工具就是执行器;策略是事件门禁 From 867dc446976a9277e41f5f0b9f60c50697e8d4f6 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 11:23:32 +0800 Subject: [PATCH 07/28] docs(notes): record the canonical image admission decision --- ...-08-20-canonical-image-admission.i18n.yaml | 6 ++++ .../2026-08-20-canonical-image-admission.md | 28 +++++++++++++++++++ ...2026-08-20-canonical-image-admission.zh.md | 28 +++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 .agents/notes/implemented/feature/2026-08-20-canonical-image-admission.i18n.yaml create mode 100644 .agents/notes/implemented/feature/2026-08-20-canonical-image-admission.md create mode 100644 .agents/notes/implemented/feature/2026-08-20-canonical-image-admission.zh.md diff --git a/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.i18n.yaml b/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.i18n.yaml new file mode 100644 index 0000000000..28a4b212df --- /dev/null +++ b/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-08-20-canonical-image-admission.md +2026-08-20-canonical-image-admission.md: bae3ce8b93b7fbfc4d3233cb3ed019f6ab09c0b4 +2026-08-20-canonical-image-admission.zh.md: a8c55383eb0c8b244dc1fefe1186004e2b869d3e diff --git a/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.md b/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.md new file mode 100644 index 0000000000..bae3ce8b93 --- /dev/null +++ b/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.md @@ -0,0 +1,28 @@ +# Agent Note: Canonical image admission + +Status: implemented + +English | [中文](2026-08-20-canonical-image-admission.zh.md) + +## Problem + +Admission used to refuse any image above 2000px per side or 3.5 MiB, because an admitted image rides every later request and deployed routes reject oversized images. Refusal pushed the problem onto the user (downscale by hand, re-attach), and the byte size of admitted images was uncontrolled below the cap, so long sessions accumulated large request payloads. The unified image-pipeline design (PR #2676) needs a canonical, deterministic stored form as the basis for content-addressed dedup, stable request bytes, and a later provider-files upload path. + +## Decision + +`AttachmentStore.saveImage` resolves `SavedImageAttachment`: the durable `ref` describing stored bytes beside `source` facts of the submitted raster. The local store validates a wide source envelope (32 MiB, 100 MP, 16384px per side) and persists a deterministic canonical encoding: EXIF orientation baked in, metadata stripped, long edge downscaled to `canonicalMaxDimension` (default 2048px), palette PNG for alpha/PNG/GIF lineage and JPEG for photographic sources, stepping a fixed quality ladder (85/75/60/45) until `canonicalMaxBytes` (default 1 MiB) holds. An in-budget PNG/JPEG/WebP source passes through byte-identically, so equal originals keep one content address; GIF always becomes the PNG of its first frame, pinning the first-frame meaning providers apply. Encoder parameters are fixed, not configurable — a parameter change would silently split the content-addressed space — so deployments choose only the source envelope and the canonical budget. The canonical ref keeps the pre-existing field order (`mediaType`, `width`, `height`, `bytes`) so logged references stay byte-identical. `read_image` reports the on-disk dimensions and the coordinate multiplier whenever storage downscaled the file. + +## Alternatives considered + +- **Keep refusing oversized sources.** Simple, but hostile at exactly the moment a user pastes a normal screenshot from a HiDPI display, and it leaves admitted byte sizes unbounded below the cap. +- **Canonicalize at request time.** Re-encoding per request breaks byte-stable prefixes (provider context caching) and violates the design's rule that durable content is written once; the request layer only projects. +- **Make encoder quality configurable.** Two deployments with different quality would address the same source at different ids, silently defeating dedup; fixed parameters keep the space whole and an encoder upgrade re-addresses only future saves. +- **Pin a resize transcript snapshot.** A fixture embedding re-encoded bytes depends on cross-platform encoder byte-stability (libvips resize and palette quantization across arm64/x86), which is unverified in CI; the assembled snapshot instead pins the acceptance passthrough (2001x1 admitted byte-identically), and re-encode branches are pinned by package tests. + +## Verification + +Package tests cover passthrough identity, resize determinism and idempotence, GIF-to-PNG, alpha-to-PNG, JPEG ladder descent, ladder exhaustion refusal, encoder-fault mapping, and the store round-trip of a downscaled save. The read-image suite pins the downscale envelope text. The `read-image-dimension` keyless snapshot now pins the acceptance the 2000px cap used to refuse, using passthrough bytes so the fixture is platform-independent. + +## Consequences + +Ordinary large sources are admitted and bounded (≤2048px, ≤1 MiB by default), shrinking per-request image payload roughly 3.5x at the old cap and making the planned request-level budgets rarely reachable. Stored bytes may differ from the submitted file; consumers that map coordinates use the saved `source` facts, as `read_image` does. A cross-platform byte-stability check for the re-encode path remains open before any fixture may embed re-encoded bytes. diff --git a/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.zh.md b/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.zh.md new file mode 100644 index 0000000000..a8c55383eb --- /dev/null +++ b/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.zh.md @@ -0,0 +1,28 @@ +# Agent Note: 规范化图片准入 + +Status: implemented + +[English](2026-08-20-canonical-image-admission.md) | 中文 + +## 问题 + +准入过去拒绝任何单边超过 2000px 或超过 3.5 MiB 的图片,因为已接纳的图片会随之后每次请求发送,而已部署路由会拒绝过大的图片。拒绝把问题推给了用户(手动缩图再重新附上),而且上限以内的已接纳图片字节数不受控制,长会话会累积出很大的请求载荷。统一图片管线设计(PR #2676)需要一个规范且确定性的存储形态,作为内容寻址去重、请求字节稳定以及后续 provider files 上传路径的基础。 + +## 决定 + +`AttachmentStore.saveImage` 解析为 `SavedImageAttachment`:描述实际存储字节的持久 `ref`,加上所提交光栅的 `source` 事实。本地存储按宽松的源图上限(32 MiB、1 亿像素、单边 16384px)校验,然后持久保存确定性的规范编码:EXIF 方向落实到像素、剥离元数据、长边等比缩放到 `canonicalMaxDimension`(默认 2048px),带透明通道或源自 PNG/GIF 的图片编码为 palette PNG,摄影类图片编码为 JPEG,并沿固定质量阶梯(85/75/60/45)递降直到满足 `canonicalMaxBytes`(默认 1 MiB)。已在预算内的 PNG/JPEG/WebP 源图按字节原样存储,相同原图保持同一个内容地址;GIF 一律转为首帧 PNG,在准入时固化提供方实际采用的首帧语义。编码器参数固定而不可配置,因为参数变化会悄悄割裂内容寻址空间;部署只选择源图上限与规范预算。规范 ref 保持原有字段顺序(`mediaType`、`width`、`height`、`bytes`),已记录的引用保持字节一致。存储缩小了文件时,`read_image` 会报告磁盘上的原始尺寸和坐标换算倍率。 + +## 考虑过的替代方案 + +- **继续拒绝超限源图。** 简单,但恰恰在用户从 HiDPI 屏幕粘贴一张普通截图的时刻表现得不友好,而且上限以内的已接纳字节数仍然无界。 +- **在请求时规范化。** 按请求重编码会破坏字节稳定前缀(provider 上下文缓存),也违反设计中「持久内容只写一次、请求层只做投影」的规则。 +- **让编码质量可配置。** 两个部署用不同质量会把同一源图寻址到不同 id,悄悄破坏去重;固定参数保持寻址空间完整,编码器升级只影响之后的保存。 +- **钉一个缩放的 transcript 快照。** 嵌入重编码字节的 fixture 依赖跨平台编码器字节稳定性(libvips 缩放与调色板量化在 arm64/x86 上的表现),CI 尚未验证;组装快照改为钉住接纳直通行为(2001x1 按字节原样接纳),重编码分支由包测试钉住。 + +## 验证 + +包测试覆盖直通恒等、缩放确定性与幂等、GIF 转 PNG、透明通道转 PNG、JPEG 阶梯递降、阶梯穷尽拒绝、编码器故障映射,以及缩小保存的存储往返。read-image 测试钉住缩放信封文本。`read-image-dimension` keyless 快照现在钉住 2000px 上限过去拒绝的接纳行为,使用直通字节因此 fixture 与平台无关。 + +## 后果 + +普通大图会被接纳并受约束(默认 ≤2048px、≤1 MiB),在旧上限处把单请求图片载荷缩小约 3.5 倍,使计划中的请求级预算正常情况下难以触达。存储字节可能与提交的文件不同;需要换算坐标的消费方使用保存的 `source` 事实,`read_image` 即如此。在任何 fixture 嵌入重编码字节之前,重编码路径的跨平台字节稳定性检查仍是待办。 From c90a944abda6f54d6fc25378d4b820ca0ec630d8 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 11:30:05 +0800 Subject: [PATCH 08/28] docs: bring the zh config catalog along; pin read_image source fields in the code-mode prompt sidecar --- docs/config-catalog.i18n.yaml | 4 ++-- docs/config-catalog.zh.md | 12 ++++++++---- .../code-mode-read-image/system-prompt.expected.md | 2 ++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/config-catalog.i18n.yaml b/docs/config-catalog.i18n.yaml index c5450730eb..a17400de81 100644 --- a/docs/config-catalog.i18n.yaml +++ b/docs/config-catalog.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/config-catalog.md -config-catalog.md: fa0e4caa36a3754356876b13507530de82ceb37b -config-catalog.zh.md: 8125a8a80de3f82f6135292f0b834fde2867a840 +config-catalog.md: c3ae3421d52c2a6c4432b6c7784c1bae53625a24 +config-catalog.zh.md: 4e6b57a42e3269935cae8765cd0c7998c39115f4 diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 8125a8a80d..4e6b57a42e 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -329,20 +329,24 @@ export interface Config { export interface Config { /** Explicit harness home; omitted follows `DSH_HOME`, then `~/.dsh`. */ dshHome?: string - /** Maximum encoded bytes accepted for one image. */ + /** Maximum encoded bytes accepted for one submitted image. */ maxImageBytes?: number /** Maximum image count accepted in one submitted message. */ maxImagesPerMessage?: number /** Maximum aggregate encoded image bytes accepted in one submitted message. */ maxMessageImageBytes?: number - /** Maximum intrinsic width multiplied by height accepted for one image. */ + /** Maximum intrinsic width multiplied by height accepted for one submitted image. */ maxImagePixels?: number - /** Maximum intrinsic width and maximum intrinsic height accepted for one image. */ + /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. */ maxImageDimension?: number + /** Long-edge pixel target of the stored canonical encoding. */ + canonicalMaxDimension?: number + /** Encoded-byte target of the stored canonical encoding. */ + canonicalMaxBytes?: number } ``` -来源:[`packages/attachment/attachment-local/src/index.ts:31`](../packages/attachment/attachment-local/src/index.ts) +来源:[`packages/attachment/attachment-local/src/index.ts:36`](../packages/attachment/attachment-local/src/index.ts) diff --git a/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md b/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md index 786aa8fe80..e3fdc4ace2 100644 --- a/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md +++ b/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md @@ -363,6 +363,8 @@ interface ToolOutputMap { width: number; height: number; name?: string; + sourceWidth?: number; + sourceHeight?: number; }; }; send_message: { From 118f244420de3bc43ae02a440e5f74e3395b858d Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 12:08:37 +0800 Subject: [PATCH 09/28] fix(attachment-local): exclude metadata carriers and animation from passthrough; validate the canonical budget up front Review-round hardening of canonical admission: - passthrough now requires a single-frame source free of EXIF/XMP/IPTC metadata, so location/device metadata never enters durable storage and stored dimensions always describe the perceived pixels; animated WebP joins GIF on the always-re-encode path (first frame only) - SourceImageInfo records orientation-applied dimensions, keeping source and stored raster on shared axes for coordinate mapping - validateImage runs a canonical-encoding dry run, so a validated batch can no longer be refused mid-write by the byte target (no partial writes) - read_image names per-axis multipliers when rounding splits the two ratios and maps IMAGE_TOO_LARGE to actionable downscale guidance --- ...-08-20-canonical-image-admission.i18n.yaml | 4 +-- .../2026-08-20-canonical-image-admission.md | 2 +- ...2026-08-20-canonical-image-admission.zh.md | 2 +- .../attachment-local/README.i18n.yaml | 4 +-- .../attachment/attachment-local/README.md | 2 +- .../attachment/attachment-local/README.zh.md | 2 +- .../attachment-local/src/canonical.ts | 20 +++++++---- .../attachment/attachment-local/src/image.ts | 19 +++++++++- .../attachment/attachment-local/src/index.ts | 2 +- .../attachment/attachment-local/src/store.ts | 33 +++++++++++------ .../attachment-local/tests/canonical.spec.ts | 35 ++++++++++++++----- .../attachment-local/tests/image.spec.ts | 25 +++++++++++-- .../attachment-local/tests/index.spec.ts | 18 ++++++++++ .../attachment/attachment/README.i18n.yaml | 4 +-- packages/attachment/attachment/README.md | 2 +- packages/attachment/attachment/README.zh.md | 2 +- packages/attachment/attachment/src/types.ts | 4 +-- packages/fs/tool-fs/src/read-image.ts | 20 +++++++++-- packages/fs/tool-fs/tests/read-image.spec.ts | 12 +++++++ 19 files changed, 167 insertions(+), 45 deletions(-) diff --git a/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.i18n.yaml b/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.i18n.yaml index 28a4b212df..d8a89613e9 100644 --- a/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-08-20-canonical-image-admission.md -2026-08-20-canonical-image-admission.md: bae3ce8b93b7fbfc4d3233cb3ed019f6ab09c0b4 -2026-08-20-canonical-image-admission.zh.md: a8c55383eb0c8b244dc1fefe1186004e2b869d3e +2026-08-20-canonical-image-admission.md: a30031ef72942a61865525b9ed22f97afd71e18b +2026-08-20-canonical-image-admission.zh.md: d5402a6e7bfd2d8c7de6e2a7ce611c74ec2843d2 diff --git a/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.md b/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.md index bae3ce8b93..a30031ef72 100644 --- a/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.md +++ b/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.md @@ -10,7 +10,7 @@ Admission used to refuse any image above 2000px per side or 3.5 MiB, because an ## Decision -`AttachmentStore.saveImage` resolves `SavedImageAttachment`: the durable `ref` describing stored bytes beside `source` facts of the submitted raster. The local store validates a wide source envelope (32 MiB, 100 MP, 16384px per side) and persists a deterministic canonical encoding: EXIF orientation baked in, metadata stripped, long edge downscaled to `canonicalMaxDimension` (default 2048px), palette PNG for alpha/PNG/GIF lineage and JPEG for photographic sources, stepping a fixed quality ladder (85/75/60/45) until `canonicalMaxBytes` (default 1 MiB) holds. An in-budget PNG/JPEG/WebP source passes through byte-identically, so equal originals keep one content address; GIF always becomes the PNG of its first frame, pinning the first-frame meaning providers apply. Encoder parameters are fixed, not configurable — a parameter change would silently split the content-addressed space — so deployments choose only the source envelope and the canonical budget. The canonical ref keeps the pre-existing field order (`mediaType`, `width`, `height`, `bytes`) so logged references stay byte-identical. `read_image` reports the on-disk dimensions and the coordinate multiplier whenever storage downscaled the file. +`AttachmentStore.saveImage` resolves `SavedImageAttachment`: the durable `ref` describing stored bytes beside `source` facts of the submitted raster. The local store validates a wide source envelope (32 MiB, 100 MP, 16384px per side) and persists a deterministic canonical encoding: EXIF orientation baked in, metadata stripped, long edge downscaled to `canonicalMaxDimension` (default 2048px), palette PNG for alpha/PNG/GIF lineage and JPEG for photographic sources, stepping a fixed quality ladder (85/75/60/45) until `canonicalMaxBytes` (default 1 MiB) holds. An in-budget PNG/JPEG/WebP source passes through byte-identically only when it is single-frame and free of EXIF/XMP/IPTC metadata and non-default orientation, so equal originals keep one content address while location and device metadata never survive admission; GIF and every animated or metadata-carrying source re-encodes, and GIF always becomes the PNG of its first frame, pinning the first-frame meaning providers apply. Encoder parameters are fixed, not configurable — a parameter change would silently split the content-addressed space — so deployments choose only the source envelope and the canonical budget. `SourceImageInfo` records orientation-applied dimensions so source and stored raster share axes, and `validateImage` includes a canonical-encoding dry run so a validated batch can never be refused mid-write by the byte target. The canonical ref keeps the pre-existing field order (`mediaType`, `width`, `height`, `bytes`) so logged references stay byte-identical. `read_image` reports the on-disk dimensions and the coordinate multiplier whenever storage downscaled the file, naming per-axis multipliers when integer rounding makes the two ratios differ. ## Alternatives considered diff --git a/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.zh.md b/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.zh.md index a8c55383eb..d5402a6e7b 100644 --- a/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.zh.md +++ b/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.zh.md @@ -10,7 +10,7 @@ Status: implemented ## 决定 -`AttachmentStore.saveImage` 解析为 `SavedImageAttachment`:描述实际存储字节的持久 `ref`,加上所提交光栅的 `source` 事实。本地存储按宽松的源图上限(32 MiB、1 亿像素、单边 16384px)校验,然后持久保存确定性的规范编码:EXIF 方向落实到像素、剥离元数据、长边等比缩放到 `canonicalMaxDimension`(默认 2048px),带透明通道或源自 PNG/GIF 的图片编码为 palette PNG,摄影类图片编码为 JPEG,并沿固定质量阶梯(85/75/60/45)递降直到满足 `canonicalMaxBytes`(默认 1 MiB)。已在预算内的 PNG/JPEG/WebP 源图按字节原样存储,相同原图保持同一个内容地址;GIF 一律转为首帧 PNG,在准入时固化提供方实际采用的首帧语义。编码器参数固定而不可配置,因为参数变化会悄悄割裂内容寻址空间;部署只选择源图上限与规范预算。规范 ref 保持原有字段顺序(`mediaType`、`width`、`height`、`bytes`),已记录的引用保持字节一致。存储缩小了文件时,`read_image` 会报告磁盘上的原始尺寸和坐标换算倍率。 +`AttachmentStore.saveImage` 解析为 `SavedImageAttachment`:描述实际存储字节的持久 `ref`,加上所提交光栅的 `source` 事实。本地存储按宽松的源图上限(32 MiB、1 亿像素、单边 16384px)校验,然后持久保存确定性的规范编码:EXIF 方向落实到像素、剥离元数据、长边等比缩放到 `canonicalMaxDimension`(默认 2048px),带透明通道或源自 PNG/GIF 的图片编码为 palette PNG,摄影类图片编码为 JPEG,并沿固定质量阶梯(85/75/60/45)递降直到满足 `canonicalMaxBytes`(默认 1 MiB)。已在预算内的 PNG/JPEG/WebP 源图只有在单帧且不携带 EXIF/XMP/IPTC 元数据、方向为默认值时才按字节原样直通,相同原图保持同一个内容地址,位置与设备元数据绝不越过准入;GIF 以及任何动图或携带元数据的源图都会重编码,GIF 一律转为首帧 PNG,在准入时固化提供方实际采用的首帧语义。编码器参数固定而不可配置,因为参数变化会悄悄割裂内容寻址空间;部署只选择源图上限与规范预算。`SourceImageInfo` 记录应用方向之后的尺寸,使源图与存储光栅共享坐标轴;`validateImage` 包含规范编码干跑,通过校验的批次绝不会在写入中途被字节目标拒绝。规范 ref 保持原有字段顺序(`mediaType`、`width`、`height`、`bytes`),已记录的引用保持字节一致。存储缩小了文件时,`read_image` 会报告磁盘上的原始尺寸和坐标换算倍率,取整使两轴比例不一致时分轴给出。 ## 考虑过的替代方案 diff --git a/packages/attachment/attachment-local/README.i18n.yaml b/packages/attachment/attachment-local/README.i18n.yaml index 11216d1aa1..4393ddbe9d 100644 --- a/packages/attachment/attachment-local/README.i18n.yaml +++ b/packages/attachment/attachment-local/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/attachment/attachment-local/README.md -README.md: e8f89906f7bedb20b80a04ab6d2aa4b80c6d746f -README.zh.md: 61e1dde94751436bb4d8ff4dde1b68b1b10fc005 +README.md: afa38ccc125f4fb36d35bb4b94b1aea278107551 +README.zh.md: 9de7ce65447a91741810bbcd41d397a70275247b diff --git a/packages/attachment/attachment-local/README.md b/packages/attachment/attachment-local/README.md index e8f89906f7..afa38ccc12 100644 --- a/packages/attachment/attachment-local/README.md +++ b/packages/attachment/attachment-local/README.md @@ -2,7 +2,7 @@ English | [中文](README.zh.md) -The private local implementation of [`@deepseek-ai/dsh-attachment`](../attachment). Objects land at `/attachments/v1/objects//` and are addressed by an opaque `sha256:` id. Each process proves a home durable once by syncing every ancestor entry to the filesystem root, so a directory another process created but has not yet synced is never mistaken for a safe boundary. Writes then use a private staging directory, owner-only files, a synced temporary file, an atomic exclusive hard-link publish, and directory syncs on the publication path (POSIX; Windows relies on filesystem metadata journaling) so the reported reference survives a crash. Write admission fully decodes the raster against a wide source envelope — byte, total-pixel, and per-side caps (defaults 32MiB, 100MP, 16384px) — and then persists a deterministic canonical encoding instead of the submitted bytes: EXIF orientation is baked into pixels, metadata is stripped, the long edge is downscaled to the configured canonical target (default 2048px), sources with alpha or PNG/GIF lineage encode as palette PNG and photographic sources as JPEG, stepping down a fixed quality ladder (85/75/60/45) until the configured canonical byte target holds (default 1MiB). A PNG/JPEG/WebP source already inside the canonical budget is stored byte-identically, so equal originals keep deduplicating to one content address; GIF always re-encodes to the PNG of its first frame, pinning at admission the first-frame meaning providers apply. Encoder parameters are deliberately fixed rather than configurable, because a parameter change would silently split the content-addressed space; the deployment chooses only the source envelope and the canonical budget. An admitted image rides every later request of its session, so canonicalizing at admission is what bounds durable history without refusing ordinary large sources. Reads re-check the digest and logged metadata, and a later policy reduction does not make already-admitted history unreadable. +The private local implementation of [`@deepseek-ai/dsh-attachment`](../attachment). Objects land at `/attachments/v1/objects//` and are addressed by an opaque `sha256:` id. Each process proves a home durable once by syncing every ancestor entry to the filesystem root, so a directory another process created but has not yet synced is never mistaken for a safe boundary. Writes then use a private staging directory, owner-only files, a synced temporary file, an atomic exclusive hard-link publish, and directory syncs on the publication path (POSIX; Windows relies on filesystem metadata journaling) so the reported reference survives a crash. Write admission fully decodes the raster against a wide source envelope — byte, total-pixel, and per-side caps (defaults 32MiB, 100MP, 16384px) — and then persists a deterministic canonical encoding instead of the submitted bytes: EXIF orientation is baked into pixels, metadata is stripped, the long edge is downscaled to the configured canonical target (default 2048px), sources with alpha or PNG/GIF lineage encode as palette PNG and photographic sources as JPEG, stepping down a fixed quality ladder (85/75/60/45) until the configured canonical byte target holds (default 1MiB). A PNG/JPEG/WebP source already inside the canonical budget passes through byte-identically only when it is a single frame and carries no EXIF/XMP/IPTC metadata and no non-default orientation, so equal originals keep deduplicating to one content address while location and device metadata never survive admission; GIF and every animated or metadata-carrying source re-encodes, and GIF always becomes the PNG of its first frame, pinning at admission the first-frame meaning providers apply. Encoder parameters are deliberately fixed rather than configurable, because a parameter change would silently split the content-addressed space; the deployment chooses only the source envelope and the canonical budget. An admitted image rides every later request of its session, so canonicalizing at admission is what bounds durable history without refusing ordinary large sources. `validateImage` runs the same policy including a canonical-encoding dry run, so a validated batch can never be refused mid-write by the byte target. Reads re-check the digest and logged metadata, and a later policy reduction does not make already-admitted history unreadable. `DSH_HOME` resolves through the shared path policy: explicit config, `$DSH_HOME`, then `~/.dsh`. Session logs contain only the reference and verified metadata, never this host path. `readImage` forwards optional cancellation into the filesystem read, observes it around verification, and preserves it instead of wrapping it as `ATTACHMENT_READ_FAILED`. diff --git a/packages/attachment/attachment-local/README.zh.md b/packages/attachment/attachment-local/README.zh.md index 61e1dde947..9de7ce6544 100644 --- a/packages/attachment/attachment-local/README.zh.md +++ b/packages/attachment/attachment-local/README.zh.md @@ -2,7 +2,7 @@ [English](README.md) | 中文 -这是 [`@deepseek-ai/dsh-attachment`](../attachment) 的私有本地实现。对象存放在 `/attachments/v1/objects//`,并通过不透明的 `sha256:` 标识符寻址。每个进程都会通过将每个祖先目录项逐级同步到文件系统根目录,为某个 home 一次性证明其持久性,因此绝不会把另一个进程已经创建但尚未同步的目录误认为安全边界。随后,写入过程使用私有暂存目录、仅所有者可访问的文件、经过同步的临时文件、原子且排他的硬链接发布,并对发布路径执行目录同步(适用于 POSIX;Windows 依赖文件系统元数据日志),确保已报告的引用能够在崩溃后继续存在。写入准入会按宽松的源图上限(字节、总像素、单边,默认 32MiB、1 亿像素、16384px)完整解码光栅图片,然后持久保存确定性的规范编码而不是提交的原始字节:EXIF 方向落实到像素并剥离元数据,长边等比缩放到配置的规范目标(默认 2048px),带透明通道或源自 PNG/GIF 的图片编码为 palette PNG,摄影类图片编码为 JPEG,并沿固定的质量阶梯(85/75/60/45)递降,直到满足配置的规范字节目标(默认 1MiB)。已在规范预算内的 PNG/JPEG/WebP 源图按字节原样存储,因此相同原图始终去重到同一个内容地址;GIF 一律重编码为其首帧的 PNG,在准入时就固化提供方实际采用的首帧语义。编码器参数刻意固定而不可配置,因为参数变化会悄悄割裂内容寻址空间;部署只选择源图上限与规范预算。一张已接纳的图片会随会话之后的每次请求发送,所以在准入时规范化才能在不拒绝普通大图的前提下约束持久历史。读取会重新校验摘要和已记录的元数据,后续收紧限制不会导致已经接纳的历史记录变得不可读。 +这是 [`@deepseek-ai/dsh-attachment`](../attachment) 的私有本地实现。对象存放在 `/attachments/v1/objects//`,并通过不透明的 `sha256:` 标识符寻址。每个进程都会通过将每个祖先目录项逐级同步到文件系统根目录,为某个 home 一次性证明其持久性,因此绝不会把另一个进程已经创建但尚未同步的目录误认为安全边界。随后,写入过程使用私有暂存目录、仅所有者可访问的文件、经过同步的临时文件、原子且排他的硬链接发布,并对发布路径执行目录同步(适用于 POSIX;Windows 依赖文件系统元数据日志),确保已报告的引用能够在崩溃后继续存在。写入准入会按宽松的源图上限(字节、总像素、单边,默认 32MiB、1 亿像素、16384px)完整解码光栅图片,然后持久保存确定性的规范编码而不是提交的原始字节:EXIF 方向落实到像素并剥离元数据,长边等比缩放到配置的规范目标(默认 2048px),带透明通道或源自 PNG/GIF 的图片编码为 palette PNG,摄影类图片编码为 JPEG,并沿固定的质量阶梯(85/75/60/45)递降,直到满足配置的规范字节目标(默认 1MiB)。已在规范预算内的 PNG/JPEG/WebP 源图只有在单帧且不携带 EXIF/XMP/IPTC 元数据、方向为默认值时才按字节原样直通,因此相同原图始终去重到同一个内容地址,而位置与设备元数据绝不会越过准入;GIF 以及任何动图或携带元数据的源图都会重编码,GIF 一律变为其首帧的 PNG,在准入时就固化提供方实际采用的首帧语义。编码器参数刻意固定而不可配置,因为参数变化会悄悄割裂内容寻址空间;部署只选择源图上限与规范预算。一张已接纳的图片会随会话之后的每次请求发送,所以在准入时规范化才能在不拒绝普通大图的前提下约束持久历史。`validateImage` 执行同一套策略并包含规范编码的干跑,因此通过校验的批次绝不会在写入中途被字节目标拒绝。读取会重新校验摘要和已记录的元数据,后续收紧限制不会导致已经接纳的历史记录变得不可读。 `DSH_HOME` 按共享路径策略解析:显式配置、`$DSH_HOME`,最后是 `~/.dsh`。会话日志只包含引用和经过校验的元数据,绝不包含这个宿主路径。`readImage` 会把可选取消信号传入文件系统读取、在校验前后观察该信号,并保留取消语义,而不会将其包装成 `ATTACHMENT_READ_FAILED`。 diff --git a/packages/attachment/attachment-local/src/canonical.ts b/packages/attachment/attachment-local/src/canonical.ts index ada2164566..db4295c404 100644 --- a/packages/attachment/attachment-local/src/canonical.ts +++ b/packages/attachment/attachment-local/src/canonical.ts @@ -38,18 +38,26 @@ async function encode(pipeline: Sharp, mediaType: 'image/png' | 'image/jpeg'): P /** * Whether stored bytes may be the submitted bytes unchanged. Byte-identical - * passthrough is preferred whenever the source already fits the budget: it - * keeps re-submissions of the same original deduplicating to the same object - * and never re-encodes what no policy requires changing. GIF is excluded — - * only its first frame is model-visible, so admission pins that meaning into - * the stored object instead of letting each provider drop frames differently. - * @param detected - verified source format and dimensions. + * passthrough is preferred whenever the source already fits the budget and + * carries nothing the canonical form forbids: it keeps re-submissions of the + * same original deduplicating to the same object and never re-encodes what no + * policy requires changing. Excluded from passthrough — and therefore always + * re-encoded — are GIF and any animated container (only the first frame is + * model-visible, so admission pins that meaning instead of letting each + * provider drop frames differently) and any source carrying EXIF/XMP/IPTC + * metadata or a non-default orientation (stored objects ride every later + * request, so location and device metadata must not survive admission, and a + * stored orientation would let the recorded dimensions diverge from the + * pixels a model perceives). + * @param detected - verified source format, dimensions, and metadata facts. * @param bytes - submitted encoded byte length. * @param policy - resolved canonical budget. * @returns whether the submitted encoding already is canonical. */ export function isCanonical(detected: DetectedImage, bytes: number, policy: CanonicalImagePolicy): boolean { return detected.mediaType !== 'image/gif' + && !detected.animated + && !detected.carriesMetadata && bytes <= policy.maxBytes && Math.max(detected.width, detected.height) <= policy.maxDimension } diff --git a/packages/attachment/attachment-local/src/image.ts b/packages/attachment/attachment-local/src/image.ts index b067ea80ff..991e5dc051 100644 --- a/packages/attachment/attachment-local/src/image.ts +++ b/packages/attachment/attachment-local/src/image.ts @@ -7,8 +7,14 @@ import type { ImageMediaType } from '@deepseek-ai/dsh-attachment' /** Decoded metadata from a supported image. */ export interface DetectedImage { mediaType: ImageMediaType + /** Intrinsic width with EXIF orientation applied — the width a viewer perceives. */ width: number + /** Intrinsic height with EXIF orientation applied — the height a viewer perceives. */ height: number + /** Whether the container carries more than one frame. */ + animated: boolean + /** Whether the bytes carry EXIF/XMP/IPTC metadata or a non-default orientation. */ + carriesMetadata: boolean } const MEDIA_TYPES: Readonly> = { @@ -24,7 +30,18 @@ async function imageMetadata(image: Sharp): Promise { if (mediaType === undefined) { throw new AttachmentError('Unsupported or malformed image data.', 'INVALID_IMAGE') } - return { mediaType, width: metadata.width, height: metadata.height } + // EXIF orientations 5-8 transpose the stored raster; report the perceived + // axes so limits, source facts, and coordinate advice all share them. + const transposed = metadata.orientation !== undefined && metadata.orientation >= 5 + return { + mediaType, + width: transposed ? metadata.height : metadata.width, + height: transposed ? metadata.width : metadata.height, + animated: (metadata.pages ?? 1) > 1, + // orientation is EXIF-derived for every whitelisted format, so exif + // presence already covers a non-default orientation. + carriesMetadata: metadata.exif !== undefined || metadata.xmp !== undefined || metadata.iptc !== undefined, + } } /** diff --git a/packages/attachment/attachment-local/src/index.ts b/packages/attachment/attachment-local/src/index.ts index cbd702c2bf..cbe535ae7d 100644 --- a/packages/attachment/attachment-local/src/index.ts +++ b/packages/attachment/attachment-local/src/index.ts @@ -89,7 +89,7 @@ export class LocalAttachmentStore extends AttachmentStore { } async validateImage(input: SaveImageAttachment): Promise { - await validateImageFile(input, this.imageLimits) + await validateImageFile(input, this.imageLimits, this.canonicalPolicy) } async saveImage(input: SaveImageAttachment): Promise { diff --git a/packages/attachment/attachment-local/src/store.ts b/packages/attachment/attachment-local/src/store.ts index 27152d89cc..9964c2a94f 100644 --- a/packages/attachment/attachment-local/src/store.ts +++ b/packages/attachment/attachment-local/src/store.ts @@ -13,11 +13,13 @@ import type { ImageAttachmentRef, SaveImageAttachment, SavedImageAttachment, + SourceImageInfo, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' import { canonicalizeImage } from './canonical.ts' import type { CanonicalImagePolicy } from './canonical.ts' import { detectImage, probeImage } from './image.ts' +import type { DetectedImage } from './image.ts' const ID_PATTERN = /^sha256:([a-f0-9]{64})$/ const durableHomes = new Set() @@ -50,24 +52,35 @@ async function inspectMetadata( data: Uint8Array, declaredMediaType: ImageAttachmentRef['mediaType'], limits: ImageAttachmentLimits, -): Promise> { +): Promise<{ detected: DetectedImage; source: SourceImageInfo }> { if (data.byteLength === 0) throw new AttachmentError('Image is empty.', 'INVALID_IMAGE') const detected = await detectImage(data, { maxPixels: limits.maxImagePixels, maxDimension: limits.maxImageDimension }) if (detected.mediaType !== declaredMediaType) throw new AttachmentError('Declared image type does not match its bytes.', 'IMAGE_TYPE_MISMATCH') - return { ...detected, bytes: data.byteLength } + return { + detected, + source: { mediaType: detected.mediaType, bytes: data.byteLength, width: detected.width, height: detected.height }, + } } /** - * Run the full admission policy for one image without touching storage. + * Run the full admission policy for one image without touching storage, + * including a canonical-encoding dry run: a batch whose members all validate + * cannot later be refused mid-write by the canonical byte target. * @param input - encoded bytes and declared metadata. - * @param limits - resolved storage policy. - * @returns completion after the encoded raster has been fully decoded. + * @param limits - resolved source admission policy. + * @param policy - resolved canonical encoding budget. + * @returns completion after the raster has been fully decoded and its canonical encoding proven to fit. */ -export async function validateImageFile(input: SaveImageAttachment, limits: ImageAttachmentLimits): Promise { +export async function validateImageFile( + input: SaveImageAttachment, + limits: ImageAttachmentLimits, + policy: CanonicalImagePolicy, +): Promise { if (input.data.byteLength > limits.maxImageBytes) { throw new AttachmentError('Image exceeds the configured byte limit.', 'IMAGE_TOO_LARGE') } - await inspectMetadata(input.data, input.mediaType, limits) + const { detected } = await inspectMetadata(input.data, input.mediaType, limits) + await canonicalizeImage(input.data, detected, policy) } /** @@ -147,8 +160,8 @@ export async function saveImageFile( policy: CanonicalImagePolicy, ): Promise { if (input.data.byteLength > limits.maxImageBytes) throw new AttachmentError('Image exceeds the configured byte limit.', 'IMAGE_TOO_LARGE') - const metadata = await inspectMetadata(input.data, input.mediaType, limits) - const canonical = await canonicalizeImage(input.data, metadata, policy) + const { detected, source } = await inspectMetadata(input.data, input.mediaType, limits) + const canonical = await canonicalizeImage(input.data, detected, policy) const sha256 = digest(canonical.data) const bucket = join(root, 'objects', sha256.slice(0, 2)) const staging = join(root, 'tmp') @@ -208,7 +221,7 @@ export async function saveImageFile( bytes: canonical.data.byteLength, ...(name !== undefined ? { name } : {}), }, - source: metadata, + source, } } diff --git a/packages/attachment/attachment-local/tests/canonical.spec.ts b/packages/attachment/attachment-local/tests/canonical.spec.ts index 0441fbc462..12d286a2bc 100644 --- a/packages/attachment/attachment-local/tests/canonical.spec.ts +++ b/packages/attachment/attachment-local/tests/canonical.spec.ts @@ -30,11 +30,14 @@ async function flatImage(width: number, height: number, format: 'png' | 'jpeg' | } describe('isCanonical', () => { - it('accepts an in-budget PNG/JPEG/WebP and refuses GIF, oversized edges, and oversized bytes', () => { - expect(isCanonical({ mediaType: 'image/png', width: 2048, height: 4 }, 100, POLICY)).toBe(true) - expect(isCanonical({ mediaType: 'image/gif', width: 4, height: 4 }, 100, POLICY)).toBe(false) - expect(isCanonical({ mediaType: 'image/jpeg', width: 2049, height: 4 }, 100, POLICY)).toBe(false) - expect(isCanonical({ mediaType: 'image/webp', width: 4, height: 4 }, POLICY.maxBytes + 1, POLICY)).toBe(false) + it('accepts an in-budget clean PNG/JPEG/WebP and refuses GIF, animation, metadata, oversized edges, and oversized bytes', () => { + const clean = { animated: false, carriesMetadata: false } + expect(isCanonical({ mediaType: 'image/png', width: 2048, height: 4, ...clean }, 100, POLICY)).toBe(true) + expect(isCanonical({ mediaType: 'image/gif', width: 4, height: 4, ...clean }, 100, POLICY)).toBe(false) + expect(isCanonical({ mediaType: 'image/webp', width: 4, height: 4, animated: true, carriesMetadata: false }, 100, POLICY)).toBe(false) + expect(isCanonical({ mediaType: 'image/jpeg', width: 4, height: 4, animated: false, carriesMetadata: true }, 100, POLICY)).toBe(false) + expect(isCanonical({ mediaType: 'image/jpeg', width: 2049, height: 4, ...clean }, 100, POLICY)).toBe(false) + expect(isCanonical({ mediaType: 'image/webp', width: 4, height: 4, ...clean }, POLICY.maxBytes + 1, POLICY)).toBe(false) }) }) @@ -56,7 +59,7 @@ describe('canonicalizeImage', () => { const canonical = await canonicalizeImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes }) expect(canonical).toMatchObject({ mediaType: 'image/png', width: 5, height: 3 }) - await expect(detectImage(canonical.data)).resolves.toEqual({ mediaType: 'image/png', width: 5, height: 3 }) + await expect(detectImage(canonical.data)).resolves.toEqual({ mediaType: 'image/png', width: 5, height: 3, animated: false, carriesMetadata: false }) const again = await canonicalizeImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes }) expect(again.data).toEqual(canonical.data) }) @@ -77,7 +80,7 @@ describe('canonicalizeImage', () => { const canonical = await canonicalizeImage(data, detected, POLICY) expect(canonical.mediaType).toBe('image/png') - await expect(detectImage(canonical.data)).resolves.toEqual({ mediaType: 'image/png', width: 6, height: 4 }) + await expect(detectImage(canonical.data)).resolves.toEqual({ mediaType: 'image/png', width: 6, height: 4, animated: false, carriesMetadata: false }) }) it('keeps alpha sources on PNG when the budget holds', async () => { @@ -132,8 +135,24 @@ describe('canonicalizeImage', () => { .rejects.toMatchObject({ code: 'IMAGE_TOO_LARGE' }) }) + it('re-encodes an in-budget oriented JPEG, baking rotation and stripping metadata', async () => { + const data = new Uint8Array(await sharp({ + create: { width: 4, height: 2, channels: 3, background: { r: 1, g: 2, b: 3 } }, + }).jpeg().withMetadata({ orientation: 6 }).toBuffer()) + const detected = await detectImage(data) + // Orientation 6 rotates 90°: the perceived source is 2x4. + expect(detected).toMatchObject({ width: 2, height: 4, carriesMetadata: true }) + + const canonical = await canonicalizeImage(data, detected, POLICY) + + expect(canonical.data).not.toBe(data) + expect(canonical).toMatchObject({ width: 2, height: 4 }) + await expect(detectImage(canonical.data)).resolves.toMatchObject({ width: 2, height: 4, carriesMetadata: false }) + }) + it('maps an encoder fault on undecodable bytes to a storage failure', async () => { - await expect(canonicalizeImage(Uint8Array.of(1, 2, 3), { mediaType: 'image/png', width: 5000, height: 5000 }, POLICY)) + const detected = { mediaType: 'image/png', width: 5000, height: 5000, animated: false, carriesMetadata: false } as const + await expect(canonicalizeImage(Uint8Array.of(1, 2, 3), detected, POLICY)) .rejects.toMatchObject({ code: 'ATTACHMENT_WRITE_FAILED' }) }) }) diff --git a/packages/attachment/attachment-local/tests/image.spec.ts b/packages/attachment/attachment-local/tests/image.spec.ts index 6b1cea6bfb..4398f986b7 100644 --- a/packages/attachment/attachment-local/tests/image.spec.ts +++ b/packages/attachment/attachment-local/tests/image.spec.ts @@ -18,7 +18,7 @@ describe('raster decoding', () => { ['gif', 'image/gif'], ] as const) { await expect(detectImage(await raster(format))) - .resolves.toEqual({ mediaType, width: 3, height: 2 }) + .resolves.toEqual({ mediaType, width: 3, height: 2, animated: false, carriesMetadata: false }) } }) @@ -31,7 +31,7 @@ describe('raster decoding', () => { await expect(detectImage(await raster('png'), { maxDimension: 2 })) .rejects.toMatchObject({ code: 'IMAGE_DIMENSION_TOO_LARGE' }) await expect(detectImage(await raster('png'), { maxDimension: 3 })) - .resolves.toEqual({ mediaType: 'image/png', width: 3, height: 2 }) + .resolves.toEqual({ mediaType: 'image/png', width: 3, height: 2, animated: false, carriesMetadata: false }) }) it('rejects malformed bytes and truncated payloads with readable headers', async () => { @@ -47,6 +47,27 @@ describe('raster decoding', () => { await expect(detectImage(truncated)).rejects.toMatchObject({ code: 'INVALID_IMAGE' }) }) + it('reports animation from a multi-frame container and perceived axes from EXIF orientation', async () => { + const header = Buffer.from('47494638396101000100800000000000ffffff', 'hex') + const frame = Buffer.from('21f90401000000002c0000000001000100000202440100', 'hex') + const twoFrameGif = Uint8Array.from(Buffer.concat([header, frame, frame, Buffer.from('3b', 'hex')])) + await expect(detectImage(twoFrameGif)).resolves.toMatchObject({ mediaType: 'image/gif', animated: true }) + + const oriented = new Uint8Array(await sharp({ + create: { width: 4, height: 2, channels: 3, background: { r: 1, g: 2, b: 3 } }, + }).jpeg().withMetadata({ orientation: 6 }).toBuffer()) + await expect(detectImage(oriented)).resolves.toEqual({ + mediaType: 'image/jpeg', width: 2, height: 4, animated: false, carriesMetadata: true, + }) + + const flipped = new Uint8Array(await sharp({ + create: { width: 4, height: 2, channels: 3, background: { r: 1, g: 2, b: 3 } }, + }).jpeg().withMetadata({ orientation: 3 }).toBuffer()) + await expect(detectImage(flipped)).resolves.toEqual({ + mediaType: 'image/jpeg', width: 4, height: 2, animated: false, carriesMetadata: true, + }) + }) + it('probes malformed bytes and unsupported formats into the same stable error', async () => { await expect(probeImage(Uint8Array.of(1, 2, 3))) .rejects.toMatchObject({ code: 'INVALID_IMAGE' }) diff --git a/packages/attachment/attachment-local/tests/index.spec.ts b/packages/attachment/attachment-local/tests/index.spec.ts index 0e86957f82..c4be530480 100644 --- a/packages/attachment/attachment-local/tests/index.spec.ts +++ b/packages/attachment/attachment-local/tests/index.spec.ts @@ -47,6 +47,24 @@ describe('local attachment service', () => { } }) + it('refuses a batch during validation when a member cannot meet the canonical byte target, before any write', async () => { + const dshHome = await mkdtemp(join(tmpdir(), 'dsh-attachment-batch-')) + try { + const service = new LocalAttachmentStore(new Context(), { dshHome, canonicalMaxBytes: 10 }) + const valid = Uint8Array.from(Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=', + 'base64', + )) + await expect(service.saveImages([ + { data: valid, mediaType: 'image/png' }, + { data: valid, mediaType: 'image/png' }, + ])).rejects.toMatchObject({ code: 'IMAGE_TOO_LARGE' }) + expect(existsSync(service.root)).toBe(false) + } finally { + await rm(dshHome, { recursive: true, force: true }) + } + }) + it('validates without persisting: a rejected image leaves no storage root behind', async () => { const dshHome = await mkdtemp(join(tmpdir(), 'dsh-attachment-validate-')) try { diff --git a/packages/attachment/attachment/README.i18n.yaml b/packages/attachment/attachment/README.i18n.yaml index 97ec722871..9c61d2fe81 100644 --- a/packages/attachment/attachment/README.i18n.yaml +++ b/packages/attachment/attachment/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/attachment/attachment/README.md -README.md: 89bc3ca3a288450c43fefb5dde38da7f65218f43 -README.zh.md: ca13c9e66234b280f7fa9d01fc80bd026604831f +README.md: 3b80444804a345bd019fe94f25954933aa549518 +README.zh.md: 37be4a4a9f54a7e7ddb5fdceb57711378c2f2cfc diff --git a/packages/attachment/attachment/README.md b/packages/attachment/attachment/README.md index 89bc3ca3a2..3b80444804 100644 --- a/packages/attachment/attachment/README.md +++ b/packages/attachment/attachment/README.md @@ -4,7 +4,7 @@ English | [中文](README.zh.md) The durable attachment seam. `ctx.attachments` validates and durably commits immutable image bytes, then returns a serializable `ImageAttachmentRef`; consumers never persist browser paths, object URLs, provider URLs, or base64 in session events. -Unsent composer images remain browser-owned temporary drafts. `validateImage` runs the same admission policy without persisting. `saveImages` owns batch count and aggregate-byte limits, validates every member before writing any member, then commits in order and returns references only after the complete batch succeeds. A later storage failure returns no partial references, although an earlier immutable content-addressed object may remain unreachable until reference-aware garbage collection exists. `AttachmentError.code` uses the closed `AttachmentErrorCode` string union. Its `ImageAdmissionErrorCode` subset marks caller-correctable image-input failures; `isImageAdmissionError` recognizes that subset at runtime so each protocol adapter can map its own error vocabulary. `saveImage` commits one accepted image before any model-visible session event is published and resolves `SavedImageAttachment`: an implementation may persist a canonical re-encoding of the submitted raster, so the returned `ref` always describes the stored bytes while `source` (`SourceImageInfo`) preserves the submitted raster's media type, byte length, and dimensions for callers that report or map coordinates against the original. `readImage` verifies the content-addressed object against its logged metadata. Callers may cancel `readImage`; implementations observe cancellation around backend and verification work and preserve it instead of translating it into a storage failure. +Unsent composer images remain browser-owned temporary drafts. `validateImage` runs the complete admission policy without persisting, including any canonical-encoding dry run the implementation applies, so batch validation proves every member can also be committed. `saveImages` owns batch count and aggregate-byte limits, validates every member before writing any member, then commits in order and returns references only after the complete batch succeeds. A later storage failure returns no partial references, although an earlier immutable content-addressed object may remain unreachable until reference-aware garbage collection exists. `AttachmentError.code` uses the closed `AttachmentErrorCode` string union. Its `ImageAdmissionErrorCode` subset marks caller-correctable image-input failures; `isImageAdmissionError` recognizes that subset at runtime so each protocol adapter can map its own error vocabulary. `saveImage` commits one accepted image before any model-visible session event is published and resolves `SavedImageAttachment`: an implementation may persist a canonical re-encoding of the submitted raster, so the returned `ref` always describes the stored bytes while `source` (`SourceImageInfo`) preserves the submitted raster's media type, byte length, and dimensions for callers that report or map coordinates against the original. `readImage` verifies the content-addressed object against its logged metadata. Callers may cancel `readImage`; implementations observe cancellation around backend and verification work and preserve it instead of translating it into a storage failure. `admitEncodedImages(attachments, images)` is the shared wire entry used by every RPC endpoint that accepts browser uploads (the session prompt endpoint and the command executor): it enforces canonical base64 on every member, then delegates batch admission — limits, validation, ordered commit — to `saveImages`. The base64 upload form is `EncodedImageAttachment`, exported from `@deepseek-ai/dsh-attachment/types` so wire contracts can reference it. diff --git a/packages/attachment/attachment/README.zh.md b/packages/attachment/attachment/README.zh.md index ca13c9e662..37be4a4a9f 100644 --- a/packages/attachment/attachment/README.zh.md +++ b/packages/attachment/attachment/README.zh.md @@ -4,7 +4,7 @@ 持久附件服务边界。`ctx.attachments` 校验并持久提交不可变图片字节,随后返回可序列化的 `ImageAttachmentRef`;消费方绝不会在会话事件中持久保存浏览器路径、对象 URL、提供方 URL 或 base64。 -未发送的输入区图片仍是由浏览器持有的临时草稿。`validateImage` 运行相同的准入策略,但不执行持久化。`saveImages` 负责批次图片数量和总字节限制,先校验全部成员,再按顺序提交,并且只在完整批次成功后返回引用。后续存储失败不会返回部分引用,但较早写入的不可变内容寻址对象可能保持不可达,直至具备按引用感知的垃圾回收。`AttachmentError.code` 使用封闭的 `AttachmentErrorCode` 字符串联合类型。其 `ImageAdmissionErrorCode` 子集标记可由调用方修正的图片输入失败;`isImageAdmissionError` 在运行时识别该子集,使每个协议适配器可以映射自己的错误词汇。`saveImage` 会在发布任何模型可见的会话事件前提交一张已接受的图片,并解析为 `SavedImageAttachment`:实现可以持久保存所提交光栅的规范重编码,因此返回的 `ref` 始终描述实际存储的字节,而 `source`(`SourceImageInfo`)保留所提交光栅的媒体类型、字节长度和尺寸,供需要对照原图汇报或换算坐标的调用方使用。`readImage` 则根据已记录的元数据校验内容寻址对象。调用方可以取消 `readImage`;实现会在后端读取与校验工作的边界观察取消,并保留取消语义,而不会将其转换为存储失败。 +未发送的输入区图片仍是由浏览器持有的临时草稿。`validateImage` 运行完整的准入策略但不执行持久化,包含实现所应用的规范编码干跑,因此批量校验能证明每个成员随后也能提交成功。`saveImages` 负责批次图片数量和总字节限制,先校验全部成员,再按顺序提交,并且只在完整批次成功后返回引用。后续存储失败不会返回部分引用,但较早写入的不可变内容寻址对象可能保持不可达,直至具备按引用感知的垃圾回收。`AttachmentError.code` 使用封闭的 `AttachmentErrorCode` 字符串联合类型。其 `ImageAdmissionErrorCode` 子集标记可由调用方修正的图片输入失败;`isImageAdmissionError` 在运行时识别该子集,使每个协议适配器可以映射自己的错误词汇。`saveImage` 会在发布任何模型可见的会话事件前提交一张已接受的图片,并解析为 `SavedImageAttachment`:实现可以持久保存所提交光栅的规范重编码,因此返回的 `ref` 始终描述实际存储的字节,而 `source`(`SourceImageInfo`)保留所提交光栅的媒体类型、字节长度和尺寸,供需要对照原图汇报或换算坐标的调用方使用。`readImage` 则根据已记录的元数据校验内容寻址对象。调用方可以取消 `readImage`;实现会在后端读取与校验工作的边界观察取消,并保留取消语义,而不会将其转换为存储失败。 `admitEncodedImages(attachments, images)` 是每个接受浏览器上传的 RPC 端点(会话 prompt 端点与命令执行器)共用的 wire 入口:它对每个成员强制执行规范 base64,随后把批量准入——限额、校验、有序提交——委托给 `saveImages`。base64 上传形式为 `EncodedImageAttachment`,从 `@deepseek-ai/dsh-attachment/types` 导出,供 wire 契约引用。 diff --git a/packages/attachment/attachment/src/types.ts b/packages/attachment/attachment/src/types.ts index 93cbf6a3db..22db4c6d23 100644 --- a/packages/attachment/attachment/src/types.ts +++ b/packages/attachment/attachment/src/types.ts @@ -65,9 +65,9 @@ export interface SourceImageInfo { mediaType: ImageMediaType /** Exact submitted encoded byte length. */ bytes: number - /** Intrinsic width of the submitted raster in pixels. */ + /** Perceived source width in pixels, with any EXIF orientation applied, so it shares axes with the stored raster. */ width: number - /** Intrinsic height of the submitted raster in pixels. */ + /** Perceived source height in pixels, with any EXIF orientation applied, so it shares axes with the stored raster. */ height: number } diff --git a/packages/fs/tool-fs/src/read-image.ts b/packages/fs/tool-fs/src/read-image.ts index 0cfaa93903..cde24a2a35 100644 --- a/packages/fs/tool-fs/src/read-image.ts +++ b/packages/fs/tool-fs/src/read-image.ts @@ -104,9 +104,17 @@ export function imageRefFromValue(image: ImageReadValue['image']): ImageAttachme * @returns the model-facing envelope; the image itself rides the adjacent image block. */ export function formatImageReadOutput(displayPath: string, image: ImageReadValue['image']): string { - const scaled = image.sourceWidth !== undefined && image.sourceHeight !== undefined - ? ` (downscaled from ${image.sourceWidth}x${image.sourceHeight} px; multiply coordinates by ${(image.sourceWidth / image.width).toFixed(2)} to locate features in the original file)` - : '' + let scaled = '' + if (image.sourceWidth !== undefined && image.sourceHeight !== undefined) { + // Integer rounding can give the two axes slightly different ratios, so the + // advice names one multiplier only when both round to the same value. + const x = (image.sourceWidth / image.width).toFixed(2) + const y = (image.sourceHeight / image.height).toFixed(2) + const advice = x === y + ? `multiply coordinates by ${x}` + : `multiply x coordinates by ${x} and y coordinates by ${y}` + scaled = ` (downscaled from ${image.sourceWidth}x${image.sourceHeight} px; ${advice} to locate features in the original file)` + } return `${displayPath} image @@ -219,6 +227,12 @@ export function applyReadImageTool(ctx: Context): void { { cause: error }, ) } + if (error.code === 'IMAGE_TOO_LARGE') { + throw new Error( + `cannot read "${target.displayPath}": the image cannot be stored within the deployment's byte limits; downscale the image and read the smaller copy`, + { cause: error }, + ) + } if (error.code !== 'IMAGE_TYPE_MISMATCH') throw error const extension = extname(target.displayPath).toLowerCase() throw new Error( diff --git a/packages/fs/tool-fs/tests/read-image.spec.ts b/packages/fs/tool-fs/tests/read-image.spec.ts index 6cea2cf18f..dcc6ab7d5e 100644 --- a/packages/fs/tool-fs/tests/read-image.spec.ts +++ b/packages/fs/tool-fs/tests/read-image.spec.ts @@ -438,6 +438,11 @@ describe('image admission failures', () => { expect(storageFault.isError).toBe(true) expect(text(storageFault)).toContain('Unable to persist image attachment.') + FailingStore.failure = new AttachmentError('Image cannot be encoded within the configured canonical byte target.', 'IMAGE_TOO_LARGE') + const overBudget = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model')) + expect(overBudget.isError).toBe(true) + expect(text(overBudget)).toContain('cannot be stored within the deployment\'s byte limits; downscale the image and read the smaller copy') + FailingStore.failure = new Error('unrelated infrastructure failure') const unrelated = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model')) expect(unrelated.isError).toBe(true) @@ -529,6 +534,13 @@ describe('image admission failures', () => { expect(result.isError).toBe(false) expect(text(result)).toContain('image/png image, 2x1 px, 7 bytes (downscaled from 4x2 px; multiply coordinates by 2.00 to locate features in the original file)') }) + + it('names per-axis multipliers when integer rounding makes the ratios differ', () => { + const envelope = formatImageReadOutput('/img/photo.jpg', { + attachmentId: 'sha256:feed', mediaType: 'image/jpeg', bytes: 9, width: 2, height: 1, sourceWidth: 5, sourceHeight: 2, + }) + expect(envelope).toContain('downscaled from 5x2 px; multiply x coordinates by 2.50 and y coordinates by 2.00 to locate features in the original file') + }) }) describe('registration surface', () => { From c1bdac69398c624cce8f87f67cdbd74ba61667f3 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 14:55:31 +0800 Subject: [PATCH 10/28] docs: propose attachment read quarantine --- ...6-07-05-reconstructable-requests.i18n.yaml | 4 +- .../2026-07-05-reconstructable-requests.md | 1 + .../2026-07-05-reconstructable-requests.zh.md | 1 + ...08-20-attachment-read-quarantine.i18n.yaml | 6 +++ .../2026-08-20-attachment-read-quarantine.md | 38 +++++++++++++++++++ ...026-08-20-attachment-read-quarantine.zh.md | 38 +++++++++++++++++++ 6 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 .agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.i18n.yaml create mode 100644 .agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.md create mode 100644 .agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.zh.md diff --git a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.i18n.yaml b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.i18n.yaml index b2478911dc..47c4c2d198 100644 --- a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.md -2026-07-05-reconstructable-requests.md: 63146fa2d392a45543daa32ce2b00158782fddb2 -2026-07-05-reconstructable-requests.zh.md: 94c1d323be0107eb8b6072a05d1e8832ebd1fffc +2026-07-05-reconstructable-requests.md: 3f49ba71a6b98a84b05530c900e902b0cf9f6449 +2026-07-05-reconstructable-requests.zh.md: 8eee44449140d656a669ac506057e4fa09c2f747 diff --git a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.md b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.md index 63146fa2d3..3f49ba71a6 100644 --- a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.md +++ b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.md @@ -51,5 +51,6 @@ Like MiniCode, the conversation advances append-only and resets only when model- - What still costs full price at the provider is inherent and logged: compaction (its `compaction/*` events and replacement entry), a real prompt, tool, or config change (`request/header` with reason `change`), or a process boundary with drift (a differing `resume` snapshot). The provider's own reasoning-content exclusion is managed server-side. - `agent/pre-step` is the current-request message channel; direct inbox mutation is the eventual later-request channel. - Tool-result trimming needs no new mechanism: a logged single-entry surface replace (`start === end`) carrying a trimmed `tool/result` under the same `callId` — compaction-family, replay-correct, cache-bust batched by the same pressure logic. +- Unreadable referenced attachment objects still fail model requests; [automatic attachment quarantine](../../proposed/bug-fix/2026-08-20-attachment-read-quarantine.md) records the proposed recovery without weakening byte-exact reconstruction. - Session logs grow one `request/header` snapshot per loop instance plus snapshots on real changes. This is larger than a delta codec but small beside chunk-heavy logs and retains one replay representation. `SESSION_FORMAT_VERSION` stays `0`; legacy delta events are rejected rather than migrated. - Snapshot expected outputs changed once (every transcript gains its header events); the fs-writing fixtures are stored in the normalized authored form with cwd-relative tool arguments, because replay only round-trips cwd-independent argument paths. diff --git a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.zh.md b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.zh.md index 94c1d323be..8eee444491 100644 --- a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.zh.md +++ b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.zh.md @@ -51,5 +51,6 @@ Status: implemented - 在提供方处仍需全价计算的内容是固有的且已记录的:压缩(其 `compaction/*` 事件和替换条目)、真正的提示词、工具或配置变更(reason 为 `change` 的 `request/header`),或带漂移的进程边界(不同的 `resume` 快照)。提供方自身的 reasoning-content 排除由服务端管理。 - `agent/pre-step` 是当前请求的消息通道;直接修改 inbox 则是最终进入后续请求的通道。 - 工具结果裁剪无需新机制:一个已记录的单条目 surface replace(`start === end`),携带同一 `callId` 下裁剪后的 `tool/result`——属压缩家族,回放正确,缓存失效由相同的压力逻辑批量处理。 +- 无法读取的被引用附件对象仍会让模型请求失败;[附件自动隔离](../../proposed/bug-fix/2026-08-20-attachment-read-quarantine.md)记录了不削弱字节精确重建的拟议恢复方案。 - 会话日志每个循环实例增长一个 `request/header` 快照,并在真正变更时增加快照。它比 delta 编解码器更大,但相对分片密集型日志仍然很小,并只保留一种回放表示。`SESSION_FORMAT_VERSION` 保持 `0`;旧的 delta 事件被拒绝而非迁移。 - 快照预期输出变更一次(每个 transcript(文本记录)增加其 header 事件);写入文件系统的 fixture(测试前置数据)以规范化的撰写形式存储,工具参数使用 cwd 相对路径,因为回放只对 cwd 无关的参数路径做往返。 diff --git a/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.i18n.yaml b/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.i18n.yaml new file mode 100644 index 0000000000..ce37d7b8d3 --- /dev/null +++ b/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.md +2026-08-20-attachment-read-quarantine.md: 28e0f26cee2ec1e257fd4d43b4edc4300e2c6f23 +2026-08-20-attachment-read-quarantine.zh.md: bdc1d580a5159edcd288552e1bde9d80ea1eafd8 diff --git a/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.md b/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.md new file mode 100644 index 0000000000..28e0f26cee --- /dev/null +++ b/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.md @@ -0,0 +1,38 @@ +# Agent Note: Quarantine unreadable historical attachments + +Status: proposed + +English | [中文](2026-08-20-attachment-read-quarantine.zh.md) + +## Problem + +An admitted `ImageAttachmentRef` remains in durable history and therefore participates in every later request until compaction replaces it. `AttachmentStore.readImage()` fails with `ATTACHMENT_NOT_FOUND`, `ATTACHMENT_CORRUPT`, or `ATTACHMENT_READ_FAILED` when the referenced object disappears, fails integrity verification, or cannot be read. The unchanged history then makes every later model request fail on the same object, leaving the session unable to continue even though the remaining messages are usable. This is the unavailable-object case left fail-loud by [reconstructable requests](../../implemented/architecture/2026-07-05-reconstructable-requests.md). + +## Proposal + +A session-backed image-request projection records unreadable references before provider dispatch. `ATTACHMENT_NOT_FOUND` and `ATTACHMENT_CORRUPT` immediately append `attachment/quarantine`; `ATTACHMENT_READ_FAILED` receives one cancellation-aware read retry and appends the same event with a retryable reason if the retry fails. Cancellation and unclassified failures do not quarantine data. + +The quarantine event identifies the attachment and failure class. Projection replaces each quarantined image with deterministic text containing its display name when present, attachment-id prefix, and failure class. Later requests derive the same replacement from the log and skip `readImage()` for that reference, while the original image block remains in append-only history. A request that discovers and records a quarantine reprojects before calling the provider, so the failed read does not become a terminal model-request attempt. + +Explicit recovery calls `readImage()` and appends `attachment/recovered` only after digest and metadata verification succeeds. Projection then restores the original image reference. Missing or corrupt bytes are never overwritten automatically, and clearing quarantine without verification is invalid. + +The shared request-projection consumer owns this policy. Attachment storage continues to report exact read failures, and provider adapters do not invent independent placeholders or recovery state. + +## Alternatives considered + +- **Keep failing every request.** This preserves strict error reporting but makes an otherwise usable durable session permanently unavailable after one storage fault. +- **Delete or rewrite the historical image block.** That loses evidence, violates append-only history, and prevents a repaired content-addressed object from restoring the original request. +- **Catch the error independently in each adapter.** An unlogged placeholder would make replay depend on which adapter and storage state happened to be present, while duplicated policies would drift. +- **Replace missing or corrupt bytes automatically.** The reference names verified immutable content; substituting different bytes under that identity would defeat integrity checking. + +## Acceptance criteria + +- A missing or corrupt historical image produces one durable quarantine transition and a stable placeholder; later model requests do not read that object or fail because of it. +- A general read failure is retried once without ignoring cancellation, then follows the retryable quarantine path. +- Restart and fork reconstruct the same quarantined request from the session log. +- Recovery restores image projection only after the original reference passes complete read verification. +- Package tests cover error classification, idempotent quarantine, cancellation, retry, recovery, and nested tool-result images; a keyless runnable snapshot pins the model-visible placeholder and durable events. + +## Risks + +Quarantine and recovery each change the provider prefix once. The implementation must identify the exact failing reference before recording state and must coordinate concurrent requests so duplicate failures produce one effective transition. Auxiliary calls without a live session cannot record recovery state; their failure policy remains explicit implementation scope rather than an adapter fallback. diff --git a/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.zh.md b/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.zh.md new file mode 100644 index 0000000000..bdc1d580a5 --- /dev/null +++ b/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.zh.md @@ -0,0 +1,38 @@ +# Agent Note: 隔离无法读取的历史附件 + +Status: proposed + +[English](2026-08-20-attachment-read-quarantine.md) | 中文 + +## 问题 + +已接纳的 `ImageAttachmentRef` 会留在持久历史中,因此在被压缩替换前都会参与之后的每次请求。引用对象丢失、完整性校验失败或无法读取时,`AttachmentStore.readImage()` 会返回 `ATTACHMENT_NOT_FOUND`、`ATTACHMENT_CORRUPT` 或 `ATTACHMENT_READ_FAILED`。未变化的历史随后会让之后每次模型请求在同一对象上失败,使会话无法继续,即使其余消息仍可使用。这是[可重建请求](../../implemented/architecture/2026-07-05-reconstructable-requests.md)保留为明确失败的对象不可用情况。 + +## 提案 + +由会话支撑的图片请求投影在分派给提供方之前记录无法读取的引用。`ATTACHMENT_NOT_FOUND` 和 `ATTACHMENT_CORRUPT` 立即追加 `attachment/quarantine`;`ATTACHMENT_READ_FAILED` 先执行一次服从取消信号的读取重试,重试仍失败时追加同一事件并标记为可重试原因。取消和未分类失败不会隔离数据。 + +隔离事件标识附件和失败类别。投影把每张已隔离图片替换为确定性文本,包含可用时的显示名称、附件 ID 前缀和失败类别。之后的请求从日志派生相同替换结果,并跳过该引用的 `readImage()`,原始图片块仍留在仅追加历史中。请求发现并记录隔离后,会在调用提供方前重新投影,因此读取失败不会成为终止性的模型请求尝试。 + +显式恢复会调用 `readImage()`,且仅在内容摘要和元数据校验成功后追加 `attachment/recovered`。投影随后恢复原始图片引用。系统绝不会自动覆盖丢失或损坏的字节,也不允许未经验证就清除隔离。 + +共享请求投影消费方拥有这项策略。附件存储继续报告准确的读取失败,提供方适配器不会各自生成占位或恢复状态。 + +## 考虑过的替代方案 + +- **让每次请求继续失败。** 这保留了严格错误报告,但一次存储故障会让其他部分仍可使用的持久会话永久不可用。 +- **删除或重写历史图片块。** 这会丢失证据、违反仅追加历史,并使修复后的内容寻址对象无法恢复原始请求。 +- **由每个适配器分别捕获错误。** 未记录的占位会让回放取决于当时存在的适配器和存储状态,重复策略也会发生偏差。 +- **自动替换丢失或损坏的字节。** 引用标识经过验证的不可变内容;在该身份下替换成其他字节会破坏完整性校验。 + +## 接受标准 + +- 缺失或损坏的历史图片产生一次持久隔离转换和稳定占位;之后的模型请求不再读取该对象,也不会因它失败。 +- 一般读取失败会在服从取消信号的前提下重试一次,随后进入可重试隔离路径。 +- 重启和 fork 后会从会话日志重建相同的隔离请求。 +- 仅在原始引用通过完整读取校验后,恢复操作才恢复图片投影。 +- 包测试覆盖错误分类、幂等隔离、取消、重试、恢复和嵌套工具结果图片;一个无需密钥的可运行快照钉住模型可见占位和持久事件。 + +## 风险 + +隔离和恢复各会改变一次提供方前缀。实现必须在记录状态前识别准确的失败引用,并协调并发请求,使重复失败只产生一次有效转换。没有活跃会话的辅助调用无法记录恢复状态;它们的失败策略属于明确的实现范围,不能退回到适配器自行处理。 From d29855f97c406893a4167d74a53db521ab8b308b Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 18:19:23 +0800 Subject: [PATCH 11/28] feat(images): unify master and Files request pipeline --- ...i-route-default-input-modalities.i18n.yaml | 4 +- ...12-pi-ai-route-default-input-modalities.md | 8 +- ...pi-ai-route-default-input-modalities.zh.md | 8 +- ...07-29-atomic-web-image-admission.i18n.yaml | 4 +- .../2026-07-29-atomic-web-image-admission.md | 12 +- ...026-07-29-atomic-web-image-admission.zh.md | 12 +- ...-image-dimension-admission-limit.i18n.yaml | 6 - ...6-08-17-image-dimension-admission-limit.md | 30 -- ...8-17-image-dimension-admission-limit.zh.md | 30 -- ...8-18-request-image-payload-bound.i18n.yaml | 6 - .../2026-08-18-request-image-payload-bound.md | 36 -- ...26-08-18-request-image-payload-bound.zh.md | 36 -- ...ge-input-and-durable-attachments.i18n.yaml | 4 +- ...dal-image-input-and-durable-attachments.md | 30 +- ...-image-input-and-durable-attachments.zh.md | 30 +- ...26-08-10-minimal-read-image-tool.i18n.yaml | 4 +- .../2026-08-10-minimal-read-image-tool.md | 19 +- .../2026-08-10-minimal-read-image-tool.zh.md | 19 +- ...mage-intake-and-limits-alignment.i18n.yaml | 4 +- ...2-web-image-intake-and-limits-alignment.md | 2 +- ...eb-image-intake-and-limits-alignment.zh.md | 2 +- ...2026-08-19-direct-deepseek-vision-input.md | 34 -- ...6-08-19-direct-deepseek-vision-input.zh.md | 34 -- ...-08-20-canonical-image-admission.i18n.yaml | 6 - .../2026-08-20-canonical-image-admission.md | 28 -- ...2026-08-20-canonical-image-admission.zh.md | 28 -- ...-unified-image-request-pipeline.i18n.yaml} | 6 +- ...26-08-20-unified-image-request-pipeline.md | 71 ++++ ...08-20-unified-image-request-pipeline.zh.md | 71 ++++ docs/config-catalog.i18n.yaml | 4 +- docs/config-catalog.md | 42 ++- docs/config-catalog.zh.md | 42 ++- docs/event-producer-consumer.i18n.yaml | 4 +- docs/event-producer-consumer.md | 2 +- docs/event-producer-consumer.zh.md | 2 +- docs/subsystems/attachment.i18n.yaml | 4 +- docs/subsystems/attachment.md | 105 +++++- docs/subsystems/attachment.zh.md | 105 +++++- docs/subsystems/llm-streaming.i18n.yaml | 4 +- docs/subsystems/llm-streaming.md | 12 + docs/subsystems/llm-streaming.zh.md | 12 + docs/tool-catalog.i18n.yaml | 4 +- docs/tool-catalog.md | 55 ++- docs/tool-catalog.zh.md | 55 ++- examples/acp-agent/tests/acp.snapshot.ts | 125 ++++--- .../tests/fixtures/image-offload.cordis.yml | 3 +- .../system-prompt.expected.md | 40 ++ .../read-image/tool-schemas.expected.json | 46 +++ .../attachment-local/README.i18n.yaml | 4 +- .../attachment/attachment-local/README.md | 10 +- .../attachment/attachment-local/README.zh.md | 10 +- .../attachment-local/src/canonical.ts | 230 ++++++++---- .../src/compression-limiter.ts | 43 +++ .../attachment-local/src/encoding.ts | 45 +++ .../attachment/attachment-local/src/image.ts | 26 +- .../attachment/attachment-local/src/index.ts | 172 +++++++-- .../attachment-local/src/request-image.ts | 353 ++++++++++++++++++ .../attachment/attachment-local/src/store.ts | 111 ++++-- .../attachment-local/tests/canonical.spec.ts | 208 +++++++++-- .../attachment-local/tests/encoding.spec.ts | 70 ++++ .../attachment-local/tests/image.spec.ts | 20 +- .../attachment-local/tests/index.spec.ts | 50 ++- .../tests/request-image.spec.ts | 209 +++++++++++ .../attachment-local/tests/store.spec.ts | 8 +- .../attachment/attachment/README.i18n.yaml | 4 +- packages/attachment/attachment/README.md | 6 +- packages/attachment/attachment/README.zh.md | 6 +- packages/attachment/attachment/src/brand.ts | 12 + packages/attachment/attachment/src/error.ts | 1 + packages/attachment/attachment/src/index.ts | 84 ++++- packages/attachment/attachment/src/types.ts | 58 ++- .../attachment/attachment/tests/index.spec.ts | 35 ++ .../extensions/tool-cordis/src/api-catalog.ts | 58 ++- packages/fs/tool-fs/README.i18n.yaml | 4 +- packages/fs/tool-fs/README.md | 15 +- packages/fs/tool-fs/README.zh.md | 15 +- packages/fs/tool-fs/src/read-image.ts | 191 +++++++++- packages/fs/tool-fs/tests/read-image.spec.ts | 83 +++- packages/host/apiproxy/src/api-proxy.ts | 18 +- .../apiproxy/tests/api-proxy-models.spec.ts | 17 +- packages/llm/llm-deepseek/README.i18n.yaml | 4 +- packages/llm/llm-deepseek/README.md | 29 +- packages/llm/llm-deepseek/README.zh.md | 29 +- packages/llm/llm-deepseek/package.json | 6 + packages/llm/llm-deepseek/src/adapter.ts | 332 ++++++++++++---- packages/llm/llm-deepseek/src/file-id.ts | 27 ++ packages/llm/llm-deepseek/src/file-store.ts | 257 +++++++++++++ packages/llm/llm-deepseek/src/files-api.ts | 257 +++++++++++++ packages/llm/llm-deepseek/src/index.ts | 128 ++++++- packages/llm/llm-deepseek/src/serialize.ts | 136 ++++--- packages/llm/llm-deepseek/src/types.ts | 10 +- packages/llm/llm-deepseek/src/upload-index.ts | 225 +++++++++++ .../llm/llm-deepseek/tests/adapter.e2e.ts | 138 +++++-- .../llm/llm-deepseek/tests/adapter.spec.ts | 286 +++++++++++++- .../llm-deepseek/tests/dynamic-config.spec.ts | 31 +- .../llm/llm-deepseek/tests/file-store.spec.ts | 135 +++++++ .../llm/llm-deepseek/tests/files-api.spec.ts | 102 +++++ .../llm/llm-deepseek/tests/mock-server.ts | 130 +++++-- .../llm/llm-deepseek/tests/serialize.spec.ts | 206 +++++----- .../llm-deepseek/tests/upload-index.spec.ts | 73 ++++ packages/llm/llm-deepseek/tsconfig.json | 12 + packages/llm/llm-pi-ai/README.i18n.yaml | 4 +- packages/llm/llm-pi-ai/README.md | 13 +- packages/llm/llm-pi-ai/README.zh.md | 13 +- packages/llm/llm-pi-ai/src/adapter.ts | 58 ++- packages/llm/llm-pi-ai/src/config.ts | 24 ++ packages/llm/llm-pi-ai/src/context.ts | 68 +++- packages/llm/llm-pi-ai/tests/adapter.spec.ts | 50 ++- packages/llm/llm-pi-ai/tests/context.spec.ts | 91 +++-- packages/llm/llm-pi-ai/tests/convert.spec.ts | 34 +- .../llm/llm-pi-ai/tests/provider-apis.e2e.ts | 22 +- packages/llm/llm/README.i18n.yaml | 4 +- packages/llm/llm/README.md | 8 +- packages/llm/llm/README.zh.md | 8 +- packages/llm/llm/src/content.ts | 130 ++++++- packages/llm/llm/src/index.ts | 101 ++++- packages/llm/llm/tests/content.spec.ts | 32 +- packages/llm/llm/tests/service.spec.ts | 73 ++++ pnpm-lock.yaml | 9 + scripts/gen-cordis-catalog.ts | 3 + scripts/gen-tool-catalog.ts | 8 +- scripts/type-equiv.manifest.json | 20 + 122 files changed, 5566 insertions(+), 1186 deletions(-) delete mode 100644 .agents/notes/implemented/bug-fix/2026-08-17-image-dimension-admission-limit.i18n.yaml delete mode 100644 .agents/notes/implemented/bug-fix/2026-08-17-image-dimension-admission-limit.md delete mode 100644 .agents/notes/implemented/bug-fix/2026-08-17-image-dimension-admission-limit.zh.md delete mode 100644 .agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.i18n.yaml delete mode 100644 .agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.md delete mode 100644 .agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.zh.md delete mode 100644 .agents/notes/implemented/feature/2026-08-19-direct-deepseek-vision-input.md delete mode 100644 .agents/notes/implemented/feature/2026-08-19-direct-deepseek-vision-input.zh.md delete mode 100644 .agents/notes/implemented/feature/2026-08-20-canonical-image-admission.i18n.yaml delete mode 100644 .agents/notes/implemented/feature/2026-08-20-canonical-image-admission.md delete mode 100644 .agents/notes/implemented/feature/2026-08-20-canonical-image-admission.zh.md rename .agents/notes/implemented/feature/{2026-08-19-direct-deepseek-vision-input.i18n.yaml => 2026-08-20-unified-image-request-pipeline.i18n.yaml} (56%) create mode 100644 .agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md create mode 100644 .agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md create mode 100644 packages/attachment/attachment-local/src/compression-limiter.ts create mode 100644 packages/attachment/attachment-local/src/encoding.ts create mode 100644 packages/attachment/attachment-local/src/request-image.ts create mode 100644 packages/attachment/attachment-local/tests/encoding.spec.ts create mode 100644 packages/attachment/attachment-local/tests/request-image.spec.ts create mode 100644 packages/llm/llm-deepseek/src/file-id.ts create mode 100644 packages/llm/llm-deepseek/src/file-store.ts create mode 100644 packages/llm/llm-deepseek/src/files-api.ts create mode 100644 packages/llm/llm-deepseek/src/upload-index.ts create mode 100644 packages/llm/llm-deepseek/tests/file-store.spec.ts create mode 100644 packages/llm/llm-deepseek/tests/files-api.spec.ts create mode 100644 packages/llm/llm-deepseek/tests/upload-index.spec.ts diff --git a/.agents/notes/implemented/architecture/2026-08-12-pi-ai-route-default-input-modalities.i18n.yaml b/.agents/notes/implemented/architecture/2026-08-12-pi-ai-route-default-input-modalities.i18n.yaml index 4ab07f7f07..9338e0cf6c 100644 --- a/.agents/notes/implemented/architecture/2026-08-12-pi-ai-route-default-input-modalities.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-08-12-pi-ai-route-default-input-modalities.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-08-12-pi-ai-route-default-input-modalities.md -2026-08-12-pi-ai-route-default-input-modalities.md: efd20b2cd73979208bb777fa42536bc5b918e29e -2026-08-12-pi-ai-route-default-input-modalities.zh.md: 069a7916c8d4ffe738ff910a851e0a9cd1f66d0a +2026-08-12-pi-ai-route-default-input-modalities.md: eb03d5330a1283d439e16262325965cf2e7e8087 +2026-08-12-pi-ai-route-default-input-modalities.zh.md: dfbbd2ae6db7a78e82955db66fe506d3506209fd diff --git a/.agents/notes/implemented/architecture/2026-08-12-pi-ai-route-default-input-modalities.md b/.agents/notes/implemented/architecture/2026-08-12-pi-ai-route-default-input-modalities.md index efd20b2cd7..eb03d5330a 100644 --- a/.agents/notes/implemented/architecture/2026-08-12-pi-ai-route-default-input-modalities.md +++ b/.agents/notes/implemented/architecture/2026-08-12-pi-ai-route-default-input-modalities.md @@ -18,17 +18,17 @@ The assumption was justified in the source as the adapter's real capability rath **The route value is a fallback, not an override — the catalog outranks it.** This is the `default*` ordering rather than `compat`'s, and the two are not interchangeable: `compat` shadows the catalog because a route-level protocol repoint invalidates the catalog's reasoning-dispatch facts wholesale, while a modality is a per-model property the catalog states accurately for the models it ships. Making the route value win would mean `defaultInput: [text]` silently strips images from every catalog vision model on the route — a footgun with no matching benefit, since narrowing one such model is what that model's own `input` is for. -**Undeclared means `[text]`, and that is the absence of a declaration rather than a guess at the endpoint.** Nothing can interrogate a gateway for its modalities — no OpenAI-compatible listing endpoint reports them — so the only honest floor is the modality every supported protocol certainly carries. This is where the modality fallback parts company with the capacity ones: 262,144 tokens is merely plausible and wrong in both directions (a gateway serving 8k overflows, one serving 1M is wasted), while text is safe in one direction. The two wrong answers do not cost the same either. Under-claiming refuses the image before it is attached, naming the model, and the remedy is one documented line. Over-claiming admits an image the provider then rejects mid-turn, *after* prompt admission has committed the message durably, so the session keeps re-sending a request that cannot succeed and model selection refuses a switch to any text-only model. A cheap refusal at the earliest resolvable point beats an expensive one at the latest. +**Undeclared means `[text]`, and that is the absence of a declaration rather than a guess at the endpoint.** Nothing can interrogate a gateway for its modalities because no OpenAI-compatible listing endpoint reports them. The only safe floor is the modality every supported protocol certainly carries. Under-claiming refuses the image before it is attached, names the model, and has a documented configuration remedy. Over-claiming admits and persists an image before the provider can reject it. Later requests to that same incorrectly declared route will encounter the image again, although the user can select a text-only model because request assembly projects durable images to placeholders. **An entry's empty list means the same as an absent one; the route's is refused.** `[]` describes a model that accepts nothing and could serve no request, so it states no answer and resolution continues past it. That reading is not cosmetic: the config schema materializes `[]` for an absent array, so treating it as "accepts nothing" would silently strip images from every catalog vision model a `models` list happens to name. The route value has nothing below it to answer instead, so its empty list is refused where it is written. The route's `models` list already resolves absent-and-empty the same way for the same reason. **No configuration surface edits `input`.** It joins `compat`, `reasoningEfforts`, `thinkingBudgets`, and `headers` as a settings-document field, and the model-list editor stays a hand-written form over id, name, and the two capacities. This costs nothing durable because that card was already built to carry fields it does not edit: its row patch spreads the stored row before applying changes, and adoption keeps an existing row over a rediscovered candidate, so a hand-written `input` survives both. -The DeepSeek chat-completions adapter is untouched. Its `['text']` is a fact about its serializer, not a missing declaration, and it keeps refusing before the send. +The direct DeepSeek adapter owns a separate exact-model catalog. Its supported vision entry declares image input, while its text models and unlisted pass-through ids remain text-only. ## Alternatives considered -- **An optimistic `[text, image]` default** — makes the motivating case work with zero configuration, and the web form writes no modality at all, so a conservative default leaves the remedy in a file a web-only user has no reason to open. Rejected on the severity of being wrong: a refused attachment is a speed bump with a documented fix, while a provider rejection poisons the session, presents as an unexplained repeating failure, and is escapable only by switching models or starting over. Documenting the remedy on the model-configuration page closes the discoverability gap; nothing closes the poisoned session. +- **An optimistic `[text, image]` default** — makes the motivating case work with zero configuration, and the web form writes no modality at all, so a conservative default leaves the remedy in the settings document. Rejected because a false positive persists an image before the provider refuses it and causes repeated failure on that route. Text-only request projection provides recovery but does not make the declaration true. - **A route value that overrides the catalog** (`compat`'s ordering: entry → route → catalog) — lets a deployment that repoints a catalog route at its own gateway declare "no vision here" once. Rejected because the same sentence then silently disables every catalog vision model on a route where someone wrote it by analogy with the capacity fields, and the legitimate case is served by that model's own `input`. An override would also have to be named `input` at the route, since calling it `default*` beside two genuine fallbacks would misdescribe it. - **No route field at all, only the entry one** — closest to upstream, which has no route-level concept. Rejected on the bulk case the product's own flow produces: "fetch available models" adopts thirty ids with no modality, and an all-vision gateway would need `input` hand-written on each. - **A route-level `defaultInput` with no entry field** — cannot mix modalities on one route or correct a single catalog model, leaving "split the provider across two route keys" as the only workaround, at the cost of a second permanent provider id and a duplicate entry in every model selector. @@ -42,7 +42,7 @@ A vision model on a custom provider costs one line, `input: [text, image]`, writ The image-admission gate keeps its meaning everywhere, because every modality it reads is now either recorded by the installed catalog or written by a person. Nothing claims a capability on a deployment's behalf. -A model that declares images its endpoint does not serve is not caught locally — the claim is not verified — and the resulting failure is expensive. Prompt admission commits the user message durably (`agent/inbox/spliced`) before the request is built, so the rejected image stays in the session log: that model keeps re-sending it, and model selection refuses a switch to any text-only model. Recovery is to select a model that does serve images, fork before the image, or start a session. Making that failure non-destructive — rolling an unconsumed image message back out of the log when the send fails — is the change that would make an optimistic default reconsiderable, and is not attempted here. +A model that declares image input its endpoint does not serve is not caught locally because the claim is not verified. Prompt admission commits the user message durably before request construction, so the rejected image stays in the session log and later requests to that route can fail again. Recovery is to correct the declaration, select an image-capable route, or select a text-only route whose request projection replaces durable images with placeholders. ## Testing diff --git a/.agents/notes/implemented/architecture/2026-08-12-pi-ai-route-default-input-modalities.zh.md b/.agents/notes/implemented/architecture/2026-08-12-pi-ai-route-default-input-modalities.zh.md index 069a7916c8..dfbbd2ae6d 100644 --- a/.agents/notes/implemented/architecture/2026-08-12-pi-ai-route-default-input-modalities.zh.md +++ b/.agents/notes/implemented/architecture/2026-08-12-pi-ai-route-default-input-modalities.zh.md @@ -18,17 +18,17 @@ Harness 把缺失的模态当作否定能力,并有三个准入点在构造任 **路由值是回退值而非覆盖值——catalog 的优先级更高。** 这采用的是 `default*` 的顺序而非 `compat` 的,两者不可互换:`compat` 之所以盖住 catalog,是因为路由级的协议改指会整体作废 catalog 关于推理分派的事实;而模态是按模型的属性,对 catalog 自己出货的那些模型,它记录得准确无误。让路由值获胜就意味着 `defaultInput: [text]` 会悄悄剥掉该路由上每一个 catalog 视觉模型的图片能力——一个没有对应收益的坑,因为收窄其中某个模型正是该模型自己的 `input` 要做的事。 -**未声明即 `[text]`,而这是「尚未声明」,不是对端点的猜测。** 没有任何环节能去询问网关的模态——没有任何 OpenAI 兼容的列表端点会报告它们——因此唯一诚实的底线是每个受支持协议都确定携带的那个模态。这也正是模态回退值与容量回退值分道扬镳之处:262,144 只是个说得过去的数字,且两个方向都会错(网关只给 8k 会溢出,给 1M 则被浪费),而 text 在一个方向上是安全的。两种猜错的代价同样并不对等。少声明会在图片被附加之前就拒绝并点名该模型,补救办法是一行有文档可依的配置。多声明会接纳一张图片、再由提供方在轮次中途拒绝——而此时 prompt 准入**早已**把消息持久化提交,于是会话会不断重发一个不可能成功的请求,且模型选择拒绝切换到任何纯文本模型。在最早可解析点付出一次廉价的拒绝,胜过在最晚点付出一次昂贵的。 +**未声明即 `[text]`,而这是「尚未声明」,不是对端点的猜测。** 没有任何环节能询问网关的模态,因为 OpenAI 兼容列表端点不会报告它们。安全的底线是每个受支持协议都确定携带的模态。少声明会在图片附加之前拒绝、点名模型,并给出有文档的配置补救方法。多声明会先接纳并持久化图片,再由提供方拒绝。之后对同一错误声明路由的请求还会再次遇到图片,但用户可以选择纯文本模型,因为请求组装会把持久图片投影为占位符。 **条目的空列表与缺省同义;路由的空列表则被拒绝。** `[]` 描述的是一个什么都不接受、无法服务任何请求的模型,因此不作答,解析继续往下走。这个读法不是修辞:配置 schema 会为缺省数组物化出 `[]`,把它当作“什么都不接受”,会悄悄剥掉 `models` 列表恰好点到的每一个 catalog 视觉模型的图片能力。而路由值下面没有可以代为作答的层级,因此它的空列表在写入处即被拒绝。路由的 `models` 列表出于同样的理由,早已用同一种方式解析缺省与空。 **没有任何配置界面编辑 `input`。** 它和 `compat`、`reasoningEfforts`、`thinkingBudgets`、`headers` 一样是 settings 文档字段,而模型列表编辑器仍是一张只覆盖 id、名称和两个容量的手写表单。这不会带来持久代价,因为那张卡片本来就是按“承载自己并不编辑的字段”建造的:它的行 patch 会先展开已存储的行再应用改动,而采纳候选时已有行优先于重新发现的候选,因此手写的 `input` 在两条路径上都能存活。 -DeepSeek chat-completions 适配器保持不动。它的 `['text']` 是关于其序列化器的事实,而不是一处缺失的声明,它继续在发送前拒绝。 +DeepSeek 直接适配器拥有独立的精确模型目录。支持视觉的条目声明图片输入,纯文本模型和未列出的透传 ID 保持纯文本。 ## 备选方案 -- **乐观的 `[text, image]` 默认值** —— 让触发本次变更的场景零配置即可工作;而且网页表单不会写入任何模态,因此保守默认值会把补救办法留在一个纯 Web 用户没有理由打开的文件里。被否决的理由是猜错时的严重程度:被拒绝的附件是一个有文档可依的减速带,而提供方拒绝会毒化整个会话、表现为一次无从解释的反复失败,且只能靠换模型或重开会话脱身。把补救办法写进配置模型页即可补上可发现性的缺口;而毒化的会话没有任何东西能补。 +- **乐观的 `[text, image]` 默认值** —— 让触发场景无需配置即可工作,而网页表单不会写入模态,因此保守默认值会把补救方法留在 settings 文档里。否决原因是错误的肯定声明会在提供方拒绝之前持久化图片,并让该路由重复失败。纯文本请求投影提供了恢复方法,但不能让错误声明变成事实。 - **让路由值盖住 catalog**(`compat` 的顺序:条目 → 路由 → catalog)—— 可以让把 catalog 路由改指到自家网关的部署,一句话声明「这里没有视觉能力」。被否决是因为同一句话也会在有人照着容量字段类比写下它的路由上,悄悄禁用每一个 catalog 视觉模型;而那个正当场景由该模型自己的 `input` 承担。覆盖值还必须在路由级改名叫 `input`,因为在两个货真价实的回退值旁边把它叫作 `default*` 是名不副实。 - **完全不要路由字段,只要条目字段** —— 最贴近上游(上游没有路由级概念)。被否决的理由是产品自身流程会产生的批量场景:「获取可用模型」一次采纳三十个不带模态的 id,全是视觉模型的网关就得逐个手写 `input`。 - **只要路由级 `defaultInput`,不要条目字段** —— 无法在一条路由上混合模态,也无法修正单个 catalog 模型,唯一的变通办法只剩「把该提供方拆成两个路由键」,代价是多一个永久的 provider id 和每个模型选择器里的一项重复。 @@ -42,7 +42,7 @@ DeepSeek chat-completions 适配器保持不动。它的 `['text']` 是关于其 图片准入门禁在各处都保住了自己的意义,因为它读到的每一个模态,如今要么由已安装 catalog 记录,要么由人写下。没有任何环节会替部署宣称一项能力。 -声明了端点并不提供的图片能力的模型不会在本地被拦下——该断言不经验证——而由此产生的失败代价高昂。prompt 准入在构造请求之前就把用户消息持久化提交(`agent/inbox/spliced`),因此被拒绝的图片会留在会话日志里:该模型会不断重发它,而模型选择拒绝切换到任何纯文本模型。恢复途径是选择一个确实提供图片能力的模型、fork 到图片之前,或者开启新会话。让这次失败不具破坏性——发送失败时把尚未消费的图片消息从日志中回滚出去——才是能让乐观默认值重新可考虑的那项改动,本次未做尝试。 +声明了端点并不提供的图片能力时,本地无法发现该错误,因为声明不会被远端验证。prompt 准入会在请求构造前持久化用户消息,因此被拒绝的图片留在会话日志中,之后对该路由的请求可能再次失败。恢复方法是修正声明、选择支持图片的路由,或选择由请求投影把持久图片替换为占位符的纯文本路由。 ## 测试 diff --git a/.agents/notes/implemented/bug-fix/2026-07-29-atomic-web-image-admission.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-07-29-atomic-web-image-admission.i18n.yaml index d5c2e38a49..a897753a57 100644 --- a/.agents/notes/implemented/bug-fix/2026-07-29-atomic-web-image-admission.i18n.yaml +++ b/.agents/notes/implemented/bug-fix/2026-07-29-atomic-web-image-admission.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/bug-fix/2026-07-29-atomic-web-image-admission.md -2026-07-29-atomic-web-image-admission.md: c09d376f101a41994df3a10c22c06da4e59f06f6 -2026-07-29-atomic-web-image-admission.zh.md: 8785f7489b0c433cba43a1747533b1d38aada3d3 +2026-07-29-atomic-web-image-admission.md: dd2faf1e14c6147c80bcba571d5310899c2e8e22 +2026-07-29-atomic-web-image-admission.zh.md: 8f15f8848dcb38fe6be178b86082a72b4ff0c8eb diff --git a/.agents/notes/implemented/bug-fix/2026-07-29-atomic-web-image-admission.md b/.agents/notes/implemented/bug-fix/2026-07-29-atomic-web-image-admission.md index c09d376f10..dd2faf1e14 100644 --- a/.agents/notes/implemented/bug-fix/2026-07-29-atomic-web-image-admission.md +++ b/.agents/notes/implemented/bug-fix/2026-07-29-atomic-web-image-admission.md @@ -6,24 +6,24 @@ English | [中文](2026-07-29-atomic-web-image-admission.zh.md) ## Problem -Image prompt admission and `session.selectModel` each read session modality state across asynchronous model and attachment lookups. Without one ordering boundary, an image prompt could validate an image-capable target while a concurrent selection installed a text-only target, or selection could miss a prompt after inbox dequeue but before its durable message event. Scanning the immutable event log avoided the second race but permanently blocked a text-only selection even after compaction removed the image from current model history. +Image prompt admission and `session.selectModel` each cross asynchronous model and attachment lookups. Without one ordering point, an image prompt could validate an image-capable target while a concurrent selection installed a text-only target. Selection could also change the route after admission had begun but before the durable message event was published. ## Decision -Each live Web agent has one private promise chain shared by image-bearing prompt admission and model selection. A failed operation settles its caller normally and leaves the chain usable. Text-only prompts bypass the chain because they cannot change the modality constraint. +Each live Web agent has one private promise chain shared by image-bearing prompt admission and model selection. A failed operation settles its caller normally and leaves the chain usable. Text-only prompts bypass the chain because they cannot create this ordering conflict. -The pending-publication set records a queued occurrence at dequeue and a steering occurrence already at enqueue (steering items never enter the queued UI mirror), and retains each until its matching `user/message` or `steering/message` event publishes. If admission ends without publishing, the transition to idle retires the entries; inbox discard retires the listed work, and session disposal retires every remaining entry. Model selection checks that set, the queued UI mirror, and `Session.deriveMessages()`, which is the current model-visible history after compaction. +The chain gives the two operations a deterministic order. When selection runs first, later image admission observes the selected model and refuses an unsupported image before persistence. When image admission runs first, its attachment and event publication complete before selection changes the route. The shared LLM runtime can then project durable image blocks to deterministic text placeholders for a text-only request without rewriting the session log. Steering uses the same admission chain even though it does not enter the queued UI mirror. Provider adapters remain the final enforcement boundary. The host ordering only prevents its mutable route and pending image state from contradicting each other before request assembly. ## Alternatives considered -**Scan every immutable session event.** This catches published images but treats compacted-away content as permanently model-visible, preventing a valid later switch to a text-only route. +**Scan durable or derived history before selection.** This prevented a text-only route from being selected whenever history contained an image. Request-local projection now supports that route directly, so history is no longer a selection constraint. -**Retire the pending mirror at inbox dequeue.** Dequeue precedes the durable message append and leaves the exact interval in which model selection can miss both pending and published state. +**Track pending publication separately.** A queued occurrence could be retained from dequeue through its matching event. The promise chain already keeps selection behind the complete admission operation, so a second lifecycle mirror is unnecessary. **Serialize every prompt and session mutation.** Text-only prompts and unrelated session operations cannot introduce an image requirement. A broader lock would add latency and ownership without closing another modality race. ## Consequences -An image prompt and a concurrent model selection have deterministic order, and a text-only target cannot strand an image that has been admitted but not yet published. Selection may wait for an in-flight image admission, while unrelated prompts retain their existing concurrency. Compaction can make a text-only target valid once no pending or derived image remains. +An image prompt and a concurrent model selection have deterministic order. Selection may wait for in-flight image admission, while unrelated text prompts retain their existing concurrency. Text-only model selection remains available after images enter durable history because request assembly projects those images to placeholders. diff --git a/.agents/notes/implemented/bug-fix/2026-07-29-atomic-web-image-admission.zh.md b/.agents/notes/implemented/bug-fix/2026-07-29-atomic-web-image-admission.zh.md index 8785f7489b..8f15f8848d 100644 --- a/.agents/notes/implemented/bug-fix/2026-07-29-atomic-web-image-admission.zh.md +++ b/.agents/notes/implemented/bug-fix/2026-07-29-atomic-web-image-admission.zh.md @@ -6,24 +6,24 @@ Status: implemented ## 问题 -包含图片的提示词准入与 `session.selectModel` 都会在跨越异步模型查询与附件查询的过程中读取会话模态状态。如果没有统一的顺序边界,包含图片的提示词可能在支持图片的目标上通过校验,并发的选择操作却设置了纯文本目标;选择操作也可能在提示词已从 inbox 出队、但其持久消息事件尚未发布时漏掉该提示词。扫描不可变事件日志可以避免第二种竞态,但即使压缩(compaction)已经从当前模型历史中移除图片,仍会永久阻止选择纯文本目标。 +包含图片的提示词准入与 `session.selectModel` 都会跨越异步模型查询和附件查询。没有统一的排序点时,包含图片的提示词可能在支持图片的目标上通过校验,并发选择却设置了纯文本目标。选择也可能在准入已经开始、持久消息事件尚未发布时改变路由。 ## 决策 -每个活跃 Web agent(智能体)都有一条私有 promise 链,由包含图片的提示词准入与模型选择共享。操作失败会照常传递给调用方,且不会使该链失效。纯文本提示词绕过该链,因为它们不会改变模态约束。 +每个活跃 Web agent(智能体)都有一条私有 promise 链,由包含图片的提示词准入与模型选择共享。操作失败会照常传递给调用方,且不会使该链失效。纯文本提示词绕过该链,因为它们不会产生这类排序冲突。 -待发布集合会在排队条目出队时记录它,而 steering 条目在入队时即被记录(steering 条目从不进入排队 UI 镜像),并各自保留到匹配的 `user/message` 或 `steering/message` 事件发布。若准入结束时未发布事件,转为空闲状态会移除这些条目;inbox 丢弃会移除列出的工作项,会话 dispose(资源释放)则会移除所有剩余条目。模型选择会检查该集合、排队 UI 镜像以及 `Session.deriveMessages()`;后者表示压缩后模型当前可见的历史。 +该链为两个操作提供确定顺序。模型选择先执行时,后续图片准入会看到已选模型,并在持久化之前拒绝不支持的图片。图片准入先执行时,附件和事件会在模型选择改变路由之前完成发布。之后,共享 LLM 运行时可以在纯文本请求中把持久图片块投影为确定的文本占位符,无需改写会话日志。steering 不进入排队 UI 镜像,但仍使用同一条准入链。 提供方适配器仍是最终的强制检查边界。宿主的顺序控制仅用于避免其可变路由与待发布图片状态在请求组装前彼此矛盾。 ## 曾考虑的替代方案 -**扫描每个不可变会话事件。** 这能捕获已发布的图片,但会把经压缩移除的内容视为永久对模型可见,从而阻止之后合法切换到纯文本路由。 +**选择前扫描持久历史或派生历史。** 这会在历史包含图片时阻止选择纯文本路由。请求期投影已经可以直接支持该路由,因此历史不再是选择约束。 -**在 inbox 出队时退役待处理镜像。** 出队早于持久消息追加,因此恰好会留下一个时间区间,让模型选择既看不到待处理状态,也看不到已发布状态。 +**单独跟踪待发布状态。** 排队条目可以从出队一直保留到匹配事件发布。promise 链已经让模型选择等待完整的准入操作,因此不需要第二套生命周期镜像。 **序列化每个提示词和会话变更。** 纯文本提示词和无关的会话操作无法引入图片要求。更宽的锁会增加延迟与所有权复杂度,却不会再消除任何模态竞态。 ## 后果 -包含图片的提示词准入与并发模型选择之间具有确定的先后顺序,纯文本目标无法使已获准入但尚未发布的图片搁浅。模型选择可能等待正在进行的图片准入完成,而无关提示词仍按现有方式并发处理。当没有图片等待发布,且派生历史经过压缩后也不再含图片时,纯文本目标可以变得有效。 +包含图片的提示词准入与并发模型选择之间具有确定顺序。模型选择可能等待正在进行的图片准入完成,无关的纯文本提示词仍按现有方式并发处理。图片进入持久历史后仍可选择纯文本模型,因为请求组装会把图片投影为占位符。 diff --git a/.agents/notes/implemented/bug-fix/2026-08-17-image-dimension-admission-limit.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-08-17-image-dimension-admission-limit.i18n.yaml deleted file mode 100644 index ec3cea9dc2..0000000000 --- a/.agents/notes/implemented/bug-fix/2026-08-17-image-dimension-admission-limit.i18n.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each -# side as of the last confirmed-consistent state. Both languages carry equal authority; -# after editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write .agents/notes/implemented/bug-fix/2026-08-17-image-dimension-admission-limit.md -2026-08-17-image-dimension-admission-limit.md: 027259c0949d142ce8d8af27e7daa2abd54769ab -2026-08-17-image-dimension-admission-limit.zh.md: 38422615aa93b7f1639877d7d3751322c77de1eb diff --git a/.agents/notes/implemented/bug-fix/2026-08-17-image-dimension-admission-limit.md b/.agents/notes/implemented/bug-fix/2026-08-17-image-dimension-admission-limit.md deleted file mode 100644 index 027259c094..0000000000 --- a/.agents/notes/implemented/bug-fix/2026-08-17-image-dimension-admission-limit.md +++ /dev/null @@ -1,30 +0,0 @@ -# Agent Note: Per-side image dimension admission limit - -Status: implemented - -English | [中文](2026-08-17-image-dimension-admission-limit.zh.md) - -## Problem - -`read_image` durably committed an image and appended its block to session history before any dimension check beyond byte count and total pixels. Deployed model routes reject a request with HTTP 400 when it carries many images and any of them has a side above 2000px. An admitted image rides every later request of its session, so one oversized read poisoned the durable history: the next model request failed, and so did every retry, permanently killing the session. The same gap applied to every other image producer (host uploads, MCP tool images) because admission had no per-side bound at all. - -## Decision - -`ImageAttachmentLimits` carries `maxImageDimension`, enforced during the admission full decode (`detectImage`) as `IMAGE_DIMENSION_TOO_LARGE`, so every producer that commits through the attachment service refuses an oversized image before anything reaches durable history. `LocalAttachmentStore` exposes it as the `maxImageDimension` config field with default `DEFAULT_MAX_IMAGE_DIMENSION = 2000`, the strictest per-side bound deployed routes enforce; deployments with laxer routes raise it from cordis.yml. `read_image` maps `IMAGE_DIMENSION_TOO_LARGE` and `IMAGE_TOO_MANY_PIXELS` to model-facing errors that name the resolved path and the limit and tell the model to downscale and retry — the turn continues as a recoverable tool error. The Web composer surfaces `IMAGE_DIMENSION_TOO_LARGE` with dedicated copy naming the limit. The `read-image-dimension` snapshot scenario replays the refusal keylessly through the assembled app: a 2001x1 workspace fixture, a recoverable tool error, and a completed turn. - -## Alternatives considered - -- **Downscale at admission instead of refusing.** Resampling changes the stored bytes away from what the caller supplied, adds a resampling-quality policy, and hides the limit from the model. Refusal keeps admission a pure gate; the model or user can downscale with full knowledge. Worth revisiting only if refusals prove frequent in practice. -- **Enforce at the provider adapter per route.** Too late: by the time a request is assembled the image is already durable history, so every route and every retry re-fails. Admission is the last point where a provider-rejected image can be kept out. -- **Repair already-poisoned sessions** (drop or replace the oversized block on later requests). Out of scope for this fix; admission prevents new poisonings, and history rewriting needs its own design against the model-visible ⟺ logged invariant. - -## Related - -- [Minimal read_image tool](../feature/2026-08-10-minimal-read-image-tool.md) — the tool whose admission gap this closes. -- [Web image intake and limits alignment](../feature/2026-08-12-web-image-intake-and-limits-alignment.md) — the composer-side surfacing of the same `ImageAttachmentLimits`. - -## Consequences - -- One oversized `read_image` can no longer break a session; the model sees an actionable error and the turn completes. -- Images with a side above 2000px are refused even in compositions whose routes would accept them on small requests; such deployments must raise `maxImageDimension` explicitly. -- Sessions that already carry an oversized image remain broken; this change does not repair existing history. diff --git a/.agents/notes/implemented/bug-fix/2026-08-17-image-dimension-admission-limit.zh.md b/.agents/notes/implemented/bug-fix/2026-08-17-image-dimension-admission-limit.zh.md deleted file mode 100644 index 38422615aa..0000000000 --- a/.agents/notes/implemented/bug-fix/2026-08-17-image-dimension-admission-limit.zh.md +++ /dev/null @@ -1,30 +0,0 @@ -# Agent Note: 图片单边尺寸准入上限 - -Status: implemented - -[English](2026-08-17-image-dimension-admission-limit.md) | 中文 - -## Problem - -`read_image` 在字节数与总像素之外没有任何尺寸检查,就把图片持久提交并追加进会话历史。已部署的模型路由在请求携带多张图片且其中任何一张单边超过 2000px 时会以 HTTP 400 拒绝整个请求。已接纳的图片会随该会话之后的每次请求发送,因此一次超限读取就毒化了持久历史:下一次模型请求失败,之后的每次重试同样失败,会话被永久杀死。其他图片来源(宿主上传、MCP 工具图片)存在同样的缺口,因为准入完全没有单边上限。 - -## Decision - -`ImageAttachmentLimits` 增加 `maxImageDimension`,在准入完整解码(`detectImage`)中以 `IMAGE_DIMENSION_TOO_LARGE` 强制执行,因此所有经附件服务提交的来源都会在任何内容进入持久历史之前拒绝超限图片。`LocalAttachmentStore` 将其暴露为 `maxImageDimension` 配置项,默认值 `DEFAULT_MAX_IMAGE_DIMENSION = 2000`,即已部署路由强制执行的最严格单边上限;路由更宽松的部署可在 cordis.yml 中调高。`read_image` 把 `IMAGE_DIMENSION_TOO_LARGE` 与 `IMAGE_TOO_MANY_PIXELS` 映射为面向模型的错误,指明解析后的路径与上限并提示缩图重试,本轮以可恢复的工具错误继续。Web 输入框对 `IMAGE_DIMENSION_TOO_LARGE` 给出指明上限的专用文案。`read-image-dimension` 快照场景通过组装后的应用无 key 回放这次拒绝:2001x1 的工作区 fixture、一条可恢复的工具错误、一个正常完成的轮次。 - -## Alternatives considered - -- **准入时缩图而非拒绝。** 重采样会让存储字节偏离调用方提供的内容,引入重采样质量策略,还会对模型隐藏上限。拒绝让准入保持为纯粹的门禁;模型或用户可以在知情的前提下自行缩图。只有当拒绝在实践中频繁出现时才值得重新考虑。 -- **在 provider 适配器按路由强制执行。** 为时已晚:组装请求时图片已是持久历史,每条路由、每次重试都会再次失败。准入是把必然被上游拒绝的图片挡在外面的最后一道关口。 -- **修复已被毒化的会话**(在之后的请求中丢弃或替换超限图片块)。不在本次修复范围内;准入阻止新的毒化,而重写历史需要针对「模型可见 ⟺ 已记录」不变量单独设计。 - -## Related - -- [最小 read_image 工具](../feature/2026-08-10-minimal-read-image-tool.zh.md),本次修复补上的正是该工具的准入缺口。 -- [Web 图片摄入与限制对齐](../feature/2026-08-12-web-image-intake-and-limits-alignment.zh.md),同一组 `ImageAttachmentLimits` 在输入框侧的呈现。 - -## Consequences - -- 一次超限的 `read_image` 不再能弄坏会话;模型看到可操作的错误,轮次正常完成。 -- 单边超过 2000px 的图片即使在其路由本可接受(小请求)的组合中也会被拒绝;这类部署必须显式调高 `maxImageDimension`。 -- 已经携带超限图片的会话仍然是坏的;本次改动不修复既有历史。 diff --git a/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.i18n.yaml deleted file mode 100644 index 8e355b809e..0000000000 --- a/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.i18n.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each -# side as of the last confirmed-consistent state. Both languages carry equal authority; -# after editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write .agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.md -2026-08-18-request-image-payload-bound.md: 0ec4594888db6157fb8cfd3e7bdb231b842d53c1 -2026-08-18-request-image-payload-bound.zh.md: 7cdf6bb768251cb792b6d590fafa094646e77ada diff --git a/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.md b/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.md deleted file mode 100644 index 0ec4594888..0000000000 --- a/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.md +++ /dev/null @@ -1,36 +0,0 @@ -# Agent Note: Request-level image payload bound - -Status: implemented - -English | [中文](2026-08-18-request-image-payload-bound.zh.md) - -## Problem - -Every image in session history is base64-inlined into every model request by the pi-ai adapter, so a long session's request body grows monotonically with each admitted image. Gateways cap request-body size; once the accumulated payload crossed such a cap the request was rejected with 413 (`Failed to buffer the request body: length limit exceeded`), and because nothing bounds or trims the assembled request, every retry resent the same oversized body. The session was permanently unusable, and the failure text matched no `classifyPiAiError` rule, so it surfaced as the generic `PI_AI_ERROR`. Admission bounds (per image, per message) cannot prevent this: each image is individually admissible, and the sum still grows without bound. Two screenshots were enough to trigger it in production. - -## Decision - -The pi-ai provider profile and direct DeepSeek adapter carry `maxRequestImageBytes` (default `DEFAULT_MAX_REQUEST_IMAGE_BYTES = 20MiB`, a positive integer, changeable from cordis.yml and settings). The provider-neutral `offloadRequestImages` conversion sums the base64 length of every image in history from `ImageAttachmentRef.bytes` without reading data and, while the sum exceeds the bound, replaces the oldest image occurrences with a fixed model-facing placeholder. The placeholder tells the model to read the file again when a path is available or ask the user to attach the image again. The most recent images are omitted last; an image larger than the bound is itself omitted. Occurrence-order replacement does not depend on object identity, so replaying the same JSON log produces the same request. Offloaded images are never read from the attachment store. Both adapters classify 413 as `INVALID_REQUEST`; pi-ai also recognizes specific request-body-cap wording. Four images admitted at the attachment store's 3.5MiB raw-image default occupy at most 18.67MiB after base64 expansion. The 20MiB default therefore retains four such images and leaves headroom under the direct API's 30MiB request limit, while deployments behind stricter gateways lower the value per route. - -## Offload is conversion, not history - -The placeholder is model-visible but not logged as a session event. It stays within the model-visible ⟺ logged invariant the same way the adapter's other serialization does (`(no output)` fallbacks, text-only folding): the offload locations are a pure function of the logged history and the route configuration, so the exact request remains reconstructable from the session log plus the composition. A logged elision event becomes necessary only when offload decisions gain non-deterministic inputs (for example live gateway feedback), which belongs to the deferred capability-metadata design. - -## Alternatives considered - -- **Fail the request with a clear error instead of offloading.** Keeps the model informed but leaves the session wedged: the user cannot remove images from durable history, so a hard failure at the bound is permanent. Offload keeps the session serviceable, which is the point of the fix. -- **Upload images once and reference them by URL / file id.** Removes the linear body growth entirely and is the right medium-term shape (providers and the internal gateway both document a Files path), but it introduces upload lifecycle management across providers and is far beyond a P0 hotfix. -- **Count the full request body, not only images.** Text and tools contribute little and their sizes are only known after full serialization per protocol; bounding the dominant term with explicit headroom is accurate enough for the failure being fixed and much simpler. Revisit inside the route-capability design. -- **Trim at admission instead.** Admission cannot see future accumulation; only the assembled request knows its total. Admission-side bounds (per-side dimension, bytes) remain as the first layer and are owned by [the dimension-limit note](2026-08-17-image-dimension-admission-limit.md). - -## Related - -- [Per-side image dimension admission limit](2026-08-17-image-dimension-admission-limit.md) — the admission-layer companion fix; together they close the two observed session-poisoning failures (400 dimension, 413 body size). -- [Direct DeepSeek vision input](../feature/2026-08-19-direct-deepseek-vision-input.md) — applies this provider-neutral conversion to the official multimodal route. - -## Consequences - -- An image-heavy long session keeps completing requests. The oldest images are omitted first; the most recent image is omitted only when it cannot fit within the bound. -- Crossing the bound rewrites an early message, so the provider prompt-cache prefix ends at the newly offloaded image until the offloaded prefix stabilizes. -- The bound counts base64 image payload only; deployments must keep it below their gateway's request-body cap with headroom, and the shipped default cannot know a private gateway's cap. -- Route capability metadata driving admission and assembly together (image count, per-image size, request size, provider token formulas) remains deferred design work tracked outside this fix. diff --git a/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.zh.md b/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.zh.md deleted file mode 100644 index 7cdf6bb768..0000000000 --- a/.agents/notes/implemented/bug-fix/2026-08-18-request-image-payload-bound.zh.md +++ /dev/null @@ -1,36 +0,0 @@ -# Agent Note: 请求级图片载荷上限 - -Status: implemented - -[English](2026-08-18-request-image-payload-bound.md) | 中文 - -## Problem - -pi-ai 适配器把会话历史中的每张图片 base64 内联进每一个模型请求,长会话的请求体随每张入库图片单调增长。网关对请求体大小设有上限;累积载荷一旦越线,请求被以 413 拒绝(`Failed to buffer the request body: length limit exceeded`),而组装层没有任何约束或裁剪,每次重试都会原样重发同一个超限请求体,会话永久不可用。该报错文本不匹配 `classifyPiAiError` 的任何规则,只能落进笼统的 `PI_AI_ERROR`。准入上限(单图、单消息)无法阻止这一点:每张图片单独看都合规,总和仍然无界增长。线上两张截图即可触发。 - -## Decision - -pi-ai provider profile 与直接 DeepSeek 适配器都提供 `maxRequestImageBytes`(默认 `DEFAULT_MAX_REQUEST_IMAGE_BYTES = 20MiB`,正整数,可从 cordis.yml 与 settings 修改)。提供方无关的 `offloadRequestImages` 转换由 `ImageAttachmentRef.bytes` 推算每张历史图片的 base64 长度(无需读取数据)求和,总和超过上限时从最老的图片出现位置开始替换为一段固定的模型可见占位文本。占位文本要求模型在有路径时重新读取文件,否则请用户重新附上图片。越新的图片越晚被省略;单张图片本身超过上限时也会被省略。按出现顺序替换不依赖对象身份,因此重放同一份 JSON 日志会产生相同请求。被 offload 的图片不会从附件存储读取。两个适配器都把 413 归类为 `INVALID_REQUEST`;pi-ai 还会识别明确的请求体上限措辞。四张按附件存储默认上限准入的 3.5MiB 原始图片,经 base64 膨胀后最多占 18.67MiB。20MiB 默认上限因此可保留四张这样的图片,并在直接 API 的 30MiB 请求上限下留出余量;网关更严格的部署则按路由调低该值。 - -## offload 是转换而非历史 - -占位文本模型可见,但不记录为会话事件。它与适配器的其他序列化(`(no output)` 回退、纯文本折叠)以同样的方式满足「模型可见 ⟺ 已记录」不变量:offload 位置是已记录历史与路由配置的纯函数,确切请求仍可由会话日志加组合配置重建。只有当 offload 决策引入非确定性输入(例如网关的实时反馈)时才需要记录省略事件,那属于暂缓的能力元数据设计。 - -## Alternatives considered - -- **在上限处直接报错而不 offload。** 模型知情,但会话仍然卡死:用户无法从持久历史中删除图片,越线即永久失败。offload 让会话保持可用,这正是本修复的目标。 -- **图片上传一次、按 URL / file id 引用。** 从结构上消除请求体线性增长,是正确的中期形态(各提供方与内部网关都有 Files 路径),但要跨提供方管理上传生命周期,远超 P0 热修复范围。 -- **统计完整请求体而非只统计图片。** 文本与工具占比很小,且其大小要到按协议完整序列化后才可知;对主导项设上限并留出显式余量,对所修故障足够精确且简单得多。留到路由能力设计中再议。 -- **改在准入侧裁剪。** 准入看不到未来的累积,只有组装后的请求知道自己的总量。准入侧上限(单边尺寸、字节)作为第一层保留,归[尺寸上限笔记](2026-08-17-image-dimension-admission-limit.zh.md)所有。 - -## Related - -- [图片单边尺寸准入上限](2026-08-17-image-dimension-admission-limit.zh.md),准入层的配套修复;两者合起来封住已观测到的两类会话毒化故障(400 尺寸、413 请求体)。 -- [直接 DeepSeek 视觉输入](../feature/2026-08-19-direct-deepseek-vision-input.zh.md)把这项提供方无关转换应用于官方多模态路由。 - -## Consequences - -- 图片较多的长会话持续可用。最老的图片优先省略;仅当最新图片本身无法装进上限时才会省略它。 -- 越过上限会改写较早的一条消息,提供方 prompt cache 前缀在新被 offload 的图片处截止,直到被 offload 的前缀稳定。 -- 上限只统计 base64 图片载荷;部署必须让它低于自家网关的请求体上限并留出余量,发行默认值无法预知私有网关的上限。 -- 由路由能力元数据同时驱动准入与组装(图片数量、单图大小、请求大小、提供方 token 公式)的设计仍为暂缓工作,在本修复之外跟踪。 diff --git a/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.i18n.yaml b/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.i18n.yaml index 55710b6e95..be716462f5 100644 --- a/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.md -2026-07-22-web-multimodal-image-input-and-durable-attachments.md: 3f77ab8d55f8eca821cd12a4591c6239c2ea10f5 -2026-07-22-web-multimodal-image-input-and-durable-attachments.zh.md: c95c5abe664635f3cde3a1fc2d569c9474c69665 +2026-07-22-web-multimodal-image-input-and-durable-attachments.md: 30ac1dcff9e6400a3bcf58f7b8e5237e20bd5c04 +2026-07-22-web-multimodal-image-input-and-durable-attachments.zh.md: 68c370c3dd2234e67717429bed417755ed20305d diff --git a/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.md b/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.md index 3f77ab8d55..30ac1dcff9 100644 --- a/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.md +++ b/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.md @@ -69,7 +69,7 @@ interface ComposerAttachment { This split uses the session provide channel's input hook and actions as the single subscription path for live composer state while keeping non-serializable browser objects out of persisted JSON. Only the plain-text draft mirror uses `localStorage`; attachment identifiers, browser `File` objects, and object URLs remain scoped to the live session input shell. Unsent images therefore do not survive reload or session-scope disposal. A Workspace switch moves a mixed text-and-image draft only when the destination shell accepts the complete image batch; refusal leaves both parts with the source. A native client may stage input in an OS temporary directory, but it must treat that path exactly like the browser object URL: delete it when no longer needed and copy the bytes into the durable store before message acceptance. -The local attachment backend resolves an explicit `dshHome`, then `$DSH_HOME`, then `~/.dsh`. It stores content-addressed objects below `$DSH_HOME/attachments/v1/objects//` with owner-only directory and file permissions. On each process's first save for one home, it creates that home and synchronizes every ancestor entry to the filesystem root; existence is not treated as durability because another process may still be between `mkdir` and parent `fsync`. A temporary file is then written, synchronized, atomically published, and made durable with directory syncs on the publication path (POSIX; Windows relies on filesystem metadata journaling) before the service returns a reference. The content digest is encoded in the opaque `sha256:` identifier. Admission and reads fully decode supported rasters before accepting their format and dimensions, and every read also verifies the digest, byte length, and logged metadata. +The local attachment backend resolves an explicit `dshHome`, then `$DSH_HOME`, then `~/.dsh`. It stores content-addressed objects below `$DSH_HOME/attachments/v1/objects//` with owner-only directory and file permissions. On each process's first save for one home, it creates that home and synchronizes every ancestor entry to the filesystem root; existence is not treated as durability because another process may still be between `mkdir` and parent `fsync`. A temporary file is then written, synchronized, atomically published, and made durable with directory syncs on the publication path (POSIX; Windows relies on filesystem metadata journaling) before the service returns a reference. The content digest is encoded in the opaque `sha256:` identifier. Admission prepares a provider-independent master by applying orientation, removing metadata, converting to 8-bit sRGB/sRGBA, and preserving aspect ratio under independent dimension and byte limits. Reads verify the digest, byte length, and logged metadata. Route-specific deterministic request versions are cached separately; the full policy is recorded in [Unified image masters, request versions, and provider files](2026-08-20-unified-image-request-pipeline.md). The store performs no automatic deletion in version one. Sent user images and model-generated images remain reachable for history, resume, and fork. Reference-aware garbage collection needs a separate design because an age-only rule can delete data still referenced by a durable session. Deployment byte and pixel limits are admission policy on writes; reads verify the digest and recorded metadata without reapplying current admission limits, so lowering policy does not invalidate older history. @@ -114,7 +114,7 @@ type PromptInputPart = } ``` -Base64 crosses a wire boundary once and is discarded after persistence. Each front door validates canonical base64 and declared MIME shape, then calls `AttachmentStore.saveImages()` with the whole decoded batch. The service owns image count, aggregate bytes, individual bytes, fully decoded raster/MIME agreement, intrinsic dimensions, and decoded-pixel count; it validates every batch member before saving any member, so one malformed image cannot strand the batch's valid members as unreferenced objects. Storage commits then run in submission order to bound full-raster decoder memory. If a later storage I/O operation fails, the caller appends no model-visible event and receives no partial references, but an earlier immutable content-addressed object may remain unreferenced; version one leaves cleanup to future reference-aware garbage collection instead of adding destructive rollback to the deduplicated store. Only after every image succeeds does the front door call the agent with normalized text and durable image blocks in wire order. A failure exposes no attachment path or raw bytes. +Base64 crosses a wire boundary once and is discarded after persistence. Each front door validates canonical base64 and declared MIME fields, then calls `AttachmentStore.saveImages()` with the whole decoded batch. The service owns image count, aggregate bytes, individual bytes, fully decoded raster/MIME agreement, intrinsic dimensions, decoded-pixel count, and master preparation. It prepares and verifies every batch member once before publishing any member, so one malformed image cannot create partial references and large images are not decoded and encoded again at commit. Storage commits then run in submission order. If a later storage I/O operation fails, the caller appends no model-visible event and receives no partial references, but an earlier immutable content-addressed object may remain unreferenced under the existing storage rule. Only after every image succeeds does the front door call the agent with normalized text and durable image blocks in wire order. A failure exposes no attachment path or raw bytes. `session.attachment` is a read-only, session-scoped endpoint. The host serves bytes only when a durable event in that session references the requested attachment identifier. The client deduplicates loads by session and attachment identifier while that session is rendered, revokes resolved URLs on rendered-session disposal, and rejects invalidated late loads before allocating an object URL so an unmounted session or disposed service cannot repopulate the cache. @@ -122,15 +122,15 @@ Base64 crosses a wire boundary once and is discarded after persistence. Each fro Model catalog entries gain optional merge-extensible input modality declarations. A missing declaration means unknown; a present list without `image` is an explicit negative capability. -The host is the authoritative preflight boundary. It resolves the session's latest routed provider/model, falling back through agent options to host defaults; if that model explicitly excludes image input, it rejects the prompt before writing any attachment or event, and the client restores the draft. Image-bearing prompt admission and model selection share one per-agent serial boundary, and a dequeued prompt remains pending until its durable message event publishes ([ordering decision](../bug-fix/2026-07-29-atomic-web-image-admission.md)); a steering carrier gates from its enqueue until its `steering/message` event publishes, closing the outbox hop that never enters the queued mirror. Selection rejects a text-only target while an image is pending publication or remains in the session's current derived history. Compaction can remove old images and make a later text-only selection valid; idle without publication releases a claimed queued carrier, while steering retained in the outbox stays gated until publication or discard. `session.updateQueue` edits accept text content only, so a queue edit cannot inject an image past this admission boundary. Unknown capability proceeds to the adapter guard so uncatalogued model identifiers remain usable. The browser rejects unsupported declared image media types before allocating preview URLs, but it does not snapshot deployment limits or model capability: a handshake snapshot cannot represent a session's current target after `session.selectModel`, and deployment policy may change independently. The host validates the complete batch against current byte, count, aggregate, media, dimension, pixel, and routed-model policy before writing any attachment or event; its rejection announces through the composer's transient toast. +The host is the authoritative preflight point. It resolves the session's latest routed provider and model, falling back through agent options to host defaults; if that model explicitly excludes image input, it rejects a new image prompt before writing an attachment or event, and the client restores the draft. Image-bearing prompt admission and model selection share one per-agent serial chain ([ordering decision](../bug-fix/2026-07-29-atomic-web-image-admission.md)), including steering that does not enter the queued UI mirror. This gives a prompt and concurrent selection a deterministic order. Selection itself may target a text-only model after images enter durable history; the shared LLM runtime replaces retained image blocks with deterministic text placeholders for that request. `session.updateQueue` edits accept text content only, so a queue edit cannot inject an image past admission. Unknown capability proceeds to the adapter guard so uncatalogued model identifiers remain usable. The browser rejects unsupported declared image media types before allocating preview URLs, but it does not snapshot deployment limits or model capability. The host validates the complete batch against current byte, count, aggregate, media, dimension, pixel, and routed-model policy before writing an attachment or event; its rejection appears through the composer's transient toast. -Pi-AI and the direct DeepSeek adapter resolve `ctx.attachments` at request time, recursively convert each durable image reference including references nested inside tool results, and emit native image content only for models that declare image input. The direct route advertises `deepseek-v4-flash-vision-exp` as image-capable and accepts configured image-capable catalog entries; its Flash, Pro, custom models without an image declaration, and unlisted pass-through ids remain text-only. Request-time service resolution keeps Cordis load order from freezing optional attachment availability. No adapter may flatten or skip a retained image; unsupported roles and models fail with typed `UNSUPPORTED_CONTENT`. +Pi-AI and the direct DeepSeek adapter resolve `ctx.attachments` at request time, recursively convert each retained image reference including references nested inside tool results, and emit native image content only for models that declare image input. Both adapters request the same deterministic route-specific version from the durable normalized attachment. Pi-AI carries it inline under a base64-aware request budget. The built-in DeepSeek route advertises `deepseek-v4-flash-vision-exp`, uploads every retained version through Files API, and sends `file_id` blocks with indexed reuse, expiry, bounded stale-id retry, quota cleanup, and explicit deletion. DeepSeek text models, custom models without an image declaration, and unlisted pass-through ids remain text-only. Request-time service resolution keeps Cordis load order from freezing optional attachment availability. No adapter may flatten or silently skip a retained image; unsupported roles and models fail with typed `UNSUPPORTED_CONTENT`. Core supports structured assistant image blocks, but no current production provider route is certified for image output. Any future output-capable adapter must retrieve provider bytes under bounded size and time policy, validate them through the same attachment service, persist them, and only then publish the atomic `ImageBlock`. A URL in assistant Markdown remains text and is never downloaded automatically. Provider-neutral token estimation does not guess visual pricing from image dimensions; provider-reported usage remains authoritative. ACP advertises image prompts only when its configured exact route and attachment deployment can accept them, persists inline input before publishing the user event, and re-reads committed assistant image references for native ACP image updates. MCP keeps canonical raw blocks for programmatic callers while projecting admitted images to durable core blocks; Code Mode carries any settled image-bearing sub-result through the outer result as logged source-attributed context. -Compaction replays the selected conversation prefix, including image references, into the configured summarization route. A visual-capable route resolves those references through its adapter; a text-only route fails explicitly instead of silently dropping the visual context. The synthesized checkpoint remains text-only, and `compaction-basic` rejects image summary output with `UNSUPPORTED_CONTENT`. +Compaction replays the selected conversation prefix, including image references, into the configured summarization route. A visual-capable route uses the same deterministic request versions as ordinary turns. A text-only route receives the same deterministic attachment placeholders as any other LLM request. The synthesized checkpoint remains text-only, and `compaction-basic` rejects image summary output with `UNSUPPORTED_CONTENT`. ### History rendering and original preview @@ -140,7 +140,7 @@ Composer thumbnails and each `MessageImage` own ephemeral original-preview state ### Limits and trust boundaries -Version one accepts PNG, JPEG, WebP, and GIF only. SVG and remote URLs are excluded. Default limits are 3.5 MiB per image, 20 images and 100 MiB aggregate image bytes per message, 40 million intrinsic pixels per image, and 2000 pixels on either side. These deployment-varying limits are validated backend configuration and enforced by the host before persistence. The client connection carrier has an independent configurable `maxRequestBodyBytes` cap (160 MiB by default) for every API request and fails load if it cannot hold the attachment service's aggregate image limit after base64 and envelope expansion; lowering image policy therefore never silently lowers the carrier limit for valid text or other RPCs. A body without a declared length is rejected the moment it crosses the cap rather than drained to its end. +Version one accepts PNG, JPEG, WebP, and GIF only. SVG and remote URLs are excluded. Source intake defaults are 32 MiB per image, 20 images and 100 MiB aggregate image bytes per message, 100 million decoded pixels per image, and 16384px on either side. The provider-independent master defaults to a 2048px long edge and 4 MiB safety cap. Provider request pixel and encoded-byte limits are separate route policies. These deployment-varying limits are validated backend configuration and enforced before persistence or request transmission. The client connection carrier has an independent configurable `maxRequestBodyBytes` cap, 160 MiB by default, and fails load if it cannot hold the aggregate source limit after base64 and envelope expansion. A body without a declared length is rejected when it crosses the cap rather than drained to its end. Malformed base64, unsupported or mismatched media, truncated image payloads, excess bytes, excess image count, excess pixels, excess per-side dimensions, missing objects, and integrity mismatches return stable structured failures. Original filenames are reduced to a display basename, control characters are removed, and no local path is logged or returned to the browser. @@ -148,11 +148,11 @@ Malformed base64, unsupported or mismatched media, truncated image payloads, exc | Surface | Responsibility | | --- | --- | -| `packages/attachment/attachment` | Opaque attachment identifier, image reference, limits, failures, and single/batch admission through `ctx.attachments`. | -| `packages/attachment/attachment-local` | Private content-addressed storage, complete raster decoding, integrity verification, and configuration. | -| `packages/llm/llm` | Role-neutral `ImageBlock` and input-modality metadata. | -| `packages/llm/llm-pi-ai` | Resolve durable supported image input into native provider content. | -| `packages/llm/llm-deepseek` | Resolve declared official vision input and reject images for text-only models. | +| `packages/attachment/attachment` | Opaque attachment and request-version identifiers, image references, policies, failures, batch admission, derived reads, and crops through `ctx.attachments`. | +| `packages/attachment/attachment-local` | Private content-addressed masters, deterministic request cache, complete raster decoding, integrity verification, and configuration. | +| `packages/llm/llm` | Role-neutral `ImageBlock`, input-modality metadata, exact adapter generations, and text-only request projection. | +| `packages/llm/llm-pi-ai` | Resolve durable images to deterministic inline request versions. | +| `packages/llm/llm-deepseek` | Resolve official vision input to deterministic request versions and Files API ids. | | `packages/compaction/compaction-basic` | Preserve images in summary input and reject non-text checkpoint output explicitly. | | `packages/host/apiproxy` and `packages/bundle/base` | Narrow upload wire, shared batch admission, limits and routed-model preflight, persist-before-event ordering, session-authorized reads, and default profile composition. | | `packages/client/connection` and `packages/client/runtime` | Bounded request buffering, wire types, fixture images, prompt uploads, attachment reads, and durable-reference folding. | @@ -165,7 +165,7 @@ The attachment packages form the interface/implementation side of one capability ### Implementation -The implemented slice includes the attachment seam and shared batch admission, role-neutral image block, Pi-AI and direct DeepSeek input conversion, durable Web/ACP/MCP ordering, Web upload/read protocol, conditional ACP image wire support, lossless MCP canonical results with durable image projection, generic Code Mode rich-result forwarding, current image-limit enforcement, bounded Web request bodies, in-memory draft images, paste/drop rail, user and assistant history rendering, single-click preview, compaction handling, and keyless assembled Web and ACP coverage. +The implemented capability includes shared prepare-once batch admission, provider-independent masters, deterministic request versions, DeepSeek Files reuse, stable crop handles, role-neutral image blocks, Pi-AI and DeepSeek input conversion, durable Web/ACP/MCP ordering, Web upload/read protocol, conditional ACP image support, lossless MCP results with durable image projection, Code Mode rich-result forwarding, bounded Web requests, draft and historical image UI, compaction handling, and keyless assembled coverage. No compatibility shim is required for the pre-release prompt wire; all call sites and fixtures change with the introducing slice. @@ -210,11 +210,11 @@ Rejected because tool renderers are pure, synchronous, and replayable. MCP prepa ## Testing - Storage tests cover content-addressed deduplication, private permissions, admission failures, corruption/missing-object failures, and reading history after deployment limits are lowered. -- Host and protocol tests cover persist-before-event ordering, absence of base64 in logs, session-scoped authorization, capability rejection, upload limits, bounded HTTP request bodies, image-admission/model-selection races (queued and steering placements), pending publication, idle release without publication, text-only queue edits, and selection against current derived history after compaction. +- Host and protocol tests cover persist-before-event ordering, absence of base64 in logs, session-scoped authorization, capability rejection, upload limits, bounded HTTP request bodies, image-admission/model-selection ordering, text-only queue edits, and text-only request projection. - Client unit tests cover paste and drop, mixed clipboard text, image-only send, draft restoration, ordering, draft/session-scope/application object-URL cleanup, and a deferred historical read that completes after disposal; the keyless assembled built-client lane (`apps/web/tests/image-display.snapshot.ts`, `DSH_EXAMPLE_MODE=lib pnpm run test:snapshot`) covers the historical user and assistant galleries over the authorized attachment route, the original-size lightbox, and the composer paste rail. -- Adapter and compaction tests cover native Pi-AI image conversion, late attachment-service composition, text-only rejection, recursively nested tool-result images, preserved summary input, and explicit image-output rejection. +- Adapter and compaction tests cover deterministic Pi-AI request versions, DeepSeek Files upload and reuse, stale-id recovery, text-only projection, recursively nested tool-result images, shared summary request versions, and explicit image-output rejection. - Attachment, MCP, ACP, and Code Mode tests cover all-member validation before writes, mixed text/image ordering, no inline base64 in durable events, exact route-capability gates, explicit unsupported-content diagnostics, post-execute replacement/block precedence, cancellation during admission, verified assistant-image delivery, and generic nested-image forwarding. A keyless assembled ACP snapshot sends a real inline PNG and pins only its durable reference in the session log. -- A credentialed real-API test sends a PNG through the Anthropic `claude-opus-4-8` route and requires the model to identify its QR code. +- Credentialed real-API tests cover the configured Anthropic route and the built-in `deepseek-official` Files path. The DeepSeek test does not use a custom provider entry. - The current production adapter set has no certified image-output route; output-provider certification remains outside version one. ## Consequences diff --git a/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.zh.md b/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.zh.md index c95c5abe66..68c370c3dd 100644 --- a/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.zh.md +++ b/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.zh.md @@ -69,7 +69,7 @@ interface ComposerAttachment { 这一拆分把会话 provide 通道的输入 hook 与 actions 用作实时输入区状态的唯一订阅路径,同时避免把不可序列化的浏览器对象写进持久 JSON。只有纯文本草稿镜像使用 `localStorage`;附件标识符、浏览器 `File` 对象和对象 URL 都限定在实时会话输入外壳的 scope 内。未发送图片因此无法跨重载或会话 scope 释放保留。切换 Workspace 时,只有目标外壳接受完整图片批次,图文混合草稿才会移动;拒绝时,文本和图片都留在来源外壳。原生客户端可以在操作系统临时目录中暂存输入,但必须像对待浏览器对象 URL 一样对待该路径:不再需要时删除,并在消息被接受前把字节复制进持久存储。 -本地附件后端依次解析显式 `dshHome`、`$DSH_HOME` 和 `~/.dsh`。它把内容寻址对象存储在 `$DSH_HOME/attachments/v1/objects//` 下,并为目录和文件设置仅所有者可访问的权限。每个进程首次为某个 home 保存对象时,都会创建该 home,并逐级同步每个祖先目录项直至文件系统根目录;不能把存在视为持久性,因为另一个进程可能仍处于 `mkdir` 与父目录 `fsync` 之间。随后,服务写入并同步临时文件,再以原子方式发布,并对发布路径执行目录同步使其持久(POSIX;Windows 依赖文件系统元数据日志),之后才返回引用。内容摘要编码在不透明的 `sha256:` 标识符中。写入准入与读取都会完整解码受支持的光栅图片,之后才接受其格式和尺寸;每次读取还会校验摘要、字节长度和已记录的元数据。 +本地附件后端依次解析显式 `dshHome`、`$DSH_HOME` 和 `~/.dsh`。它把内容寻址对象存储在 `$DSH_HOME/attachments/v1/objects//` 下,并为目录和文件设置仅所有者可访问的权限。每个进程首次为某个 home 保存对象时,都会创建该 home,并逐级同步每个祖先目录项直至文件系统根目录;不能把存在视为持久性,因为另一个进程可能仍处于 `mkdir` 与父目录 `fsync` 之间。随后,服务写入并同步临时文件,再以原子方式发布,并对发布路径执行目录同步使其持久(POSIX;Windows 依赖文件系统元数据日志),之后才返回引用。内容摘要编码在不透明的 `sha256:` 标识符中。准入会应用方向、删除元数据、转换为 8-bit sRGB/sRGBA,并在独立尺寸和字节上限内保持宽高比,生成与提供方无关的主版本。读取会校验摘要、字节长度和已记录元数据。路由专用的确定性请求版本单独缓存,完整策略见[统一图片主版本、请求版本和提供方文件](2026-08-20-unified-image-request-pipeline.md)。 第一版不对存储执行自动删除。已发送的用户图片和模型生成图片会一直保留,以供历史记录、恢复和 fork 使用。按引用感知的垃圾回收需要单独设计,因为仅按时间清理可能删除仍被持久会话引用的数据。部署的字节和像素限制是写入时的准入策略;读取时会校验摘要和已记录的元数据,但不重新应用当前准入限制,因此收紧策略不会导致旧历史记录失效。 @@ -114,7 +114,7 @@ type PromptInputPart = } ``` -Base64 只跨越一次协议边界,并在持久化后丢弃。每个入口都会校验规范 base64 与声明的 MIME 形状,再用完整解码批次调用 `AttachmentStore.saveImages()`。服务负责图片数量、总字节数、单张图片字节数、声明 MIME 与完整解码后的光栅图片是否一致、固有尺寸和解码像素数;它会在保存任何成员之前校验每个批次成员,因此一张畸形图片不会把批次中的有效成员留成无引用对象。随后按提交顺序执行存储提交,以限制完整光栅解码器的内存占用。如果后续存储 I/O 操作失败,调用方不会追加模型可见事件,也不会收到部分引用,但先前的不可变内容寻址对象可能保持无引用状态;第一版将清理留给未来按引用感知的垃圾回收,而不向去重存储添加破坏性回滚。只有每张图片都成功后,入口才会用规范化文本和按协议顺序排列的持久图片块调用 agent。失败时不公开任何附件路径或原始字节。 +Base64 只跨越一次协议边界,并在持久化后丢弃。每个入口都会校验规范 base64 与声明的 MIME 字段,再用完整解码批次调用 `AttachmentStore.saveImages()`。服务负责图片数量、总字节数、单张图片字节数、声明 MIME 与完整解码后的光栅图片是否一致、固有尺寸、解码像素数和主版本准备。它会在发布任何成员之前只准备并验证每个批次成员一次,因此一张畸形图片不会产生部分引用,大图也不会在提交时重复解码和编码。随后按顺序提交存储。如果后续存储 I/O 操作失败,调用方不会追加模型可见事件,也不会收到部分引用,但先前的不可变内容寻址对象可能按现有存储规则保持无引用状态。只有每张图片都成功后,入口才会用规范化文本和按协议顺序排列的持久图片块调用 agent。失败时不公开任何附件路径或原始字节。 `session.attachment` 是只读且限定于会话作用域的端点。只有该会话中的持久事件引用了所请求的附件标识符,宿主才提供字节。会话处于渲染状态时,客户端会按会话和附件标识符对加载操作去重;已渲染会话释放时会撤销已解析的 URL,并在分配对象 URL 前拒绝已失效的延迟加载,以免已卸载的会话或已释放的服务重新写入缓存。 @@ -122,15 +122,15 @@ Base64 只跨越一次协议边界,并在持久化后丢弃。每个入口都 模型目录项增加可选且可合并扩展的输入模态声明。缺少声明表示未知;声明存在但不含 `image`,则明确表示不支持图片。 -宿主是权威的前置检查边界。它会解析会话最新路由到的提供方和模型,并在缺失时依次回退到 agent 选项和宿主默认值;如果该模型明确排除图片输入,宿主会在写入任何附件或事件前拒绝提示词,客户端则恢复草稿。包含图片的提示词准入与模型选择共用一个逐 agent 的串行边界,而且已经出队的提示词在其持久消息事件发布前仍保持待发布状态([顺序决策](../bug-fix/2026-07-29-atomic-web-image-admission.zh.md));steering 载体则从入队起就参与门槛,直到其 `steering/message` 事件发布为止,堵住了从不进入排队镜像的 outbox 窗口。当图片正等待发布或仍存在于会话当前的派生历史中时,模型选择会拒绝纯文本目标。压缩(compaction)可以移除旧图片,使之后选择纯文本目标变得有效;未发布任何事件即转入空闲时,已认领的 queued 载体会被释放,而保留在 outbox 中的 steering 在发布或丢弃前始终受门槛约束。`session.updateQueue` 的编辑只接受文本内容,因此队列编辑无法绕过该准入边界注入图片。能力未知时继续进入适配器强制检查,使未收录的模型标识符仍然可用。浏览器会在分配预览 URL 前拒绝声明不支持的图片媒体类型,但不会为部署限制或模型能力保留快照:握手快照无法表达 `session.selectModel` 之后会话的当前目标,部署策略也可能独立变化。宿主会根据当前的单张字节数、图片数量、总字节数、媒体类型、尺寸、像素数和路由模型策略校验整个批次,再写入任何附件或事件;其拒绝通过 composer 的短时 toast 播报。 +宿主是权威的前置检查点。它会解析会话最新路由到的提供方和模型,并在缺失时依次回退到 agent 选项和宿主默认值;如果模型明确排除图片输入,宿主会在写入附件或事件前拒绝新的图片提示词,客户端则恢复草稿。包含图片的提示词准入与模型选择共用一条逐 agent 串行链([顺序决策](../bug-fix/2026-07-29-atomic-web-image-admission.md)),也包括不进入排队 UI 镜像的 steering。这会为提示词和并发选择提供确定顺序。图片进入持久历史后仍可选择纯文本模型;共享 LLM 运行时会在该请求中把保留的图片块替换为确定的文本占位符。`session.updateQueue` 只接受文本内容,因此队列编辑无法绕过准入注入图片。能力未知时继续进入适配器强制检查,使未收录的模型标识符仍然可用。浏览器会在分配预览 URL 前拒绝声明不支持的图片媒体类型,但不会为部署限制或模型能力保留快照。宿主会根据当前的单张字节数、图片数量、总字节数、媒体类型、尺寸、像素数和路由模型策略校验整个批次,再写入附件或事件;拒绝会通过 composer 的短时 toast 显示。 -Pi-AI 与直接 DeepSeek 适配器都会在请求时解析 `ctx.attachments`,递归转换每个持久图片引用,包括嵌套在工具结果中的引用,并且仅为声明支持图片输入的模型生成提供方原生图片内容。直接路由会将 `deepseek-v4-flash-vision-exp` 公布为支持图片,并接受已配置且支持图片的 catalog 配置项;其 Flash、Pro、未声明图片能力的自定义模型和未列出原样传递 id 仍仅支持文本。在请求时解析服务,可避免 Cordis 加载顺序将可选附件服务的可用性固化。任何适配器都不得将保留的图片展平或跳过;不支持的角色与模型会以类型化的 `UNSUPPORTED_CONTENT` 失败。 +Pi-AI 与直接 DeepSeek 适配器都会在请求时解析 `ctx.attachments`,递归转换每个保留的图片引用,包括嵌套在工具结果中的引用,并且仅为声明支持图片输入的模型生成提供方原生图片内容。两个适配器都从持久主版本请求同一个确定性路由版本。Pi-AI 在考虑 base64 扩张的请求预算内内联携带它。内置 DeepSeek 路由公布 `deepseek-v4-flash-vision-exp`,把每个保留的版本上传到 Files API,并通过索引复用、过期处理、有界陈旧 ID 重试、配额清理和显式删除发送 `file_id` 块。DeepSeek 纯文本模型、未声明图片能力的自定义模型和未列出的透传 ID 保持纯文本。在请求时解析服务,可避免 Cordis 加载顺序将可选附件服务的可用性固化。适配器不得展平或静默跳过保留图片;不支持的角色与模型会以类型化的 `UNSUPPORTED_CONTENT` 失败。 核心层支持结构化助手图片块,但当前没有任何生产提供方路径通过图片输出认证。未来任何支持输出的适配器都必须在有界的大小和时间策略下获取提供方字节,通过同一个附件服务校验并持久化字节,之后才能以原子方式发布 `ImageBlock`。助手 Markdown 中的 URL 仍是文本,绝不自动下载。 提供方无关的 token 估算不会根据图片尺寸猜测视觉定价;提供方返回的用量仍是权威值。只有配置的确切路由与附件部署可以接受图片时,ACP(Agent Client Protocol)才公布图片提示词能力;它会在发布用户事件前持久化内联输入,并重新读取已提交的助手图片引用来发送原生 ACP 图片更新。MCP 为程序化调用方保留规范原始块,同时把已准入图片投影为持久核心块;Code Mode 会把任何已经结算且含图片的子结果经外层结果转运为带来源归属且写入日志的上下文。 -压缩会把选定的会话前缀(包含图片引用)回放到已配置的摘要生成路径中。支持视觉的路径会通过适配器解析这些引用;仅文本路径会明确失败,而不是静默丢弃视觉上下文。合成的检查点仍仅包含文本,`compaction-basic` 会以 `UNSUPPORTED_CONTENT` 拒绝包含图片的摘要输出。 +压缩会把选定的会话前缀和其中的图片引用回放到已配置的摘要生成路径。支持视觉的路径使用与普通轮次相同的确定性请求版本。纯文本路径接收与其他 LLM 请求相同的确定性附件占位符。合成的检查点仍仅包含文本,`compaction-basic` 会以 `UNSUPPORTED_CONTENT` 拒绝包含图片的摘要输出。 ### 历史渲染与原图预览 @@ -140,7 +140,7 @@ Pi-AI 与直接 DeepSeek 适配器都会在请求时解析 `ctx.attachments`, ### 限制与信任边界 -第一版仅接受 PNG、JPEG、WebP 和 GIF。不接受 SVG 和远程 URL。默认限制为每张图片 3.5 MiB、每条消息 20 张图片和 100 MiB 图片总字节数、每张图片 4,000 万个固有像素,以及任一边 2,000 像素。这些随部署变化的限制属于经过校验的后端配置,并由宿主在持久化前强制执行。客户端连接载体为每个 API 请求设置独立且可配置的 `maxRequestBodyBytes` 上限(默认 160 MiB);如果该上限无法容纳附件服务的图片总量限制经 base64 和请求封装膨胀后的大小,加载就会失败。因此,降低图片策略绝不会静默降低有效文本或其他 RPC 的载体上限。未声明长度的请求体在越过上限的瞬间即被拒绝,而不是先读完再拒。 +第一版仅接受 PNG、JPEG、WebP 和 GIF。不接受 SVG 和远程 URL。源文件输入默认限制为每张图片 32 MiB、每条消息 20 张图片和 100 MiB 图片总字节数、每张图片一亿解码像素,以及任一边 16384px。与提供方无关的主版本默认长边 2048px,独立安全上限 4 MiB。提供方请求的像素和编码字节上限是单独的路由策略。这些随部署变化的限制属于经过校验的后端配置,并在持久化或请求发送前强制执行。客户端连接载体为每个 API 请求设置独立且可配置的 `maxRequestBodyBytes` 上限,默认 160 MiB;如果该上限无法容纳源文件总量限制经 base64 和请求封装膨胀后的大小,加载就会失败。未声明长度的请求体在越过上限时即被拒绝,而不是先读完再拒。 格式错误的 base64、不支持或不匹配的媒体、截断的图片数据、超出字节限制、超出图片数量、超出像素限制、超出单边尺寸限制、对象缺失和完整性不匹配都会返回稳定的结构化错误。原始文件名只保留用于显示的末段,控制字符会被移除,并且任何本地路径都不会写入日志或返回浏览器。 @@ -148,11 +148,11 @@ Pi-AI 与直接 DeepSeek 适配器都会在请求时解析 `ctx.attachments`, | 接口 | 职责 | | --- | --- | -| `packages/attachment/attachment` | 不透明附件标识符、图片引用、限制、错误,以及通过 `ctx.attachments` 提供的单张/批量准入。 | -| `packages/attachment/attachment-local` | 私有内容寻址存储、完整光栅解码、完整性校验和配置。 | -| `packages/llm/llm` | 角色无关的 `ImageBlock` 和输入模态元数据。 | -| `packages/llm/llm-pi-ai` | 将持久且受支持的图片输入解析为提供方原生内容。 | -| `packages/llm/llm-deepseek` | 解析已声明的官方视觉输入,并拒绝纯文本模型的图片。 | +| `packages/attachment/attachment` | 不透明附件和请求版本标识符、图片引用、策略、错误,以及通过 `ctx.attachments` 提供的批量准入、派生读取和裁剪。 | +| `packages/attachment/attachment-local` | 私有内容寻址主版本、确定性请求缓存、完整光栅解码、完整性校验和配置。 | +| `packages/llm/llm` | 角色无关的 `ImageBlock`、输入模态元数据、精确适配器代次和纯文本请求投影。 | +| `packages/llm/llm-pi-ai` | 把持久图片解析为确定性内联请求版本。 | +| `packages/llm/llm-deepseek` | 把官方视觉输入解析为确定性请求版本和 Files API ID。 | | `packages/compaction/compaction-basic` | 在摘要输入中保留图片,并明确拒绝非文本检查点输出。 | | `packages/host/apiproxy` 和 `packages/bundle/base` | 范围狭窄的上传协议、共享批量准入、限制和路由模型前置检查、先持久化再追加事件的顺序、会话授权读取,以及默认 profile 组合。 | | `packages/client/connection` 和 `packages/client/runtime` | 有界请求缓冲、协议类型、fixture(测试前置数据)图片、提示词上传、附件读取和持久引用折叠。 | @@ -165,7 +165,7 @@ Pi-AI 与直接 DeepSeek 适配器都会在请求时解析 `ctx.attachments`, ### 实现 -已实现的范围包括附件服务边界与共享批量准入、角色无关的图片块、Pi-AI 与直接 DeepSeek 输入转换、Web/ACP/MCP 的持久化顺序、Web 上传与读取协议、条件式 ACP 图片协议支持、无损 MCP 规范结果与持久图片投影、通用 Code Mode 丰富结果转发、当前图片限制执行、大小受限的 Web 请求体、内存草稿图片、粘贴与拖放附件栏、用户与助手历史图片渲染、单击预览、压缩处理,以及组装后无需密钥的 Web 与 ACP 覆盖。 +已实现能力包括只准备一次的共享批量准入、与提供方无关的主版本、确定性请求版本、DeepSeek Files 复用、稳定裁剪句柄、角色无关图片块、Pi-AI 和 DeepSeek 输入转换、Web/ACP/MCP 持久化顺序、Web 上传与读取协议、条件式 ACP 图片支持、带持久图片投影的无损 MCP 结果、Code Mode 丰富结果转发、有界 Web 请求、草稿与历史图片 UI、压缩处理,以及组装后的无密钥覆盖。 预发布提示词协议不需要兼容包装层;引入相应切片时会同时修改所有调用点和 fixture。 @@ -210,11 +210,11 @@ UI 状态可能陈旧,也无法保护直接 SDK、ACP、回放或未收录模 ## 测试 - 存储测试覆盖内容寻址去重、私有权限、准入失败、对象损坏或缺失时的失败,以及收紧部署限制后读取历史数据。 -- 宿主与协议测试覆盖先持久化再追加事件的顺序、日志中不含 base64、会话作用域授权、能力拒绝、上传限制、大小受限的 HTTP 请求体、图片准入与模型选择的竞态(排队与 steering 两种放置)、待发布状态、未发布即空闲时的门槛释放、仅文本的队列编辑,以及压缩后依据当前派生历史进行的选择。 +- 宿主与协议测试覆盖先持久化再追加事件的顺序、日志中不含 base64、会话作用域授权、能力拒绝、上传限制、大小受限的 HTTP 请求体、图片准入与模型选择的排序、仅文本的队列编辑,以及纯文本请求投影。 - 客户端单元测试覆盖粘贴与拖放、混合剪贴板文本、仅图片发送、草稿恢复、顺序、草稿、会话作用域和应用层级的对象 URL 清理,以及一项在释放后才完成的延迟历史读取;keyless 的组装后构建产物通道(`apps/web/tests/image-display.snapshot.ts`,`DSH_EXAMPLE_MODE=lib pnpm run test:snapshot`)覆盖经授权附件路由渲染的历史用户与助手图片画廊、原图 lightbox,以及 composer 粘贴缩略图条。 -- 适配器与压缩测试覆盖 Pi-AI 原生图片转换、后置附件服务组合、仅文本拒绝、递归嵌套在工具结果中的图片、保留摘要输入,以及明确拒绝图片输出。 +- 适配器与压缩测试覆盖确定性 Pi-AI 请求版本、DeepSeek Files 上传与复用、陈旧 ID 恢复、纯文本投影、递归嵌套在工具结果中的图片、共享摘要请求版本,以及明确拒绝图片输出。 - 附件、MCP、ACP 与 Code Mode 测试覆盖写入前校验全部成员、图文混合顺序、持久事件不含内联 base64、确切路由能力门禁、明确的不支持内容诊断、post-execute 替换/阻止优先级、准入期间取消、经过校验的助手图片交付,以及通用嵌套图片转发。组装后的无密钥 ACP 快照发送真实内联 PNG,并在会话日志中只固定其持久引用。 -- 需要凭据的实际 API 测试会通过 Anthropic `claude-opus-4-8` 路径发送一张 PNG,并要求模型识别其中的二维码。 +- 需要凭据的实际 API 测试会覆盖配置的 Anthropic 路由和内置 `deepseek-official` Files 路径。DeepSeek 测试不使用自定义提供方条目。 - 当前生产适配器集合没有经过认证的图片输出路由;输出提供方认证仍不在第一版范围内。 ## 后果 diff --git a/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.i18n.yaml b/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.i18n.yaml index dcd01fc6d3..6c37530274 100644 --- a/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.md -2026-08-10-minimal-read-image-tool.md: a43e53d70e98bac7a50aa6bbabbb1e177237df01 -2026-08-10-minimal-read-image-tool.zh.md: a94e4b296425ad50876b0b45a689442c896a85a1 +2026-08-10-minimal-read-image-tool.md: 0c0c6a95fa3d8be1dbe895ecd83ff44e1e1eac17 +2026-08-10-minimal-read-image-tool.zh.md: c3c2fe1095637a19c3ebaa21cf23a501fe83c480 diff --git a/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.md b/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.md index a43e53d70e..0c0c6a95fa 100644 --- a/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.md +++ b/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.md @@ -6,28 +6,29 @@ English | [中文](2026-08-10-minimal-read-image-tool.zh.md) ## Problem -The multimodal attachment work gave user uploads a complete durable path — bytes committed to the content-addressed attachment store before the owning `user/message`, an `ImageBlock` carrying only the `sha256:` reference, and the pi-ai route re-reading verified bytes per request — but the model itself had no way to look at an image on disk. `read` rejects binary content by contract, so an agent asked about a screenshot or a rendered chart either failed or shelled out to lossy workarounds. A first standalone attempt (PR #598) solved this together with loop-level route scoping: an `agent/request-ready` extension point publishing exact-model modalities before assembly, per-route schema/guidance visibility, and a reversible `image-placeholder-v1` history projection so text routes could continue over placeholder text. That design worked but coupled a tool to new agent-loop machinery, three new session-log concepts, and per-step registration churn — far more surface than the capability needs. +The multimodal attachment work gave user uploads a complete durable path, but the model itself had no way to inspect an image on disk or crop a durable user upload that had no path. `read` rejects binary content by contract, so an agent asked about a screenshot or rendered chart either failed or used a lossy workaround. A standalone attempt in PR #598 combined the tool with loop-level route scoping, per-route schema visibility, and new session-log concepts. Those features were not required to publish a logged image tool result. ## Decision -Ship the smallest tool that loads an image into the next request's context, entirely over existing seams; the withdrawn PR #598 design is the explicit counter-example this note records. +Both image-reading operations live in `dsh-tool-fs` and publish ordinary logged tool results over existing extension points. -- **`read_image` lives in `dsh-tool-fs`** beside `read`/`write`/`edit`. Extension selects the declared PNG/JPEG/WebP/GIF media type; the attachment store's magic-byte and pixel validation stays authoritative. Bytes travel `ctx.fs.stat` → bounded `ctx.fs.readBytes` → `ctx.attachments.saveImage` → `fs/observed`, and the tool result is the metadata envelope plus a real `ImageBlock` — `ToolResultBlock.content` already admits image blocks, the pi-ai adapter already renders them, and the Web host's model-switch guard already scans tool results, so nothing downstream changes. +- **`read_image` reads a filesystem path.** Extension selects the declared PNG/JPEG/WebP/GIF media type; the attachment store's magic-byte and pixel validation stays authoritative. Bytes travel `ctx.fs.stat` → bounded `ctx.fs.readBytes` → `ctx.attachments.saveImage` → `fs/observed`. The tool result contains metadata and an `ImageBlock`. +- **`read_image_region` crops a durable session attachment.** The request names the complete attachment id, current preview dimensions, and a preview-coordinate rectangle. The tool authorizes the id against images already referenced by the calling session, maps the rectangle to the durable master, crops that master, and persists the result as a new attachment. Its result contains the cropped `ImageBlock`, so the model-visible crop is reconstructable from the log. This is the path for pasted or dragged images that have no filesystem location. - **`FileSystem.readBytes(target, signal, maxBytes)`** is a new required provider primitive: the byte bound lives at the seam so no backend can buffer an unbounded file, with the stat-size short-circuit and a one-byte-past-cap stream guard against post-stat growth (`FS_TOO_LARGE`). -- **Registration is composition-conditional, execution is route-gated.** The tool registers only under `ctx.inject(['attachments'], …)` — no store, no tool. At execution, before any I/O, the strict gate resolves the calling route (latest `request/header` config, falling back to agent options) through `ctx.llm.resolveModelInfo` and requires `image` in `inputModalities`; unknown capability refuses. A refusal is a plain `isError` result, so a text route's durable history never acquires an image block and the session cannot brick its own route. +- **Registration is composition-conditional, execution is route-gated.** The tools register only under `ctx.inject(['attachments'], …)`. Before I/O, the strict gate resolves the calling route through `ctx.llm.resolveModelInfo` and requires `image` in `inputModalities`; unknown capability refuses. A text-only route can still consume prior durable images because the shared LLM runtime projects them to placeholders at request assembly. - **Code Mode forwards the image out-of-band**: a nested dispatch returns the canonical value (execution-local, no image block) and defers a `user`-role context message carrying the envelope and image, so the picture still reaches the next request. -- **llm-replay models may declare `inputModalities`**, which is what lets the two keyless ACP snapshots pin both sides of the gate — the sha256-referenced success on an image-capable replay route and the verbatim refusal on a text-only one. +- **llm-replay models may declare `inputModalities`**, which lets keyless ACP snapshots cover the image-capable result and the text-only refusal. ## Alternatives considered -- **PR #598's route-scoped design** (request-ready seam, per-route schema/guidance visibility, reversible history projection) — withdrawn in favor of this note's shape. What it bought: text routes could keep running after images entered history, and the tool disappeared from prompts where it cannot succeed. What it cost: agent-loop changes, three new durable concepts (`agent/request-ready`, `messageProjection`, availability notices), and registration that churned per step. The capability itself — see an image on the next request — never needed any of it. If per-route projection becomes a real requirement, that PR's history is the reference implementation. +- **PR #598's route-scoped design** used a request-ready extension point, per-route schema visibility, reversible projection, and three durable concepts. Shared LLM request projection now handles text-only routes without putting tool registration or session formats into agent-loop. - **`agent.inject()` instead of the image-bearing tool result** — routes the image around the tool result as a separate injected user message. Rejected: the image *is* the tool's result; splitting them adds a second logged message with no gain, and the tool-result path already works end to end. - **Magic-byte sniffing instead of extension declaration** — sniffing duplicates detection the attachment store already owns (sharp-backed, authoritative). The extension is only a *declaration*; a mismatch fails closed with a rename remedy rather than being silently accepted, which also keeps the model's mental map (file name ↔ content) honest. - **Registering unconditionally and failing on a missing store** — rejected; a deployment without an attachment store cannot ever satisfy the tool, so its schema would be a standing lie. The route gate, by contrast, is per-call state and correctly lives at the execution boundary. ## Consequences -- A text-only route refuses instead of degrading: no placeholder projection means no delegated-viewing story here — that is deliberately the next PR (subagent image readback rebuilt on the current subagent seams). -- The route gate races a concurrent model switch; the Web host's image-aware switch guard covers its surface, and other front doors own their equivalent. Recorded as a tool-fs Known Limitation. -- Repeated image results accumulate request-token cost until compaction; content addressing deduplicates bytes only. +- The tools refuse execution on a text-only route, while existing images in session history are represented by request-local placeholders. +- Pasted and dragged images can be cropped without exposing local paths. Session reference authorization prevents access to attachments outside the current session. +- Repeated image results accumulate request cost until request projection or compaction removes them; content addressing deduplicates durable bytes. - The tool-result card renders the durable reference, not pixels; inline preview is deferred to the UI packages. diff --git a/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.zh.md b/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.zh.md index a94e4b2964..c3c2fe1095 100644 --- a/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.zh.md +++ b/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.zh.md @@ -6,28 +6,29 @@ Status: implemented ## 问题 -多模态附件工作为用户上传建立了完整的持久路径:字节在所属 `user/message` 之前提交到内容寻址的附件存储,`ImageBlock` 只携带 `sha256:` 引用,pi-ai 路由在每次请求时重新读取并校验字节。但模型自己没有查看磁盘图像的手段。`read` 按约定拒绝二进制内容,因此被问到截图或渲染图表的 agent 要么失败,要么退到有损的变通做法。第一次独立尝试(PR #598)把这个问题与循环级路由作用域一起解决:新增在组装前发布确切模型模态的 `agent/request-ready` 扩展点、按路由控制 schema/指导可见性,以及可逆的 `image-placeholder-v1` 历史投影让文本路由能在占位符上继续。该设计可行,但让一个工具耦合了新的 agent-loop 机制、三个新的会话日志概念和每步的注册变动,远超这项能力本身的需要。 +多模态附件工作为用户上传建立了完整的持久路径,但模型无法查看磁盘图片,也无法裁剪没有文件路径的持久用户上传。`read` 按约定拒绝二进制内容,因此被问到截图或渲染图表的 agent 要么失败,要么使用有损的变通方法。PR #598 的独立尝试把工具与循环级路由作用域、按路由控制 schema 可见性和新的会话日志概念放在一起。这些能力不是发布一条带图片且已记录的工具结果所必需的。 ## 决定 -只交付能把图像载入下一次请求上下文的最小工具,完全建立在既有 seam 之上;撤回的 PR #598 设计是本记录明确保留的反例。 +两个图片读取操作都放在 `dsh-tool-fs`,通过现有扩展点发布普通的持久工具结果。 -- **`read_image` 放在 `dsh-tool-fs`**,与 `read`/`write`/`edit` 并列。扩展名选择声明的 PNG/JPEG/WebP/GIF 媒体类型;附件存储的魔数与像素校验保持权威。字节沿 `ctx.fs.stat` → 有界 `ctx.fs.readBytes` → `ctx.attachments.saveImage` → `fs/observed` 流动,工具结果是元数据信封加真正的 `ImageBlock`——`ToolResultBlock.content` 本就允许图像块,pi-ai 适配器本就会渲染它们,Web 宿主的模型切换防护本就会扫描工具结果,下游无需任何改动。 +- **`read_image` 读取文件系统路径。** 扩展名选择声明的 PNG/JPEG/WebP/GIF 媒体类型,附件存储的魔数与像素校验保持权威。字节沿 `ctx.fs.stat` → 有界 `ctx.fs.readBytes` → `ctx.attachments.saveImage` → `fs/observed` 流动。工具结果包含元数据和一个 `ImageBlock`。 +- **`read_image_region` 裁剪会话中的持久附件。** 请求给出完整附件 ID、当前预览尺寸和预览坐标矩形。工具根据当前会话已引用的图片授权该 ID,把矩形映射到持久主版本,从主版本裁剪,并把结果保存为新附件。结果包含裁剪后的 `ImageBlock`,因此模型可见裁剪可以从日志重建。这也是粘贴或拖入且没有文件路径的图片所使用的入口。 - **`FileSystem.readBytes(target, signal, maxBytes)`** 是新的必备提供方原语:字节上限放在 seam 上,任何后端都无法无界缓冲文件;stat 大小先短路,随后的流最多多读一个字节以防 stat 之后的增长(`FS_TOO_LARGE`)。 -- **注册随组合条件挂载,执行按路由门禁。** 工具只在 `ctx.inject(['attachments'], …)` 作用域内注册——没有存储就没有工具。执行时在任何 I/O 之前,严格门禁通过 `ctx.llm.resolveModelInfo` 解析调用路由(最新 `request/header` 配置,缺失时回退到 agent 选项),要求 `inputModalities` 包含 `image`;能力未知即拒绝。拒绝是普通的 `isError` 结果,因此文本路由的持久历史绝不会出现图像块,会话不会毁掉自己的路由。 +- **注册随组合条件挂载,执行按路由门禁。** 工具只在 `ctx.inject(['attachments'], …)` 作用域内注册。执行时在 I/O 之前通过 `ctx.llm.resolveModelInfo` 解析调用路由,并要求 `inputModalities` 包含 `image`;能力未知即拒绝。纯文本路由仍可使用此前的持久图片,因为共享 LLM 运行时会在请求组装时把图片投影为占位符。 - **Code Mode 以带外方式转发图像**:嵌套分派返回规范值(仅限本次执行,不含图像块),并延迟提交一条携带信封和图像的 `user` 角色上下文消息,图片仍会到达下一次请求。 -- **llm-replay 模型可以声明 `inputModalities`**,这正是两个 keyless ACP 快照能钉住门禁两侧的原因:图像路由上以 sha256 引用的成功结果,和纯文本路由上逐字的拒绝。 +- **llm-replay 模型可以声明 `inputModalities`**,因此 keyless ACP 快照可以覆盖支持图片的结果和纯文本拒绝。 ## 考虑过的替代方案 -- **PR #598 的路由作用域设计**(request-ready 扩展点、按路由的 schema/指导可见性、可逆历史投影)——被本记录的形态取代后撤回。它换来的是:图像进入历史后文本路由仍能运行,工具在注定失败的提示词里消失。它付出的是:改动 agent-loop、三个新的持久概念(`agent/request-ready`、`messageProjection`、可用性通知)和每步变动的注册。而这项能力本身——下一次请求看到图像——从不需要这些。如果按路由投影将来成为真实需求,该 PR 的历史就是参考实现。 +- **PR #598 的路由作用域设计**使用 request-ready 扩展点、按路由控制 schema 可见性、可逆投影和三个持久概念。共享 LLM 请求投影现在可以处理纯文本路由,无需把工具注册或会话格式放进 agent-loop。 - **用 `agent.inject()` 代替带图像的工具结果**——把图像绕过工具结果,作为单独注入的用户消息。拒绝:图像就是工具的结果;拆开只会多一条无收益的日志消息,而工具结果路径本就端到端可用。 - **用魔数嗅探代替扩展名声明**——嗅探重复了附件存储已拥有的检测(基于 sharp,权威)。扩展名只是声明;不匹配时按改名修复提示失败关闭,而不是被静默接受,这也让模型对文件名与内容的对应保持诚实。 - **无条件注册、缺存储时执行报错**——拒绝;没有附件存储的部署永远无法满足该工具,其 schema 会是常态谎言。相反,路由门禁是逐调用状态,正确的位置就是执行边界。 ## 后果 -- 纯文本路由得到拒绝而不是降级:没有占位符投影意味着这里没有委托查看的方案——那有意留给下一个 PR(基于当前 subagent seam 重建的 subagent image readback)。 -- 路由门禁与并发模型切换存在竞态;Web 宿主的图像感知切换防护覆盖其表面,其他前端拥有各自的等价防护。已记入 tool-fs 的已知限制。 -- 重复的图像结果在压缩之前持续累积请求 token 成本;内容寻址只去重字节。 +- 工具在纯文本路由上拒绝执行,而会话历史中已经存在的图片会由请求期占位符表示。 +- 粘贴和拖入的图片无需暴露本地路径即可裁剪。会话引用授权会阻止访问当前会话范围外的附件。 +- 重复的图片结果会累积请求成本,直到请求投影或压缩将其移除;内容寻址只去重持久字节。 - 工具结果卡片渲染持久引用而非像素;内嵌预览延后到 UI 包处理。 diff --git a/.agents/notes/implemented/feature/2026-08-12-web-image-intake-and-limits-alignment.i18n.yaml b/.agents/notes/implemented/feature/2026-08-12-web-image-intake-and-limits-alignment.i18n.yaml index 0720d5d9ee..3c2be099df 100644 --- a/.agents/notes/implemented/feature/2026-08-12-web-image-intake-and-limits-alignment.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-08-12-web-image-intake-and-limits-alignment.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-08-12-web-image-intake-and-limits-alignment.md -2026-08-12-web-image-intake-and-limits-alignment.md: 00cf7ea99d63e848c4b5839da1d97d94c9fb8464 -2026-08-12-web-image-intake-and-limits-alignment.zh.md: 7bf7f3621d6d305baf8e7c1c060bbc5810f28b77 +2026-08-12-web-image-intake-and-limits-alignment.md: 0bb8cadc8db4b4c28cf311bc9420c32744e173fb +2026-08-12-web-image-intake-and-limits-alignment.zh.md: fafa7652756554a54a0a0952e843bf5f4b81b00d diff --git a/.agents/notes/implemented/feature/2026-08-12-web-image-intake-and-limits-alignment.md b/.agents/notes/implemented/feature/2026-08-12-web-image-intake-and-limits-alignment.md index 00cf7ea99d..0bb8cadc8d 100644 --- a/.agents/notes/implemented/feature/2026-08-12-web-image-intake-and-limits-alignment.md +++ b/.agents/notes/implemented/feature/2026-08-12-web-image-intake-and-limits-alignment.md @@ -16,7 +16,7 @@ The second alignment step for issue #2248, after the [attachment display note](2 **History thumbnails (DeepSeek Chat rules).** A message's lone image renders at 240px on its long edge with the displayed ratio clamped to [0.25, 4], cropped by `cover` with the anchor at the top of very tall images and the left of very wide ones, never upscaled; several images render as fixed 64px square tiles in one wrapping row (10px gap, user messages right-aligned). Consecutive assistant `image` blocks merge into one gallery so they tile instead of each opening a one-image row. -**Limits aligned and projected.** Defaults are 20 images / 3.5 MiB per image / 100 MiB aggregate (`attachment-local`), with the HTTP carrier cap raised to one shared `DEFAULT_MAX_REQUEST_BODY_BYTES = 160 MiB` (http-bridge, previously two independent 32 MiB literals) to satisfy the load-time capacity assertion (aggregate × 4/3 + headroom ≈ 134.3 MiB). Consumer products cluster at 10–20 attachments (ChatGPT 10, Gemini 10, Claude 20; DeepSeek Chat's 50 is the outlier), and a vision-model image costs roughly 1300–4800 tokens, so 50 images can fill a 200k context in one message. Including base64 padding, a 3.5 MiB encoded file occupies at most 4.67 MiB and leaves 0.33 MiB below a 5 MiB route check. Deployments using only routes with larger limits can override it. A 512 MiB aggregate cannot pass this transport because base64-in-JSON would need a single JSON string past V8's ~512 MiB string ceiling. The limits reach clients as the `imageLimits` session projection — a constant-per-boot unit (`apply` returns the same state reference, so baselines alone carry it and no change frames exist) registered by **apiproxy**, not the attachment Service Definition: `dsh-llm` depends on `dsh-attachment` (`ImageBlock` → `ImageAttachmentRef`), so the seam package referencing `dsh-session-projection` (whose graph reaches `dsh-llm` through `dsh-session`) closes a project-reference cycle, and the per-message count/aggregate rules the value describes are the proxy's own admission checks anyway. The `SessionProjectionMap` merge rides the proxy's sessions wire-contract file, which every client program already includes through the carrier's type re-exports. +**Limits aligned and projected.** Intake defaults are 20 images, 32 MiB per source, 100 MiB aggregate source bytes, 100 million decoded pixels, and 16384px per source side. The attachment backend prepares a separate durable master with a 2048px long edge and 4 MiB safety cap. Model requests have their own route-specific pixel and encoded-byte budgets, so source admission does not use provider request limits. The HTTP carrier uses one shared `DEFAULT_MAX_REQUEST_BODY_BYTES = 160 MiB` to satisfy the load-time capacity assertion for the 100 MiB aggregate after base64 and envelope expansion. A 512 MiB aggregate cannot pass this transport because base64-in-JSON would require a JSON string near V8's string-size limit. The intake limits reach clients as the `imageLimits` session projection, a constant-per-boot unit registered by **apiproxy** rather than the attachment Service Definition. `dsh-llm` depends on `dsh-attachment`, while `dsh-session-projection` reaches `dsh-llm` through `dsh-session`; registering the projection in the seam package would create a project-reference cycle. The per-message count and aggregate rules are also enforced by the proxy. The `SessionProjectionMap` merge remains in the proxy sessions wire file, which clients already consume through carrier type re-exports. **Intake pre-check and error copy.** Both intake gestures converge on one `intakeImages` wrapper in InputBar that checks count, per-image bytes, and aggregate bytes against the projection before `addImages`: a violating batch is refused whole (DeepSeek Chat semantics) with an immediate banner naming the limit — no submit-time rollback theater. The host checks stay as the backstop for callers that bypass the composer. Banner copy follows one principle the user set: reasons a user can act on (model without vision, count, size, resolution, format — now a positive list of supported formats instead of echoing the rejected MIME type) get product sentences naming the way out; reasons they cannot act on (corrupt base64, lost references, read failures) fold into one send-failed sentence that keeps the reason code, because the product currently faces developers and a reportable code beats a dead end. Non-attachment error codes keep the raw message + code presentation. diff --git a/.agents/notes/implemented/feature/2026-08-12-web-image-intake-and-limits-alignment.zh.md b/.agents/notes/implemented/feature/2026-08-12-web-image-intake-and-limits-alignment.zh.md index 7bf7f3621d..fafa765275 100644 --- a/.agents/notes/implemented/feature/2026-08-12-web-image-intake-and-limits-alignment.zh.md +++ b/.agents/notes/implemented/feature/2026-08-12-web-image-intake-and-limits-alignment.zh.md @@ -16,7 +16,7 @@ issue #2248 的第二步对齐,接在[附件展示 note](2026-08-11-web-attach **历史缩略图(DeepSeek Chat 规则)。** 一条消息仅有的一张图长边 240px、展示比例钳制在 [0.25, 4],`cover` 裁切,特别高的图锚定顶部、特别宽的锚定左侧,从不放大;多张图渲染为固定 64px 方块,单个可换行的横排(10px 间距,用户消息右对齐)。assistant 连续的 `image` 块合并进同一个画廊,平铺而不是各占一行。 -**上限对齐并投影。** 默认值为每条消息 20 张、单图 3.5 MiB、总量 100 MiB(`attachment-local`),HTTP 载体上限提为唯一共享的 `DEFAULT_MAX_REQUEST_BODY_BYTES = 160 MiB`(http-bridge,原先是两个独立的 32 MiB 字面量),以满足加载时的容量断言(总量 × 4/3 加余量 ≈ 134.3 MiB)。消费级产品集中在 10 到 20 个附件(ChatGPT 10、Gemini 10、Claude 20;DeepSeek Chat 的 50 是例外),且视觉模型一张图约 1300 到 4800 token,因此 50 张图可在一条消息中填满 200k 上下文。3.5 MiB 编码文件包括 base64 填充在内最多占 4.67 MiB,在 5 MiB 路由检查下保留 0.33 MiB 余量。仅使用较大上限路由的部署可以覆盖该值。512 MiB 总量无法通过当前传输,因为 base64 进 JSON 需要一个超过 V8 约 512 MiB 字符串上限的单个 JSON 字符串。限额以 `imageLimits` 会话投影到达客户端。它是每次启动恒定的单元(`apply` 返回同一状态引用,因此只靠基线携带、不存在变更帧),由 **apiproxy** 而非 attachment Service Definition 注册:`dsh-llm` 依赖 `dsh-attachment`(`ImageBlock` → `ImageAttachmentRef`),seam 包引用 `dsh-session-projection`(其图谱经 `dsh-session` 到达 `dsh-llm`)会闭合 project-reference 环,而该值描述的每消息数量与总量规则本来就是 proxy 自己的准入检查。`SessionProjectionMap` 合并放在 proxy 的 sessions 协议文件里,每个客户端程序都经载体的类型再导出包含它。 +**上限对齐并投影。** 输入默认值是每条消息 20 张、每个源文件 32 MiB、源文件总量 100 MiB、每张图片一亿解码像素,以及源文件任一边 16384px。附件后端另行生成长边 2048px、独立安全上限 4 MiB 的持久主版本。模型请求使用各路由自己的像素和编码字节预算,因此源文件准入不采用提供方请求限制。HTTP 载体统一使用 `DEFAULT_MAX_REQUEST_BODY_BYTES = 160 MiB`,满足 100 MiB 总量经过 base64 和请求封装扩张后的加载时容量断言。512 MiB 总量无法通过当前传输,因为 base64 进入 JSON 后会需要一个接近 V8 字符串大小上限的 JSON 字符串。输入上限通过 `imageLimits` 会话投影到达客户端。它是每次启动恒定的单元,由 **apiproxy** 而非 attachment Service Definition 注册。`dsh-llm` 依赖 `dsh-attachment`,而 `dsh-session-projection` 经 `dsh-session` 到达 `dsh-llm`;在 seam 包注册投影会形成 project-reference 环。每条消息的数量和总量规则也由 proxy 强制执行。`SessionProjectionMap` 合并继续放在 proxy 的 sessions 协议文件中,客户端已经通过载体类型再导出使用它。 **加入预检与错误文案。** 两种加入手势汇合到 InputBar 的一个 `intakeImages` 包装:在 `addImages` 之前按投影检查数量、单图字节与总字节,违规的一批整体拒收(DeepSeek Chat 语义)并立刻弹出点名上限的横幅——不再有提交时的回滚戏码。宿主检查保留,兜底绕过 composer 的调用方。横幅文案遵循用户定下的一条原则:用户能解决的原因(模型不支持视觉、数量、大小、分辨率、格式——格式改为正面列出支持列表而不是回显被拒的 MIME 类型)用点明出路的产品句子;用户无法解决的原因(base64 损坏、引用丢失、读取失败)折叠为一条保留原因码的发送失败句子,因为产品当前面向开发者,可上报的码好过死胡同。非附件错误码保留原文加错误码的展示。 diff --git a/.agents/notes/implemented/feature/2026-08-19-direct-deepseek-vision-input.md b/.agents/notes/implemented/feature/2026-08-19-direct-deepseek-vision-input.md deleted file mode 100644 index 76d3244e67..0000000000 --- a/.agents/notes/implemented/feature/2026-08-19-direct-deepseek-vision-input.md +++ /dev/null @@ -1,34 +0,0 @@ -# Agent Note: Direct DeepSeek vision input - -Status: implemented - -English | [中文](2026-08-19-direct-deepseek-vision-input.zh.md) - -## Problem - -DeepSeek vision deployments use the chat-completions image protocol, but the direct `deepseek-official` adapter declares every catalog and pass-through model text-only and rejects every `ImageBlock`. The durable attachment path therefore works only through configurable pi-ai routes, and a deployment cannot pass user uploads or image-bearing tool results through the direct provider. - -## Decision - -The shipped catalog declares `deepseek-v4-flash-vision-exp` with `inputModalities: [text, image]`; configured catalogs use the same declaration to opt another exact model into image input, and validation rejects empty, unknown, or duplicate modalities. Flash, Pro, unlisted ids, and configured models that omit `inputModalities` remain explicitly text-only. - -The adapter resolves `ctx.attachments` per image request, reads each retained durable reference with the request signal, and serializes verified bytes as ordered OpenAI-compatible `image_url` data URLs. Text-only user messages retain string content. Tool results retain string-only `tool` messages; image-only results use `(see attached image)`, and consecutive retained tool-result images follow in one `user` message beginning `Attached image(s) from tool result:`. System and assistant history images fail with `UNSUPPORTED_CONTENT` before attachment or network I/O. - -The direct adapter and pi-ai conversion share the deterministic [request-level image payload bound](../bug-fix/2026-08-18-request-image-payload-bound.md). Both default to 20 MiB of accumulated base64 payload, replace oldest image occurrences with the same fixed placeholder, and never read omitted attachments. Direct HTTP 413 responses are `INVALID_REQUEST`; attachment failures retain their stable attachment code rather than becoming `TRANSPORT`. - -Canonical messages continue to store only `ImageAttachmentRef`. Data URLs exist only while preparing one provider request, so no session event, persistence format, API schema, or SDK projection changes. The route accepts PNG, JPEG, WebP, and GIF already admitted by the attachment service. External image URLs, the Files API, and image output remain unsupported. - -## Alternatives considered - -- **Use only the pi-ai DeepSeek provider.** Its generic multimodal path proves the content conversion, but it does not make the direct official route truthful or usable with the official model id. -- **Declare the whole provider image-capable.** This would let Flash, Pro, and unknown pass-through ids accept durable images that their exact wire model cannot promise to consume. Capability remains exact-model metadata. -- **Send images inside `tool` message content.** The documented compatible form keeps tool content a string. A following user message avoids relying on an undocumented multimodal tool-role form while preserving call-result order. -- **Add external URLs or Files uploads.** Both require new canonical input, authorization, lifetime, cleanup, and replay decisions. Transient base64 uses the existing durable attachment contract without expanding those concerns. - -## Verification - -Package tests pin model discovery and fallback capabilities, configuration validation and live settings updates, user and tool-result wire messages, all admitted MIME types, cancellation, attachment failures, 413 classification, exact image-bound behavior, and pi-ai equivalence. A keyless assembled ACP request records the native adapter's tool-result data URL and oldest-image placeholder. A real-API smoke test with an explicit image-capable catalog entry sends a deterministic image only when `DEEPSEEK_VISION_E2E=1` is set in addition to the provider key. - -## Consequences - -The official DeepSeek vision route and configured vision routes can consume durable user and tool-result images without changing session durability or response streaming. Repeated history still expands request bodies, but deterministic oldest-first offload bounds the dominant payload and leaves headroom below the official 30 MiB request-body limit. Image token pricing remains provider-owned because the official image token formula is not available. diff --git a/.agents/notes/implemented/feature/2026-08-19-direct-deepseek-vision-input.zh.md b/.agents/notes/implemented/feature/2026-08-19-direct-deepseek-vision-input.zh.md deleted file mode 100644 index a772311041..0000000000 --- a/.agents/notes/implemented/feature/2026-08-19-direct-deepseek-vision-input.zh.md +++ /dev/null @@ -1,34 +0,0 @@ -# Agent Note: 直接 DeepSeek 视觉输入 - -Status: implemented - -[English](2026-08-19-direct-deepseek-vision-input.md) | 中文 - -## Problem - -DeepSeek 视觉部署使用 chat-completions 图片协议,但直接 `deepseek-official` 适配器把所有 catalog 与原样传递模型都声明为仅文本,并拒绝每一个 `ImageBlock`。因此,持久附件路径只能经可配置 pi-ai 路由工作,部署方无法通过直接提供方传递用户上传或包含图片的工具结果。 - -## Decision - -随附目录为 `deepseek-v4-flash-vision-exp` 声明 `inputModalities: [text, image]`;已配置目录可以用同一声明让另一个确切模型支持图片输入,校验会拒绝空列表、未知模态或重复模态。Flash、Pro、未列出 id,以及省略 `inputModalities` 的已配置模型仍明确仅支持文本。 - -适配器会对每个图片请求解析 `ctx.attachments`,用请求 signal 读取每个保留的持久引用,并将校验后的字节按顺序序列化为 OpenAI 兼容的 `image_url` data URL。纯文本 user 消息保留字符串内容。工具结果保留仅字符串的 `tool` 消息;仅含图片的结果使用 `(see attached image)`,连续工具结果中保留的图片随后合并进一条以 `Attached image(s) from tool result:` 开头的 `user` 消息。System 与 assistant 历史图片会在附件或网络 I/O 前以 `UNSUPPORTED_CONTENT` 失败。 - -直接适配器与 pi-ai 转换共享确定性的[请求级图片载荷上限](../bug-fix/2026-08-18-request-image-payload-bound.zh.md)。两者都以 20 MiB 累计 base64 payload 为默认值,用相同固定占位文本替换最旧的图片出现位置,并且绝不读取被省略的附件。直接 HTTP 413 响应归类为 `INVALID_REQUEST`;附件失败会保留其稳定附件 code,不会变成 `TRANSPORT`。 - -规范消息继续只存储 `ImageAttachmentRef`。Data URL 只在准备单次提供方请求时存在,因此无需修改会话事件、持久化格式、API schema 或 SDK 投影。路由接受已经由附件服务准入的 PNG、JPEG、WebP 和 GIF。不支持外部图片 URL、Files API 和图片输出。 - -## Alternatives considered - -- **只使用 pi-ai DeepSeek 提供方。** 其通用多模态路径验证了内容转换,但无法让直接官方路由如实公布能力,也无法让它配合官方模型 id 使用。 -- **把整个提供方声明为支持图片。** 这样会让 Flash、Pro 和未知的原样传递 id 接受持久图片,但其确切协议模型无法承诺消费这些图片。能力仍属于确切模型元数据。 -- **在 `tool` 消息内容中发送图片。** 已记录的兼容形式要求工具内容保持字符串。随后发送 user 消息可避免依赖未记录的多模态 tool role 形式,同时保留调用结果顺序。 -- **增加外部 URL 或 Files 上传。** 两者都需要新的规范输入、授权、生命周期、清理和重放决策。瞬态 base64 可以复用现有持久附件约定,不扩展这些问题。 - -## Verification - -包测试固定模型发现与回退能力、配置校验与存活 settings 更新、user 和工具结果协议消息、所有已准入 MIME 类型、取消、附件失败、413 分类、确切图片上限行为和 pi-ai 等价性。无需密钥的组装 ACP 请求会记录原生适配器的工具结果 data URL 与最旧图片占位文本。真实 API 冒烟测试会配置明确支持图片的目录项,并且仅在提供方密钥之外还设置 `DEEPSEEK_VISION_E2E=1` 时发送确定性图片。 - -## Consequences - -官方 DeepSeek 视觉路由与已配置视觉路由可以消费持久 user 与工具结果图片,而无需改变会话持久性或响应流。重复历史仍会扩张请求正文,但确定性的最旧优先 offload 会限制主导 payload,并在官方 30 MiB 请求正文上限下保留余量。由于官方图片 token 公式尚不可用,图片 token 定价仍由提供方掌握。 diff --git a/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.i18n.yaml b/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.i18n.yaml deleted file mode 100644 index d8a89613e9..0000000000 --- a/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.i18n.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each -# side as of the last confirmed-consistent state. Both languages carry equal authority; -# after editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-08-20-canonical-image-admission.md -2026-08-20-canonical-image-admission.md: a30031ef72942a61865525b9ed22f97afd71e18b -2026-08-20-canonical-image-admission.zh.md: d5402a6e7bfd2d8c7de6e2a7ce611c74ec2843d2 diff --git a/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.md b/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.md deleted file mode 100644 index a30031ef72..0000000000 --- a/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.md +++ /dev/null @@ -1,28 +0,0 @@ -# Agent Note: Canonical image admission - -Status: implemented - -English | [中文](2026-08-20-canonical-image-admission.zh.md) - -## Problem - -Admission used to refuse any image above 2000px per side or 3.5 MiB, because an admitted image rides every later request and deployed routes reject oversized images. Refusal pushed the problem onto the user (downscale by hand, re-attach), and the byte size of admitted images was uncontrolled below the cap, so long sessions accumulated large request payloads. The unified image-pipeline design (PR #2676) needs a canonical, deterministic stored form as the basis for content-addressed dedup, stable request bytes, and a later provider-files upload path. - -## Decision - -`AttachmentStore.saveImage` resolves `SavedImageAttachment`: the durable `ref` describing stored bytes beside `source` facts of the submitted raster. The local store validates a wide source envelope (32 MiB, 100 MP, 16384px per side) and persists a deterministic canonical encoding: EXIF orientation baked in, metadata stripped, long edge downscaled to `canonicalMaxDimension` (default 2048px), palette PNG for alpha/PNG/GIF lineage and JPEG for photographic sources, stepping a fixed quality ladder (85/75/60/45) until `canonicalMaxBytes` (default 1 MiB) holds. An in-budget PNG/JPEG/WebP source passes through byte-identically only when it is single-frame and free of EXIF/XMP/IPTC metadata and non-default orientation, so equal originals keep one content address while location and device metadata never survive admission; GIF and every animated or metadata-carrying source re-encodes, and GIF always becomes the PNG of its first frame, pinning the first-frame meaning providers apply. Encoder parameters are fixed, not configurable — a parameter change would silently split the content-addressed space — so deployments choose only the source envelope and the canonical budget. `SourceImageInfo` records orientation-applied dimensions so source and stored raster share axes, and `validateImage` includes a canonical-encoding dry run so a validated batch can never be refused mid-write by the byte target. The canonical ref keeps the pre-existing field order (`mediaType`, `width`, `height`, `bytes`) so logged references stay byte-identical. `read_image` reports the on-disk dimensions and the coordinate multiplier whenever storage downscaled the file, naming per-axis multipliers when integer rounding makes the two ratios differ. - -## Alternatives considered - -- **Keep refusing oversized sources.** Simple, but hostile at exactly the moment a user pastes a normal screenshot from a HiDPI display, and it leaves admitted byte sizes unbounded below the cap. -- **Canonicalize at request time.** Re-encoding per request breaks byte-stable prefixes (provider context caching) and violates the design's rule that durable content is written once; the request layer only projects. -- **Make encoder quality configurable.** Two deployments with different quality would address the same source at different ids, silently defeating dedup; fixed parameters keep the space whole and an encoder upgrade re-addresses only future saves. -- **Pin a resize transcript snapshot.** A fixture embedding re-encoded bytes depends on cross-platform encoder byte-stability (libvips resize and palette quantization across arm64/x86), which is unverified in CI; the assembled snapshot instead pins the acceptance passthrough (2001x1 admitted byte-identically), and re-encode branches are pinned by package tests. - -## Verification - -Package tests cover passthrough identity, resize determinism and idempotence, GIF-to-PNG, alpha-to-PNG, JPEG ladder descent, ladder exhaustion refusal, encoder-fault mapping, and the store round-trip of a downscaled save. The read-image suite pins the downscale envelope text. The `read-image-dimension` keyless snapshot now pins the acceptance the 2000px cap used to refuse, using passthrough bytes so the fixture is platform-independent. - -## Consequences - -Ordinary large sources are admitted and bounded (≤2048px, ≤1 MiB by default), shrinking per-request image payload roughly 3.5x at the old cap and making the planned request-level budgets rarely reachable. Stored bytes may differ from the submitted file; consumers that map coordinates use the saved `source` facts, as `read_image` does. A cross-platform byte-stability check for the re-encode path remains open before any fixture may embed re-encoded bytes. diff --git a/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.zh.md b/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.zh.md deleted file mode 100644 index d5402a6e7b..0000000000 --- a/.agents/notes/implemented/feature/2026-08-20-canonical-image-admission.zh.md +++ /dev/null @@ -1,28 +0,0 @@ -# Agent Note: 规范化图片准入 - -Status: implemented - -[English](2026-08-20-canonical-image-admission.md) | 中文 - -## 问题 - -准入过去拒绝任何单边超过 2000px 或超过 3.5 MiB 的图片,因为已接纳的图片会随之后每次请求发送,而已部署路由会拒绝过大的图片。拒绝把问题推给了用户(手动缩图再重新附上),而且上限以内的已接纳图片字节数不受控制,长会话会累积出很大的请求载荷。统一图片管线设计(PR #2676)需要一个规范且确定性的存储形态,作为内容寻址去重、请求字节稳定以及后续 provider files 上传路径的基础。 - -## 决定 - -`AttachmentStore.saveImage` 解析为 `SavedImageAttachment`:描述实际存储字节的持久 `ref`,加上所提交光栅的 `source` 事实。本地存储按宽松的源图上限(32 MiB、1 亿像素、单边 16384px)校验,然后持久保存确定性的规范编码:EXIF 方向落实到像素、剥离元数据、长边等比缩放到 `canonicalMaxDimension`(默认 2048px),带透明通道或源自 PNG/GIF 的图片编码为 palette PNG,摄影类图片编码为 JPEG,并沿固定质量阶梯(85/75/60/45)递降直到满足 `canonicalMaxBytes`(默认 1 MiB)。已在预算内的 PNG/JPEG/WebP 源图只有在单帧且不携带 EXIF/XMP/IPTC 元数据、方向为默认值时才按字节原样直通,相同原图保持同一个内容地址,位置与设备元数据绝不越过准入;GIF 以及任何动图或携带元数据的源图都会重编码,GIF 一律转为首帧 PNG,在准入时固化提供方实际采用的首帧语义。编码器参数固定而不可配置,因为参数变化会悄悄割裂内容寻址空间;部署只选择源图上限与规范预算。`SourceImageInfo` 记录应用方向之后的尺寸,使源图与存储光栅共享坐标轴;`validateImage` 包含规范编码干跑,通过校验的批次绝不会在写入中途被字节目标拒绝。规范 ref 保持原有字段顺序(`mediaType`、`width`、`height`、`bytes`),已记录的引用保持字节一致。存储缩小了文件时,`read_image` 会报告磁盘上的原始尺寸和坐标换算倍率,取整使两轴比例不一致时分轴给出。 - -## 考虑过的替代方案 - -- **继续拒绝超限源图。** 简单,但恰恰在用户从 HiDPI 屏幕粘贴一张普通截图的时刻表现得不友好,而且上限以内的已接纳字节数仍然无界。 -- **在请求时规范化。** 按请求重编码会破坏字节稳定前缀(provider 上下文缓存),也违反设计中「持久内容只写一次、请求层只做投影」的规则。 -- **让编码质量可配置。** 两个部署用不同质量会把同一源图寻址到不同 id,悄悄破坏去重;固定参数保持寻址空间完整,编码器升级只影响之后的保存。 -- **钉一个缩放的 transcript 快照。** 嵌入重编码字节的 fixture 依赖跨平台编码器字节稳定性(libvips 缩放与调色板量化在 arm64/x86 上的表现),CI 尚未验证;组装快照改为钉住接纳直通行为(2001x1 按字节原样接纳),重编码分支由包测试钉住。 - -## 验证 - -包测试覆盖直通恒等、缩放确定性与幂等、GIF 转 PNG、透明通道转 PNG、JPEG 阶梯递降、阶梯穷尽拒绝、编码器故障映射,以及缩小保存的存储往返。read-image 测试钉住缩放信封文本。`read-image-dimension` keyless 快照现在钉住 2000px 上限过去拒绝的接纳行为,使用直通字节因此 fixture 与平台无关。 - -## 后果 - -普通大图会被接纳并受约束(默认 ≤2048px、≤1 MiB),在旧上限处把单请求图片载荷缩小约 3.5 倍,使计划中的请求级预算正常情况下难以触达。存储字节可能与提交的文件不同;需要换算坐标的消费方使用保存的 `source` 事实,`read_image` 即如此。在任何 fixture 嵌入重编码字节之前,重编码路径的跨平台字节稳定性检查仍是待办。 diff --git a/.agents/notes/implemented/feature/2026-08-19-direct-deepseek-vision-input.i18n.yaml b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml similarity index 56% rename from .agents/notes/implemented/feature/2026-08-19-direct-deepseek-vision-input.i18n.yaml rename to .agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml index 467a951f7c..53c15e1755 100644 --- a/.agents/notes/implemented/feature/2026-08-19-direct-deepseek-vision-input.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml @@ -1,6 +1,6 @@ # Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: -# pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-08-19-direct-deepseek-vision-input.md -2026-08-19-direct-deepseek-vision-input.md: 76d3244e67a73c1cdf4419a6537ada38e0a75bd5 -2026-08-19-direct-deepseek-vision-input.zh.md: a77231104156371fe698f8a8ad386cfa03251990 +# pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md +2026-08-20-unified-image-request-pipeline.md: c487f583e4770b8d495404f08de67fd877dc48fd +2026-08-20-unified-image-request-pipeline.zh.md: a82312d55ba59403e71e97ee483e2b5bbfebfb03 diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md new file mode 100644 index 0000000000..c487f583e4 --- /dev/null +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md @@ -0,0 +1,71 @@ +# Agent Note: Unified image masters, request versions, and provider files + +Status: implemented + +English | [中文](2026-08-20-unified-image-request-pipeline.zh.md) + +## Problem + +Durable image history, provider resolution, inline request size, and remote file reuse have different limits. Treating an admitted image as the bytes sent on every later request forced one byte cap and one raster to serve all four concerns. Large but ordinary input was refused, clean 16-bit PNG could pass into history and fail at DeepSeek, repeated base64 expanded long requests, and a provider rejection repeated because the same durable image stayed in every future request. A model also had no stable way to crop a user upload that had no filesystem path. + +## Decision + +The image path has two explicit versions. The attachment backend owns a provider-independent durable master. Each image-capable model route owns a deterministic request policy, and the attachment backend derives and caches the exact request version from the master. Session history contains only the master reference; inline bytes and provider file ids remain transient request projections. + +### Provider-independent master + +Admission fully decodes each source under a configurable 32MiB, 100MP, and 16384px-per-side envelope. It applies EXIF orientation, removes metadata and color profiles, converts to 8-bit sRGB/sRGBA, and preserves aspect ratio while limiting the long edge to `masterMaxDimension`, 2048px by default. `sourceWidth` and `sourceHeight` record orientation-applied dimensions when preparation reduces the raster. + +The master has an independent `masterMaxBytes` safety cap, 4MiB by default. Alpha is never flattened. A nearest-neighbour bounded sample classifies color complexity without averaging high-frequency pixels. Confirmed low-color input tries PNG, with palette encoding only when no alpha channel is present, followed by WebP qualities 85, 80, and 75. Other alpha input tries WebP at those qualities; other opaque input tries JPEG. Candidates execute in order and stop at the first result within the cap. Dimensions shrink only after every candidate at one size exceeds the cap. The source extension does not classify a PNG as low color. A clean, single-frame 8-bit sRGB/sRGBA PNG, JPEG, or WebP within both master limits passes through byte-identically and retains content-addressed deduplication. GIF, animation, metadata, orientation, 16-bit PNG, and incompatible color spaces force conversion. The source and a converted output are each fully decoded once; the output must match its format, dimensions, depth, color space, and alpha facts before its digest enters the reference. + +Batch admission prepares and verifies every master once before publishing any member. Validation failure starts no writes. Publication uses those prepared bytes directly, so a large batch does not repeat full decoding and encoding during commit. A later storage failure returns no partial references; already published immutable objects may remain unreachable under the existing storage rule. + +### Deterministic request versions + +`AttachmentStore.readImageRequest` derives a request version under route-owned total-pixel and encoded-byte budgets. Scaling is `min(1, sqrt(maxPixels / (width * height)))`, with no enlargement, followed by inward integer rounding so the encoded raster never exceeds the total-pixel cap. DeepSeek V4 Flash Vision Exp uses 640,000 total pixels and 1MiB raw encoded bytes by default; low detail uses 512 by 512 total pixels. A 2048 by 1024 master projects to 1130 by 565 under the hard cap. Request encoding uses the same color branches, with PNG (palette only without alpha) then WebP 85 and 80 for low-color input, WebP 85 then 80 for other alpha input, and JPEG 85 then 80 for other opaque input. Each fallback runs only after the previous result exceeds 1MiB, and dimensions shrink only after both quality attempts exceed it. The same derivation is used by normal agent turns, direct `ctx.llm.stream` calls, compaction, and other auxiliary streams. + +The `variantId` and cache path cover the master attachment id, transform version, route pixel and byte budgets, optional master-coordinate crop, and fixed encoder parameters. Cached output is fully decoded before reuse. DeepSeek Files and pi-ai inline base64 therefore use the same deterministic bytes for the same policy. Inline accounting uses the derived byte length after base64 expansion, not the master byte count. Equal in-process `variantId` calls share one transform and cache write; cancellation rejects only that waiter. `AttachmentStore.readImageRequests` preserves input order while the local implementation runs master and request transforms through one FIFO limiter. `imageCompressionConcurrency` is configurable from 1 through 8 and defaults to 2. Batch publication remains sequential after every master has been prepared. + +Request-size offload is a deterministic oldest-first projection. DeepSeek defaults to 128MiB and 600 referenced images. Its removed prefix advances past successive 64MiB byte boundaries and in 20-image count quanta, so 129 one-megabyte images remove the oldest 65, retain 64MiB, and keep that prefix stable until total history passes 192MiB. Pi-ai retains a configurable base64 request bound. A text-only route receives deterministic attachment placeholders, including nested tool-result images, while append-only session history keeps the original references. + +### Stable handles and master-coordinate crops + +Every retained request image is preceded by its complete attachment id, actual request dimensions, and the preview-coordinate arguments for `read_image_region`. The tool accepts only an attachment already referenced by the calling session. It maps the supplied preview rectangle to the 2048px master with floor-at-origin and ceil-at-far-edge rounding, crops the master rather than the preview, and persists the result as a new attachment. The tool result contains the new `ImageBlock`, so model-visible output and the durable log remain equivalent. + +### DeepSeek Files lifecycle + +The direct `deepseek-official` adapter uploads every retained request version through the OpenAI-compatible Files API and sends only `file_id` content blocks. There is no inline fallback. The default catalog advertises `deepseek-v4-flash-vision-exp` as image-capable. Uploaded ids are indexed by endpoint and API-key scope plus `variantId`. Uploads request seven days by default and record the returned `expires_at`; a mapping with no more than one hour remaining is replaced without a preceding retrieve call. The index never stores the API key. + +An upload is indexed only after the response returns a complete file object, matching byte count, and `expires_at`. A missing or inconsistent response leaves no local mapping, so a later request uploads again. A malformed upload index is an empty cache and is replaced on the next successful upload; filesystem I/O failures remain errors. If chat reports an expired, deleted, missing, or invalid id and names one used id, only that mapping is removed. A stale-file response without a specific id removes every mapping used by that chat attempt. The affected request bytes are uploaded again and chat is retried once. A second stale rejection clears the mappings identified by its response and returns the error without a third chat attempt. One upload quota error deletes the configured number of oldest harness-owned `dsh-` files and retries once. Public file operations expose list, retrieve, delete, one-variant release, and namespace-wide release. The client enforces the documented 128MiB upload limit, 32MiB chat-image limit, 10,000-file and 25GiB quotas, and one-hour to 30-day expiry range. + +### Diagnostics + +A 16-bit RGB or RGBA PNG is normal admitted input and converts to 8-bit sRGB/sRGBA. If local conversion fails, `read_image` names the path, detected 16-bit PNG, required canonical form, and manual conversion remedy. If DeepSeek rejects a normalized request version, the primary error names the attachment or display name, durable message and image position, normalized media type, 8-bit sRGB/sRGBA depth, dimensions, and provider message. An ambiguous multi-image rejection lists every candidate. The raw provider body remains the error cause rather than the only visible message. + +Historical attachment objects that later disappear or fail integrity verification remain fail-loud. Durable quarantine and verified recovery require session events and are tracked by [Quarantine unreadable historical attachments](../../proposed/bug-fix/2026-08-20-attachment-read-quarantine.md). + +## Alternatives considered + +**Use one 1MiB canonical image for storage and requests.** This makes model resolution determine durable quality, reduces the source for later crops, and combines local storage, inline expansion, Files quota, and model pixels into one setting. Independent master and request policies keep those responsibilities explicit. + +**Reject images above provider dimensions or at the encoding quality floor.** A provider limit is route-specific and future requests may use another model. Proportional master preparation and request projection accept ordinary large images while bounding each later representation. + +**Treat PNG as a screenshot and reject 16-bit PNG.** File format does not reveal pixel complexity, and 16-bit RGB/RGBA is a convertible sample depth rather than an unsupported image type. Pixel sampling and post-conversion probes give the required facts. + +**Keep DeepSeek data URLs.** Inline base64 repeats bytes on every request and caps usable image history by request-body size. Files API references reuse uploaded deterministic request bytes and provide explicit expiry and deletion. + +**Trust a locally indexed file id indefinitely.** Remote expiry, deletion, and lost upload responses make local and provider state diverge. Response-directed invalidation and one re-upload recover without an unbounded retry loop; an ambiguous stale-file response must invalidate every file used by that attempt because it provides no safe exact target. + +**Crop the request preview.** Repeated crops would compound the 640,000-pixel reduction and make coordinates depend on previous encodes. Mapping back to the master preserves the available local detail. + +**Refuse text-only model selection after any image.** Durable history can outlive the model that first consumed it. Request-local placeholders keep the session usable without rewriting history. + +**Remove one image whenever a request crosses its limit.** That changes an early request message after nearly every new upload. Quantized removed prefixes keep cache invalidation occasional while honoring the configured high bound. + +## Verification + +Package tests generate 16-bit RGB and RGBA PNG fixtures, prove 8-bit conversion and clean 8-bit passthrough, retain alpha under byte pressure, distinguish high-frequency and ordinary photos from low-color graphics, stop lazy encoding after the first fitting candidate, cover square and wide 640,000-pixel projections, enforce 1MiB request bytes, singleflight equal variants, bound transform concurrency, preserve cache and upload identity, map preview crops to the master, prepare batches once, reject inconsistent Files responses, refresh near-expiry ids without retrieve, recover once from exact and ambiguous stale-id responses, delete quota files, normalize provider diagnostics, project text-only history, and share normal/compaction request bytes. Keyless assembled snapshots cover the real tool schemas and image request path. A credentialed test uses the built-in `deepseek-official` route and its configured endpoint, never a custom provider entry. + +## Consequences + +Durable masters consume up to the independent local safety cap, while request caches and remote Files consume additional derived storage. Deterministic identities and singleflight make that work reusable across turns and sessions sharing the same DSH home. Two simultaneous transforms reduce batch latency while increasing peak RSS relative to serial execution; deployments with tighter memory can set the limit to one. Encoder or transform-version changes create new future identities without rewriting existing history. DeepSeek image requests now depend on Files API availability; bounded stale-id recovery handles inconsistent remote state, while a general Files outage remains a visible request failure. Missing or corrupt durable masters still require the separate quarantine design. diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md new file mode 100644 index 0000000000..a82312d55b --- /dev/null +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md @@ -0,0 +1,71 @@ +# Agent Note: 统一图片主版本、请求版本与提供方文件 + +Status: implemented + +[English](2026-08-20-unified-image-request-pipeline.md) | 中文 + +## Problem + +持久图片历史、提供方分辨率、内联请求大小和远端文件复用有不同限制。过去把已接纳图片直接作为之后每次请求发送的字节,导致一个字节上限和一份光栅同时承担四种职责。普通大图会被拒绝;干净的 16-bit PNG 可以进入历史,之后才被 DeepSeek 拒绝;重复 base64 使长会话请求持续增长;提供方拒绝后,同一持久图片还会进入每次后续请求。模型也无法稳定裁剪没有文件系统路径的用户上传图片。 + +## Decision + +图片路径有两个显式版本。附件后端拥有提供方无关的持久主版本。每条支持图片的模型路由拥有确定性请求策略,附件后端从主版本派生并缓存确切请求版本。会话历史只包含主版本引用;内联字节和提供方文件 ID 都是瞬时请求投影。 + +### 提供方无关的主版本 + +准入在可配置的 32MiB、1 亿像素和单边 16384px 源图范围内完整解码每张图片。处理会应用 EXIF 方向,删除元数据和色彩配置文件,转换为 8-bit sRGB/sRGBA,并保持宽高比把长边限制到 `masterMaxDimension`,默认 2048px。处理缩小光栅时,`sourceWidth` 和 `sourceHeight` 记录应用方向后的源尺寸。 + +主版本有独立的 `masterMaxBytes` 安全上限,默认 4MiB。透明通道绝不铺平。系统通过 nearest-neighbour 对有界样本判断色彩复杂度,不会通过像素平均把高频图片误判为低色数。确认的低色数输入先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80、75 的 WebP;其他透明输入依次尝试这些质量的 WebP;其他非透明输入依次尝试这些质量的 JPEG。候选按顺序执行,首个不超过上限的结果会立即返回。同一尺寸的候选全部超限后才会缩小尺寸。源扩展名不会把 PNG 归类为低色数图片。处于两个主版本上限内的干净、单帧、8-bit sRGB/sRGBA PNG、JPEG 或 WebP 按字节原样直通,并保留内容寻址去重。GIF、动图、元数据、方向、16-bit PNG 和不兼容色彩空间都会触发转换。源图和转换输出各完整解码一次;输出的格式、尺寸、位深、色彩空间和透明通道事实通过校验后,其摘要才会进入引用。 + +批量准入在发布任何成员前,为每张图片各准备并验证一次主版本。校验失败不会开始写入。发布直接使用这些已准备字节,因此大批次不会在提交时重复完整解码和编码。之后发生的存储失败不会返回部分引用;按现有存储规则,已经发布的不可变对象可能保持不可达。 + +### 确定性请求版本 + +`AttachmentStore.readImageRequest` 按路由拥有的总像素和编码字节预算派生请求版本。缩放公式为 `min(1, sqrt(maxPixels / (width * height)))`,不会放大小图,随后向预算内取整,确保编码光栅不超过总像素上限。DeepSeek V4 Flash Vision Exp 默认使用总像素 640,000 和原始编码字节 1MiB;low detail 使用总像素 512×512。2048×1024 主版本在这个硬上限下会投影为 1130×565。请求编码使用相同的分类分支:低色数输入先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80 的 WebP;其他透明输入依次尝试质量 85、80 的 WebP;其他非透明输入依次尝试质量 85、80 的 JPEG。只有前一结果超过 1MiB 时才执行下一个候选;两个质量档都超限后才缩小尺寸。普通 agent 轮次、直接 `ctx.llm.stream` 调用、压缩和其他辅助流都使用同一派生过程。 + +`variantId` 和缓存路径覆盖主附件 ID、变换策略版本、路由像素和字节预算、可选的主版本坐标裁剪区域及固定编码参数。缓存输出会在复用前完整解码。因此,同一策略下的 DeepSeek Files 和 pi-ai 内联 base64 使用相同的确定性字节。内联计量使用派生字节经过 base64 膨胀后的长度,不使用主版本字节数。同一进程内相同 `variantId` 的调用共享一次变换和缓存写入;取消只拒绝对应等待方。`AttachmentStore.readImageRequests` 保持输入顺序,本地实现则通过一个 FIFO 限流器运行主版本和请求版本变换。`imageCompressionConcurrency` 的可配置范围为 1 至 8,默认值为 2。全部主版本准备完成后,批次仍按顺序发布。 + +请求大小 offload 是确定性的从旧到新投影。DeepSeek 默认上限为 128MiB 和 600 张引用图片。被移除前缀会越过连续的 64MiB 字节边界,并按 20 张图片数量步长递增,因此 129 张 1MiB 图片会移除最旧的 65 张并保留 64MiB;持久历史超过 192MiB 前,该前缀保持不变。Pi-ai 保留可配置的 base64 请求上限。纯文本路由会收到确定性的附件占位文本,其中包括嵌套工具结果图片;追加式会话历史继续保留原始引用。 + +### 稳定句柄与主版本坐标裁剪 + +每张保留请求图片前都有完整附件 ID、实际请求尺寸和 `read_image_region` 所需的预览坐标参数。该工具只接受调用会话已经引用的附件。它按起点向下取整、远端边界向上取整,把提交的预览矩形映射到 2048px 主版本,从主版本而非预览图裁剪,并把结果保存为新附件。工具结果包含新的 `ImageBlock`,因此模型可见输出与持久日志保持一致。 + +### DeepSeek Files 生命周期 + +直接 `deepseek-official` 适配器通过 OpenAI 兼容 Files API 上传每张保留的请求版本,只发送 `file_id` 内容块,不提供内联回退。默认 catalog 把 `deepseek-v4-flash-vision-exp` 公布为支持图片。上传 ID 按端点和 API key 作用域以及 `variantId` 写入索引。上传默认请求 7 天有效期,并记录返回的 `expires_at`;本地映射剩余时间不超过一小时时会直接替换,不会先查询远端文件。索引绝不存储 API key。 + +只有上传响应返回完整文件对象、匹配的字节数和 `expires_at` 时,上传结果才会写入索引。缺失或不一致的响应不会留下本地映射,后续请求会重新上传。格式损坏的上传索引按空缓存处理,并在下一次成功上传时替换;文件系统 I/O 失败仍是错误。如果 chat 报告 ID 已过期、删除、缺失或无效,并指出本次请求使用的某个 ID,适配器只删除该映射。如果响应只说明文件状态失效而没有指出具体 ID,适配器会删除该次 chat 使用的全部映射。受影响的请求字节会重新上传,chat 只重试一次。第二次仍报告文件失效时,适配器会按响应清理映射并返回错误,不会发起第三次 chat。一次上传配额错误会删除配置数量的最旧 `dsh-` 文件,然后重试一次。公开文件操作提供列表、查询、删除、单个变体释放和整个作用域释放。客户端执行文档规定的 Files 单次上传 128MiB、chat 单图 32MiB、10,000 个文件、25GiB,以及一小时到 30 天有效期限制。 + +### 诊断 + +16-bit RGB 或 RGBA PNG 属于普通可接纳输入,会转换为 8-bit sRGB/sRGBA。本地转换失败时,`read_image` 会写明路径、检测到的 16-bit PNG、所需规范形式和手工转换方法。如果 DeepSeek 拒绝已规范化请求版本,主错误会写明附件 ID 或显示名称、持久消息和图片位置、规范化媒体类型、8-bit sRGB/sRGBA 位深、尺寸和提供方消息。多图片错误无法确定对象时会列出全部候选图片。原始提供方正文保留为错误 cause,不会成为唯一可见消息。 + +持久附件对象之后缺失或无法通过完整性校验时,系统仍会明确失败。持久隔离和经校验恢复需要新增会话事件,由[隔离不可读历史附件](../../proposed/bug-fix/2026-08-20-attachment-read-quarantine.md)继续跟踪。 + +## Alternatives considered + +**使用一份 1MiB 规范图片同时负责存储和请求。** 这种做法让模型分辨率决定持久质量,降低之后裁剪可用的源信息,并把本地存储、内联膨胀、Files 配额和模型像素合并成一个设置。独立的主版本和请求策略会明确区分这些职责。 + +**拒绝超过提供方尺寸或达到编码质量下限的图片。** 提供方限制属于具体路由,未来请求可能改用另一个模型。按比例准备主版本和投影请求版本可以接纳普通大图,同时约束每种后续表示。 + +**把 PNG 当作截图,并拒绝 16-bit PNG。** 文件格式不能说明像素复杂度,16-bit RGB/RGBA 是可转换位深,不是不支持的图片类型。像素采样和转换后探测能提供所需事实。 + +**继续向 DeepSeek 发送 data URL。** 内联 base64 会在每次请求中重复字节,并按请求正文大小限制可用图片历史。Files API 引用会复用上传后的确定性请求字节,并提供显式有效期和删除操作。 + +**永久信任本地索引中的文件 ID。** 远端过期、删除和上传响应丢失会使本地与提供方状态不一致。按响应失效和一次重新上传可以恢复,同时避免无界重试;响应没有给出可安全使用的精确目标时,必须使该次请求使用的全部文件失效。 + +**从请求预览图裁剪。** 重复裁剪会叠加 640,000 像素缩小,坐标也会依赖之前的编码。映射回主版本能保留本地可用细节。 + +**历史中出现图片后拒绝选择纯文本模型。** 持久历史可能比最初读取它的模型存活更久。按请求生成的占位文本可以保持会话可用,无需改写历史。 + +**请求每次越过上限就移除一张图片。** 这种做法会在几乎每次新增图片后改写较早的请求消息。按固定步长递增的移除前缀会降低缓存失效频率,同时遵守配置的上限。 + +## Verification + +包测试会生成 16-bit RGB 和 RGBA PNG fixture,验证 8-bit 转换与干净 8-bit 字节直通、字节压力下保留透明通道、区分高频和普通照片与低色数图形、首个候选合规后停止编码、正方形和宽屏 640,000 像素投影、请求字节不超过 1MiB、相同变体 singleflight、变换并发上限、缓存与上传身份、预览到主版本坐标映射、批量只准备一次、Files 响应不一致、进入刷新余量时不查询远端并更新 ID、精确和模糊失效响应只恢复一次、配额删除、规范化提供方诊断、纯文本投影,以及普通请求与压缩共享请求字节。无需密钥的组装快照覆盖真实工具 schema 和图片请求路径。使用凭据的测试只使用内置 `deepseek-official` 路由及其已配置端点,不使用自定义提供方条目。 + +## Consequences + +持久主版本最多占用独立的本地安全上限,请求缓存和远端 Files 还会占用额外派生存储。确定性身份和 singleflight 使这些成本可以被共享同一 DSH home 的轮次和会话复用。同时执行两个变换会降低批次延迟,但峰值 RSS 高于串行执行;内存更紧张的部署可以把上限设为 1。编码器或变换策略版本变化会为未来内容产生新身份,不会改写已有历史。DeepSeek 图片请求现在依赖 Files API 可用性;有界的陈旧 ID 恢复会处理远端状态不一致,一般 Files 故障仍会成为可见请求失败。缺失或损坏的持久主版本仍需要单独的隔离设计。 diff --git a/docs/config-catalog.i18n.yaml b/docs/config-catalog.i18n.yaml index a17400de81..3523b633cd 100644 --- a/docs/config-catalog.i18n.yaml +++ b/docs/config-catalog.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/config-catalog.md -config-catalog.md: c3ae3421d52c2a6c4432b6c7784c1bae53625a24 -config-catalog.zh.md: 4e6b57a42e3269935cae8765cd0c7998c39115f4 +config-catalog.md: dd91a870ecb338e784acdd1ffa0a470fa33d8813 +config-catalog.zh.md: a412a4f0afe652863cda1edad0e344b17e1697ac diff --git a/docs/config-catalog.md b/docs/config-catalog.md index c3ae3421d5..dd91a870ec 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -337,14 +337,16 @@ export interface Config { maxImagePixels?: number /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. */ maxImageDimension?: number - /** Long-edge pixel target of the stored canonical encoding. */ - canonicalMaxDimension?: number - /** Encoded-byte target of the stored canonical encoding. */ - canonicalMaxBytes?: number + /** Long-edge pixel cap of the stored provider-independent master version. */ + masterMaxDimension?: number + /** Encoded-byte safety cap of the stored provider-independent master version. */ + masterMaxBytes?: number + /** Maximum simultaneous master or request-image transformations in this service instance. */ + imageCompressionConcurrency?: number } ``` -Source: [`packages/attachment/attachment-local/src/index.ts:36`](../packages/attachment/attachment-local/src/index.ts) +Source: [`packages/attachment/attachment-local/src/index.ts:53`](../packages/attachment/attachment-local/src/index.ts) @@ -936,8 +938,20 @@ export interface Config { models?: DeepSeekCatalogModel[] /** Maximum provider idle time while one stream read is outstanding (default five minutes). */ streamIdleTimeoutMs?: number - /** Maximum accumulated base64 image payload per request (default 20 MiB). */ - maxRequestImageBytes?: number + /** Maximum accumulated file-referenced image bytes per chat request (default 128 MiB). */ + maxRequestFilesBytes?: number + /** Maximum number of file-referenced images per chat request (default 600). */ + maxImagesPerRequest?: number + /** Raw-byte removal step after the request exceeds its file bound (default 64 MiB). */ + imageOffloadByteQuantum?: number + /** Image-count removal step after the request exceeds its count bound (default 20). */ + imageOffloadCountQuantum?: number + /** Explicit lifetime assigned to each uploaded image (default seven days). */ + fileExpiresAfterSeconds?: number + /** Remaining lifetime below which an indexed file is replaced (default one hour). */ + fileRefreshMarginSeconds?: number + /** Oldest harness-owned files deleted before one quota-recovery upload retry (default 100). */ + fileQuotaCleanupBatch?: number /** Provider-owned model-request retry policy; omission uses normal mode with five retries. */ retryPolicy?: RetryPolicyConfig } @@ -956,12 +970,18 @@ export interface DeepSeekCatalogModel { maxTokens?: number /** Accepted request modalities; omission is text-only. */ inputModalities?: ModelModality[] + /** Total-pixel budget for one deterministic request preview. */ + imagePixelBudget?: number + /** Encoded-byte cap for one deterministic request preview. */ + imageMaxBytes?: number + /** Provider detail tier; `low` uses the 512-by-512 total-pixel default. */ + imageDetail?: 'auto' | 'low' } ``` Depends on: [`ModelModality`](../packages/llm/llm/src/index.ts) · [`RetryPolicyConfig`](../packages/llm/llm/src/index.ts) -Source: [`packages/llm/llm-deepseek/src/index.ts:72`](../packages/llm/llm-deepseek/src/index.ts) +Source: [`packages/llm/llm-deepseek/src/index.ts:100`](../packages/llm/llm-deepseek/src/index.ts) @@ -1063,6 +1083,10 @@ export interface PiAiProviderProfile { * requests instead of being rejected by a request-size cap. */ maxRequestImageBytes?: number + /** Total-pixel budget for each deterministic inline request version. */ + requestImagePixelBudget?: number + /** Raw encoded-byte cap for each deterministic inline request version. */ + requestImageMaxBytes?: number /** Provider-owned model-request retry policy; omission uses normal mode with five retries. */ retryPolicy?: RetryPolicyConfig } @@ -1211,7 +1235,7 @@ export type PiAiThinkingFormat = NonNullable diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 4e6b57a42e..a412a4f0af 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -339,14 +339,16 @@ export interface Config { maxImagePixels?: number /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. */ maxImageDimension?: number - /** Long-edge pixel target of the stored canonical encoding. */ - canonicalMaxDimension?: number - /** Encoded-byte target of the stored canonical encoding. */ - canonicalMaxBytes?: number + /** Long-edge pixel cap of the stored provider-independent master version. */ + masterMaxDimension?: number + /** Encoded-byte safety cap of the stored provider-independent master version. */ + masterMaxBytes?: number + /** Maximum simultaneous master or request-image transformations in this service instance. */ + imageCompressionConcurrency?: number } ``` -来源:[`packages/attachment/attachment-local/src/index.ts:36`](../packages/attachment/attachment-local/src/index.ts) +来源:[`packages/attachment/attachment-local/src/index.ts:53`](../packages/attachment/attachment-local/src/index.ts) @@ -938,8 +940,20 @@ export interface Config { models?: DeepSeekCatalogModel[] /** Maximum provider idle time while one stream read is outstanding (default five minutes). */ streamIdleTimeoutMs?: number - /** Maximum accumulated base64 image payload per request (default 20 MiB). */ - maxRequestImageBytes?: number + /** Maximum accumulated file-referenced image bytes per chat request (default 128 MiB). */ + maxRequestFilesBytes?: number + /** Maximum number of file-referenced images per chat request (default 600). */ + maxImagesPerRequest?: number + /** Raw-byte removal step after the request exceeds its file bound (default 64 MiB). */ + imageOffloadByteQuantum?: number + /** Image-count removal step after the request exceeds its count bound (default 20). */ + imageOffloadCountQuantum?: number + /** Explicit lifetime assigned to each uploaded image (default seven days). */ + fileExpiresAfterSeconds?: number + /** Remaining lifetime below which an indexed file is replaced (default one hour). */ + fileRefreshMarginSeconds?: number + /** Oldest harness-owned files deleted before one quota-recovery upload retry (default 100). */ + fileQuotaCleanupBatch?: number /** Provider-owned model-request retry policy; omission uses normal mode with five retries. */ retryPolicy?: RetryPolicyConfig } @@ -958,12 +972,18 @@ export interface DeepSeekCatalogModel { maxTokens?: number /** Accepted request modalities; omission is text-only. */ inputModalities?: ModelModality[] + /** Total-pixel budget for one deterministic request preview. */ + imagePixelBudget?: number + /** Encoded-byte cap for one deterministic request preview. */ + imageMaxBytes?: number + /** Provider detail tier; `low` uses the 512-by-512 total-pixel default. */ + imageDetail?: 'auto' | 'low' } ``` 依赖:[`ModelModality`](../packages/llm/llm/src/index.ts) · [`RetryPolicyConfig`](../packages/llm/llm/src/index.ts) -来源:[`packages/llm/llm-deepseek/src/index.ts:72`](../packages/llm/llm-deepseek/src/index.ts) +来源:[`packages/llm/llm-deepseek/src/index.ts:100`](../packages/llm/llm-deepseek/src/index.ts) @@ -1065,6 +1085,10 @@ export interface PiAiProviderProfile { * requests instead of being rejected by a request-size cap. */ maxRequestImageBytes?: number + /** Total-pixel budget for each deterministic inline request version. */ + requestImagePixelBudget?: number + /** Raw encoded-byte cap for each deterministic inline request version. */ + requestImageMaxBytes?: number /** Provider-owned model-request retry policy; omission uses normal mode with five retries. */ retryPolicy?: RetryPolicyConfig } @@ -1213,7 +1237,7 @@ export type PiAiThinkingFormat = NonNullable diff --git a/docs/event-producer-consumer.i18n.yaml b/docs/event-producer-consumer.i18n.yaml index 9aef71c868..c37d706383 100644 --- a/docs/event-producer-consumer.i18n.yaml +++ b/docs/event-producer-consumer.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/event-producer-consumer.md -event-producer-consumer.md: d68f92d317dec2fd05813c1ce487bb88c369bd6e -event-producer-consumer.zh.md: 5b3454e6000f4e1e017cb494423736f2c0f75f31 +event-producer-consumer.md: 1fb65d55f5a0d8121f4f171c956196fde746103f +event-producer-consumer.zh.md: d8db21e5266f83a5fc403a9b825bc05530e61b8d diff --git a/docs/event-producer-consumer.md b/docs/event-producer-consumer.md index d68f92d317..1fb65d55f5 100644 --- a/docs/event-producer-consumer.md +++ b/docs/event-producer-consumer.md @@ -38,7 +38,7 @@ This matrix shows which packages dispatch each harness-owned event and which pac | `fs/write-intent` | `waterfall` | [`packages/fs/fs/src/index.ts:58`](../packages/fs/fs/src/index.ts) | [`tool-fs`](../packages/fs/tool-fs) (`waterfall`), [`tool-str-replace-editor`](../packages/fs/tool-str-replace-editor) (`waterfall`) | [`fs-observation-policy`](../packages/fs/fs-observation-policy) | | `goal/changed` | `emit` | [`packages/goal/goal/src/domain.ts:114`](../packages/goal/goal/src/domain.ts) | [`goal`](../packages/goal/goal) (`emit`) | [`goal-round-driver`](../packages/goal/goal-round-driver) | | `llm/adapters-updated` | `emit` | [`packages/llm/llm/src/types.ts:23`](../packages/llm/llm/src/types.ts) | [`llm`](../packages/llm/llm) (`events.dispatch`) | `apiproxy`, [`llm`](../packages/llm/llm) | -| `llm/stream` | `waterfall` | [`packages/llm/llm/src/index.ts:64`](../packages/llm/llm/src/index.ts) | [`llm`](../packages/llm/llm) (`waterfall`) | [`agent-loop`](../packages/core/agent-loop), [`llm`](../packages/llm/llm), [`llm-replay`](../packages/test-support/llm-replay), [`session-checkpoint-policy`](../packages/session/session-checkpoint-policy), [`session-title`](../packages/session/session-title) | +| `llm/stream` | `waterfall` | [`packages/llm/llm/src/index.ts:65`](../packages/llm/llm/src/index.ts) | [`llm`](../packages/llm/llm) (`waterfall`) | [`agent-loop`](../packages/core/agent-loop), [`llm`](../packages/llm/llm), [`llm-replay`](../packages/test-support/llm-replay), [`session-checkpoint-policy`](../packages/session/session-checkpoint-policy), [`session-title`](../packages/session/session-title) | | `session-telemetry/record` | `waterfall` | [`packages/session/session-telemetry/src/index.ts:43`](../packages/session/session-telemetry/src/index.ts) | [`session-telemetry`](../packages/session/session-telemetry) (`waterfall`) | - | | `session/created` | `emit` | [`packages/core/session/src/index.ts:54`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | `apiproxy`, [`compaction`](../packages/compaction/compaction), [`goal`](../packages/goal/goal), [`hook-protocol`](../packages/hooks/hook-protocol), [`llm-retry`](../packages/llm/llm-retry), [`permission-presets`](../packages/interaction/permission-presets), [`plan-mode`](../packages/plan/plan-mode), [`schedule`](../packages/schedule/schedule), `server`, [`session`](../packages/core/session), [`session-persistence`](../packages/session/session-persistence), [`session-telemetry`](../packages/session/session-telemetry), [`time-context`](../packages/context/time-context), [`tool-workflow`](../packages/workflow/tool-workflow), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval) | | `session/disposed` | `emit` | [`packages/core/session/src/index.ts:64`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`agent-loop`](../packages/core/agent-loop), `agent-team`, `apiproxy`, [`session-persistence`](../packages/session/session-persistence), [`session-projection-cache`](../packages/session/session-projection-cache), [`session-telemetry`](../packages/session/session-telemetry), [`session-title`](../packages/session/session-title) | diff --git a/docs/event-producer-consumer.zh.md b/docs/event-producer-consumer.zh.md index 5b3454e600..d8db21e526 100644 --- a/docs/event-producer-consumer.zh.md +++ b/docs/event-producer-consumer.zh.md @@ -40,7 +40,7 @@ | `fs/write-intent` | `waterfall` | [`packages/fs/fs/src/index.ts:58`](../packages/fs/fs/src/index.ts) | [`tool-fs`](../packages/fs/tool-fs) (`waterfall`), [`tool-str-replace-editor`](../packages/fs/tool-str-replace-editor) (`waterfall`) | [`fs-observation-policy`](../packages/fs/fs-observation-policy) | | `goal/changed` | `emit` | [`packages/goal/goal/src/domain.ts:114`](../packages/goal/goal/src/domain.ts) | [`goal`](../packages/goal/goal) (`emit`) | [`goal-round-driver`](../packages/goal/goal-round-driver) | | `llm/adapters-updated` | `emit` | [`packages/llm/llm/src/types.ts:23`](../packages/llm/llm/src/types.ts) | [`llm`](../packages/llm/llm) (`events.dispatch`) | `apiproxy`, [`llm`](../packages/llm/llm) | -| `llm/stream` | `waterfall` | [`packages/llm/llm/src/index.ts:64`](../packages/llm/llm/src/index.ts) | [`llm`](../packages/llm/llm) (`waterfall`) | [`agent-loop`](../packages/core/agent-loop), [`llm`](../packages/llm/llm), [`llm-replay`](../packages/test-support/llm-replay), [`session-checkpoint-policy`](../packages/session/session-checkpoint-policy), [`session-title`](../packages/session/session-title) | +| `llm/stream` | `waterfall` | [`packages/llm/llm/src/index.ts:65`](../packages/llm/llm/src/index.ts) | [`llm`](../packages/llm/llm) (`waterfall`) | [`agent-loop`](../packages/core/agent-loop), [`llm`](../packages/llm/llm), [`llm-replay`](../packages/test-support/llm-replay), [`session-checkpoint-policy`](../packages/session/session-checkpoint-policy), [`session-title`](../packages/session/session-title) | | `session-telemetry/record` | `waterfall` | [`packages/session/session-telemetry/src/index.ts:43`](../packages/session/session-telemetry/src/index.ts) | [`session-telemetry`](../packages/session/session-telemetry) (`waterfall`) | - | | `session/created` | `emit` | [`packages/core/session/src/index.ts:54`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | `apiproxy`, [`compaction`](../packages/compaction/compaction), [`goal`](../packages/goal/goal), [`hook-protocol`](../packages/hooks/hook-protocol), [`llm-retry`](../packages/llm/llm-retry), [`permission-presets`](../packages/interaction/permission-presets), [`plan-mode`](../packages/plan/plan-mode), [`schedule`](../packages/schedule/schedule), `server`, [`session`](../packages/core/session), [`session-persistence`](../packages/session/session-persistence), [`session-telemetry`](../packages/session/session-telemetry), [`time-context`](../packages/context/time-context), [`tool-workflow`](../packages/workflow/tool-workflow), [`tools`](../packages/core/tools), [`user-approval`](../packages/interaction/user-approval) | | `session/disposed` | `emit` | [`packages/core/session/src/index.ts:64`](../packages/core/session/src/index.ts) | [`session`](../packages/core/session) (`events.dispatch`) | [`agent-loop`](../packages/core/agent-loop), `agent-team`, `apiproxy`, [`session-persistence`](../packages/session/session-persistence), [`session-projection-cache`](../packages/session/session-projection-cache), [`session-telemetry`](../packages/session/session-telemetry), [`session-title`](../packages/session/session-title) | diff --git a/docs/subsystems/attachment.i18n.yaml b/docs/subsystems/attachment.i18n.yaml index f904af9a27..7c236d6480 100644 --- a/docs/subsystems/attachment.i18n.yaml +++ b/docs/subsystems/attachment.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/subsystems/attachment.md -attachment.md: 780d4744dc7ca8cada6209476cd208cf8ef95bc2 -attachment.zh.md: 843eca1c4deda9d3499207a2d0f163e401a50c9b +attachment.md: cdbb528d30eabc74a9c3607d67e91af053c45e7e +attachment.zh.md: 79ee753d22c3adecaca153659181e113f2b3e728 diff --git a/docs/subsystems/attachment.md b/docs/subsystems/attachment.md index 780d4744dc..cdbb528d30 100644 --- a/docs/subsystems/attachment.md +++ b/docs/subsystems/attachment.md @@ -32,6 +32,10 @@ interface ImageAttachmentRef { height: number /** Optional display name stripped of local path information. */ name?: string + /** Perceived source width before master-version downscaling; present only when it differs from {@link width}. */ + sourceWidth?: number + /** Perceived source height before master-version downscaling; present only when it differs from {@link height}. */ + sourceHeight?: number } ``` @@ -83,7 +87,65 @@ interface StoredImageAttachment { } ``` -`saveImage()` validates bytes and atomically commits one object before returning its reference. `validateImage()` runs the same admission checks without persisting anything; batch callers validate every member through it before saving any member, so validation rejection leaves no partial objects behind. `admitEncodedImages()` is the wire entry for base64 uploads: it enforces canonical base64, then delegates batch admission to `saveImages()`, which owns the count and aggregate-byte limits and the validate-all-before-save order. `readImage()` accepts a reference from an authorized session path and returns bytes only after integrity verification. The service is deliberately retention-neutral: resumed and forked sessions may share objects, so reference-aware garbage collection is deferred rather than tied to any one session's deletion. +```ts type-equiv +/** Pixel rectangle in the oriented 2048px master-version coordinate system. */ +interface MasterImageCrop { + x: number + y: number + width: number + height: number +} +``` + +```ts type-equiv +/** Deterministic request-image policy selected by one exact model route. */ +interface ImageRequestPolicy { + /** Maximum width multiplied by height after aspect-preserving projection. */ + maxPixels: number + /** Encoded-byte cap before base64 expansion or Files API upload. */ + maxBytes: number + /** Optional master-coordinate crop applied before pixel-budget scaling. */ + crop?: MasterImageCrop +} +``` + +```ts type-equiv +/** Crop coordinates measured by a model on the request preview it received. */ +interface PreviewImageCrop { + previewWidth: number + previewHeight: number + x: number + y: number + width: number + height: number +} +``` + +```ts type-equiv +/** Cached request version derived from one provider-independent master attachment. */ +interface RequestImageAttachment { + /** Cache and upload-index key over the master id, policy, crop, and fixed encoder parameters. */ + variantId: ImageVariantId + /** Durable master reference from which this request version was derived. */ + master: ImageAttachmentRef + /** Encoded request bytes. */ + data: Uint8Array + mediaType: ImageMediaType + bytes: number + width: number + height: number + /** Provider-compatible sample depth proven after request encoding. */ + depth: 'uchar' + /** Provider-compatible color space proven after request encoding. */ + space: 'srgb' + /** Whether the encoded request version retains an alpha channel. */ + hasAlpha: boolean + /** Applied master-coordinate crop, when present. */ + crop?: MasterImageCrop +} +``` + +`saveImage()` prepares a provider-independent 2048px, 4MiB master and atomically commits it before returning its reference. `saveImages()` prepares every validated master once before publishing the batch, so validation rejection leaves no partial objects and publication does not repeat decoding or quality selection. `admitEncodedImages()` is the wire entry for base64 uploads and delegates count, aggregate-byte, and ordered batch admission to `saveImages()`. `readImage()` verifies a master from an authorized session path. `readImageRequest()` derives and caches one request version under an exact route pixel and byte budget; `readImageRequests()` lets an implementation apply its configured bounded transform concurrency to an ordered batch. The local implementation lazily encodes preferred candidates, singleflights equal request identities, and defaults to two simultaneous transformations. `cropImage()` maps model preview coordinates back to the master and returns another durable attachment. The service is retention-neutral: resumed and forked sessions may share objects, so reference-aware garbage collection is deferred rather than tied to one session's deletion. @@ -109,18 +171,15 @@ Immutable binary attachment service. Implementations validate bytes before publi abstract validateImage(input: SaveImageAttachment): Promise /** - * Validate one ordered image batch before committing any member. - * Validation failures start no writes; storage failures return no partial - * references, although already published content-addressed objects may stay - * unreachable until a future retention policy collects them. - * @param inputs - encoded images in their owning message order. - * @returns durable references in the exact input order. + * Validate and durably commit one ordered image batch. + * @param inputs - encoded images in owning-message order. + * @returns durable master references in the same order after every member succeeds. */ async saveImages(inputs: readonly SaveImageAttachment[]): Promise /** * Validate and durably commit one image before its owning session event is appended. - * Implementations may store a canonical re-encoding of the submitted raster; + * Implementations may store a prepared master version of the submitted raster; * the returned reference always describes the stored bytes, while `source` * preserves the submitted raster's intrinsic facts for callers that report * or map coordinates against the original. @@ -133,10 +192,38 @@ abstract saveImage(input: SaveImageAttachment): Promise * Read one image and verify that bytes still match the recorded reference. * @param ref - durable reference from the session log. * @param signal - optional cancellation for backend read and verification work. - * @returns the verified bytes and canonical reference. + * @returns the verified bytes and master reference. * @throws the signal reason when aborted, or a storage error when verification fails. */ abstract readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise + +/** + * Generate or read one deterministic model-request version from the stored master image. + * @param ref - durable provider-independent master reference. + * @param policy - exact route pixel and encoded-byte budget. + * @param signal - optional cancellation. + * @returns request bytes and the cache/upload identity covering every transform input. + */ +async readImageRequest( ref: ImageAttachmentRef, policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise + +/** + * Generate or read an ordered batch of deterministic model-request versions. + * Implementations may use their own bounded transform concurrency while preserving input order. + * @param refs - durable provider-independent master references in request order. + * @param policy - exact route pixel and encoded-byte budget shared by the batch. + * @param signal - optional cancellation. + * @returns request versions in the same order as `refs`. + */ +async readImageRequests( refs: readonly ImageAttachmentRef[], policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise + +/** + * Crop the stored master by coordinates measured on a model request preview and persist the result. + * @param ref - session-authorized master attachment. + * @param crop - preview dimensions and preview-coordinate rectangle. + * @param signal - optional cancellation. + * @returns a new durable attachment reference suitable for a logged tool result. + */ +async cropImage( ref: ImageAttachmentRef, crop: PreviewImageCrop, signal?: AbortSignal, ): Promise ``` Source: [`packages/attachment/attachment/src/index.ts`](../../packages/attachment/attachment/src/index.ts) diff --git a/docs/subsystems/attachment.zh.md b/docs/subsystems/attachment.zh.md index 843eca1c4d..79ee753d22 100644 --- a/docs/subsystems/attachment.zh.md +++ b/docs/subsystems/attachment.zh.md @@ -32,6 +32,10 @@ interface ImageAttachmentRef { height: number /** Optional display name stripped of local path information. */ name?: string + /** Perceived source width before master-version downscaling; present only when it differs from {@link width}. */ + sourceWidth?: number + /** Perceived source height before master-version downscaling; present only when it differs from {@link height}. */ + sourceHeight?: number } ``` @@ -83,7 +87,65 @@ interface StoredImageAttachment { } ``` -`saveImage()` 校验字节并以原子方式提交一个对象,之后才返回其引用。`validateImage()` 执行相同的准入检查,但不持久化任何内容;批量调用方会在保存任何成员前通过它校验所有成员,因此校验拒绝不会留下部分对象。`admitEncodedImages()` 是面向 base64 上传的 wire 入口:强制执行规范 base64,随后把批量准入委托给 `saveImages()`,由后者负责张数与聚合字节上限以及先全量校验再保存的顺序。`readImage()` 接受来自已授权会话路径的引用,只在完整性校验通过后返回字节。该服务刻意不规定保留策略:恢复和 fork 后的会话可能共享对象,因此基于引用的垃圾回收会延期实现,而不是与任何一个会话的删除绑定。 +```ts type-equiv +/** Pixel rectangle in the oriented 2048px master-version coordinate system. */ +interface MasterImageCrop { + x: number + y: number + width: number + height: number +} +``` + +```ts type-equiv +/** Deterministic request-image policy selected by one exact model route. */ +interface ImageRequestPolicy { + /** Maximum width multiplied by height after aspect-preserving projection. */ + maxPixels: number + /** Encoded-byte cap before base64 expansion or Files API upload. */ + maxBytes: number + /** Optional master-coordinate crop applied before pixel-budget scaling. */ + crop?: MasterImageCrop +} +``` + +```ts type-equiv +/** Crop coordinates measured by a model on the request preview it received. */ +interface PreviewImageCrop { + previewWidth: number + previewHeight: number + x: number + y: number + width: number + height: number +} +``` + +```ts type-equiv +/** Cached request version derived from one provider-independent master attachment. */ +interface RequestImageAttachment { + /** Cache and upload-index key over the master id, policy, crop, and fixed encoder parameters. */ + variantId: ImageVariantId + /** Durable master reference from which this request version was derived. */ + master: ImageAttachmentRef + /** Encoded request bytes. */ + data: Uint8Array + mediaType: ImageMediaType + bytes: number + width: number + height: number + /** Provider-compatible sample depth proven after request encoding. */ + depth: 'uchar' + /** Provider-compatible color space proven after request encoding. */ + space: 'srgb' + /** Whether the encoded request version retains an alpha channel. */ + hasAlpha: boolean + /** Applied master-coordinate crop, when present. */ + crop?: MasterImageCrop +} +``` + +`saveImage()` 准备提供方无关的 2048px、4MiB 主版本,并在返回引用前以原子方式提交。`saveImages()` 在发布批次前为每个成员各准备一次经过验证的主版本,因此校验拒绝不会留下部分对象,发布也不会重复解码或选择质量。`admitEncodedImages()` 是面向 base64 上传的 wire 入口,把张数、聚合字节和有序批量准入交给 `saveImages()`。`readImage()` 校验来自已授权会话路径的主版本。`readImageRequest()` 按确切路由的像素和字节预算派生并缓存请求版本;`readImageRequests()` 允许实现按自身配置的有界变换并发处理有序批次。本地实现按需编码首选候选、合并相同请求身份的并发任务,默认同时执行两项变换。`cropImage()` 把模型预览坐标映射回主版本,并返回另一个持久附件。该服务不规定保留策略:恢复和 fork 后的会话可能共享对象,因此基于引用的垃圾回收会延期实现,不与单个会话的删除绑定。 @@ -109,18 +171,15 @@ Immutable binary attachment service. Implementations validate bytes before publi abstract validateImage(input: SaveImageAttachment): Promise /** - * Validate one ordered image batch before committing any member. - * Validation failures start no writes; storage failures return no partial - * references, although already published content-addressed objects may stay - * unreachable until a future retention policy collects them. - * @param inputs - encoded images in their owning message order. - * @returns durable references in the exact input order. + * Validate and durably commit one ordered image batch. + * @param inputs - encoded images in owning-message order. + * @returns durable master references in the same order after every member succeeds. */ async saveImages(inputs: readonly SaveImageAttachment[]): Promise /** * Validate and durably commit one image before its owning session event is appended. - * Implementations may store a canonical re-encoding of the submitted raster; + * Implementations may store a prepared master version of the submitted raster; * the returned reference always describes the stored bytes, while `source` * preserves the submitted raster's intrinsic facts for callers that report * or map coordinates against the original. @@ -133,10 +192,38 @@ abstract saveImage(input: SaveImageAttachment): Promise * Read one image and verify that bytes still match the recorded reference. * @param ref - durable reference from the session log. * @param signal - optional cancellation for backend read and verification work. - * @returns the verified bytes and canonical reference. + * @returns the verified bytes and master reference. * @throws the signal reason when aborted, or a storage error when verification fails. */ abstract readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise + +/** + * Generate or read one deterministic model-request version from the stored master image. + * @param ref - durable provider-independent master reference. + * @param policy - exact route pixel and encoded-byte budget. + * @param signal - optional cancellation. + * @returns request bytes and the cache/upload identity covering every transform input. + */ +async readImageRequest( ref: ImageAttachmentRef, policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise + +/** + * Generate or read an ordered batch of deterministic model-request versions. + * Implementations may use their own bounded transform concurrency while preserving input order. + * @param refs - durable provider-independent master references in request order. + * @param policy - exact route pixel and encoded-byte budget shared by the batch. + * @param signal - optional cancellation. + * @returns request versions in the same order as `refs`. + */ +async readImageRequests( refs: readonly ImageAttachmentRef[], policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise + +/** + * Crop the stored master by coordinates measured on a model request preview and persist the result. + * @param ref - session-authorized master attachment. + * @param crop - preview dimensions and preview-coordinate rectangle. + * @param signal - optional cancellation. + * @returns a new durable attachment reference suitable for a logged tool result. + */ +async cropImage( ref: ImageAttachmentRef, crop: PreviewImageCrop, signal?: AbortSignal, ): Promise ``` Source: [`packages/attachment/attachment/src/index.ts`](../../packages/attachment/attachment/src/index.ts) diff --git a/docs/subsystems/llm-streaming.i18n.yaml b/docs/subsystems/llm-streaming.i18n.yaml index 5c118e9782..cd1adacbe3 100644 --- a/docs/subsystems/llm-streaming.i18n.yaml +++ b/docs/subsystems/llm-streaming.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/subsystems/llm-streaming.md -llm-streaming.md: 4f322ca1024b9d74a4906e34f93fc6f8e4082cbf -llm-streaming.zh.md: c74cabe27c1f5fdd44711ac0aae7cd6b0a7ba7dd +llm-streaming.md: 1b2356983be4045666f7a9d40d8d191bdb4910a2 +llm-streaming.zh.md: 0c2b64830dda74595deac7797c759be1970ccb32 diff --git a/docs/subsystems/llm-streaming.md b/docs/subsystems/llm-streaming.md index 4f322ca102..1b2356983b 100644 --- a/docs/subsystems/llm-streaming.md +++ b/docs/subsystems/llm-streaming.md @@ -675,6 +675,8 @@ interface PreparedLlmCall { readonly retryPolicy: ResolvedRetryPolicy /** Detached context metadata resolved with the registration-bound call. */ readonly context?: LlmModelContext + /** Exact model modalities captured with the adapter dispatch generation. */ + readonly inputModalities?: readonly ModelModality[] /** Config fields materialized by the captured adapter rather than proposed by the caller. */ readonly adapterDefaults: LlmCallConfigAdapterDefaults /** @@ -730,6 +732,16 @@ declare abstract class LlmAdapter { model: string, _signal?: AbortSignal, ): Promise; + /** + * Bind exact model metadata and the eventual request dispatch to one adapter generation. + * Dynamic adapters override this so settings changes between preparation and + * dispatch cannot combine one generation's capabilities with another's endpoint. + * @param provider - registered provider route. + * @param model - exact model id. + * @param signal - cancellation for model resolution. + * @returns model metadata and a one-generation stream entry point. + */ + async prepareCall(provider: string, model: string, signal?: AbortSignal): Promise; /** * Stream one model call as raw chunks. The only required method. * @param options - the fully-assembled request; implementations must honor `options.signal`. diff --git a/docs/subsystems/llm-streaming.zh.md b/docs/subsystems/llm-streaming.zh.md index c74cabe27c..0c2b64830d 100644 --- a/docs/subsystems/llm-streaming.zh.md +++ b/docs/subsystems/llm-streaming.zh.md @@ -681,6 +681,8 @@ interface PreparedLlmCall { readonly retryPolicy: ResolvedRetryPolicy /** Detached context metadata resolved with the registration-bound call. */ readonly context?: LlmModelContext + /** Exact model modalities captured with the adapter dispatch generation. */ + readonly inputModalities?: readonly ModelModality[] /** Config fields materialized by the captured adapter rather than proposed by the caller. */ readonly adapterDefaults: LlmCallConfigAdapterDefaults /** @@ -736,6 +738,16 @@ declare abstract class LlmAdapter { model: string, _signal?: AbortSignal, ): Promise; + /** + * Bind exact model metadata and the eventual request dispatch to one adapter generation. + * Dynamic adapters override this so settings changes between preparation and + * dispatch cannot combine one generation's capabilities with another's endpoint. + * @param provider - registered provider route. + * @param model - exact model id. + * @param signal - cancellation for model resolution. + * @returns model metadata and a one-generation stream entry point. + */ + async prepareCall(provider: string, model: string, signal?: AbortSignal): Promise; /** * Stream one model call as raw chunks. The only required method. * @param options - the fully-assembled request; implementations must honor `options.signal`. diff --git a/docs/tool-catalog.i18n.yaml b/docs/tool-catalog.i18n.yaml index 9dfb15cdf6..d219a4c8ad 100644 --- a/docs/tool-catalog.i18n.yaml +++ b/docs/tool-catalog.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/tool-catalog.md -tool-catalog.md: 92b6d8b92050d2dc822f016c18d31a81b43ef447 -tool-catalog.zh.md: e57eaf0a74c4a3cb5858d991a73decc694c7abc0 +tool-catalog.md: 11a7aead7938fca40d20096e3689890258fbe31c +tool-catalog.zh.md: f29d489441b36318523e0afa2eeab9104e639fd0 diff --git a/docs/tool-catalog.md b/docs/tool-catalog.md index 92b6d8b920..11a7aead79 100644 --- a/docs/tool-catalog.md +++ b/docs/tool-catalog.md @@ -24,7 +24,7 @@ This table connects model-visible tool names to the plugin package and service s | `@deepseek-ai/dsh-tool-bash-persistent` | `bash` | `ctx.tools`, `ctx.terminals`, `an owning Agent at execution time` | `tool/call`, `PTY shell state`, `tool/result` | - | One owner-isolated persistent bash tool; deployment composition supplies the PTY backend and may override the model-facing environment description. | | `@deepseek-ai/dsh-tool-pwsh-persistent` | `pwsh` | `ctx.tools`, `ctx.terminals`, `an owning Agent at execution time` | `tool/call`, `PTY shell state`, `tool/result` | - | One owner-isolated persistent pwsh tool, the Windows counterpart of the persistent bash tool; deployment composition supplies a pwsh-dialect PTY backend and may override the model-facing environment description. | | `@deepseek-ai/dsh-tool-str-replace-editor` | `str_replace_editor` | `ctx.tools`, `ctx.fs` | `tool/call`, `fs/observed after view presence/absence, edit absence, or successful mutation`, `tool/result` | - | Standalone view/create/unique literal replace/line insert tool over the filesystem seam; it composes with any shell or terminal API. | -| `@deepseek-ai/dsh-tool-fs` | `edit`, `read`, `read_image`, `write` | `ctx.tools`, `ctx.fs`, `ctx.systemPrompt`, `ctx.attachments (read_image registration)`, `ctx.llm + an image-capable route (read_image execution)` | `tool/call`, `fs/write-intent or fs/edit-intent for mutations`, `fs/observed after read presence/absence or successful file operation`, `durable attachment (read_image)`, `tool/result` | - | The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-observation-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. `read_image` is not registered without `ctx.attachments`; its schema is route-independent, and execution refuses unless the exact routed model declares image input. | +| `@deepseek-ai/dsh-tool-fs` | `edit`, `read`, `read_image`, `read_image_region`, `write` | `ctx.tools`, `ctx.fs`, `ctx.systemPrompt`, `ctx.attachments (image-tool registration)`, `ctx.llm + an image-capable route (image-tool execution)` | `tool/call`, `fs/write-intent or fs/edit-intent for mutations`, `fs/observed after read presence/absence or successful file operation`, `durable attachment (read_image and read_image_region)`, `tool/result` | - | The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-observation-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. The image tools are not registered without `ctx.attachments`; their schemas are route-independent, and execution refuses unless the exact routed model declares image input. | | `@deepseek-ai/dsh-tool-fs-search` | `glob`, `grep` | `ctx.tools`, `ctx.subprocess`, `ctx.systemPrompt` | `tool/call`, `tool/result` | - | glob and grep are unconditional discovery tools that spawn the packaged ripgrep binary (`@vscode/ripgrep`) through ctx.subprocess as ordinary foreground calls (never background jobs) — no host `rg` install and no shell layer. The catalog uses `sampleOverCapGlobResults: true`; deployments must choose that behavior explicitly. Capped results save the complete formatted list through the optional ctx.spillStore backend; returned locators are follow-up-readable/searchable when the backend exposes local paths in co-located deployments. | | `@deepseek-ai/dsh-tool-terminal` | `terminal_close`, `terminal_list`, `terminal_open`, `terminal_read`, `terminal_send`, `terminal_signal` | `ctx.tools`, `ctx.terminals`, `ctx.systemPrompt`, `ctx.jobs at call time for run_in_background` | `tool/call`, `tool/result` | - | The six terminal tools are opt-in and complement one-shot shell/filesystem tools. `terminal_send(run_in_background: true)` registers with `ctx.jobs`; TUI, named key sequences, BEL, resize, auto-start, and cross-agent sharing are absent from the schema. | | `@deepseek-ai/dsh-tool-goal` | `create_goal`, `get_goal`, `update_goal` | `ctx.tools`, `ctx.agents`, `ctx.goals`, `ctx.systemPrompt`, `a calling Agent in an authorized open turn` | `tool/call`, `goal/change for mutations`, `tool/result` | - | create, edit, pause, and resume require direct-human root authority; complete and blocked also accept the exact current goal round. The default blocked lower bound is three admitted rounds. | @@ -714,6 +714,57 @@ Read a PNG/JPEG/WebP/GIF file and return the image itself. Requires the current Source: [`packages/fs/tool-fs/src/index.ts`](../packages/fs/tool-fs/src/index.ts) +### `read_image_region` + +Crop a region from an image attachment already visible in this session. Coordinates use the preview dimensions supplied beside that image. + +```json +{ + "type": "object", + "properties": { + "attachment_id": { + "type": "string", + "description": "Complete attachment id shown beside the image." + }, + "preview_width": { + "type": "integer", + "description": "Width of the preview shown to the model." + }, + "preview_height": { + "type": "integer", + "description": "Height of the preview shown to the model." + }, + "x": { + "type": "integer", + "description": "Left edge in preview pixels." + }, + "y": { + "type": "integer", + "description": "Top edge in preview pixels." + }, + "width": { + "type": "integer", + "description": "Crop width in preview pixels." + }, + "height": { + "type": "integer", + "description": "Crop height in preview pixels." + } + }, + "required": [ + "attachment_id", + "preview_width", + "preview_height", + "x", + "y", + "width", + "height" + ] +} +``` + +Source: [`packages/fs/tool-fs/src/index.ts`](../packages/fs/tool-fs/src/index.ts) + ### `write` Create or fully replace a UTF-8 text file. @@ -740,7 +791,7 @@ Create or fully replace a UTF-8 text file. Source: [`packages/fs/tool-fs/src/index.ts`](../packages/fs/tool-fs/src/index.ts) -The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-observation-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. `read_image` is not registered without `ctx.attachments`; its schema is route-independent, and execution refuses unless the exact routed model declares image input. +The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-observation-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. The image tools are not registered without `ctx.attachments`; their schemas are route-independent, and execution refuses unless the exact routed model declares image input. diff --git a/docs/tool-catalog.zh.md b/docs/tool-catalog.zh.md index e57eaf0a74..f29d489441 100644 --- a/docs/tool-catalog.zh.md +++ b/docs/tool-catalog.zh.md @@ -28,7 +28,7 @@ | `@deepseek-ai/dsh-tool-bash-persistent` | `bash` | `ctx.tools`、`ctx.terminals`、`an owning Agent at execution time` | `tool/call`、`PTY shell state`、`tool/result` | - | 一个按所有者隔离的持久 bash 工具;部署组合提供 PTY 后端,并可覆盖面向模型的环境描述。 | | `@deepseek-ai/dsh-tool-pwsh-persistent` | `pwsh` | `ctx.tools`、`ctx.terminals`、`an owning Agent at execution time` | `tool/call`、`PTY shell state`、`tool/result` | - | 一个按所有者隔离的持久 pwsh 工具,持久 bash 工具的 Windows 对应物;部署组合提供 pwsh 方言的 PTY 后端,并可覆盖面向模型的环境描述。 | | `@deepseek-ai/dsh-tool-str-replace-editor` | `str_replace_editor` | `ctx.tools`、`ctx.fs` | `tool/call`、`fs/observed after view presence/absence, edit absence, or successful mutation`、`tool/result` | - | 基于文件系统 seam 的独立查看/创建/唯一字面量替换/按行插入工具;可与任何 shell 或终端接口组合。 | -| `@deepseek-ai/dsh-tool-fs` | `edit`、`read`、`read_image`、`write` | `ctx.tools`、`ctx.fs`、`ctx.systemPrompt`、`ctx.attachments (read_image registration)`、`ctx.llm + an image-capable route (read_image execution)` | `tool/call`、`fs/write-intent or fs/edit-intent for mutations`、`fs/observed after read presence/absence or successful file operation`、`durable attachment (read_image)`、`tool/result` | - | 先读后写/编辑策略由 `@deepseek-ai/dsh-fs-observation-policy` 添加;它是一个 `fs/*` 事件门禁插件,不会改变 schema。加载这些工具的部署按预期也应加载该插件。没有 `ctx.attachments` 时 `read_image` 不会注册;其 schema 与路由无关,执行时除非确切路由的模型声明图像输入,否则拒绝。 | +| `@deepseek-ai/dsh-tool-fs` | `edit`、`read`、`read_image`、`read_image_region`、`write` | `ctx.tools`、`ctx.fs`、`ctx.systemPrompt`、`ctx.attachments (image-tool registration)`、`ctx.llm + an image-capable route (image-tool execution)` | `tool/call`、`fs/write-intent or fs/edit-intent for mutations`、`fs/observed after read presence/absence or successful file operation`、`durable attachment (read_image and read_image_region)`、`tool/result` | - | 先读后写/编辑策略由 `@deepseek-ai/dsh-fs-observation-policy` 添加;它是一个 `fs/*` 事件门禁插件,不会改变 schema。加载这些工具的部署按预期也应加载该插件。没有 `ctx.attachments` 时图片工具不会注册;其 schema 与路由无关,执行时除非确切路由的模型声明图片输入,否则拒绝。 | | `@deepseek-ai/dsh-tool-fs-search` | `glob`、`grep` | `ctx.tools`、`ctx.subprocess`、`ctx.systemPrompt` | `tool/call`、`tool/result` | - | glob 和 grep 是无条件可用的发现工具,通过 ctx.subprocess spawn 随包提供的 ripgrep 二进制文件(`@vscode/ripgrep`),并作为普通前台调用运行,绝不作为后台任务;无需在宿主机安装 `rg`,也不经过 shell 层。本目录使用 `sampleOverCapGlobResults: true`;部署必须显式选择该行为。结果超过上限时,会通过可选的 ctx.spillStore 后端保存完整的格式化列表;在共置部署中,如果后端公开本地路径,返回的定位信息可供后续读取/搜索。 | | `@deepseek-ai/dsh-tool-terminal` | `terminal_close`、`terminal_list`、`terminal_open`、`terminal_read`、`terminal_send`、`terminal_signal` | `ctx.tools`、`ctx.terminals`、`ctx.systemPrompt`、`ctx.jobs at call time for run_in_background` | `tool/call`、`tool/result` | - | 这 6 个终端工具需要选择启用,用于补充一次性 bash/文件系统工具。`terminal_send(run_in_background: true)` 会注册到 `ctx.jobs`;schema 不包含 TUI、具名按键序列、BEL、调整尺寸、自动启动和跨 agent 共享。 | | `@deepseek-ai/dsh-tool-goal` | `create_goal`、`get_goal`、`update_goal` | `ctx.tools`、`ctx.agents`、`ctx.goals`、`ctx.systemPrompt`、`a calling Agent in an authorized open turn` | `tool/call`、`goal/change for mutations`、`tool/result` | - | create、edit、pause 和 resume 要求直接来自人类的根权限;complete 和 blocked 也接受确切的当前 Goal Round。blocked 的默认下限是 3 个获准的 Round。 | @@ -720,6 +720,57 @@ pwsh 工具是 Windows 组合中 bash 执行器 seam 的 PowerShell 方言消费 来源:[`packages/fs/tool-fs/src/index.ts`](../packages/fs/tool-fs/src/index.ts) +### `read_image_region` + +裁剪当前会话中模型已经可见的图片附件。坐标采用该图片旁给出的预览尺寸。 + +```json +{ + "type": "object", + "properties": { + "attachment_id": { + "type": "string", + "description": "Complete attachment id shown beside the image." + }, + "preview_width": { + "type": "integer", + "description": "Width of the preview shown to the model." + }, + "preview_height": { + "type": "integer", + "description": "Height of the preview shown to the model." + }, + "x": { + "type": "integer", + "description": "Left edge in preview pixels." + }, + "y": { + "type": "integer", + "description": "Top edge in preview pixels." + }, + "width": { + "type": "integer", + "description": "Crop width in preview pixels." + }, + "height": { + "type": "integer", + "description": "Crop height in preview pixels." + } + }, + "required": [ + "attachment_id", + "preview_width", + "preview_height", + "x", + "y", + "width", + "height" + ] +} +``` + +来源:[`packages/fs/tool-fs/src/index.ts`](../packages/fs/tool-fs/src/index.ts) + ### `write` 创建或完全替换 UTF-8 文本文件。 @@ -746,7 +797,7 @@ pwsh 工具是 Windows 组合中 bash 执行器 seam 的 PowerShell 方言消费 来源:[`packages/fs/tool-fs/src/index.ts`](../packages/fs/tool-fs/src/index.ts) -先读后写/编辑策略由 `@deepseek-ai/dsh-fs-observation-policy` 添加;它是一个 `fs/*` 事件门禁插件,不会改变 schema。加载这些工具的部署按预期也应加载该插件。没有 `ctx.attachments` 时 `read_image` 不会注册;其 schema 与路由无关,执行时除非确切路由的模型声明图像输入,否则拒绝。 +先读后写/编辑策略由 `@deepseek-ai/dsh-fs-observation-policy` 添加;它是一个 `fs/*` 事件门禁插件,不会改变 schema。加载这些工具的部署按预期也应加载该插件。没有 `ctx.attachments` 时图片工具不会注册;其 schema 与路由无关,执行时除非确切路由的模型声明图片输入,否则拒绝。 diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index a6f12fdad5..8da7ac71b1 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -702,30 +702,63 @@ defineAcpSnapshotSuite({ hasPwsh, }) -it('pins native DeepSeek image offload in the request sent by the assembled app', async () => { +it('pins native DeepSeek Files image offload in the request sent by the assembled app', async () => { const requests: Record[] = [] + const fileRequests: Array<{ method: string; path: string; bytes: number }> = [] const server = createServer((request: IncomingMessage, response: ServerResponse) => { - let body = '' - request.setEncoding('utf8') - request.on('data', (chunk: string) => { body += chunk }) + const chunks: Buffer[] = [] + request.on('data', (chunk: Buffer) => { chunks.push(chunk) }) request.on('end', () => { - requests.push(JSON.parse(body) as Record) - response.writeHead(200, { 'content-type': 'text/event-stream' }) - const events = requests.length === 1 - ? [ - 'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"native-read-image","type":"function","function":{"name":"read_image","arguments":"{\\"file_path\\":\\"red.png\\"}"}}]},"index":0,"finish_reason":null}]}', - 'data: {"choices":[{"delta":{},"index":0,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}', - 'data: [DONE]', - '', - ] - : [ - 'data: {"choices":[{"delta":{"role":"assistant","content":""},"index":0,"finish_reason":null}]}', - 'data: {"choices":[{"delta":{"content":"DONE"},"index":0,"finish_reason":null}]}', - 'data: {"choices":[{"delta":{},"index":0,"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}', - 'data: [DONE]', - '', - ] - response.end(events.join('\n\n')) + void (async () => { + const url = new URL(request.url ?? '/', 'http://localhost') + const body = Buffer.concat(chunks) + if (url.pathname === '/files' && request.method === 'POST') { + const headers = new Headers() + for (const [name, value] of Object.entries(request.headers)) { + if (value !== undefined) headers.set(name, Array.isArray(value) ? value.join(', ') : value) + } + const form = await new Request('http://localhost/files', { + method: 'POST', headers, body, + }).formData() + const file = form.get('file') + if (!(file instanceof Blob)) throw new Error('snapshot Files upload omitted file') + fileRequests.push({ method: 'POST', path: url.pathname, bytes: file.size }) + const createdAt = Math.floor(Date.now() / 1_000) + response.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ + id: 'file-api-snapshot-1', + object: 'file', + bytes: file.size, + created_at: createdAt, + filename: 'dsh-snapshot.png', + purpose: 'user_data', + expires_at: createdAt + Number(form.get('expires_after[seconds]')), + })) + return + } + if (url.pathname !== '/chat/completions') { + response.writeHead(404).end() + return + } + requests.push(JSON.parse(body.toString('utf8')) as Record) + response.writeHead(200, { 'content-type': 'text/event-stream' }) + const events = requests.length === 1 + ? [ + 'data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"native-read-image","type":"function","function":{"name":"read_image","arguments":"{\\"file_path\\":\\"red.png\\"}"}}]},"index":0,"finish_reason":null}]}', + 'data: {"choices":[{"delta":{},"index":0,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}', + 'data: [DONE]', + '', + ] + : [ + 'data: {"choices":[{"delta":{"role":"assistant","content":""},"index":0,"finish_reason":null}]}', + 'data: {"choices":[{"delta":{"content":"DONE"},"index":0,"finish_reason":null}]}', + 'data: {"choices":[{"delta":{},"index":0,"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}', + 'data: [DONE]', + '', + ] + response.end(events.join('\n\n')) + })().catch((error: unknown) => { + response.writeHead(500, { 'content-type': 'text/plain' }).end(String(error)) + }) }) }) await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) @@ -764,34 +797,22 @@ it('pins native DeepSeek image offload in the request sent by the assembled app' }) expect(result.stderr).toBe('') expect(requests).toHaveLength(2) + expect(fileRequests).toEqual([{ method: 'POST', path: '/files', bytes: 69 }]) const messages = requests[0]?.messages as { content?: unknown }[] | undefined const offloaded = messages?.find(message => JSON.stringify(message.content).includes('[image omitted')) - expect(offloaded?.content).toMatchInlineSnapshot(` - [ - { - "text": "Compare the older image ", - "type": "text", - }, - { - "text": "[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]", - "type": "text", - }, - { - "text": " with the newer image ", - "type": "text", - }, - { - "image_url": { - "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC", - }, - "type": "image_url", - }, - { - "text": ", then use read_image on red.png and reply with DONE.", - "type": "text", - }, - ] - `) + expect(offloaded?.content).toEqual([ + { type: 'text', text: 'Compare the older image ' }, + { type: 'text', text: OFFLOADED_IMAGE_TEXT }, + { type: 'text', text: ' with the newer image ' }, + { + type: 'text', + text: '\nImage sha256:b1ff9c8ea3a780bad09b346c423d2d0e46815926879b18e841d928376a946640; ' + + 'preview 1x1px. Crop coordinates use this preview. Call read_image_region with this attachment_id, ' + + 'preview_width=1, preview_height=1, x, y, width, and height.', + }, + { type: 'file', file_id: 'file-api-snapshot-1' }, + { type: 'text', text: ', then use read_image on red.png and reply with DONE.' }, + ]) const followup = structuredClone((requests[1]?.messages as unknown[]).slice(1)) as Array<{ role?: unknown @@ -830,16 +851,16 @@ it('pins native DeepSeek image offload in the request sent by the assembled app' { role: 'tool', tool_call_id: 'native-read-image', - content: '{{cwd}}/red.png\nimage\n\nimage/png image, 1x1 px, 69 bytes\n', + content: '{{cwd}}/red.png\nimage\n\nimage/png image, 1x1 px, 69 bytes\n' + + '\nImage sha256:b1ff9c8ea3a780bad09b346c423d2d0e46815926879b18e841d928376a946640; preview 1x1px. ' + + 'Crop coordinates use this preview. Call read_image_region with this attachment_id, preview_width=1, ' + + 'preview_height=1, x, y, width, and height.', }, { role: 'user', content: [ { type: 'text', text: 'Attached image(s) from tool result:' }, - { - type: 'image_url', - image_url: { url: `data:image/png;base64,${image}` }, - }, + { type: 'file', file_id: 'file-api-snapshot-1' }, ], }, ]) diff --git a/examples/acp-agent/tests/fixtures/image-offload.cordis.yml b/examples/acp-agent/tests/fixtures/image-offload.cordis.yml index 530f7b9663..320e66fe06 100644 --- a/examples/acp-agent/tests/fixtures/image-offload.cordis.yml +++ b/examples/acp-agent/tests/fixtures/image-offload.cordis.yml @@ -12,7 +12,8 @@ apiKeyEnv: DSH_SNAPSHOT_API_KEY baseURL: !!js process.env.DSH_SNAPSHOT_BASE_URL thinking: disabled - maxRequestImageBytes: 92 + maxRequestFilesBytes: 92 + imageOffloadByteQuantum: 1 models: - id: deepseek-v4-flash-vision-exp contextWindow: 32768 diff --git a/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md b/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md index e3fdc4ace2..7408ddb329 100644 --- a/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md +++ b/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md @@ -130,6 +130,23 @@ interface ToolArgsMap { /** Path to the image file, resolved by the filesystem backend. */ file_path: string; } & Record; + /** Crop a region from an image attachment already visible in this session. Coordinates use the preview dimensions supplied beside that image. */ + read_image_region: { + /** Complete attachment id shown beside the image. */ + attachment_id: string; + /** Width of the preview shown to the model. */ + preview_width: number; + /** Height of the preview shown to the model. */ + preview_height: number; + /** Left edge in preview pixels. */ + x: number; + /** Top edge in preview pixels. */ + y: number; + /** Crop width in preview pixels. */ + width: number; + /** Crop height in preview pixels. */ + height: number; + } & Record; /** Send a message to a background subagent by its subagent id, continuing the same conversation. It becomes the subagent's next turn: if it is still working, the message waits until its current turn finishes, so it cannot redirect work already underway. This call returns no answer from the subagent — only confirmation that the message was delivered — so use it to give it more work. A failure means the message was NOT delivered. */ send_message: { /** The subagent id returned when the background subagent was started. */ @@ -367,6 +384,29 @@ interface ToolOutputMap { sourceHeight?: number; }; }; + read_image_region: { + sourceAttachmentId: string; + preview: { + width: number; + height: number; + }; + crop: { + x: number; + y: number; + width: number; + height: number; + }; + image: { + attachmentId: string; + mediaType: "image/png" | "image/jpeg" | "image/webp" | "image/gif"; + bytes: number; + width: number; + height: number; + name?: string; + sourceWidth?: number; + sourceHeight?: number; + }; + }; send_message: { messageId: string; }; diff --git a/examples/acp-agent/tests/snapshots/read-image/tool-schemas.expected.json b/examples/acp-agent/tests/snapshots/read-image/tool-schemas.expected.json index cce80e04c8..dec4bd85ab 100644 --- a/examples/acp-agent/tests/snapshots/read-image/tool-schemas.expected.json +++ b/examples/acp-agent/tests/snapshots/read-image/tool-schemas.expected.json @@ -260,6 +260,52 @@ ] } }, + { + "name": "read_image_region", + "description": "Crop a region from an image attachment already visible in this session. Coordinates use the preview dimensions supplied beside that image.", + "parameters": { + "type": "object", + "properties": { + "attachment_id": { + "type": "string", + "description": "Complete attachment id shown beside the image." + }, + "preview_width": { + "type": "integer", + "description": "Width of the preview shown to the model." + }, + "preview_height": { + "type": "integer", + "description": "Height of the preview shown to the model." + }, + "x": { + "type": "integer", + "description": "Left edge in preview pixels." + }, + "y": { + "type": "integer", + "description": "Top edge in preview pixels." + }, + "width": { + "type": "integer", + "description": "Crop width in preview pixels." + }, + "height": { + "type": "integer", + "description": "Crop height in preview pixels." + } + }, + "required": [ + "attachment_id", + "preview_width", + "preview_height", + "x", + "y", + "width", + "height" + ] + } + }, { "name": "send_message", "description": "Send a message to a background subagent by its subagent id, continuing the same conversation. It becomes the subagent's next turn: if it is still working, the message waits until its current turn finishes, so it cannot redirect work already underway. This call returns no answer from the subagent — only confirmation that the message was delivered — so use it to give it more work. A failure means the message was NOT delivered.", diff --git a/packages/attachment/attachment-local/README.i18n.yaml b/packages/attachment/attachment-local/README.i18n.yaml index 4393ddbe9d..0ebf6a80fe 100644 --- a/packages/attachment/attachment-local/README.i18n.yaml +++ b/packages/attachment/attachment-local/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/attachment/attachment-local/README.md -README.md: afa38ccc125f4fb36d35bb4b94b1aea278107551 -README.zh.md: 9de7ce65447a91741810bbcd41d397a70275247b +README.md: 77b68357d5a961549bef0a015b8e48ba02fbd702 +README.zh.md: 05932c93e40d42a7f8fcdcf906f6669f6f8f7073 diff --git a/packages/attachment/attachment-local/README.md b/packages/attachment/attachment-local/README.md index afa38ccc12..77b68357d5 100644 --- a/packages/attachment/attachment-local/README.md +++ b/packages/attachment/attachment-local/README.md @@ -2,7 +2,11 @@ English | [中文](README.zh.md) -The private local implementation of [`@deepseek-ai/dsh-attachment`](../attachment). Objects land at `/attachments/v1/objects//` and are addressed by an opaque `sha256:` id. Each process proves a home durable once by syncing every ancestor entry to the filesystem root, so a directory another process created but has not yet synced is never mistaken for a safe boundary. Writes then use a private staging directory, owner-only files, a synced temporary file, an atomic exclusive hard-link publish, and directory syncs on the publication path (POSIX; Windows relies on filesystem metadata journaling) so the reported reference survives a crash. Write admission fully decodes the raster against a wide source envelope — byte, total-pixel, and per-side caps (defaults 32MiB, 100MP, 16384px) — and then persists a deterministic canonical encoding instead of the submitted bytes: EXIF orientation is baked into pixels, metadata is stripped, the long edge is downscaled to the configured canonical target (default 2048px), sources with alpha or PNG/GIF lineage encode as palette PNG and photographic sources as JPEG, stepping down a fixed quality ladder (85/75/60/45) until the configured canonical byte target holds (default 1MiB). A PNG/JPEG/WebP source already inside the canonical budget passes through byte-identically only when it is a single frame and carries no EXIF/XMP/IPTC metadata and no non-default orientation, so equal originals keep deduplicating to one content address while location and device metadata never survive admission; GIF and every animated or metadata-carrying source re-encodes, and GIF always becomes the PNG of its first frame, pinning at admission the first-frame meaning providers apply. Encoder parameters are deliberately fixed rather than configurable, because a parameter change would silently split the content-addressed space; the deployment chooses only the source envelope and the canonical budget. An admitted image rides every later request of its session, so canonicalizing at admission is what bounds durable history without refusing ordinary large sources. `validateImage` runs the same policy including a canonical-encoding dry run, so a validated batch can never be refused mid-write by the byte target. Reads re-check the digest and logged metadata, and a later policy reduction does not make already-admitted history unreadable. +The private local implementation of [`@deepseek-ai/dsh-attachment`](../attachment). Objects land at `/attachments/v1/objects//` and are addressed by an opaque `sha256:` id. Each process proves a home durable once by syncing every ancestor entry to the filesystem root. Writes use a private staging directory, owner-only files, a synced temporary file, an atomic exclusive hard-link publish, and directory syncs on the publication path (POSIX; Windows relies on filesystem metadata journaling) so the reported reference survives a crash. + +Admission fully decodes the raster against a wide source envelope: 32MiB, 100MP, and 16384px per side by default. It then prepares a provider-independent master. EXIF orientation is applied, metadata and color profiles are removed, pixels become 8-bit sRGB/sRGBA, and the long edge is reduced proportionally to `masterMaxDimension` (2048px by default). The master has its own `masterMaxBytes` safety cap (4MiB by default). Alpha is retained. A nearest-neighbour bounded sample classifies color complexity without averaging high-frequency pixels. Confirmed low-color images try PNG, using a palette only when the input has no alpha channel, then WebP at qualities 85, 80, and 75. Other alpha images try WebP at those qualities; other opaque images try JPEG. Each candidate runs only after the preceding candidate exceeds the cap. Dimensions shrink only after every candidate at one size exceeds the cap. A clean, single-frame 8-bit sRGB/sRGBA PNG, JPEG, or WebP already within both master limits passes through byte-identically; 16-bit PNG, GIF, animated input, metadata, orientation, and incompatible color spaces force conversion. The source and a converted master are each fully decoded once. `saveImages` prepares and verifies every master once before publishing the batch, so validation failure leaves no partial references and commit does not repeat full image encoding. + +Request versions live below `/attachments/v1/request-images/`. `readImageRequest` scales the stored master under a total-pixel budget without enlargement, then enforces a separate encoded-byte cap. The request encoder uses the same color branches, with PNG (palette only without alpha) before WebP 85 and 80 for low-color images, WebP 85 then 80 for other alpha images, and JPEG 85 then 80 for other opaque images. It also executes candidates lazily and reduces dimensions only after both quality attempts exceed the request cap. Its cache identity includes the master id, transform version, pixel and byte budgets, optional master-coordinate crop, and fixed encoder settings. Cached bytes are fully decoded and checked as 8-bit sRGB/sRGBA before use. Concurrent calls for one identity share one transform and cache write; cancelling one waiter does not cancel the shared work. `readImageRequests` schedules batches through the service's FIFO limiter. `imageCompressionConcurrency` controls simultaneous master and request transforms from 1 through 8 and defaults to 2; file publication remains ordered after preparation. `cropImage` maps coordinates measured on a model preview back to the master, crops the master rather than the preview, and commits the crop as another durable attachment. `DSH_HOME` resolves through the shared path policy: explicit config, `$DSH_HOME`, then `~/.dsh`. Session logs contain only the reference and verified metadata, never this host path. `readImage` forwards optional cancellation into the filesystem read, observes it around verification, and preserves it instead of wrapping it as `ATTACHMENT_READ_FAILED`. @@ -12,11 +16,11 @@ Indirectly, through durable replay of historical user images and structured mode #### KV Cache effect -Canonicalization happens once at admission and is deterministic, so a stored image contributes identical request bytes on every later turn; nothing here re-encodes per request. +Master preparation and request projection are deterministic. An unchanged master and route policy reuse identical cached request bytes on later turns. ## Known Limitations and Deferred Work - Objects are retained indefinitely; reference-aware garbage collection is deferred. - The local backend assumes the host and provider adapter share this filesystem service. - Animated GIF sources keep only their first frame; animation is outside the version-one image contract. -- The canonical encoder is pinned by the installed sharp/libvips build; an encoder upgrade re-addresses future saves of the same source while already-stored objects stay valid. +- The master and request encoders are pinned by the installed sharp/libvips build; an encoder or transform-version upgrade re-addresses future masters or request variants while existing objects stay valid. diff --git a/packages/attachment/attachment-local/README.zh.md b/packages/attachment/attachment-local/README.zh.md index 9de7ce6544..05932c93e4 100644 --- a/packages/attachment/attachment-local/README.zh.md +++ b/packages/attachment/attachment-local/README.zh.md @@ -2,7 +2,11 @@ [English](README.md) | 中文 -这是 [`@deepseek-ai/dsh-attachment`](../attachment) 的私有本地实现。对象存放在 `/attachments/v1/objects//`,并通过不透明的 `sha256:` 标识符寻址。每个进程都会通过将每个祖先目录项逐级同步到文件系统根目录,为某个 home 一次性证明其持久性,因此绝不会把另一个进程已经创建但尚未同步的目录误认为安全边界。随后,写入过程使用私有暂存目录、仅所有者可访问的文件、经过同步的临时文件、原子且排他的硬链接发布,并对发布路径执行目录同步(适用于 POSIX;Windows 依赖文件系统元数据日志),确保已报告的引用能够在崩溃后继续存在。写入准入会按宽松的源图上限(字节、总像素、单边,默认 32MiB、1 亿像素、16384px)完整解码光栅图片,然后持久保存确定性的规范编码而不是提交的原始字节:EXIF 方向落实到像素并剥离元数据,长边等比缩放到配置的规范目标(默认 2048px),带透明通道或源自 PNG/GIF 的图片编码为 palette PNG,摄影类图片编码为 JPEG,并沿固定的质量阶梯(85/75/60/45)递降,直到满足配置的规范字节目标(默认 1MiB)。已在规范预算内的 PNG/JPEG/WebP 源图只有在单帧且不携带 EXIF/XMP/IPTC 元数据、方向为默认值时才按字节原样直通,因此相同原图始终去重到同一个内容地址,而位置与设备元数据绝不会越过准入;GIF 以及任何动图或携带元数据的源图都会重编码,GIF 一律变为其首帧的 PNG,在准入时就固化提供方实际采用的首帧语义。编码器参数刻意固定而不可配置,因为参数变化会悄悄割裂内容寻址空间;部署只选择源图上限与规范预算。一张已接纳的图片会随会话之后的每次请求发送,所以在准入时规范化才能在不拒绝普通大图的前提下约束持久历史。`validateImage` 执行同一套策略并包含规范编码的干跑,因此通过校验的批次绝不会在写入中途被字节目标拒绝。读取会重新校验摘要和已记录的元数据,后续收紧限制不会导致已经接纳的历史记录变得不可读。 +这是 [`@deepseek-ai/dsh-attachment`](../attachment) 的私有本地实现。对象存放在 `/attachments/v1/objects//`,并通过不透明的 `sha256:` 标识符寻址。每个进程都会把每级祖先目录项同步到文件系统根目录,以此一次性证明 home 已持久化。写入使用私有暂存目录、仅所有者可访问的文件、经过同步的临时文件、原子且排他的硬链接发布,并对发布路径执行目录同步(适用于 POSIX;Windows 依赖文件系统元数据日志),确保已报告的引用能够在崩溃后继续存在。 + +准入针对宽松的源图范围完整解码光栅,默认上限为 32MiB、1 亿像素和单边 16384px。随后生成提供方无关的主版本:应用 EXIF 方向,删除元数据和色彩配置文件,转换为 8-bit sRGB/sRGBA,并保持宽高比把长边限制到 `masterMaxDimension`(默认 2048px)。主版本有独立的 `masterMaxBytes` 安全上限(默认 4MiB)。透明通道会保留。系统用 nearest-neighbour 对有界样本分类,不会通过像素平均把高频图片误判为低色数。确认的低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80、75 的 WebP;其他透明图片依次尝试这些质量的 WebP;其他非透明图片依次尝试这些质量的 JPEG。只有前一个候选超限时才会执行下一个候选;同一尺寸的候选全部超限后才缩小尺寸。已经处于两个主版本上限内的干净、单帧、8-bit sRGB/sRGBA PNG、JPEG 或 WebP 按字节原样直通;16-bit PNG、GIF、动图、元数据、方向和不兼容色彩空间都会触发转换。源图和转换后的主版本各完整解码一次。`saveImages` 在发布任何批次成员前为每张图片各准备并验证一次主版本,因此校验失败不会留下部分引用,提交阶段也不会重复执行完整图片编码。 + +请求版本保存在 `/attachments/v1/request-images/`。`readImageRequest` 在不放大小图的前提下,把存储的主版本缩放到总像素预算内,再执行独立的编码字节上限。请求编码器使用同一分类分支:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。候选仍按需执行,两个质量档均超限后才缩小尺寸。缓存身份包含主版本 ID、变换策略版本、像素和字节预算、可选的主版本坐标裁剪区域以及固定编码参数。缓存字节在使用前会完整解码并校验为 8-bit sRGB/sRGBA。同一身份的并发调用共享一次变换和缓存写入;取消一个等待方不会取消共享任务。`readImageRequests` 通过服务的 FIFO 限流器调度批次。`imageCompressionConcurrency` 控制同时执行的主版本和请求版本变换,范围为 1 至 8,默认值为 2;文件发布仍在准备结束后按顺序执行。`cropImage` 把模型在预览图上测得的坐标映射回主版本,从主版本而非预览图裁剪,并把裁剪结果提交为另一个持久附件。 `DSH_HOME` 按共享路径策略解析:显式配置、`$DSH_HOME`,最后是 `~/.dsh`。会话日志只包含引用和经过校验的元数据,绝不包含这个宿主路径。`readImage` 会把可选取消信号传入文件系统读取、在校验前后观察该信号,并保留取消语义,而不会将其包装成 `ATTACHMENT_READ_FAILED`。 @@ -12,11 +16,11 @@ #### KV 缓存影响 -规范化只在准入时发生一次且是确定性的,因此一张已存储的图片在之后每一轮贡献完全相同的请求字节;这里没有任何按请求重编码的环节。 +主版本准备和请求投影都是确定性的。主版本和路由策略不变时,之后各轮会复用相同的缓存请求字节。 ## 已知限制与待完成工作 - 对象会无限期保留;基于引用的垃圾回收尚未实现。 - 本地后端假定宿主与提供方适配器共享同一个文件系统服务。 - 动态 GIF 源图只保留首帧;动画在版本一图片契约之外。 -- 规范编码器由安装的 sharp/libvips 构建钉定;编码器升级会让同一源图之后的保存得到新地址,已存储对象保持有效。 +- 主版本和请求版本编码器由安装的 sharp/libvips 构建钉定;编码器或变换策略版本升级会让未来的主版本或请求变体产生新地址,已有对象保持有效。 diff --git a/packages/attachment/attachment-local/src/canonical.ts b/packages/attachment/attachment-local/src/canonical.ts index db4295c404..8a4193aaed 100644 --- a/packages/attachment/attachment-local/src/canonical.ts +++ b/packages/attachment/attachment-local/src/canonical.ts @@ -1,111 +1,201 @@ -/** - * Deterministic canonical image encoding. Admission stores this encoding, so - * the same source bytes always publish the same content address on one - * runtime: encoder parameters are fixed here, never configurable, because a - * parameter change would silently split the content-addressed space. The - * deployment chooses only the canonical budget (long edge and byte target). - */ +/** Deterministic provider-independent master-image encoding. */ import sharp, { type Sharp } from 'sharp' import { AttachmentError } from '@deepseek-ai/dsh-attachment' import type { ImageMediaType } from '@deepseek-ai/dsh-attachment' +import { encodeFirstWithinLimit, isExhaustedEncoding } from './encoding.ts' +import { detectImage } from './image.ts' import type { DetectedImage } from './image.ts' -/** Deployment-resolved canonical encoding budget. */ -export interface CanonicalImagePolicy { - /** Long-edge target in pixels; a larger source is downscaled proportionally. */ +/** Deployment-resolved storage policy for the provider-independent master version. */ +export interface MasterImagePolicy { + /** Long-edge cap in pixels; larger sources are downscaled proportionally. */ maxDimension: number - /** Encoded-byte target; a larger encoding falls down the fixed quality ladder. */ + /** Independent safety cap for encoded master bytes. */ maxBytes: number } -/** Canonical bytes beside the facts a durable reference records about them. */ -export interface CanonicalImage { +/** Master bytes beside the facts recorded by a durable reference. */ +export interface MasterImage { data: Uint8Array mediaType: ImageMediaType width: number height: number } -/** JPEG quality ladder tried in order once the preferred encoding exceeds the byte target. */ -const JPEG_QUALITIES = [85, 75, 60, 45] as const +const MASTER_QUALITIES = [85, 80, 75] as const +const LOW_COLOUR_SAMPLE_EDGE = 128 +const LOW_COLOUR_LIMIT = 256 +const MIN_SCALE_STEP = 0.9 -/** Encode one prepared pipeline and report the exact output facts. */ -async function encode(pipeline: Sharp, mediaType: 'image/png' | 'image/jpeg'): Promise { - const { data, info } = await pipeline.toBuffer({ resolveWithObject: true }) +/** Encode one prepared pipeline and report exact output facts. */ +async function encode( + pipeline: Sharp, + mediaType: 'image/png' | 'image/jpeg' | 'image/webp', + quality?: number, + palette = true, +): Promise { + const encoded = mediaType === 'image/png' + ? pipeline.png({ compressionLevel: 9, palette }) + : mediaType === 'image/webp' + ? pipeline.webp({ quality }) + : pipeline.jpeg({ quality }) + const { data, info } = await encoded.toBuffer({ resolveWithObject: true }) return { data: new Uint8Array(data), mediaType, width: info.width, height: info.height } } /** - * Whether stored bytes may be the submitted bytes unchanged. Byte-identical - * passthrough is preferred whenever the source already fits the budget and - * carries nothing the canonical form forbids: it keeps re-submissions of the - * same original deduplicating to the same object and never re-encodes what no - * policy requires changing. Excluded from passthrough — and therefore always - * re-encoded — are GIF and any animated container (only the first frame is - * model-visible, so admission pins that meaning instead of letting each - * provider drop frames differently) and any source carrying EXIF/XMP/IPTC - * metadata or a non-default orientation (stored objects ride every later - * request, so location and device metadata must not survive admission, and a - * stored orientation would let the recorded dimensions diverge from the - * pixels a model perceives). - * @param detected - verified source format, dimensions, and metadata facts. - * @param bytes - submitted encoded byte length. - * @param policy - resolved canonical budget. - * @returns whether the submitted encoding already is canonical. + * Whether bytes already satisfy the master-version storage contract. + * @param detected - fully decoded source facts. + * @param bytes - encoded source length. + * @param policy - resolved master limits. + * @returns whether the source can pass through byte-identically. */ -export function isCanonical(detected: DetectedImage, bytes: number, policy: CanonicalImagePolicy): boolean { +export function isMasterImage(detected: DetectedImage, bytes: number, policy: MasterImagePolicy): boolean { return detected.mediaType !== 'image/gif' && !detected.animated && !detected.carriesMetadata + && detected.depth === 'uchar' + && detected.space === 'srgb' && bytes <= policy.maxBytes && Math.max(detected.width, detected.height) <= policy.maxDimension } /** - * Produce the canonical encoding of one fully validated source raster. - * Passthrough returns the submitted array; every re-encode bakes EXIF - * orientation into pixels, strips metadata, downscales to the policy's long - * edge, and encodes with fixed parameters: PNG (palette) for sources that - * carry alpha or were PNG/GIF, JPEG for photographic sources, falling down - * one fixed JPEG quality ladder until the byte target holds. - * @param data - submitted encoded bytes, already fully decoded by admission. - * @param detected - verified source format and dimensions. - * @param policy - resolved canonical budget. - * @returns canonical bytes and their reference facts. - * @throws AttachmentError `IMAGE_TOO_LARGE` when the smallest ladder step still exceeds the byte target. + * Classify a bounded pixel sample without assuming that a PNG source is a screenshot. + * @param pipeline - oriented sRGB source pipeline before output resizing. + * @returns whether the nearest-neighbour sample stays within the low-color threshold. */ -export async function canonicalizeImage( +export async function hasLowColourCount(pipeline: Sharp): Promise { + const { data, info } = await pipeline.clone().resize({ + width: LOW_COLOUR_SAMPLE_EDGE, + height: LOW_COLOUR_SAMPLE_EDGE, + fit: 'inside', + withoutEnlargement: true, + kernel: sharp.kernel.nearest, + fastShrinkOnLoad: false, + }).raw().toBuffer({ resolveWithObject: true }) + const colours = new Set() + for (let offset = 0; offset < data.length; offset += info.channels) { + const red = data[offset] ?? 0 + const green = data[offset + 1] ?? red + const blue = data[offset + 2] ?? red + const alpha = info.channels === 2 + ? data[offset + 1] ?? 255 + : info.channels === 4 ? data[offset + 3] ?? 255 : 255 + colours.add(((red >> 3) << 15) | ((green >> 3) << 10) | ((blue >> 3) << 5) | (alpha >> 3)) + if (colours.size > LOW_COLOUR_LIMIT) return false + } + return true +} + +/** Assert that a re-encoded master is an 8-bit sRGB/sRGBA single-frame image with matching facts. */ +async function verifyMaster(image: MasterImage, expectedAlpha: boolean | undefined): Promise { + const detected = await detectImage(image.data) + if (detected.mediaType !== image.mediaType + || detected.width !== image.width + || detected.height !== image.height + || detected.animated + || detected.carriesMetadata + || detected.depth !== 'uchar' + || detected.space !== 'srgb' + || (expectedAlpha !== undefined && detected.hasAlpha !== expectedAlpha)) { + throw new AttachmentError( + 'Canonical image conversion did not produce a single-frame 8-bit sRGB image with matching metadata.', + 'ATTACHMENT_WRITE_FAILED', + ) + } + return image +} + +/** Build one fixed-size, oriented, metadata-free sRGB pipeline from submitted bytes. */ +function preparedPipeline(data: Uint8Array, width: number, height: number): Sharp { + return sharp(data, { failOn: 'error', limitInputPixels: false }) + .rotate() + .toColourspace('srgb') + .resize({ width, height, fit: 'inside', withoutEnlargement: true }) +} + +/** Dimensions after the long edge is capped without changing aspect ratio. */ +function initialDimensions(detected: DetectedImage, maxDimension: number): { width: number; height: number } { + const scale = Math.min(1, maxDimension / Math.max(detected.width, detected.height)) + return { + width: Math.max(1, Math.round(detected.width * scale)), + height: Math.max(1, Math.round(detected.height * scale)), + } +} + +/** Lazy encoding order for one size, separated by sampled colour complexity and alpha. */ +function encodingAttemptsAtSize( + data: Uint8Array, + width: number, + height: number, + hasAlpha: boolean, + lowColour: boolean, +): Array<() => Promise> { + const prepared = preparedPipeline(data, width, height) + const webp = MASTER_QUALITIES.map(quality => ( + () => encode(prepared.clone(), 'image/webp', quality) + )) + if (lowColour) { + return [() => encode(prepared.clone(), 'image/png', undefined, !hasAlpha), ...webp] + } + if (hasAlpha) return webp + return MASTER_QUALITIES.map(quality => ( + () => encode(prepared.clone(), 'image/jpeg', quality) + )) +} + +/** + * Produce the 2048px provider-independent master version of one fully decoded source. + * The source is passed through only when it is already clean, single-frame, 8-bit sRGB/sRGBA, + * and inside both master limits. Re-encoding never removes transparency. After the fixed + * quality floor is reached, dimensions continue shrinking until the independent byte cap holds. + * @param data - complete admitted source bytes. + * @param detected - fully decoded source facts. + * @param policy - resolved independent master limits. + * @returns verified provider-independent master bytes and metadata. + */ +export async function prepareMasterImage( data: Uint8Array, detected: DetectedImage, - policy: CanonicalImagePolicy, -): Promise { - if (isCanonical(detected, data.byteLength, policy)) { + policy: MasterImagePolicy, +): Promise { + if (isMasterImage(detected, data.byteLength, policy)) { return { data, mediaType: detected.mediaType, width: detected.width, height: detected.height } } try { - const source = sharp(data, { failOn: 'error', limitInputPixels: false }) - const { hasAlpha } = await source.metadata() - const prepared = source.rotate().resize({ - width: policy.maxDimension, - height: policy.maxDimension, - fit: 'inside', - withoutEnlargement: true, - }) - const preferPng = hasAlpha || detected.mediaType === 'image/png' || detected.mediaType === 'image/gif' - if (preferPng) { - const png = await encode(prepared.clone().png({ compressionLevel: 9, palette: true }), 'image/png') - if (png.data.byteLength <= policy.maxBytes) return png - } - for (const quality of JPEG_QUALITIES) { - const jpeg = await encode( - prepared.clone().flatten({ background: '#ffffff' }).jpeg({ quality }), - 'image/jpeg', + let { width, height } = initialDimensions(detected, policy.maxDimension) + const classificationPipeline = sharp(data, { failOn: 'error', limitInputPixels: false }) + .rotate() + .toColourspace('srgb') + const lowColour = await hasLowColourCount(classificationPipeline) + for (;;) { + const encoded = await encodeFirstWithinLimit( + encodingAttemptsAtSize(data, width, height, detected.hasAlpha, lowColour), + policy.maxBytes, ) - if (jpeg.data.byteLength <= policy.maxBytes) return jpeg + if (!isExhaustedEncoding(encoded)) { + return await verifyMaster(encoded, detected.mediaType === 'image/gif' ? undefined : detected.hasAlpha) + } + if (width === 1 && height === 1) break + const sizeScale = Math.sqrt(policy.maxBytes / encoded.smallest.data.byteLength) * 0.95 + const scale = Math.min(MIN_SCALE_STEP, sizeScale) + const nextWidth = Math.max(1, Math.floor(width * scale)) + const nextHeight = Math.max(1, Math.floor(height * scale)) + width = nextWidth === width && width > 1 ? width - 1 : nextWidth + height = nextHeight === height && height > 1 ? height - 1 : nextHeight } } catch (error) { - throw new AttachmentError('Unable to canonicalize image attachment.', 'ATTACHMENT_WRITE_FAILED', { cause: error }) + if (error instanceof AttachmentError) throw error + const source = detected.mediaType === 'image/png' && detected.depth !== 'uchar' + ? `${detected.depth === 'ushort' ? '16-bit' : detected.depth} PNG` + : `${detected.depth} ${detected.mediaType.slice('image/'.length).toUpperCase()}` + throw new AttachmentError( + `The ${source} could not be converted to the canonical 8-bit sRGB form.`, + 'ATTACHMENT_WRITE_FAILED', + { cause: error }, + ) } - throw new AttachmentError('Image cannot be encoded within the configured canonical byte target.', 'IMAGE_TOO_LARGE') + throw new AttachmentError('Image cannot be encoded within the configured master-image byte cap.', 'IMAGE_TOO_LARGE') } diff --git a/packages/attachment/attachment-local/src/compression-limiter.ts b/packages/attachment/attachment-local/src/compression-limiter.ts new file mode 100644 index 0000000000..3935f262a1 --- /dev/null +++ b/packages/attachment/attachment-local/src/compression-limiter.ts @@ -0,0 +1,43 @@ +/** Instance-owned concurrency bound for native image transformations. */ + +/** FIFO limiter for asynchronous compression work. */ +export class CompressionLimiter { + private active = 0 + private readonly waiting: Array<() => void> = [] + + /** + * @param concurrency - positive maximum number of active tasks. + */ + constructor(readonly concurrency: number) {} + + /** + * Run one task after an instance slot becomes available. + * @param task - compression operation occupying one slot until settlement. + * @returns the task result. + */ + run(task: () => Promise): Promise { + return new Promise((resolve, reject) => { + const start = (): void => { + this.active += 1 + const release = (): void => { + this.active -= 1 + this.waiting.shift()?.() + } + void Promise.resolve().then(task).then( + (value) => { + release() + resolve(value) + }, + (error: unknown) => { + release() + reject(error instanceof Error + ? error + : new Error('Image compression task rejected with a non-Error value.', { cause: error })) + }, + ) + } + if (this.active < this.concurrency) start() + else this.waiting.push(start) + }) + } +} diff --git a/packages/attachment/attachment-local/src/encoding.ts b/packages/attachment/attachment-local/src/encoding.ts new file mode 100644 index 0000000000..8099046c95 --- /dev/null +++ b/packages/attachment/attachment-local/src/encoding.ts @@ -0,0 +1,45 @@ +/** Shared lazy candidate execution for master and request-image encoders. */ + +/** One encoded candidate carrying its complete bytes. */ +export interface EncodedCandidate { + data: Uint8Array +} + +/** Result of exhausting candidates at one raster size without a fitting output. */ +export interface ExhaustedEncoding { + smallest: T +} + +/** + * Execute encoding candidates in preference order and stop after the first fitting output. + * @param attempts - lazy encoders ordered from preferred to fallback representation. + * @param maxBytes - positive encoded-byte cap. + * @returns the first fitting candidate, otherwise the smallest completed fallback. + */ +export async function encodeFirstWithinLimit( + attempts: readonly (() => Promise)[], + maxBytes: number, +): Promise> { + if (attempts.length === 0) throw new Error('image encoding requires at least one candidate') + let smallest: T | undefined + for (const attempt of attempts) { + const candidate = await attempt() + if (candidate.data.byteLength <= maxBytes) return candidate + if (smallest === undefined || candidate.data.byteLength < smallest.data.byteLength) { + smallest = candidate + } + } + if (smallest === undefined) throw new Error('image encoding did not execute a candidate') + return { smallest } +} + +/** + * Whether a lazy encoding result exhausted every candidate at one size. + * @param result - first fitting candidate or exhausted result. + * @returns whether every candidate exceeded the byte cap. + */ +export function isExhaustedEncoding( + result: T | ExhaustedEncoding, +): result is ExhaustedEncoding { + return 'smallest' in result +} diff --git a/packages/attachment/attachment-local/src/image.ts b/packages/attachment/attachment-local/src/image.ts index 991e5dc051..beedd3b8c0 100644 --- a/packages/attachment/attachment-local/src/image.ts +++ b/packages/attachment/attachment-local/src/image.ts @@ -13,8 +13,14 @@ export interface DetectedImage { height: number /** Whether the container carries more than one frame. */ animated: boolean - /** Whether the bytes carry EXIF/XMP/IPTC metadata or a non-default orientation. */ + /** Whether the bytes carry descriptive metadata, a color profile, or orientation. */ carriesMetadata: boolean + /** Sharp sample depth reported for the decoded channels. */ + depth: string + /** Sharp colour space reported for the decoded pixels. */ + space: string + /** Whether decoded pixels carry an alpha channel. */ + hasAlpha: boolean } const MEDIA_TYPES: Readonly> = { @@ -24,6 +30,17 @@ const MEDIA_TYPES: Readonly> = { gif: 'image/gif', } +function carriesRetainedMetadata(metadata: Awaited>): boolean { + return metadata.exif !== undefined + || metadata.xmp !== undefined + || metadata.iptc !== undefined + || metadata.icc !== undefined + || metadata.hasProfile + || metadata.tifftagPhotoshop !== undefined + || metadata.comments !== undefined + || metadata.orientation !== undefined +} + async function imageMetadata(image: Sharp): Promise { const metadata = await image.metadata() const mediaType = MEDIA_TYPES[metadata.format as string] @@ -38,9 +55,10 @@ async function imageMetadata(image: Sharp): Promise { width: transposed ? metadata.height : metadata.width, height: transposed ? metadata.width : metadata.height, animated: (metadata.pages ?? 1) > 1, - // orientation is EXIF-derived for every whitelisted format, so exif - // presence already covers a non-default orientation. - carriesMetadata: metadata.exif !== undefined || metadata.xmp !== undefined || metadata.iptc !== undefined, + carriesMetadata: carriesRetainedMetadata(metadata), + depth: metadata.depth, + space: metadata.space, + hasAlpha: metadata.hasAlpha, } } diff --git a/packages/attachment/attachment-local/src/index.ts b/packages/attachment/attachment-local/src/index.ts index cbe535ae7d..e43153247a 100644 --- a/packages/attachment/attachment-local/src/index.ts +++ b/packages/attachment/attachment-local/src/index.ts @@ -4,14 +4,27 @@ import { join, resolve } from 'node:path' import { Context } from '@deepseek-ai/cordis' import z from '@deepseek-ai/schemastery' import { AttachmentStore } from '@deepseek-ai/dsh-attachment' -import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, SavedImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' +import type { + ImageAttachmentLimits, + ImageAttachmentRef, + ImageRequestPolicy, + PreviewImageCrop, + RequestImageAttachment, + SaveImageAttachment, + SavedImageAttachment, + StoredImageAttachment, +} from '@deepseek-ai/dsh-attachment' import { resolveDshHome } from '@deepseek-ai/dsh-home-paths' -import type { CanonicalImagePolicy } from './canonical.ts' -import { readImageFile, saveImageFile, validateImageFile } from './store.ts' +import type { MasterImagePolicy } from './canonical.ts' +import { CompressionLimiter } from './compression-limiter.ts' +import { commitPreparedImageFile, prepareImageFile, readImageFile, validateImageFile } from './store.ts' +import { previewCropToMaster, readRequestImageFile, requestImageVariantId } from './request-image.ts' -export { canonicalizeImage, isCanonical } from './canonical.ts' -export type { CanonicalImage, CanonicalImagePolicy } from './canonical.ts' -export { readImageFile, saveImageFile, validateImageFile } from './store.ts' +export { isMasterImage, prepareMasterImage } from './canonical.ts' +export type { MasterImage, MasterImagePolicy } from './canonical.ts' +export { commitPreparedImageFile, prepareImageFile, readImageFile, saveImageFile, validateImageFile } from './store.ts' +export type { PreparedImageFile } from './store.ts' +export { previewCropToMaster, readRequestImageFile, requestImageDimensions, requestImageVariantId } from './request-image.ts' /** Default maximum encoded bytes for one submitted image; oversized sources are refused, not shrunk. */ export const DEFAULT_MAX_IMAGE_BYTES = 32 * 1024 * 1024 @@ -24,13 +37,17 @@ export const DEFAULT_MAX_IMAGE_PIXELS = 100_000_000 /** Default per-side pixel cap for one submitted image. */ export const DEFAULT_MAX_IMAGE_DIMENSION = 16384 /** - * Default long-edge target of the stored canonical encoding. A larger source + * Default long-edge target of the stored image master. A larger source * is admitted and downscaled to this edge, so admission bounds what rides * every later model request without refusing ordinary large sources. */ -export const DEFAULT_CANONICAL_MAX_DIMENSION = 2048 -/** Default byte target of the stored canonical encoding. */ -export const DEFAULT_CANONICAL_MAX_BYTES = 1024 * 1024 +export const DEFAULT_MASTER_MAX_DIMENSION = 2048 +/** Default independent safety cap for one stored master version. */ +export const DEFAULT_MASTER_MAX_BYTES = 4 * 1024 * 1024 +/** Conservative default number of simultaneous native image transformations per store. */ +export const DEFAULT_IMAGE_COMPRESSION_CONCURRENCY = 2 +/** Maximum configurable native image transformations per store. */ +export const MAX_IMAGE_COMPRESSION_CONCURRENCY = 8 /** Local attachment backend configuration. */ export interface Config { @@ -46,10 +63,29 @@ export interface Config { maxImagePixels?: number /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. */ maxImageDimension?: number - /** Long-edge pixel target of the stored canonical encoding. */ - canonicalMaxDimension?: number - /** Encoded-byte target of the stored canonical encoding. */ - canonicalMaxBytes?: number + /** Long-edge pixel cap of the stored provider-independent master version. */ + masterMaxDimension?: number + /** Encoded-byte safety cap of the stored provider-independent master version. */ + masterMaxBytes?: number + /** Maximum simultaneous master or request-image transformations in this service instance. */ + imageCompressionConcurrency?: number +} + +function waitForShared(operation: Promise, signal: AbortSignal | undefined): Promise { + if (signal === undefined) return operation + signal.throwIfAborted() + return new Promise((resolve, reject) => { + const abort = (): void => { + const reason: unknown = signal.reason + reject(reason instanceof Error + ? reason + : new Error('Attachment request cancelled with a non-Error reason.', { cause: reason })) + } + signal.addEventListener('abort', abort, { once: true }) + void operation.then(resolve, reject).finally(() => { + signal.removeEventListener('abort', abort) + }) + }) } /** Persistent content-addressed local attachment store. */ @@ -61,15 +97,21 @@ export class LocalAttachmentStore extends AttachmentStore { maxMessageImageBytes: z.number().step(1).min(1).default(DEFAULT_MAX_MESSAGE_IMAGE_BYTES), maxImagePixels: z.number().step(1).min(1).default(DEFAULT_MAX_IMAGE_PIXELS), maxImageDimension: z.number().step(1).min(1).default(DEFAULT_MAX_IMAGE_DIMENSION), - canonicalMaxDimension: z.number().step(1).min(1).default(DEFAULT_CANONICAL_MAX_DIMENSION), - canonicalMaxBytes: z.number().step(1).min(1).default(DEFAULT_CANONICAL_MAX_BYTES), + masterMaxDimension: z.number().step(1).min(1).default(DEFAULT_MASTER_MAX_DIMENSION), + masterMaxBytes: z.number().step(1).min(1).default(DEFAULT_MASTER_MAX_BYTES), + imageCompressionConcurrency: z.number().step(1).min(1).max(MAX_IMAGE_COMPRESSION_CONCURRENCY) + .default(DEFAULT_IMAGE_COMPRESSION_CONCURRENCY), }) /** Absolute versioned storage root. */ readonly root: string readonly imageLimits: ImageAttachmentLimits - /** Resolved canonical encoding budget applied by every save. */ - readonly canonicalPolicy: Readonly + /** Resolved provider-independent master-version storage policy. */ + readonly masterPolicy: Readonly + /** Resolved instance-level compression limit. */ + readonly imageCompressionConcurrency: number + private readonly compression: CompressionLimiter + private readonly requestInflight = new Map>() constructor(ctx: Context, config: Config) { super(ctx) @@ -82,23 +124,107 @@ export class LocalAttachmentStore extends AttachmentStore { maxImageDimension: config.maxImageDimension ?? DEFAULT_MAX_IMAGE_DIMENSION, mediaTypes: Object.freeze(['image/png', 'image/jpeg', 'image/webp', 'image/gif'] as const), }) - this.canonicalPolicy = Object.freeze({ - maxDimension: config.canonicalMaxDimension ?? DEFAULT_CANONICAL_MAX_DIMENSION, - maxBytes: config.canonicalMaxBytes ?? DEFAULT_CANONICAL_MAX_BYTES, + this.masterPolicy = Object.freeze({ + maxDimension: config.masterMaxDimension ?? DEFAULT_MASTER_MAX_DIMENSION, + maxBytes: config.masterMaxBytes ?? DEFAULT_MASTER_MAX_BYTES, }) + const compressionConcurrency = config.imageCompressionConcurrency ?? DEFAULT_IMAGE_COMPRESSION_CONCURRENCY + if (!Number.isSafeInteger(compressionConcurrency) + || compressionConcurrency < 1 + || compressionConcurrency > MAX_IMAGE_COMPRESSION_CONCURRENCY) { + throw new Error( + `attachment-local: imageCompressionConcurrency must be an integer from 1 through ${MAX_IMAGE_COMPRESSION_CONCURRENCY}`, + ) + } + this.imageCompressionConcurrency = compressionConcurrency + this.compression = new CompressionLimiter(compressionConcurrency) } async validateImage(input: SaveImageAttachment): Promise { - await validateImageFile(input, this.imageLimits, this.canonicalPolicy) + await this.compression.run(() => validateImageFile(input, this.imageLimits, this.masterPolicy)) + } + + override async saveImages(inputs: readonly SaveImageAttachment[]): Promise { + this.validateImageBatch(inputs) + const prepared = await Promise.all(inputs.map(input => this.compression.run( + () => prepareImageFile(input, this.imageLimits, this.masterPolicy), + ))) + const refs: ImageAttachmentRef[] = [] + for (const image of prepared) refs.push((await commitPreparedImageFile(this.root, image)).ref) + return refs } async saveImage(input: SaveImageAttachment): Promise { - return saveImageFile(this.root, input, this.imageLimits, this.canonicalPolicy) + const prepared = await this.compression.run( + () => prepareImageFile(input, this.imageLimits, this.masterPolicy), + ) + return commitPreparedImageFile(this.root, prepared) } async readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise { return readImageFile(this.root, ref, signal) } + + override async readImageRequest( + ref: ImageAttachmentRef, + policy: ImageRequestPolicy, + signal?: AbortSignal, + ): Promise { + return this.requestVersion(ref, policy, undefined, signal) + } + + override async readImageRequests( + refs: readonly ImageAttachmentRef[], + policy: ImageRequestPolicy, + signal?: AbortSignal, + ): Promise { + return Promise.all(refs.map(ref => this.requestVersion(ref, policy, undefined, signal))) + } + + private requestVersion( + ref: ImageAttachmentRef, + policy: ImageRequestPolicy, + master: StoredImageAttachment | undefined, + signal: AbortSignal | undefined, + ): Promise { + signal?.throwIfAborted() + const variantId = requestImageVariantId(ref, policy) + const key = String(variantId) + let operation = this.requestInflight.get(key) + if (operation === undefined) { + operation = this.compression.run(async () => readRequestImageFile( + this.root, + master ?? await this.readImage(ref), + policy, + )) + this.requestInflight.set(key, operation) + void operation.finally(() => { + if (this.requestInflight.get(key) === operation) this.requestInflight.delete(key) + }).catch(() => {}) + } + return waitForShared(operation, signal) + } + + override async cropImage( + ref: ImageAttachmentRef, + crop: PreviewImageCrop, + signal?: AbortSignal, + ): Promise { + const master = await this.readImage(ref, signal) + const region = previewCropToMaster(ref.width, ref.height, crop) + const version = await this.requestVersion(ref, { + maxPixels: region.width * region.height, + maxBytes: this.masterPolicy.maxBytes, + crop: region, + }, master, signal) + signal?.throwIfAborted() + const stem = ref.name?.replace(/\.[^.]+$/u, '') ?? String(ref.attachmentId).slice(0, 15) + return this.saveImage({ + data: version.data, + mediaType: version.mediaType, + name: `${stem}-crop.${version.mediaType.slice('image/'.length).replace('jpeg', 'jpg')}`, + }) + } } export default LocalAttachmentStore diff --git a/packages/attachment/attachment-local/src/request-image.ts b/packages/attachment/attachment-local/src/request-image.ts new file mode 100644 index 0000000000..9c92d78181 --- /dev/null +++ b/packages/attachment/attachment-local/src/request-image.ts @@ -0,0 +1,353 @@ +/** Deterministic cached image versions for model requests and region reads. */ + +import { createHash, randomUUID } from 'node:crypto' +import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import sharp, { type Sharp } from 'sharp' +import { AttachmentError, ImageVariantId } from '@deepseek-ai/dsh-attachment' +import type { + ImageMediaType, + ImageAttachmentRef, + ImageRequestPolicy, + MasterImageCrop, + PreviewImageCrop, + RequestImageAttachment, + StoredImageAttachment, +} from '@deepseek-ai/dsh-attachment' +import { hasLowColourCount } from './canonical.ts' +import { encodeFirstWithinLimit, isExhaustedEncoding } from './encoding.ts' +import { detectImage, probeImage } from './image.ts' + +/** Transform version included in every cache and upload-index identity. */ +export const REQUEST_IMAGE_TRANSFORM_VERSION = 'request-image-v2' +/** DeepSeek request versions normally fit at these two preferred qualities. */ +export const REQUEST_IMAGE_QUALITIES = [85, 80] as const + +interface EncodedRequestImage { + data: Uint8Array + mediaType: ImageMediaType + width: number + height: number +} + +interface VerifiedRequestImage extends EncodedRequestImage { + hasAlpha: boolean +} + +function digest(value: string | Uint8Array): string { + return createHash('sha256').update(value).digest('hex') +} + +/** + * Compute aspect-preserving integer dimensions within a hard total-pixel budget. + * @param width - positive source width. + * @param height - positive source height. + * @param maxPixels - positive width-times-height cap. + * @returns inward-rounded dimensions; small images are not enlarged. + */ +export function requestImageDimensions( + width: number, + height: number, + maxPixels: number, +): { width: number; height: number } { + const scale = Math.min(1, Math.sqrt(maxPixels / (width * height))) + if (scale === 1) return { width, height } + if (width >= height) { + let projectedWidth = Math.max(1, Math.floor(width * scale)) + let projectedHeight = Math.max(1, Math.round(projectedWidth * height / width)) + while (projectedWidth * projectedHeight > maxPixels && projectedWidth > 1) { + projectedWidth -= 1 + projectedHeight = Math.max(1, Math.round(projectedWidth * height / width)) + } + return { width: projectedWidth, height: projectedHeight } + } + let projectedHeight = Math.max(1, Math.floor(height * scale)) + let projectedWidth = Math.max(1, Math.round(projectedHeight * width / height)) + while (projectedWidth * projectedHeight > maxPixels && projectedHeight > 1) { + projectedHeight -= 1 + projectedWidth = Math.max(1, Math.round(projectedHeight * width / height)) + } + return { width: projectedWidth, height: projectedHeight } +} + +function checkedInteger(value: number, name: string): number { + if (!Number.isSafeInteger(value) || value <= 0) { + throw new AttachmentError(`${name} must be a positive integer.`, 'INVALID_ATTACHMENT_REF') + } + return value +} + +function validatePolicy(policy: ImageRequestPolicy): void { + checkedInteger(policy.maxPixels, 'Image request maxPixels') + checkedInteger(policy.maxBytes, 'Image request maxBytes') + if (policy.crop !== undefined) { + if (!Number.isSafeInteger(policy.crop.x) || policy.crop.x < 0 + || !Number.isSafeInteger(policy.crop.y) || policy.crop.y < 0) { + throw new AttachmentError('Image crop origin must use non-negative integer pixels.', 'INVALID_ATTACHMENT_REF') + } + checkedInteger(policy.crop.width, 'Image crop width') + checkedInteger(policy.crop.height, 'Image crop height') + } +} + +function checkedCrop(master: StoredImageAttachment, crop: MasterImageCrop | undefined): MasterImageCrop | undefined { + if (crop === undefined) return undefined + if (crop.x + crop.width > master.ref.width || crop.y + crop.height > master.ref.height) { + throw new AttachmentError('Image crop extends beyond the stored master image.', 'INVALID_ATTACHMENT_REF') + } + return crop +} + +function descriptor(master: ImageAttachmentRef, policy: ImageRequestPolicy): string { + return JSON.stringify({ + transformVersion: REQUEST_IMAGE_TRANSFORM_VERSION, + masterAttachmentId: master.attachmentId, + routePixelBudget: policy.maxPixels, + encodedByteBudget: policy.maxBytes, + crop: policy.crop ?? null, + encoding: { + png: { compressionLevel: 9, palette: 'opaque-only' }, + webpQualities: REQUEST_IMAGE_QUALITIES, + jpegQualities: REQUEST_IMAGE_QUALITIES, + order: ['low-colour:png-webp', 'alpha:webp', 'opaque:jpeg'], + colourspace: 'srgb', + }, + }) +} + +/** + * Complete deterministic identity for one master and route-owned request policy. + * @param master - provider-independent durable master reference. + * @param policy - route-owned pixel, byte, and crop policy. + * @returns branded digest over every request transform input. + */ +export function requestImageVariantId( + master: ImageAttachmentRef, + policy: ImageRequestPolicy, +): ReturnType { + return ImageVariantId(`sha256:${digest(descriptor(master, policy))}`) +} + +function pipeline(master: StoredImageAttachment, crop: MasterImageCrop | undefined, width: number, height: number): Sharp { + return sourcePipeline(master, crop) + .resize({ width, height, fit: 'inside', withoutEnlargement: true }) +} + +function sourcePipeline(master: StoredImageAttachment, crop: MasterImageCrop | undefined): Sharp { + let image = sharp(master.data, { failOn: 'error', limitInputPixels: false }).toColourspace('srgb') + if (crop !== undefined) image = image.extract({ + left: crop.x, + top: crop.y, + width: crop.width, + height: crop.height, + }) + return image +} + +async function encoded( + image: Sharp, + mediaType: 'image/png' | 'image/jpeg' | 'image/webp', + quality?: number, + palette = true, +): Promise { + const output = mediaType === 'image/png' + ? image.png({ compressionLevel: 9, palette }) + : mediaType === 'image/webp' + ? image.webp({ quality }) + : image.jpeg({ quality }) + const { data, info } = await output.toBuffer({ resolveWithObject: true }) + return { data: new Uint8Array(data), mediaType, width: info.width, height: info.height } +} + +function encodingAttempts( + master: StoredImageAttachment, + crop: MasterImageCrop | undefined, + width: number, + height: number, + hasAlpha: boolean, + lowColour: boolean, +): Array<() => Promise> { + const prepared = pipeline(master, crop, width, height) + const webp = REQUEST_IMAGE_QUALITIES.map(quality => ( + () => encoded(prepared.clone(), 'image/webp', quality) + )) + if (lowColour) return [() => encoded(prepared.clone(), 'image/png', undefined, !hasAlpha), ...webp] + if (hasAlpha) return webp + return REQUEST_IMAGE_QUALITIES.map(quality => ( + () => encoded(prepared.clone(), 'image/jpeg', quality) + )) +} + +async function createRequestImage( + master: StoredImageAttachment, + policy: ImageRequestPolicy, + hasAlpha: boolean, +): Promise { + const crop = checkedCrop(master, policy.crop) + const sourceWidth = crop?.width ?? master.ref.width + const sourceHeight = crop?.height ?? master.ref.height + let dimensions = requestImageDimensions(sourceWidth, sourceHeight, policy.maxPixels) + if (crop === undefined + && dimensions.width === master.ref.width + && dimensions.height === master.ref.height + && master.data.byteLength <= policy.maxBytes) { + return { + data: master.data, + mediaType: master.ref.mediaType, + width: master.ref.width, + height: master.ref.height, + } + } + const lowColour = await hasLowColourCount(sourcePipeline(master, crop)) + for (;;) { + const encodedVersion = await encodeFirstWithinLimit( + encodingAttempts(master, crop, dimensions.width, dimensions.height, hasAlpha, lowColour), + policy.maxBytes, + ) + if (!isExhaustedEncoding(encodedVersion)) return encodedVersion + if (dimensions.width === 1 && dimensions.height === 1) break + const scale = Math.min(0.9, Math.sqrt(policy.maxBytes / encodedVersion.smallest.data.byteLength) * 0.95) + dimensions = { + width: Math.max(1, Math.floor(dimensions.width * scale)), + height: Math.max(1, Math.floor(dimensions.height * scale)), + } + } + throw new AttachmentError('Image cannot be encoded within the model-request byte budget.', 'IMAGE_TOO_LARGE') +} + +function cachePath(root: string, hash: string): string { + return join(root, 'request-images', hash.slice(0, 2), hash) +} + +async function readCached( + path: string, + master: StoredImageAttachment, + policy: ImageRequestPolicy, + expectedAlpha: boolean, + signal?: AbortSignal, +): Promise { + try { + const data = new Uint8Array(await readFile(path, { signal })) + const detected = await detectImage(data) + const crop = policy.crop + const maximum = requestImageDimensions(crop?.width ?? master.ref.width, crop?.height ?? master.ref.height, policy.maxPixels) + if (data.byteLength > policy.maxBytes || detected.depth !== 'uchar' || detected.space !== 'srgb' + || detected.width > maximum.width || detected.height > maximum.height + || detected.hasAlpha !== expectedAlpha) return undefined + return { data, mediaType: detected.mediaType, width: detected.width, height: detected.height, hasAlpha: detected.hasAlpha } + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException | null)?.code === 'ENOENT') return undefined + signal?.throwIfAborted() + return undefined + } +} + +async function verifyRequestImage( + image: EncodedRequestImage, + expectedAlpha: boolean, +): Promise { + const detected = await detectImage(image.data) + if (detected.depth !== 'uchar' || detected.space !== 'srgb' + || detected.width !== image.width || detected.height !== image.height + || detected.mediaType !== image.mediaType || detected.hasAlpha !== expectedAlpha) { + throw new AttachmentError( + 'Encoded model-request image does not match its verified 8-bit sRGB metadata.', + 'ATTACHMENT_WRITE_FAILED', + ) + } + return { ...image, hasAlpha: detected.hasAlpha } +} + +async function writeCached(path: string, data: Uint8Array): Promise { + await mkdir(dirname(path), { recursive: true, mode: 0o700 }) + const temporary = `${path}.${randomUUID()}.tmp` + try { + await writeFile(temporary, data, { mode: 0o600, flag: 'wx' }) + try { + await rename(temporary, path) + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException | null)?.code !== 'EEXIST') throw error + } + } finally { + await rm(temporary, { force: true }) + } +} + +/** + * Generate or reuse one request image below the local attachment root. + * @param root - absolute versioned attachment storage root. + * @param master - verified stored master bytes and reference. + * @param policy - exact route request-image policy. + * @param signal - optional cancellation for cache I/O. + * @returns verified request bytes and deterministic variant identity. + */ +export async function readRequestImageFile( + root: string, + master: StoredImageAttachment, + policy: ImageRequestPolicy, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted() + validatePolicy(policy) + checkedCrop(master, policy.crop) + const source = await probeImage(master.data) + const variantId = requestImageVariantId(master.ref, policy) + const hash = String(variantId).slice('sha256:'.length) + const path = cachePath(root, hash) + const cached = await readCached(path, master, policy, source.hasAlpha, signal) + const created = cached ?? await createRequestImage(master, policy, source.hasAlpha) + const version = cached ?? (created.data === master.data + ? { ...created, hasAlpha: source.hasAlpha } + : await verifyRequestImage(created, source.hasAlpha)) + signal?.throwIfAborted() + if (cached === undefined && version.data !== master.data) await writeCached(path, version.data) + return { + variantId, + master: master.ref, + data: version.data, + mediaType: version.mediaType, + bytes: version.data.byteLength, + width: version.width, + height: version.height, + depth: 'uchar', + space: 'srgb', + hasAlpha: version.hasAlpha, + ...policy.crop === undefined ? {} : { crop: policy.crop }, + } +} + +/** + * Map a preview-coordinate rectangle to the oriented stored master. + * @param masterWidth - stored master width. + * @param masterHeight - stored master height. + * @param crop - rectangle measured on the model-visible preview. + * @returns covering integer rectangle in master coordinates. + */ +export function previewCropToMaster( + masterWidth: number, + masterHeight: number, + crop: PreviewImageCrop, +): MasterImageCrop { + checkedInteger(masterWidth, 'Master image width') + checkedInteger(masterHeight, 'Master image height') + checkedInteger(crop.previewWidth, 'Preview width') + checkedInteger(crop.previewHeight, 'Preview height') + if (!Number.isSafeInteger(crop.x) || crop.x < 0 || !Number.isSafeInteger(crop.y) || crop.y < 0) { + throw new AttachmentError('Preview crop origin must use non-negative integer pixels.', 'INVALID_ATTACHMENT_REF') + } + checkedInteger(crop.width, 'Preview crop width') + checkedInteger(crop.height, 'Preview crop height') + if (crop.x + crop.width > crop.previewWidth || crop.y + crop.height > crop.previewHeight) { + throw new AttachmentError('Preview crop extends beyond the image shown to the model.', 'INVALID_ATTACHMENT_REF') + } + const x = Math.floor(crop.x * masterWidth / crop.previewWidth) + const y = Math.floor(crop.y * masterHeight / crop.previewHeight) + const right = Math.ceil((crop.x + crop.width) * masterWidth / crop.previewWidth) + const bottom = Math.ceil((crop.y + crop.height) * masterHeight / crop.previewHeight) + return { + x, + y, + width: Math.max(1, Math.min(masterWidth, right) - x), + height: Math.max(1, Math.min(masterHeight, bottom) - y), + } +} diff --git a/packages/attachment/attachment-local/src/store.ts b/packages/attachment/attachment-local/src/store.ts index 9964c2a94f..ba45256416 100644 --- a/packages/attachment/attachment-local/src/store.ts +++ b/packages/attachment/attachment-local/src/store.ts @@ -16,8 +16,8 @@ import type { SourceImageInfo, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' -import { canonicalizeImage } from './canonical.ts' -import type { CanonicalImagePolicy } from './canonical.ts' +import { prepareMasterImage } from './canonical.ts' +import type { MasterImagePolicy } from './canonical.ts' import { detectImage, probeImage } from './image.ts' import type { DetectedImage } from './image.ts' @@ -64,23 +64,60 @@ async function inspectMetadata( /** * Run the full admission policy for one image without touching storage, - * including a canonical-encoding dry run: a batch whose members all validate - * cannot later be refused mid-write by the canonical byte target. + * including master-version preparation: a batch whose members all validate + * cannot later be refused by the master byte cap during publication. * @param input - encoded bytes and declared metadata. * @param limits - resolved source admission policy. - * @param policy - resolved canonical encoding budget. - * @returns completion after the raster has been fully decoded and its canonical encoding proven to fit. + * @param policy - resolved master-version storage policy. + * @returns completion after the raster has been decoded and its master version proven to fit. */ export async function validateImageFile( input: SaveImageAttachment, limits: ImageAttachmentLimits, - policy: CanonicalImagePolicy, + policy: MasterImagePolicy, ): Promise { + await prepareImageFile(input, limits, policy) +} + +/** Fully prepared master object, verified before any batch member is persisted. */ +export interface PreparedImageFile extends SavedImageAttachment { + /** Deterministic master bytes whose digest is {@link ref.attachmentId}. */ + data: Uint8Array +} + +/** + * Decode, normalize, and verify one submitted image without touching storage. + * @param input - submitted encoded bytes and declared media type. + * @param limits - source admission policy. + * @param policy - independent master-version storage policy. + * @returns immutable reference facts beside bytes ready for atomic publication. + */ +export async function prepareImageFile( + input: SaveImageAttachment, + limits: ImageAttachmentLimits, + policy: MasterImagePolicy, +): Promise { if (input.data.byteLength > limits.maxImageBytes) { throw new AttachmentError('Image exceeds the configured byte limit.', 'IMAGE_TOO_LARGE') } - const { detected } = await inspectMetadata(input.data, input.mediaType, limits) - await canonicalizeImage(input.data, detected, policy) + const { detected, source } = await inspectMetadata(input.data, input.mediaType, limits) + const master = await prepareMasterImage(input.data, detected, policy) + const sha256 = digest(master.data) + const name = displayName(input.name) + const downscaled = source.width !== master.width || source.height !== master.height + return { + data: master.data, + ref: { + attachmentId: AttachmentId(`sha256:${sha256}`), + mediaType: master.mediaType, + width: master.width, + height: master.height, + bytes: master.data.byteLength, + ...(name !== undefined ? { name } : {}), + ...downscaled ? { sourceWidth: source.width, sourceHeight: source.height } : {}, + }, + source, + } } /** @@ -143,26 +180,20 @@ async function ensureDurableHome(path: string): Promise { } /** - * Save and verify one image below a versioned attachment root. Admission - * validates the submitted source, then stores its deterministic canonical - * encoding; the returned reference describes the stored canonical bytes while - * `source` preserves the submitted raster's facts. + * Publish one already verified master below a versioned attachment root. * @param root - absolute `DSH_HOME/attachments/v1` root. - * @param input - encoded bytes and declared metadata. - * @param limits - resolved source admission policy. - * @param policy - resolved canonical encoding budget. + * @param prepared - deterministic master bytes, reference, and source facts. * @returns durable content-addressed reference beside the submitted source facts. */ -export async function saveImageFile( +export async function commitPreparedImageFile( root: string, - input: SaveImageAttachment, - limits: ImageAttachmentLimits, - policy: CanonicalImagePolicy, + prepared: PreparedImageFile, ): Promise { - if (input.data.byteLength > limits.maxImageBytes) throw new AttachmentError('Image exceeds the configured byte limit.', 'IMAGE_TOO_LARGE') - const { detected, source } = await inspectMetadata(input.data, input.mediaType, limits) - const canonical = await canonicalizeImage(input.data, detected, policy) - const sha256 = digest(canonical.data) + const master = prepared.data + const sha256 = ensureReference(prepared.ref) + if (digest(master) !== sha256 || master.byteLength !== prepared.ref.bytes) { + throw new AttachmentError('Prepared attachment bytes do not match their reference.', 'ATTACHMENT_CORRUPT') + } const bucket = join(root, 'objects', sha256.slice(0, 2)) const staging = join(root, 'tmp') // Establish DSH_HOME itself against the filesystem root once per process. @@ -176,7 +207,7 @@ export async function saveImageFile( let handle try { handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600) - await handle.writeFile(canonical.data) + await handle.writeFile(master) await handle.sync() await handle.close() handle = undefined @@ -211,18 +242,24 @@ export async function saveImageFile( if (error instanceof AttachmentError) throw error throw new AttachmentError('Unable to persist image attachment.', 'ATTACHMENT_WRITE_FAILED', { cause: error }) } - const name = displayName(input.name) - return { - ref: { - attachmentId: AttachmentId(`sha256:${sha256}`), - mediaType: canonical.mediaType, - width: canonical.width, - height: canonical.height, - bytes: canonical.data.byteLength, - ...(name !== undefined ? { name } : {}), - }, - source, - } + return { ref: prepared.ref, source: prepared.source } +} + +/** + * Decode and normalize one image once, then publish the prepared object. + * @param root - absolute `DSH_HOME/attachments/v1` root. + * @param input - submitted encoded bytes and declared media type. + * @param limits - resolved source admission policy. + * @param policy - resolved master-version storage policy. + * @returns durable content-addressed reference beside submitted source facts. + */ +export async function saveImageFile( + root: string, + input: SaveImageAttachment, + limits: ImageAttachmentLimits, + policy: MasterImagePolicy, +): Promise { + return commitPreparedImageFile(root, await prepareImageFile(input, limits, policy)) } /** diff --git a/packages/attachment/attachment-local/tests/canonical.spec.ts b/packages/attachment/attachment-local/tests/canonical.spec.ts index 12d286a2bc..c1307b5848 100644 --- a/packages/attachment/attachment-local/tests/canonical.spec.ts +++ b/packages/attachment/attachment-local/tests/canonical.spec.ts @@ -1,17 +1,19 @@ import { describe, expect, it } from 'vitest' import sharp from 'sharp' -import { canonicalizeImage, isCanonical } from '../src/canonical.ts' -import type { CanonicalImagePolicy } from '../src/canonical.ts' +import { hasLowColourCount, isMasterImage, prepareMasterImage } from '../src/canonical.ts' +import type { MasterImagePolicy } from '../src/canonical.ts' import { detectImage } from '../src/image.ts' -const POLICY: CanonicalImagePolicy = { maxDimension: 2048, maxBytes: 1024 * 1024 } +const POLICY: MasterImagePolicy = { maxDimension: 2048, maxBytes: 4 * 1024 * 1024 } /** Deterministic pseudo-random RGB noise; PNG cannot compress it below raw size. */ function noisePixels(width: number, height: number): Uint8Array { const pixels = new Uint8Array(width * height * 3) let state = 0x2545f491 for (let index = 0; index < pixels.length; index += 1) { - state = (state * 1103515245 + 12345) & 0x7fffffff + state ^= state << 13 + state ^= state >>> 17 + state ^= state << 5 pixels[index] = state & 0xff } return pixels @@ -29,46 +31,64 @@ async function flatImage(width: number, height: number, format: 'png' | 'jpeg' | return new Uint8Array(await image.toFormat(format, format === 'webp' && alpha ? { lossless: true } : {}).toBuffer()) } -describe('isCanonical', () => { +describe('isMasterImage', () => { it('accepts an in-budget clean PNG/JPEG/WebP and refuses GIF, animation, metadata, oversized edges, and oversized bytes', () => { - const clean = { animated: false, carriesMetadata: false } - expect(isCanonical({ mediaType: 'image/png', width: 2048, height: 4, ...clean }, 100, POLICY)).toBe(true) - expect(isCanonical({ mediaType: 'image/gif', width: 4, height: 4, ...clean }, 100, POLICY)).toBe(false) - expect(isCanonical({ mediaType: 'image/webp', width: 4, height: 4, animated: true, carriesMetadata: false }, 100, POLICY)).toBe(false) - expect(isCanonical({ mediaType: 'image/jpeg', width: 4, height: 4, animated: false, carriesMetadata: true }, 100, POLICY)).toBe(false) - expect(isCanonical({ mediaType: 'image/jpeg', width: 2049, height: 4, ...clean }, 100, POLICY)).toBe(false) - expect(isCanonical({ mediaType: 'image/webp', width: 4, height: 4, ...clean }, POLICY.maxBytes + 1, POLICY)).toBe(false) + const clean = { animated: false, carriesMetadata: false, depth: 'uchar', space: 'srgb', hasAlpha: false } + expect(isMasterImage({ mediaType: 'image/png', width: 2048, height: 4, ...clean }, 100, POLICY)).toBe(true) + expect(isMasterImage({ mediaType: 'image/gif', width: 4, height: 4, ...clean }, 100, POLICY)).toBe(false) + expect(isMasterImage({ mediaType: 'image/webp', width: 4, height: 4, animated: true, carriesMetadata: false, depth: 'uchar', space: 'srgb', hasAlpha: false }, 100, POLICY)).toBe(false) + expect(isMasterImage({ mediaType: 'image/jpeg', width: 4, height: 4, animated: false, carriesMetadata: true, depth: 'uchar', space: 'srgb', hasAlpha: false }, 100, POLICY)).toBe(false) + expect(isMasterImage({ mediaType: 'image/png', width: 4, height: 4, ...clean, depth: 'ushort' }, 100, POLICY)).toBe(false) + expect(isMasterImage({ mediaType: 'image/png', width: 4, height: 4, ...clean, space: 'rgb16' }, 100, POLICY)).toBe(false) + expect(isMasterImage({ mediaType: 'image/jpeg', width: 2049, height: 4, ...clean }, 100, POLICY)).toBe(false) + expect(isMasterImage({ mediaType: 'image/webp', width: 4, height: 4, ...clean }, POLICY.maxBytes + 1, POLICY)).toBe(false) }) }) -describe('canonicalizeImage', () => { +describe('prepareMasterImage', () => { it('passes an already-canonical source through byte-identically', async () => { const data = await flatImage(6, 4, 'webp') const detected = await detectImage(data) - const canonical = await canonicalizeImage(data, detected, POLICY) + const canonical = await prepareMasterImage(data, detected, POLICY) expect(canonical.data).toBe(data) expect(canonical).toMatchObject({ mediaType: 'image/webp', width: 6, height: 4 }) }) + it.each([3, 4] as const)('converts a 16-bit %s-channel PNG to 8-bit sRGB without passthrough', async (channels) => { + const data = new Uint8Array(await sharp({ + create: { width: 7, height: 5, channels, background: { r: 12, g: 34, b: 56, alpha: 0.5 } }, + }).toColourspace('rgb16').png().toBuffer()) + const detected = await detectImage(data) + expect(detected).toMatchObject({ depth: 'ushort', space: 'rgb16', hasAlpha: channels === 4 }) + + const canonical = await prepareMasterImage(data, detected, POLICY) + + expect(canonical.data).not.toBe(data) + expect(canonical.data).not.toEqual(data) + await expect(detectImage(canonical.data)).resolves.toMatchObject({ + depth: 'uchar', space: 'srgb', hasAlpha: channels === 4, width: 7, height: 5, + }) + }) + it('downscales an oversized PNG to the long-edge target and stays PNG', async () => { const data = await flatImage(10, 6, 'png') const detected = await detectImage(data) - const canonical = await canonicalizeImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes }) + const canonical = await prepareMasterImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes }) expect(canonical).toMatchObject({ mediaType: 'image/png', width: 5, height: 3 }) - await expect(detectImage(canonical.data)).resolves.toEqual({ mediaType: 'image/png', width: 5, height: 3, animated: false, carriesMetadata: false }) - const again = await canonicalizeImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes }) + await expect(detectImage(canonical.data)).resolves.toMatchObject({ mediaType: 'image/png', width: 5, height: 3, animated: false, carriesMetadata: false, depth: 'uchar', space: 'srgb' }) + const again = await prepareMasterImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes }) expect(again.data).toEqual(canonical.data) }) it('re-encodes the canonical output of a resize into itself (idempotence)', async () => { const data = await flatImage(10, 6, 'png') - const first = await canonicalizeImage(data, await detectImage(data), { maxDimension: 5, maxBytes: POLICY.maxBytes }) + const first = await prepareMasterImage(data, await detectImage(data), { maxDimension: 5, maxBytes: POLICY.maxBytes }) - const second = await canonicalizeImage(first.data, await detectImage(first.data), { maxDimension: 5, maxBytes: POLICY.maxBytes }) + const second = await prepareMasterImage(first.data, await detectImage(first.data), { maxDimension: 5, maxBytes: POLICY.maxBytes }) expect(second.data).toBe(first.data) }) @@ -77,31 +97,66 @@ describe('canonicalizeImage', () => { const data = await flatImage(6, 4, 'gif') const detected = await detectImage(data) - const canonical = await canonicalizeImage(data, detected, POLICY) + const canonical = await prepareMasterImage(data, detected, POLICY) expect(canonical.mediaType).toBe('image/png') - await expect(detectImage(canonical.data)).resolves.toEqual({ mediaType: 'image/png', width: 6, height: 4, animated: false, carriesMetadata: false }) + await expect(detectImage(canonical.data)).resolves.toMatchObject({ mediaType: 'image/png', width: 6, height: 4, animated: false, carriesMetadata: false, depth: 'uchar', space: 'srgb' }) }) - it('keeps alpha sources on PNG when the budget holds', async () => { + it('keeps a low-colour alpha source on PNG when the budget holds', async () => { const data = await flatImage(9, 5, 'webp', true) const detected = await detectImage(data) - const canonical = await canonicalizeImage(data, detected, { maxDimension: 4, maxBytes: POLICY.maxBytes }) + const canonical = await prepareMasterImage(data, detected, { maxDimension: 4, maxBytes: POLICY.maxBytes }) expect(canonical).toMatchObject({ mediaType: 'image/png', width: 4, height: 2 }) }) + it('retains an all-opaque alpha channel while converting a low-colour image', async () => { + const data = new Uint8Array(await sharp({ + create: { width: 10, height: 6, channels: 4, background: { r: 12, g: 200, b: 64, alpha: 1 } }, + }).png().toBuffer()) + + const canonical = await prepareMasterImage(data, await detectImage(data), { + maxDimension: 5, + maxBytes: POLICY.maxBytes, + }) + + expect(canonical).toMatchObject({ mediaType: 'image/png', width: 5, height: 3 }) + await expect(detectImage(canonical.data)).resolves.toMatchObject({ hasAlpha: true }) + }) + + it('keeps transparency when the byte cap requires another encoding and smaller dimensions', async () => { + const side = 128 + const pixels = new Uint8Array(side * side * 4) + const noise = noisePixels(side, side) + for (let pixel = 0; pixel < side * side; pixel += 1) { + const target = pixel * 4 + const source = pixel * 3 + pixels[target] = noise[source] ?? 0 + pixels[target + 1] = noise[source + 1] ?? 0 + pixels[target + 2] = noise[source + 2] ?? 0 + pixels[target + 3] = pixel & 0xff + } + const data = new Uint8Array(await sharp(pixels, { raw: { width: side, height: side, channels: 4 } }).png().toBuffer()) + + const canonical = await prepareMasterImage(data, await detectImage(data), { maxDimension: side, maxBytes: 1_024 }) + + expect(canonical.data.byteLength).toBeLessThanOrEqual(1_024) + expect(canonical.width).toBeLessThan(side) + await expect(detectImage(canonical.data)).resolves.toMatchObject({ hasAlpha: true, depth: 'uchar', space: 'srgb' }) + }) + it('re-encodes an oversized photographic JPEG as JPEG', async () => { const data = await noiseImage(64, 32, 'jpeg') const detected = await detectImage(data) - const canonical = await canonicalizeImage(data, detected, { maxDimension: 32, maxBytes: POLICY.maxBytes }) + const canonical = await prepareMasterImage(data, detected, { maxDimension: 32, maxBytes: POLICY.maxBytes }) expect(canonical).toMatchObject({ mediaType: 'image/jpeg', width: 32, height: 16 }) }) - it('falls from PNG to the JPEG ladder when palette PNG exceeds the byte target', async () => { + it('classifies a photographic PNG by pixels and uses an opaque photographic encoding', async () => { // A smooth gradient: palette quantization dithers it into a sizable PNG // while JPEG at quality 85 stays far smaller, so the budget between the // two forces exactly one ladder hop. @@ -117,22 +172,23 @@ describe('canonicalizeImage', () => { } const data = new Uint8Array(await sharp(pixels, { raw: { width: side, height: side, channels: 3 } }).png().toBuffer()) const detected = await detectImage(data) - const paletteSize = (await sharp(data).png({ compressionLevel: 9, palette: true }).toBuffer()).byteLength - const jpegSize = (await sharp(data).flatten({ background: '#ffffff' }).jpeg({ quality: 85 }).toBuffer()).byteLength - expect(jpegSize).toBeLessThan(paletteSize) - const budget = { maxDimension: 2048, maxBytes: paletteSize - 1 } + const budget = { maxDimension: 128, maxBytes: POLICY.maxBytes } - const canonical = await canonicalizeImage(data, detected, budget) + const canonical = await prepareMasterImage(data, detected, budget) expect(canonical.mediaType).toBe('image/jpeg') + expect(canonical).toMatchObject({ width: 128, height: 128 }) expect(canonical.data.byteLength).toBeLessThanOrEqual(budget.maxBytes) }) - it('refuses a source that no ladder step fits into the byte target', async () => { + it('shrinks dimensions after the quality floor instead of refusing an oversized encoding', async () => { const data = await noiseImage(64, 64, 'png') - await expect(canonicalizeImage(data, await detectImage(data), { maxDimension: 2048, maxBytes: 10 })) - .rejects.toMatchObject({ code: 'IMAGE_TOO_LARGE' }) + const canonical = await prepareMasterImage(data, await detectImage(data), { maxDimension: 2048, maxBytes: 512 }) + + expect(canonical.data.byteLength).toBeLessThanOrEqual(512) + expect(canonical.width).toBeLessThan(64) + expect(canonical.height).toBeLessThan(64) }) it('re-encodes an in-budget oriented JPEG, baking rotation and stripping metadata', async () => { @@ -143,16 +199,94 @@ describe('canonicalizeImage', () => { // Orientation 6 rotates 90°: the perceived source is 2x4. expect(detected).toMatchObject({ width: 2, height: 4, carriesMetadata: true }) - const canonical = await canonicalizeImage(data, detected, POLICY) + const canonical = await prepareMasterImage(data, detected, POLICY) expect(canonical.data).not.toBe(data) expect(canonical).toMatchObject({ width: 2, height: 4 }) await expect(detectImage(canonical.data)).resolves.toMatchObject({ width: 2, height: 4, carriesMetadata: false }) }) + it('re-encodes an in-budget image with an ICC profile and strips the profile', async () => { + const data = new Uint8Array(await sharp({ + create: { width: 4, height: 2, channels: 3, background: { r: 1, g: 2, b: 3 } }, + }).png().withIccProfile('p3').toBuffer()) + const detected = await detectImage(data) + expect(detected.carriesMetadata).toBe(true) + + const canonical = await prepareMasterImage(data, detected, POLICY) + + expect(canonical.data).not.toBe(data) + await expect(detectImage(canonical.data)).resolves.toMatchObject({ carriesMetadata: false }) + }) + it('maps an encoder fault on undecodable bytes to a storage failure', async () => { - const detected = { mediaType: 'image/png', width: 5000, height: 5000, animated: false, carriesMetadata: false } as const - await expect(canonicalizeImage(Uint8Array.of(1, 2, 3), detected, POLICY)) - .rejects.toMatchObject({ code: 'ATTACHMENT_WRITE_FAILED' }) + const detected = { + mediaType: 'image/png', width: 5000, height: 5000, animated: false, carriesMetadata: false, + depth: 'ushort', space: 'rgb16', hasAlpha: true, + } as const + await expect(prepareMasterImage(Uint8Array.of(1, 2, 3), detected, POLICY)) + .rejects.toMatchObject({ + code: 'ATTACHMENT_WRITE_FAILED', + message: 'The 16-bit PNG could not be converted to the canonical 8-bit sRGB form.', + }) + }) +}) + +describe('hasLowColourCount', () => { + it('distinguishes photographic rasters from low-colour graphics without averaged sampling', async () => { + const side = 512 + const highFrequency = sharp(noisePixels(side, side), { raw: { width: side, height: side, channels: 3 } }) + const gradientPixels = new Uint8Array(side * side * 3) + for (let y = 0; y < side; y += 1) { + for (let x = 0; x < side; x += 1) { + const offset = (y * side + x) * 3 + gradientPixels[offset] = x & 0xff + gradientPixels[offset + 1] = y & 0xff + gradientPixels[offset + 2] = (x * 3 + y * 5) & 0xff + } + } + const ordinaryPhoto = sharp(gradientPixels, { raw: { width: side, height: side, channels: 3 } }) + const solid = sharp({ + create: { width: side, height: side, channels: 3, background: { r: 12, g: 34, b: 56 } }, + }) + const text = sharp(Buffer.from(` + + + DeepSeek 16-bit + + `)) + const transparentData = await sharp({ + create: { width: side, height: side, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 } }, + }).composite([{ input: Buffer.from(` + + + + `) }]).png().toBuffer() + const transparent = sharp(transparentData) + + await expect(hasLowColourCount(highFrequency)).resolves.toBe(false) + await expect(hasLowColourCount(ordinaryPhoto)).resolves.toBe(false) + await expect(hasLowColourCount(solid)).resolves.toBe(true) + await expect(hasLowColourCount(text)).resolves.toBe(true) + await expect(hasLowColourCount(transparent)).resolves.toBe(true) + }) + + it('keeps an antialiased text screenshot readable on the low-colour PNG path', async () => { + const source = new Uint8Array(await sharp(Buffer.from(` + + + Readable text + + `)).removeAlpha().png().toBuffer()) + + const master = await prepareMasterImage(source, await detectImage(source), { + maxDimension: 512, + maxBytes: POLICY.maxBytes, + }) + const stats = await sharp(master.data).greyscale().stats() + + expect(master).toMatchObject({ mediaType: 'image/png', width: 512, height: 256 }) + expect(stats.channels[0]?.min).toBeLessThan(80) + expect(stats.channels[0]?.max).toBeGreaterThan(240) }) }) diff --git a/packages/attachment/attachment-local/tests/encoding.spec.ts b/packages/attachment/attachment-local/tests/encoding.spec.ts new file mode 100644 index 0000000000..c95d09c43c --- /dev/null +++ b/packages/attachment/attachment-local/tests/encoding.spec.ts @@ -0,0 +1,70 @@ +import { describe, expect, it, vi } from 'vitest' +import { CompressionLimiter } from '../src/compression-limiter.ts' +import { encodeFirstWithinLimit } from '../src/encoding.ts' + +describe('lazy image encoding', () => { + it('does not execute fallback qualities after the first fitting candidate', async () => { + const first = vi.fn(() => Promise.resolve({ data: new Uint8Array(8), quality: 85 })) + const fallback = vi.fn(() => Promise.resolve({ data: new Uint8Array(4), quality: 80 })) + + await expect(encodeFirstWithinLimit([first, fallback], 8)).resolves.toMatchObject({ quality: 85 }) + expect(first).toHaveBeenCalledTimes(1) + expect(fallback).not.toHaveBeenCalled() + }) + + it('executes later candidates only after earlier candidates exceed the cap', async () => { + const first = vi.fn(() => Promise.resolve({ data: new Uint8Array(12), quality: 85 })) + const second = vi.fn(() => Promise.resolve({ data: new Uint8Array(7), quality: 80 })) + const third = vi.fn(() => Promise.resolve({ data: new Uint8Array(5), quality: 75 })) + + await expect(encodeFirstWithinLimit([first, second, third], 8)).resolves.toMatchObject({ quality: 80 }) + expect(first).toHaveBeenCalledTimes(1) + expect(second).toHaveBeenCalledTimes(1) + expect(third).not.toHaveBeenCalled() + }) +}) + +describe('CompressionLimiter', () => { + it('starts at most the configured number of tasks and preserves queued progress', async () => { + const limiter = new CompressionLimiter(2) + const gates = Array.from({ length: 4 }, () => Promise.withResolvers()) + let active = 0 + let maximum = 0 + const started: number[] = [] + const tasks = gates.map((gate, index) => limiter.run(async () => { + active += 1 + maximum = Math.max(maximum, active) + started.push(index) + await gate.promise + active -= 1 + return index + })) + + await Promise.resolve() + expect(started).toEqual([0, 1]) + gates[0]!.resolve(undefined) + await tasks[0] + await Promise.resolve() + expect(started).toEqual([0, 1, 2]) + gates[1]!.resolve(undefined) + gates[2]!.resolve(undefined) + await Promise.all([tasks[1], tasks[2]]) + await Promise.resolve() + expect(started).toEqual([0, 1, 2, 3]) + gates[3]!.resolve(undefined) + + await expect(Promise.all(tasks)).resolves.toEqual([0, 1, 2, 3]) + expect(maximum).toBe(2) + }) + + it('releases a slot when a task throws before returning a promise', async () => { + const limiter = new CompressionLimiter(1) + const failed = limiter.run(() => { + throw new Error('synchronous setup failure') + }) + const next = limiter.run(() => Promise.resolve('next')) + + await expect(failed).rejects.toThrow('synchronous setup failure') + await expect(next).resolves.toBe('next') + }) +}) diff --git a/packages/attachment/attachment-local/tests/image.spec.ts b/packages/attachment/attachment-local/tests/image.spec.ts index 4398f986b7..848aa3ea28 100644 --- a/packages/attachment/attachment-local/tests/image.spec.ts +++ b/packages/attachment/attachment-local/tests/image.spec.ts @@ -18,7 +18,7 @@ describe('raster decoding', () => { ['gif', 'image/gif'], ] as const) { await expect(detectImage(await raster(format))) - .resolves.toEqual({ mediaType, width: 3, height: 2, animated: false, carriesMetadata: false }) + .resolves.toMatchObject({ mediaType, width: 3, height: 2, animated: false, carriesMetadata: false, depth: 'uchar', space: 'srgb' }) } }) @@ -31,7 +31,7 @@ describe('raster decoding', () => { await expect(detectImage(await raster('png'), { maxDimension: 2 })) .rejects.toMatchObject({ code: 'IMAGE_DIMENSION_TOO_LARGE' }) await expect(detectImage(await raster('png'), { maxDimension: 3 })) - .resolves.toEqual({ mediaType: 'image/png', width: 3, height: 2, animated: false, carriesMetadata: false }) + .resolves.toMatchObject({ mediaType: 'image/png', width: 3, height: 2, animated: false, carriesMetadata: false, depth: 'uchar', space: 'srgb' }) }) it('rejects malformed bytes and truncated payloads with readable headers', async () => { @@ -56,18 +56,30 @@ describe('raster decoding', () => { const oriented = new Uint8Array(await sharp({ create: { width: 4, height: 2, channels: 3, background: { r: 1, g: 2, b: 3 } }, }).jpeg().withMetadata({ orientation: 6 }).toBuffer()) - await expect(detectImage(oriented)).resolves.toEqual({ + await expect(detectImage(oriented)).resolves.toMatchObject({ mediaType: 'image/jpeg', width: 2, height: 4, animated: false, carriesMetadata: true, }) const flipped = new Uint8Array(await sharp({ create: { width: 4, height: 2, channels: 3, background: { r: 1, g: 2, b: 3 } }, }).jpeg().withMetadata({ orientation: 3 }).toBuffer()) - await expect(detectImage(flipped)).resolves.toEqual({ + await expect(detectImage(flipped)).resolves.toMatchObject({ mediaType: 'image/jpeg', width: 4, height: 2, animated: false, carriesMetadata: true, }) }) + it('reports color profiles and encoder metadata as metadata', async () => { + const profiled = new Uint8Array(await sharp({ + create: { width: 4, height: 2, channels: 3, background: { r: 1, g: 2, b: 3 } }, + }).png().withIccProfile('p3').toBuffer()) + await expect(detectImage(profiled)).resolves.toMatchObject({ carriesMetadata: true }) + + const commented = new Uint8Array(await sharp({ + create: { width: 4, height: 2, channels: 3, background: { r: 1, g: 2, b: 3 } }, + }).png().withMetadata().toBuffer()) + await expect(detectImage(commented)).resolves.toMatchObject({ carriesMetadata: true }) + }) + it('probes malformed bytes and unsupported formats into the same stable error', async () => { await expect(probeImage(Uint8Array.of(1, 2, 3))) .rejects.toMatchObject({ code: 'INVALID_IMAGE' }) diff --git a/packages/attachment/attachment-local/tests/index.spec.ts b/packages/attachment/attachment-local/tests/index.spec.ts index c4be530480..872aa5a3f7 100644 --- a/packages/attachment/attachment-local/tests/index.spec.ts +++ b/packages/attachment/attachment-local/tests/index.spec.ts @@ -4,9 +4,11 @@ import { mkdtemp, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { describe, expect, it } from 'vitest' +import sharp from 'sharp' import LocalAttachmentStore, { - DEFAULT_CANONICAL_MAX_BYTES, - DEFAULT_CANONICAL_MAX_DIMENSION, + DEFAULT_MASTER_MAX_BYTES, + DEFAULT_MASTER_MAX_DIMENSION, + DEFAULT_IMAGE_COMPRESSION_CONCURRENCY, DEFAULT_MAX_IMAGE_BYTES, DEFAULT_MAX_IMAGE_DIMENSION, DEFAULT_MAX_IMAGE_PIXELS, @@ -26,10 +28,19 @@ describe('local attachment service', () => { maxImageDimension: DEFAULT_MAX_IMAGE_DIMENSION, mediaTypes: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'], }) - expect(service.canonicalPolicy).toEqual({ - maxDimension: DEFAULT_CANONICAL_MAX_DIMENSION, - maxBytes: DEFAULT_CANONICAL_MAX_BYTES, + expect(service.masterPolicy).toEqual({ + maxDimension: DEFAULT_MASTER_MAX_DIMENSION, + maxBytes: DEFAULT_MASTER_MAX_BYTES, }) + expect(service.imageCompressionConcurrency).toBe(DEFAULT_IMAGE_COMPRESSION_CONCURRENCY) + }) + + it('resolves and validates the instance image-compression concurrency', () => { + expect(new LocalAttachmentStore(new Context(), { imageCompressionConcurrency: 1 }).imageCompressionConcurrency).toBe(1) + for (const imageCompressionConcurrency of [0, 1.5, 9]) { + expect(() => new LocalAttachmentStore(new Context(), { imageCompressionConcurrency })) + .toThrow(/imageCompressionConcurrency must be an integer from 1 through 8/) + } }) it('saves and reads through the service boundary', async () => { @@ -37,7 +48,7 @@ describe('local attachment service', () => { try { const service = new LocalAttachmentStore(new Context(), { dshHome }) const data = Uint8Array.from(Buffer.from( - 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=', + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC', 'base64', )) const { ref } = await service.saveImage({ data, mediaType: 'image/png' }) @@ -47,12 +58,31 @@ describe('local attachment service', () => { } }) - it('refuses a batch during validation when a member cannot meet the canonical byte target, before any write', async () => { + it.each([3, 4] as const)('admits a 16-bit %s-channel PNG as an 8-bit master object', async (channels) => { + const dshHome = await mkdtemp(join(tmpdir(), 'dsh-attachment-16-bit-')) + try { + const service = new LocalAttachmentStore(new Context(), { dshHome }) + const source = new Uint8Array(await sharp({ + create: { width: 7, height: 5, channels, background: { r: 12, g: 34, b: 56, alpha: 0.5 } }, + }).toColourspace('rgb16').png().toBuffer()) + + const saved = await service.saveImage({ data: source, mediaType: 'image/png' }) + const stored = await service.readImage(saved.ref) + const metadata = await sharp(stored.data).metadata() + + expect(stored.data).not.toEqual(source) + expect(metadata).toMatchObject({ depth: 'uchar', space: 'srgb', hasAlpha: channels === 4 }) + } finally { + await rm(dshHome, { recursive: true, force: true }) + } + }) + + it('prepares every batch member before any write', async () => { const dshHome = await mkdtemp(join(tmpdir(), 'dsh-attachment-batch-')) try { - const service = new LocalAttachmentStore(new Context(), { dshHome, canonicalMaxBytes: 10 }) + const service = new LocalAttachmentStore(new Context(), { dshHome, masterMaxBytes: 1 }) const valid = Uint8Array.from(Buffer.from( - 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=', + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC', 'base64', )) await expect(service.saveImages([ @@ -72,7 +102,7 @@ describe('local attachment service', () => { await expect(service.validateImage({ data: Uint8Array.of(1, 2, 3), mediaType: 'image/png' })) .rejects.toThrow(/Unsupported or malformed image data/) const valid = Uint8Array.from(Buffer.from( - 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=', + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC', 'base64', )) const limited = new LocalAttachmentStore(new Context(), { dshHome, maxImageBytes: 1 }) diff --git a/packages/attachment/attachment-local/tests/request-image.spec.ts b/packages/attachment/attachment-local/tests/request-image.spec.ts new file mode 100644 index 0000000000..69bcfdf36c --- /dev/null +++ b/packages/attachment/attachment-local/tests/request-image.spec.ts @@ -0,0 +1,209 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Context } from '@deepseek-ai/cordis' +import sharp from 'sharp' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { CompressionLimiter } from '../src/compression-limiter.ts' +import LocalAttachmentStore, { previewCropToMaster, requestImageDimensions } from '../src/index.ts' + +const homes: string[] = [] + +async function store(): Promise { + const dshHome = await mkdtemp(join(tmpdir(), 'dsh-request-image-')) + homes.push(dshHome) + return new LocalAttachmentStore(new Context(), { dshHome }) +} + +async function image(width: number, height: number): Promise { + return new Uint8Array(await sharp({ + create: { width, height, channels: 3, background: { r: 12, g: 34, b: 56 } }, + }).png().toBuffer()) +} + +afterEach(async () => { + await Promise.all(homes.splice(0).map(home => rm(home, { recursive: true, force: true }))) +}) + +describe('request image dimensions', () => { + it.each([ + [4096, 4096, 800, 800], + [4096, 2048, 1130, 565], + [3840, 2160, 1066, 600], + [320, 240, 320, 240], + ])('projects %sx%s under 640,000 pixels as %sx%s', (width, height, expectedWidth, expectedHeight) => { + const projected = requestImageDimensions(width, height, 640_000) + expect(projected).toEqual({ + width: expectedWidth, + height: expectedHeight, + }) + expect(projected.width * projected.height).toBeLessThanOrEqual(640_000) + }) +}) + +describe('local request-image cache', () => { + it('derives stable square and wide previews and separates route budgets in the cache key', async () => { + const attachments = await store() + const square = (await attachments.saveImage({ + data: await image(2048, 2048), mediaType: 'image/png', name: 'square.png', + })).ref + const wide = (await attachments.saveImage({ + data: await image(2048, 1024), mediaType: 'image/png', name: 'wide.png', + })).ref + + const squareRequest = await attachments.readImageRequest(square, { maxPixels: 640_000, maxBytes: 1024 * 1024 }) + const wideRequest = await attachments.readImageRequest(wide, { maxPixels: 640_000, maxBytes: 1024 * 1024 }) + const repeated = await attachments.readImageRequest(wide, { maxPixels: 640_000, maxBytes: 1024 * 1024 }) + const low = await attachments.readImageRequest(wide, { maxPixels: 512 * 512, maxBytes: 1024 * 1024 }) + + expect(squareRequest).toMatchObject({ width: 800, height: 800 }) + expect(wideRequest).toMatchObject({ width: 1130, height: 565 }) + expect(repeated.variantId).toBe(wideRequest.variantId) + expect(repeated.data).toEqual(wideRequest.data) + expect(Buffer.from(repeated.data).toString('base64')).toBe(Buffer.from(wideRequest.data).toString('base64')) + expect(low.variantId).not.toBe(wideRequest.variantId) + expect(low.width * low.height).toBeLessThanOrEqual(512 * 512 + low.width) + }) + + it('maps preview coordinates to the 2048px master and crops the master instead of the preview', async () => { + const attachments = await store() + const pixels = Buffer.alloc(2048 * 1024 * 3) + for (let y = 0; y < 1024; y += 1) { + for (let x = 0; x < 2048; x += 1) { + const offset = (y * 2048 + x) * 3 + pixels[offset] = x < 1024 ? 255 : 0 + pixels[offset + 1] = x < 1024 ? 0 : 255 + pixels[offset + 2] = 0 + } + } + const source = new Uint8Array(await sharp(pixels, { raw: { width: 2048, height: 1024, channels: 3 } }).png().toBuffer()) + const master = (await attachments.saveImage({ data: source, mediaType: 'image/png', name: 'halves.png' })).ref + const preview = await attachments.readImageRequest(master, { maxPixels: 640_000, maxBytes: 1024 * 1024 }) + const previewCrop = { + previewWidth: preview.width, + previewHeight: preview.height, + x: Math.floor(preview.width / 2), + y: 0, + width: preview.width - Math.floor(preview.width / 2), + height: preview.height, + } + const mapped = previewCropToMaster(master.width, master.height, previewCrop) + + const cropped = await attachments.cropImage(master, previewCrop) + const stored = await attachments.readImage(cropped.ref) + const pixel = await sharp(stored.data).resize(1, 1).removeAlpha().raw().toBuffer() + + expect(mapped).toEqual({ x: 1024, y: 0, width: 1024, height: 1024 }) + expect(cropped.ref.width).toBe(mapped.width) + expect(cropped.ref.height).toBe(mapped.height) + expect(pixel[1]).toBeGreaterThan(pixel[0] ?? 0) + }) + + it('classifies opaque PNG pixels and preserves alpha while enforcing the request budget', async () => { + const attachments = await store() + const side = 256 + const photoPixels = new Uint8Array(side * side * 3) + const alphaPixels = new Uint8Array(side * side * 4) + let state = 0x2545f491 + for (let pixel = 0; pixel < side * side; pixel += 1) { + state ^= state << 13 + state ^= state >>> 17 + state ^= state << 5 + const photo = pixel * 3 + const alpha = pixel * 4 + photoPixels[photo] = state & 0xff + photoPixels[photo + 1] = state >> 8 & 0xff + photoPixels[photo + 2] = state >> 16 & 0xff + alphaPixels[alpha] = photoPixels[photo] ?? 0 + alphaPixels[alpha + 1] = photoPixels[photo + 1] ?? 0 + alphaPixels[alpha + 2] = photoPixels[photo + 2] ?? 0 + alphaPixels[alpha + 3] = pixel & 0xff + } + const photoSource = new Uint8Array(await sharp(photoPixels, { + raw: { width: side, height: side, channels: 3 }, + }).png().toBuffer()) + const alphaSource = new Uint8Array(await sharp(alphaPixels, { + raw: { width: side, height: side, channels: 4 }, + }).png().toBuffer()) + const photo = (await attachments.saveImage({ data: photoSource, mediaType: 'image/png' })).ref + const alpha = (await attachments.saveImage({ data: alphaSource, mediaType: 'image/png' })).ref + + const photoRequest = await attachments.readImageRequest(photo, { maxPixels: 128 * 128, maxBytes: 1024 * 1024 }) + const alphaRequest = await attachments.readImageRequest(alpha, { maxPixels: 128 * 128, maxBytes: 4_096 }) + + expect(photoRequest.mediaType).toBe('image/jpeg') + expect(alphaRequest.bytes).toBeLessThanOrEqual(4_096) + expect(alphaRequest.width).toBeLessThan(128) + await expect(sharp(alphaRequest.data).metadata()).resolves.toMatchObject({ hasAlpha: true, depth: 'uchar', space: 'srgb' }) + }) + + it.each([3, 4] as const)('projects a 16-bit %s-channel PNG as a bounded 8-bit request image', async (channels) => { + const attachments = await store() + const source = new Uint8Array(await sharp({ + create: { width: 64, height: 32, channels, background: { r: 12, g: 34, b: 56, alpha: 0.5 } }, + }).toColourspace('rgb16').png().toBuffer()) + const master = (await attachments.saveImage({ data: source, mediaType: 'image/png' })).ref + + const request = await attachments.readImageRequest(master, { maxPixels: 16 * 16, maxBytes: 1024 * 1024 }) + + expect(request.bytes).toBeLessThanOrEqual(1024 * 1024) + expect(request.width * request.height).toBeLessThanOrEqual(16 * 16) + await expect(sharp(request.data).metadata()).resolves.toMatchObject({ + depth: 'uchar', space: 'srgb', hasAlpha: channels === 4, + }) + }) + + it('retains an all-opaque alpha channel in a resized request version', async () => { + const attachments = await store() + const source = new Uint8Array(await sharp({ + create: { width: 64, height: 32, channels: 4, background: { r: 12, g: 34, b: 56, alpha: 1 } }, + }).png().toBuffer()) + const master = (await attachments.saveImage({ data: source, mediaType: 'image/png' })).ref + + const request = await attachments.readImageRequest(master, { maxPixels: 16 * 16, maxBytes: 1024 * 1024 }) + + await expect(sharp(request.data).metadata()).resolves.toMatchObject({ hasAlpha: true }) + }) + + it('keeps a complex 640,000-pixel request version below 1 MiB', async () => { + const attachments = await store() + const side = 1024 + const pixels = new Uint8Array(side * side * 3) + let state = 0x6d2b79f5 + for (let index = 0; index < pixels.length; index += 1) { + state ^= state << 13 + state ^= state >>> 17 + state ^= state << 5 + pixels[index] = state & 0xff + } + const source = new Uint8Array(await sharp(pixels, { + raw: { width: side, height: side, channels: 3 }, + }).png().toBuffer()) + const master = (await attachments.saveImage({ data: source, mediaType: 'image/png' })).ref + + const request = await attachments.readImageRequest(master, { maxPixels: 640_000, maxBytes: 1024 * 1024 }) + + expect(request).toMatchObject({ width: 800, height: 800 }) + expect(request.bytes).toBeLessThanOrEqual(1024 * 1024) + }) + + it('shares one request transform between concurrent callers without sharing cancellation', async () => { + const attachments = await store() + const master = (await attachments.saveImage({ + data: await image(2048, 1024), mediaType: 'image/png', name: 'shared.png', + })).ref + const run = vi.spyOn(CompressionLimiter.prototype, 'run') + const controller = new AbortController() + const policy = { maxPixels: 640_000, maxBytes: 1024 * 1024 } + + const cancelled = attachments.readImageRequest(master, policy, controller.signal) + const completed = attachments.readImageRequest(master, policy) + const reason = new Error('cancel one waiter') + controller.abort(reason) + + await expect(cancelled).rejects.toBe(reason) + await expect(completed).resolves.toMatchObject({ width: 1130, height: 565 }) + expect(run).toHaveBeenCalledTimes(1) + run.mockRestore() + }) +}) diff --git a/packages/attachment/attachment-local/tests/store.spec.ts b/packages/attachment/attachment-local/tests/store.spec.ts index 8fdd076f6e..97445c2f85 100644 --- a/packages/attachment/attachment-local/tests/store.spec.ts +++ b/packages/attachment/attachment-local/tests/store.spec.ts @@ -7,7 +7,7 @@ import { mkdtemp, rm } from 'node:fs/promises' import { afterEach, describe, expect, it, vi } from 'vitest' import sharp from 'sharp' import type { ImageAttachmentLimits } from '@deepseek-ai/dsh-attachment' -import type { CanonicalImagePolicy } from '../src/canonical.ts' +import type { MasterImagePolicy } from '../src/canonical.ts' import { readImageFile, saveImageFile } from '../src/store.ts' const fsControl = vi.hoisted(() => ({ @@ -35,11 +35,11 @@ vi.mock('node:fs/promises', async (importOriginal) => { }) const PNG = Uint8Array.from(Buffer.from( - 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=', + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC', 'base64', )) -const POLICY: CanonicalImagePolicy = { maxDimension: 2048, maxBytes: 1024 * 1024 } +const POLICY: MasterImagePolicy = { maxDimension: 2048, maxBytes: 1024 * 1024 } const LIMITS: ImageAttachmentLimits = { maxImageBytes: 1024, @@ -139,7 +139,7 @@ describe('local attachment store', () => { await expect(readImageFile(storageRoot, first.ref)).resolves.toEqual({ ref: first.ref, data: PNG }) }) - it('stores the canonical encoding of an oversized source and reads it back verified', async () => { + it('stores the image master of an oversized source and reads it back verified', async () => { const storageRoot = await root() const oversized = new Uint8Array(await sharp({ create: { width: 4, height: 4, channels: 3, background: { r: 9, g: 9, b: 9 } }, diff --git a/packages/attachment/attachment/README.i18n.yaml b/packages/attachment/attachment/README.i18n.yaml index 9c61d2fe81..221699165d 100644 --- a/packages/attachment/attachment/README.i18n.yaml +++ b/packages/attachment/attachment/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/attachment/attachment/README.md -README.md: 3b80444804a345bd019fe94f25954933aa549518 -README.zh.md: 37be4a4a9f54a7e7ddb5fdceb57711378c2f2cfc +README.md: c4925addf079cdd65defb733e6bc40f91ed6384f +README.zh.md: 5623e0944c6f67e2cdaa90076d794cd617c46d5f diff --git a/packages/attachment/attachment/README.md b/packages/attachment/attachment/README.md index 3b80444804..c4925addf0 100644 --- a/packages/attachment/attachment/README.md +++ b/packages/attachment/attachment/README.md @@ -2,15 +2,15 @@ English | [中文](README.zh.md) -The durable attachment seam. `ctx.attachments` validates and durably commits immutable image bytes, then returns a serializable `ImageAttachmentRef`; consumers never persist browser paths, object URLs, provider URLs, or base64 in session events. +The durable attachment seam. `ctx.attachments` validates and durably commits a provider-independent master image, then returns a serializable `ImageAttachmentRef`; consumers never persist browser paths, object URLs, provider URLs, or base64 in session events. -Unsent composer images remain browser-owned temporary drafts. `validateImage` runs the complete admission policy without persisting, including any canonical-encoding dry run the implementation applies, so batch validation proves every member can also be committed. `saveImages` owns batch count and aggregate-byte limits, validates every member before writing any member, then commits in order and returns references only after the complete batch succeeds. A later storage failure returns no partial references, although an earlier immutable content-addressed object may remain unreachable until reference-aware garbage collection exists. `AttachmentError.code` uses the closed `AttachmentErrorCode` string union. Its `ImageAdmissionErrorCode` subset marks caller-correctable image-input failures; `isImageAdmissionError` recognizes that subset at runtime so each protocol adapter can map its own error vocabulary. `saveImage` commits one accepted image before any model-visible session event is published and resolves `SavedImageAttachment`: an implementation may persist a canonical re-encoding of the submitted raster, so the returned `ref` always describes the stored bytes while `source` (`SourceImageInfo`) preserves the submitted raster's media type, byte length, and dimensions for callers that report or map coordinates against the original. `readImage` verifies the content-addressed object against its logged metadata. Callers may cancel `readImage`; implementations observe cancellation around backend and verification work and preserve it instead of translating it into a storage failure. +Unsent composer images remain browser-owned temporary drafts. `validateImage` runs the complete admission policy without persisting. `saveImages` owns batch count and aggregate-byte limits, prepares every validated master once before publishing any member, then commits in order and returns references only after the complete batch succeeds. A later storage failure returns no partial references, although an earlier immutable content-addressed object may remain unreachable until reference-aware garbage collection exists. `AttachmentError.code` uses the closed `AttachmentErrorCode` string union. Its `ImageAdmissionErrorCode` subset marks caller-correctable image-input failures; `isImageAdmissionError` recognizes that subset at runtime so each protocol adapter can map its own error vocabulary. `saveImage` commits one accepted image before any model-visible session event is published and resolves `SavedImageAttachment`: the returned `ref` describes the stored master while `source` (`SourceImageInfo`) preserves the submitted raster's media type, byte length, and orientation-applied dimensions. `readImage` verifies that master against its logged metadata. `readImageRequest` deterministically derives a route-sized request version whose identity covers the master id, transform version, pixel and byte budgets, crop, and encoder settings; `readImageRequests` preserves ordered results while implementations apply their own bounded concurrency. `cropImage` maps preview coordinates to the stored master and persists the result as a new attachment. Callers may cancel reads and projections; implementations preserve cancellation instead of translating it into a storage failure. `admitEncodedImages(attachments, images)` is the shared wire entry used by every RPC endpoint that accepts browser uploads (the session prompt endpoint and the command executor): it enforces canonical base64 on every member, then delegates batch admission — limits, validation, ordered commit — to `saveImages`. The base64 upload form is `EncodedImageAttachment`, exported from `@deepseek-ai/dsh-attachment/types` so wire contracts can reference it. ## Model Experience -Indirectly, through the role-neutral core `ImageBlock` and provider adapters that resolve its durable reference. +Indirectly, through the role-neutral core `ImageBlock` and provider adapters that resolve its durable reference into an exact request version. Request descriptors expose the complete attachment id, actual preview dimensions, and the `read_image_region` coordinate system. #### KV Cache effect diff --git a/packages/attachment/attachment/README.zh.md b/packages/attachment/attachment/README.zh.md index 37be4a4a9f..5623e0944c 100644 --- a/packages/attachment/attachment/README.zh.md +++ b/packages/attachment/attachment/README.zh.md @@ -2,15 +2,15 @@ [English](README.md) | 中文 -持久附件服务边界。`ctx.attachments` 校验并持久提交不可变图片字节,随后返回可序列化的 `ImageAttachmentRef`;消费方绝不会在会话事件中持久保存浏览器路径、对象 URL、提供方 URL 或 base64。 +持久附件服务边界。`ctx.attachments` 校验并持久提交提供方无关的图片主版本,随后返回可序列化的 `ImageAttachmentRef`;消费方绝不会在会话事件中持久保存浏览器路径、对象 URL、提供方 URL 或 base64。 -未发送的输入区图片仍是由浏览器持有的临时草稿。`validateImage` 运行完整的准入策略但不执行持久化,包含实现所应用的规范编码干跑,因此批量校验能证明每个成员随后也能提交成功。`saveImages` 负责批次图片数量和总字节限制,先校验全部成员,再按顺序提交,并且只在完整批次成功后返回引用。后续存储失败不会返回部分引用,但较早写入的不可变内容寻址对象可能保持不可达,直至具备按引用感知的垃圾回收。`AttachmentError.code` 使用封闭的 `AttachmentErrorCode` 字符串联合类型。其 `ImageAdmissionErrorCode` 子集标记可由调用方修正的图片输入失败;`isImageAdmissionError` 在运行时识别该子集,使每个协议适配器可以映射自己的错误词汇。`saveImage` 会在发布任何模型可见的会话事件前提交一张已接受的图片,并解析为 `SavedImageAttachment`:实现可以持久保存所提交光栅的规范重编码,因此返回的 `ref` 始终描述实际存储的字节,而 `source`(`SourceImageInfo`)保留所提交光栅的媒体类型、字节长度和尺寸,供需要对照原图汇报或换算坐标的调用方使用。`readImage` 则根据已记录的元数据校验内容寻址对象。调用方可以取消 `readImage`;实现会在后端读取与校验工作的边界观察取消,并保留取消语义,而不会将其转换为存储失败。 +未发送的输入区图片仍是由浏览器持有的临时草稿。`validateImage` 运行完整准入策略但不执行持久化。`saveImages` 负责批次图片数量和总字节限制,在发布任何成员前为全部成员各准备一次经过验证的主版本,然后按顺序提交,并且只在完整批次成功后返回引用。后续存储失败不会返回部分引用,但较早写入的不可变内容寻址对象可能保持不可达,直至具备按引用感知的垃圾回收。`AttachmentError.code` 使用封闭的 `AttachmentErrorCode` 字符串联合类型。其 `ImageAdmissionErrorCode` 子集标记可由调用方修正的图片输入失败;`isImageAdmissionError` 在运行时识别该子集,使每个协议适配器可以映射自己的错误词汇。`saveImage` 会在发布任何模型可见的会话事件前提交一张已接受的图片,并解析为 `SavedImageAttachment`:返回的 `ref` 描述实际存储的主版本,而 `source`(`SourceImageInfo`)保留所提交光栅的媒体类型、字节长度和应用方向后的尺寸。`readImage` 根据已记录的元数据校验该主版本。`readImageRequest` 确定性派生路由所需的请求版本,其身份覆盖主版本 ID、变换策略版本、像素和字节预算、裁剪区域及编码参数;`readImageRequests` 保持结果顺序,并由实现施加自己的有界并发。`cropImage` 把预览坐标映射到存储的主版本,并把结果保存为新附件。调用方可以取消读取和投影;实现保留取消结果,不把它转换为存储失败。 `admitEncodedImages(attachments, images)` 是每个接受浏览器上传的 RPC 端点(会话 prompt 端点与命令执行器)共用的 wire 入口:它对每个成员强制执行规范 base64,随后把批量准入——限额、校验、有序提交——委托给 `saveImages`。base64 上传形式为 `EncodedImageAttachment`,从 `@deepseek-ai/dsh-attachment/types` 导出,供 wire 契约引用。 ## 模型体验 -该包通过角色无关的核心 `ImageBlock`,以及解析其持久引用的提供方适配器,间接影响模型。 +该包通过角色无关的核心 `ImageBlock`,以及把持久引用解析为确定请求版本的提供方适配器,间接影响模型。请求描述会公开完整附件 ID、实际预览尺寸和 `read_image_region` 使用的坐标系。 #### KV 缓存影响 diff --git a/packages/attachment/attachment/src/brand.ts b/packages/attachment/attachment/src/brand.ts index 6df4014f74..e783076982 100644 --- a/packages/attachment/attachment/src/brand.ts +++ b/packages/attachment/attachment/src/brand.ts @@ -13,3 +13,15 @@ export type AttachmentId = Branded<'AttachmentId'> export function AttachmentId(value: string): AttachmentId { return value as AttachmentId } + +/** Opaque deterministic identity for one request-image transformation. */ +export type ImageVariantId = Branded<'ImageVariantId'> + +/** + * Brand a validated request-image transformation identifier. + * @param value - attachment-provider-produced opaque identifier. + * @returns the branded identifier. + */ +export function ImageVariantId(value: string): ImageVariantId { + return value as ImageVariantId +} diff --git a/packages/attachment/attachment/src/error.ts b/packages/attachment/attachment/src/error.ts index 2e2d695dae..c19229872b 100644 --- a/packages/attachment/attachment/src/error.ts +++ b/packages/attachment/attachment/src/error.ts @@ -23,6 +23,7 @@ export type AttachmentErrorCode = | 'ATTACHMENT_WRITE_FAILED' | 'ATTACHMENT_NOT_FOUND' | 'ATTACHMENT_READ_FAILED' + | 'ATTACHMENT_PROJECTION_UNSUPPORTED' /** Runtime membership for structurally compatible errors crossing package boundaries. */ const IMAGE_ADMISSION_ERROR_CODE_SET: ReadonlySet = new Set(IMAGE_ADMISSION_ERROR_CODES) diff --git a/packages/attachment/attachment/src/index.ts b/packages/attachment/attachment/src/index.ts index 8b3f81a98f..705346d4cc 100644 --- a/packages/attachment/attachment/src/index.ts +++ b/packages/attachment/attachment/src/index.ts @@ -5,12 +5,15 @@ import { AttachmentError } from './error.ts' import type { ImageAttachmentLimits, ImageAttachmentRef, + ImageRequestPolicy, + PreviewImageCrop, + RequestImageAttachment, SaveImageAttachment, SavedImageAttachment, StoredImageAttachment, } from './types.ts' -export { AttachmentId } from './brand.ts' +export { AttachmentId, ImageVariantId } from './brand.ts' export { AttachmentError, isImageAdmissionError } from './error.ts' export type { AttachmentErrorCode, ImageAdmissionErrorCode } from './error.ts' export { admitEncodedImages } from './admission.ts' @@ -19,7 +22,11 @@ export type { EncodedImageAttachment, ImageAttachmentLimits, ImageAttachmentRef, + ImageRequestPolicy, ImageMediaType, + MasterImageCrop, + PreviewImageCrop, + RequestImageAttachment, SaveImageAttachment, SavedImageAttachment, SourceImageInfo, @@ -57,7 +64,7 @@ export abstract class AttachmentStore extends Service { * @param inputs - encoded images in their owning message order. * @returns durable references in the exact input order. */ - async saveImages(inputs: readonly SaveImageAttachment[]): Promise { + protected validateImageBatch(inputs: readonly SaveImageAttachment[]): void { const { maxImagesPerMessage, maxMessageImageBytes, mediaTypes } = this.imageLimits if (inputs.length > maxImagesPerMessage) { throw new AttachmentError('Image batch exceeds the configured image-count limit.', 'TOO_MANY_IMAGES') @@ -71,6 +78,15 @@ export abstract class AttachmentStore extends Service { throw new AttachmentError(`Image type ${input.mediaType} is not accepted by this deployment.`, 'UNSUPPORTED_IMAGE_TYPE') } } + } + + /** + * Validate and durably commit one ordered image batch. + * @param inputs - encoded images in owning-message order. + * @returns durable master references in the same order after every member succeeds. + */ + async saveImages(inputs: readonly SaveImageAttachment[]): Promise { + this.validateImageBatch(inputs) for (const input of inputs) await this.validateImage(input) const refs: ImageAttachmentRef[] = [] @@ -80,7 +96,7 @@ export abstract class AttachmentStore extends Service { /** * Validate and durably commit one image before its owning session event is appended. - * Implementations may store a canonical re-encoding of the submitted raster; + * Implementations may store a prepared master version of the submitted raster; * the returned reference always describes the stored bytes, while `source` * preserves the submitted raster's intrinsic facts for callers that report * or map coordinates against the original. @@ -93,10 +109,70 @@ export abstract class AttachmentStore extends Service { * Read one image and verify that bytes still match the recorded reference. * @param ref - durable reference from the session log. * @param signal - optional cancellation for backend read and verification work. - * @returns the verified bytes and canonical reference. + * @returns the verified bytes and master reference. * @throws the signal reason when aborted, or a storage error when verification fails. */ abstract readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise + + /** + * Generate or read one deterministic model-request version from the stored master image. + * @param ref - durable provider-independent master reference. + * @param policy - exact route pixel and encoded-byte budget. + * @param signal - optional cancellation. + * @returns request bytes and the cache/upload identity covering every transform input. + */ + readImageRequest( + ref: ImageAttachmentRef, + policy: ImageRequestPolicy, + signal?: AbortSignal, + ): Promise { + signal?.throwIfAborted() + void ref + void policy + return Promise.reject(new AttachmentError( + 'The mounted attachment provider cannot derive model-request images.', + 'ATTACHMENT_PROJECTION_UNSUPPORTED', + )) + } + + /** + * Generate or read an ordered batch of deterministic model-request versions. + * Implementations may use their own bounded transform concurrency while preserving input order. + * @param refs - durable provider-independent master references in request order. + * @param policy - exact route pixel and encoded-byte budget shared by the batch. + * @param signal - optional cancellation. + * @returns request versions in the same order as `refs`. + */ + async readImageRequests( + refs: readonly ImageAttachmentRef[], + policy: ImageRequestPolicy, + signal?: AbortSignal, + ): Promise { + const versions: RequestImageAttachment[] = [] + for (const ref of refs) versions.push(await this.readImageRequest(ref, policy, signal)) + return versions + } + + /** + * Crop the stored master by coordinates measured on a model request preview and persist the result. + * @param ref - session-authorized master attachment. + * @param crop - preview dimensions and preview-coordinate rectangle. + * @param signal - optional cancellation. + * @returns a new durable attachment reference suitable for a logged tool result. + */ + cropImage( + ref: ImageAttachmentRef, + crop: PreviewImageCrop, + signal?: AbortSignal, + ): Promise { + signal?.throwIfAborted() + void ref + void crop + return Promise.reject(new AttachmentError( + 'The mounted attachment provider cannot crop stored images.', + 'ATTACHMENT_PROJECTION_UNSUPPORTED', + )) + } } export default AttachmentStore diff --git a/packages/attachment/attachment/src/types.ts b/packages/attachment/attachment/src/types.ts index 22db4c6d23..1d83cf1afa 100644 --- a/packages/attachment/attachment/src/types.ts +++ b/packages/attachment/attachment/src/types.ts @@ -1,6 +1,6 @@ /** Durable attachment vocabulary. @module @deepseek-ai/dsh-attachment/types */ -import type { AttachmentId } from './brand.ts' +import type { AttachmentId, ImageVariantId } from './brand.ts' export type { AttachmentId } from './brand.ts' @@ -21,6 +21,10 @@ export interface ImageAttachmentRef { height: number /** Optional display name stripped of local path information. */ name?: string + /** Perceived source width before master-version downscaling; present only when it differs from {@link width}. */ + sourceWidth?: number + /** Perceived source height before master-version downscaling; present only when it differs from {@link height}. */ + sourceHeight?: number } /** Deployment-resolved limits used by upload admission and request buffering. */ @@ -59,7 +63,57 @@ export interface StoredImageAttachment { data: Uint8Array } -/** Intrinsic facts of the submitted source raster, before any canonical re-encoding. */ +/** Pixel rectangle in the oriented 2048px master-version coordinate system. */ +export interface MasterImageCrop { + x: number + y: number + width: number + height: number +} + +/** Deterministic request-image policy selected by one exact model route. */ +export interface ImageRequestPolicy { + /** Maximum width multiplied by height after aspect-preserving projection. */ + maxPixels: number + /** Encoded-byte cap before base64 expansion or Files API upload. */ + maxBytes: number + /** Optional master-coordinate crop applied before pixel-budget scaling. */ + crop?: MasterImageCrop +} + +/** Cached request version derived from one provider-independent master attachment. */ +export interface RequestImageAttachment { + /** Cache and upload-index key over the master id, policy, crop, and fixed encoder parameters. */ + variantId: ImageVariantId + /** Durable master reference from which this request version was derived. */ + master: ImageAttachmentRef + /** Encoded request bytes. */ + data: Uint8Array + mediaType: ImageMediaType + bytes: number + width: number + height: number + /** Provider-compatible sample depth proven after request encoding. */ + depth: 'uchar' + /** Provider-compatible color space proven after request encoding. */ + space: 'srgb' + /** Whether the encoded request version retains an alpha channel. */ + hasAlpha: boolean + /** Applied master-coordinate crop, when present. */ + crop?: MasterImageCrop +} + +/** Crop coordinates measured by a model on the request preview it received. */ +export interface PreviewImageCrop { + previewWidth: number + previewHeight: number + x: number + y: number + width: number + height: number +} + +/** Intrinsic facts of the submitted source raster, before master-version preparation. */ export interface SourceImageInfo { /** Media type verified from the submitted bytes. */ mediaType: ImageMediaType diff --git a/packages/attachment/attachment/tests/index.spec.ts b/packages/attachment/attachment/tests/index.spec.ts index b3460a77ab..3a8fa23cbe 100644 --- a/packages/attachment/attachment/tests/index.spec.ts +++ b/packages/attachment/attachment/tests/index.spec.ts @@ -3,9 +3,12 @@ import { describe, expect, it } from 'vitest' import AttachmentStore, { AttachmentError, AttachmentId, + ImageVariantId, isImageAdmissionError, type ImageAttachmentRef, type ImageMediaType, + type ImageRequestPolicy, + type RequestImageAttachment, type SaveImageAttachment, type SavedImageAttachment, type StoredImageAttachment, @@ -52,6 +55,25 @@ class RecordingStore extends AttachmentStore { readImage(_ref: ImageAttachmentRef): Promise { throw new Error('not used') } + + override readImageRequest( + ref: ImageAttachmentRef, + _policy: ImageRequestPolicy, + ): Promise { + this.calls.push(`request:${ref.name}`) + return Promise.resolve({ + variantId: ImageVariantId(`sha256:${String(ref.bytes).padStart(64, '0')}`), + master: ref, + data: Uint8Array.of(ref.bytes), + mediaType: ref.mediaType, + bytes: 1, + width: ref.width, + height: ref.height, + depth: 'uchar', + space: 'srgb', + hasAlpha: false, + }) + } } function image(value: number, mediaType: ImageMediaType = 'image/png'): SaveImageAttachment { @@ -101,6 +123,19 @@ describe('AttachmentStore.saveImages', () => { }) }) +describe('AttachmentStore.readImageRequests', () => { + it('uses the default serial projection and preserves input order', async () => { + const store = new RecordingStore(new Context()) + const refs = await store.saveImages([image(1), image(2)]) + store.calls.length = 0 + + const versions = await store.readImageRequests(refs, { maxPixels: 1, maxBytes: 1 }) + + expect(store.calls).toEqual(['request:1.png', 'request:2.png']) + expect(versions.map(version => version.master.name)).toEqual(['1.png', '2.png']) + }) +}) + describe('isImageAdmissionError', () => { it('separates caller-correctable image admission failures from storage faults', () => { expect(isImageAdmissionError(new AttachmentError('bad bytes', 'INVALID_IMAGE'))).toBe(true) diff --git a/packages/extensions/tool-cordis/src/api-catalog.ts b/packages/extensions/tool-cordis/src/api-catalog.ts index 60f8ac66f3..ad5d04fd5d 100644 --- a/packages/extensions/tool-cordis/src/api-catalog.ts +++ b/packages/extensions/tool-cordis/src/api-catalog.ts @@ -438,13 +438,13 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ }, { signature: 'async saveImages(inputs: readonly SaveImageAttachment[]): Promise', - description: 'Validate one ordered image batch before committing any member. Validation failures start no writes; storage failures return no partial references, although already published content-addressed objects may stay unreachable until a future retention policy collects them.', - parameters: [{ name: 'inputs', description: 'encoded images in their owning message order.' }], - returns: 'durable references in the exact input order.', + description: 'Validate and durably commit one ordered image batch.', + parameters: [{ name: 'inputs', description: 'encoded images in owning-message order.' }], + returns: 'durable master references in the same order after every member succeeds.', }, { signature: 'abstract saveImage(input: SaveImageAttachment): Promise', - description: 'Validate and durably commit one image before its owning session event is appended. Implementations may store a canonical re-encoding of the submitted raster; the returned reference always describes the stored bytes, while `source` preserves the submitted raster\'s intrinsic facts for callers that report or map coordinates against the original.', + description: 'Validate and durably commit one image before its owning session event is appended. Implementations may store a prepared master version of the submitted raster; the returned reference always describes the stored bytes, while `source` preserves the submitted raster\'s intrinsic facts for callers that report or map coordinates against the original.', parameters: [{ name: 'input', description: 'encoded bytes, declared media type, and optional display name.' }], returns: 'the durable content-addressed reference beside the submitted source facts.', }, @@ -452,9 +452,27 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ signature: 'abstract readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise', description: 'Read one image and verify that bytes still match the recorded reference.', parameters: [{ name: 'ref', description: 'durable reference from the session log.' }, { name: 'signal', description: 'optional cancellation for backend read and verification work.' }], - returns: 'the verified bytes and canonical reference.', + returns: 'the verified bytes and master reference.', throws: ['the signal reason when aborted, or a storage error when verification fails.'], }, + { + signature: 'async readImageRequest( ref: ImageAttachmentRef, policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise', + description: 'Generate or read one deterministic model-request version from the stored master image.', + parameters: [{ name: 'ref', description: 'durable provider-independent master reference.' }, { name: 'policy', description: 'exact route pixel and encoded-byte budget.' }, { name: 'signal', description: 'optional cancellation.' }], + returns: 'request bytes and the cache/upload identity covering every transform input.', + }, + { + signature: 'async readImageRequests( refs: readonly ImageAttachmentRef[], policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise', + description: 'Generate or read an ordered batch of deterministic model-request versions. Implementations may use their own bounded transform concurrency while preserving input order.', + parameters: [{ name: 'refs', description: 'durable provider-independent master references in request order.' }, { name: 'policy', description: 'exact route pixel and encoded-byte budget shared by the batch.' }, { name: 'signal', description: 'optional cancellation.' }], + returns: 'request versions in the same order as `refs`.', + }, + { + signature: 'async cropImage( ref: ImageAttachmentRef, crop: PreviewImageCrop, signal?: AbortSignal, ): Promise', + description: 'Crop the stored master by coordinates measured on a model request preview and persist the result.', + parameters: [{ name: 'ref', description: 'session-authorized master attachment.' }, { name: 'crop', description: 'preview dimensions and preview-coordinate rectangle.' }, { name: 'signal', description: 'optional cancellation.' }], + returns: 'a new durable attachment reference suitable for a logged tool result.', + }, ], }, { @@ -3450,7 +3468,7 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'ImageAttachmentRef', - declaration: 'export interface ImageAttachmentRef {\n attachmentId: AttachmentId;\n mediaType: ImageMediaType;\n bytes: number;\n width: number;\n height: number;\n name?: string;\n}', + declaration: 'export interface ImageAttachmentRef {\n attachmentId: AttachmentId;\n mediaType: ImageMediaType;\n bytes: number;\n width: number;\n height: number;\n name?: string;\n sourceWidth?: number;\n sourceHeight?: number;\n}', }, { name: 'ImageBlock', @@ -3460,6 +3478,14 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'ImageMediaType', declaration: 'export type ImageMediaType = \'image/png\' | \'image/jpeg\' | \'image/webp\' | \'image/gif\';', }, + { + name: 'ImageRequestPolicy', + declaration: 'export interface ImageRequestPolicy {\n maxPixels: number;\n maxBytes: number;\n crop?: MasterImageCrop;\n}', + }, + { + name: 'ImageVariantId', + declaration: 'export type ImageVariantId = Branded<\'ImageVariantId\'>;', + }, { name: 'Inbox', declaration: 'export class Inbox {\n constructor(private readonly session: Session, private readonly notifications: InboxNotifications);\n get nextTurn(): readonly UserMessage[];\n get nextStep(): readonly UserMessage[];\n get hasPending(): boolean;\n clear(): void;\n claim(target: InboxTarget, turn: number): UserMessage[];\n append(target: InboxTarget, message: UserMessage): void;\n prepend(target: InboxTarget, message: UserMessage): void;\n replace(messageId: MessageId, newMessage: UserMessage): boolean;\n remove(messageId: MessageId): boolean;\n splice(target: InboxTarget, start: number, deleteCount: number, inserted: UserMessage[]): UserMessage[];\n}', @@ -3586,7 +3612,7 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'LlmAdapter', - declaration: 'export abstract class LlmAdapter {\n providerInfo(provider: string): LlmProviderInfo;\n providerRetryPolicy(_provider: string): ResolvedRetryPolicy | undefined;\n listModels(_provider: string): Promise;\n resolveModel(provider: string, model: string, _signal?: AbortSignal): Promise;\n abstract stream(options: GenerateOptions): AsyncIterable;\n}', + declaration: 'export abstract class LlmAdapter {\n providerInfo(provider: string): LlmProviderInfo;\n providerRetryPolicy(_provider: string): ResolvedRetryPolicy | undefined;\n listModels(_provider: string): Promise;\n resolveModel(provider: string, model: string, _signal?: AbortSignal): Promise;\n async prepareCall(provider: string, model: string, signal?: AbortSignal): Promise;\n abstract stream(options: GenerateOptions): AsyncIterable;\n}', }, { name: 'LlmCallConfig', @@ -3684,6 +3710,10 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'ManualCompactAgentContext', declaration: 'export interface ManualCompactAgentContext extends CompactionAgentContext {\n runMaintenance(task: (signal: AbortSignal) => Promise): Promise;\n}', }, + { + name: 'MasterImageCrop', + declaration: 'export interface MasterImageCrop {\n x: number;\n y: number;\n width: number;\n height: number;\n}', + }, { name: 'Message', declaration: 'export interface Message {\n readonly id: MessageId;\n readonly role: \'system\' | \'user\' | \'assistant\';\n readonly content: ContentBlock[];\n readonly source: MessageSource;\n}', @@ -3804,9 +3834,13 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'PostToolDecision', declaration: 'export type PostToolDecision = {\n kind: \'accept\';\n content?: ContentBlock[];\n value?: never;\n additionalContexts?: UserMessage[];\n} | {\n kind: \'accept\';\n value: JsonValue;\n content?: never;\n additionalContexts?: UserMessage[];\n} | {\n kind: \'block\';\n feedback: ContentBlock[];\n additionalContexts?: UserMessage[];\n};', }, + { + name: 'PreparedAdapterCall', + declaration: 'export interface PreparedAdapterCall {\n readonly model: LlmResolvedModelInfo;\n stream(options: GenerateOptions): AsyncIterable;\n}', + }, { name: 'PreparedLlmCall', - declaration: 'export interface PreparedLlmCall {\n readonly config: LlmCallConfig;\n readonly retryPolicy: ResolvedRetryPolicy;\n readonly context?: LlmModelContext;\n readonly adapterDefaults: LlmCallConfigAdapterDefaults;\n stream(options: GenerateOptions): AsyncIterable;\n}', + declaration: 'export interface PreparedLlmCall {\n readonly config: LlmCallConfig;\n readonly retryPolicy: ResolvedRetryPolicy;\n readonly context?: LlmModelContext;\n readonly inputModalities?: readonly ModelModality[];\n readonly adapterDefaults: LlmCallConfigAdapterDefaults;\n stream(options: GenerateOptions): AsyncIterable;\n}', }, { name: 'PreparedReferencedMessage', @@ -3836,6 +3870,10 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'PreToolDecision', declaration: 'export type PreToolDecision = {\n kind: \'allow\';\n} | {\n kind: \'deny\';\n reason: string;\n} | {\n kind: \'ask\';\n reason?: string;\n};', }, + { + name: 'PreviewImageCrop', + declaration: 'export interface PreviewImageCrop {\n previewWidth: number;\n previewHeight: number;\n x: number;\n y: number;\n width: number;\n height: number;\n}', + }, { name: 'ProjectionChangeListener', declaration: 'export type ProjectionChangeListener = (session: Session, key: Extract, value: unknown, seq: number) => void;', @@ -3916,6 +3954,10 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'RequestHeaderReason', declaration: 'export type RequestHeaderReason = \'initial\' | \'resume\' | \'change\';', }, + { + name: 'RequestImageAttachment', + declaration: 'export interface RequestImageAttachment {\n variantId: ImageVariantId;\n master: ImageAttachmentRef;\n data: Uint8Array;\n mediaType: ImageMediaType;\n bytes: number;\n width: number;\n height: number;\n depth: \'uchar\';\n space: \'srgb\';\n hasAlpha: boolean;\n crop?: MasterImageCrop;\n}', + }, { name: 'RequestRunOutcome', declaration: 'export type RequestRunOutcome = \'approved\' | \'completed\' | \'rejected\' | \'cancelled\' | \'failed\';', diff --git a/packages/fs/tool-fs/README.i18n.yaml b/packages/fs/tool-fs/README.i18n.yaml index 3d9c4606c4..47084a3435 100644 --- a/packages/fs/tool-fs/README.i18n.yaml +++ b/packages/fs/tool-fs/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/fs/tool-fs/README.md -README.md: 22384ddb18f2b36e9b8a177ee62eed9424ddcd6a -README.zh.md: 74c41f4f25d19089c40a52cff4e3dfa654630b1a +README.md: 94af10c501bcb86465d685f1f20c7d42f3b9d117 +README.zh.md: 4b8e826db3ae15b825d2f888e7d37fc3cafd1b23 diff --git a/packages/fs/tool-fs/README.md b/packages/fs/tool-fs/README.md index 22384ddb18..94af10c501 100644 --- a/packages/fs/tool-fs/README.md +++ b/packages/fs/tool-fs/README.md @@ -2,7 +2,7 @@ English | [中文](README.zh.md) -The **model-facing filesystem tools** — `read`, `read_image`, `write`, `edit` — and their **executor**. This is the consumer layer of the filesystem stack: it owns tool names, JSON schemas, argument validation, prompt sections, **read windowing**, and result formatting. It reads/writes/edits through the `ctx.fs` provider contract ([`@deepseek-ai/dsh-fs`](../fs)) **directly**. The freshness/observation policy is contributed by a separate plugin ([`@deepseek-ai/dsh-fs-observation-policy`](../fs-observation-policy)) through the `fs/*` event gate; the tool is not method-coupled to it. Under a confining provider, the shared sandbox-policy service is required for per-session execution and the tool exposes escalation for filesystem mutations. +The **model-facing filesystem tools** — `read`, `read_image`, `read_image_region`, `write`, `edit` — and their **executor**. This is the consumer layer of the filesystem stack: it owns tool names, JSON schemas, argument validation, prompt sections, **read windowing**, and result formatting. It reads/writes/edits through the `ctx.fs` provider contract ([`@deepseek-ai/dsh-fs`](../fs)) **directly**. The freshness/observation policy is contributed by a separate plugin ([`@deepseek-ai/dsh-fs-observation-policy`](../fs-observation-policy)) through the `fs/*` event gate; the tool is not method-coupled to it. Under a confining provider, the shared sandbox-policy service is required for per-session execution and the tool exposes escalation for filesystem mutations. ```ts ignore-check // Default deployment: a ctx.fs provider, the policy plugin, then the tools. @@ -14,7 +14,7 @@ await ctx.plugin(ToolFs) // this package — re `@deepseek-ai/dsh-fs-observation-policy` is **optional**: omit it and the tools run against the bare provider (unconditional write/overwrite/edit, no observed-state). A deployment that loads these tools is expected to also load it, so the behavior is read-before-write/edit. -`read_image` registers only while a durable `ctx.attachments` service is mounted — without one the deployment cannot commit image bytes, so the tool never appears. Execution additionally requires the exact routed model to declare `image` input (resolved through `ctx.llm.resolveModelInfo` from the session's latest request header, falling back to agent options); an unknown or text-only route gets a refusal result before any filesystem I/O, so a text route's durable history stays free of image blocks. +`read_image` and `read_image_region` register only while a durable `ctx.attachments` service is mounted. Execution additionally requires the exact routed model to declare `image` input (resolved through `ctx.llm.resolveModelInfo` from the session's latest request header, falling back to agent options). `read_image_region` accepts only a complete attachment id already referenced by the calling session, so it can crop a user upload without a filesystem path but cannot cross session scope. ## Config @@ -33,12 +33,13 @@ All keys are optional; the defaults are the shipped read caps. |---|---|---| | `read` | `file_path`, `offset?`, `limit?` | Line-numbered UTF-8 content with a pagination footer. `offset` is 1-based; `limit` defaults to and caps at the configured `readLimit` (2000). | | `read_image` | `file_path` | Reads a PNG/JPEG/WebP/GIF file through the bounded byte seam, persists it through `ctx.attachments.saveImage`, and returns an image block beside a small metadata envelope. It succeeds only when the exact routed model declares image input. | +| `read_image_region` | `attachment_id`, `preview_width`, `preview_height`, `x`, `y`, `width`, `height` | Resolves a session-authorized image, maps the preview-coordinate rectangle to its stored master, persists the crop, and returns the new image block. | | `write` | `file_path`, `content` | Create or fully replace a file. With the policy plugin: overwriting an existing file requires a prior `read` at the unchanged version; creating a new file does not. Without it: unconditional. | | `edit` | `file_path`, non-empty `old_string`, `new_string`, `replace_all?` | Literal replacement; unique match required unless `replace_all` is true. With the policy plugin: requires a prior `read` (any window) and the file unchanged since. Without it: unconditional. | Field names are snake_case to match Claude Code and existing harness tool schemas. -Canonical successes are `read` → `{ path, offset, lines: [{ number, text }], totalLines }`, `read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name?, sourceWidth?, sourceHeight? } }` (the source fields appear only when the attachment store's canonical encoding downscaled the file, and the envelope then names the coordinate multiplier back to the original), `write` → `{ path, operation: 'create' | 'update', before: string | null, after }`, and `edit` → `{ path, before, after }`. Native renderers preserve the line-numbered read and mutation acknowledgements below. `write`/`edit` derive replayable diff-card metadata, and `read` derives a replayable read-card window `{ path, offset, lines, totalLines, lang? }`, from these canonical values; the canonical values themselves are execution-local and are not added to `tool/result`, only the derived presentation metadata is persisted. +Structured successes are `read` → `{ path, offset, lines: [{ number, text }], totalLines }`, `read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name?, sourceWidth?, sourceHeight? } }`, `read_image_region` → `{ sourceAttachmentId, preview, crop, image }`, `write` → `{ path, operation: 'create' | 'update', before: string | null, after }`, and `edit` → `{ path, before, after }`. The image source fields appear only when master preparation downscaled the submitted raster. Native renderers preserve the line-numbered read and mutation acknowledgements below. `write`/`edit` derive replayable diff-card metadata, and `read` derives a replayable read-card window `{ path, offset, lines, totalLines, lang? }`; execution-local structured values are not added to `tool/result`, while image renderers emit the durable image blocks that the result logs. ## The tool is the executor; policy is an event gate @@ -46,6 +47,7 @@ The tools do **not** inject a policy service or inspect any cache. Each tool res - **read** — one `ctx.fs.stat` (type + size routing + version), then `readText`/`streamText`, then builds the line window, then emits `fs/observed` with a plain `ctx.emit`. (1 stat.) - **read_image** — validates the argument, extension, attachment availability, deployment media types, and the image-capable route before any I/O; then one `ctx.fs.stat` (recording an `absent` observation for a missing target, like `read`), a bounded `ctx.fs.readBytes` capped at the smaller of `imageLimits.maxImageBytes` and `imageLimits.maxMessageImageBytes` (the result is one message carrying one image), `attachments.saveImage` (content-addressed, so the image block references a durably committed object by the time `tool/result` is appended), and finally `fs/observed`. (1 stat.) +- **read_image_region** — resolves the full attachment id only from current session messages, validates integer preview coordinates, maps the rectangle to the stored master through `attachments.cropImage`, and returns the persisted crop as an image block. It performs no filesystem-path operation and emits no `fs/observed` event. - **write** — `ctx.waterfall('fs/write-intent', target, exec, () => undefined)` for the optional guard, then `ctx.fs.writeText(target, content, intent)`, then `fs/observed`. (0 stat.) - **edit** — `ctx.waterfall('fs/edit-intent', target, exec, () => undefined)` for the optional guard, then `ctx.fs.editText(target, edit, intent)`, then `fs/observed`. (0 stat.) @@ -99,7 +101,7 @@ Prefix-stable while the plugin scope and guidance text are unchanged. Tool restr #### What the model sees -The model sees the generated [`read`, `read_image`, `write`, and `edit` schemas](../../../docs/tool-catalog.md#deepseek-aidsh-tool-fs), with snake_case arguments. `read_image` appears only while a durable attachment store is mounted; the schema itself is route-independent, and the strict gate refuses at execution. Scoped tool restrictions can remove any definition for one agent. +The model sees the generated [`read`, `read_image`, `read_image_region`, `write`, and `edit` schemas](../../../docs/tool-catalog.md#deepseek-aidsh-tool-fs), with snake_case arguments. The image tools appear only while a durable attachment store is mounted; their schemas are route-independent, and the strict gate refuses at execution. Scoped tool restrictions can remove any definition for one agent. #### Token effect @@ -127,7 +129,7 @@ Append-only; newly visible content follows the reusable request prefix and does #### What the model sees -A successful `read_image` returns ``, `image`, and a `` envelope naming the media type, dimensions, and byte size, followed by the image itself as a native image block. The session log stores only the durable `sha256:` attachment reference; the routed provider re-reads and digest-verifies the bytes on each request. +A successful `read_image` returns ``, `image`, and a `` envelope naming the media type, master dimensions, and byte size, followed by the image itself as a native image block. A successful `read_image_region` returns an `image-region` envelope naming the source attachment, supplied preview dimensions and rectangle, and result dimensions, followed by the crop as a native image block. The result is logged with its new durable reference before the next model request. Request adapters derive previews from the master, so later region reads never crop an already reduced preview. #### Token effect @@ -155,7 +157,7 @@ Append-only; newly visible content follows the reusable request prefix and does #### What the model sees -Failures are normalized as `Error: `. This package's stable validation and read messages are `file_path must be a non-empty string`, `limit must be less than or equal to `, `old_string must be a non-empty string`, `old_string and new_string must differ`, `cannot read "": not found`, `cannot read "": not a regular file`, `offset is out of range for "" ( lines)`, `cannot read "": read_image only accepts PNG/JPEG/WebP/GIF paths`, `cannot read "" as an image: model "" does not declare image input; switch to an image-capable model to read images`, and the mismatch repair `cannot read "": the extension declares , but the bytes use a different image format; rename the file to match its actual format if it is PNG/JPEG/WebP/GIF, or convert it to one of those formats`; provider and policy templates are quoted in their package READMEs. Guarded-mutation failures additionally carry their recovery instruction in the message, appended by this package's model-facing error wrapper: `FS_STALE_VERSION` gets `— re-read the file, then retry`, and `FS_NOT_OBSERVED` gets `— read the file, then retry`; the structured code is preserved. After that reread confirms absence, edit reports `FS_NOT_FOUND` instead of repeating a stale remedy, while write uses guarded creation. +Failures are normalized as `Error: `. This package's stable validation and read messages are `file_path must be a non-empty string`, `limit must be less than or equal to `, `old_string must be a non-empty string`, `old_string and new_string must differ`, `cannot read "": not found`, `cannot read "": not a regular file`, `offset is out of range for "" ( lines)`, `cannot read "": read_image only accepts PNG/JPEG/WebP/GIF paths`, `cannot read "" as an image: model "" does not declare image input; switch to an image-capable model to read images`, and the mismatch repair `cannot read "": the extension declares , but the bytes use a different image format; rename the file to match its actual format if it is PNG/JPEG/WebP/GIF, or convert it to one of those formats`. A failed 16-bit conversion reports `cannot read "": the 16-bit PNG could not be converted to the canonical 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`. Region reads reject empty or out-of-scope attachment ids and invalid preview rectangles before storage mutation. Provider and policy templates are quoted in their package READMEs. Guarded-mutation failures additionally carry their recovery instruction in the message, appended by this package's model-facing error wrapper: `FS_STALE_VERSION` gets `— re-read the file, then retry`, and `FS_NOT_OBSERVED` gets `— read the file, then retry`; the structured code is preserved. After that reread confirms absence, edit reports `FS_NOT_FOUND` instead of repeating a stale remedy, while write uses guarded creation. #### Token effect @@ -169,7 +171,6 @@ Append-only; newly visible content follows the reusable request prefix and does - **No model-facing directory listing ships** — `ctx.fs.listDir` serves provider code such as skill discovery, while the sibling [`dsh-tool-fs-search`](../tool-fs-search/) package supplies ripgrep-backed `glob` and `grep` rather than extending the filesystem seam. - **`read` handles UTF-8 text files only** — images use the separate extension-routed `read_image` tool; PDF, audio, and video remain deferred. A directory target is `FS_NOT_REGULAR_FILE`. -- **The route gate races a concurrent model switch** — `read_image` checks the latest routed model at execution; a switch committed between that check and the next request can leave an image block on a route that rejects image content. The Web host already refuses switching an image-bearing session to a text-only model; other front doors own their equivalent guard. - **Extension-declared media type** — the extension selects the declared type and the attachment store's magic-byte validation stays authoritative; a correctly formatted image under a wrong extension is refused with the rename remedy rather than sniffed. - **No inline image preview on the tool-result card** — UI surfaces render the image result generically (the durable reference, not pixels); inline rendering is deferred to the UI packages. - **No timeout surface** — `read`/`write`/`edit` take no timeout argument and declare no `timeout-policy` budget; cancellation rides `exec.signal` only ([provider rationale](../README.md#no-timeouts-on-file-io)). diff --git a/packages/fs/tool-fs/README.zh.md b/packages/fs/tool-fs/README.zh.md index 74c41f4f25..4b8e826db3 100644 --- a/packages/fs/tool-fs/README.zh.md +++ b/packages/fs/tool-fs/README.zh.md @@ -2,7 +2,7 @@ [English](README.md) | 中文 -**面向模型的文件系统工具**(`read`、`read_image`、`write`、`edit`)及其**执行器**。这是文件系统栈的消费方层:拥有工具名称、JSON Schema、参数校验、提示词段、**读取窗口逻辑**和结果格式化。它**直接**通过 `ctx.fs` 提供方约定([`@deepseek-ai/dsh-fs`](../fs))读取/写入/编辑。新鲜度/观察策略由独立插件([`@deepseek-ai/dsh-fs-observation-policy`](../fs-observation-policy))通过 `fs/*` 事件门禁贡献;工具不与其方法耦合。使用施加沙箱限制的提供方时,逐会话执行需要共享沙箱策略服务,工具还会为文件系统变更提供升权路径。 +**面向模型的文件系统工具**(`read`、`read_image`、`read_image_region`、`write`、`edit`)及其**执行器**。这是文件系统栈的消费方层:拥有工具名称、JSON Schema、参数校验、提示词段、**读取窗口逻辑**和结果格式化。它**直接**通过 `ctx.fs` 提供方约定([`@deepseek-ai/dsh-fs`](../fs))读取、写入和编辑。新鲜度与观察策略由独立插件([`@deepseek-ai/dsh-fs-observation-policy`](../fs-observation-policy))通过 `fs/*` 事件门禁贡献;工具不与其方法耦合。使用施加沙箱限制的提供方时,逐会话执行需要共享沙箱策略服务,工具还会为文件系统变更提供升权路径。 ```ts ignore-check // Default deployment: a ctx.fs provider, the policy plugin, then the tools. @@ -14,7 +14,7 @@ await ctx.plugin(ToolFs) // this package — re `@deepseek-ai/dsh-fs-observation-policy` 是**可选的**:省略时,工具直接使用裸提供方(无条件写入/覆盖/编辑,无已观察状态)。加载这些工具的部署也应加载该插件,从而提供写入/编辑前读取行为。 -`read_image` 只在持久 `ctx.attachments` 服务已挂载时注册:没有它,部署无法持久提交图像字节,工具就不会出现。执行时还要求确切路由的模型声明 `image` 输入(通过 `ctx.llm.resolveModelInfo` 从会话最新请求 header 解析,缺失时回退到 agent 选项);未知或纯文本路由在任何文件系统 I/O 之前就得到拒绝结果,因此文本路由的持久历史不会出现图像块。 +`read_image` 和 `read_image_region` 只在持久 `ctx.attachments` 服务已挂载时注册。执行时还要求确切路由的模型声明 `image` 输入,通过 `ctx.llm.resolveModelInfo` 从会话最新请求 header 解析,缺失时回退到 agent 选项。`read_image_region` 只接受调用会话已经引用的完整附件 ID,因此可以裁剪没有文件路径的用户上传图片,但不能越过会话范围。 ## 配置 @@ -33,12 +33,13 @@ await ctx.plugin(ToolFs) // this package — re |---|---|---| | `read` | `file_path`、`offset?`、`limit?` | 带行号的 UTF-8 内容和分页 footer。`offset` 从 1 开始;`limit` 默认为配置的 `readLimit`(2000),上限也为该值。 | | `read_image` | `file_path` | 通过有界字节 seam 读取 PNG/JPEG/WebP/GIF 文件,经 `ctx.attachments.saveImage` 持久保存,并在小型元数据信封旁返回图像块。只有确切路由的模型声明图像输入时才会成功。 | +| `read_image_region` | `attachment_id`、`preview_width`、`preview_height`、`x`、`y`、`width`、`height` | 解析会话有权访问的图片,把预览坐标矩形映射到存储主版本,持久保存裁剪结果并返回新图片块。 | | `write` | `file_path`、`content` | 创建文件或完整替换文件。有策略插件时:覆盖现有文件要求先在未变版本上执行 `read`;创建新文件不需要。没有插件时:无条件执行。 | | `edit` | `file_path`、非空 `old_string`、`new_string`、`replace_all?` | 字面量替换;除非 `replace_all` 为 true,否则要求唯一匹配。有策略插件时:要求先执行 `read`(任何窗口),且文件此后未变。没有插件时:无条件执行。 | 字段名使用 snake_case,与 Claude Code 和现有 harness 工具 schema 一致。 -规范成功值分别为:`read` → `{ path, offset, lines: [{ number, text }], totalLines }`,`read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name?, sourceWidth?, sourceHeight? } }`(source 两个字段仅在附件存储的规范编码缩小了该文件时出现,此时信封会写明换算回原图的坐标倍率),`write` → `{ path, operation: 'create' | 'update', before: string | null, after }`,`edit` → `{ path, before, after }`。原生渲染器会保留下方带行号的读取结果和变更确认。`write`/`edit` 从这些规范值派生可回放的 diff 卡片元数据,`read` 派生可回放的读取卡片窗口 `{ path, offset, lines, totalLines, lang? }`;规范值本身仅限于本次执行,不会添加到 `tool/result`,只有派生出的呈现元数据会被持久化。 +结构化成功值分别为:`read` → `{ path, offset, lines: [{ number, text }], totalLines }`,`read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name?, sourceWidth?, sourceHeight? } }`,`read_image_region` → `{ sourceAttachmentId, preview, crop, image }`,`write` → `{ path, operation: 'create' | 'update', before: string | null, after }`,`edit` → `{ path, before, after }`。图片 source 字段只在主版本准备缩小了提交光栅时出现。原生渲染器会保留下方带行号的读取结果和变更确认。`write` 和 `edit` 从这些值派生可回放的 diff 卡片元数据,`read` 派生可回放的读取卡片窗口 `{ path, offset, lines, totalLines, lang? }`;仅用于执行的结构化值不会添加到 `tool/result`,图片渲染器则会发出由结果记录的持久图片块。 ## 工具就是执行器;策略是事件门禁 @@ -46,6 +47,7 @@ await ctx.plugin(ToolFs) // this package — re - **read**:一次 `ctx.fs.stat`(用于类型、大小路由和版本),随后调用 `readText`/`streamText`,构建行窗口,再发出 `fs/observed`,使用普通 `ctx.emit`。(1 次 stat。) - **read_image**:在任何 I/O 之前校验参数、扩展名、附件可用性、部署接受的媒体类型和图像路由;随后一次 `ctx.fs.stat`(目标缺失时与 `read` 一样记录 `absent` 观察)、以 `imageLimits.maxImageBytes` 与 `imageLimits.maxMessageImageBytes` 中较小者为上限的有界 `ctx.fs.readBytes`(结果是携带一张图像的一条消息)、`attachments.saveImage`(内容寻址,因此在 `tool/result` 事件追加时图像块引用的对象已持久提交),最后发出 `fs/observed`。(1 次 stat。) +- **read_image_region**:只从当前会话消息解析完整附件 ID,校验整数预览坐标,通过 `attachments.cropImage` 把矩形映射到存储主版本,并把持久裁剪结果作为图片块返回。它不执行文件系统路径操作,也不发出 `fs/observed` 事件。 - **write**:调用 `ctx.waterfall('fs/write-intent', target, exec, () => undefined)` 取得可选防护,然后调用 `ctx.fs.writeText(target, content, intent)`,再发出 `fs/observed`。(0 次 stat。) - **edit**:调用 `ctx.waterfall('fs/edit-intent', target, exec, () => undefined)` 取得可选防护,然后调用 `ctx.fs.editText(target, edit, intent)`,再发出 `fs/observed`。(0 次 stat。) @@ -99,7 +101,7 @@ Use the edit tool for targeted changes to existing UTF-8 text files. It replaces #### 模型看到的内容 -模型会看到已生成的 [`read`、`read_image`、`write` 和 `edit` schema](../../../docs/tool-catalog.zh.md#deepseek-aidsh-tool-fs),参数使用 snake_case。`read_image` 只在持久附件存储已挂载时出现;schema 本身与路由无关,严格门禁在执行时拒绝。作用域工具限制可以为某个 agent 移除任一定义。 +模型会看到已生成的 [`read`、`read_image`、`read_image_region`、`write` 和 `edit` schema](../../../docs/tool-catalog.zh.md#deepseek-aidsh-tool-fs),参数使用 snake_case。图片工具只在持久附件存储已挂载时出现;schema 本身与路由无关,严格门禁在执行时拒绝。作用域工具限制可以为某个 agent 移除任一定义。 #### Token 影响 @@ -127,7 +129,7 @@ Use the edit tool for targeted changes to existing UTF-8 text files. It replaces #### 模型看到的内容 -成功的 `read_image` 返回 ``、`image` 和写明媒体类型、尺寸与字节数的 `` 信封,随后是作为原生图像块的图像本身。会话日志只存储持久的 `sha256:` 附件引用;路由到的提供方在每次请求时重新读取并校验字节摘要。 +成功的 `read_image` 返回 ``、`image` 和写明媒体类型、主版本尺寸与字节数的 `` 信封,随后是作为原生图像块的图像本身。成功的 `read_image_region` 返回 `image-region` 信封,写明源附件、提交的预览尺寸和矩形及结果尺寸,随后是作为原生图像块的裁剪结果。新持久引用会随结果写入会话日志,然后才进入下一次模型请求。请求适配器从主版本派生预览,因此之后的局部读取不会从已经缩小的预览继续裁剪。 #### Token 影响 @@ -155,7 +157,7 @@ Use the edit tool for targeted changes to existing UTF-8 text files. It replaces #### 模型看到的内容 -失败会规范化为 `Error: `。本包稳定的校验和读取消息是 `file_path must be a non-empty string`、`limit must be less than or equal to `、`old_string must be a non-empty string`、`old_string and new_string must differ`、`cannot read "": not found`、`cannot read "": not a regular file`、`offset is out of range for "" ( lines)`、`cannot read "": read_image only accepts PNG/JPEG/WebP/GIF paths`、`cannot read "" as an image: model "" does not declare image input; switch to an image-capable model to read images`,以及类型不匹配的修复消息 `cannot read "": the extension declares , but the bytes use a different image format; rename the file to match its actual format if it is PNG/JPEG/WebP/GIF, or convert it to one of those formats`;提供方和策略模板在各自包的 README 中逐字列出。防护变更失败还会在消息中携带恢复指令,由本包面向模型的错误包装追加:`FS_STALE_VERSION` 追加 `— re-read the file, then retry`,`FS_NOT_OBSERVED` 追加 `— read the file, then retry`;结构化错误码保持不变。该次重新读取确认缺失后,edit 会报告 `FS_NOT_FOUND`,而不会重复陈旧恢复指令;write 则使用带防护的创建。 +失败会规范化为 `Error: `。本包稳定的校验和读取消息是 `file_path must be a non-empty string`、`limit must be less than or equal to `、`old_string must be a non-empty string`、`old_string and new_string must differ`、`cannot read "": not found`、`cannot read "": not a regular file`、`offset is out of range for "" ( lines)`、`cannot read "": read_image only accepts PNG/JPEG/WebP/GIF paths`、`cannot read "" as an image: model "" does not declare image input; switch to an image-capable model to read images`,以及类型不匹配的修复消息 `cannot read "": the extension declares , but the bytes use a different image format; rename the file to match its actual format if it is PNG/JPEG/WebP/GIF, or convert it to one of those formats`。16-bit 转换失败会报告 `cannot read "": the 16-bit PNG could not be converted to the canonical 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`。局部读取会在改变存储前拒绝空白或超出会话范围的附件 ID 以及无效预览矩形。提供方和策略模板在各自包的 README 中逐字列出。防护变更失败还会在消息中携带恢复指令,由本包面向模型的错误包装追加:`FS_STALE_VERSION` 追加 `re-read the file, then retry`,`FS_NOT_OBSERVED` 追加 `read the file, then retry`;结构化错误码保持不变。该次重新读取确认缺失后,edit 会报告 `FS_NOT_FOUND`,不会重复陈旧恢复指令;write 则使用带防护的创建。 #### Token 影响 @@ -169,7 +171,6 @@ Use the edit tool for targeted changes to existing UTF-8 text files. It replaces - **未交付面向模型的目录列表工具**:`ctx.fs.listDir` 服务于 skill(技能)发现等提供方代码,同级 [`dsh-tool-fs-search`](../tool-fs-search/) 包则提供基于 ripgrep 的 `glob` 与 `grep`,而不是扩展文件系统 seam。 - **`read` 只处理 UTF-8 文本文件**:图像使用独立的、按扩展名路由的 `read_image` 工具;PDF、音频和视频仍延期处理。目录目标为 `FS_NOT_REGULAR_FILE`。 -- **路由门禁与并发模型切换存在竞态**:`read_image` 在执行时检查最新路由的模型;在该检查与下一次请求之间提交的切换,可能让图像块落在拒绝图像内容的路由上。Web 宿主已拒绝把含图像的会话切到纯文本模型;其他前端拥有各自的等价防护。 - **媒体类型按扩展名声明**:扩展名选择声明类型,附件存储的魔数校验保持权威;扩展名错误但格式正确的图像会得到改名修复提示,而不是被嗅探接受。 - **工具结果卡片没有内嵌图像预览**:UI 表面以通用形式渲染图像结果(持久引用而非像素);内嵌渲染延后到 UI 包处理。 - **没有超时接口**:`read`/`write`/`edit` 不接受超时参数,也不声明 `timeout-policy` 预算;取消只通过 `exec.signal` 传递(见[提供方理由](../README.zh.md#no-timeouts-on-file-io))。 diff --git a/packages/fs/tool-fs/src/read-image.ts b/packages/fs/tool-fs/src/read-image.ts index cde24a2a35..4766bea6ba 100644 --- a/packages/fs/tool-fs/src/read-image.ts +++ b/packages/fs/tool-fs/src/read-image.ts @@ -1,20 +1,19 @@ /** - * The model-facing `read_image` tool: reads a PNG/JPEG/WebP/GIF file, durably - * commits its bytes through the attachment service (the same lifecycle as a - * user-uploaded image), and returns an image block so the image enters model - * context from the next request onward. + * The model-facing image tools: `read_image` commits a PNG/JPEG/WebP/GIF file, + * while `read_image_region` crops a session-authorized durable attachment by + * coordinates measured on the exact preview shown to the model. * - * The route gate is deliberately stricter than the host upload preflight: a - * tool result enters durable session history, so emitting an image on a route - * that cannot carry it would break that route's continuation. Unknown - * capability therefore refuses instead of relying on the adapter guard. + * The route gate is deliberately stricter than the host upload preflight. An + * image-reading tool is useful only when the exact calling route can inspect + * its result, so unknown capability refuses instead of relying on an adapter + * failure after filesystem and attachment work. * @module @deepseek-ai/dsh-tool-fs/src/read-image */ import { basename, extname } from 'node:path' import type { Context } from '@deepseek-ai/cordis' import { AttachmentError, AttachmentId } from '@deepseek-ai/dsh-attachment' -import type { ImageAttachmentRef, ImageMediaType } from '@deepseek-ai/dsh-attachment' +import type { ImageAttachmentRef, ImageMediaType, PreviewImageCrop } from '@deepseek-ai/dsh-attachment' import type { ContentBlock } from '@deepseek-ai/dsh-llm' import { defineTool } from '@deepseek-ai/dsh-tools' import type { GenericCallView, ToolExecution } from '@deepseek-ai/dsh-tools' @@ -30,7 +29,7 @@ const IMAGE_EXTENSIONS: Readonly> = { '.gif': 'image/gif', } -/** The canonical outcome declared by the `read_image` output schema. */ +/** The structured outcome declared by the `read_image` output schema. */ export interface ImageReadValue { path: string image: { @@ -47,6 +46,14 @@ export interface ImageReadValue { } } +/** Structured result of cropping a session-authorized image attachment. */ +export interface ImageRegionReadValue { + sourceAttachmentId: string + preview: { width: number; height: number } + crop: { x: number; y: number; width: number; height: number } + image: ImageReadValue['image'] +} + /** * Map a model-supplied path to its declared image media type by extension. * @param filePath - the raw `file_path` argument (not yet resolved). @@ -79,9 +86,9 @@ export async function assertImageCapableRoute(ctx: Context, exec: ToolExecution, } /** - * Re-brand a canonical image outcome into the durable attachment reference an + * Re-brand a structured image outcome into the durable attachment reference an * `ImageBlock` carries. - * @param image - the canonical image metadata from the output schema. + * @param image - the image metadata from the output schema. * @returns the branded attachment reference. */ export function imageRefFromValue(image: ImageReadValue['image']): ImageAttachmentRef { @@ -92,15 +99,66 @@ export function imageRefFromValue(image: ImageReadValue['image']): ImageAttachme width: image.width, height: image.height, ...image.name === undefined ? {} : { name: image.name }, + ...image.sourceWidth === undefined ? {} : { sourceWidth: image.sourceWidth }, + ...image.sourceHeight === undefined ? {} : { sourceHeight: image.sourceHeight }, } } +function findImageRef( + content: readonly ContentBlock[], + attachmentId: string, +): ImageAttachmentRef | undefined { + for (const block of content) { + if (block.type === 'image' && block.attachment.attachmentId === attachmentId) return block.attachment + if (block.type === 'tool-result') { + const nested = findImageRef(block.content, attachmentId) + if (nested !== undefined) return nested + } + } + return undefined +} + +function sessionImageRef(exec: ToolExecution, attachmentId: string): ImageAttachmentRef { + const session = exec.agent?.session + if (session === undefined) { + throw new Error('read_image_region requires an active agent session') + } + for (const message of session.deriveMessages()) { + const ref = findImageRef(message.content, attachmentId) + if (ref !== undefined) return ref + } + throw new Error(`attachment "${attachmentId}" is not referenced by the current session`) +} + +function positiveInteger(value: number, name: string): number { + if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`) + return value +} + +function nonNegativeInteger(value: number, name: string): number { + if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`) + return value +} + +function regionReadContent(value: ImageRegionReadValue): ContentBlock[] { + return [ + { + type: 'text', + text: `${value.sourceAttachmentId}\nimage-region\n\n` + + `preview ${value.preview.width}x${value.preview.height} px; crop ` + + `x=${value.crop.x}, y=${value.crop.y}, width=${value.crop.width}, height=${value.crop.height}; ` + + `result ${value.image.width}x${value.image.height} px\n`, + }, + { type: 'image', attachment: imageRefFromValue(value.image) }, + ] +} + /** * Format an image read as the model-facing envelope beside its image block. * A downscaled read names the on-disk dimensions and the multiplier that maps * coordinates measured on the attached image back onto the original file. * @param displayPath - the backend-resolved path rendered in the envelope's `` element. - * @param image - the canonical image metadata to summarize. + * @param image - the image metadata to summarize. * @returns the model-facing envelope; the image itself rides the adjacent image block. */ export function formatImageReadOutput(displayPath: string, image: ImageReadValue['image']): string { @@ -123,8 +181,8 @@ ${image.mediaType} image, ${image.width}x${image.height} px, ${image.bytes} byte } /** - * Project one canonical image read into its model-facing envelope and image. - * @param value - the canonical image-read outcome. + * Project one structured image read into its model-facing envelope and image. + * @param value - the image-read outcome. * @returns the two content blocks used by native and nested dispatches. */ function imageReadContent(value: ImageReadValue): ContentBlock[] { @@ -233,6 +291,12 @@ export function applyReadImageTool(ctx: Context): void { { cause: error }, ) } + if (error.code === 'ATTACHMENT_WRITE_FAILED' && /16-bit PNG/iu.test(error.message)) { + throw new Error( + `cannot read "${target.displayPath}": the 16-bit PNG could not be converted to the canonical 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`, + { cause: error }, + ) + } if (error.code !== 'IMAGE_TYPE_MISMATCH') throw error const extension = extname(target.displayPath).toLowerCase() throw new Error( @@ -267,4 +331,101 @@ export function applyReadImageTool(ctx: Context): void { } }, })) + + ctx.tools.register(defineTool({ + name: 'read_image_region', + description: 'Crop a region from an image attachment already visible in this session. Coordinates use the preview dimensions supplied beside that image.', + parameters: { + attachment_id: { type: 'string', required: true, description: 'Complete attachment id shown beside the image.' }, + preview_width: { type: 'integer', required: true, description: 'Width of the preview shown to the model.' }, + preview_height: { type: 'integer', required: true, description: 'Height of the preview shown to the model.' }, + x: { type: 'integer', required: true, description: 'Left edge in preview pixels.' }, + y: { type: 'integer', required: true, description: 'Top edge in preview pixels.' }, + width: { type: 'integer', required: true, description: 'Crop width in preview pixels.' }, + height: { type: 'integer', required: true, description: 'Crop height in preview pixels.' }, + }, + output: { + schema: { + type: 'object', + additionalProperties: false, + properties: { + sourceAttachmentId: { type: 'string', required: true }, + preview: { + type: 'object', + additionalProperties: false, + required: true, + properties: { + width: { type: 'integer', required: true }, + height: { type: 'integer', required: true }, + }, + }, + crop: { + type: 'object', + additionalProperties: false, + required: true, + properties: { + x: { type: 'integer', required: true }, + y: { type: 'integer', required: true }, + width: { type: 'integer', required: true }, + height: { type: 'integer', required: true }, + }, + }, + image: { + type: 'object', + additionalProperties: false, + required: true, + properties: { + attachmentId: { type: 'string', required: true }, + mediaType: { type: 'string', enum: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'], required: true }, + bytes: { type: 'integer', required: true }, + width: { type: 'integer', required: true }, + height: { type: 'integer', required: true }, + name: { type: 'string' }, + sourceWidth: { type: 'integer' }, + sourceHeight: { type: 'integer' }, + }, + }, + }, + }, + render: (_args, value) => regionReadContent(value), + }, + isConcurrencySafe: () => true, + async execute(args, exec) { + const attachmentId = args.attachment_id.trim() + if (attachmentId.length === 0) throw new Error('attachment_id must be a non-empty string') + const ref = sessionImageRef(exec, attachmentId) + await assertImageCapableRoute(ctx, exec, attachmentId) + const crop: PreviewImageCrop = { + previewWidth: positiveInteger(args.preview_width, 'preview_width'), + previewHeight: positiveInteger(args.preview_height, 'preview_height'), + x: nonNegativeInteger(args.x, 'x'), + y: nonNegativeInteger(args.y, 'y'), + width: positiveInteger(args.width, 'width'), + height: positiveInteger(args.height, 'height'), + } + const saved = await ctx.attachments.cropImage(ref, crop, exec.signal) + return { + sourceAttachmentId: ref.attachmentId, + preview: { width: crop.previewWidth, height: crop.previewHeight }, + crop: { x: crop.x, y: crop.y, width: crop.width, height: crop.height }, + image: { + attachmentId: saved.ref.attachmentId, + mediaType: saved.ref.mediaType, + bytes: saved.ref.bytes, + width: saved.ref.width, + height: saved.ref.height, + ...saved.ref.name === undefined ? {} : { name: saved.ref.name }, + ...saved.ref.sourceWidth === undefined ? {} : { sourceWidth: saved.ref.sourceWidth }, + ...saved.ref.sourceHeight === undefined ? {} : { sourceHeight: saved.ref.sourceHeight }, + }, + } + }, + presentCall(args): GenericCallView { + return { + card: 'generic', + title: `Read image region ${args.attachment_id}`, + kind: 'read', + } + }, + })) } diff --git a/packages/fs/tool-fs/tests/read-image.spec.ts b/packages/fs/tool-fs/tests/read-image.spec.ts index dcc6ab7d5e..2f67a464fd 100644 --- a/packages/fs/tool-fs/tests/read-image.spec.ts +++ b/packages/fs/tool-fs/tests/read-image.spec.ts @@ -12,8 +12,8 @@ import { join } from 'node:path' import { Context } from '@deepseek-ai/cordis' import { CodeRuntime } from '@deepseek-ai/dsh-code-runtime' import type { CodeRunRequest, CodeRunResult } from '@deepseek-ai/dsh-code-runtime' -import { CallId, LlmAdapter, LlmRuntime } from '@deepseek-ai/dsh-llm' -import type { GenerateOptions, LlmModelInfo, LlmResolvedModelInfo, StreamChunk } from '@deepseek-ai/dsh-llm' +import { CallId, createUserMessage, LlmAdapter, LlmRuntime } from '@deepseek-ai/dsh-llm' +import type { GenerateOptions, LlmModelInfo, LlmResolvedModelInfo, Message, StreamChunk } from '@deepseek-ai/dsh-llm' import SystemPrompt from '@deepseek-ai/dsh-system-prompt' import ToolRuntime, { RUN_CODE_NAME } from '@deepseek-ai/dsh-tools' import type { Config as ToolConfig } from '@deepseek-ai/dsh-tools' @@ -122,12 +122,13 @@ async function setup(options: SetupOptions = {}) { } /** A fake calling agent pinned to one routed provider/model. */ -function agentOn(model: string | undefined, provider = 'visual'): object { +function agentOn(model: string | undefined, provider = 'visual', messages: readonly Message[] = []): object { return { options: {}, session: { header: { cwd: dir }, requestHeader: () => (model === undefined ? undefined : { config: { provider, model } }), + deriveMessages: () => [...messages], append: () => undefined, }, } @@ -169,6 +170,61 @@ describe('imageRefFromValue', () => { const base = { attachmentId: 'sha256:00', mediaType: 'image/png' as const, bytes: 1, width: 1, height: 1 } expect(imageRefFromValue(base)).toEqual(base) expect(imageRefFromValue({ ...base, name: 'a.png' })).toEqual({ ...base, name: 'a.png' }) + expect(imageRefFromValue({ ...base, sourceWidth: 4, sourceHeight: 2 })) + .toEqual({ ...base, sourceWidth: 4, sourceHeight: 2 }) + }) +}) + +describe('read_image_region', () => { + it('crops a session-visible attachment and returns a new logged image reference', async () => { + const ctx = await setup() + const attachments = ctx.attachments + const source = await attachments.saveImage({ data: PNG_3X3, mediaType: 'image/png', name: 'grid.png' }) + const history = [createUserMessage({ + content: [{ type: 'image', attachment: source.ref }], + source: { kind: 'plugin', plugin: 'test' }, + })] + + const result = await call(ctx, 'read_image_region', { + attachment_id: source.ref.attachmentId, + preview_width: 3, + preview_height: 3, + x: 1, + y: 0, + width: 2, + height: 2, + }, agentOn('vision-model', 'visual', history)) + + expect(result.isError).toBe(false) + expect(result.content[0]).toMatchObject({ + type: 'text', + text: expect.stringContaining('crop x=1, y=0, width=2, height=2') as string, + }) + expect(result.content[1]).toMatchObject({ + type: 'image', + attachment: { width: 2, height: 2, name: 'grid-crop.png' }, + }) + const cropped = result.content[1] + if (cropped?.type !== 'image') throw new Error('expected cropped image block') + await expect(attachments.readImage(cropped.attachment)).resolves.toMatchObject({ + ref: { attachmentId: cropped.attachment.attachmentId }, + }) + }) + + it('refuses an attachment that is absent from the current session', async () => { + const ctx = await setup() + const result = await call(ctx, 'read_image_region', { + attachment_id: `sha256:${'f'.repeat(64)}`, + preview_width: 800, + preview_height: 800, + x: 0, + y: 0, + width: 100, + height: 100, + }, agentOn('vision-model')) + + expect(result.isError).toBe(true) + expect(text(result)).toContain('not referenced by the current session') }) }) @@ -438,6 +494,15 @@ describe('image admission failures', () => { expect(storageFault.isError).toBe(true) expect(text(storageFault)).toContain('Unable to persist image attachment.') + FailingStore.failure = new AttachmentError( + 'The 16-bit PNG could not be converted to the canonical 8-bit sRGB form.', + 'ATTACHMENT_WRITE_FAILED', + ) + const sixteenBit = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model')) + expect(text(sixteenBit)).toContain( + `cannot read "${join(dir, 'red.png')}": the 16-bit PNG could not be converted to the canonical 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`, + ) + FailingStore.failure = new AttachmentError('Image cannot be encoded within the configured canonical byte target.', 'IMAGE_TOO_LARGE') const overBudget = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model')) expect(overBudget.isError).toBe(true) @@ -501,7 +566,7 @@ describe('image admission failures', () => { }) it('names the on-disk dimensions and coordinate multiplier when storage downscales', async () => { - /** Store whose canonical encoding halves the source on both sides. */ + /** Store whose image master halves the source on both sides. */ class DownscalingStore extends AttachmentStore { readonly imageLimits: ImageAttachmentLimits = Object.freeze({ maxImageBytes: 1024, @@ -553,7 +618,7 @@ describe('registration surface', () => { const attachmentsFiber = await ctx.plugin(LocalAttachmentStore, { dshHome: home }) const toolFsFiber = await ctx.plugin(ToolFs) const names = () => ctx.tools.schemas().map(schema => schema.name).sort() - expect(names()).toEqual(['edit', 'read', 'read_image', 'write']) + expect(names()).toEqual(['edit', 'read', 'read_image', 'read_image_region', 'write']) // Disposing only the attachment store tears down the scoped inject fiber: // read_image withdraws while the unconditional tools stay registered. @@ -562,7 +627,7 @@ describe('registration surface', () => { // Remounting the store restores the conditional registration. const remounted = await ctx.plugin(LocalAttachmentStore, { dshHome: home }) - expect(names()).toEqual(['edit', 'read', 'read_image', 'write']) + expect(names()).toEqual(['edit', 'read', 'read_image', 'read_image_region', 'write']) // Disposing the whole plugin withdraws every tool, read_image included. await toolFsFiber.dispose() @@ -581,6 +646,12 @@ describe('registration surface', () => { kind: 'read', locations: [{ path: 'shot.png' }], }) + expect(ctx.tools.executionMode({ + signal: testToolSignal, + callId: CallId('region-parallel'), + name: 'read_image_region', + arguments: { attachment_id: 'sha256:a', preview_width: 1, preview_height: 1, x: 0, y: 0, width: 1, height: 1 }, + })).toEqual({ kind: 'parallel' }) }) }) diff --git a/packages/host/apiproxy/src/api-proxy.ts b/packages/host/apiproxy/src/api-proxy.ts index e708353c00..dd1268fe00 100644 --- a/packages/host/apiproxy/src/api-proxy.ts +++ b/packages/host/apiproxy/src/api-proxy.ts @@ -14,7 +14,7 @@ import type { Agent, ModelSelection, ModelSelectionRef, AgentOptions, AgentStatu import type {} from '@deepseek-ai/dsh-agent-presets/types' import { AttachmentError, admitEncodedImages } from '@deepseek-ai/dsh-attachment' import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment' -import { contentHasImage, createUserMessage, freezeMessage, ReasoningEffortId } from '@deepseek-ai/dsh-llm' +import { createUserMessage, freezeMessage, ReasoningEffortId } from '@deepseek-ai/dsh-llm' import { errorChain } from '@deepseek-ai/dsh-llm' import type { ContentBlock, MessageSource } from '@deepseek-ai/dsh-llm' import { isAppendSurfaceEvent, isJsonValue } from '@deepseek-ai/dsh-session' @@ -186,10 +186,6 @@ function imageInEvent(event: SessionEvent, match: (ref: ImageAttachmentRef) => b } /** True when the current model-visible surface contains an image. */ -function messagesHaveImage(messages: readonly { content: readonly ContentBlock[] }[]): boolean { - return messages.some(message => contentHasImage(message.content)) -} - /** Resolve the first reference matching one opaque id. */ function referencedImage(events: readonly SessionEvent[], attachmentId: string): ImageAttachmentRef | undefined { for (const event of events) { @@ -2221,18 +2217,6 @@ export function createApiProxy(ctx: Context, defaults: ApiProxyDefaults): ApiPro ? {} : { reasoningEffort: ReasoningEffortId(reasoningEffort) }, }) - const pendingImage = [...found.agent.inbox.nextTurn, ...found.agent.inbox.nextStep] - .some(message => contentHasImage(message.content)) - if (pendingImage || messagesHaveImage(found.agent.session.deriveMessages())) { - const info = await ctx.llm.resolveModelInfo(resolved.provider, resolved.model) - if (info.inputModalities !== undefined && !info.inputModalities.includes('image')) { - return err(request, { - code: 'model-unavailable', - message: `Model "${resolved.model}" does not accept image input, but this session already contains images; select an image-capable model.`, - details: { provider, model }, - }) - } - } const selected: ModelSelection = { provider: resolved.provider, model: resolved.model, diff --git a/packages/host/apiproxy/tests/api-proxy-models.spec.ts b/packages/host/apiproxy/tests/api-proxy-models.spec.ts index 55cb15ca9f..99f99c3432 100644 --- a/packages/host/apiproxy/tests/api-proxy-models.spec.ts +++ b/packages/host/apiproxy/tests/api-proxy-models.spec.ts @@ -156,12 +156,7 @@ describe('Web session model selection', () => { validateImage, saveImage, } - ctx.provide('attachments', { - ...attachments, - saveImages(inputs: readonly Parameters[0][]) { - return AttachmentStore.prototype.saveImages.call(attachments, inputs) - }, - } as never) + ctx.provide('attachments', Object.setPrototypeOf(attachments, AttachmentStore.prototype) as never) const followup = vi.fn() Object.assign(agent, { followup }) const api = createApiProxy(ctx, { @@ -207,7 +202,7 @@ describe('Web session model selection', () => { await ctx.fiber.dispose() }) - it('refuses a text-only selection while durable or pending image content remains visible', async () => { + it('allows a text-only selection while durable or pending images remain available for later models', async () => { const { ctx, agent, sessionId } = await harness() registerTextOnly(ctx) const api = createApiProxy(ctx, { @@ -221,9 +216,9 @@ describe('Web session model selection', () => { agent.session.append('user/message', { id: 'image-message', role: 'user', source: { kind: 'user' }, content: [image], } as never, { surfaceOp: 'append' }) - expect((await api.sessions.selectModel(request({ + expect(expectValue(await api.sessions.selectModel(request({ sessionId, provider: 'text-only', model: 'plain', - }))).result).toMatchObject({ ok: false, error: { code: 'model-unavailable' } }) + }))).selected).toEqual({ provider: 'text-only', model: 'plain' }) agent.session.append('user/message', { id: 'summary', role: 'user', source: { kind: 'plugin', plugin: 'compact' }, @@ -235,10 +230,6 @@ describe('Web session model selection', () => { ;(agent.inbox.nextTurn as UserMessage[]).push({ id: 'pending-image', role: 'user', source: { kind: 'user' }, content: [image], } as never) - expect((await api.sessions.selectModel(request({ - sessionId, provider: 'text-only', model: 'plain', - }))).result.ok).toBe(false) - ;(agent.inbox.nextTurn as UserMessage[]).length = 0 expect(expectValue(await api.sessions.selectModel(request({ sessionId, provider: 'text-only', model: 'plain', }))).selected).toEqual({ provider: 'text-only', model: 'plain' }) diff --git a/packages/llm/llm-deepseek/README.i18n.yaml b/packages/llm/llm-deepseek/README.i18n.yaml index 82da16e4f0..c5aa0c7e8c 100644 --- a/packages/llm/llm-deepseek/README.i18n.yaml +++ b/packages/llm/llm-deepseek/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/llm/llm-deepseek/README.md -README.md: bae9011135a9cbc14467086e4b6ebc6f052ed230 -README.zh.md: 0a5f0224dbebd62766775822260585825579f4a7 +README.md: da2044abe6f5201c1bed1ca6b529b34c34282ea8 +README.zh.md: d17d7a739640c31e9e88f154a11d5e24011e54f7 diff --git a/packages/llm/llm-deepseek/README.md b/packages/llm/llm-deepseek/README.md index bae9011135..da2044abe6 100644 --- a/packages/llm/llm-deepseek/README.md +++ b/packages/llm/llm-deepseek/README.md @@ -20,7 +20,12 @@ The package root exposes the Cordis plugin contract and `DeepSeekAdapter`; wire reasoningEffort: high # optional; off | low | high | max — omitted ⇒ high maxTokens: 256000 # optional positive per-request output cap; this is the default streamIdleTimeoutMs: 300000 # optional; positive finite Node timer delay; five-minute default - maxRequestImageBytes: 20971520 # optional positive integer; 20 MiB base64-payload default + maxRequestFilesBytes: 134217728 # optional positive integer; 128 MiB raw request-image default + maxImagesPerRequest: 600 # provider request image-count limit + imageOffloadByteQuantum: 67108864 # oldest-image removal advances in 64 MiB steps + fileExpiresAfterSeconds: 604800 # uploaded image lifetime; 1 hour to 30 days + fileRefreshMarginSeconds: 3600 # replace ids with less lifetime remaining + fileQuotaCleanupBatch: 100 # oldest harness-owned files deleted before one quota retry retryPolicy: # optional; omission uses normal mode with five retries mode: always # normal | always backoff: @@ -34,16 +39,22 @@ The package root exposes the Cordis plugin contract and `DeepSeekAdapter`; wire - id: deepseek-v4-flash-vision-exp name: DeepSeek-V4-Flash-Vision-Exp inputModalities: [text, image] + imagePixelBudget: 640000 + imageMaxBytes: 1048576 - id: private-reasoner description: Company-hosted reasoning model contextWindow: 512000 ``` -The plugin registers the single provider route `deepseek-official` together with its resolved `retryPolicy`; omission resolves to normal mode with five retries. A request selects it with `provider: deepseek-official`; its `model` is passed through as the wire `model` string, so changing DeepSeek models does not require lifecycle-time registration. Omitting `models` advertises `deepseek-v4-flash`, `deepseek-v4-pro`, and the image-capable `deepseek-v4-flash-vision-exp`, each with a 1,000,000-token context window; an explicit list replaces those defaults, while `models: []` advertises none. Catalog entries are exposed through `ctx.llm.listModels('deepseek-official')` for clients such as ACP editors and the Web selector, but remain advisory: unlisted model ids still pass through unchanged. An omitted entry name defaults to its id, and omitted `inputModalities` means `text` only. +The plugin registers the single provider route `deepseek-official` together with its resolved `retryPolicy`; omission resolves to normal mode with five retries. A request selects it with `provider: deepseek-official`; its `model` is passed through as the wire `model` string, so changing DeepSeek models does not require lifecycle-time registration. Omitting `models` advertises `deepseek-v4-flash`, `deepseek-v4-pro`, and the image-capable `deepseek-v4-flash-vision-exp`, each with a 1,000,000-token context window; an explicit list replaces those defaults, while `models: []` advertises none. Catalog entries are exposed through `ctx.llm.listModels('deepseek-official')` for clients such as ACP editors and the Web selector, but remain advisory: unlisted model ids still pass through unchanged as text-only routes. An omitted entry name defaults to its id, and omitted `inputModalities` means `text` only. -An image-capable catalog entry may declare `inputModalities: [text, image]`. The adapter resolves user and tool-result `ImageBlock` references through `ctx.attachments`, verifies the stored bytes, and sends transient `data:;base64,...` `image_url` parts without changing the durable session message. Text-only and unlisted models reject image input before credential, attachment, or network I/O. System and assistant history remain image-free; tool-result images follow their string-only `tool` messages in a separate `user` message. +An image-capable catalog entry declares `inputModalities: [text, image]` and may set `imagePixelBudget`, `imageMaxBytes`, or `imageDetail: low`. The ordinary default is 640,000 total pixels and 1MiB encoded bytes; low detail defaults to 512 by 512 total pixels. The attachment store scales by `min(1, sqrt(pixelBudget / (width * height)))` and rounds inward to keep the pixel count at or below the hard cap, so a 2048 by 1024 master becomes about 1130 by 565 instead of a forced square. Request encoders run lazily: low-color images try PNG (palette only without alpha) then WebP 85 and 80, other alpha images try WebP 85 then 80, and other opaque images try JPEG 85 then 80; dimensions shrink only when both quality attempts exceed 1MiB. Concurrent generation of one `variantId` shares one transform. The adapter uploads the exact derived request bytes through `POST /files` and sends `{type: "file", file_id}` blocks. It never falls back to an inline data URL. Every retained image is preceded by stable text naming the complete attachment id, actual request dimensions, and the preview-coordinate arguments for `read_image_region`. User, tool-result, agent-loop, compaction, and direct `ctx.llm.stream` requests all use this projection. Text-only routes receive stable attachment placeholders while durable history keeps its image references. -`maxRequestImageBytes` bounds accumulated base64 image payload and defaults to 20 MiB, leaving headroom below the official 30 MiB request-body limit for text, tools, and JSON framing. When history exceeds the bound, the oldest images become the fixed model-visible placeholder `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]` until the request fits; omitted attachments are not read. Attachment admission continues to own per-image and per-message raw-byte, media, dimension, and pixel limits. +`maxRequestFilesBytes` and `maxImagesPerRequest` bound the retained request versions at 128MiB and 600 images by default. When the byte bound is crossed, the oldest prefix advances past the next 64MiB boundary; 129 one-megabyte images remove the oldest 65 and retain 64MiB, and that prefix stays unchanged until durable history exceeds 192MiB. Count overflow advances independently in `imageOffloadCountQuantum` steps. Removed images become the fixed model-visible placeholder `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`. This high-watermark projection avoids changing an old request prefix after every new image. + +Uploaded ids are indexed below `DSH_HOME` by endpoint/API-key scope and request `variantId`. The variant covers the master attachment id, transform version, route pixel and byte budgets, crop, and encoder parameters, so Files API and inline-capable adapters refer to the same deterministic bytes. Uploads request a seven-day lifetime by default and store the server's `expires_at`. A local mapping with no more than one hour remaining is replaced before use; the adapter does not retrieve every remote file before chat. If chat reports an expired, deleted, missing, or invalid file id and names a used id, the adapter removes only that mapping. If the provider identifies stale file state without naming an id, it removes every file mapping used by that chat attempt. It then uploads the affected request versions again and retries chat once. A second stale-file rejection clears the mappings identified by that response and is returned without a third chat attempt. An upload response without a complete file object, matching byte count, and `expires_at` is never indexed; a later request therefore uploads again instead of trusting inconsistent local state. A malformed local upload index is treated as an empty cache and replaced by the next successful upload; permission and filesystem I/O failures still fail the request. + +One quota upload failure triggers deletion of the configured number of oldest `dsh-` files and one upload retry. `DeepSeekFilesClient.delete`, `DeepSeekFileStore.release`, and `releaseAll` expose explicit remote-space reclamation. The current provider limits represented by this package are 128MiB per Files upload, 32MiB per chat-referenced image, 10,000 stored files, and 25GiB per API key; the default 1MiB request version remains below the two per-file limits. `contextWindow` is optional per configured model and is not exposed through the advisory catalog. `ctx.llm.resolveModelInfo('deepseek-official', model).context` returns an exact model value first, then `defaultContextWindow` for an entry without capacity or an unlisted pass-through id. The adapter default is 1,000,000; pressure-sensitive plugins therefore get deployment-owned capacity without treating the model selector as authoritative. Registering another adapter for `deepseek-official` throws `LlmError('DUPLICATE_ADAPTER')`. @@ -53,11 +64,11 @@ The same exact-model result exposes ordered `off`, `low`, `high`, and `max` effo `thinking: disabled` is a deployment lock that publishes only `off` with `off` as its default. Omitting `reasoningEffort` or configuring it as `off` is valid; configuring `low`, `high`, or `max` fails plugin loading, and a direct per-request attempt to enable thinking fails before network I/O. A request with `GenerateOptions.purpose: 'session-title'` also forces thinking disabled and omits the already-resolved effort, reserving its bounded output for visible title text without changing conversation or compaction defaults. -`streamIdleTimeoutMs` bounds each outstanding provider read, including the initial `fetch`, without counting time the consumer spends between chunks. DeepSeek SSE comments rearm an outstanding read as transport activity but never become `StreamChunk` values or session-log events. One stable abort signal reaches the request and body reader for the whole call; expiry stops the transport and throws `LlmError('TIMEOUT')`, while an earlier caller abort throws `LlmError('ABORTED')`. The adapter makes exactly one provider request per `stream()` call; it registers the configured policy as provider metadata, and `dsh-llm-retry` separately executes it at durable agent-step boundaries. +`streamIdleTimeoutMs` bounds each outstanding provider read, including the initial `fetch`, without counting time the consumer spends between chunks. DeepSeek SSE comments rearm an outstanding read as transport activity but never become `StreamChunk` values or session-log events. One stable abort signal reaches the request and body reader for the whole call; expiry stops the transport and throws `LlmError('TIMEOUT')`, while an earlier caller abort throws `LlmError('ABORTED')`. The adapter normally makes one chat request per `stream()` call and makes a second only for the stale-file recovery described above. It registers the configured retry policy as provider metadata, and `dsh-llm-retry` separately executes that policy at durable agent-step boundaries. ## Dynamic configuration (settings + credentials) -Connection facts are not frozen at load. `resolveAdapterOptions` is the one explicit resolve step from raw config to validated facts, and the adapter re-reads them through a thunk **once per operation**: base URL, catalog, request defaults, image bound, and idle budget all take effect on the next request, while an in-flight stream keeps the facts it started with. Three optional seams feed that thunk: +Connection facts are not frozen at load. `resolveAdapterOptions` is the one explicit resolve step from raw config to validated facts, and the adapter re-reads them through a thunk **once per operation**: base URL, catalog, request defaults, image and Files policies, and idle budget all take effect on the next request, while an in-flight stream keeps the facts it started with. Three optional seams feed that thunk: - **`ctx.settings`** — the plugin registers the `llm-deepseek` namespace with this same `Config` schema and its `cordis.yml` entry as the composition `base`, so a `llm-deepseek:` section in the user settings document overrides any field without a restart. Without a mounted settings service the entry config alone drives the adapter, unchanged. A live settings snapshot that passes the schema but fails a beyond-schema bound (a duplicate catalog id, a broken thinking/effort pair) keeps the last good facts and logs the failure; the entry config itself still fails plugin load. - **`ctx.credentials`** — the API key resolves per stream call, from the *same* resolved snapshot that supplies the endpoint. Configuration carries only `apiKeyEnv`, never a literal key: the reference resolves through the credential seam, and without a mounted seam through the trusted environment layers. Because credential facts travel with the connection facts, a settings snapshot the resolver rejects contributes neither its endpoint nor its key: the whole previous generation keeps serving. Every resolved key is format-checked before use, so a value no HTTP header can carry is refused with `LlmError('INVALID_CREDENTIAL')` naming the failing entry point — never any part of the key — instead of surfacing as an opaque `fetch` `TypeError`. A request with no key anywhere fails with `MISSING_CREDENTIAL` naming every configuration entry point, while the route stays registered and the catalog stays browsable — first-run onboarding is "browse models, store the key, prompt again", with no restart between. @@ -84,7 +95,7 @@ DeepSeek request identity is separate from app attribution. After credential res ## Errors -Non-2xx responses throw `LlmError` with stable codes: `AUTH` (401/403), `QUOTA` (a response whose provider details identify exhausted quota, balance, or credits), `RATE_LIMIT` (other 429s), `CONTEXT_WINDOW_EXCEEDED` (a 400 whose provider code, type, or message identifies context overflow), `INVALID_REQUEST` (other 400s and 413), `SERVER` (5xx), `HTTP_` otherwise. Its serializable `failure` retains the HTTP status plus a valid positive `Retry-After` seconds/date delay and `x-request-id` / `x-deepseek-request-id` when present. Attachment reads retain their stable attachment failure code rather than becoming transport failures. A pre-response transport failure (DNS, refused connection, TLS, proxy) throws `TRANSPORT` naming the configured endpoint and chaining the original rejection as `cause`; caller aborts throw `ABORTED`, and the loop's cancellation signal remains authoritative. Protocol violations throw `STREAM_CLOSED` (no `[DONE]`) or `MALFORMED_RESPONSE` (bad JSON payload). Unknown wire `finish_reason`s (e.g. `content_filter`, `insufficient_system_resource`) become `finish {kind: 'error', failure}` chunks, and a completed stream whose `stop` (or absent) finish opened no content blocks becomes a `finish {kind: 'error'}` with code `EMPTY_RESPONSE` (retried by default policy). +Non-2xx responses throw `LlmError` with stable codes: `AUTH` (401/403), `QUOTA` (a response whose provider details identify exhausted quota, balance, or credits), `RATE_LIMIT` (other 429s), `CONTEXT_WINDOW_EXCEEDED` (a 400 whose provider code, type, or message identifies context overflow), `INVALID_REQUEST` (other 400s and 413), `SERVER` (5xx), `HTTP_` otherwise. Its serializable `failure` retains the HTTP status plus a valid positive `Retry-After` seconds/date delay and `x-request-id` / `x-deepseek-request-id` when present. If DeepSeek rejects a normalized image, the primary message names the attachment or display name, durable message and image position, normalized media type, 8-bit sRGB/sRGBA depth, dimensions, and provider message. With several candidates and no file id in the provider detail, it lists each possible image instead of assigning the failure to the first one. The raw response remains the error `cause`; it is never the only user-visible diagnostic. Attachment reads retain their stable attachment failure code rather than becoming transport failures. A pre-response transport failure (DNS, refused connection, TLS, proxy) throws `TRANSPORT` naming the configured endpoint and chaining the original rejection as `cause`; caller aborts throw `ABORTED`, and the loop's cancellation signal remains authoritative. Protocol violations throw `STREAM_CLOSED` (no `[DONE]`) or `MALFORMED_RESPONSE` (bad JSON payload). Unknown wire `finish_reason`s (e.g. `content_filter`, `insufficient_system_resource`) become `finish {kind: 'error', failure}` chunks, and a completed stream whose `stop` (or absent) finish opened no content blocks becomes a `finish {kind: 'error'}` with code `EMPTY_RESPONSE` (retried by default policy). ## Model Experience @@ -92,7 +103,7 @@ Non-2xx responses throw `LlmError` with stable codes: `AUTH` (401/403), `QUOTA` #### What the model sees -The selected DeepSeek model receives the harness system prompt, message history, tool schemas, stop sequences, and call config without adapter-authored prompt prose. The vision model also receives retained user and tool-result images as base64 data URLs; an over-budget older image is represented by the documented placeholder. Reasoning content from a prior assistant turn is passed back verbatim, whether or not that turn called a tool. +The selected DeepSeek model receives the harness system prompt, message history, tool schemas, stop sequences, and call config. The vision model receives retained user and tool-result images as Files API references beside stable attachment handles and preview dimensions; an over-budget older image is represented by the documented placeholder. Reasoning content from a prior assistant turn is passed back verbatim, whether or not that turn called a tool. #### Token effect @@ -122,4 +133,4 @@ Loop-retained response blocks append to the next request and preserve its earlie - **`tool_choice` is not mapped** — not part of the core vocabulary (MVP cut, shared with the pi-ai twin). - **Requests use raw `fetch`, not `@cordisjs/plugin-http`** — no shared proxy/interception configuration; adoption is deferred until a second adapter wants it (`TODO(http)`). - **Plugin-added content block types are skipped** — core text and supported image blocks are serialized, and empty tool output crosses the wire as the literal `(no output)`. -- **Images are input-only durable attachments** — direct external URLs, the Files API, and assistant image output are not supported. +- **Images are input-only durable attachments** — direct external URLs and assistant image output are not supported; DeepSeek input uses the Files API. diff --git a/packages/llm/llm-deepseek/README.zh.md b/packages/llm/llm-deepseek/README.zh.md index 0a5f0224db..d17d7a7396 100644 --- a/packages/llm/llm-deepseek/README.zh.md +++ b/packages/llm/llm-deepseek/README.zh.md @@ -20,7 +20,12 @@ harness LLM(大语言模型)seam 的 DeepSeek chat-completions 适配器: reasoningEffort: high # optional; off | low | high | max — omitted ⇒ high maxTokens: 256000 # optional positive per-request output cap; this is the default streamIdleTimeoutMs: 300000 # optional; positive finite Node timer delay; five-minute default - maxRequestImageBytes: 20971520 # optional positive integer; 20 MiB base64-payload default + maxRequestFilesBytes: 134217728 # optional positive integer; 128 MiB raw request-image default + maxImagesPerRequest: 600 # provider request image-count limit + imageOffloadByteQuantum: 67108864 # oldest-image removal advances in 64 MiB steps + fileExpiresAfterSeconds: 604800 # uploaded image lifetime; 1 hour to 30 days + fileRefreshMarginSeconds: 3600 # replace ids with less lifetime remaining + fileQuotaCleanupBatch: 100 # oldest harness-owned files deleted before one quota retry retryPolicy: # optional; omission uses normal mode with five retries mode: always # normal | always backoff: @@ -34,16 +39,22 @@ harness LLM(大语言模型)seam 的 DeepSeek chat-completions 适配器: - id: deepseek-v4-flash-vision-exp name: DeepSeek-V4-Flash-Vision-Exp inputModalities: [text, image] + imagePixelBudget: 640000 + imageMaxBytes: 1048576 - id: private-reasoner description: Company-hosted reasoning model contextWindow: 512000 ``` -该插件注册唯一提供方路由 `deepseek-official`,并一同注册解析后的 `retryPolicy`;省略时会解析为 normal 模式并重试五次。请求使用 `provider: deepseek-official` 选择该路由;其 `model` 会作为协议 `model` 字符串原样传递,因此更改 DeepSeek 模型不需要生命周期时注册。省略 `models` 会公布 `deepseek-v4-flash`、`deepseek-v4-pro` 与支持图片输入的 `deepseek-v4-flash-vision-exp`,三者的上下文窗口均为 1,000,000 token;显式列表会替换这些默认值,`models: []` 则不公布任何模型。Catalog 配置项通过 `ctx.llm.listModels('deepseek-official')` 公开给 ACP(Agent Client Protocol)编辑器和 Web 选择器等客户端,但仍只提供建议:未列出模型 id 仍原样传递。省略配置项 name 默认为其 id,省略 `inputModalities` 则表示仅支持 `text`。 +该插件注册唯一提供方路由 `deepseek-official`,并一同注册解析后的 `retryPolicy`;省略时会解析为 normal 模式并重试五次。请求使用 `provider: deepseek-official` 选择该路由;其 `model` 会作为协议 `model` 字符串原样传递,因此更改 DeepSeek 模型不需要生命周期时注册。省略 `models` 会公布 `deepseek-v4-flash`、`deepseek-v4-pro` 与支持图片输入的 `deepseek-v4-flash-vision-exp`,三者的上下文窗口均为 1,000,000 token;显式列表会替换这些默认值,`models: []` 则不公布任何模型。Catalog 配置项通过 `ctx.llm.listModels('deepseek-official')` 公开给 ACP(Agent Client Protocol)编辑器和 Web 选择器等客户端,但仍只提供建议:未列出模型 id 仍原样传递,并按纯文本路由处理。省略配置项 name 默认为其 id,省略 `inputModalities` 则表示仅支持 `text`。 -支持图片的 catalog 配置项可以声明 `inputModalities: [text, image]`。适配器通过 `ctx.attachments` 解析 user 和工具结果中的 `ImageBlock` 引用,校验已存储字节,再发送瞬态 `data:;base64,...` `image_url` 部分,不改变持久会话消息。纯文本模型与未列出模型会在凭据、附件或网络 I/O 前拒绝图片输入。System 和 assistant 历史仍不能包含图片;工具结果图片会在仅含字符串的 `tool` 消息后,通过单独的 `user` 消息发送。 +支持图片的 catalog 配置项声明 `inputModalities: [text, image]`,并可设置 `imagePixelBudget`、`imageMaxBytes` 或 `imageDetail: low`。普通默认值为总像素 640,000、编码字节 1MiB;low detail 的默认总像素为 512×512。附件存储按 `min(1, sqrt(pixelBudget / (width * height)))` 缩放,并向预算内取整,确保总像素不超过硬上限。因此 2048×1024 主版本会得到约 1130×565 的请求版本,而不会被强制变成正方形。请求编码按需执行:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。两个质量档均超过 1MiB 时才缩小尺寸。同一 `variantId` 的并发生成共享一次变换。适配器通过 `POST /files` 上传确切的派生请求字节,再发送 `{type: "file", file_id}` 块,不会回退到内联 data URL。每张保留图片前都有稳定文本,写明完整附件 ID、实际请求尺寸,以及 `read_image_region` 所需的预览坐标参数。User、工具结果、agent loop、压缩和直接 `ctx.llm.stream` 请求都使用该投影。纯文本路由会收到稳定的附件占位文本,持久历史继续保留图片引用。 -`maxRequestImageBytes` 限制累计 base64 图片 payload,默认值为 20 MiB,为官方 30 MiB 请求正文限制中的文本、工具和 JSON 分帧保留余量。历史超过上限时,适配器会从最旧图片开始替换为固定模型可见占位文本 `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`,直至请求可容纳;被省略的附件不会被读取。附件准入仍负责单图和单消息原始字节数、媒体类型、尺寸与像素限制。 +`maxRequestFilesBytes` 和 `maxImagesPerRequest` 限制请求中保留的请求版本,默认值分别为 128MiB 和 600 张。字节数越过上限时,被移除的最旧前缀会越过下一个 64MiB 边界。由 1MiB 图片组成的历史达到 129MiB 时会移除最旧的 65 张并保留 64MiB;直到持久历史超过 192MiB,这个前缀才再次变化。图片数量超限时则按 `imageOffloadCountQuantum` 独立递增。移除的图片会变成固定模型可见占位文本 `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`。这种定量投影不会因每新增一张图片就改写较早的请求前缀。 + +上传 ID 按端点和 API key 作用域以及请求 `variantId` 记录在 `DSH_HOME` 下。变体身份覆盖主附件 ID、变换策略版本、路由像素和字节预算、裁剪区域及编码参数,因此 Files API 和支持内联的适配器引用同一份确定性字节。上传默认请求 7 天有效期,并保存服务端返回的 `expires_at`。本地映射剩余时间不超过一小时时会在使用前替换;适配器不会在每次 chat 前查询远端文件。如果 chat 报告文件 ID 已过期、删除、缺失或无效,并指出本次请求使用的某个 ID,适配器只删除该映射。如果响应只说明文件状态失效而没有指出 ID,适配器会删除该次 chat 使用的全部文件映射。随后重新上传受影响的请求版本,并重试一次 chat。第二次 chat 仍报告文件失效时,适配器会按该响应清理映射并返回错误,不会发起第三次 chat。上传响应若没有完整文件对象、匹配的字节数和 `expires_at`,就不会写入索引;后续请求会再次上传,而不是信任不一致的本地状态。本地上传索引格式损坏时按空缓存处理,并由下一次成功上传替换;权限和文件系统 I/O 错误仍使请求失败。 + +一次上传配额错误会触发删除配置数量的最旧 `dsh-` 文件,然后重试一次上传。`DeepSeekFilesClient.delete`、`DeepSeekFileStore.release` 和 `releaseAll` 提供主动远端空间回收。本包记录的当前提供方限制为 Files 单次上传 128MiB、chat 单图引用 32MiB、每个 API key 最多 10,000 个文件和 25GiB;默认 1MiB 请求版本低于两个单文件上限。 `contextWindow` 对每个已配置模型都可选,不会通过建议 catalog 公开。`ctx.llm.resolveModelInfo('deepseek-official', model).context` 先返回精确模型值,再对不含容量的配置项或未列出原样传递 id 返回 `defaultContextWindow`。适配器默认值为 1,000,000;因此,压力敏感插件可以获得由部署决定的容量,不会将模型 selector 视为权威。为 `deepseek-official` 注册另一个适配器会抛出 `LlmError('DUPLICATE_ADAPTER')`。 @@ -53,11 +64,11 @@ harness LLM(大语言模型)seam 的 DeepSeek chat-completions 适配器: `thinking: disabled` 是部署锁定:它只公布 `off`,并以 `off` 为默认值。省略 `reasoningEffort` 或将其配置为 `off` 均有效;配置 `low`、`high` 或 `max` 会使插件加载失败,直接按请求启用思考也会在网络 I/O 前失败。携带 `GenerateOptions.purpose: 'session-title'` 的请求也会强制禁用思考并省略已解析的推理强度,将有界输出保留给可见标题文本,不改变会话或压缩(compaction)默认值。 -`streamIdleTimeoutMs` 会限制每次未完成提供方读取,包括初始 `fetch`,但不计入消费方在分片间花费的时间。DeepSeek SSE 注释会作为传输活动使尚未完成的读取重新布防,但绝不会成为 `StreamChunk` 值或会话日志事件。同一个稳定的 abort 信号会在整个调用期间传递给请求与 body reader;过期会停止传输并抛出 `LlmError('TIMEOUT')`,较早的调用方 abort 则抛出 `LlmError('ABORTED')`。适配器每次 `stream()` 调用恰好发起一次提供方请求;它把已配置策略注册为提供方元数据,再由 `dsh-llm-retry` 在持久化的 agent(智能体)步骤边界单独执行该策略。 +`streamIdleTimeoutMs` 会限制每次未完成提供方读取,包括初始 `fetch`,但不计入消费方在分片间花费的时间。DeepSeek SSE 注释会作为传输活动使尚未完成的读取重新布防,但绝不会成为 `StreamChunk` 值或会话日志事件。同一个稳定的 abort 信号会在整个调用期间传递给请求与 body reader;过期会停止传输并抛出 `LlmError('TIMEOUT')`,较早的调用方 abort 则抛出 `LlmError('ABORTED')`。适配器通常每次 `stream()` 调用发起一次 chat 请求,只有上述失效文件恢复会发起第二次。适配器把已配置重试策略注册为提供方元数据,再由 `dsh-llm-retry` 在持久化的 agent(智能体)步骤边界单独执行该策略。 ## 动态配置(settings + credentials) -连接事实不在加载时冻结。`resolveAdapterOptions` 是从原始配置到已校验事实的唯一显式 resolve 步骤,适配器经由一个 thunk **每操作重读一次**:base URL、catalog、请求默认值、图片上限与 idle 预算都在下一次请求生效,进行中的流则保持其起始事实。三个可选 seam 供给该 thunk: +连接事实不在加载时冻结。`resolveAdapterOptions` 是从原始配置到已校验事实的唯一显式 resolve 步骤,适配器经由一个 thunk **每操作重读一次**:base URL、catalog、请求默认值、图片和 Files 策略与 idle 预算都在下一次请求生效,进行中的流则保持其起始事实。三个可选 seam 供给该 thunk: - **`ctx.settings`**——插件用同一份 `Config` schema 注册 `llm-deepseek` namespace,并以其 `cordis.yml` 条目为组合 `base`,因此用户设置文档中的 `llm-deepseek:` 分节可以免重启覆盖任何字段。未挂载 settings 服务时,仅由 entry 配置驱动适配器,行为不变。存活 settings 快照若通过 schema 却违反 schema 之外的约束(重复的 catalog id、无法成立的 thinking/推理强度组合),则保留最后可用事实并记录失败;entry 配置本身仍会使插件加载失败。 - **`ctx.credentials`**——API 密钥按每次 stream 调用解析,取自与端点*同一*份解析后的快照。配置只携带 `apiKeyEnv`,从不携带字面密钥:该引用经凭据 seam 解析,未挂载 seam 时则经受信环境层解析。由于凭据事实与连接事实同行,被 resolver 拒绝的 settings 快照既不贡献自己的端点,也不贡献自己的密钥:整个先前世代继续服务。每个解析出的密钥在使用前都会被校验格式,因此 HTTP 标头无法承载的值会以 `LlmError('INVALID_CREDENTIAL')` 被拒绝,点名失败的入口,但绝不透露密钥的任何部分,而不是以语义不明的 `fetch` `TypeError` 形式浮现。任何地方都没有密钥的请求以 `MISSING_CREDENTIAL` 失败,并点名每个配置入口,同时路由保持注册、catalog 保持可浏览——首次运行的上手流程就是「浏览模型、存入密钥、再次发起提示」,中间无需任何重启。 @@ -84,7 +95,7 @@ DeepSeek 请求身份独立于应用归因。凭据解析成功后,每个提 ## 错误 -非 2xx 响应会抛出稳定 code 的 `LlmError`:`AUTH`(401/403)、`QUOTA`(提供方详细信息标识配额、余额或点数耗尽的响应)、`RATE_LIMIT`(其他 429)、`CONTEXT_WINDOW_EXCEEDED`(提供方 code、type 或 message 标识上下文溢出的 400)、`INVALID_REQUEST`(其他 400 和 413)、`SERVER`(5xx),其他情况为 `HTTP_`。其可序列化 `failure` 保留 HTTP 状态,以及有效的正 `Retry-After` 秒数/日期延迟和存在时的 `x-request-id` / `x-deepseek-request-id`。附件读取会保留稳定的附件失败 code,不会变成传输失败。响应前传输失败(DNS、连接被拒绝、TLS、proxy)会抛出命名已配置端点的 `TRANSPORT`,并将原始拒绝作为 `cause`;调用方 abort 抛出 `ABORTED`,仍以 loop 的取消信号为准。协议违例抛出 `STREAM_CLOSED`(没有 `[DONE]`)或 `MALFORMED_RESPONSE`(JSON payload 格式错误)。未知协议 `finish_reason`(例如 `content_filter`、`insufficient_system_resource`)会变为 `finish {kind: 'error', failure}` 分片;已完成流如果使用 `stop`(或缺失)finish 但没有开启内容块,就会变为 `finish {kind: 'error'}`,code 为 `EMPTY_RESPONSE`(默认策略会重试)。 +非 2xx 响应会抛出稳定 code 的 `LlmError`:`AUTH`(401/403)、`QUOTA`(提供方详细信息标识配额、余额或点数耗尽的响应)、`RATE_LIMIT`(其他 429)、`CONTEXT_WINDOW_EXCEEDED`(提供方 code、type 或 message 标识上下文溢出的 400)、`INVALID_REQUEST`(其他 400 和 413)、`SERVER`(5xx),其他情况为 `HTTP_`。其可序列化 `failure` 保留 HTTP 状态,以及有效的正 `Retry-After` 秒数/日期延迟和存在时的 `x-request-id` / `x-deepseek-request-id`。如果 DeepSeek 拒绝一张已规范化图片,主错误会写明附件 ID 或显示名称、持久消息和图片位置、规范化后的媒体类型、8-bit sRGB/sRGBA 位深、尺寸和提供方消息。存在多张候选图片且提供方详细信息没有 file id 时,错误会列出全部可能图片,不会把错误归给第一张。原始响应保留为错误 `cause`,不会成为唯一的用户可见诊断。附件读取会保留稳定的附件失败 code,不会变成传输失败。响应前传输失败(DNS、连接被拒绝、TLS、proxy)会抛出命名已配置端点的 `TRANSPORT`,并将原始拒绝作为 `cause`;调用方 abort 抛出 `ABORTED`,仍以 loop 的取消信号为准。协议违例抛出 `STREAM_CLOSED`(没有 `[DONE]`)或 `MALFORMED_RESPONSE`(JSON payload 格式错误)。未知协议 `finish_reason`(例如 `content_filter`、`insufficient_system_resource`)会变为 `finish {kind: 'error', failure}` 分片;已完成流如果使用 `stop`(或缺失)finish 但没有开启内容块,就会变为 `finish {kind: 'error'}`,code 为 `EMPTY_RESPONSE`(默认策略会重试)。 ## 模型体验 @@ -92,7 +103,7 @@ DeepSeek 请求身份独立于应用归因。凭据解析成功后,每个提 #### 模型看到的内容 -所选 DeepSeek 模型会收到 harness 系统提示词、消息历史、工具 schema、stop sequence 和调用配置,不含适配器撰写的提示词文本。视觉模型还会通过 base64 data URL 收到保留的 user 与工具结果图片;超出上限的较旧图片由已记录的占位文本表示。之前 assistant 轮次的推理内容会原文回传,无论该轮次是否调用了工具。 +所选 DeepSeek 模型会收到 harness 系统提示词、消息历史、工具 schema、stop sequence 和调用配置。视觉模型会通过 Files API 引用收到保留的 user 与工具结果图片,旁边带有稳定附件句柄和预览尺寸;超出上限的较旧图片由已记录的占位文本表示。之前 assistant 轮次的推理内容会原文回传,无论该轮次是否调用了工具。 #### Token 影响 @@ -122,4 +133,4 @@ loop 保留的响应块会追加到下一个请求,并保留其较早可复用 - **未映射 `tool_choice`**:它不属于核心词汇(MVP 取舍,与 pi-ai twin 共享)。 - **请求使用原始 `fetch`,而非 `@cordisjs/plugin-http`**:没有共享 proxy/拦截配置;采用暂缓到第二个适配器需要该功能时(`TODO(http)`)。 - **会跳过插件添加的内容块类型**:核心文本与支持的图片块会被序列化,空工具输出会以字面 `(no output)` 通过协议发送。 -- **图片是仅输入的持久附件**:不支持直接外部 URL、Files API 和 assistant 图片输出。 +- **图片是仅输入的持久附件**:不支持直接外部 URL 和 assistant 图片输出;DeepSeek 图片输入使用 Files API。 diff --git a/packages/llm/llm-deepseek/package.json b/packages/llm/llm-deepseek/package.json index 1c7e8aadf0..effb77d4c0 100644 --- a/packages/llm/llm-deepseek/package.json +++ b/packages/llm/llm-deepseek/package.json @@ -33,10 +33,13 @@ "license": "MIT", "peerDependencies": { "@deepseek-ai/dsh-attachment": "workspace:^", + "@deepseek-ai/dsh-atomic-write": "workspace:^", + "@deepseek-ai/dsh-brand": "workspace:^", "@deepseek-ai/dsh-credentials": "workspace:^", "@deepseek-ai/dsh-launch-environment": "workspace:^", "@deepseek-ai/dsh-invariants": "workspace:^", "@deepseek-ai/dsh-llm": "workspace:^", + "@deepseek-ai/dsh-home-paths": "workspace:^", "@deepseek-ai/dsh-settings": "workspace:^", "@deepseek-ai/dsh-timeout": "workspace:^", "@deepseek-ai/dsh-anonymous-user-id": "workspace:^", @@ -48,10 +51,13 @@ }, "devDependencies": { "@deepseek-ai/dsh-attachment": "workspace:^", + "@deepseek-ai/dsh-atomic-write": "workspace:^", + "@deepseek-ai/dsh-brand": "workspace:^", "@deepseek-ai/dsh-credentials": "workspace:^", "@deepseek-ai/dsh-launch-environment": "workspace:^", "@deepseek-ai/dsh-invariants": "workspace:^", "@deepseek-ai/dsh-llm": "workspace:^", + "@deepseek-ai/dsh-home-paths": "workspace:^", "@deepseek-ai/dsh-settings": "workspace:^", "@deepseek-ai/dsh-timeout": "workspace:^", "@deepseek-ai/dsh-anonymous-user-id": "workspace:^", diff --git a/packages/llm/llm-deepseek/src/adapter.ts b/packages/llm/llm-deepseek/src/adapter.ts index 638d555b1e..8d9381c67f 100644 --- a/packages/llm/llm-deepseek/src/adapter.ts +++ b/packages/llm/llm-deepseek/src/adapter.ts @@ -10,20 +10,31 @@ import { attributionHeaders, contentHasImage, CONTEXT_WINDOW_EXCEEDED_CODE, isContextWindowExceededError, isQuotaExceededError, LlmAdapter, LlmError, ProviderRequestId, QUOTA_EXCEEDED_CODE, ReasoningEffortId } from '@deepseek-ai/dsh-llm' import type { + ContentBlock, GenerateOptions, LlmModelInfo, LlmProviderInfo, + PreparedAdapterCall, LlmResolvedModelInfo, ModelModality, ResolvedRetryPolicy, StreamChunk, } from '@deepseek-ai/dsh-llm' -import type { AttachmentStore } from '@deepseek-ai/dsh-attachment' +import type { + AttachmentId, + AttachmentStore, + ImageAttachmentRef, + ImageRequestPolicy, + RequestImageAttachment, +} from '@deepseek-ai/dsh-attachment' import type { CredentialRef } from '@deepseek-ai/dsh-credentials' import { idleWatchdog, timeoutOf } from '@deepseek-ai/dsh-timeout' import type { AnonymousUserId } from '@deepseek-ai/dsh-anonymous-user-id' import { serializeRequest, serializeRequestWithImages } from './serialize.ts' -import type { RequestDefaults } from './serialize.ts' +import type { ImageWireLocation, RequestDefaults } from './serialize.ts' +import { DeepSeekFileStore } from './file-store.ts' +import type { DeepSeekFilePolicy } from './file-store.ts' +import type { DeepSeekFileId } from './file-id.ts' import { parseSse } from './sse.ts' import { translate } from './translate.ts' import type { WireError } from './types.ts' @@ -42,6 +53,12 @@ export interface DeepSeekCatalogModel { maxTokens?: number /** Accepted request modalities; omission is text-only. */ inputModalities?: ModelModality[] + /** Total-pixel budget for one deterministic request preview. */ + imagePixelBudget?: number + /** Encoded-byte cap for one deterministic request preview. */ + imageMaxBytes?: number + /** Provider detail tier; `low` uses the 512-by-512 total-pixel default. */ + imageDetail?: 'auto' | 'low' } /** @@ -70,8 +87,16 @@ export interface DeepSeekConnectionOptions { models: readonly DeepSeekCatalogModel[] /** Maximum provider idle time while one stream read is outstanding. */ streamIdleTimeoutMs: number - /** Maximum accumulated base64 image payload in one request. */ - maxRequestImageBytes: number + /** Maximum accumulated file-referenced image bytes in one request. */ + maxRequestFilesBytes: number + /** Maximum number of file-referenced images in one request. */ + maxImagesPerRequest: number + /** Raw-byte removal step after the file-reference bound is exceeded. */ + imageOffloadByteQuantum: number + /** Image-count removal step after the count bound is exceeded. */ + imageOffloadCountQuantum: number + /** Upload expiry, refresh, and quota-recovery policy. */ + filePolicy: DeepSeekFilePolicy /** Provider-owned model-request retry policy, already resolved. */ retryPolicy: ResolvedRetryPolicy } @@ -91,6 +116,8 @@ export interface DeepSeekAdapterOptions { resolveUserId: () => AnonymousUserId /** Resolve the current durable attachment service; absence rejects image input. */ resolveAttachments?: () => AttachmentStore | undefined + /** Resolve the process-wide upload reuse store. */ + resolveFiles?: () => DeepSeekFileStore } /** Default maximum idle interval while an adapter stream read is outstanding. */ @@ -99,8 +126,26 @@ export const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 300_000 export const DEFAULT_CONTEXT_WINDOW = 1_000_000 /** Default per-request output-token cap. */ export const DEFAULT_MAX_TOKENS = 256_000 -/** Default bound on accumulated base64 image payload per request. */ -export const DEFAULT_MAX_REQUEST_IMAGE_BYTES = 20 * 1024 * 1024 +/** Default bound on accumulated file-referenced image bytes per request. */ +export const DEFAULT_MAX_REQUEST_FILES_BYTES = 128 * 1024 * 1024 +/** Provider request image-count limit. */ +export const DEFAULT_MAX_IMAGES_PER_REQUEST = 600 +/** Total-pixel budget matching DeepSeek's normal vision projection. */ +export const DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET = 640_000 +/** Total-pixel budget matching provider low-detail image input. */ +export const DEFAULT_LOW_DETAIL_IMAGE_PIXEL_BUDGET = 512 * 512 +/** Encoded-byte cap for one deterministic model-request image. */ +export const DEFAULT_REQUEST_IMAGE_MAX_BYTES = 1024 * 1024 +/** Deterministic raw-byte removal step. */ +export const DEFAULT_IMAGE_OFFLOAD_BYTE_QUANTUM = 64 * 1024 * 1024 +/** Deterministic image-count removal step. */ +export const DEFAULT_IMAGE_OFFLOAD_COUNT_QUANTUM = 20 +/** Default explicit lifetime for uploaded images. */ +export const DEFAULT_FILE_EXPIRY_SECONDS = 7 * 24 * 60 * 60 +/** Default proactive refresh window for indexed file ids. */ +export const DEFAULT_FILE_REFRESH_MARGIN_SECONDS = 60 * 60 +/** Default number of oldest harness-owned files removed on quota recovery. */ +export const DEFAULT_FILE_QUOTA_CLEANUP_BATCH = 100 const STREAM_IDLE_TIMEOUT_CODE = 'LLM_STREAM_IDLE_TIMEOUT' const OFF_REASONING_EFFORT = ReasoningEffortId('off') const LOW_REASONING_EFFORT = ReasoningEffortId('low') @@ -116,6 +161,112 @@ const OFF_ONLY_REASONING_EFFORTS = [ { id: OFF_REASONING_EFFORT, name: 'Off' }, ] as const +function collectImageRefs( + content: readonly ContentBlock[], + refs: Map, +): void { + for (const block of content) { + if (block.type === 'image') refs.set(block.attachment.attachmentId, block.attachment) + else if (block.type === 'tool-result') collectImageRefs(block.content, refs) + } +} + +function requestImagePolicy(model: DeepSeekCatalogModel): ImageRequestPolicy { + return { + maxPixels: model.imagePixelBudget + ?? (model.imageDetail === 'low' + ? DEFAULT_LOW_DETAIL_IMAGE_PIXEL_BUDGET + : DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET), + maxBytes: model.imageMaxBytes ?? DEFAULT_REQUEST_IMAGE_MAX_BYTES, + } +} + +async function prepareRequestImages( + options: GenerateOptions, + attachments: AttachmentStore, + model: DeepSeekCatalogModel, + signal: AbortSignal, +): Promise> { + const refs = new Map() + for (const message of options.messages) collectImageRefs(message.content, refs) + const policy = requestImagePolicy(model) + const orderedRefs = [...refs.values()] + const projected = await attachments.readImageRequests(orderedRefs, policy, signal) + return new Map(orderedRefs.map((ref, index) => ( + [ref.attachmentId, projected[index] as RequestImageAttachment] + ))) +} + +function providerRejectedNormalizedImage(detail: string): boolean { + const reasonBeforeImage = /(?:unsupported|invalid|cannot read|failed to (?:decode|process)).{0,40}image/iu + const imageBeforeReason = /image.{0,40}(?:unsupported|invalid|cannot be decoded)/iu + return reasonBeforeImage.test(detail) || imageBeforeReason.test(detail) +} + +interface UsedRequestFile { + version: RequestImageAttachment + fileId: DeepSeekFileId + location: ImageWireLocation +} + +function providerRejectedFileId(detail: string): boolean { + const file = /\bfile(?:[_ -]?(?:id|api|not[_ -]?found|deleted|expired))?/iu.test(detail) + const missing = /(?:expired|not[_ -]?found|deleted|does not exist)/iu.test(detail) + const invalidId = /(?:invalid.{0,20}file[_ -]?(?:id|api)|file[_ -]?(?:id|api).{0,20}invalid)/iu.test(detail) + return file && (missing || invalidId) +} + +function detailNamesFileId(detail: string, fileId: DeepSeekFileId): boolean { + let index = detail.indexOf(fileId) + while (index >= 0) { + const before = detail[index - 1] + const after = detail[index + fileId.length] + if ((before === undefined || !/[\p{L}\p{N}_-]/u.test(before)) + && (after === undefined || !/[\p{L}\p{N}_-]/u.test(after))) return true + index = detail.indexOf(fileId, index + 1) + } + return false +} + +function staleMappings( + files: readonly UsedRequestFile[], + detail: string, +): UsedRequestFile[] { + const unique = [...new Map(files.map(file => [`${file.version.variantId}\0${file.fileId}`, file])).values()] + const exact = unique.filter(file => detailNamesFileId(detail, file.fileId)) + return exact.length > 0 ? exact : unique +} + +function normalizedImageFacts( + file: { version: RequestImageAttachment; location: ImageWireLocation }, +): string { + const version = file.version + const name = version.master.name ?? version.master.attachmentId + const colour = version.hasAlpha ? 'sRGBA' : 'sRGB' + return `"${name}" at message ${file.location.message}, image ${file.location.image} ` + + `(${version.mediaType}, 8-bit ${colour}, ${version.width}x${version.height})` +} + +function normalizedImageDiagnostic( + files: readonly UsedRequestFile[], + providerMessage: string, + providerDetail: string, +): string { + const exact = files.find(file => detailNamesFileId(providerDetail, file.fileId)) + const target = exact ?? (files.length === 1 ? files[0] : undefined) + if (target !== undefined) { + return `DeepSeek rejected normalized image ${normalizedImageFacts(target)}: ${providerMessage}. ` + + 'The provider rejected bytes already normalized by the harness; PNG, JPEG, WebP, and GIF remain supported input formats.' + } + const candidates = [...new Map(files.map(file => [ + `${file.version.variantId}\0${file.location.message}\0${file.location.image}`, + file, + ])).values()] + return `DeepSeek rejected a normalized request image: ${providerMessage}. Candidate images: ` + + `${candidates.map(normalizedImageFacts).join('; ')}. ` + + 'The provider rejected bytes already normalized by the harness; PNG, JPEG, WebP, and GIF remain supported input formats.' +} + function modelInfo(provider: string, model: DeepSeekCatalogModel): LlmModelInfo { return { provider, @@ -169,8 +320,11 @@ export function httpErrorCode(status: number, error?: WireError['error']): strin * map to `ABORTED`; the configured per-read idle watchdog maps to `TIMEOUT`. */ export class DeepSeekAdapter extends LlmAdapter { + private readonly files: DeepSeekFileStore + constructor(private readonly config: DeepSeekAdapterOptions) { super() + this.files = config.resolveFiles?.() ?? new DeepSeekFileStore() } override providerInfo(provider: string): LlmProviderInfo { @@ -190,11 +344,18 @@ export class DeepSeekAdapter extends LlmAdapter { model: string, _signal?: AbortSignal, ): Promise { - const connection = this.config.options() + return Promise.resolve(this.modelInfoFor(this.config.options(), provider, model)) + } + + private modelInfoFor( + connection: DeepSeekConnectionOptions, + provider: string, + model: string, + ): LlmResolvedModelInfo { const configured = connection.models.find(entry => entry.id === model) const contextWindow = configured?.contextWindow ?? connection.defaultContextWindow - return Promise.resolve({ + return { // An uncatalogued endpoint is safely treated as text-only. Declaring an // unverified image capability would let the host persist input that the // endpoint may reject on every later turn. @@ -222,16 +383,30 @@ export class DeepSeekAdapter extends LlmAdapter { : HIGH_REASONING_EFFORT, }, }, + } + } + + override prepareCall(provider: string, model: string, _signal?: AbortSignal): Promise { + const connection = this.config.options() + return Promise.resolve({ + model: this.modelInfoFor(connection, provider, model), + stream: options => this.streamWithConnection(options, connection), }) } - async * stream(options: GenerateOptions): AsyncIterable { + stream(options: GenerateOptions): AsyncIterable { + return this.streamWithConnection(options, this.config.options()) + } + + private async * streamWithConnection( + options: GenerateOptions, + connection: DeepSeekConnectionOptions, + ): AsyncIterable { // One resolution per stream call: connection facts and the credential // freeze here and hold for this whole request, so an in-flight stream // never observes a configuration change and the next call re-resolves. // The key resolves *from this snapshot*, so an endpoint and the secret // sent to it can never come from different configuration generations. - const connection = this.config.options() const hasImages = options.messages.some(message => contentHasImage(message.content)) let attachments: AttachmentStore | undefined if (hasImages) { @@ -310,16 +485,6 @@ export class DeepSeekAdapter extends LlmAdapter { attachments: AttachmentStore | undefined, onComment: () => void, ): AsyncIterable { - const body = attachments === undefined - ? serializeRequest(options, connection.defaults) - : await serializeRequestWithImages(options, { - attachments, - maxRequestImageBytes: connection.maxRequestImageBytes, - signal, - }, connection.defaults) - // Prepared outside the try so the TRANSPORT label below covers exactly the - // transport boundary, never a serialization failure. - const payload = JSON.stringify(body) const headers = { 'authorization': `Bearer ${apiKey}`, 'content-type': 'application/json', @@ -334,53 +499,92 @@ export class DeepSeekAdapter extends LlmAdapter { : {}, } - // TODO(http): adopt the Cordis HTTP service when shared transport configuration - // outweighs its additional runtime dependencies. - let response: Response - try { - response = await fetch(`${connection.baseURL}/chat/completions`, { - method: 'POST', - headers, - body: payload, - signal, - }) - } catch (error: unknown) { - // The outer stream distinguishes caller cancellation and watchdog expiry. - if (signal.aborted) throw error - // fetch wraps every transport failure (DNS, refused connection, TLS, - // proxy) in a bare `TypeError: fetch failed` whose actionable detail - // lives on `cause`. Wrapping with the endpoint and chaining the cause - // lets `errorChain` render the full diagnosis at every reporting boundary. - throw new LlmError( - `DeepSeek API request to ${connection.baseURL} failed`, - 'TRANSPORT', - { cause: error }, - ) - } + const fileConnection = { baseURL: connection.baseURL, apiKey } + const model = connection.models.find(entry => entry.id === options.model) + const requestImages = attachments === undefined || model === undefined + ? new Map() + : await prepareRequestImages(options, attachments, model, signal) + for (let fileAttempt = 0; fileAttempt < 2; fileAttempt += 1) { + const usedFiles: UsedRequestFile[] = [] + const body = attachments === undefined + ? serializeRequest(options, connection.defaults) + : await serializeRequestWithImages(options, { + requestImages, + resolveFileId: async (version, _block, location) => { + const resolved = await this.files.ensureUploaded( + version, + fileConnection, + connection.filePolicy, + signal, + ) + usedFiles.push({ version, fileId: resolved.record.fileId, location }) + return resolved.record.fileId + }, + maxRequestFilesBytes: connection.maxRequestFilesBytes, + maxImagesPerRequest: connection.maxImagesPerRequest, + byteQuantum: connection.imageOffloadByteQuantum, + countQuantum: connection.imageOffloadCountQuantum, + }, connection.defaults) + const payload = JSON.stringify(body) - if (!response.ok) { - let message = `DeepSeek API error (HTTP ${response.status})` - let providerError: WireError['error'] + // TODO(http): adopt the Cordis HTTP service when shared transport configuration + // outweighs its additional runtime dependencies. + let response: Response try { - const parsed = await response.json() as WireError - providerError = parsed.error - if (providerError?.message) message = providerError.message - } catch { - // Only swallow error-body parsing: the HTTP status still identifies the - // failure, so malformed gateway JSON must not mask it. + response = await fetch(`${connection.baseURL}/chat/completions`, { + method: 'POST', + headers, + body: payload, + signal, + }) + } catch (error: unknown) { + if (signal.aborted) throw error + throw new LlmError( + `DeepSeek API request to ${connection.baseURL} failed`, + 'TRANSPORT', + { cause: error }, + ) } - const delay = providerRetryAfterMs(response.headers.get('retry-after')) - const id = requestId(response.headers) - throw new LlmError(message, httpErrorCode(response.status, providerError), { - status: response.status, - ...delay === undefined ? {} : { providerRetryAfterMs: delay }, - ...id === undefined ? {} : { requestId: id }, - }) - } - if (!response.body) { - throw new LlmError('DeepSeek API returned no response body', 'EMPTY_RESPONSE') - } - yield* translate(parseSse(response.body, onComment)) + if (!response.ok) { + let message = `DeepSeek API error (HTTP ${response.status})` + let providerError: WireError['error'] + const rawResponse = await response.text() + try { + const parsed = JSON.parse(rawResponse) as WireError + providerError = parsed.error + if (providerError?.message) message = providerError.message + } catch { + // The HTTP status remains authoritative when a gateway returns malformed JSON. + } + const detail = [providerError?.code, providerError?.type, providerError?.message] + .filter((field): field is string => typeof field === 'string') + .join(' ') + const staleFile = usedFiles.length > 0 && providerRejectedFileId(detail) + if (staleFile) { + await Promise.all(staleMappings(usedFiles, detail).map(file => ( + this.files.invalidate(file.version, file.fileId, fileConnection) + ))) + if (fileAttempt === 0) continue + } + if (response.status === 400 && usedFiles.length > 0 && providerRejectedNormalizedImage(detail)) { + message = normalizedImageDiagnostic(usedFiles, message, detail) + } + const delay = providerRetryAfterMs(response.headers.get('retry-after')) + const id = requestId(response.headers) + throw new LlmError(message, httpErrorCode(response.status, providerError), { + cause: new Error(rawResponse.length > 0 ? rawResponse : `DeepSeek HTTP ${response.status}`), + status: response.status, + ...delay === undefined ? {} : { providerRetryAfterMs: delay }, + ...id === undefined ? {} : { requestId: id }, + }) + } + if (!response.body) { + throw new LlmError('DeepSeek API returned no response body', 'EMPTY_RESPONSE') + } + + yield* translate(parseSse(response.body, onComment)) + return + } } } diff --git a/packages/llm/llm-deepseek/src/file-id.ts b/packages/llm/llm-deepseek/src/file-id.ts new file mode 100644 index 0000000000..fd77de372f --- /dev/null +++ b/packages/llm/llm-deepseek/src/file-id.ts @@ -0,0 +1,27 @@ +/** DeepSeek Files API identifiers. @module dsh-llm-deepseek/file-id */ + +import type { Branded } from '@deepseek-ai/dsh-brand' + +/** Opaque identifier returned by the DeepSeek Files API. */ +export type DeepSeekFileId = Branded<'DeepSeekFileId'> + +/** + * Brand a provider-returned file identifier after wire validation. + * @param id - non-empty Files API identifier. + * @returns the same string with its provider identity attached at type level. + */ +export function DeepSeekFileId(id: string): DeepSeekFileId { + return id as DeepSeekFileId +} + +/** Non-secret digest identifying one endpoint and API-key file namespace. */ +export type DeepSeekFileScope = Branded<'DeepSeekFileScope'> + +/** + * Brand a locally derived namespace digest. + * @param scope - SHA-256 digest of endpoint and API key. + * @returns the same string with namespace identity attached at type level. + */ +export function DeepSeekFileScope(scope: string): DeepSeekFileScope { + return scope as DeepSeekFileScope +} diff --git a/packages/llm/llm-deepseek/src/file-store.ts b/packages/llm/llm-deepseek/src/file-store.ts new file mode 100644 index 0000000000..1ec943a7f9 --- /dev/null +++ b/packages/llm/llm-deepseek/src/file-store.ts @@ -0,0 +1,257 @@ +/** DeepSeek Files API upload reuse, invalidation, and quota recovery. @module dsh-llm-deepseek/file-store */ + +import type { RequestImageAttachment } from '@deepseek-ai/dsh-attachment' +import { LlmError } from '@deepseek-ai/dsh-llm' +import { DeepSeekFilesClient, isFilesQuotaError } from './files-api.ts' +import type { DeepSeekFileId } from './file-id.ts' +import { deepSeekFileScope, DeepSeekUploadIndex } from './upload-index.ts' +import type { DeepSeekUploadRecord } from './upload-index.ts' + +/** DeepSeek chat accepts at most 32 MiB per image even when it is referenced by file id. */ +export const MAX_CHAT_IMAGE_BYTES = 32 * 1024 * 1024 +const OWNED_FILE_PREFIX = 'dsh-' + +/** Resolved file-store policy from the plugin configuration. */ +export interface DeepSeekFilePolicy { + expiresAfterSeconds: number + refreshMarginSeconds: number + quotaCleanupBatch: number +} + +/** Connection facts needed by file operations. */ +export interface DeepSeekFileConnection { + baseURL: string + apiKey: string +} + +/** Result of one file-id resolution. */ +export interface DeepSeekFileReference { + record: DeepSeekUploadRecord + uploaded: boolean +} + +interface FileStoreOptions { + index?: DeepSeekUploadIndex + now?: () => number + fetch?: typeof fetch +} + +function extension(mediaType: RequestImageAttachment['mediaType']): 'png' | 'jpeg' | 'webp' | 'gif' { + switch (mediaType) { + case 'image/png': return 'png' + case 'image/jpeg': return 'jpeg' + case 'image/webp': return 'webp' + case 'image/gif': return 'gif' + } +} + +function filename(version: RequestImageAttachment): string { + const master = String(version.master.attachmentId).slice('sha256:'.length, 'sha256:'.length + 16) + const variant = String(version.variantId).slice('sha256:'.length, 'sha256:'.length + 8) + return `${OWNED_FILE_PREFIX}${master}-${variant}.${extension(version.mediaType)}` +} + +/** User-scoped durable file-id reuse for the DeepSeek route. */ +export class DeepSeekFileStore { + private readonly index: DeepSeekUploadIndex + private readonly now: () => number + private readonly fetchImpl: typeof fetch | undefined + private readonly inflight = new Map>() + + /** + * @param options - testable index, clock, and transport boundaries. + */ + constructor(options: FileStoreOptions = {}) { + this.index = options.index ?? new DeepSeekUploadIndex() + this.now = options.now ?? Date.now + this.fetchImpl = options.fetch + } + + private client(connection: DeepSeekFileConnection): DeepSeekFilesClient { + return new DeepSeekFilesClient({ + baseURL: connection.baseURL, + apiKey: connection.apiKey, + ...this.fetchImpl === undefined ? {} : { fetch: this.fetchImpl }, + }) + } + + /** + * Resolve or upload one deterministic request image. Concurrent calls in this process share one promise. + * @param version - deterministic model-request bytes and complete transformation identity. + * @param connection - endpoint and API-key snapshot. + * @param policy - expiry and quota-recovery policy. + * @param signal - request cancellation. + * @returns a reusable file id and whether this call published a new upload. + */ + ensureUploaded( + version: RequestImageAttachment, + connection: DeepSeekFileConnection, + policy: DeepSeekFilePolicy, + signal?: AbortSignal, + ): Promise { + const scope = deepSeekFileScope(connection.baseURL, connection.apiKey) + const key = `${scope}\0${version.variantId}` + const active = this.inflight.get(key) + if (active !== undefined) return active + const operation = this.ensureUploadedOnce(version, connection, policy, signal) + this.inflight.set(key, operation) + void operation.finally(() => { + if (this.inflight.get(key) === operation) this.inflight.delete(key) + }).catch(() => {}) + return operation + } + + private async ensureUploadedOnce( + version: RequestImageAttachment, + connection: DeepSeekFileConnection, + policy: DeepSeekFilePolicy, + signal?: AbortSignal, + ): Promise { + if (version.bytes > MAX_CHAT_IMAGE_BYTES) { + throw new LlmError('DeepSeek chat image exceeds the 32 MiB per-image limit.', 'INVALID_REQUEST') + } + const scope = deepSeekFileScope(connection.baseURL, connection.apiKey) + const now = this.now() + const marginMs = policy.refreshMarginSeconds * 1_000 + const cached = await this.index.get(scope, version.variantId, now, marginMs) + if (cached !== undefined) return { record: cached, uploaded: false } + + const client = this.client(connection) + const upload = async (): Promise => { + const remote = await client.upload({ + data: version.data, + mediaType: version.mediaType, + filename: filename(version), + expiresAfterSeconds: policy.expiresAfterSeconds, + ...signal === undefined ? {} : { signal }, + }) + if (remote.bytes !== version.data.byteLength || remote.expiresAt === undefined) { + throw new LlmError('DeepSeek Files API upload response does not match the submitted image.', 'INVALID_RESPONSE') + } + return { + scope, + masterAttachmentId: version.master.attachmentId, + variantId: version.variantId, + fileId: remote.id, + bytes: remote.bytes, + createdAt: remote.createdAt * 1_000, + expiresAt: remote.expiresAt * 1_000, + } + } + + let candidate: DeepSeekUploadRecord + try { + candidate = await upload() + } catch (error: unknown) { + if (!isFilesQuotaError(error)) throw error + const deleted = await this.reclaimOldestOwned(connection, policy.quotaCleanupBatch, signal) + if (deleted === 0) throw error + candidate = await upload() + } + const committed = await this.index.commit(candidate, this.now(), marginMs) + if (!committed.accepted) { + try { + await client.delete(candidate.fileId, signal) + } catch { + // The winning mapping is durable. A failed duplicate cleanup affects quota only and is retried by recovery. + } + } + return { record: committed.record, uploaded: committed.accepted } + } + + /** + * Invalidate one exact local mapping after the chat endpoint rejects its remote id. + * @param version - request-image version whose remote generation failed. + * @param fileId - exact rejected file id. + * @param connection - endpoint and API-key snapshot. + */ + async invalidate( + version: RequestImageAttachment, + fileId: DeepSeekFileId, + connection: DeepSeekFileConnection, + ): Promise { + await this.index.remove( + deepSeekFileScope(connection.baseURL, connection.apiKey), + version.variantId, + fileId, + ) + } + + /** + * Delete the indexed remote file for one attachment and remove its local mapping. + * @param version - exact request-image version to release. + * @param connection - endpoint and API-key snapshot. + * @param policy - expiry policy used to locate a reusable mapping. + * @param signal - request cancellation. + * @returns whether an indexed file existed and was deleted. + */ + async release( + version: RequestImageAttachment, + connection: DeepSeekFileConnection, + policy: DeepSeekFilePolicy, + signal?: AbortSignal, + ): Promise { + const scope = deepSeekFileScope(connection.baseURL, connection.apiKey) + const record = await this.index.get( + scope, + version.variantId, + this.now(), + policy.refreshMarginSeconds * 1_000, + ) + if (record === undefined) return false + await this.client(connection).delete(record.fileId, signal) + await this.index.remove(scope, version.variantId, record.fileId) + return true + } + + /** + * Delete the oldest provider files whose names identify harness ownership. + * @param connection - endpoint and API-key snapshot. + * @param count - positive maximum number of files to delete. + * @param signal - request cancellation. + * @returns number of successfully deleted files. + */ + async reclaimOldestOwned( + connection: DeepSeekFileConnection, + count: number, + signal?: AbortSignal, + ): Promise { + const client = this.client(connection) + let after: DeepSeekFileId | undefined + let deleted = 0 + while (deleted < count) { + const page = await client.list({ + ...after === undefined ? {} : { after }, + limit: 1_000, + order: 'asc', + ...signal === undefined ? {} : { signal }, + }) + for (const file of page.data) { + if (!file.filename.startsWith(OWNED_FILE_PREFIX)) continue + await client.delete(file.id, signal) + deleted += 1 + if (deleted === count) break + } + if (!page.hasMore || page.lastId === undefined || page.lastId === after) break + after = page.lastId + } + return deleted + } + + /** + * Delete every remote harness-owned file in the active API-key namespace and clear its index. + * @param connection - endpoint and API-key snapshot. + * @param signal - request cancellation. + * @returns number of deleted files. + */ + async releaseAll(connection: DeepSeekFileConnection, signal?: AbortSignal): Promise { + let total = 0 + for (;;) { + const deleted = await this.reclaimOldestOwned(connection, 1_000, signal) + total += deleted + if (deleted < 1_000) break + } + await this.index.clear(deepSeekFileScope(connection.baseURL, connection.apiKey)) + return total + } +} diff --git a/packages/llm/llm-deepseek/src/files-api.ts b/packages/llm/llm-deepseek/src/files-api.ts new file mode 100644 index 0000000000..90ddf20b8c --- /dev/null +++ b/packages/llm/llm-deepseek/src/files-api.ts @@ -0,0 +1,257 @@ +/** OpenAI-compatible DeepSeek Files API transport. @module dsh-llm-deepseek/files-api */ + +import { LlmError } from '@deepseek-ai/dsh-llm' +import type { ImageMediaType } from '@deepseek-ai/dsh-attachment' +import { DeepSeekFileId } from './file-id.ts' +import type { DeepSeekFileId as DeepSeekFileIdType } from './file-id.ts' + +/** Minimum provider-supported file lifetime. */ +export const MIN_FILE_EXPIRY_SECONDS = 3_600 +/** Maximum provider-supported file lifetime. */ +export const MAX_FILE_EXPIRY_SECONDS = 2_592_000 +/** Maximum Files API upload size. */ +export const MAX_FILE_UPLOAD_BYTES = 128 * 1024 * 1024 +/** Current per-key file-count quota. */ +export const MAX_STORED_FILE_COUNT = 10_000 +/** Current per-key storage quota. */ +export const MAX_STORED_FILE_BYTES = 25 * 1024 * 1024 * 1024 + +/** Validated file object returned by the OpenAI-compatible endpoint. */ +export interface DeepSeekFileObject { + id: DeepSeekFileIdType + bytes: number + createdAt: number + filename: string + purpose: 'user_data' + expiresAt?: number +} + +/** One page returned by `GET /files`. */ +export interface DeepSeekFilePage { + data: DeepSeekFileObject[] + firstId?: DeepSeekFileIdType + lastId?: DeepSeekFileIdType + hasMore: boolean +} + +/** Files API operation failure with its HTTP status retained for recovery policy. */ +export class DeepSeekFilesError extends LlmError { + /** Parsed provider detail used only for error classification. */ + readonly detail: string + + /** + * @param message - user-readable provider failure. + * @param status - HTTP status returned by the Files API. + * @param detail - provider error fields joined for classification. + */ + constructor(message: string, status: number, detail: string) { + super(message, status === 401 || status === 403 + ? 'AUTH' + : status === 429 + ? 'RATE_LIMIT' + : status >= 500 + ? 'SERVER' + : 'FILES_API', { status }) + this.name = 'DeepSeekFilesError' + this.detail = detail + } +} + +/** + * Whether an upload failure reports a provider storage or file-count quota. + * @param error - Files API operation failure. + * @returns whether one bounded remote cleanup and upload retry may recover. + */ +export function isFilesQuotaError(error: unknown): error is DeepSeekFilesError { + return error instanceof DeepSeekFilesError + && /(?:quota|storage|stored files|file count|too many files)/iu.test(error.detail) +} + +interface FilesApiOptions { + baseURL: string + apiKey: string + fetch?: typeof fetch +} + +interface WireFileObject { + id?: unknown + object?: unknown + bytes?: unknown + created_at?: unknown + filename?: unknown + purpose?: unknown + expires_at?: unknown +} + +function invalidResponse(operation: string): LlmError { + return new LlmError(`DeepSeek Files API returned an invalid ${operation} response.`, 'INVALID_RESPONSE') +} + +function parseFileObject(value: unknown, operation: string): DeepSeekFileObject { + if (value === null || typeof value !== 'object' || Array.isArray(value)) throw invalidResponse(operation) + const wire = value as WireFileObject + if (typeof wire.id !== 'string' || wire.id.length === 0 + || wire.object !== 'file' + || !Number.isSafeInteger(wire.bytes) || (wire.bytes as number) < 0 + || !Number.isSafeInteger(wire.created_at) || (wire.created_at as number) < 0 + || typeof wire.filename !== 'string' || wire.filename.length === 0 + || wire.purpose !== 'user_data' + || (wire.expires_at !== undefined + && (!Number.isSafeInteger(wire.expires_at) || (wire.expires_at as number) < 0))) { + throw invalidResponse(operation) + } + return { + id: DeepSeekFileId(wire.id), + bytes: wire.bytes as number, + createdAt: wire.created_at as number, + filename: wire.filename, + purpose: 'user_data', + ...wire.expires_at === undefined ? {} : { expiresAt: wire.expires_at as number }, + } +} + +function providerErrorDetail(value: unknown): { message?: string; detail: string } { + if (value === null || typeof value !== 'object' || Array.isArray(value)) return { detail: '' } + const error = (value as { error?: unknown }).error + if (error === null || typeof error !== 'object' || Array.isArray(error)) return { detail: '' } + const fields = error as { message?: unknown; type?: unknown; code?: unknown } + const message = typeof fields.message === 'string' ? fields.message : undefined + return { + ...message === undefined ? {} : { message }, + detail: [fields.code, fields.type, fields.message] + .filter((field): field is string => typeof field === 'string') + .join(' '), + } +} + +/** Direct client for the OpenAI-compatible `/files` endpoints. */ +export class DeepSeekFilesClient { + private readonly baseURL: string + private readonly apiKey: string + private readonly fetchImpl: typeof fetch + + /** + * @param options - endpoint, API-key snapshot, and optional test transport. + */ + constructor(options: FilesApiOptions) { + this.baseURL = options.baseURL.replace(/\/+$/u, '') + this.apiKey = options.apiKey + this.fetchImpl = options.fetch ?? globalThis.fetch + } + + private async request(path: string, init: RequestInit, signal?: AbortSignal): Promise { + let response: Response + try { + const headers = new Headers(init.headers) + headers.set('authorization', `Bearer ${this.apiKey}`) + response = await this.fetchImpl(`${this.baseURL}${path}`, { + ...init, + headers, + ...signal === undefined ? {} : { signal }, + }) + } catch (error: unknown) { + if (signal?.aborted) throw error + throw new LlmError(`DeepSeek Files API request to ${this.baseURL} failed`, 'TRANSPORT', { cause: error }) + } + if (response.ok) return response + let parsed: unknown + try { + parsed = await response.json() + } catch { + // A status remains sufficient to report the provider failure. + } + const { message, detail } = providerErrorDetail(parsed) + throw new DeepSeekFilesError( + message ?? `DeepSeek Files API error (HTTP ${response.status})`, + response.status, + detail, + ) + } + + /** + * Upload one image with an explicit expiry. + * @param input - deterministic request-version bytes, media type, filename, lifetime, and cancellation. + * @returns the validated provider file object, including `expires_at`. + */ + async upload(input: { + data: Uint8Array + mediaType: ImageMediaType + filename: string + expiresAfterSeconds: number + signal?: AbortSignal + }): Promise { + if (input.data.byteLength > MAX_FILE_UPLOAD_BYTES) { + throw new LlmError('DeepSeek Files API upload exceeds 128 MiB.', 'INVALID_REQUEST') + } + if (!Number.isSafeInteger(input.expiresAfterSeconds) + || input.expiresAfterSeconds < MIN_FILE_EXPIRY_SECONDS + || input.expiresAfterSeconds > MAX_FILE_EXPIRY_SECONDS) { + throw new LlmError('DeepSeek file expiry must be between 3600 and 2592000 seconds.', 'INVALID_REQUEST') + } + const form = new FormData() + form.set('purpose', 'user_data') + form.set('expires_after[anchor]', 'created_at') + form.set('expires_after[seconds]', String(input.expiresAfterSeconds)) + form.set('file', new Blob([Uint8Array.from(input.data).buffer], { type: input.mediaType }), input.filename) + const response = await this.request('/files', { method: 'POST', body: form }, input.signal) + const file = parseFileObject(await response.json(), 'upload') + if (file.expiresAt === undefined) throw invalidResponse('upload') + return file + } + + /** + * List one ascending or descending page of user-data files. + * @param options - pagination, ordering, and cancellation. + * @returns the validated page. + */ + async list(options: { + after?: DeepSeekFileIdType + limit?: number + order?: 'asc' | 'desc' + signal?: AbortSignal + } = {}): Promise { + const query = new URLSearchParams({ purpose: 'user_data' }) + if (options.after !== undefined) query.set('after', options.after) + if (options.limit !== undefined) query.set('limit', String(options.limit)) + if (options.order !== undefined) query.set('order', options.order) + const response = await this.request(`/files?${query.toString()}`, { method: 'GET' }, options.signal) + const value = await response.json() as unknown + if (value === null || typeof value !== 'object' || Array.isArray(value)) throw invalidResponse('list') + const wire = value as { object?: unknown; data?: unknown; first_id?: unknown; last_id?: unknown; has_more?: unknown } + if (wire.object !== 'list' || !Array.isArray(wire.data) || typeof wire.has_more !== 'boolean' + || (wire.first_id !== undefined && typeof wire.first_id !== 'string') + || (wire.last_id !== undefined && typeof wire.last_id !== 'string')) { + throw invalidResponse('list') + } + return { + data: wire.data.map(item => parseFileObject(item, 'list')), + ...typeof wire.first_id === 'string' ? { firstId: DeepSeekFileId(wire.first_id) } : {}, + ...typeof wire.last_id === 'string' ? { lastId: DeepSeekFileId(wire.last_id) } : {}, + hasMore: wire.has_more, + } + } + + /** + * Retrieve one file object. + * @param fileId - provider file identifier. + * @param signal - request cancellation. + * @returns the validated file object. + */ + async retrieve(fileId: DeepSeekFileIdType, signal?: AbortSignal): Promise { + const response = await this.request(`/files/${encodeURIComponent(fileId)}`, { method: 'GET' }, signal) + return parseFileObject(await response.json(), 'retrieve') + } + + /** + * Delete one provider file. + * @param fileId - provider file identifier. + * @param signal - request cancellation. + */ + async delete(fileId: DeepSeekFileIdType, signal?: AbortSignal): Promise { + const response = await this.request(`/files/${encodeURIComponent(fileId)}`, { method: 'DELETE' }, signal) + const value = await response.json() as unknown + if (value === null || typeof value !== 'object' || Array.isArray(value)) throw invalidResponse('delete') + const wire = value as { id?: unknown; object?: unknown; deleted?: unknown } + if (wire.id !== fileId || wire.object !== 'file' || wire.deleted !== true) throw invalidResponse('delete') + } +} diff --git a/packages/llm/llm-deepseek/src/index.ts b/packages/llm/llm-deepseek/src/index.ts index 65b8aa3e3c..6def44f2d3 100644 --- a/packages/llm/llm-deepseek/src/index.ts +++ b/packages/llm/llm-deepseek/src/index.ts @@ -22,8 +22,17 @@ import { MAX_TIMER_DELAY_MS } from '@deepseek-ai/dsh-timeout' import { getOrCreateAnonymousUserId, type AnonymousUserId } from '@deepseek-ai/dsh-anonymous-user-id' import { DEFAULT_CONTEXT_WINDOW, - DEFAULT_MAX_REQUEST_IMAGE_BYTES, + DEFAULT_FILE_EXPIRY_SECONDS, + DEFAULT_FILE_QUOTA_CLEANUP_BATCH, + DEFAULT_FILE_REFRESH_MARGIN_SECONDS, + DEFAULT_IMAGE_OFFLOAD_BYTE_QUANTUM, + DEFAULT_IMAGE_OFFLOAD_COUNT_QUANTUM, + DEFAULT_LOW_DETAIL_IMAGE_PIXEL_BUDGET, + DEFAULT_MAX_IMAGES_PER_REQUEST, + DEFAULT_MAX_REQUEST_FILES_BYTES, DEFAULT_MAX_TOKENS, + DEFAULT_REQUEST_IMAGE_MAX_BYTES, + DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET, DEFAULT_STREAM_IDLE_TIMEOUT_MS, DeepSeekAdapter, } from './adapter.ts' @@ -31,12 +40,29 @@ import type { DeepSeekCatalogModel, DeepSeekConnectionOptions } from './adapter. export { DEFAULT_CONTEXT_WINDOW, - DEFAULT_MAX_REQUEST_IMAGE_BYTES, + DEFAULT_FILE_EXPIRY_SECONDS, + DEFAULT_FILE_QUOTA_CLEANUP_BATCH, + DEFAULT_FILE_REFRESH_MARGIN_SECONDS, + DEFAULT_IMAGE_OFFLOAD_BYTE_QUANTUM, + DEFAULT_IMAGE_OFFLOAD_COUNT_QUANTUM, + DEFAULT_LOW_DETAIL_IMAGE_PIXEL_BUDGET, + DEFAULT_MAX_IMAGES_PER_REQUEST, + DEFAULT_MAX_REQUEST_FILES_BYTES, DEFAULT_MAX_TOKENS, + DEFAULT_REQUEST_IMAGE_MAX_BYTES, + DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET, DEFAULT_STREAM_IDLE_TIMEOUT_MS, DeepSeekAdapter, } from './adapter.ts' export type { DeepSeekAdapterOptions, DeepSeekCatalogModel, DeepSeekConnectionOptions } from './adapter.ts' +export { DeepSeekFileStore, MAX_CHAT_IMAGE_BYTES } from './file-store.ts' +export type { DeepSeekFileConnection, DeepSeekFilePolicy, DeepSeekFileReference } from './file-store.ts' +export { DeepSeekFilesClient, MAX_FILE_EXPIRY_SECONDS, MAX_FILE_UPLOAD_BYTES, MAX_STORED_FILE_BYTES, MAX_STORED_FILE_COUNT, MIN_FILE_EXPIRY_SECONDS } from './files-api.ts' +export type { DeepSeekFileObject, DeepSeekFilePage } from './files-api.ts' +export { DeepSeekFileId } from './file-id.ts' +export type { DeepSeekFileId as DeepSeekFileIdType } from './file-id.ts' +export { DeepSeekUploadIndex, deepSeekFileScope } from './upload-index.ts' +export type { DeepSeekUploadRecord } from './upload-index.ts' export type { RequestDefaults } from './serialize.ts' export type * from './types.ts' @@ -56,6 +82,8 @@ const DEFAULT_MODELS: DeepSeekCatalogModel[] = [ name: 'DeepSeek-V4-Flash-Vision-Exp', contextWindow: DEFAULT_CONTEXT_WINDOW, inputModalities: ['text', 'image'], + imagePixelBudget: DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET, + imageMaxBytes: DEFAULT_REQUEST_IMAGE_MAX_BYTES, }, ] @@ -86,8 +114,20 @@ export interface Config { models?: DeepSeekCatalogModel[] /** Maximum provider idle time while one stream read is outstanding (default five minutes). */ streamIdleTimeoutMs?: number - /** Maximum accumulated base64 image payload per request (default 20 MiB). */ - maxRequestImageBytes?: number + /** Maximum accumulated file-referenced image bytes per chat request (default 128 MiB). */ + maxRequestFilesBytes?: number + /** Maximum number of file-referenced images per chat request (default 600). */ + maxImagesPerRequest?: number + /** Raw-byte removal step after the request exceeds its file bound (default 64 MiB). */ + imageOffloadByteQuantum?: number + /** Image-count removal step after the request exceeds its count bound (default 20). */ + imageOffloadCountQuantum?: number + /** Explicit lifetime assigned to each uploaded image (default seven days). */ + fileExpiresAfterSeconds?: number + /** Remaining lifetime below which an indexed file is replaced (default one hour). */ + fileRefreshMarginSeconds?: number + /** Oldest harness-owned files deleted before one quota-recovery upload retry (default 100). */ + fileQuotaCleanupBatch?: number /** Provider-owned model-request retry policy; omission uses normal mode with five retries. */ retryPolicy?: RetryPolicyConfig } @@ -99,6 +139,9 @@ const catalogModel: z = z.object({ contextWindow: z.number().step(1).min(1), maxTokens: z.number().step(1).min(1), inputModalities: z.array(z.union(MODEL_MODALITIES)).min(1).default(['text']), + imagePixelBudget: z.number().step(1).min(1), + imageMaxBytes: z.number().step(1).min(1), + imageDetail: z.union(['auto', 'low']), }) export const Config: z = z.object({ @@ -110,7 +153,13 @@ export const Config: z = z.object({ defaultContextWindow: z.number().step(1).min(1).default(DEFAULT_CONTEXT_WINDOW), models: z.array(catalogModel).default(DEFAULT_MODELS), streamIdleTimeoutMs: z.number().min(Number.MIN_VALUE).max(MAX_TIMER_DELAY_MS).default(DEFAULT_STREAM_IDLE_TIMEOUT_MS), - maxRequestImageBytes: z.number().step(1).min(1).default(DEFAULT_MAX_REQUEST_IMAGE_BYTES), + maxRequestFilesBytes: z.number().step(1).min(1).default(DEFAULT_MAX_REQUEST_FILES_BYTES), + maxImagesPerRequest: z.number().step(1).min(1).default(DEFAULT_MAX_IMAGES_PER_REQUEST), + imageOffloadByteQuantum: z.number().step(1).min(1).default(DEFAULT_IMAGE_OFFLOAD_BYTE_QUANTUM), + imageOffloadCountQuantum: z.number().step(1).min(1).default(DEFAULT_IMAGE_OFFLOAD_COUNT_QUANTUM), + fileExpiresAfterSeconds: z.number().step(1).min(3_600).max(2_592_000).default(DEFAULT_FILE_EXPIRY_SECONDS), + fileRefreshMarginSeconds: z.number().step(1).min(0).default(DEFAULT_FILE_REFRESH_MARGIN_SECONDS), + fileQuotaCleanupBatch: z.number().step(1).min(1).max(1_000).default(DEFAULT_FILE_QUOTA_CLEANUP_BATCH), retryPolicy: RetryPolicySchema, }) @@ -160,6 +209,19 @@ function resolveModels(models: readonly DeepSeekCatalogModel[] | undefined): Dee if (new Set(inputModalities).size !== inputModalities.length) { throw new Error(`llm-deepseek: catalog model "${model.id}" inputModalities must not contain duplicates`) } + const hasImage = inputModalities.includes('image') + if (!hasImage && (model.imagePixelBudget !== undefined + || model.imageMaxBytes !== undefined || model.imageDetail !== undefined)) { + throw new Error(`llm-deepseek: text-only catalog model "${model.id}" cannot declare image request limits`) + } + if (model.imagePixelBudget !== undefined + && (!Number.isSafeInteger(model.imagePixelBudget) || model.imagePixelBudget <= 0)) { + throw new Error(`llm-deepseek: catalog model "${model.id}" imagePixelBudget must be a positive safe integer`) + } + if (model.imageMaxBytes !== undefined + && (!Number.isSafeInteger(model.imageMaxBytes) || model.imageMaxBytes <= 0)) { + throw new Error(`llm-deepseek: catalog model "${model.id}" imageMaxBytes must be a positive safe integer`) + } if (seen.has(model.id)) throw new Error(`llm-deepseek: duplicate catalog model "${model.id}"`) seen.add(model.id) return { @@ -169,6 +231,16 @@ function resolveModels(models: readonly DeepSeekCatalogModel[] | undefined): Dee ...model.contextWindow === undefined ? {} : { contextWindow: model.contextWindow }, ...model.maxTokens === undefined ? {} : { maxTokens: model.maxTokens }, inputModalities: [...inputModalities], + ...hasImage + ? { + imagePixelBudget: model.imagePixelBudget + ?? (model.imageDetail === 'low' + ? DEFAULT_LOW_DETAIL_IMAGE_PIXEL_BUDGET + : DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET), + imageMaxBytes: model.imageMaxBytes ?? DEFAULT_REQUEST_IMAGE_MAX_BYTES, + ...model.imageDetail === undefined ? {} : { imageDetail: model.imageDetail }, + } + : {}, } }) } @@ -207,9 +279,39 @@ export function resolveAdapterOptions(config: Config, environment?: LaunchEnviro `llm-deepseek: streamIdleTimeoutMs must be a positive finite number no greater than ${MAX_TIMER_DELAY_MS}`, ) } - const maxRequestImageBytes = config.maxRequestImageBytes ?? DEFAULT_MAX_REQUEST_IMAGE_BYTES - if (!Number.isSafeInteger(maxRequestImageBytes) || maxRequestImageBytes <= 0) { - throw new Error('llm-deepseek: maxRequestImageBytes must be a positive safe integer') + const maxRequestFilesBytes = config.maxRequestFilesBytes ?? DEFAULT_MAX_REQUEST_FILES_BYTES + if (!Number.isSafeInteger(maxRequestFilesBytes) || maxRequestFilesBytes <= 0) { + throw new Error('llm-deepseek: maxRequestFilesBytes must be a positive safe integer') + } + const maxImagesPerRequest = config.maxImagesPerRequest ?? DEFAULT_MAX_IMAGES_PER_REQUEST + if (!Number.isSafeInteger(maxImagesPerRequest) || maxImagesPerRequest <= 0) { + throw new Error('llm-deepseek: maxImagesPerRequest must be a positive safe integer') + } + const imageOffloadByteQuantum = config.imageOffloadByteQuantum ?? DEFAULT_IMAGE_OFFLOAD_BYTE_QUANTUM + if (!Number.isSafeInteger(imageOffloadByteQuantum) || imageOffloadByteQuantum <= 0) { + throw new Error('llm-deepseek: imageOffloadByteQuantum must be a positive safe integer') + } + const imageOffloadCountQuantum = config.imageOffloadCountQuantum ?? DEFAULT_IMAGE_OFFLOAD_COUNT_QUANTUM + if (!Number.isSafeInteger(imageOffloadCountQuantum) || imageOffloadCountQuantum <= 0) { + throw new Error('llm-deepseek: imageOffloadCountQuantum must be a positive safe integer') + } + const fileExpiresAfterSeconds = config.fileExpiresAfterSeconds ?? DEFAULT_FILE_EXPIRY_SECONDS + if (!Number.isSafeInteger(fileExpiresAfterSeconds) + || fileExpiresAfterSeconds < 3_600 + || fileExpiresAfterSeconds > 2_592_000) { + throw new Error('llm-deepseek: fileExpiresAfterSeconds must be an integer from 3600 through 2592000') + } + const fileRefreshMarginSeconds = config.fileRefreshMarginSeconds ?? DEFAULT_FILE_REFRESH_MARGIN_SECONDS + if (!Number.isSafeInteger(fileRefreshMarginSeconds) + || fileRefreshMarginSeconds < 0 + || fileRefreshMarginSeconds >= fileExpiresAfterSeconds) { + throw new Error('llm-deepseek: fileRefreshMarginSeconds must be a non-negative integer below fileExpiresAfterSeconds') + } + const fileQuotaCleanupBatch = config.fileQuotaCleanupBatch ?? DEFAULT_FILE_QUOTA_CLEANUP_BATCH + if (!Number.isSafeInteger(fileQuotaCleanupBatch) + || fileQuotaCleanupBatch < 1 + || fileQuotaCleanupBatch > 1_000) { + throw new Error('llm-deepseek: fileQuotaCleanupBatch must be an integer from 1 through 1000') } return { apiKeyEnv: credentialRef(config.apiKeyEnv ?? DEFAULT_API_KEY_ENV), @@ -224,7 +326,15 @@ export function resolveAdapterOptions(config: Config, environment?: LaunchEnviro defaultContextWindow: config.defaultContextWindow ?? DEFAULT_CONTEXT_WINDOW, models: resolveModels(config.models), streamIdleTimeoutMs, - maxRequestImageBytes, + maxRequestFilesBytes, + maxImagesPerRequest, + imageOffloadByteQuantum, + imageOffloadCountQuantum, + filePolicy: { + expiresAfterSeconds: fileExpiresAfterSeconds, + refreshMarginSeconds: fileRefreshMarginSeconds, + quotaCleanupBatch: fileQuotaCleanupBatch, + }, retryPolicy: resolveRetryPolicy(config.retryPolicy, 'llm-deepseek: retryPolicy'), } } diff --git a/packages/llm/llm-deepseek/src/serialize.ts b/packages/llm/llm-deepseek/src/serialize.ts index 498b3fb2f7..f066808b99 100644 --- a/packages/llm/llm-deepseek/src/serialize.ts +++ b/packages/llm/llm-deepseek/src/serialize.ts @@ -1,19 +1,19 @@ /** * Serialize harness messages into DeepSeek chat completions. Text-only * requests retain string user content; the image path resolves durable - * attachments into ordered data-URL parts. Tool-result images follow their + * attachments into ordered Files API parts. Tool-result images follow their * string-only tool messages in a separate user message. * @module dsh-llm-deepseek/serialize */ -import { contentHasImage, LlmError, offloadRequestImages } from '@deepseek-ai/dsh-llm' +import { contentHasImage, LlmError, offloadRequestImagesWithPolicy, requestImagePreviewText } from '@deepseek-ai/dsh-llm' import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm' -import { AttachmentError } from '@deepseek-ai/dsh-attachment' -import type { AttachmentStore } from '@deepseek-ai/dsh-attachment' +import type { ImageAttachmentRef, RequestImageAttachment } from '@deepseek-ai/dsh-attachment' import type { - WireImageContentPart, + WireFileContentPart, WireMessage, WireRequest, + WireTextContentPart, WireTool, WireUserContentPart, } from './types.ts' @@ -31,12 +31,28 @@ interface ResolvedThinking { /** Dependencies required only when the request contains image input. */ export interface ImageSerializationOptions { - /** Durable resolver for canonical image references. */ - attachments: AttachmentStore - /** Positive bound on accumulated base64 image payload. */ - maxRequestImageBytes: number - /** Cancellation shared with the provider request. */ - signal: AbortSignal + /** Resolve a retained request version to a reusable DeepSeek file id. */ + resolveFileId: ( + version: RequestImageAttachment, + block: Extract, + location: ImageWireLocation, + ) => Promise + /** Request versions prepared before offload selection, keyed by master attachment id. */ + requestImages: ReadonlyMap + /** Positive bound on accumulated referenced image bytes. */ + maxRequestFilesBytes: number + /** Maximum referenced images in one request. */ + maxImagesPerRequest?: number + /** Raw-byte removal step applied after the request exceeds its byte bound. */ + byteQuantum?: number + /** Image-count removal step applied after the request exceeds its count bound. */ + countQuantum?: number +} + +/** Durable message and image ordinal used in provider diagnostics. */ +export interface ImageWireLocation { + message: number + image: number } const TOOL_RESULT_IMAGE_TEXT = 'Attached image(s) from tool result:' @@ -98,33 +114,40 @@ function assertSupportedImageRoles(messages: readonly Message[]): void { } } -/** Resolve one durable image into its transient DeepSeek data-URL part. */ -async function imagePart( - block: Extract, - attachments: AttachmentStore, - signal: AbortSignal, -): Promise { - try { - const stored = await attachments.readImage(block.attachment, signal) - return { - type: 'image_url', - image_url: { - url: `data:${stored.ref.mediaType};base64,${Buffer.from(stored.data).toString('base64')}`, - }, - } - } catch (error: unknown) { - if (error instanceof AttachmentError) { - throw new LlmError(error.message, error.code, { cause: error }) - } - throw error +/** Describe the exact request preview and its model-callable coordinate system. */ +function imageHandle(version: RequestImageAttachment, precededByContent: boolean): WireTextContentPart { + return { + type: 'text', + text: `${precededByContent ? '\n' : ''}${requestImagePreviewText(version)}`, } } +/** Resolve one durable image into its descriptor and transient DeepSeek file-id part. */ +async function imageParts( + block: Extract, + images: ImageSerializationOptions, + location: ImageWireLocation, + precededByContent: boolean, +): Promise<[WireTextContentPart, WireFileContentPart]> { + const version = images.requestImages.get(block.attachment.attachmentId) + if (version === undefined) { + throw new LlmError( + `DeepSeek request image ${block.attachment.attachmentId} was not prepared.`, + 'INVALID_REQUEST', + ) + } + return [ + imageHandle(version, precededByContent), + { type: 'file', file_id: await images.resolveFileId(version, block, location) }, + ] +} + /** Convert user or nested tool-result blocks into ordered wire parts. */ async function contentParts( blocks: readonly ContentBlock[], - attachments: AttachmentStore, - signal: AbortSignal, + images: ImageSerializationOptions, + message: number, + nextImage: { value: number }, ): Promise { const parts: WireUserContentPart[] = [] for (const block of blocks) { @@ -133,10 +156,11 @@ async function contentParts( if (block.text.length > 0) parts.push({ type: 'text', text: block.text }) break case 'image': - parts.push(await imagePart(block, attachments, signal)) + nextImage.value += 1 + parts.push(...await imageParts(block, images, { message, image: nextImage.value }, parts.length > 0)) break case 'tool-result': - parts.push(...await contentParts(block.content, attachments, signal)) + parts.push(...await contentParts(block.content, images, message, nextImage)) break default: // Other merge-extensible blocks are not DeepSeek user-input vocabulary. @@ -150,7 +174,7 @@ async function contentParts( function userContent(parts: readonly WireUserContentPart[]): string | WireUserContentPart[] { const text: string[] = [] for (const part of parts) { - if (part.type === 'image_url') return [...parts] + if (part.type === 'file') return [...parts] text.push(part.text) } return text.join('') @@ -236,18 +260,16 @@ export function serializeMessages(messages: Message[]): WireMessage[] { * Consecutive tool results keep string `tool` messages and share one following * user message containing their images. * @param messages - transient request history after request-size offloading. - * @param attachments - durable image resolver. - * @param signal - cancellation for attachment reads. + * @param images - prepared request versions and reusable provider file-id resolver. * @returns ordered DeepSeek wire messages. */ export async function serializeMessagesWithImages( messages: readonly Message[], - attachments: AttachmentStore, - signal: AbortSignal, + images: ImageSerializationOptions, ): Promise { assertSupportedImageRoles(messages) const wire: WireMessage[] = [] - let pendingToolImages: WireImageContentPart[] = [] + let pendingToolImages: WireFileContentPart[] = [] const flushToolImages = (): void => { if (pendingToolImages.length === 0) return wire.push({ @@ -257,7 +279,8 @@ export async function serializeMessagesWithImages( pendingToolImages = [] } - for (const message of messages) { + for (const [messageIndex, message] of messages.entries()) { + const nextImage = { value: 0 } if (message.role === 'system') { flushToolImages() wire.push({ role: 'system', content: flattenText(message.content) }) @@ -273,7 +296,7 @@ export async function serializeMessagesWithImages( const toolResults = message.content.filter((block): block is Extract => ( block.type === 'tool-result' )) - const content = userContent(await contentParts(regular, attachments, signal)) + const content = userContent(await contentParts(regular, images, messageIndex + 1, nextImage)) if (content.length > 0 || toolResults.length === 0) { flushToolImages() wire.push({ @@ -282,15 +305,15 @@ export async function serializeMessagesWithImages( }) } for (const result of toolResults) { - const parts = await contentParts(result.content, attachments, signal) - const images = parts.filter((part): part is WireImageContentPart => part.type === 'image_url') + const parts = await contentParts(result.content, images, messageIndex + 1, nextImage) + const fileParts = parts.filter((part): part is WireFileContentPart => part.type === 'file') const text = parts.filter(part => part.type === 'text').map(part => part.text).join('') wire.push({ role: 'tool', tool_call_id: result.toolCallId, - content: text || (images.length > 0 ? '(see attached image)' : '(no output)'), + content: text || (fileParts.length > 0 ? '(see attached image)' : '(no output)'), }) - pendingToolImages.push(...images) + pendingToolImages.push(...fileParts) } } flushToolImages() @@ -351,8 +374,8 @@ export function serializeRequest( /** * Build one image-capable request while keeping durable bytes out of session - * messages. Oversized oldest images become deterministic text before any - * attachment read. + * messages. Oversized oldest images become deterministic text after their + * exact request-version byte lengths are known and before provider upload. * @param options - harness request containing image-capable user content. * @param images - attachment resolver, request bound, and cancellation. * @param defaults - adapter-level thinking defaults. @@ -364,11 +387,24 @@ export async function serializeRequestWithImages( defaults: RequestDefaults = {}, ): Promise { assertSupportedImageRoles(options.messages) - const requestMessages = offloadRequestImages(options.messages, images.maxRequestImageBytes) + const requestMessages = offloadRequestImagesWithPolicy(options.messages, { + representation: 'raw', + byteLength: (ref) => { + const version = images.requestImages.get(ref.attachmentId) + if (version === undefined) { + throw new LlmError(`DeepSeek request image ${ref.attachmentId} was not prepared.`, 'INVALID_REQUEST') + } + return version.bytes + }, + maxBytes: images.maxRequestFilesBytes, + ...images.maxImagesPerRequest === undefined ? {} : { maxImages: images.maxImagesPerRequest }, + ...images.byteQuantum === undefined ? {} : { byteQuantum: images.byteQuantum }, + ...images.countQuantum === undefined ? {} : { countQuantum: images.countQuantum }, + }) const messages: WireMessage[] = [] if (options.system !== undefined) { messages.push({ role: 'system', content: options.system }) } - messages.push(...await serializeMessagesWithImages(requestMessages, images.attachments, images.signal)) + messages.push(...await serializeMessagesWithImages(requestMessages, images)) return requestWithMessages(options, messages, defaults) } diff --git a/packages/llm/llm-deepseek/src/types.ts b/packages/llm/llm-deepseek/src/types.ts index 93c7f48a36..54f39b095b 100644 --- a/packages/llm/llm-deepseek/src/types.ts +++ b/packages/llm/llm-deepseek/src/types.ts @@ -41,14 +41,14 @@ export interface WireTextContentPart { text: string } -/** Base64 data URL part inside a multimodal user message. */ -export interface WireImageContentPart { - type: 'image_url' - image_url: { url: string } +/** Files API reference inside a multimodal user message. */ +export interface WireFileContentPart { + type: 'file' + file_id: string } /** Ordered input part accepted by a multimodal user message. */ -export type WireUserContentPart = WireTextContentPart | WireImageContentPart +export type WireUserContentPart = WireTextContentPart | WireFileContentPart /** User-role message: text-only string or ordered multimodal input. */ export interface WireUserMessage { diff --git a/packages/llm/llm-deepseek/src/upload-index.ts b/packages/llm/llm-deepseek/src/upload-index.ts new file mode 100644 index 0000000000..12d433b760 --- /dev/null +++ b/packages/llm/llm-deepseek/src/upload-index.ts @@ -0,0 +1,225 @@ +/** Durable DeepSeek attachment-to-file-id index. @module dsh-llm-deepseek/upload-index */ + +import { createHash } from 'node:crypto' +import { readFile, mkdir } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { withFileLock, writeFileAtomic } from '@deepseek-ai/dsh-atomic-write' +import { resolveDshHome } from '@deepseek-ai/dsh-home-paths' +import { ImageVariantId } from '@deepseek-ai/dsh-attachment' +import type { AttachmentId, ImageVariantId as ImageVariantIdType } from '@deepseek-ai/dsh-attachment' +import { DeepSeekFileId, DeepSeekFileScope } from './file-id.ts' +import type { DeepSeekFileId as DeepSeekFileIdType, DeepSeekFileScope as DeepSeekFileScopeType } from './file-id.ts' + +/** One durable remote upload mapping. Unix times are milliseconds. */ +export interface DeepSeekUploadRecord { + scope: DeepSeekFileScopeType + /** Provider-independent master attachment from which the uploaded request version was derived. */ + masterAttachmentId: AttachmentId + /** Complete request transformation identity, including crop and encoder parameters. */ + variantId: ImageVariantIdType + fileId: DeepSeekFileIdType + bytes: number + createdAt: number + expiresAt: number +} + +interface StoredIndex { + formatVersion: 2 + records: DeepSeekUploadRecord[] +} + +class InvalidUploadIndexError extends Error {} + +/** Candidate commit outcome when another process already published a reusable upload. */ +export interface UploadIndexCommit { + record: DeepSeekUploadRecord + accepted: boolean +} + +/** + * Derive a non-secret stable index namespace without persisting or logging the API key. + * @param baseURL - normalized provider endpoint namespace. + * @param apiKey - resolved credential used only as hash input. + * @returns branded SHA-256 namespace digest. + */ +export function deepSeekFileScope(baseURL: string, apiKey: string): DeepSeekFileScopeType { + const digest = createHash('sha256') + .update(baseURL.replace(/\/+$/u, '')) + .update('\0') + .update(apiKey) + .digest('hex') + return DeepSeekFileScope(digest) +} + +function absent(error: unknown): boolean { + return (error as NodeJS.ErrnoException | null)?.code === 'ENOENT' +} + +function parseRecord(value: unknown): DeepSeekUploadRecord { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new InvalidUploadIndexError('llm-deepseek: upload index contains a non-object record') + } + const record = value as Record + if (typeof record.scope !== 'string' || !/^[0-9a-f]{64}$/u.test(record.scope) + || typeof record.masterAttachmentId !== 'string' || !/^sha256:[0-9a-f]{64}$/u.test(record.masterAttachmentId) + || typeof record.variantId !== 'string' || !/^sha256:[0-9a-f]{64}$/u.test(record.variantId) + || typeof record.fileId !== 'string' || record.fileId.length === 0 + || !Number.isSafeInteger(record.bytes) || (record.bytes as number) < 0 + || !Number.isSafeInteger(record.createdAt) || (record.createdAt as number) < 0 + || !Number.isSafeInteger(record.expiresAt) || (record.expiresAt as number) < 0) { + throw new InvalidUploadIndexError('llm-deepseek: upload index contains an invalid record') + } + return { + scope: DeepSeekFileScope(record.scope), + masterAttachmentId: record.masterAttachmentId as AttachmentId, + variantId: ImageVariantId(record.variantId), + fileId: DeepSeekFileId(record.fileId), + bytes: record.bytes as number, + createdAt: record.createdAt as number, + expiresAt: record.expiresAt as number, + } +} + +function parseIndex(text: string): StoredIndex { + let value: unknown + try { + value = JSON.parse(text) + } catch (error: unknown) { + throw new InvalidUploadIndexError('llm-deepseek: upload index is not valid JSON', { cause: error }) + } + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw new InvalidUploadIndexError('llm-deepseek: upload index is not an object') + } + const index = value as { formatVersion?: unknown; records?: unknown } + if (index.formatVersion !== 2 || !Array.isArray(index.records)) { + throw new InvalidUploadIndexError('llm-deepseek: unsupported upload index format') + } + const records = index.records.map(parseRecord) + const keys = new Set() + for (const record of records) { + const key = `${record.scope}\0${record.variantId}` + if (keys.has(key)) throw new InvalidUploadIndexError('llm-deepseek: upload index contains duplicate mappings') + keys.add(key) + } + return { formatVersion: 2, records } +} + +function reusable(record: DeepSeekUploadRecord, now: number, refreshMarginMs: number): boolean { + return record.expiresAt - now > refreshMarginMs +} + +/** Atomic local index shared by every DeepSeek session in this DSH home. */ +export class DeepSeekUploadIndex { + /** Absolute owner-private JSON index path. */ + readonly path: string + + /** + * @param path - explicit test path; omission uses `DSH_HOME/llm-deepseek/files-v2.json`. + */ + constructor(path = join(resolveDshHome(), 'llm-deepseek', 'files-v2.json')) { + this.path = path + } + + private async load(): Promise { + try { + return parseIndex(await readFile(this.path, 'utf8')) + } catch (error: unknown) { + if (absent(error) || error instanceof InvalidUploadIndexError) { + return { formatVersion: 2, records: [] } + } + throw error + } + } + + private async save(index: StoredIndex): Promise { + await writeFileAtomic(this.path, `${JSON.stringify(index, undefined, 2)}\n`, { + mode: 0o600, + dirMode: 0o700, + }) + } + + /** + * Read one reusable mapping. + * @param scope - endpoint/API-key namespace. + * @param variantId - complete request-image transformation identity. + * @param now - current Unix time in milliseconds. + * @param refreshMarginMs - remaining lifetime below which a mapping is not reused. + * @returns the mapping when it has enough lifetime remaining. + */ + async get( + scope: DeepSeekFileScopeType, + variantId: ImageVariantIdType, + now: number, + refreshMarginMs: number, + ): Promise { + const record = (await this.load()).records.find(candidate => ( + candidate.scope === scope && candidate.variantId === variantId + )) + return record !== undefined && reusable(record, now, refreshMarginMs) ? record : undefined + } + + /** + * Publish a completed upload unless another process already published a reusable mapping. + * @param candidate - completed remote upload. + * @param now - current Unix time in milliseconds. + * @param refreshMarginMs - minimum reusable remaining lifetime. + * @returns the winning record and whether the candidate entered the index. + */ + async commit( + candidate: DeepSeekUploadRecord, + now: number, + refreshMarginMs: number, + ): Promise { + await mkdir(dirname(this.path), { recursive: true, mode: 0o700 }) + return withFileLock(this.path, async () => { + const index = await this.load() + const existing = index.records.find(record => ( + record.scope === candidate.scope + && record.variantId === candidate.variantId + && reusable(record, now, refreshMarginMs) + )) + if (existing !== undefined) return { record: existing, accepted: false } + const records = index.records.filter(record => ( + reusable(record, now, refreshMarginMs) + && !(record.scope === candidate.scope && record.variantId === candidate.variantId) + )) + records.push(candidate) + await this.save({ formatVersion: 2, records }) + return { record: candidate, accepted: true } + }) + } + + /** + * Remove one exact mapping without deleting a concurrently installed successor. + * @param scope - endpoint/API-key namespace. + * @param variantId - complete request-image transformation identity. + * @param fileId - exact remote generation being invalidated. + */ + async remove( + scope: DeepSeekFileScopeType, + variantId: ImageVariantIdType, + fileId: DeepSeekFileIdType, + ): Promise { + await mkdir(dirname(this.path), { recursive: true, mode: 0o700 }) + await withFileLock(this.path, async () => { + const index = await this.load() + const records = index.records.filter(record => !( + record.scope === scope && record.variantId === variantId && record.fileId === fileId + )) + if (records.length !== index.records.length) await this.save({ formatVersion: 2, records }) + }) + } + + /** + * Remove every local mapping for one remote namespace. + * @param scope - endpoint/API-key namespace. + */ + async clear(scope: DeepSeekFileScopeType): Promise { + await mkdir(dirname(this.path), { recursive: true, mode: 0o700 }) + await withFileLock(this.path, async () => { + const index = await this.load() + const records = index.records.filter(record => record.scope !== scope) + if (records.length !== index.records.length) await this.save({ formatVersion: 2, records }) + }) + } +} diff --git a/packages/llm/llm-deepseek/tests/adapter.e2e.ts b/packages/llm/llm-deepseek/tests/adapter.e2e.ts index 5ac490b110..c52195433b 100644 --- a/packages/llm/llm-deepseek/tests/adapter.e2e.ts +++ b/packages/llm/llm-deepseek/tests/adapter.e2e.ts @@ -1,15 +1,19 @@ +import { readFileSync } from 'node:fs' import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' -import { createHash } from 'node:crypto' +import { randomBytes } from 'node:crypto' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { Context } from '@deepseek-ai/cordis' -import LlmRuntime, { createUserMessage, CallId, ReasoningEffortId , createMessage } from '@deepseek-ai/dsh-llm' +import LlmRuntime, { createUserMessage, CallId, ReasoningEffortId, createMessage } from '@deepseek-ai/dsh-llm' import type { Message, ToolSchema } from '@deepseek-ai/dsh-llm' -import AttachmentStore, { AttachmentId } from '@deepseek-ai/dsh-attachment' +import AttachmentStore, { AttachmentId, ImageVariantId } from '@deepseek-ai/dsh-attachment' import type { ImageAttachmentLimits, ImageAttachmentRef, + ImageRequestPolicy, + RequestImageAttachment, + SavedImageAttachment, SaveImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' @@ -29,42 +33,70 @@ const FLASH = 'deepseek-v4-flash' const PRO = 'deepseek-v4-pro' const VISION = 'deepseek-v4-flash-vision-exp' const VISION_E2E_ENABLED = process.env.DEEPSEEK_VISION_E2E === '1' -const RED_IMAGE = Buffer.from( - 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC', - 'base64', -) -const RED_IMAGE_REF: ImageAttachmentRef = { - attachmentId: AttachmentId(`sha256:${createHash('sha256').update(RED_IMAGE).digest('hex')}`), - mediaType: 'image/png', - bytes: RED_IMAGE.byteLength, - width: 1, - height: 1, -} +const TEST_PNG = Uint8Array.from(readFileSync( + new URL('../../llm-pi-ai/tests/fixtures/qr-code.png', import.meta.url), +)) +const contexts: Context[] = [] +let identityHome: string class E2eAttachmentStore extends AttachmentStore { readonly imageLimits: ImageAttachmentLimits = { - maxImageBytes: 1024, + maxImageBytes: TEST_PNG.byteLength, maxImagesPerMessage: 1, - maxMessageImageBytes: 1024, - maxImagePixels: 1, - maxImageDimension: 1, + maxMessageImageBytes: TEST_PNG.byteLength, + maxImagePixels: 256 * 256, + maxImageDimension: 256, mediaTypes: ['image/png'], } + readonly ref: ImageAttachmentRef = { + attachmentId: AttachmentId(`sha256:${randomBytes(32).toString('hex')}`), + mediaType: 'image/png', + bytes: TEST_PNG.byteLength, + width: 256, + height: 256, + name: 'files-api-e2e.png', + } + readonly version: RequestImageAttachment = { + variantId: ImageVariantId(`sha256:${randomBytes(32).toString('hex')}`), + master: this.ref, + data: TEST_PNG, + mediaType: 'image/png', + bytes: TEST_PNG.byteLength, + width: 256, + height: 256, + depth: 'uchar', + space: 'srgb', + hasAlpha: false, + } validateImage(_input: SaveImageAttachment): Promise { return Promise.resolve() } - saveImage(_input: SaveImageAttachment): Promise { - return Promise.resolve(RED_IMAGE_REF) + saveImage(_input: SaveImageAttachment): Promise { + return Promise.resolve({ + ref: this.ref, + source: { + mediaType: this.ref.mediaType, + bytes: this.ref.bytes, + width: this.ref.width, + height: this.ref.height, + }, + }) } - readImage(_ref: ImageAttachmentRef, _signal?: AbortSignal): Promise { - return Promise.resolve({ ref: RED_IMAGE_REF, data: RED_IMAGE }) + readImage(ref: ImageAttachmentRef, _signal?: AbortSignal): Promise { + return Promise.resolve({ ref, data: TEST_PNG }) + } + + override readImageRequest( + _ref: ImageAttachmentRef, + _policy: ImageRequestPolicy, + _signal?: AbortSignal, + ): Promise { + return Promise.resolve(this.version) } } -const contexts: Context[] = [] -let identityHome: string beforeEach(async () => { identityHome = await mkdtemp(join(tmpdir(), 'dsh-e2e-user-id-')) @@ -82,6 +114,7 @@ async function harness(_model: string, config: Partial = {}) { afterEach(async () => { await Promise.all(contexts.splice(0).map(ctx => ctx.fiber.dispose())) + vi.unstubAllGlobals() vi.unstubAllEnvs() await rm(identityHome, { recursive: true, force: true }) }) @@ -111,23 +144,46 @@ const weatherTool: ToolSchema = { } describe.skipIf(!process.env.DEEPSEEK_API_KEY)('llm-deepseek e2e (real API)', () => { - it.skipIf(!VISION_E2E_ENABLED)('recognizes a deterministic image with the official vision model', async () => { - const ctx = await harness(VISION, { - thinking: 'disabled', - }) - const result = await assemble(ctx, { - model: VISION, - messages: [createUserMessage({ - content: [ - { type: 'text', text: 'This image is one solid color. Reply with only its English color name.' }, - { type: 'image', attachment: RED_IMAGE_REF }, - ], - source: { kind: 'plugin', plugin: 'test' }, - })], - maxTokens: 50, - }) - expect(result.finish.kind).toBe('stop') - expect(textOf(result).toLowerCase()).toContain('red') + it.skipIf(!VISION_E2E_ENABLED)('uses the built-in official route to upload, reference, and delete one image', async () => { + const key = process.env.DEEPSEEK_API_KEY + if (key === undefined) throw new Error('e2e ran without DEEPSEEK_API_KEY') + const baseURL = process.env.DEEPSEEK_BASE_URL ?? LlmDeepSeek.PUBLIC_BASE_URL + const ctx = await harness(VISION, { baseURL }) + await ctx.plugin(E2eAttachmentStore) + const attachments = ctx.attachments as E2eAttachmentStore + let uploadedFile: LlmDeepSeek.DeepSeekFileIdType | undefined + const nativeFetch = globalThis.fetch + const observedFetch: typeof fetch = async (input, init) => { + const response = await nativeFetch(input, init) + const url = new URL(input instanceof Request ? input.url : input) + const method = init?.method ?? (input instanceof Request ? input.method : 'GET') + if (method === 'POST' && url.pathname.endsWith('/files') && response.ok) { + const value = await response.clone().json() as { id?: unknown } + if (typeof value.id === 'string') uploadedFile = LlmDeepSeek.DeepSeekFileId(value.id) + } + return response + } + vi.stubGlobal('fetch', observedFetch) + const files = new LlmDeepSeek.DeepSeekFilesClient({ baseURL, apiKey: key }) + + try { + const result = await assemble(ctx, { + model: VISION, + messages: [createUserMessage({ + content: [ + { type: 'text', text: 'Briefly describe this image.' }, + { type: 'image', attachment: attachments.ref }, + ], + source: { kind: 'plugin', plugin: 'test' }, + })], + maxTokens: 100, + }) + expect(result.finish.kind).toBe('stop') + expect(textOf(result).trim().length).toBeGreaterThan(0) + expect(uploadedFile).toMatch(/^file-api-/u) + } finally { + if (uploadedFile !== undefined) await files.delete(uploadedFile) + } }) it('serves a real request with the key held only by a credentials-local document', async () => { diff --git a/packages/llm/llm-deepseek/tests/adapter.spec.ts b/packages/llm/llm-deepseek/tests/adapter.spec.ts index 5bcccb9d99..e37a1a882e 100644 --- a/packages/llm/llm-deepseek/tests/adapter.spec.ts +++ b/packages/llm/llm-deepseek/tests/adapter.spec.ts @@ -3,8 +3,8 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { Context } from '@deepseek-ai/cordis' -import { AttachmentId } from '@deepseek-ai/dsh-attachment' -import type { AttachmentStore, ImageAttachmentRef } from '@deepseek-ai/dsh-attachment' +import { AttachmentId, ImageVariantId } from '@deepseek-ai/dsh-attachment' +import type { AttachmentStore, ImageAttachmentRef, RequestImageAttachment } from '@deepseek-ai/dsh-attachment' import { createLaunchEnvironmentSnapshot } from '@deepseek-ai/dsh-launch-environment' import LlmRuntime, { createUserMessage, CONTEXT_WINDOW_EXCEEDED_CODE, @@ -74,6 +74,41 @@ const imageRef: ImageAttachmentRef = { height: 1, } +function requestImage(ref = imageRef): RequestImageAttachment { + return { + variantId: ImageVariantId(`sha256:${'b'.repeat(64)}`), + master: ref, + data: Uint8Array.of(1, 2, 3), + mediaType: 'image/png', + bytes: 3, + width: 1, + height: 1, + depth: 'uchar', + space: 'srgb', + hasAlpha: true, + } +} + +function attachmentStoreOf( + project: (ref: ImageAttachmentRef, policy: unknown, signal?: AbortSignal) => Promise, +): { + store: AttachmentStore + readImageRequest: ReturnType> + readImageRequests: ReturnType +} { + const readImageRequest = vi.fn(project) + const readImageRequests = vi.fn(async ( + refs: readonly ImageAttachmentRef[], + policy: unknown, + signal?: AbortSignal, + ) => Promise.all(refs.map(ref => readImageRequest(ref, policy, signal)))) + return { + store: { readImageRequest, readImageRequests } as unknown as AttachmentStore, + readImageRequest, + readImageRequests, + } +} + describe('DeepSeekAdapter against a mock server', () => { it('streams a text generation end to end through the assembler', async () => { const server = await mockServer([{ kind: 'sse', events: textEvents }]) @@ -108,16 +143,16 @@ describe('DeepSeekAdapter against a mock server', () => { expect(server.headers[0]).not.toHaveProperty('x-deepseek-harness-compact') }) - it('sends a durable image as a base64 data URL for the vision model', async () => { + it('uploads a durable image once and sends only its Files API id to the vision model', async () => { const server = await mockServer([{ kind: 'sse', events: textEvents }]) const signalSeen: (AbortSignal | undefined)[] = [] - const attachments = { - readImage: vi.fn((ref: ImageAttachmentRef, signal?: AbortSignal) => { - signalSeen.push(signal) - return Promise.resolve({ ref, data: Uint8Array.of(1, 2, 3) }) - }), - } as unknown as AttachmentStore - const adapter = adapterOf({ baseURL: server.url }, attachments) + const policies: unknown[] = [] + const attachmentMocks = attachmentStoreOf((ref, policy, signal) => { + signalSeen.push(signal) + policies.push(policy) + return Promise.resolve(requestImage(ref)) + }) + const adapter = adapterOf({ baseURL: server.url }, attachmentMocks.store) await drain(adapter.stream({ provider: 'deepseek-official', @@ -137,11 +172,228 @@ describe('DeepSeekAdapter against a mock server', () => { role: 'user', content: [ { type: 'text', text: 'describe ' }, - { type: 'image_url', image_url: { url: 'data:image/png;base64,AQID' } }, + { type: 'text', text: expect.stringContaining(`Image ${imageRef.attachmentId}`) as string }, + { type: 'file', file_id: 'file-api-1' }, ], }], }) + expect(server.fileRequests).toEqual([{ + method: 'POST', + path: '/files', + filename: `dsh-${'a'.repeat(16)}-${'b'.repeat(8)}.png`, + bytes: 3, + }]) expect(signalSeen[0]).toBeInstanceOf(AbortSignal) + expect(policies).toEqual([{ maxPixels: 640_000, maxBytes: 1024 * 1024 }]) + }) + + it('reuses the exact request version between agent and compaction calls', async () => { + const server = await mockServer([ + { kind: 'sse', events: textEvents }, + { kind: 'sse', events: textEvents }, + ]) + const attachments = attachmentStoreOf(ref => Promise.resolve(requestImage(ref))).store + const adapter = adapterOf({ + baseURL: server.url, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachments) + const messages = [createUserMessage({ + content: [{ type: 'image' as const, attachment: imageRef }], + source: { kind: 'plugin' as const, plugin: 'test' }, + })] + + await drain(adapter.stream({ provider: 'deepseek-official', model: 'deepseek-v4-flash-vision-exp', messages })) + await drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages, + purpose: 'compaction', + })) + + expect(server.fileRequests.filter(request => request.method === 'POST')).toHaveLength(1) + expect(server.requests).toMatchObject([ + { messages: [{ content: [expect.objectContaining({ type: 'text' }), { file_id: 'file-api-1' }] }] }, + { messages: [{ content: [expect.objectContaining({ type: 'text' }), { file_id: 'file-api-1' }] }] }, + ]) + expect(server.headers[1]?.['x-deepseek-harness-compact']).toBe('1') + }) + + it('explains a provider rejection of a normalized image and retains the raw response as cause', async () => { + const raw = JSON.stringify({ error: { message: 'unsupported image payload' } }) + const server = await mockServer([{ kind: 'http-error', status: 400, body: raw }]) + const attachments = attachmentStoreOf(ref => Promise.resolve(requestImage(ref))).store + const adapter = adapterOf({ + baseURL: server.url, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachments) + + let failure: unknown + try { + await drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [{ type: 'image', attachment: imageRef }], + source: { kind: 'plugin', plugin: 'test' }, + })], + })) + } catch (error: unknown) { + failure = error + } + expect(failure).toMatchObject({ + code: 'INVALID_REQUEST', + message: expect.stringContaining( + `normalized image "${imageRef.attachmentId}" at message 1, image 1`, + ) as string, + cause: { message: raw }, + }) + expect((failure as Error).message).toContain('image/png, 8-bit sRGBA, 1x1') + expect((failure as Error).message).toContain('unsupported image payload') + expect((failure as Error).message).not.toBe(raw) + }) + + it.each([ + 'file_id file-api-1 expired', + 'file_not_found', + 'file_id file-api-1 deleted', + 'invalid file_id file-api-1', + ])('reuploads once when chat rejects a Files API reference as %s', async (providerMessage) => { + const server = await mockServer([ + { + kind: 'http-error', + status: 400, + body: JSON.stringify({ error: { message: providerMessage } }), + }, + { kind: 'sse', events: textEvents }, + ]) + const attachmentMocks = attachmentStoreOf(ref => Promise.resolve(requestImage(ref))) + const adapter = adapterOf({ + baseURL: server.url, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachmentMocks.store) + const options = { + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [{ type: 'image' as const, attachment: imageRef }], + source: { kind: 'plugin' as const, plugin: 'test' }, + })], + } + + await drain(adapter.stream(options)) + + expect(server.fileRequests.filter(request => request.method === 'POST')).toHaveLength(2) + expect(server.requests).toMatchObject([ + { messages: [{ content: [expect.objectContaining({ type: 'text' }), { file_id: 'file-api-1' }] }] }, + { messages: [{ content: [expect.objectContaining({ type: 'text' }), { file_id: 'file-api-2' }] }] }, + ]) + expect(attachmentMocks.readImageRequest).toHaveBeenCalledTimes(1) + }) + + it('invalidates only the identified mapping when a multi-image request names one stale file id', async () => { + const secondRef: ImageAttachmentRef = { + ...imageRef, + attachmentId: AttachmentId(`sha256:${'c'.repeat(64)}`), + } + const server = await mockServer([ + { + kind: 'http-error', + status: 400, + body: JSON.stringify({ error: { message: 'file_id file-api-2 expired' } }), + }, + { kind: 'sse', events: textEvents }, + ]) + const attachments = attachmentStoreOf(ref => Promise.resolve({ + ...requestImage(ref), + variantId: ImageVariantId(`sha256:${(ref.attachmentId === imageRef.attachmentId ? 'b' : 'd').repeat(64)}`), + })).store + const adapter = adapterOf({ + baseURL: server.url, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachments) + + await drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [ + { type: 'image', attachment: imageRef }, + { type: 'image', attachment: secondRef }, + ], + source: { kind: 'plugin', plugin: 'test' }, + })], + })) + + expect(server.fileRequests.filter(request => request.method === 'POST')).toHaveLength(3) + const retries = server.requests as Array<{ messages: Array<{ content: Array<{ type: string; file_id?: string }> }> }> + expect(retries[0]?.messages[0]?.content.filter(block => block.type === 'file')) + .toEqual([{ type: 'file', file_id: 'file-api-1' }, { type: 'file', file_id: 'file-api-2' }]) + expect(retries[1]?.messages[0]?.content.filter(block => block.type === 'file')) + .toEqual([{ type: 'file', file_id: 'file-api-1' }, { type: 'file', file_id: 'file-api-3' }]) + }) + + it('invalidates every used mapping when a stale-file response does not identify one file id', async () => { + const secondRef: ImageAttachmentRef = { + ...imageRef, + attachmentId: AttachmentId(`sha256:${'c'.repeat(64)}`), + } + const server = await mockServer([ + { + kind: 'http-error', + status: 400, + body: JSON.stringify({ error: { message: 'file reference expired' } }), + }, + { kind: 'sse', events: textEvents }, + ]) + const attachments = attachmentStoreOf(ref => Promise.resolve({ + ...requestImage(ref), + variantId: ImageVariantId(`sha256:${(ref.attachmentId === imageRef.attachmentId ? 'b' : 'd').repeat(64)}`), + })).store + const adapter = adapterOf({ + baseURL: server.url, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachments) + + await drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [ + { type: 'image', attachment: imageRef }, + { type: 'image', attachment: secondRef }, + ], + source: { kind: 'plugin', plugin: 'test' }, + })], + })) + + expect(server.fileRequests.filter(request => request.method === 'POST')).toHaveLength(4) + const retries = server.requests as Array<{ messages: Array<{ content: Array<{ type: string; file_id?: string }> }> }> + expect(retries[1]?.messages[0]?.content.filter(block => block.type === 'file')) + .toEqual([{ type: 'file', file_id: 'file-api-3' }, { type: 'file', file_id: 'file-api-4' }]) + }) + + it('returns the second stale-file rejection without a third chat attempt', async () => { + const stale = JSON.stringify({ error: { message: 'file_id file-api-1 expired' } }) + const server = await mockServer([ + { kind: 'http-error', status: 400, body: stale }, + { kind: 'http-error', status: 400, body: stale }, + ]) + const attachments = attachmentStoreOf(ref => Promise.resolve(requestImage(ref))).store + const adapter = adapterOf({ + baseURL: server.url, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachments) + + await expect(drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [{ type: 'image', attachment: imageRef }], + source: { kind: 'plugin', plugin: 'test' }, + })], + }))).rejects.toMatchObject({ code: 'INVALID_REQUEST', message: 'file_id file-api-1 expired' }) + expect(server.requests).toHaveLength(2) + expect(server.fileRequests.filter(request => request.method === 'POST')).toHaveLength(2) }) it.each(['deepseek-v4-flash', 'unlisted-pass-through'])( @@ -1057,17 +1309,17 @@ describe('plugin registration and config', () => { ) it.each([0, 1.5, Number.MAX_SAFE_INTEGER + 1])( - 'rejects invalid request image bound %s', - async (maxRequestImageBytes) => { - expect(() => resolveAdapterOptions({ maxRequestImageBytes })) - .toThrow(/maxRequestImageBytes must be a positive safe integer/) + 'rejects invalid request file bound %s', + async (maxRequestFilesBytes) => { + expect(() => resolveAdapterOptions({ maxRequestFilesBytes })) + .toThrow(/maxRequestFilesBytes must be a positive safe integer/) const ctx = new Context() await ctx.plugin(LlmRuntime) await expect(ctx.plugin(LlmDeepSeek, { baseURL: 'http://127.0.0.1:1', - maxRequestImageBytes, - })).rejects.toThrow(/maxRequestImageBytes/) + maxRequestFilesBytes, + })).rejects.toThrow(/maxRequestFilesBytes/) expect(ctx.llm.listProviders()).toEqual([]) }, ) diff --git a/packages/llm/llm-deepseek/tests/dynamic-config.spec.ts b/packages/llm/llm-deepseek/tests/dynamic-config.spec.ts index c048f68920..c0a2e29750 100644 --- a/packages/llm/llm-deepseek/tests/dynamic-config.spec.ts +++ b/packages/llm/llm-deepseek/tests/dynamic-config.spec.ts @@ -4,10 +4,12 @@ import { access, mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import LlmRuntime, { createUserMessage, INVALID_CREDENTIAL_CODE } from '@deepseek-ai/dsh-llm' -import AttachmentStore, { AttachmentId } from '@deepseek-ai/dsh-attachment' +import AttachmentStore, { AttachmentId, ImageVariantId } from '@deepseek-ai/dsh-attachment' import type { ImageAttachmentLimits, ImageAttachmentRef, + ImageRequestPolicy, + RequestImageAttachment, SavedImageAttachment, SaveImageAttachment, StoredImageAttachment, @@ -54,6 +56,25 @@ class StaticAttachmentStore extends AttachmentStore { readImage(ref: ImageAttachmentRef, _signal?: AbortSignal): Promise { return Promise.resolve({ ref, data: Uint8Array.of(1, 2, 3) }) } + + override readImageRequest( + ref: ImageAttachmentRef, + _policy: ImageRequestPolicy, + _signal?: AbortSignal, + ): Promise { + return Promise.resolve({ + variantId: ImageVariantId(`sha256:${'b'.repeat(64)}`), + master: ref, + data: Uint8Array.of(1, 2, 3), + mediaType: ref.mediaType, + bytes: 3, + width: ref.width, + height: ref.height, + depth: 'uchar', + space: 'srgb', + hasAlpha: true, + }) + } } const cleanups: Array<() => Promise> = [] @@ -168,7 +189,7 @@ describe('request-level dynamic configuration', () => { ]) }) - it('applies a changed request image bound to the next request', async () => { + it('applies changed request file limits to the next request', async () => { vi.stubEnv('DEEPSEEK_API_KEY', 'test-key') const dir = await home() const server = await mockServer([ @@ -185,14 +206,14 @@ describe('request-level dynamic configuration', () => { })] await assemble(ctx, { model: 'deepseek-v4-flash-vision-exp', messages }) - await ctx.settings.update(NS, { maxRequestImageBytes: 4 }) + await ctx.settings.update(NS, { maxRequestFilesBytes: 4, imageOffloadByteQuantum: 2 }) await assemble(ctx, { model: 'deepseek-v4-flash-vision-exp', messages }) const first = (server.requests[0] as { messages: Array<{ content: unknown }> }).messages[0]?.content const second = (server.requests[1] as { messages: Array<{ content: unknown }> }).messages[0]?.content - expect(JSON.stringify(first).match(/"type":"image_url"/g)).toHaveLength(2) + expect(JSON.stringify(first).match(/"type":"file"/g)).toHaveLength(2) expect(JSON.stringify(second)).toContain('[image omitted to keep the request within its image limit') - expect(JSON.stringify(second).match(/"type":"image_url"/g)).toHaveLength(1) + expect(JSON.stringify(second).match(/"type":"file"/g)).toHaveLength(1) }) it('re-registers the route in place when the captured retry policy changes, without an empty-registry window', async () => { diff --git a/packages/llm/llm-deepseek/tests/file-store.spec.ts b/packages/llm/llm-deepseek/tests/file-store.spec.ts new file mode 100644 index 0000000000..041fd0a0c6 --- /dev/null +++ b/packages/llm/llm-deepseek/tests/file-store.spec.ts @@ -0,0 +1,135 @@ +import { mkdtemp } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it, vi } from 'vitest' +import { AttachmentId, ImageVariantId } from '@deepseek-ai/dsh-attachment' +import type { ImageAttachmentRef, RequestImageAttachment } from '@deepseek-ai/dsh-attachment' +import { DeepSeekFileStore } from '../src/file-store.ts' +import { DeepSeekUploadIndex } from '../src/upload-index.ts' + +const REF: ImageAttachmentRef = { + attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`), + mediaType: 'image/png', + bytes: 3, + width: 1, + height: 1, +} +const VERSION: RequestImageAttachment = { + variantId: ImageVariantId(`sha256:${'b'.repeat(64)}`), + master: REF, + data: Uint8Array.of(1, 2, 3), + mediaType: 'image/png', + bytes: 3, + width: 1, + height: 1, + depth: 'uchar', + space: 'srgb', + hasAlpha: true, +} +const CONNECTION = { baseURL: 'https://api.deepseek.com', apiKey: 'key' } +const POLICY = { expiresAfterSeconds: 604_800, refreshMarginSeconds: 3_600, quotaCleanupBatch: 100 } +const NOW = 1_700_000_000_000 + +function requestUrl(input: string | URL | Request): string { + if (typeof input === 'string') return input + return input instanceof URL ? input.href : input.url +} + +function uploadFetch(now: () => number = () => NOW) { + let uploads = 0 + const fetchImpl = vi.fn(async (_url: string | URL | Request, init?: RequestInit) => { + if (init?.method === 'POST') { + uploads += 1 + const createdAt = now() / 1_000 + return new Response(JSON.stringify({ + id: `file-api-${uploads}`, + object: 'file', + bytes: 3, + created_at: createdAt, + filename: `dsh-${'a'.repeat(16)}-${'b'.repeat(8)}.png`, + purpose: 'user_data', + expires_at: createdAt + POLICY.expiresAfterSeconds, + }), { status: 200 }) + } + if (init?.method === 'DELETE') { + const id = requestUrl(_url).split('/').at(-1) + return new Response(JSON.stringify({ id, object: 'file', deleted: true }), { status: 200 }) + } + throw new Error('unexpected Files API request') + }) as typeof fetch + return { fetchImpl, uploads: () => uploads } +} + +describe('DeepSeekFileStore', () => { + it('singleflights the first upload and reuses the durable mapping across store instances', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + const index = new DeepSeekUploadIndex(join(dir, 'index.json')) + const remote = uploadFetch() + const first = new DeepSeekFileStore({ index, now: () => NOW, fetch: remote.fetchImpl }) + + const [a, b] = await Promise.all([ + first.ensureUploaded(VERSION, CONNECTION, POLICY), + first.ensureUploaded(VERSION, CONNECTION, POLICY), + ]) + expect(a.record.fileId).toBe('file-api-1') + expect(b.record.fileId).toBe('file-api-1') + expect(remote.uploads()).toBe(1) + + const resumed = new DeepSeekFileStore({ index, now: () => NOW, fetch: remote.fetchImpl }) + await expect(resumed.ensureUploaded(VERSION, CONNECTION, POLICY)) + .resolves.toMatchObject({ record: { fileId: 'file-api-1' }, uploaded: false }) + expect(remote.uploads()).toBe(1) + }) + + it('does not persist an upload whose response is missing and retries on the next request', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + const index = new DeepSeekUploadIndex(join(dir, 'index.json')) + const good = uploadFetch() + let first = true + const fetchImpl = vi.fn((url: string | URL | Request, init?: RequestInit) => { + if (first) { + first = false + return Promise.resolve(new Response('', { status: 204 })) + } + return good.fetchImpl(url, init) + }) as typeof fetch + const store = new DeepSeekFileStore({ index, now: () => NOW, fetch: fetchImpl }) + + await expect(store.ensureUploaded(VERSION, CONNECTION, POLICY)) + .rejects.toBeInstanceOf(Error) + await expect(store.ensureUploaded(VERSION, CONNECTION, POLICY)) + .resolves.toMatchObject({ record: { fileId: 'file-api-1' }, uploaded: true }) + }) + + it('reuses local expires_at above the refresh margin and uploads again at the margin', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + const index = new DeepSeekUploadIndex(join(dir, 'index.json')) + let now = NOW + const remote = uploadFetch(() => now) + const store = new DeepSeekFileStore({ index, now: () => now, fetch: remote.fetchImpl }) + + await expect(store.ensureUploaded(VERSION, CONNECTION, POLICY)) + .resolves.toMatchObject({ record: { fileId: 'file-api-1' }, uploaded: true }) + now = NOW + (POLICY.expiresAfterSeconds - POLICY.refreshMarginSeconds) * 1_000 - 1 + await expect(store.ensureUploaded(VERSION, CONNECTION, POLICY)) + .resolves.toMatchObject({ record: { fileId: 'file-api-1' }, uploaded: false }) + now += 1 + await expect(store.ensureUploaded(VERSION, CONNECTION, POLICY)) + .resolves.toMatchObject({ record: { fileId: 'file-api-2' }, uploaded: true }) + + expect(remote.uploads()).toBe(2) + expect(vi.mocked(remote.fetchImpl).mock.calls.every(([, init]) => init?.method === 'POST')).toBe(true) + }) + + it('releases an indexed file through DELETE and removes only that mapping', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + const index = new DeepSeekUploadIndex(join(dir, 'index.json')) + const remote = uploadFetch() + const store = new DeepSeekFileStore({ index, now: () => NOW, fetch: remote.fetchImpl }) + await store.ensureUploaded(VERSION, CONNECTION, POLICY) + + await expect(store.release(VERSION, CONNECTION, POLICY)).resolves.toBe(true) + await expect(store.release(VERSION, CONNECTION, POLICY)).resolves.toBe(false) + expect(remote.fetchImpl).toHaveBeenCalledTimes(2) + }) +}) diff --git a/packages/llm/llm-deepseek/tests/files-api.spec.ts b/packages/llm/llm-deepseek/tests/files-api.spec.ts new file mode 100644 index 0000000000..752c659a7f --- /dev/null +++ b/packages/llm/llm-deepseek/tests/files-api.spec.ts @@ -0,0 +1,102 @@ +import { describe, expect, it, vi } from 'vitest' +import { DeepSeekFileId } from '../src/file-id.ts' +import { DeepSeekFilesClient, isFilesQuotaError } from '../src/files-api.ts' + +function requestUrl(input: string | URL | Request): string { + if (typeof input === 'string') return input + return input instanceof URL ? input.href : input.url +} + +function file(overrides: Record = {}) { + return { + id: 'file-api-one', + object: 'file', + bytes: 3, + created_at: 1_700_000_000, + filename: 'image.png', + purpose: 'user_data', + expires_at: 1_700_604_800, + ...overrides, + } +} + +describe('DeepSeekFilesClient', () => { + it('uploads multipart bytes with the required purpose and explicit expiry', async () => { + const fetchImpl = vi.fn(async (url: string | URL | Request, init?: RequestInit) => { + expect(requestUrl(url)).toBe('https://api.deepseek.com/files') + expect(init?.method).toBe('POST') + expect(new Headers(init?.headers).get('authorization')).toBe('Bearer key') + const form = init?.body + expect(form).toBeInstanceOf(FormData) + if (!(form instanceof FormData)) throw new Error('expected multipart body') + expect(form.get('purpose')).toBe('user_data') + expect(form.get('expires_after[anchor]')).toBe('created_at') + expect(form.get('expires_after[seconds]')).toBe('604800') + const blob = form.get('file') + expect(blob).toBeInstanceOf(Blob) + expect((blob as Blob).size).toBe(3) + return new Response(JSON.stringify(file()), { status: 200 }) + }) as typeof fetch + const client = new DeepSeekFilesClient({ baseURL: 'https://api.deepseek.com/', apiKey: 'key', fetch: fetchImpl }) + + await expect(client.upload({ + data: Uint8Array.of(1, 2, 3), + mediaType: 'image/png', + filename: 'image.png', + expiresAfterSeconds: 604_800, + })).resolves.toEqual({ + id: DeepSeekFileId('file-api-one'), + bytes: 3, + createdAt: 1_700_000_000, + filename: 'image.png', + purpose: 'user_data', + expiresAt: 1_700_604_800, + }) + }) + + it('validates list, retrieve, and delete responses', async () => { + const fetchImpl = vi.fn(async (url: string | URL | Request, init?: RequestInit) => { + const target = requestUrl(url) + if (target.includes('?')) { + return new Response(JSON.stringify({ + object: 'list', data: [file()], first_id: 'file-api-one', last_id: 'file-api-one', has_more: false, + }), { status: 200 }) + } + if (init?.method === 'DELETE') { + return new Response(JSON.stringify({ id: 'file-api-one', object: 'file', deleted: true }), { status: 200 }) + } + return new Response(JSON.stringify(file()), { status: 200 }) + }) as typeof fetch + const client = new DeepSeekFilesClient({ baseURL: 'https://api.deepseek.com', apiKey: 'key', fetch: fetchImpl }) + + await expect(client.list({ limit: 20, order: 'desc' })).resolves.toMatchObject({ + data: [{ id: 'file-api-one' }], firstId: 'file-api-one', lastId: 'file-api-one', hasMore: false, + }) + await expect(client.retrieve(DeepSeekFileId('file-api-one'))).resolves.toMatchObject({ id: 'file-api-one' }) + await expect(client.delete(DeepSeekFileId('file-api-one'))).resolves.toBeUndefined() + }) + + it('refuses an upload response that omits the requested expiry', async () => { + const fetchImpl = vi.fn(() => Promise.resolve(new Response( + JSON.stringify(file({ expires_at: undefined })), + { status: 200 }, + ))) as typeof fetch + const client = new DeepSeekFilesClient({ baseURL: 'https://api.deepseek.com', apiKey: 'key', fetch: fetchImpl }) + + await expect(client.upload({ + data: Uint8Array.of(1), mediaType: 'image/png', filename: 'image.png', expiresAfterSeconds: 3_600, + })).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }) + }) + + it('retains quota error detail for the one cleanup retry policy', async () => { + const fetchImpl = vi.fn(() => Promise.resolve(new Response(JSON.stringify({ + error: { message: 'user storage quota exceeded', type: 'invalid_request_error', code: 'file_quota' }, + }), { status: 400 }))) as typeof fetch + const client = new DeepSeekFilesClient({ baseURL: 'https://api.deepseek.com', apiKey: 'key', fetch: fetchImpl }) + + const error = await client.upload({ + data: Uint8Array.of(1), mediaType: 'image/png', filename: 'image.png', expiresAfterSeconds: 3_600, + }).catch((caught: unknown) => caught) + expect(isFilesQuotaError(error)).toBe(true) + }) +}) diff --git a/packages/llm/llm-deepseek/tests/mock-server.ts b/packages/llm/llm-deepseek/tests/mock-server.ts index cdb499e143..819a945e93 100644 --- a/packages/llm/llm-deepseek/tests/mock-server.ts +++ b/packages/llm/llm-deepseek/tests/mock-server.ts @@ -13,6 +13,8 @@ export interface MockServer { requests: unknown[] /** Header bags of received requests, in order (parallel to `requests`). */ headers: IncomingMessage['headers'][] + /** Parsed Files API operations, excluded from chat request ordering. */ + fileRequests: Array<{ method: string; path: string; filename?: string; bytes?: number }> script: Behavior[] close(): Promise } @@ -36,36 +38,111 @@ export const textEvents = [ export async function mockServer(script: Behavior[]): Promise { const requests: unknown[] = [] const headers: IncomingMessage['headers'][] = [] + const fileRequests: MockServer['fileRequests'] = [] + const files = new Map() + let nextFile = 1 const server = createServer((request: IncomingMessage, response: ServerResponse) => { - let body = '' - request.on('data', (chunk: Buffer) => { body += chunk.toString('utf8') }) + const chunks: Buffer[] = [] + request.on('data', (chunk: Buffer) => { chunks.push(chunk) }) request.on('end', () => { - requests.push(JSON.parse(body)) - headers.push(request.headers) - const behavior = script.shift() - if (!behavior) { - response.writeHead(500).end('mock script exhausted') - return - } - if (behavior.kind === 'http-error') { - response.writeHead(behavior.status, { - 'content-type': behavior.contentType ?? 'application/json', - ...behavior.headers, - }) - response.end(behavior.body) - return - } - response.writeHead(200, { 'content-type': 'text/event-stream' }) - const write = (index: number): void => { - if (index >= behavior.events.length) { - if (behavior.kind === 'sse') response.end() - else response.destroy() // close-early: drop the socket mid-stream + void (async () => { + const url = new URL(request.url ?? '/', 'http://localhost') + const body = Buffer.concat(chunks) + if (url.pathname === '/files' && request.method === 'POST') { + const headers = new Headers() + for (const [name, value] of Object.entries(request.headers)) { + if (value !== undefined) headers.set(name, Array.isArray(value) ? value.join(', ') : value) + } + const form = await new Request('http://localhost/files', { + method: 'POST', + headers, + body, + }).formData() + const blob = form.get('file') + if (!(blob instanceof Blob)) throw new Error('mock upload omitted file') + const name = 'name' in blob && typeof blob.name === 'string' ? blob.name : 'uploaded_file' + const id = `file-api-${nextFile}` + const createdAt = Math.floor(Date.now() / 1_000) + nextFile += 1 + const expiresSeconds = Number(form.get('expires_after[seconds]')) + const file = { + id, + object: 'file' as const, + bytes: blob.size, + created_at: createdAt, + filename: name, + purpose: 'user_data' as const, + expires_at: createdAt + expiresSeconds, + } + files.set(id, file) + fileRequests.push({ method: 'POST', path: url.pathname, filename: name, bytes: blob.size }) + response.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify(file)) return } - response.write(`data: ${behavior.events[index]}\n\n`) - setTimeout(() => { write(index + 1) }, behavior.kind === 'sse' ? behavior.delayMs ?? 0 : 5) - } - write(0) + if (url.pathname === '/files' && request.method === 'GET') { + fileRequests.push({ method: 'GET', path: `${url.pathname}${url.search}` }) + const data = [...files.values()] + response.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ + object: 'list', + data, + first_id: data[0]?.id, + last_id: data.at(-1)?.id, + has_more: false, + })) + return + } + if (url.pathname.startsWith('/files/') && request.method === 'DELETE') { + const id = decodeURIComponent(url.pathname.slice('/files/'.length)) + files.delete(id) + fileRequests.push({ method: 'DELETE', path: url.pathname }) + response.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ + id, object: 'file', deleted: true, + })) + return + } + if (url.pathname.startsWith('/files/') && request.method === 'GET') { + const id = decodeURIComponent(url.pathname.slice('/files/'.length)) + fileRequests.push({ method: 'GET', path: url.pathname }) + const file = files.get(id) + if (file === undefined) { + response.writeHead(404, { 'content-type': 'application/json' }).end(JSON.stringify({ + error: { message: 'file not found', code: 'file_not_found' }, + })) + } else { + response.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify(file)) + } + return + } + + requests.push(JSON.parse(body.toString('utf8'))) + headers.push(request.headers) + const behavior = script.shift() + if (!behavior) { + response.writeHead(500).end('mock script exhausted') + return + } + if (behavior.kind === 'http-error') { + response.writeHead(behavior.status, { + 'content-type': behavior.contentType ?? 'application/json', + ...behavior.headers, + }) + response.end(behavior.body) + return + } + response.writeHead(200, { 'content-type': 'text/event-stream' }) + const write = (index: number): void => { + if (index >= behavior.events.length) { + if (behavior.kind === 'sse') response.end() + else response.destroy() // close-early: drop the socket mid-stream + return + } + response.write(`data: ${behavior.events[index]}\n\n`) + setTimeout(() => { write(index + 1) }, behavior.kind === 'sse' ? behavior.delayMs ?? 0 : 5) + } + write(0) + })().catch((error: unknown) => { + response.writeHead(500, { 'content-type': 'text/plain' }).end(String(error)) + }) }) }) servers.push(server) @@ -76,6 +153,7 @@ export async function mockServer(script: Behavior[]): Promise { url: `http://127.0.0.1:${address.port}`, requests, headers, + fileRequests, script, close: () => new Promise(resolve => server.close(() => { resolve() })), } diff --git a/packages/llm/llm-deepseek/tests/serialize.spec.ts b/packages/llm/llm-deepseek/tests/serialize.spec.ts index 742efc863d..04717da6e0 100644 --- a/packages/llm/llm-deepseek/tests/serialize.spec.ts +++ b/packages/llm/llm-deepseek/tests/serialize.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest' -import { AttachmentError, AttachmentId } from '@deepseek-ai/dsh-attachment' -import type { AttachmentStore, ImageAttachmentRef, ImageMediaType } from '@deepseek-ai/dsh-attachment' +import { AttachmentId, ImageVariantId } from '@deepseek-ai/dsh-attachment' +import type { ImageAttachmentRef, ImageMediaType, RequestImageAttachment } from '@deepseek-ai/dsh-attachment' import { createUserMessage, CallId, ReasoningEffortId, createMessage } from '@deepseek-ai/dsh-llm' import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm' import { @@ -9,14 +9,21 @@ import { serializeRequest, serializeRequestWithImages, } from '../src/serialize.ts' +import type { ImageSerializationOptions } from '../src/serialize.ts' function request(overrides: Partial = {}): GenerateOptions { return { provider: 'deepseek-official', model: 'deepseek-v4-flash', messages: [], ...overrides } } function imageRef(mediaType: ImageMediaType = 'image/png', bytes = 3): ImageAttachmentRef { + const digit = ({ + 'image/png': 'a', + 'image/jpeg': 'b', + 'image/webp': 'c', + 'image/gif': 'd', + } as const)[mediaType] return { - attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`), + attachmentId: AttachmentId(`sha256:${digit.repeat(64)}`), mediaType, bytes, width: 1, @@ -24,13 +31,36 @@ function imageRef(mediaType: ImageMediaType = 'image/png', bytes = 3): ImageAtta } } -function attachmentStore( - readImage = vi.fn((ref: ImageAttachmentRef, _signal?: AbortSignal) => Promise.resolve({ - ref, - data: Uint8Array.of(1, 2, 3), - })), -): AttachmentStore { - return { readImage } as unknown as AttachmentStore +function fileResolver(id = 'file-api-image') { + return vi.fn(() => Promise.resolve(id)) +} + +function requestVersion(ref: ImageAttachmentRef): RequestImageAttachment { + const hash = String(ref.attachmentId).slice('sha256:'.length) + return { + variantId: ImageVariantId(`sha256:${hash}`), + master: ref, + data: new Uint8Array(ref.bytes), + mediaType: ref.mediaType, + bytes: ref.bytes, + width: ref.width, + height: ref.height, + depth: 'uchar', + space: 'srgb', + hasAlpha: ref.mediaType === 'image/png', + } +} + +function imageOptions( + refs: readonly ImageAttachmentRef[], + resolveFileId: ImageSerializationOptions['resolveFileId'] = fileResolver(), + maxRequestFilesBytes = 20 * 1024 * 1024, +) { + return { + resolveFileId, + requestImages: new Map(refs.map(ref => [ref.attachmentId, requestVersion(ref)])), + maxRequestFilesBytes, + } } describe('serializeMessages', () => { @@ -304,53 +334,47 @@ describe('image serialization', () => { 'image/webp', 'image/gif', ] as const)('preserves ordered text and %s image parts', async (mediaType) => { - const signal = new AbortController().signal - const readImage = vi.fn((ref: ImageAttachmentRef, received?: AbortSignal) => { - expect(received).toBe(signal) - return Promise.resolve({ ref, data: Uint8Array.of(1, 2, 3) }) - }) + const resolveFileId = fileResolver() + const ref = imageRef(mediaType) const wire = await serializeRequestWithImages(request({ model: 'deepseek-v4-flash-vision-exp', messages: [createUserMessage({ content: [ { type: 'text', text: 'before' }, - { type: 'image', attachment: imageRef(mediaType) }, + { type: 'image', attachment: ref }, { type: 'text', text: 'after' }, ], source: { kind: 'plugin', plugin: 'test' }, })], - }), { - attachments: attachmentStore(readImage), - maxRequestImageBytes: 20 * 1024 * 1024, - signal, - }) + }), imageOptions([ref], resolveFileId)) expect(wire.messages).toEqual([{ role: 'user', content: [ { type: 'text', text: 'before' }, - { type: 'image_url', image_url: { url: `data:${mediaType};base64,AQID` } }, + { type: 'text', text: expect.stringContaining(`Image ${ref.attachmentId}; preview 1x1px`) as string }, + { type: 'file', file_id: 'file-api-image' }, { type: 'text', text: 'after' }, ], }]) }) - it('serializes image-only user content without synthetic text', async () => { + it('gives image-only input a stable handle and preview coordinate system', async () => { + const ref = imageRef() const wire = await serializeRequestWithImages(request({ model: 'deepseek-v4-flash-vision-exp', messages: [createUserMessage({ - content: [{ type: 'image', attachment: imageRef() }], + content: [{ type: 'image', attachment: ref }], source: { kind: 'plugin', plugin: 'test' }, })], - }), { - attachments: attachmentStore(), - maxRequestImageBytes: 20 * 1024 * 1024, - signal: new AbortController().signal, - }) + }), imageOptions([ref])) expect(wire.messages).toEqual([{ role: 'user', - content: [{ type: 'image_url', image_url: { url: 'data:image/png;base64,AQID' } }], + content: [ + { type: 'text', text: expect.stringContaining('Call read_image_region') as string }, + { type: 'file', file_id: 'file-api-image' }, + ], }]) }) @@ -377,19 +401,28 @@ describe('image serialization', () => { }), ] - await expect(serializeMessagesWithImages( - messages, - attachmentStore(), - new AbortController().signal, - )).resolves.toEqual([ - { role: 'tool', tool_call_id: 'first', content: '(see attached image)' }, - { role: 'tool', tool_call_id: 'second', content: 'caption' }, + const png = imageRef() + const jpeg = imageRef('image/jpeg') + await expect(serializeMessagesWithImages(messages, imageOptions( + [png, jpeg], + vi.fn((version: RequestImageAttachment) => Promise.resolve(`file-api-${version.mediaType}`)), + ))).resolves.toEqual([ + { + role: 'tool', + tool_call_id: 'first', + content: expect.stringContaining(`Image ${png.attachmentId}`) as string, + }, + { + role: 'tool', + tool_call_id: 'second', + content: expect.stringContaining(`caption\nImage ${jpeg.attachmentId}`) as string, + }, { role: 'user', content: [ { type: 'text', text: 'Attached image(s) from tool result:' }, - { type: 'image_url', image_url: { url: 'data:image/png;base64,AQID' } }, - { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,AQID' } }, + { type: 'file', file_id: 'file-api-image/png' }, + { type: 'file', file_id: 'file-api-image/jpeg' }, ], }, ]) @@ -409,11 +442,7 @@ describe('image serialization', () => { source: { kind: 'plugin', plugin: 'test' }, })] - await expect(serializeMessagesWithImages( - messages, - attachmentStore(), - new AbortController().signal, - )).resolves.toEqual([ + await expect(serializeMessagesWithImages(messages, imageOptions([], fileResolver()))).resolves.toEqual([ { role: 'tool', tool_call_id: 'result', content: 'ok' }, ]) }) @@ -435,11 +464,7 @@ describe('image serialization', () => { source: { kind: 'plugin', plugin: 'test' }, })] - await expect(serializeMessagesWithImages( - messages, - attachmentStore(), - new AbortController().signal, - )).resolves.toEqual([ + await expect(serializeMessagesWithImages(messages, imageOptions([], fileResolver()))).resolves.toEqual([ { role: 'tool', tool_call_id: 'nested', content: 'inside' }, { role: 'tool', tool_call_id: 'empty', content: '(no output)' }, ]) @@ -469,113 +494,106 @@ describe('image serialization', () => { }), ] - const wire = await serializeMessagesWithImages( - messages, - attachmentStore(), - new AbortController().signal, - ) + const wire = await serializeMessagesWithImages(messages, imageOptions([imageRef()], fileResolver())) expect(wire).toEqual([ - { role: 'tool', tool_call_id: 'before-system', content: '(see attached image)' }, + { + role: 'tool', + tool_call_id: 'before-system', + content: expect.stringContaining('Call read_image_region') as string, + }, expect.objectContaining({ role: 'user' }), { role: 'system', content: 'system history' }, - { role: 'tool', tool_call_id: 'before-assistant', content: '(see attached image)' }, + { + role: 'tool', + tool_call_id: 'before-assistant', + content: expect.stringContaining('Call read_image_region') as string, + }, expect.objectContaining({ role: 'user' }), { role: 'assistant', content: 'assistant history' }, ]) }) it('offloads oldest images before reads and keeps the newest image', async () => { - const readImage = vi.fn((ref: ImageAttachmentRef) => Promise.resolve({ - ref, - data: Uint8Array.of(1, 2, 3), - })) + const resolveFileId = fileResolver() + const png = imageRef('image/png', 3) + const jpeg = imageRef('image/jpeg', 3) const wire = await serializeRequestWithImages(request({ model: 'deepseek-v4-flash-vision-exp', messages: [createUserMessage({ content: [ - { type: 'image', attachment: imageRef('image/png', 3) }, - { type: 'image', attachment: imageRef('image/jpeg', 3) }, + { type: 'image', attachment: png }, + { type: 'image', attachment: jpeg }, ], source: { kind: 'plugin', plugin: 'test' }, })], - }), { - attachments: attachmentStore(readImage), - maxRequestImageBytes: 4, - signal: new AbortController().signal, - }) + }), imageOptions([png, jpeg], resolveFileId, 4)) expect(wire.messages[0]).toMatchObject({ role: 'user', content: [ { type: 'text', text: expect.stringContaining('older images are omitted first') as string }, - { type: 'image_url', image_url: { url: 'data:image/jpeg;base64,AQID' } }, + { type: 'text', text: expect.stringContaining(`Image ${jpeg.attachmentId}`) as string }, + { type: 'file', file_id: 'file-api-image' }, ], }) - expect(readImage).toHaveBeenCalledTimes(1) - expect(readImage.mock.calls[0]?.[0]).toMatchObject({ mediaType: 'image/jpeg' }) + expect(resolveFileId).toHaveBeenCalledTimes(1) + expect(resolveFileId.mock.calls[0]?.[0]).toMatchObject({ master: { mediaType: 'image/jpeg' } }) }) it.each(['system', 'assistant'] as const)('rejects an image in %s history before reading attachments', async (role) => { - const readImage = vi.fn() + const resolveFileId = vi.fn() await expect(serializeMessagesWithImages([createMessage({ role, content: [{ type: 'image', attachment: imageRef() }], source: { kind: 'plugin', plugin: 'test' }, - })], attachmentStore(readImage), new AbortController().signal)) + })], imageOptions([imageRef()], resolveFileId))) .rejects.toMatchObject({ code: 'UNSUPPORTED_CONTENT' }) - expect(readImage).not.toHaveBeenCalled() + expect(resolveFileId).not.toHaveBeenCalled() }) it('rejects unsupported image history before request offloading can replace it', async () => { - const readImage = vi.fn() + const resolveFileId = vi.fn() await expect(serializeRequestWithImages(request({ messages: [createMessage({ role: 'system', content: [{ type: 'image', attachment: imageRef('image/png', 300) }], source: { kind: 'plugin', plugin: 'test' }, })], - }), { - attachments: attachmentStore(readImage), - maxRequestImageBytes: 1, - signal: new AbortController().signal, - })).rejects.toMatchObject({ code: 'UNSUPPORTED_CONTENT' }) - expect(readImage).not.toHaveBeenCalled() + }), imageOptions([imageRef('image/png', 300)], resolveFileId, 1))) + .rejects.toMatchObject({ code: 'UNSUPPORTED_CONTENT' }) + expect(resolveFileId).not.toHaveBeenCalled() }) it('prepends the request system prompt on the image path', async () => { + const ref = imageRef() const wire = await serializeRequestWithImages(request({ system: 'system prompt', messages: [createUserMessage({ - content: [{ type: 'image', attachment: imageRef() }], + content: [{ type: 'image', attachment: ref }], source: { kind: 'plugin', plugin: 'test' }, })], - }), { - attachments: attachmentStore(), - maxRequestImageBytes: 20 * 1024 * 1024, - signal: new AbortController().signal, - }) + }), imageOptions([ref])) expect(wire.messages[0]).toEqual({ role: 'system', content: 'system prompt' }) }) - it('preserves stable attachment failure codes', async () => { - const readImage = vi.fn(() => Promise.reject(new AttachmentError( - 'Stored attachment bytes are corrupt.', - 'ATTACHMENT_CORRUPT', - ))) + it('preserves stable file-resolution failure codes', async () => { + const failure = new Error('Stored attachment bytes are corrupt.') as Error & { code: string } + failure.code = 'ATTACHMENT_CORRUPT' + const resolveFileId = vi.fn(() => Promise.reject(failure)) await expect(serializeMessagesWithImages([createUserMessage({ content: [{ type: 'image', attachment: imageRef() }], source: { kind: 'plugin', plugin: 'test' }, - })], attachmentStore(readImage), new AbortController().signal)) + })], imageOptions([imageRef()], resolveFileId))) .rejects.toMatchObject({ code: 'ATTACHMENT_CORRUPT' }) }) it('preserves non-attachment resolver failures', async () => { const failure = new Error('resolver failed') - const readImage = vi.fn(() => Promise.reject(failure)) + const resolveFileId = vi.fn(() => Promise.reject(failure)) await expect(serializeMessagesWithImages([createUserMessage({ content: [{ type: 'image', attachment: imageRef() }], source: { kind: 'plugin', plugin: 'test' }, - })], attachmentStore(readImage), new AbortController().signal)).rejects.toBe(failure) + })], imageOptions([imageRef()], resolveFileId))).rejects.toBe(failure) }) }) diff --git a/packages/llm/llm-deepseek/tests/upload-index.spec.ts b/packages/llm/llm-deepseek/tests/upload-index.spec.ts new file mode 100644 index 0000000000..6157adc06e --- /dev/null +++ b/packages/llm/llm-deepseek/tests/upload-index.spec.ts @@ -0,0 +1,73 @@ +import { mkdtemp, readFile, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { AttachmentId, ImageVariantId } from '@deepseek-ai/dsh-attachment' +import { DeepSeekFileId } from '../src/file-id.ts' +import { deepSeekFileScope, DeepSeekUploadIndex } from '../src/upload-index.ts' + +const ATTACHMENT = AttachmentId(`sha256:${'a'.repeat(64)}`) +const VARIANT = ImageVariantId(`sha256:${'b'.repeat(64)}`) + +describe('DeepSeekUploadIndex', () => { + it('isolates API-key namespaces and reuses only records above the refresh margin', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-upload-index-')) + const index = new DeepSeekUploadIndex(join(dir, 'index.json')) + const first = deepSeekFileScope('https://api.deepseek.com', 'first-key') + const second = deepSeekFileScope('https://api.deepseek.com', 'second-key') + const record = { + scope: first, + masterAttachmentId: ATTACHMENT, + variantId: VARIANT, + fileId: DeepSeekFileId('file-api-one'), + bytes: 3, + createdAt: 1_000, + expiresAt: 10_000, + } + + await expect(index.commit(record, 1_000, 1_000)).resolves.toMatchObject({ accepted: true }) + await expect(index.get(first, VARIANT, 1_000, 1_000)).resolves.toEqual(record) + await expect(index.get(second, VARIANT, 1_000, 1_000)).resolves.toBeUndefined() + await expect(index.get(first, VARIANT, 9_000, 1_000)).resolves.toBeUndefined() + }) + + it('keeps a reusable cross-process winner and removes only an exact generation', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-upload-index-')) + const index = new DeepSeekUploadIndex(join(dir, 'index.json')) + const scope = deepSeekFileScope('https://api.deepseek.com', 'key') + const first = { + scope, masterAttachmentId: ATTACHMENT, variantId: VARIANT, + fileId: DeepSeekFileId('file-api-first'), bytes: 3, createdAt: 1, expiresAt: 10_000, + } + const duplicate = { ...first, fileId: DeepSeekFileId('file-api-duplicate') } + await index.commit(first, 1, 1) + + await expect(index.commit(duplicate, 2, 1)).resolves.toEqual({ record: first, accepted: false }) + await index.remove(scope, VARIANT, duplicate.fileId) + await expect(index.get(scope, VARIANT, 2, 1)).resolves.toEqual(first) + await index.remove(scope, VARIANT, first.fileId) + await expect(index.get(scope, VARIANT, 2, 1)).resolves.toBeUndefined() + }) + + it('treats a corrupt upload cache as empty and repairs it on the next commit', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-upload-index-')) + const path = join(dir, 'index.json') + await writeFile(path, '{bad', 'utf8') + const index = new DeepSeekUploadIndex(path) + const scope = deepSeekFileScope('https://api.deepseek.com', 'key') + const record = { + scope, + masterAttachmentId: ATTACHMENT, + variantId: VARIANT, + fileId: DeepSeekFileId('file-api-repaired'), + bytes: 3, + createdAt: 1, + expiresAt: 10_000, + } + + await expect(index.get(scope, VARIANT, 1, 1)).resolves.toBeUndefined() + await expect(index.commit(record, 1, 1)).resolves.toEqual({ record, accepted: true }) + await expect(index.get(scope, VARIANT, 1, 1)).resolves.toEqual(record) + expect(JSON.parse(await readFile(path, 'utf8'))).toMatchObject({ formatVersion: 2 }) + }) +}) diff --git a/packages/llm/llm-deepseek/tsconfig.json b/packages/llm/llm-deepseek/tsconfig.json index 0a1751aab1..0ba1a2116c 100644 --- a/packages/llm/llm-deepseek/tsconfig.json +++ b/packages/llm/llm-deepseek/tsconfig.json @@ -20,6 +20,18 @@ { "path": "../../llm/llm" }, + { + "path": "../../attachment/attachment" + }, + { + "path": "../../util/atomic-write" + }, + { + "path": "../../util/brand" + }, + { + "path": "../../util/home-paths" + }, { "path": "../../credentials/credentials" }, diff --git a/packages/llm/llm-pi-ai/README.i18n.yaml b/packages/llm/llm-pi-ai/README.i18n.yaml index 034382822f..b951aad76e 100644 --- a/packages/llm/llm-pi-ai/README.i18n.yaml +++ b/packages/llm/llm-pi-ai/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/llm/llm-pi-ai/README.md -README.md: 19dbcfa90dbefbe322800c83da6d75c70c849f05 -README.zh.md: 334c6c3166f35f5dc7d7659b9de7cff404c6b56d +README.md: 044038aa69535ad90c9dc59ad63f05ab68560d28 +README.zh.md: d4b5dff10ea0f3668038cc4d3a6876f52ae273cb diff --git a/packages/llm/llm-pi-ai/README.md b/packages/llm/llm-pi-ai/README.md index 19dbcfa90d..044038aa69 100644 --- a/packages/llm/llm-pi-ai/README.md +++ b/packages/llm/llm-pi-ai/README.md @@ -20,6 +20,9 @@ Configure credentials, the model catalog, and deployment-specific transport sett apiKeyEnv: OPENAI_API_KEY baseURL: https://proxy.example.com:8443 reasoning: high + requestImagePixelBudget: 4194304 # total pixels; 2048 by 2048 default + requestImageMaxBytes: 1048576 # raw bytes before base64 expansion + maxRequestImageBytes: 20971520 # accumulated base64 payload retryPolicy: mode: normal maxRetries: 3 @@ -120,7 +123,7 @@ A model that carries reasoning metadata — from the installed catalog or from i A model **without** that metadata — a hand-declared one whose entry declares no `reasoningEfforts`, and a catalog model pi-ai marks as non-reasoning — exposes no `reasoning` at all. pi-ai reports such a model as supporting the single level `off`, but `off` is translated to *omitting* the reasoning option, which is byte-for-byte the request that naming no effort already produces: selecting it could not disable anything, so a provider whose own default is to think would keep thinking with `off` shown as selected. Reporting the capability as unavailable leaves a surface offering the provider's default and nothing that misrepresents it. The profile `reasoning` value, including `off`, is the deployment default when configured; omitting it preserves the provider default. Per-request `GenerateOptions.reasoningEffort` takes precedence, and a level absent from the exact model capability fails the REQUEST with `UNSUPPORTED_REASONING_EFFORT` before network I/O instead of being clamped. Describing a model never fails that way: the models under one provider disagree about which levels they accept, so `resolveModel` reports a profile level the exact model cannot take as no default at all rather than throwing. A throw there would take the whole provider out of every model catalog built over it — one mis-set profile field hiding even the models that do support the level — so a bad configuration surfaces where it is acted on, not where it is described. pi-ai's common stream options represent `off` by omitting `reasoning`. -Supported profile fields are `apiKeyEnv`, `displayName`, `api`, `baseURL`, `models`, `modelOverrides`, `compat`, `defaultContextWindow`, `defaultMaxTokens`, `defaultInput`, `headers`, `reasoning`, `thinkingBudgets`, `cacheRetention`, `transport`, `timeoutMs`, `websocketConnectTimeoutMs`, `streamIdleTimeoutMs`, `maxRequestImageBytes`, and `retryPolicy`. Each resolved profile retry policy is captured with that provider route; omission uses the shared bounded normal default of five retries. The stream-idle interval is a positive finite Node timer delay, defaults to five minutes, and covers only an outstanding provider read, not consumer think time. `maxRequestImageBytes` bounds one request's base64-encoded image payload (default 20MiB, a positive integer): every image in history is re-encoded into every request, so when the accumulated payload exceeds the bound, the oldest images are replaced by a fixed text placeholder until the request fits, keeping an image-heavy session serviceable instead of permanently rejected by a gateway request-size cap. The default leaves capacity for system prompts, history, tools, and JSON; deployments behind stricter gateways lower it per route. Harness app attribution wins a conflicting configured header name. +Supported profile fields are `apiKeyEnv`, `displayName`, `api`, `baseURL`, `models`, `modelOverrides`, `compat`, `defaultContextWindow`, `defaultMaxTokens`, `defaultInput`, `headers`, `reasoning`, `thinkingBudgets`, `cacheRetention`, `transport`, `timeoutMs`, `websocketConnectTimeoutMs`, `streamIdleTimeoutMs`, `maxRequestImageBytes`, `requestImagePixelBudget`, `requestImageMaxBytes`, and `retryPolicy`. Each resolved profile retry policy is captured with that provider route; omission uses the shared bounded normal default of five retries. The stream-idle interval is a positive finite Node timer delay, defaults to five minutes, and covers only an outstanding provider read, not consumer think time. Every image route first derives a deterministic request version from the provider-independent master under `requestImagePixelBudget` (default 2048 by 2048 total pixels) and `requestImageMaxBytes` (default 1MiB raw bytes). The same version feeds inline base64, and its stable descriptor exposes the attachment id and actual preview dimensions. `maxRequestImageBytes` then bounds the accumulated base64 length (default 20MiB): the oldest request versions are replaced by a fixed text placeholder until the request fits. Harness app attribution wins a conflicting configured header name. The adapter forces pi-ai's SDK `maxRetries` to zero so one `stream()` call makes one provider request. The removed profile fields `maxRetries` and `maxRetryDelayMs` fail load instead of silently multiplying or hiding the separately composed agent-level retry budget. Idle expiry aborts the SDK's stable request signal and surfaces `TIMEOUT`; an earlier caller abort remains `ABORTED`. @@ -170,11 +173,11 @@ pi-ai installs several provider SDKs and lazy-loads the one selected by the cata #### What the model sees -The selected catalog model receives `GenerateOptions.system`, history, tools, and sampling fields supported by pi-ai's common streaming API. This package adds no prompt prose, with one exception: when a request's accumulated base64 image payload exceeds the route's `maxRequestImageBytes`, each offloaded image (oldest first) is replaced by fixed text. The text tells the model to read the file again when a path is available or ask the user to attach the image again. Provider-native replay metadata is restored only when the adapter validates it for the historical content. +The selected catalog model receives `GenerateOptions.system`, history, tools, and sampling fields supported by pi-ai's common streaming API. Each retained image is preceded by stable text naming its complete attachment id, actual request dimensions, and `read_image_region` preview coordinates. When accumulated base64 image payload exceeds the route's `maxRequestImageBytes`, each offloaded image (oldest first) is replaced by fixed text that tells the model to read the file again when a path is available or ask the user to attach it again. Provider-native replay metadata is restored only when the adapter validates it for the historical content. #### Token effect -Provider tokenization governs exact input. Conversion adds no model-visible text beyond the image-offload placeholder, which replaces the offloaded image's visual tokens with a short fixed sentence; replay metadata may let a native API reuse provider-side state. +Provider tokenization governs exact input. Retained images add the stable attachment and coordinate descriptor; the offload placeholder replaces an omitted image's visual tokens. Replay metadata may let a native API reuse provider-side state. #### KV Cache effect @@ -196,7 +199,7 @@ Recorded response content appends to the next request and does not invalidate it ## Known Limitations and Deferred Work -- **`maxRequestImageBytes` counts base64 image payload only** — text, tools, and JSON structure ride outside the bound, so it must sit below the gateway's request-body cap with headroom. Offload is decided at request conversion as a pure function of history and configuration and is not recorded as a session event; per-route capability metadata (image count, per-image size, total request size) driving admission and assembly together is deferred design work. +- **`maxRequestImageBytes` counts base64 image payload only** — text, tools, descriptors, and JSON structure ride outside the bound, so it must sit below the gateway's request-body cap with headroom. Offload is a deterministic request projection and is not recorded as a session event. - **A sign-in lives only in the process that started it** — an authorization attempt is not durable, so reloading the page mid-login abandons it and the human starts over. Signing out is `deleteRecord` on the stored record, which forgets it locally without telling the issuer. - **Provider-native discovery answers through this plugin's ambient context** — a route naming no credential defers to the catalog provider's own resolution, which asks for environment values (`AZURE_OPENAI_API_KEY`, `AWS_PROFILE`, and each provider's own set) and for local credential files. Both questions are answered here: the credential seam is consulted before the process environment, and file existence is checked against the host process's filesystem with `~` expanded. What it cannot do is *read* a credential file's contents — a provider that parses `~/.aws/credentials` itself does so directly, outside the seam. - **Settings can add or override routes, not remove composition routes** — the user layer merges over the composition `base`, so deleting a `cordis.yml`-provided provider is a composition change; `replace` on the namespace only resets the user layer. @@ -204,7 +207,7 @@ Recorded response content appends to the next request and does not invalidate it - **`headers` can carry a credential the redactor never sees** — the profile's `headers` dict is plain strings, so `Authorization` or `api-key` set there is returned verbatim by a redacted `describe()` and rendered by any configuration UI. Store credentials as `apiKeyEnv` references; making the dict write-only is deferred with the rest of the [wire-boundary work](../llm/README.md#known-limitations-and-deferred-work). - **A route's catalog never refreshes itself** — the catalog is whatever `settings.yaml` says, so a model list is only as current as its last edit. Nothing here queries a provider for the models it serves; a route gains a model when someone writes one. - **One wire protocol per route** — `api` applies to the whole route, so a mixed-protocol catalog route (an OpenAI-style catalog spanning Responses and Chat Completions) cannot host a model of the other protocol, and adding a model such a route does not describe requires naming `api` and moving every model onto it. Splitting the provider across two route keys is the workaround. -- **A modality declaration is not verified, and over-claiming outlives the turn** — nothing interrogates an endpoint for what it accepts, so a model declaring `image` its gateway does not serve is refused by the provider mid-turn rather than here. Prompt admission commits the user message durably before the request is built, so the rejected image stays in the session log: that model keeps re-sending it, and model selection refuses a switch to any text-only model. Recovery is another image-capable model, a fork before the image, or a new session; rolling an unconsumed image message back out of the log on a failed send is deferred. +- **A modality declaration is not verified** — nothing interrogates an endpoint for what it accepts, so a model declaring `image` its gateway does not serve is refused by the provider after prompt admission. The durable image remains in history and the same misdeclared model can fail again. Switching to a text-only model remains possible because the shared LLM runtime projects image references into stable text for that exact request. - **An unauthenticated route depends on its protocol** — naming no credential resolves the route as configured-but-keyless, but pi-ai's OpenAI-compatible implementation still requires an API key or an `Authorization` header, so a keyless local server needs a placeholder credential referenced by `apiKeyEnv` or an `Authorization` entry in `headers`. - **`GenerateOptions.stop` is unsupported** — pi-ai's common stream options cannot guarantee stop-sequence behavior across providers, so the adapter rejects the field. - **In-history `system` messages use pi-ai's common context conversion** — provider-specific placement follows pi-ai rather than a harness-owned wire override. diff --git a/packages/llm/llm-pi-ai/README.zh.md b/packages/llm/llm-pi-ai/README.zh.md index 334c6c3166..d4b5dff10e 100644 --- a/packages/llm/llm-pi-ai/README.zh.md +++ b/packages/llm/llm-pi-ai/README.zh.md @@ -20,6 +20,9 @@ apiKeyEnv: OPENAI_API_KEY baseURL: https://proxy.example.com:8443 reasoning: high + requestImagePixelBudget: 4194304 # total pixels; 2048 by 2048 default + requestImageMaxBytes: 1048576 # raw bytes before base64 expansion + maxRequestImageBytes: 20971520 # accumulated base64 payload retryPolicy: mode: normal maxRetries: 3 @@ -121,7 +124,7 @@ pi-ai 依据提供方 id 与 baseURL 决定每个请求的形状:系统提示 **没有**这份元数据的模型——条目未声明 `reasoningEfforts` 的手工声明模型,以及 pi-ai 标记为不具备推理能力的 catalog 模型——完全不公开 `reasoning`。pi-ai 会把这类模型报告为只支持 `off` 一档,但 `off` 会被翻译成*省略* reasoning 选项,而那与「不点名任何档位」产出的请求逐字节相同:选它关不掉任何东西,于是自身默认就在思考的提供方,会在界面显示 `off` 被选中的同时继续思考。把该能力报告为不可用,界面就只剩提供方默认这一项,不会再出现自相矛盾的控件。配置 profile 的 `reasoning` 值(包括 `off`)在存在时是部署默认值;省略它会保留提供方默认值。每次请求的 `GenerateOptions.reasoningEffort` 优先;未出现在确切模型能力中的档位会让**请求**在网络 I/O 前以 `UNSUPPORTED_REASONING_EFFORT` 失败,而不会被自动调整。**描述**一个模型则从不这样失败:同一提供方下各模型接受的档位并不一致,因此 `resolveModel` 对该模型拿不下的 profile 档位报告为「没有默认值」,而不是抛错。在那里抛错会让整个提供方从任何基于它构建的模型目录中消失——一个配错的 profile 字段连支持该档位的模型也一并藏起来——所以坏配置暴露在被执行处,而不是被描述处。pi-ai 的通用流选项通过省略 `reasoning` 表示 `off`。 -受支持的 profile 字段是 `apiKeyEnv`、`displayName`、`api`、`baseURL`、`models`、`modelOverrides`、`compat`、`defaultContextWindow`、`defaultMaxTokens`、`defaultInput`、`headers`、`reasoning`、`thinkingBudgets`、`cacheRetention`、`transport`、`timeoutMs`、`websocketConnectTimeoutMs`、`streamIdleTimeoutMs`、`maxRequestImageBytes` 和 `retryPolicy`。每条 profile 解析后的重试策略会随该提供方路由一同捕获;省略时使用共享的有界 normal 默认值并重试五次。流空闲间隔必须是正的有限 Node 定时器延迟,默认为五分钟,且只覆盖未完成提供方读取,不包括消费方思考时间。`maxRequestImageBytes` 约束单个请求的 base64 编码图片载荷(默认 20MiB,正整数):历史中的每张图片都会重新编码进每个请求,累积载荷超过上限时,从最老的图片开始替换为固定文本占位,直到请求装得下,使图片较多的会话保持可用,而不是被网关请求体上限永久拒绝。默认值为系统提示词、历史、工具与 JSON 保留请求容量;网关更严格的部署按路由调低该值。若已配置标头中有同名项,则以 Harness 应用归因为准。 +受支持的 profile 字段是 `apiKeyEnv`、`displayName`、`api`、`baseURL`、`models`、`modelOverrides`、`compat`、`defaultContextWindow`、`defaultMaxTokens`、`defaultInput`、`headers`、`reasoning`、`thinkingBudgets`、`cacheRetention`、`transport`、`timeoutMs`、`websocketConnectTimeoutMs`、`streamIdleTimeoutMs`、`maxRequestImageBytes`、`requestImagePixelBudget`、`requestImageMaxBytes` 和 `retryPolicy`。每条 profile 解析后的重试策略会随该提供方路由一同捕获;省略时使用共享的有界 normal 默认值并重试五次。流空闲间隔必须是正的有限 Node 定时器延迟,默认为五分钟,且只覆盖未完成提供方读取,不包括消费方思考时间。每条图片路由先从提供方无关的主版本派生确定性请求版本,受 `requestImagePixelBudget`(默认总像素 2048×2048)和 `requestImageMaxBytes`(默认原始字节 1MiB)约束。同一版本用于内联 base64,其稳定描述会公开附件 ID 和实际预览尺寸。`maxRequestImageBytes` 再限制累计 base64 长度(默认 20MiB);超出时从最旧请求版本开始替换为固定文本占位,直到请求可容纳。若已配置标头中有同名项,则以 Harness 应用归因为准。 适配器强制 pi-ai SDK `maxRetries` 为零,因此一次 `stream()` 调用只会发起一次提供方请求。已移除 profile 字段 `maxRetries` 和 `maxRetryDelayMs` 会使加载失败,而不是静默倍增或隐藏单独组合的 agent(智能体)级重试预算。空闲超时会 abort SDK 的稳定请求信号,并以 `TIMEOUT` 呈现;较早的调用方 abort 仍为 `ABORTED`。 @@ -171,11 +174,11 @@ pi-ai 会安装多个提供方 SDK,并延迟加载 catalog 模型所选的 SDK #### 模型看到的内容 -所选 catalog 模型会收到 `GenerateOptions.system`、历史、工具,以及 pi-ai 通用流式 API 支持的采样字段。本包不添加提示词文本,仅有一个例外:请求累积的 base64 图片载荷超过路由的 `maxRequestImageBytes` 时,被 offload 的图片(从最老开始)会被替换为一段固定文本。该文本要求模型在有路径时重新读取文件,否则请用户重新附上图片。只有当适配器验证提供方原生回放元数据与历史内容匹配时,才会恢复这些元数据。 +所选 catalog 模型会收到 `GenerateOptions.system`、历史、工具,以及 pi-ai 通用流式 API 支持的采样字段。每张保留图片前都有稳定文本,写明完整附件 ID、实际请求尺寸和 `read_image_region` 使用的预览坐标。请求累积的 base64 图片载荷超过路由的 `maxRequestImageBytes` 时,被 offload 的图片会从最老开始替换为固定文本,要求模型在有路径时重新读取文件,否则请用户重新附上图片。只有当适配器验证提供方原生回放元数据与历史内容匹配时,才会恢复这些元数据。 #### Token 影响 -精确输入取决于提供方 tokenization。除图片 offload 占位文本外,转换不添加模型可见文本;占位文本用一句固定短句替代被省略图片的视觉 token。回放元数据可能让原生 API 复用提供方侧状态。 +精确输入取决于提供方 tokenization。保留图片会增加稳定的附件与坐标描述;offload 占位文本替代被省略图片的视觉 token。回放元数据可能让原生 API 复用提供方侧状态。 #### KV Cache 影响 @@ -197,7 +200,7 @@ pi-ai 事件会变为 harness 推理、文本、工具调用、usage 与 finish ## 已知限制与暂缓事项 -- **`maxRequestImageBytes` 只统计 base64 图片载荷**:文本、工具与 JSON 结构不计入上限,因此该值必须低于网关请求体上限并留出余量。offload 在请求转换时决定,是历史与配置的纯函数,不记录为会话事件;由按路由能力元数据(图片数量、单图大小、请求总大小)同时驱动准入与组装的完整设计属于暂缓工作。 +- **`maxRequestImageBytes` 只统计 base64 图片载荷**:文本、工具、图片描述和 JSON 结构不计入上限,因此该值必须低于网关请求体上限并留出余量。offload 是确定性请求投影,不记录为会话事件。 - **一次登录只存活于发起它的进程中**:授权尝试不可持久,登录途中刷新页面会丢弃它,人需要重来。登出即对已存储记录执行 `deleteRecord`,它只在本地遗忘而不通知签发方。 - **提供方自带的凭据发现经由本插件的 ambient context 作答**:不指定凭据的路由交由 catalog 提供方自行解析,它会询问环境值(`AZURE_OPENAI_API_KEY`、`AWS_PROFILE` 以及各提供方自己的那一组)与本地凭据文件是否存在。两类问题都在这里作答:先查凭据 seam 再查进程环境,文件存在性则按宿主进程的文件系统判断并展开 `~`。它做不到的是*读取*凭据文件的内容——自行解析 `~/.aws/credentials` 的提供方是直接读盘的,不经过 seam。 - **settings 能新增或覆盖路由,但不能移除组合路由**:用户层合并在组合 `base` 之上,因此删除 `cordis.yml` 提供的提供方属于组合变更;对该 namespace 执行 `replace` 只会重置用户层。 @@ -205,7 +208,7 @@ pi-ai 事件会变为 harness 推理、文本、工具调用、usage 与 finish - **`headers` 可能承载一条脱敏器看不见的凭据**:profile 的 `headers` 是纯字符串字典,因此设在其中的 `Authorization` 或 `api-key` 会被脱敏后的 `describe()` 原样返回,并被任何配置 UI 渲染出来。请把凭据存为 `apiKeyEnv` 引用;把该字典整体改为只写与其余[协议边界工作](../llm/README.zh.md#known-limitations-and-deferred-work)一并暂缓。 - **路由的 catalog 不会自我刷新**:catalog 就是 `settings.yaml` 所写的内容,因此模型列表的新鲜度只到最近一次编辑为止。这里没有任何环节会去问提供方它服务哪些模型;路由要多一个模型,得有人写进去。 - **每条路由只有一种协议格式**:`api` 作用于整条路由,因此混合协议的 catalog 路由(跨 Responses 与 Chat Completions 的 OpenAI 式 catalog)无法承载另一种协议的模型,向这类路由添加它未描述的模型必须点名 `api` 并把全部模型一起迁过去。把该提供方拆成两个路由键是变通办法。 -- **模态声明不经验证,且多声明的后果超出本轮**:没有任何环节会去询问端点接受什么,因此声明了网关并不提供的 `image` 的模型不会在这里被拦下,而是由提供方在轮次中途拒绝。prompt 准入在构造请求之前就把用户消息持久化提交,于是被拒绝的图片留在会话日志里:该模型会不断重发它,而模型选择拒绝切换到任何纯文本模型。恢复途径是换一个确实支持图片的模型、fork 到图片之前,或开启新会话;发送失败时把尚未消费的图片消息从日志中回滚出去这件事已暂缓。 +- **模态声明不经验证**:没有任何环节会去询问端点接受什么,因此声明了网关并不提供的 `image` 的模型会在 prompt 准入后被提供方拒绝。持久图片会留在历史中,同一个错误声明的模型可能再次失败。系统仍允许切换到纯文本模型,因为共享 LLM 运行时会在该次请求中把图片引用投影为稳定文本。 - **未认证路由取决于其协议**:不点名凭据会让路由解析为「已配置但无密钥」,但 pi-ai 的 OpenAI 兼容实现仍要求 API key 或 `Authorization` 标头,因此无鉴权的本地服务需要一个由 `apiKeyEnv` 引用的占位凭据,或在 `headers` 中给出 `Authorization` 条目。 - **不支持 `GenerateOptions.stop`**:pi-ai 的通用流选项无法保证所有提供方都支持 stop sequence,因此适配器会拒绝该字段。 - **历史中的 `system` 消息使用 pi-ai 通用上下文转换**:提供方特定位置由 pi-ai 决定,而非由 harness 拥有的协议覆盖决定。 diff --git a/packages/llm/llm-pi-ai/src/adapter.ts b/packages/llm/llm-pi-ai/src/adapter.ts index 3c7ecd4a91..c5af6bff6a 100644 --- a/packages/llm/llm-pi-ai/src/adapter.ts +++ b/packages/llm/llm-pi-ai/src/adapter.ts @@ -50,6 +50,7 @@ import type { LlmModelInfo, LlmProviderInfo, LlmResolvedModelInfo, + PreparedAdapterCall, ReasoningEffortId as ReasoningEffortIdType, ResolvedRetryPolicy, StreamChunk, @@ -284,25 +285,44 @@ export class PiAiAdapter extends LlmAdapter { ): Promise { return Promise.resolve().then(() => { const snapshot = this.current() - const profile = this.profileOf(snapshot, provider) - const resolvedModel = this.modelOf(snapshot, provider, model) - const defaultLevel = describableReasoningLevel(resolvedModel, profile.reasoning) - // Only a cap the deployment configured is a request default; the - // catalog's `maxTokens` sizes the model and stops there. - const configuredMaxTokens = profile.configuredMaxTokens.get(model) - return { - provider, - id: model, - name: resolvedModel.name, - inputModalities: [...resolvedModel.input], - context: { contextWindow: resolvedModel.contextWindow }, - ...configuredMaxTokens === undefined ? {} : { defaultMaxTokens: configuredMaxTokens }, - ...reasoningInfo(resolvedModel, defaultLevel), - } + return this.modelInfo(snapshot, provider, model) }) } - async * stream(options: GenerateOptions): AsyncIterable { + private modelInfo(snapshot: PiAiSnapshot, provider: string, model: string): LlmResolvedModelInfo { + const profile = this.profileOf(snapshot, provider) + const resolvedModel = this.modelOf(snapshot, provider, model) + const defaultLevel = describableReasoningLevel(resolvedModel, profile.reasoning) + // Only a cap the deployment configured is a request default; the + // catalog's `maxTokens` sizes the model and stops there. + const configuredMaxTokens = profile.configuredMaxTokens.get(model) + return { + provider, + id: model, + name: resolvedModel.name, + inputModalities: [...resolvedModel.input], + context: { contextWindow: resolvedModel.contextWindow }, + ...configuredMaxTokens === undefined ? {} : { defaultMaxTokens: configuredMaxTokens }, + ...reasoningInfo(resolvedModel, defaultLevel), + } + } + + override prepareCall(provider: string, model: string, _signal?: AbortSignal): Promise { + const snapshot = this.current() + return Promise.resolve({ + model: this.modelInfo(snapshot, provider, model), + stream: options => this.streamWithSnapshot(options, snapshot), + }) + } + + stream(options: GenerateOptions): AsyncIterable { + return this.streamWithSnapshot(options, this.current()) + } + + private async * streamWithSnapshot( + options: GenerateOptions, + snapshot: PiAiSnapshot, + ): AsyncIterable { if (options.stop !== undefined) { throw new LlmError('llm-pi-ai does not support GenerateOptions.stop', 'UNSUPPORTED_OPTION') } @@ -311,7 +331,6 @@ export class PiAiAdapter extends LlmAdapter { // snapshot, and the credential freezes with them. A configuration change // mid-request builds a separate snapshot, so this request finishes under // the one it started with and the next call picks up the new one. - const snapshot = this.current() const profile = this.profileOf(snapshot, options.provider) const model = this.modelOf(snapshot, options.provider, options.model) const reasoning = resolveReasoningLevel( @@ -341,7 +360,10 @@ export class PiAiAdapter extends LlmAdapter { } const context = attachments === undefined ? toPiContext(options, undefined, onReplayDegrade) - : await toPiContext(options, attachments, onReplayDegrade, profile.maxRequestImageBytes) + : await toPiContext(options, attachments, onReplayDegrade, profile.maxRequestImageBytes, { + maxPixels: profile.requestImagePixelBudget, + maxBytes: profile.requestImageMaxBytes, + }) const events = snapshot.models.streamSimple(model, context, { ...profileOptions(profile, reasoning, apiKey), ...options.temperature === undefined ? {} : { temperature: options.temperature }, diff --git a/packages/llm/llm-pi-ai/src/config.ts b/packages/llm/llm-pi-ai/src/config.ts index 62d49a58ee..2fd68d4648 100644 --- a/packages/llm/llm-pi-ai/src/config.ts +++ b/packages/llm/llm-pi-ai/src/config.ts @@ -52,6 +52,10 @@ export const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 300_000 * Deployments behind stricter gateways lower it per route. */ export const DEFAULT_MAX_REQUEST_IMAGE_BYTES = 20 * 1024 * 1024 +/** Default total-pixel budget preserves the complete 2048px local master. */ +export const DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET = 2048 * 2048 +/** Default raw encoded-byte cap before inline base64 expansion. */ +export const DEFAULT_REQUEST_IMAGE_MAX_BYTES = 1024 * 1024 /** Context capacity assumed for a model neither configuration nor the catalog sizes. */ export const DEFAULT_CONTEXT_WINDOW = 262_144 @@ -163,6 +167,10 @@ export interface PiAiProviderProfile { * requests instead of being rejected by a request-size cap. */ maxRequestImageBytes?: number + /** Total-pixel budget for each deterministic inline request version. */ + requestImagePixelBudget?: number + /** Raw encoded-byte cap for each deterministic inline request version. */ + requestImageMaxBytes?: number /** Provider-owned model-request retry policy; omission uses normal mode with five retries. */ retryPolicy?: RetryPolicyConfig } @@ -180,6 +188,10 @@ export interface ResolvedPiAiProviderProfile streamIdleTimeoutMs: number /** Positive request-level base64 image payload bound after defaulting. */ maxRequestImageBytes: number + /** Positive total-pixel request-version budget after defaulting. */ + requestImagePixelBudget: number + /** Positive raw request-version byte cap after defaulting. */ + requestImageMaxBytes: number /** Immutable retry policy captured with this provider route. */ retryPolicy: ResolvedRetryPolicy /** @@ -312,6 +324,8 @@ const profile = z.object({ websocketConnectTimeoutMs: z.natural(), streamIdleTimeoutMs: z.number().min(Number.MIN_VALUE).max(MAX_TIMER_DELAY_MS).default(DEFAULT_STREAM_IDLE_TIMEOUT_MS), maxRequestImageBytes: z.number().step(1).min(1).default(DEFAULT_MAX_REQUEST_IMAGE_BYTES), + requestImagePixelBudget: z.number().step(1).min(1).default(DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET), + requestImageMaxBytes: z.number().step(1).min(1).default(DEFAULT_REQUEST_IMAGE_MAX_BYTES), retryPolicy: RetryPolicySchema, }) @@ -391,6 +405,14 @@ export function resolveProfiles( if (!Number.isInteger(maxRequestImageBytes) || maxRequestImageBytes <= 0) { throw new Error(`llm-pi-ai: provider "${provider}" maxRequestImageBytes must be a positive integer`) } + const requestImagePixelBudget = source.requestImagePixelBudget ?? DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET + if (!Number.isSafeInteger(requestImagePixelBudget) || requestImagePixelBudget <= 0) { + throw new Error(`llm-pi-ai: provider "${provider}" requestImagePixelBudget must be a positive safe integer`) + } + const requestImageMaxBytes = source.requestImageMaxBytes ?? DEFAULT_REQUEST_IMAGE_MAX_BYTES + if (!Number.isSafeInteger(requestImageMaxBytes) || requestImageMaxBytes <= 0) { + throw new Error(`llm-pi-ai: provider "${provider}" requestImageMaxBytes must be a positive safe integer`) + } // Detached from the configuration object because pi-ai types `Model.input` // mutable. The schema's explicit default covers an absent key, so an empty // list here is always one someone typed — and unlike an entry's, nothing @@ -423,6 +445,8 @@ export function resolveProfiles( ...apiKeyEnv === undefined ? {} : { apiKeyEnv: credentialRef(apiKeyEnv) }, streamIdleTimeoutMs, maxRequestImageBytes, + requestImagePixelBudget, + requestImageMaxBytes, retryPolicy: resolveRetryPolicy(retryPolicy, `llm-pi-ai: provider "${provider}" retryPolicy`), ...rest.headers === undefined ? {} : { headers: { ...rest.headers } }, ...rest.thinkingBudgets === undefined ? {} : { thinkingBudgets: { ...rest.thinkingBudgets } }, diff --git a/packages/llm/llm-pi-ai/src/context.ts b/packages/llm/llm-pi-ai/src/context.ts index d66a48115d..5cf9b7c042 100644 --- a/packages/llm/llm-pi-ai/src/context.ts +++ b/packages/llm/llm-pi-ai/src/context.ts @@ -4,11 +4,18 @@ * @module dsh-llm-pi-ai/context */ -import { CallId, contentHasImage, LlmError, offloadRequestImages } from '@deepseek-ai/dsh-llm' +import { CallId, contentHasImage, LlmError, offloadRequestImagesWithPolicy, requestImagePreviewText } from '@deepseek-ai/dsh-llm' import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm' -import type { AttachmentStore } from '@deepseek-ai/dsh-attachment' +import type { + AttachmentId, + AttachmentStore, + ImageAttachmentRef, + ImageRequestPolicy, + RequestImageAttachment, +} from '@deepseek-ai/dsh-attachment' import type { Context as PiContext, ImageContent, Message as PiMessage, TextContent, Tool as PiTool } from '@earendil-works/pi-ai' import { toPiAssistant } from './replay.ts' +import { DEFAULT_REQUEST_IMAGE_MAX_BYTES, DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET } from './config.ts' /** Join the text blocks of a harness message. */ function flattenText(message: Message): string { @@ -40,7 +47,7 @@ function assertSupportedImageRoles(messages: readonly Message[]): void { async function userContent( blocks: readonly ContentBlock[], - attachments: AttachmentStore, + requestImages: ReadonlyMap, ): Promise { const content: (TextContent | ImageContent)[] = [] for (const block of blocks) { @@ -49,17 +56,21 @@ async function userContent( if (block.text.length > 0) content.push({ type: 'text', text: block.text }) break case 'image': { - const stored = await attachments.readImage(block.attachment) + const version = requestImages.get(block.attachment.attachmentId) + if (version === undefined) { + throw new LlmError(`pi-ai request image ${block.attachment.attachmentId} was not prepared`, 'INVALID_REQUEST') + } + content.push({ type: 'text', text: requestImagePreviewText(version) }) content.push({ type: 'image', - data: Buffer.from(stored.data).toString('base64'), - mimeType: stored.ref.mediaType, + data: Buffer.from(version.data).toString('base64'), + mimeType: version.mediaType, }) break } case 'tool-result': { - const nested = await userContent(block.content, attachments) + const nested = await userContent(block.content, requestImages) if (typeof nested === 'string') { if (nested.length > 0) content.push({ type: 'text', text: nested }) } else { @@ -76,6 +87,28 @@ async function userContent( return content } +function collectImageRefs( + blocks: readonly ContentBlock[], + refs: Map, +): void { + for (const block of blocks) { + if (block.type === 'image') refs.set(block.attachment.attachmentId, block.attachment) + else if (block.type === 'tool-result') collectImageRefs(block.content, refs) + } +} + +async function prepareRequestImages( + messages: readonly Message[], + attachments: AttachmentStore, + policy: ImageRequestPolicy, +): Promise> { + const refs = new Map() + for (const message of messages) collectImageRefs(message.content, refs) + const versions = new Map() + for (const [id, ref] of refs) versions.set(id, await attachments.readImageRequest(ref, policy)) + return versions +} + function toolsOf(options: GenerateOptions): PiTool[] | undefined { return options.tools?.map(tool => ({ name: tool.name, @@ -156,6 +189,7 @@ export function toPiContext( * @param attachments - durable byte resolver for image references. * @param onReplayDegrade - forwarded to {@link toPiAssistant} for each assistant message. * @param maxRequestImageBytes - request-level bound on base64-encoded image payload; omission leaves every image in place. + * @param requestImagePolicy - route pixel and raw encoded-byte budgets. * @returns the asynchronously resolved pi-ai context. */ export function toPiContext( @@ -163,16 +197,18 @@ export function toPiContext( attachments: AttachmentStore, onReplayDegrade?: (reason: string) => void, maxRequestImageBytes?: number, + requestImagePolicy?: ImageRequestPolicy, ): Promise export function toPiContext( options: GenerateOptions, attachments?: AttachmentStore, onReplayDegrade?: (reason: string) => void, maxRequestImageBytes?: number, + requestImagePolicy?: ImageRequestPolicy, ): PiContext | Promise { return attachments === undefined ? textOnlyContext(options, onReplayDegrade) - : toPiContextWithImages(options, attachments, onReplayDegrade, maxRequestImageBytes) + : toPiContextWithImages(options, attachments, onReplayDegrade, maxRequestImageBytes, requestImagePolicy) } async function toPiContextWithImages( @@ -180,9 +216,19 @@ async function toPiContextWithImages( attachments: AttachmentStore, onReplayDegrade?: (reason: string) => void, maxRequestImageBytes?: number, + requestImagePolicy: ImageRequestPolicy = { + maxPixels: DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET, + maxBytes: DEFAULT_REQUEST_IMAGE_MAX_BYTES, + }, ): Promise { assertSupportedImageRoles(options.messages) - const requestMessages = offloadRequestImages(options.messages, maxRequestImageBytes) + const requestImages = await prepareRequestImages(options.messages, attachments, requestImagePolicy) + const requestMessages = offloadRequestImagesWithPolicy(options.messages, { + representation: 'base64', + ...maxRequestImageBytes === undefined ? {} : { maxBytes: maxRequestImageBytes }, + byteQuantum: 1, + byteLength: ref => requestImages.get(ref.attachmentId)?.bytes ?? ref.bytes, + }) const toolNames = new Map() const messages: PiMessage[] = [] @@ -204,7 +250,7 @@ async function toPiContextWithImages( } // user role: text + tool results (each result becomes its own message). const regular = message.content.filter(block => block.type !== 'tool-result') - const content = await userContent(regular, attachments) + const content = await userContent(regular, requestImages) const results = message.content.filter((block): block is Extract => ( block.type === 'tool-result' )) @@ -212,7 +258,7 @@ async function toPiContextWithImages( messages.push({ role: 'user', content, timestamp: 0 }) } for (const result of results) { - const resultContent = await userContent(result.content, attachments) + const resultContent = await userContent(result.content, requestImages) messages.push({ role: 'toolResult', toolCallId: result.toolCallId, diff --git a/packages/llm/llm-pi-ai/tests/adapter.spec.ts b/packages/llm/llm-pi-ai/tests/adapter.spec.ts index c7336cb8d7..171f898db9 100644 --- a/packages/llm/llm-pi-ai/tests/adapter.spec.ts +++ b/packages/llm/llm-pi-ai/tests/adapter.spec.ts @@ -1,9 +1,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { Context } from '@deepseek-ai/cordis' -import { AttachmentId, AttachmentStore } from '@deepseek-ai/dsh-attachment' +import { AttachmentId, AttachmentStore, ImageVariantId } from '@deepseek-ai/dsh-attachment' import type { ImageAttachmentLimits, ImageAttachmentRef, + ImageRequestPolicy, + RequestImageAttachment, SavedImageAttachment, SaveImageAttachment, StoredImageAttachment, @@ -76,6 +78,29 @@ describe('PiAiAdapter provider routing', () => { expect(server.paths).toEqual(['/chat/completions']) }) + it('keeps prepared model metadata and dispatch on one profile snapshot', async () => { + const first = await mockServer([{ events: textEvents }]) + const second = await mockServer([]) + let providers: Record = { + deepseek: { apiKeyEnv: 'PI_TEST_KEY', baseURL: first.url }, + } + const ctx = new Context() + await ctx.plugin(LlmRuntime) + ctx.llm.registerAdapter(['deepseek'], new PiAiAdapter({ + profiles: () => resolveProfiles(providers), + resolveApiKey: () => Promise.resolve('test-key'), + })) + + const prepared = await ctx.llm.prepareCall({ provider: 'deepseek', model: 'deepseek-v4-flash' }) + providers = { deepseek: { apiKeyEnv: 'PI_TEST_KEY', baseURL: second.url } } + const chunks: unknown[] = [] + for await (const chunk of prepared.stream({ ...prepared.config, messages: [] })) chunks.push(chunk) + + expect(chunks.length).toBeGreaterThan(0) + expect(first.requests).toHaveLength(1) + expect(second.requests).toHaveLength(0) + }) + it('merges profile headers with Harness attribution winning', async () => { const server = await mockServer([{ events: textEvents }]) const ctx = await harness(server.url, { @@ -213,6 +238,20 @@ describe('PiAiAdapter provider routing', () => { } const readImage = vi.fn((_ref: ImageAttachmentRef): Promise => Promise.resolve({ ref, data: Uint8Array.of(1) })) + const readImageRequest = vi.fn((value: ImageAttachmentRef, _policy: ImageRequestPolicy): Promise => ( + Promise.resolve({ + variantId: ImageVariantId(`sha256:${'b'.repeat(64)}`), + master: value, + data: Uint8Array.of(1), + mediaType: value.mediaType, + bytes: 1, + width: value.width, + height: value.height, + depth: 'uchar', + space: 'srgb', + hasAlpha: true, + }) + )) class LateAttachmentStore extends AttachmentStore { readonly imageLimits: ImageAttachmentLimits = { @@ -235,6 +274,10 @@ describe('PiAiAdapter provider routing', () => { readImage(value: ImageAttachmentRef): Promise { return readImage(value) } + + override readImageRequest(value: ImageAttachmentRef, policy: ImageRequestPolicy): Promise { + return readImageRequest(value, policy) + } } const ctx = new Context() @@ -254,7 +297,10 @@ describe('PiAiAdapter provider routing', () => { }) expect(result.finish.kind).toBe('error') - expect(readImage).toHaveBeenCalledWith(ref) + expect(readImageRequest).toHaveBeenCalledWith(ref, { + maxPixels: 2048 * 2048, + maxBytes: 1024 * 1024, + }) expect(server.paths).toEqual(['/v1/responses']) }) diff --git a/packages/llm/llm-pi-ai/tests/context.spec.ts b/packages/llm/llm-pi-ai/tests/context.spec.ts index b82c6b63f5..7163a7375d 100644 --- a/packages/llm/llm-pi-ai/tests/context.spec.ts +++ b/packages/llm/llm-pi-ai/tests/context.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest' -import { AttachmentId } from '@deepseek-ai/dsh-attachment' -import type { AttachmentStore, ImageAttachmentRef } from '@deepseek-ai/dsh-attachment' +import { AttachmentId, ImageVariantId } from '@deepseek-ai/dsh-attachment' +import type { AttachmentStore, ImageAttachmentRef, RequestImageAttachment } from '@deepseek-ai/dsh-attachment' import { CallId, createMessage, createUserMessage, OFFLOADED_IMAGE_TEXT } from '@deepseek-ai/dsh-llm' import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm' import { toPiContext } from '../src/context.ts' @@ -14,9 +14,30 @@ const ref: ImageAttachmentRef = { height: 1, } -const attachments = { - readImage: vi.fn(() => Promise.resolve({ ref, data: Uint8Array.of(1) })), -} as unknown as AttachmentStore +function requestImage(value: ImageAttachmentRef, data: Uint8Array): RequestImageAttachment { + return { + variantId: ImageVariantId(`sha256:${'b'.repeat(64)}`), + master: value, + data, + mediaType: value.mediaType, + bytes: data.byteLength, + width: value.width, + height: value.height, + depth: 'uchar', + space: 'srgb', + hasAlpha: value.mediaType === 'image/png', + } +} + +function projectionStore( + readImageRequest = vi.fn((value: ImageAttachmentRef) => ( + Promise.resolve(requestImage(value, Uint8Array.of(1))) + )), +): AttachmentStore { + return { readImageRequest } as unknown as AttachmentStore +} + +const attachments = projectionStore() function request(messages: GenerateOptions['messages']): GenerateOptions { return { @@ -116,6 +137,7 @@ describe('pi-ai request context conversion', () => { { role: 'user', content: [ + { type: 'text', text: expect.stringContaining(`Image ${ref.attachmentId}`) as string }, { type: 'image', data: 'AQ==', mimeType: 'image/png' }, { type: 'text', text: 'caption' }, ], @@ -133,7 +155,10 @@ describe('pi-ai request context conversion', () => { role: 'toolResult', toolCallId: 'missing-call', toolName: 'unknown', - content: [{ type: 'image', data: 'AQ==', mimeType: 'image/png' }], + content: [ + { type: 'text', text: expect.stringContaining(`Image ${ref.attachmentId}`) as string }, + { type: 'image', data: 'AQ==', mimeType: 'image/png' }, + ], isError: true, timestamp: 0, }, @@ -165,6 +190,7 @@ describe('pi-ai request context conversion', () => { toolName: 'unknown', content: [ { type: 'text', text: 'nested text' }, + { type: 'text', text: expect.stringContaining(`Image ${ref.attachmentId}`) as string }, { type: 'image', data: 'AQ==', mimeType: 'image/png' }, ], isError: false, @@ -194,8 +220,10 @@ describe('pi-ai request context conversion', () => { }) it('replaces the oldest images with placeholders once the request payload bound is exceeded', async () => { - const readImage = vi.fn(() => Promise.resolve({ ref: { ...ref, bytes: 3 }, data: Uint8Array.of(1, 2, 3) })) - const store = { readImage } as unknown as AttachmentStore + const readImageRequest = vi.fn((value: ImageAttachmentRef) => ( + Promise.resolve(requestImage(value, Uint8Array.of(1, 2, 3))) + )) + const store = projectionStore(readImageRequest) const sized: ImageAttachmentRef = { ...ref, bytes: 3 } const callId = CallId('shot-call') // Three 3-byte images cost 4 base64 characters each (12 total); a bound of @@ -222,14 +250,22 @@ describe('pi-ai request context conversion', () => { { role: 'user', content: [ + { type: 'text', text: expect.stringContaining(`Image ${sized.attachmentId}`) as string }, { type: 'image', data: 'AQID', mimeType: 'image/png' }, { type: 'text', text: 'newer' }, ], timestamp: 0, }, - { role: 'user', content: [{ type: 'image', data: 'AQID', mimeType: 'image/png' }], timestamp: 0 }, + { + role: 'user', + content: [ + { type: 'text', text: expect.stringContaining(`Image ${sized.attachmentId}`) as string }, + { type: 'image', data: 'AQID', mimeType: 'image/png' }, + ], + timestamp: 0, + }, ]) - expect(readImage).toHaveBeenCalledTimes(2) + expect(readImageRequest).toHaveBeenCalledTimes(1) }) it('keeps every image at exactly the payload bound and drops all of them when even the newest cannot fit', async () => { @@ -239,12 +275,22 @@ describe('pi-ai request context conversion', () => { user([{ type: 'image', attachment: sized }]), ]), attachments, undefined, 8) expect(exact.messages).toEqual([ - { role: 'user', content: [expect.objectContaining({ type: 'image' })], timestamp: 0 }, - { role: 'user', content: [expect.objectContaining({ type: 'image' })], timestamp: 0 }, + { + role: 'user', + content: [expect.objectContaining({ type: 'text' }), expect.objectContaining({ type: 'image' })], + timestamp: 0, + }, + { + role: 'user', + content: [expect.objectContaining({ type: 'text' }), expect.objectContaining({ type: 'image' })], + timestamp: 0, + }, ]) - const readImage = vi.fn() - const store = { readImage } as unknown as AttachmentStore + const readImageRequest = vi.fn((value: ImageAttachmentRef) => ( + Promise.resolve(requestImage(value, new Uint8Array(300))) + )) + const store = projectionStore(readImageRequest) const oversized = await toPiContext(request([ user([{ type: 'image', attachment: { ...ref, bytes: 300 } }]), ]), store, undefined, 8) @@ -252,14 +298,16 @@ describe('pi-ai request context conversion', () => { expect(oversized.messages).toEqual([ { role: 'user', content: OFFLOADED_IMAGE_TEXT, timestamp: 0 }, ]) - expect(readImage).not.toHaveBeenCalled() + expect(readImageRequest).toHaveBeenCalledTimes(1) }) it('offloads repeated image-block occurrences by position rather than shared object identity', async () => { const sized: ImageAttachmentRef = { ...ref, bytes: 3 } const shared: ContentBlock = { type: 'image', attachment: sized } - const readImage = vi.fn(() => Promise.resolve({ ref: sized, data: Uint8Array.of(1, 2, 3) })) - const store = { readImage } as unknown as AttachmentStore + const readImageRequest = vi.fn((value: ImageAttachmentRef) => ( + Promise.resolve(requestImage(value, Uint8Array.of(1, 2, 3))) + )) + const store = projectionStore(readImageRequest) const aliased = await toPiContext(request([user([shared, shared])]), store, undefined, 4) const replayed = await toPiContext(request([user([ { type: 'image', attachment: { ...sized } }, @@ -270,13 +318,14 @@ describe('pi-ai request context conversion', () => { role: 'user', content: [ { type: 'text', text: OFFLOADED_IMAGE_TEXT }, + { type: 'text', text: expect.stringContaining(`Image ${sized.attachmentId}`) as string }, { type: 'image', data: 'AQID', mimeType: 'image/png' }, ], timestamp: 0, }] expect(aliased.messages).toEqual(expected) expect(replayed.messages).toEqual(expected) - expect(readImage).toHaveBeenCalledTimes(2) + expect(readImageRequest).toHaveBeenCalledTimes(2) }) it('keeps empty text-only users while separating result-only messages', () => { @@ -303,12 +352,12 @@ describe('pi-ai request context conversion', () => { it('handles in-history system and assistant messages explicitly on the image path', async () => { for (const role of ['system', 'assistant'] as const) { - const readImage = vi.fn() - const store = { readImage } as unknown as AttachmentStore + const readImageRequest = vi.fn() + const store = projectionStore(readImageRequest) await expect(toPiContext(request([ history(role, [{ type: 'image', attachment: ref }]), ]), store, undefined, 1)).rejects.toMatchObject({ code: 'UNSUPPORTED_CONTENT' }) - expect(readImage).not.toHaveBeenCalled() + expect(readImageRequest).not.toHaveBeenCalled() } await expect(toPiContext(request([ diff --git a/packages/llm/llm-pi-ai/tests/convert.spec.ts b/packages/llm/llm-pi-ai/tests/convert.spec.ts index 2a3b41b0c4..e77075b0c6 100644 --- a/packages/llm/llm-pi-ai/tests/convert.spec.ts +++ b/packages/llm/llm-pi-ai/tests/convert.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest' -import { AttachmentId } from '@deepseek-ai/dsh-attachment' -import type { AttachmentStore } from '@deepseek-ai/dsh-attachment' +import { AttachmentId, ImageVariantId } from '@deepseek-ai/dsh-attachment' +import type { AttachmentStore, ImageAttachmentRef, RequestImageAttachment } from '@deepseek-ai/dsh-attachment' import { createUserMessage, CallId, CONTEXT_WINDOW_EXCEEDED_CODE, EMPTY_RESPONSE_CODE, createMessage } from '@deepseek-ai/dsh-llm' import type { ContentBlock, StreamChunk } from '@deepseek-ai/dsh-llm' import type { AssistantMessage, AssistantMessageEvent, Usage } from '@earendil-works/pi-ai' @@ -43,6 +43,21 @@ async function collect(stream: AsyncIterable): Promise { it('maps system prompt, user text, and tools', () => { const context = toPiContext({ @@ -76,7 +91,7 @@ describe('toPiContext', () => { width: 1, height: 1, } - const readImage = vi.fn().mockResolvedValue({ ref: attachment, data: Uint8Array.of(1, 2, 3) }) + const readImageRequest = vi.fn((value: ImageAttachmentRef) => Promise.resolve(requestVersion(value))) const context = await toPiContext({ provider: 'openai', model: 'gpt-4.1', @@ -84,13 +99,17 @@ describe('toPiContext', () => { content: [{ type: 'text', text: 'describe' }, { type: 'image', attachment }], source: { kind: 'plugin', plugin: 'test' }, })], - }, { readImage } as unknown as AttachmentStore) + }, { readImageRequest } as unknown as AttachmentStore) - expect(readImage).toHaveBeenCalledWith(attachment) + expect(readImageRequest).toHaveBeenCalledWith( + attachment, + { maxPixels: 2048 * 2048, maxBytes: 1024 * 1024 }, + ) expect(context.messages[0]).toEqual({ role: 'user', content: [ { type: 'text', text: 'describe' }, + { type: 'text', text: expect.stringContaining(`Image ${attachment.attachmentId}`) as string }, { type: 'image', data: 'AQID', mimeType: 'image/png' }, ], timestamp: 0, @@ -105,7 +124,7 @@ describe('toPiContext', () => { width: 1, height: 1, } - const readImage = vi.fn().mockResolvedValue({ ref: attachment, data: Uint8Array.of(1, 2, 3) }) + const readImageRequest = vi.fn((value: ImageAttachmentRef) => Promise.resolve(requestVersion(value))) const context = await toPiContext({ provider: 'openai', model: 'gpt-4.1', @@ -129,7 +148,7 @@ describe('toPiContext', () => { }], source: { kind: 'plugin', plugin: 'test' }, })], - }, { readImage } as unknown as AttachmentStore) + }, { readImageRequest } as unknown as AttachmentStore) expect(context.messages).toEqual([{ role: 'toolResult', @@ -138,6 +157,7 @@ describe('toPiContext', () => { content: [ { type: 'text', text: 'before' }, { type: 'text', text: 'middle' }, + { type: 'text', text: expect.stringContaining(`Image ${attachment.attachmentId}`) as string }, { type: 'image', data: 'AQID', mimeType: 'image/png' }, { type: 'text', text: 'after' }, ], diff --git a/packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts b/packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts index b0b1dbba9a..93732e75d3 100644 --- a/packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts +++ b/packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts @@ -1,10 +1,12 @@ import { readFile } from 'node:fs/promises' import { afterEach, describe, expect, it } from 'vitest' import { Context } from '@deepseek-ai/cordis' -import { AttachmentId, AttachmentStore } from '@deepseek-ai/dsh-attachment' +import { AttachmentId, AttachmentStore, ImageVariantId } from '@deepseek-ai/dsh-attachment' import type { ImageAttachmentLimits, ImageAttachmentRef, + ImageRequestPolicy, + RequestImageAttachment, SavedImageAttachment, SaveImageAttachment, StoredImageAttachment, @@ -89,6 +91,24 @@ async function harness(image?: StoredImageAttachment): Promise { } return Promise.resolve(fixture) } + + override readImageRequest(ref: ImageAttachmentRef, _policy: ImageRequestPolicy): Promise { + if (ref.attachmentId !== fixture.ref.attachmentId) { + return Promise.reject(new Error('unknown e2e attachment fixture')) + } + return Promise.resolve({ + variantId: ImageVariantId(`sha256:${'f'.repeat(64)}`), + master: fixture.ref, + data: fixture.data, + mediaType: fixture.ref.mediaType, + bytes: fixture.data.byteLength, + width: fixture.ref.width, + height: fixture.ref.height, + depth: 'uchar', + space: 'srgb', + hasAlpha: fixture.ref.mediaType === 'image/png', + }) + } } await ctx.plugin(E2eAttachmentStore) } diff --git a/packages/llm/llm/README.i18n.yaml b/packages/llm/llm/README.i18n.yaml index 2621de89af..08820bbfd9 100644 --- a/packages/llm/llm/README.i18n.yaml +++ b/packages/llm/llm/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/llm/llm/README.md -README.md: e6b3c4924ad4e7cf115abdbfb38d22d0f524e377 -README.zh.md: 91f1c6ede2b800671b35a90d02a23a6d79e96e0e +README.md: 59c5303bdae8d6391f3bb1595a527d677e2378bc +README.zh.md: 313fc23590de2119bf07dd5c4c4fafe9daa94c56 diff --git a/packages/llm/llm/README.md b/packages/llm/llm/README.md index e6b3c4924a..59c5303bda 100644 --- a/packages/llm/llm/README.md +++ b/packages/llm/llm/README.md @@ -25,7 +25,7 @@ Each provider adapter supplies its resolved route policy. Omitting provider conf - `ctx.llm.listModels(provider: string): Promise` Discover the models one registered provider currently advertises. - `ctx.llm.resolveModelInfo(provider: string, model: string, signal?: AbortSignal): Promise` Resolve validated exact-model identity plus available context, output-default, and reasoning metadata from the owning adapter, with optional cancellation for asynchronous adapters. - `ctx.llm.resolveCallConfig(config: LlmCallConfig, signal?: AbortSignal): Promise` Validate an explicit effort and materialize adapter-configured call defaults without clamping. -- `ctx.llm.prepareCall(config: LlmCallConfig, signal?: AbortSignal): Promise` Resolve a config plus detached context metadata and markers for fields supplied by adapter defaults in one exact-model lookup, then capture its current adapter registration and immutable retry policy as one cancellable, one-shot call. +- `ctx.llm.prepareCall(config: LlmCallConfig, signal?: AbortSignal): Promise` Resolve a config plus detached context and modality metadata and markers for fields supplied by adapter defaults in one exact-model lookup, then capture the adapter's matching dispatch generation and immutable retry policy as one cancellable, one-shot call. - `ctx.llm.stream(options: GenerateOptions): AsyncIterable` Stream one model call as raw chunks (token-level deltas). Consumers assemble the chunks into blocks/messages with `BlockAssembler`. `LlmRuntime` normalizes failures from final adapter selection, synchronous dispatch, iterator construction, and iteration into the stream protocol's single terminal form: `finish { kind: 'error' | 'aborted', failure }`. A failure after partial deltas may leave content blocks open; consumers discard that incomplete output. Errors from `llm/stream` middleware, nested calls, adapter cleanup, and downstream consumers remain thrown because they are plugin or consumer failures rather than model-request outcomes. A prepared call exposes the immutable retry policy captured with its exact adapter registration; a route handled entirely by middleware has no serving policy. @@ -38,7 +38,7 @@ Every topology commit point — adapter routes registering or disposing, directo Exact-model metadata is a separate correctness query, not a catalog decoration or global LLM setting. `resolveModelInfo()` asks the adapter that owns the exact provider/model route once; an adapter can describe an unlisted dynamic model, and absent `context`, `defaultMaxTokens`, or `reasoning` fields preserve unknown capacity, provider-owned output defaults, or unavailable reasoning capability. Invalid identity, context, output default, or reasoning metadata fails with `INVALID_MODEL_INFO`, `INVALID_MODEL_CONTEXT`, `INVALID_MODEL_MAX_TOKENS`, or `INVALID_MODEL_REASONING`. -`defaultMaxTokens` is an adapter-configured per-request output cap, not a model hard limit. `resolveCallConfig()` materializes it only when the request omits `maxTokens`; an explicit cap wins. Reasoning identifiers are opaque adapter-owned strings rather than a core enum: the same resolution accepts only an exact advertised identifier, materializes `defaultEffort` when present, and otherwise preserves the provider default. Asynchronous model resolvers receive the caller's signal and must settle promptly after cancellation. `prepareCall()` additionally exposes detached context metadata from the same lookup, reports which `maxTokens` and `reasoningEffort` fields it materialized in `adapterDefaults`, and retains the exact adapter registration through header logging and terminal dispatch, so HMR cannot combine one adapter's capability result with another adapter's request; reusing its one-shot handle or changing its call-config fields fails with `INVALID_PREPARED_CALL`. An unsupported explicit or configured effort fails with `UNSUPPORTED_REASONING_EFFORT` before provider I/O. +`defaultMaxTokens` is an adapter-configured per-request output cap, not a model hard limit. `resolveCallConfig()` materializes it only when the request omits `maxTokens`; an explicit cap wins. Reasoning identifiers are opaque adapter-owned strings rather than a core enum: the same resolution accepts only an exact advertised identifier, materializes `defaultEffort` when present, and otherwise preserves the provider default. Asynchronous model resolvers receive the caller's signal and must settle promptly after cancellation. `prepareCall()` additionally exposes detached context and input-modality metadata, reports which `maxTokens` and `reasoningEffort` fields it materialized in `adapterDefaults`, and binds those facts to the adapter generation that performs terminal dispatch. HMR or dynamic settings therefore cannot combine one generation's image capability with another generation's endpoint; reusing the one-shot handle or changing its call-config fields fails with `INVALID_PREPARED_CALL`. An unsupported explicit or configured effort fails with `UNSUPPORTED_REASONING_EFFORT` before provider I/O. ### Events @@ -55,7 +55,9 @@ Exact-model metadata is a separate correctness query, not a catalog decoration o `Message` is the shared immutable value used by delivery, durable history, and model requests. Every message has a required `MessageId`, role, content, and typed source from creation onward. `createMessage(input)` mints the identity and returns a detached deep-frozen value; `createUserMessage({ content, source })` fixes the user role; `createAssistantMessage({ content, source })` fixes the assistant role and model source kind; `createToolResultMessage({ callId, content, isError })` fixes the user role and couples the tool source to its result block; `freezeMessage(message)` imports an identity that already exists and never replaces it. Message rewrites preserve the identity and produce another frozen value. Browser code imports these value constructors from the dependency-minimal `@deepseek-ai/dsh-llm/message` entry instead of the service-bearing package root. -Message content is an array of typed blocks: `text`, `reasoning`, `tool-call`, `tool-result`. The union is derived from the merge-extensible `ContentBlockMap`, so plugins can add block types via declaration merging. Assistant messages use a model source carrying the provider and model that produced them plus optional adapter-private replay state. Before dispatch, `LlmRuntime` retains that state only when the historical provider route and target provider route are currently owned by the exact same adapter instance; the adapter then decides whether it can restore or convert the state across models/providers. The core block set is limited to blocks every shipping path honors — multimodal content (images, audio, …) has no core block type; a feature that needs one adds it via the map together with the adapter/UI/compaction support that honors it. +Message content is an array of typed blocks: `text`, `reasoning`, `image`, `tool-call`, `tool-result`. An `ImageBlock` carries only a durable `ImageAttachmentRef`; provider bytes and request dimensions are resolved later. The union remains merge-extensible through `ContentBlockMap`, so plugins can add further block types via declaration merging. Assistant messages use a model source carrying the provider and model that produced them plus optional adapter-private replay state. Before dispatch, `LlmRuntime` retains that state only when the historical provider route and target provider route are currently owned by the exact same adapter instance; the adapter then decides whether it can restore or convert the state across models/providers. + +Every dispatch uses the exact model modalities captured with its adapter generation. An image-capable adapter projects durable image references into route-specific request versions. A text-only route instead receives deterministic attachment placeholders, including nested tool-result images, without changing append-only session history. `offloadRequestImagesWithPolicy()` provides deterministic oldest-first image removal with raw or base64 accounting and count or byte quanta; adapters supply the exact derived-version byte length. Streaming is a raw chunk protocol (`block-start`, `text-delta`, `reasoning-delta`, `tool-call-delta`, `block-end`, `usage`, `finish`). Every adapter outcome reaches consumers as one terminal `finish`; operational failure uses its `error` or `aborted` reason rather than throwing across the stream API. `BlockAssembler` is the single shared implementation that assembles chunks into blocks/messages. A successful `finish` may carry a `ReplayEnvelope` — opaque response-level replay metadata plus optional per-block entries aligned with the emitted block sequence. Assembly makes one keep/drop decision for content and metadata together: a `max-tokens` finish drops tool calls that may have been truncated, and the envelope loses the entry at each dropped position, so stored metadata always describes stored content. diff --git a/packages/llm/llm/README.zh.md b/packages/llm/llm/README.zh.md index 91f1c6ede2..313fc23590 100644 --- a/packages/llm/llm/README.zh.md +++ b/packages/llm/llm/README.zh.md @@ -25,7 +25,7 @@ - `ctx.llm.listModels(provider: string): Promise` 发现某个已注册提供方当前公布的模型。 - `ctx.llm.resolveModelInfo(provider: string, model: string, signal?: AbortSignal): Promise` 从拥有该精确路由的适配器中,解析并校验确切模型身份,以及可用上下文、输出默认值和推理(reasoning)元数据;异步适配器可选地支持取消。 - `ctx.llm.resolveCallConfig(config: LlmCallConfig, signal?: AbortSignal): Promise` 校验显式推理强度,并填入适配器配置的调用默认值,但不自动调整。 -- `ctx.llm.prepareCall(config: LlmCallConfig, signal?: AbortSignal): Promise` 在一次精确模型查询中解析配置、脱耦的上下文元数据以及标明哪些字段由适配器默认值填入的标记,再将当前适配器注册和不可变重试策略捕获为一次可取消、一次性调用。 +- `ctx.llm.prepareCall(config: LlmCallConfig, signal?: AbortSignal): Promise` 在一次精确模型查询中解析配置、脱耦的上下文与模态元数据以及标明哪些字段由适配器默认值填入的标记,再把适配器匹配的分发世代和不可变重试策略捕获为一次可取消、一次性调用。 - `ctx.llm.stream(options: GenerateOptions): AsyncIterable` 将一次模型调用流式输出为原始分片(token 级增量)。消费方使用 `BlockAssembler` 将分片组装为块/消息。 `LlmRuntime` 将最终适配器选择、同步分发、迭代器构造和迭代期间的失败,统一转换为流协议唯一的终止形式:`finish { kind: 'error' | 'aborted', failure }`。部分增量输出后发生失败时,内容块可能仍未闭合;消费方会丢弃这些不完整输出。`llm/stream` middleware、嵌套调用、适配器清理和下游消费方的错误仍会抛出,因为它们属于插件或消费方失败,而非模型请求结果。已准备调用会暴露随其确切适配器注册一同捕获的不可变重试策略;完全由 middleware 处理的路由没有服务策略。 @@ -38,7 +38,7 @@ 确切模型元数据是独立的正确性查询,不是 catalog 装饰或全局 LLM 设置。`resolveModelInfo()` 会向拥有精确提供方/模型路由的适配器查询一次;适配器可以描述未列出的动态模型。缺少 `context` 表示模型容量未知;缺少 `defaultMaxTokens` 表示继续沿用提供方自身的输出默认值;缺少 `reasoning` 则表示推理能力不可用。无效的身份、上下文、输出默认值或推理元数据会以 `INVALID_MODEL_INFO`、`INVALID_MODEL_CONTEXT`、`INVALID_MODEL_MAX_TOKENS` 或 `INVALID_MODEL_REASONING` 失败。 -`defaultMaxTokens` 是适配器配置的单次请求输出上限,不是模型硬上限。仅当请求省略 `maxTokens` 时,`resolveCallConfig()` 才会填入该值;显式上限优先。推理标识符是由适配器定义的不透明字符串,而非核心枚举:同一次解析只接受与已公布标识符完全一致的值,在存在 `defaultEffort` 时填入它,否则保留提供方默认值。异步模型解析器会接收调用方信号,并且必须在取消后尽快结算。`prepareCall()` 还会返回同一次查询得到的、与适配器内部状态分离的上下文元数据,通过 `adapterDefaults` 标明填入了哪些 `maxTokens` 和 `reasoningEffort` 字段,并在请求头记录和最终分发期间始终保留同一项精确的适配器注册。因此,HMR(热模块替换)不会把一个适配器的能力结果与另一个适配器的请求混用;复用其一次性句柄或更改调用配置字段会以 `INVALID_PREPARED_CALL` 失败。不支持的显式或配置推理强度会在提供方 I/O 前以 `UNSUPPORTED_REASONING_EFFORT` 失败。 +`defaultMaxTokens` 是适配器配置的单次请求输出上限,不是模型硬上限。仅当请求省略 `maxTokens` 时,`resolveCallConfig()` 才会填入该值;显式上限优先。推理标识符是由适配器定义的不透明字符串,而非核心枚举:同一次解析只接受与已公布标识符完全一致的值,在存在 `defaultEffort` 时填入它,否则保留提供方默认值。异步模型解析器会接收调用方信号,并且必须在取消后尽快结算。`prepareCall()` 还会公开脱离内部状态的上下文和输入模态元数据,通过 `adapterDefaults` 标明填入了哪些 `maxTokens` 和 `reasoningEffort` 字段,并把这些事实绑定到执行最终分发的适配器世代。因此,HMR(热模块替换)或动态 settings 不会把一个世代的图片能力与另一个世代的端点组合;复用一次性句柄或更改调用配置字段会以 `INVALID_PREPARED_CALL` 失败。不支持的显式或配置推理强度会在提供方 I/O 前以 `UNSUPPORTED_REASONING_EFFORT` 失败。 ### 事件 @@ -55,7 +55,9 @@ `Message` 是投递、持久历史和模型请求共享的不可变值。每条消息从创建起都必须具有 `MessageId`、角色、内容和带类型的来源。`createMessage(input)` 生成标识,并返回与输入分离且深度冻结的值;`createUserMessage({ content, source })` 固定 user 角色;`createAssistantMessage({ content, source })` 固定 assistant 角色与模型来源类别;`createToolResultMessage({ callId, content, isError })` 固定 user 角色,并将工具来源与其结果块耦合;`freezeMessage(message)` 导入已有标识,绝不将其替换。改写消息时会保留标识,并产生另一个冻结值。浏览器端代码会从依赖最少的 `@deepseek-ai/dsh-llm/message` 入口导入这些值构造函数,而不是从包含服务的包根入口导入。 -消息内容是类型化内容块数组:`text`、`reasoning`、`tool-call`、`tool-result`。联合从可合并扩展的 `ContentBlockMap` 派生,因此插件可以通过 declaration merging 添加块类型。assistant 消息使用模型来源,其中携带生成该消息的提供方和模型,以及可选的适配器私有回放状态。dispatch 前,`LlmRuntime` 只在历史提供方路由与目标提供方路由当前由完全相同的适配器实例拥有时才保留该状态;随后由适配器判定能否在模型/提供方间恢复或转换该状态。核心块集只包含每条已发布路径都支持的块。多模态内容(图像、音频等)没有核心块类型;需要它的功能会通过 map 添加,并一并添加相应的适配器/UI/压缩(compaction)支持。 +消息内容是类型化内容块数组:`text`、`reasoning`、`image`、`tool-call`、`tool-result`。`ImageBlock` 只携带持久 `ImageAttachmentRef`;提供方字节和请求尺寸之后再解析。联合仍从可合并扩展的 `ContentBlockMap` 派生,因此插件可以通过 declaration merging 添加其他块类型。assistant 消息使用模型来源,其中携带生成该消息的提供方和模型,以及可选的适配器私有回放状态。dispatch 前,`LlmRuntime` 只在历史提供方路由与目标提供方路由当前由完全相同的适配器实例拥有时才保留该状态;随后由适配器判定能否在模型或提供方间恢复或转换该状态。 + +每次分发都使用随适配器世代捕获的确切模型模态。支持图片的适配器把持久图片引用投影为路由专用请求版本。纯文本路由则收到确定性的附件占位文本,其中也包括嵌套工具结果图片,追加式会话历史不会改变。`offloadRequestImagesWithPolicy()` 提供确定性的从旧到新图片移除,支持按原始字节或 base64 计数,也支持图片数量或字节量步长;适配器提供确切派生版本的字节长度。 流式输出是原始分片协议(`block-start`、`text-delta`、`reasoning-delta`、`tool-call-delta`、`block-end`、`usage`、`finish`)。每个适配器结果都以一个终止 `finish` 到达消费方;运行故障使用 `error` 或 `aborted` 作为结束原因,而不会跨流 API 抛出。`BlockAssembler` 是将分片组装为块/消息的唯一共享实现。成功的 `finish` 可以携带 `ReplayEnvelope`——不透明的响应级回放元数据,加上与发射块序列对齐的可选逐块条目。组装对内容与元数据只做一次保留/丢弃决定:`max-tokens` 结束会丢弃可能被截断的工具调用,数据在每个被丢弃的位置同步失去对应条目,因此存储的元数据始终描述存储的内容。 diff --git a/packages/llm/llm/src/content.ts b/packages/llm/llm/src/content.ts index 55c0719fb9..96c39f4f9b 100644 --- a/packages/llm/llm/src/content.ts +++ b/packages/llm/llm/src/content.ts @@ -2,11 +2,33 @@ import type { ContentBlock } from './types.ts' import type { Message } from './message.ts' +import type { ImageAttachmentRef, RequestImageAttachment } from '@deepseek-ai/dsh-attachment' /** Model-facing stand-in for an image removed to fit a provider request bound. */ export const OFFLOADED_IMAGE_TEXT = '[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]' +/** + * Stable text shown to a model that cannot accept one durable image reference. + * @param ref - durable master reference omitted from the request. + * @returns deterministic text-only placeholder. + */ +export function textOnlyImageText(ref: ImageAttachmentRef): string { + const digest = String(ref.attachmentId).slice('sha256:'.length, 'sha256:'.length + 8) + return `[image omitted because this model accepts text only; attachment sha256:${digest}]` +} + +/** + * Stable model-facing handle and coordinate description for one exact request preview. + * @param version - exact request image shown beside the text. + * @returns attachment handle, preview dimensions, and crop-coordinate guidance. + */ +export function requestImagePreviewText(version: RequestImageAttachment): string { + return `Image ${version.master.attachmentId}; preview ${version.width}x${version.height}px. ` + + 'Crop coordinates use this preview. Call read_image_region with this attachment_id, ' + + `preview_width=${version.width}, preview_height=${version.height}, x, y, width, and height.` +} + /** * True when typed model content contains an image block, walking nested * tool-result content. This is the one recursive image walk shared by every @@ -25,13 +47,34 @@ function base64Length(bytes: number): number { return Math.ceil(bytes / 3) * 4 } -/** Collect base64 payload lengths in request and nested-block order. */ -function collectImageLengths(blocks: readonly ContentBlock[], lengths: number[]): void { +/** Byte accounting and quantized removal policy for one request representation. */ +export interface RequestImageOffloadPolicy { + /** Image count accepted by the route; omission leaves count unbounded. */ + maxImages?: number + /** Accumulated image bytes accepted by the route; omission leaves bytes unbounded. */ + maxBytes?: number + /** Number of excess images removed as one deterministic step. */ + countQuantum?: number + /** Number of excess bytes removed as one deterministic step. */ + byteQuantum?: number + /** Whether byte accounting uses raw file bytes or inline base64 length. */ + representation: 'raw' | 'base64' + /** Resolve the encoded request-version length; omission uses master attachment bytes. */ + byteLength?: (ref: ImageAttachmentRef) => number +} + +/** Collect represented image lengths in request and nested-block order. */ +function collectImageLengths( + blocks: readonly ContentBlock[], + lengths: number[], + policy: RequestImageOffloadPolicy, +): void { for (const block of blocks) { if (block.type === 'image') { - lengths.push(base64Length(block.attachment.bytes)) + const bytes = policy.byteLength?.(block.attachment) ?? block.attachment.bytes + lengths.push(policy.representation === 'base64' ? base64Length(bytes) : bytes) } else if (block.type === 'tool-result') { - collectImageLengths(block.content, lengths) + collectImageLengths(block.content, lengths, policy) } } } @@ -62,6 +105,41 @@ function replaceOldestImages( return next ?? blocks as ContentBlock[] } +/** Replace every image occurrence, including nested tool results, for a text-only model. */ +function replaceImagesForTextModel(blocks: readonly ContentBlock[]): ContentBlock[] { + let next: ContentBlock[] | undefined + for (const [index, block] of blocks.entries()) { + if (block.type === 'image') { + next ??= blocks.slice(0, index) + next.push({ type: 'text', text: textOnlyImageText(block.attachment) }) + continue + } + if (block.type === 'tool-result') { + const content = replaceImagesForTextModel(block.content) + if (content !== block.content) { + next ??= blocks.slice(0, index) + next.push({ ...block, content }) + continue + } + } + next?.push(block) + } + return next ?? blocks as ContentBlock[] +} + +/** + * Project durable image history into deterministic text for an exact text-only model. + * @param messages - complete request history. + * @returns the original list without images, otherwise shallow message copies with stable placeholders. + */ +export function projectImagesForTextModel(messages: readonly Message[]): readonly Message[] { + if (!messages.some(message => contentHasImage(message.content))) return messages + return messages.map((message) => { + const content = replaceImagesForTextModel(message.content) + return content === message.content ? message : { ...message, content } + }) +} + /** * Return transient request messages whose oldest images are replaced until * their accumulated base64 payload fits the configured bound. The selection @@ -75,17 +153,47 @@ export function offloadRequestImages( messages: readonly Message[], maxRequestImageBytes: number | undefined, ): readonly Message[] { - if (maxRequestImageBytes === undefined) return messages + return offloadRequestImagesWithPolicy(messages, { + representation: 'base64', + ...maxRequestImageBytes === undefined ? {} : { maxBytes: maxRequestImageBytes }, + byteQuantum: 1, + }) +} + +/** + * Return a deterministic transient projection whose oldest images are replaced + * in whole count and byte quanta after a route budget is exceeded. The target + * depends only on complete durable history: at 129 one-megabyte images under + * a 128 MiB bound with a 64 MiB quantum, the oldest 65 images are removed so + * 64 MiB remain; that removed prefix stays fixed until total history exceeds + * 192 MiB. + * @param messages - complete request history, oldest first. + * @param policy - route representation, budgets, and removal quanta. + * @returns original messages below both bounds, otherwise shallow copies with deterministic placeholders. + */ +export function offloadRequestImagesWithPolicy( + messages: readonly Message[], + policy: RequestImageOffloadPolicy, +): readonly Message[] { const lengths: number[] = [] - for (const message of messages) collectImageLengths(message.content, lengths) - let total = lengths.reduce((sum, bytes) => sum + bytes, 0) + for (const message of messages) collectImageLengths(message.content, lengths, policy) + const total = lengths.reduce((sum, bytes) => sum + bytes, 0) + const excessCount = policy.maxImages === undefined ? 0 : Math.max(0, lengths.length - policy.maxImages) + const excessBytes = policy.maxBytes === undefined ? 0 : Math.max(0, total - policy.maxBytes) + if (excessCount === 0 && excessBytes === 0) return messages + const countQuantum = policy.countQuantum ?? 1 + const byteQuantum = policy.byteQuantum ?? 1 + const removeCount = excessCount === 0 ? 0 : Math.ceil(excessCount / countQuantum) * countQuantum + const removeBytes = excessBytes === 0 ? 0 : Math.ceil(excessBytes / byteQuantum) * byteQuantum let count = 0 - for (const bytes of lengths) { - if (total <= maxRequestImageBytes) break - total -= bytes + let removedBytes = 0 + for (const imageBytes of lengths) { + const byteTargetMet = removeBytes === 0 + || (byteQuantum === 1 ? removedBytes >= removeBytes : removedBytes > removeBytes) + if (count >= removeCount && byteTargetMet) break + removedBytes += imageBytes count += 1 } - if (count === 0) return messages const remaining = { count } return messages.map((message) => { const content = replaceOldestImages(message.content, remaining) diff --git a/packages/llm/llm/src/index.ts b/packages/llm/llm/src/index.ts index e87c428d06..82b64bf4ce 100644 --- a/packages/llm/llm/src/index.ts +++ b/packages/llm/llm/src/index.ts @@ -29,6 +29,7 @@ import type { LlmCallConfig, LlmCallConfigAdapterDefaults } from './call-config. import { HarnessError, INVALID_CREDENTIAL_CODE } from './error.ts' import { normalizeLlmFailure } from './adapter-failure.ts' import { normalizeApiKey } from './api-key.ts' +import { contentHasImage, projectImagesForTextModel } from './content.ts' export * from './attribution.ts' export * from './brand.ts' @@ -159,6 +160,8 @@ export interface PreparedLlmCall { readonly retryPolicy: ResolvedRetryPolicy /** Detached context metadata resolved with the registration-bound call. */ readonly context?: LlmModelContext + /** Exact model modalities captured with the adapter dispatch generation. */ + readonly inputModalities?: readonly ModelModality[] /** Config fields materialized by the captured adapter rather than proposed by the caller. */ readonly adapterDefaults: LlmCallConfigAdapterDefaults /** @@ -171,6 +174,14 @@ export interface PreparedLlmCall { stream(options: GenerateOptions): AsyncIterable } +/** One adapter-owned model-resolution generation bound to its eventual stream call. */ +export interface PreparedAdapterCall { + /** Exact model metadata from the same adapter generation as {@link stream}. */ + readonly model: LlmResolvedModelInfo + /** Dispatch through that generation without re-reading dynamic connection facts. */ + stream(options: GenerateOptions): AsyncIterable +} + /** * Provider-wire adapter for the harness message and stream vocabulary. Register implementations * with `ctx.llm.registerAdapter(providers, adapter)`. Every provider HTTP request must include @@ -224,6 +235,22 @@ export abstract class LlmAdapter { return Promise.resolve({ provider, id: model, name: model }) } + /** + * Bind exact model metadata and the eventual request dispatch to one adapter generation. + * Dynamic adapters override this so settings changes between preparation and + * dispatch cannot combine one generation's capabilities with another's endpoint. + * @param provider - registered provider route. + * @param model - exact model id. + * @param signal - cancellation for model resolution. + * @returns model metadata and a one-generation stream entry point. + */ + async prepareCall(provider: string, model: string, signal?: AbortSignal): Promise { + return { + model: await this.resolveModel(provider, model, signal), + stream: options => this.stream(options), + } + } + /** * Stream one model call as raw chunks. The only required method. * @param options - the fully-assembled request; implementations must honor `options.signal`. @@ -629,8 +656,17 @@ export class LlmRuntime extends Service { model: string, signal?: AbortSignal, ): Promise { + const resolved = await registration.adapter.resolveModel(registration.provider.id, model, signal) + return this.normalizeModelInfo(registration, model, resolved) + } + + /** Validate and detach one adapter-returned exact model result. */ + private normalizeModelInfo( + registration: AdapterRegistration, + model: string, + resolved: LlmResolvedModelInfo, + ): LlmResolvedModelInfo { const provider = registration.provider.id - const resolved = await registration.adapter.resolveModel(provider, model, signal) if ( typeof resolved.provider !== 'string' || resolved.provider !== provider @@ -735,8 +771,16 @@ export class LlmRuntime extends Service { registration: AdapterRegistration, config: LlmCallConfig, signal?: AbortSignal, - ): Promise<{ config: LlmCallConfig; context?: LlmModelContext }> { + ): Promise<{ config: LlmCallConfig; context?: LlmModelContext; modelInfo: LlmResolvedModelInfo }> { const info = await this.resolveModelInfoFor(registration, config.model, signal) + return this.resolveCallWithInfo(config, info) + } + + /** Validate request controls against one already-bound exact model result. */ + private resolveCallWithInfo( + config: LlmCallConfig, + info: LlmResolvedModelInfo, + ): { config: LlmCallConfig; context?: LlmModelContext; modelInfo: LlmResolvedModelInfo } { const defaulted = config.maxTokens === undefined && info.defaultMaxTokens !== undefined ? { ...config, maxTokens: info.defaultMaxTokens } : config @@ -765,6 +809,7 @@ export class LlmRuntime extends Service { return { config: resolvedConfig, ...info.context === undefined ? {} : { context: info.context }, + modelInfo: info, } } @@ -778,7 +823,9 @@ export class LlmRuntime extends Service { */ async prepareCall(config: LlmCallConfig, signal?: AbortSignal): Promise { const registration = this.registration(config.provider) - const resolved = await this.resolveCallFor(registration, config, signal) + const adapterCall = await registration.adapter.prepareCall(config.provider, config.model, signal) + const modelInfo = this.normalizeModelInfo(registration, config.model, adapterCall.model) + const resolved = this.resolveCallWithInfo(config, modelInfo) const resolvedConfig = deepFreeze(structuredClone(resolved.config)) const context = resolved.context === undefined ? undefined @@ -797,6 +844,9 @@ export class LlmRuntime extends Service { retryPolicy: registration.retryPolicy, adapterDefaults, ...context === undefined ? {} : { context }, + ...modelInfo.inputModalities === undefined + ? {} + : { inputModalities: Object.freeze([...modelInfo.inputModalities]) }, stream: (options: GenerateOptions): AsyncIterable => { if (dispatched) { throw new LlmError('a prepared LLM call can only be dispatched once', 'INVALID_PREPARED_CALL') @@ -808,7 +858,12 @@ export class LlmRuntime extends Service { ) } dispatched = true - return this.streamWithRegistration(options, { registration, config: resolvedConfig }) + return this.streamWithRegistration(options, { + registration, + config: resolvedConfig, + modelInfo, + dispatch: options => adapterCall.stream(options), + }) }, }) } @@ -842,14 +897,25 @@ export class LlmRuntime extends Service { */ private async * adapterStream( options: GenerateOptions, - prepared?: { registration: AdapterRegistration; config: LlmCallConfig }, + prepared?: PreparedDispatch, ): AsyncGenerator { let iterator: AsyncIterator try { const registration = prepared?.registration ?? this.registration(options.provider) - const resolvedConfig = prepared === undefined - ? (await this.resolveCallFor(registration, options, options.signal)).config - : prepared.config + const adapter = registration.adapter + let modelInfo: LlmResolvedModelInfo + let resolvedConfig: LlmCallConfig + let dispatch: (options: GenerateOptions) => AsyncIterable + if (prepared === undefined) { + const adapterCall = await adapter.prepareCall(options.provider, options.model, options.signal) + modelInfo = this.normalizeModelInfo(registration, options.model, adapterCall.model) + resolvedConfig = this.resolveCallWithInfo(options, modelInfo).config + dispatch = options => adapterCall.stream(options) + } else { + modelInfo = prepared.modelInfo + resolvedConfig = prepared.config + dispatch = prepared.dispatch + } if (prepared !== undefined && !callConfigEquals(options, resolvedConfig)) { throw new LlmError( 'prepared LLM call config changed before adapter dispatch', @@ -861,8 +927,14 @@ export class LlmRuntime extends Service { : Object.isFrozen(options) ? deepFreeze({ ...options, ...resolvedConfig }) : { ...options, ...resolvedConfig } - const adapter = registration.adapter - const stream = adapter.stream(this.forAdapter(resolvedOptions, adapter)) + const projectedOptions = modelInfo.inputModalities !== undefined + && !modelInfo.inputModalities.includes('image') + && resolvedOptions.messages.some(message => contentHasImage(message.content)) + ? Object.isFrozen(resolvedOptions) + ? deepFreeze({ ...resolvedOptions, messages: projectImagesForTextModel(resolvedOptions.messages) as Message[] }) + : { ...resolvedOptions, messages: projectImagesForTextModel(resolvedOptions.messages) as Message[] } + : resolvedOptions + const stream = dispatch(this.forAdapter(projectedOptions, adapter)) iterator = stream[Symbol.asyncIterator]() } catch (error: unknown) { yield adapterFailureChunk(error, options.signal) @@ -916,7 +988,7 @@ export class LlmRuntime extends Service { private streamWithRegistration( options: GenerateOptions, - prepared?: { registration: AdapterRegistration; config: LlmCallConfig }, + prepared?: PreparedDispatch, ): AsyncIterable { return this.ctx.waterfall( this, @@ -944,4 +1016,11 @@ interface AdapterRegistration { readonly retryPolicy: ResolvedRetryPolicy } +interface PreparedDispatch { + readonly registration: AdapterRegistration + readonly config: LlmCallConfig + readonly modelInfo: LlmResolvedModelInfo + readonly dispatch: (options: GenerateOptions) => AsyncIterable +} + export default LlmRuntime diff --git a/packages/llm/llm/tests/content.spec.ts b/packages/llm/llm/tests/content.spec.ts index ffb5a586bf..d1b02fa011 100644 --- a/packages/llm/llm/tests/content.spec.ts +++ b/packages/llm/llm/tests/content.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import { AttachmentId } from '@deepseek-ai/dsh-attachment' -import { CallId, createUserMessage, OFFLOADED_IMAGE_TEXT, offloadRequestImages } from '../src/index.ts' +import { CallId, createUserMessage, OFFLOADED_IMAGE_TEXT, offloadRequestImages, offloadRequestImagesWithPolicy } from '../src/index.ts' import type { ContentBlock } from '../src/index.ts' const source = { kind: 'plugin' as const, plugin: 'test' } @@ -87,3 +87,33 @@ describe('offloadRequestImages', () => { ]) }) }) + +describe('offloadRequestImagesWithPolicy', () => { + it('drops 129 MiB to 64 MiB and keeps the removed prefix stable through 192 MiB', () => { + const mib = 1024 * 1024 + const project = (count: number) => offloadRequestImagesWithPolicy([ + createUserMessage({ content: Array.from({ length: count }, () => image(mib)), source }), + ], { + representation: 'raw', + maxBytes: 128 * mib, + byteQuantum: 64 * mib, + })[0]?.content + + expect(project(128)?.filter(block => block.type === 'image')).toHaveLength(128) + expect(project(129)?.filter(block => block.type === 'text')).toHaveLength(65) + expect(project(192)?.filter(block => block.type === 'text')).toHaveLength(65) + expect(project(193)?.filter(block => block.type === 'text')).toHaveLength(129) + }) + + it('rounds a count excess up to a 20-image removal step', () => { + const projected = offloadRequestImagesWithPolicy([ + createUserMessage({ content: Array.from({ length: 601 }, () => image(1)), source }), + ], { + representation: 'raw', + maxImages: 600, + countQuantum: 20, + }) + expect(projected[0]?.content.filter(block => block.type === 'text')).toHaveLength(20) + expect(projected[0]?.content.filter(block => block.type === 'image')).toHaveLength(581) + }) +}) diff --git a/packages/llm/llm/tests/service.spec.ts b/packages/llm/llm/tests/service.spec.ts index 45f523c129..b2e5399964 100644 --- a/packages/llm/llm/tests/service.spec.ts +++ b/packages/llm/llm/tests/service.spec.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' import { Context } from '@deepseek-ai/cordis' +import { AttachmentId } from '@deepseek-ai/dsh-attachment' import LlmRuntime, { errorChain, GenerateOptions, @@ -13,6 +14,7 @@ import LlmRuntime, { resolveRetryPolicy, StreamChunk, createMessage, + createUserMessage, } from '@deepseek-ai/dsh-llm' import type { LlmModelContext, @@ -915,6 +917,77 @@ describe('LlmRuntime', () => { expect(resolutions).toBe(2) }) + it('binds adapter-owned capabilities and dispatch to one prepared generation', async () => { + const ctx = new Context() + await ctx.plugin(LlmRuntime) + let generation = 'first' + let dispatched: string | undefined + const adapter = new class extends ScriptedAdapter { + override prepareCall(provider: string, model: string) { + const captured = generation + return Promise.resolve({ + model: { provider, id: model, name: model, inputModalities: ['text'] as const }, + stream: (options: GenerateOptions) => { + dispatched = captured + return super.stream(options) + }, + }) + } + }(SCRIPT) + ctx.llm.registerAdapter(['route'], adapter) + + const prepared = await ctx.llm.prepareCall({ provider: 'route', model: 'model' }) + generation = 'second' + expect(prepared.inputModalities).toEqual(['text']) + expect(Object.isFrozen(prepared.inputModalities)).toBe(true) + await collect(prepared.stream({ ...prepared.config, messages: [] })) + expect(dispatched).toBe('first') + }) + + it('projects historical images to stable text only after the loop-visible waterfall', async () => { + const ctx = new Context() + await ctx.plugin(LlmRuntime) + const seen: GenerateOptions[] = [] + const adapter = new class extends ScriptedAdapter { + override resolveModel(provider: string, model: string): Promise { + return Promise.resolve({ provider, id: model, name: model, inputModalities: ['text'] }) + } + + override async * stream(options: GenerateOptions): AsyncIterable { + seen.push(options) + yield * super.stream(options) + } + }(SCRIPT) + ctx.llm.registerAdapter(['route'], adapter) + const attachment = { + attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`), + mediaType: 'image/png' as const, + bytes: 3, + width: 1, + height: 1, + } + const waterfall: GenerateOptions[] = [] + ctx.on('llm/stream', async function* (options, next) { + waterfall.push(options) + yield * next() + }) + + await collect(ctx.llm.stream({ + provider: 'route', + model: 'text-only', + messages: [createUserMessage({ + content: [{ type: 'image', attachment }], + source: { kind: 'plugin', plugin: 'test' }, + })], + })) + + expect(waterfall[0]?.messages[0]?.content).toEqual([{ type: 'image', attachment }]) + expect(seen[0]?.messages[0]?.content).toEqual([{ + type: 'text', + text: '[image omitted because this model accepts text only; attachment sha256:aaaaaaaa]', + }]) + }) + it('passes cancellation through exact-model resolution', async () => { const ctx = new Context() await ctx.plugin(LlmRuntime) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d190033b4f..854a86886c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5504,12 +5504,21 @@ importers: '@deepseek-ai/dsh-anonymous-user-id': specifier: workspace:^ version: link:../../identity/anonymous-user-id + '@deepseek-ai/dsh-atomic-write': + specifier: workspace:^ + version: link:../../util/atomic-write '@deepseek-ai/dsh-attachment': specifier: workspace:^ version: link:../../attachment/attachment + '@deepseek-ai/dsh-brand': + specifier: workspace:^ + version: link:../../util/brand '@deepseek-ai/dsh-credentials': specifier: workspace:^ version: link:../../credentials/credentials + '@deepseek-ai/dsh-home-paths': + specifier: workspace:^ + version: link:../../util/home-paths '@deepseek-ai/dsh-invariants': specifier: workspace:^ version: link:../../runtime-diagnostics/invariants diff --git a/scripts/gen-cordis-catalog.ts b/scripts/gen-cordis-catalog.ts index 255ff45001..558dafca6b 100644 --- a/scripts/gen-cordis-catalog.ts +++ b/scripts/gen-cordis-catalog.ts @@ -293,6 +293,9 @@ export const LINK_MAP: Readonly> = { ApprovalService: 'approval.md', EncodedImageAttachment: 'attachment.md', ImageAttachmentRef: 'attachment.md', + ImageRequestPolicy: 'attachment.md', + PreviewImageCrop: 'attachment.md', + RequestImageAttachment: 'attachment.md', SaveImageAttachment: 'attachment.md', SavedImageAttachment: 'attachment.md', SourceImageInfo: 'attachment.md', diff --git a/scripts/gen-tool-catalog.ts b/scripts/gen-tool-catalog.ts index 87eee0a7e4..48ba2255a5 100644 --- a/scripts/gen-tool-catalog.ts +++ b/scripts/gen-tool-catalog.ts @@ -314,18 +314,18 @@ const TOOL_PACKAGES: ToolPackage[] = [ pkg: '@deepseek-ai/dsh-tool-fs', dir: 'tool-fs', source: 'packages/fs/tool-fs/src/index.ts', - requires: ['ctx.tools', 'ctx.fs', 'ctx.systemPrompt', 'ctx.attachments (read_image registration)', 'ctx.llm + an image-capable route (read_image execution)'], - writes: ['tool/call', 'fs/write-intent or fs/edit-intent for mutations', 'fs/observed after read presence/absence or successful file operation', 'durable attachment (read_image)', 'tool/result'], + requires: ['ctx.tools', 'ctx.fs', 'ctx.systemPrompt', 'ctx.attachments (image-tool registration)', 'ctx.llm + an image-capable route (image-tool execution)'], + writes: ['tool/call', 'fs/write-intent or fs/edit-intent for mutations', 'fs/observed after read presence/absence or successful file operation', 'durable attachment (read_image and read_image_region)', 'tool/result'], async mount(ctx) { // The tool needs `fs`; the bare provider is sufficient because policy // changes behavior, not schema shape. The catalog seam marker opts into - // the attachments-conditional read_image schema without attachment I/O. + // both attachments-conditional image schemas without attachment I/O. await ctx.plugin(LocalFileSystem) await ctx.plugin(CatalogAttachmentStore) await ctx.plugin(ToolFs) }, note: - 'The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-observation-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. `read_image` is not registered without `ctx.attachments`; its schema is route-independent, and execution refuses unless the exact routed model declares image input.', + 'The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-observation-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. The image tools are not registered without `ctx.attachments`; their schemas are route-independent, and execution refuses unless the exact routed model declares image input.', }, { pkg: '@deepseek-ai/dsh-tool-fs-search', diff --git a/scripts/type-equiv.manifest.json b/scripts/type-equiv.manifest.json index 8b580750bd..946015d483 100644 --- a/scripts/type-equiv.manifest.json +++ b/scripts/type-equiv.manifest.json @@ -930,6 +930,26 @@ "symbol": "StoredImageAttachment", "source": "packages/attachment/attachment/src/types.ts" }, + { + "doc": "docs/subsystems/attachment.md", + "symbol": "MasterImageCrop", + "source": "packages/attachment/attachment/src/types.ts" + }, + { + "doc": "docs/subsystems/attachment.md", + "symbol": "ImageRequestPolicy", + "source": "packages/attachment/attachment/src/types.ts" + }, + { + "doc": "docs/subsystems/attachment.md", + "symbol": "PreviewImageCrop", + "source": "packages/attachment/attachment/src/types.ts" + }, + { + "doc": "docs/subsystems/attachment.md", + "symbol": "RequestImageAttachment", + "source": "packages/attachment/attachment/src/types.ts" + }, { "doc": "docs/subsystems/shell.md", "symbol": "ShellExecRequest", From c0dd8ec820cd5abd5be0345f750b957a1e926ac5 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 18:25:37 +0800 Subject: [PATCH 12/28] chore(images): align merged runtime closure --- docs/subsystems/attachment.i18n.yaml | 4 ++-- docs/subsystems/attachment.md | 4 ++-- docs/subsystems/attachment.zh.md | 4 ++-- packages/extensions/tool-cordis/src/api-catalog.ts | 4 ++-- packages/llm/llm-pi-ai/tests/adapter.spec.ts | 1 + pnpm-lock.yaml | 3 +++ python/sdk-runtime/package.json | 1 + 7 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/subsystems/attachment.i18n.yaml b/docs/subsystems/attachment.i18n.yaml index 7c236d6480..55a43dd247 100644 --- a/docs/subsystems/attachment.i18n.yaml +++ b/docs/subsystems/attachment.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/subsystems/attachment.md -attachment.md: cdbb528d30eabc74a9c3607d67e91af053c45e7e -attachment.zh.md: 79ee753d22c3adecaca153659181e113f2b3e728 +attachment.md: ec9d1f27bdde4a4d5b6e6e7328260bcb4af49948 +attachment.zh.md: e79c2df4ca168bcae4fd45e61812a4f86ce2194b diff --git a/docs/subsystems/attachment.md b/docs/subsystems/attachment.md index cdbb528d30..ec9d1f27bd 100644 --- a/docs/subsystems/attachment.md +++ b/docs/subsystems/attachment.md @@ -204,7 +204,7 @@ abstract readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise +readImageRequest( ref: ImageAttachmentRef, policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise /** * Generate or read an ordered batch of deterministic model-request versions. @@ -223,7 +223,7 @@ async readImageRequests( refs: readonly ImageAttachmentRef[], policy: ImageReque * @param signal - optional cancellation. * @returns a new durable attachment reference suitable for a logged tool result. */ -async cropImage( ref: ImageAttachmentRef, crop: PreviewImageCrop, signal?: AbortSignal, ): Promise +cropImage( ref: ImageAttachmentRef, crop: PreviewImageCrop, signal?: AbortSignal, ): Promise ``` Source: [`packages/attachment/attachment/src/index.ts`](../../packages/attachment/attachment/src/index.ts) diff --git a/docs/subsystems/attachment.zh.md b/docs/subsystems/attachment.zh.md index 79ee753d22..e79c2df4ca 100644 --- a/docs/subsystems/attachment.zh.md +++ b/docs/subsystems/attachment.zh.md @@ -204,7 +204,7 @@ abstract readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise +readImageRequest( ref: ImageAttachmentRef, policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise /** * Generate or read an ordered batch of deterministic model-request versions. @@ -223,7 +223,7 @@ async readImageRequests( refs: readonly ImageAttachmentRef[], policy: ImageReque * @param signal - optional cancellation. * @returns a new durable attachment reference suitable for a logged tool result. */ -async cropImage( ref: ImageAttachmentRef, crop: PreviewImageCrop, signal?: AbortSignal, ): Promise +cropImage( ref: ImageAttachmentRef, crop: PreviewImageCrop, signal?: AbortSignal, ): Promise ``` Source: [`packages/attachment/attachment/src/index.ts`](../../packages/attachment/attachment/src/index.ts) diff --git a/packages/extensions/tool-cordis/src/api-catalog.ts b/packages/extensions/tool-cordis/src/api-catalog.ts index ad5d04fd5d..1602c351b5 100644 --- a/packages/extensions/tool-cordis/src/api-catalog.ts +++ b/packages/extensions/tool-cordis/src/api-catalog.ts @@ -456,7 +456,7 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ throws: ['the signal reason when aborted, or a storage error when verification fails.'], }, { - signature: 'async readImageRequest( ref: ImageAttachmentRef, policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise', + signature: 'readImageRequest( ref: ImageAttachmentRef, policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise', description: 'Generate or read one deterministic model-request version from the stored master image.', parameters: [{ name: 'ref', description: 'durable provider-independent master reference.' }, { name: 'policy', description: 'exact route pixel and encoded-byte budget.' }, { name: 'signal', description: 'optional cancellation.' }], returns: 'request bytes and the cache/upload identity covering every transform input.', @@ -468,7 +468,7 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ returns: 'request versions in the same order as `refs`.', }, { - signature: 'async cropImage( ref: ImageAttachmentRef, crop: PreviewImageCrop, signal?: AbortSignal, ): Promise', + signature: 'cropImage( ref: ImageAttachmentRef, crop: PreviewImageCrop, signal?: AbortSignal, ): Promise', description: 'Crop the stored master by coordinates measured on a model request preview and persist the result.', parameters: [{ name: 'ref', description: 'session-authorized master attachment.' }, { name: 'crop', description: 'preview dimensions and preview-coordinate rectangle.' }, { name: 'signal', description: 'optional cancellation.' }], returns: 'a new durable attachment reference suitable for a logged tool result.', diff --git a/packages/llm/llm-pi-ai/tests/adapter.spec.ts b/packages/llm/llm-pi-ai/tests/adapter.spec.ts index 171f898db9..416802e246 100644 --- a/packages/llm/llm-pi-ai/tests/adapter.spec.ts +++ b/packages/llm/llm-pi-ai/tests/adapter.spec.ts @@ -89,6 +89,7 @@ describe('PiAiAdapter provider routing', () => { ctx.llm.registerAdapter(['deepseek'], new PiAiAdapter({ profiles: () => resolveProfiles(providers), resolveApiKey: () => Promise.resolve('test-key'), + auth: memoryAuth(), })) const prepared = await ctx.llm.prepareCall({ provider: 'deepseek', model: 'deepseek-v4-flash' }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 854a86886c..ca87caa7d9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8729,6 +8729,9 @@ importers: '@deepseek-ai/dsh-app-boot': specifier: workspace:^ version: link:../../packages/boot/app-boot + '@deepseek-ai/dsh-atomic-write': + specifier: workspace:^ + version: link:../../packages/util/atomic-write '@deepseek-ai/dsh-attachment': specifier: workspace:^ version: link:../../packages/attachment/attachment diff --git a/python/sdk-runtime/package.json b/python/sdk-runtime/package.json index befad374c1..abf3e11a78 100644 --- a/python/sdk-runtime/package.json +++ b/python/sdk-runtime/package.json @@ -17,6 +17,7 @@ "@deepseek-ai/dsh-agent-tool-presentation": "workspace:^", "@deepseek-ai/dsh-app-boot": "workspace:^", "@deepseek-ai/dsh-attachment": "workspace:^", + "@deepseek-ai/dsh-atomic-write": "workspace:^", "@deepseek-ai/dsh-shell": "workspace:^", "@deepseek-ai/dsh-shell-env": "workspace:^", "@deepseek-ai/dsh-bash-local": "workspace:^", From c09a42ccb51d136e214241164ebd2f91a9419ba9 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 18:44:40 +0800 Subject: [PATCH 13/28] fix(images): parse listed missing Files ids --- ...0-unified-image-request-pipeline.i18n.yaml | 4 +- ...26-08-20-unified-image-request-pipeline.md | 4 +- ...08-20-unified-image-request-pipeline.zh.md | 4 +- packages/llm/llm-deepseek/README.i18n.yaml | 4 +- packages/llm/llm-deepseek/README.md | 2 +- packages/llm/llm-deepseek/README.zh.md | 2 +- packages/llm/llm-deepseek/src/adapter.ts | 25 +- .../llm/llm-deepseek/tests/adapter.spec.ts | 215 +++++++++++++++++- 8 files changed, 238 insertions(+), 22 deletions(-) diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml index 53c15e1755..4f1b156e3a 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md -2026-08-20-unified-image-request-pipeline.md: c487f583e4770b8d495404f08de67fd877dc48fd -2026-08-20-unified-image-request-pipeline.zh.md: a82312d55ba59403e71e97ee483e2b5bbfebfb03 +2026-08-20-unified-image-request-pipeline.md: 72382b6130086ba5c36d386ffe7ebe413cd2243d +2026-08-20-unified-image-request-pipeline.zh.md: 15560ad475af669cc4a2d9c46354a4da08528e0b diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md index c487f583e4..72382b6130 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md @@ -36,7 +36,7 @@ Every retained request image is preceded by its complete attachment id, actual r The direct `deepseek-official` adapter uploads every retained request version through the OpenAI-compatible Files API and sends only `file_id` content blocks. There is no inline fallback. The default catalog advertises `deepseek-v4-flash-vision-exp` as image-capable. Uploaded ids are indexed by endpoint and API-key scope plus `variantId`. Uploads request seven days by default and record the returned `expires_at`; a mapping with no more than one hour remaining is replaced without a preceding retrieve call. The index never stores the API key. -An upload is indexed only after the response returns a complete file object, matching byte count, and `expires_at`. A missing or inconsistent response leaves no local mapping, so a later request uploads again. A malformed upload index is an empty cache and is replaced on the next successful upload; filesystem I/O failures remain errors. If chat reports an expired, deleted, missing, or invalid id and names one used id, only that mapping is removed. A stale-file response without a specific id removes every mapping used by that chat attempt. The affected request bytes are uploaded again and chat is retried once. A second stale rejection clears the mappings identified by its response and returns the error without a third chat attempt. One upload quota error deletes the configured number of oldest harness-owned `dsh-` files and retries once. Public file operations expose list, retrieve, delete, one-variant release, and namespace-wide release. The client enforces the documented 128MiB upload limit, 32MiB chat-image limit, 10,000-file and 25GiB quotas, and one-hour to 30-day expiry range. +An upload is indexed only after the response returns a complete file object, matching byte count, and `expires_at`. A missing or inconsistent response leaves no local mapping, so a later request uploads again. A malformed upload index is an empty cache and is replaced on the next successful upload; filesystem I/O failures remain errors. If chat reports expired, deleted, missing, or invalid ids and names one or more ids used by the request, only those mappings are removed. A stale-file response without a specific id removes every mapping used by that chat attempt. The affected request bytes are uploaded again and chat is retried once. A second stale rejection clears the mappings identified by its response and returns the error without a third chat attempt. One upload quota error deletes the configured number of oldest harness-owned `dsh-` files and retries once. Public file operations expose list, retrieve, delete, one-variant release, and namespace-wide release. The client enforces the documented 128MiB upload limit, 32MiB chat-image limit, 10,000-file and 25GiB quotas, and one-hour to 30-day expiry range. ### Diagnostics @@ -64,7 +64,7 @@ Historical attachment objects that later disappear or fail integrity verificatio ## Verification -Package tests generate 16-bit RGB and RGBA PNG fixtures, prove 8-bit conversion and clean 8-bit passthrough, retain alpha under byte pressure, distinguish high-frequency and ordinary photos from low-color graphics, stop lazy encoding after the first fitting candidate, cover square and wide 640,000-pixel projections, enforce 1MiB request bytes, singleflight equal variants, bound transform concurrency, preserve cache and upload identity, map preview crops to the master, prepare batches once, reject inconsistent Files responses, refresh near-expiry ids without retrieve, recover once from exact and ambiguous stale-id responses, delete quota files, normalize provider diagnostics, project text-only history, and share normal/compaction request bytes. Keyless assembled snapshots cover the real tool schemas and image request path. A credentialed test uses the built-in `deepseek-official` route and its configured endpoint, never a custom provider entry. +Package tests generate 16-bit RGB and RGBA PNG fixtures, prove 8-bit conversion and clean 8-bit passthrough, retain alpha under byte pressure, distinguish high-frequency and ordinary photos from low-color graphics, stop lazy encoding after the first fitting candidate, cover square and wide 640,000-pixel projections, enforce 1MiB request bytes, singleflight equal variants, bound transform concurrency, preserve cache and upload identity, map preview crops to the master, prepare batches once, reject inconsistent Files responses, refresh near-expiry ids without retrieve, recover once from single-id, multiple-id, and ambiguous stale responses, delete quota files, normalize provider diagnostics, project text-only history, and share normal/compaction request bytes. Keyless assembled snapshots cover the real tool schemas and image request path. A credentialed test uses the built-in `deepseek-official` route and its configured endpoint, never a custom provider entry. ## Consequences diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md index a82312d55b..15560ad475 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md @@ -36,7 +36,7 @@ Status: implemented 直接 `deepseek-official` 适配器通过 OpenAI 兼容 Files API 上传每张保留的请求版本,只发送 `file_id` 内容块,不提供内联回退。默认 catalog 把 `deepseek-v4-flash-vision-exp` 公布为支持图片。上传 ID 按端点和 API key 作用域以及 `variantId` 写入索引。上传默认请求 7 天有效期,并记录返回的 `expires_at`;本地映射剩余时间不超过一小时时会直接替换,不会先查询远端文件。索引绝不存储 API key。 -只有上传响应返回完整文件对象、匹配的字节数和 `expires_at` 时,上传结果才会写入索引。缺失或不一致的响应不会留下本地映射,后续请求会重新上传。格式损坏的上传索引按空缓存处理,并在下一次成功上传时替换;文件系统 I/O 失败仍是错误。如果 chat 报告 ID 已过期、删除、缺失或无效,并指出本次请求使用的某个 ID,适配器只删除该映射。如果响应只说明文件状态失效而没有指出具体 ID,适配器会删除该次 chat 使用的全部映射。受影响的请求字节会重新上传,chat 只重试一次。第二次仍报告文件失效时,适配器会按响应清理映射并返回错误,不会发起第三次 chat。一次上传配额错误会删除配置数量的最旧 `dsh-` 文件,然后重试一次。公开文件操作提供列表、查询、删除、单个变体释放和整个作用域释放。客户端执行文档规定的 Files 单次上传 128MiB、chat 单图 32MiB、10,000 个文件、25GiB,以及一小时到 30 天有效期限制。 +只有上传响应返回完整文件对象、匹配的字节数和 `expires_at` 时,上传结果才会写入索引。缺失或不一致的响应不会留下本地映射,后续请求会重新上传。格式损坏的上传索引按空缓存处理,并在下一次成功上传时替换;文件系统 I/O 失败仍是错误。如果 chat 报告 ID 已过期、删除、缺失或无效,并指出本次请求使用的一个或多个 ID,适配器只删除这些映射。如果响应只说明文件状态失效而没有指出具体 ID,适配器会删除该次 chat 使用的全部映射。受影响的请求字节会重新上传,chat 只重试一次。第二次仍报告文件失效时,适配器会按响应清理映射并返回错误,不会发起第三次 chat。一次上传配额错误会删除配置数量的最旧 `dsh-` 文件,然后重试一次。公开文件操作提供列表、查询、删除、单个变体释放和整个作用域释放。客户端执行文档规定的 Files 单次上传 128MiB、chat 单图 32MiB、10,000 个文件、25GiB,以及一小时到 30 天有效期限制。 ### 诊断 @@ -64,7 +64,7 @@ Status: implemented ## Verification -包测试会生成 16-bit RGB 和 RGBA PNG fixture,验证 8-bit 转换与干净 8-bit 字节直通、字节压力下保留透明通道、区分高频和普通照片与低色数图形、首个候选合规后停止编码、正方形和宽屏 640,000 像素投影、请求字节不超过 1MiB、相同变体 singleflight、变换并发上限、缓存与上传身份、预览到主版本坐标映射、批量只准备一次、Files 响应不一致、进入刷新余量时不查询远端并更新 ID、精确和模糊失效响应只恢复一次、配额删除、规范化提供方诊断、纯文本投影,以及普通请求与压缩共享请求字节。无需密钥的组装快照覆盖真实工具 schema 和图片请求路径。使用凭据的测试只使用内置 `deepseek-official` 路由及其已配置端点,不使用自定义提供方条目。 +包测试会生成 16-bit RGB 和 RGBA PNG fixture,验证 8-bit 转换与干净 8-bit 字节直通、字节压力下保留透明通道、区分高频和普通照片与低色数图形、首个候选合规后停止编码、正方形和宽屏 640,000 像素投影、请求字节不超过 1MiB、相同变体 singleflight、变换并发上限、缓存与上传身份、预览到主版本坐标映射、批量只准备一次、Files 响应不一致、进入刷新余量时不查询远端并更新 ID、单个 ID、多个 ID 和模糊失效响应只恢复一次、配额删除、规范化提供方诊断、纯文本投影,以及普通请求与压缩共享请求字节。无需密钥的组装快照覆盖真实工具 schema 和图片请求路径。使用凭据的测试只使用内置 `deepseek-official` 路由及其已配置端点,不使用自定义提供方条目。 ## Consequences diff --git a/packages/llm/llm-deepseek/README.i18n.yaml b/packages/llm/llm-deepseek/README.i18n.yaml index c5aa0c7e8c..5d0e91eae1 100644 --- a/packages/llm/llm-deepseek/README.i18n.yaml +++ b/packages/llm/llm-deepseek/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/llm/llm-deepseek/README.md -README.md: da2044abe6f5201c1bed1ca6b529b34c34282ea8 -README.zh.md: d17d7a739640c31e9e88f154a11d5e24011e54f7 +README.md: ea82956d77f3157638aa078c56630994ccc61d75 +README.zh.md: ff8780f59a3caac7e08e5ab6e08c4b2b15d1b57d diff --git a/packages/llm/llm-deepseek/README.md b/packages/llm/llm-deepseek/README.md index da2044abe6..ea82956d77 100644 --- a/packages/llm/llm-deepseek/README.md +++ b/packages/llm/llm-deepseek/README.md @@ -52,7 +52,7 @@ An image-capable catalog entry declares `inputModalities: [text, image]` and may `maxRequestFilesBytes` and `maxImagesPerRequest` bound the retained request versions at 128MiB and 600 images by default. When the byte bound is crossed, the oldest prefix advances past the next 64MiB boundary; 129 one-megabyte images remove the oldest 65 and retain 64MiB, and that prefix stays unchanged until durable history exceeds 192MiB. Count overflow advances independently in `imageOffloadCountQuantum` steps. Removed images become the fixed model-visible placeholder `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`. This high-watermark projection avoids changing an old request prefix after every new image. -Uploaded ids are indexed below `DSH_HOME` by endpoint/API-key scope and request `variantId`. The variant covers the master attachment id, transform version, route pixel and byte budgets, crop, and encoder parameters, so Files API and inline-capable adapters refer to the same deterministic bytes. Uploads request a seven-day lifetime by default and store the server's `expires_at`. A local mapping with no more than one hour remaining is replaced before use; the adapter does not retrieve every remote file before chat. If chat reports an expired, deleted, missing, or invalid file id and names a used id, the adapter removes only that mapping. If the provider identifies stale file state without naming an id, it removes every file mapping used by that chat attempt. It then uploads the affected request versions again and retries chat once. A second stale-file rejection clears the mappings identified by that response and is returned without a third chat attempt. An upload response without a complete file object, matching byte count, and `expires_at` is never indexed; a later request therefore uploads again instead of trusting inconsistent local state. A malformed local upload index is treated as an empty cache and replaced by the next successful upload; permission and filesystem I/O failures still fail the request. +Uploaded ids are indexed below `DSH_HOME` by endpoint/API-key scope and request `variantId`. The variant covers the master attachment id, transform version, route pixel and byte budgets, crop, and encoder parameters, so Files API and inline-capable adapters refer to the same deterministic bytes. Uploads request a seven-day lifetime by default and store the server's `expires_at`. A local mapping with no more than one hour remaining is replaced before use; the adapter does not retrieve every remote file before chat. If chat reports expired, deleted, missing, or invalid file ids and names one or more ids used by the request, the adapter removes exactly those mappings. If the provider identifies stale file state without naming an id, it removes every file mapping used by that chat attempt. It then uploads the affected request versions again and retries chat once. A second stale-file rejection clears the mappings identified by that response and is returned without a third chat attempt. An upload response without a complete file object, matching byte count, and `expires_at` is never indexed; a later request therefore uploads again instead of trusting inconsistent local state. A malformed local upload index is treated as an empty cache and replaced by the next successful upload; permission and filesystem I/O failures still fail the request. One quota upload failure triggers deletion of the configured number of oldest `dsh-` files and one upload retry. `DeepSeekFilesClient.delete`, `DeepSeekFileStore.release`, and `releaseAll` expose explicit remote-space reclamation. The current provider limits represented by this package are 128MiB per Files upload, 32MiB per chat-referenced image, 10,000 stored files, and 25GiB per API key; the default 1MiB request version remains below the two per-file limits. diff --git a/packages/llm/llm-deepseek/README.zh.md b/packages/llm/llm-deepseek/README.zh.md index d17d7a7396..ff8780f59a 100644 --- a/packages/llm/llm-deepseek/README.zh.md +++ b/packages/llm/llm-deepseek/README.zh.md @@ -52,7 +52,7 @@ harness LLM(大语言模型)seam 的 DeepSeek chat-completions 适配器: `maxRequestFilesBytes` 和 `maxImagesPerRequest` 限制请求中保留的请求版本,默认值分别为 128MiB 和 600 张。字节数越过上限时,被移除的最旧前缀会越过下一个 64MiB 边界。由 1MiB 图片组成的历史达到 129MiB 时会移除最旧的 65 张并保留 64MiB;直到持久历史超过 192MiB,这个前缀才再次变化。图片数量超限时则按 `imageOffloadCountQuantum` 独立递增。移除的图片会变成固定模型可见占位文本 `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`。这种定量投影不会因每新增一张图片就改写较早的请求前缀。 -上传 ID 按端点和 API key 作用域以及请求 `variantId` 记录在 `DSH_HOME` 下。变体身份覆盖主附件 ID、变换策略版本、路由像素和字节预算、裁剪区域及编码参数,因此 Files API 和支持内联的适配器引用同一份确定性字节。上传默认请求 7 天有效期,并保存服务端返回的 `expires_at`。本地映射剩余时间不超过一小时时会在使用前替换;适配器不会在每次 chat 前查询远端文件。如果 chat 报告文件 ID 已过期、删除、缺失或无效,并指出本次请求使用的某个 ID,适配器只删除该映射。如果响应只说明文件状态失效而没有指出 ID,适配器会删除该次 chat 使用的全部文件映射。随后重新上传受影响的请求版本,并重试一次 chat。第二次 chat 仍报告文件失效时,适配器会按该响应清理映射并返回错误,不会发起第三次 chat。上传响应若没有完整文件对象、匹配的字节数和 `expires_at`,就不会写入索引;后续请求会再次上传,而不是信任不一致的本地状态。本地上传索引格式损坏时按空缓存处理,并由下一次成功上传替换;权限和文件系统 I/O 错误仍使请求失败。 +上传 ID 按端点和 API key 作用域以及请求 `variantId` 记录在 `DSH_HOME` 下。变体身份覆盖主附件 ID、变换策略版本、路由像素和字节预算、裁剪区域及编码参数,因此 Files API 和支持内联的适配器引用同一份确定性字节。上传默认请求 7 天有效期,并保存服务端返回的 `expires_at`。本地映射剩余时间不超过一小时时会在使用前替换;适配器不会在每次 chat 前查询远端文件。如果 chat 报告文件 ID 已过期、删除、缺失或无效,并指出本次请求使用的一个或多个 ID,适配器只删除这些映射。如果响应只说明文件状态失效而没有指出 ID,适配器会删除该次 chat 使用的全部文件映射。随后重新上传受影响的请求版本,并重试一次 chat。第二次 chat 仍报告文件失效时,适配器会按该响应清理映射并返回错误,不会发起第三次 chat。上传响应若没有完整文件对象、匹配的字节数和 `expires_at`,就不会写入索引;后续请求会再次上传,而不是信任不一致的本地状态。本地上传索引格式损坏时按空缓存处理,并由下一次成功上传替换;权限和文件系统 I/O 错误仍使请求失败。 一次上传配额错误会触发删除配置数量的最旧 `dsh-` 文件,然后重试一次上传。`DeepSeekFilesClient.delete`、`DeepSeekFileStore.release` 和 `releaseAll` 提供主动远端空间回收。本包记录的当前提供方限制为 Files 单次上传 128MiB、chat 单图引用 32MiB、每个 API key 最多 10,000 个文件和 25GiB;默认 1MiB 请求版本低于两个单文件上限。 diff --git a/packages/llm/llm-deepseek/src/adapter.ts b/packages/llm/llm-deepseek/src/adapter.ts index 8d9381c67f..006882d745 100644 --- a/packages/llm/llm-deepseek/src/adapter.ts +++ b/packages/llm/llm-deepseek/src/adapter.ts @@ -171,13 +171,22 @@ function collectImageRefs( } } -function requestImagePolicy(model: DeepSeekCatalogModel): ImageRequestPolicy { +/** + * Resolve the request-image budgets owned by one DeepSeek model route. + * @param model - Advertised model route and its optional image overrides. + * @returns Complete pixel and encoded-byte budgets. + * @internal + */ +export function resolveRequestImagePolicy(model: DeepSeekCatalogModel): ImageRequestPolicy { + let maxPixels: number + if (model.imagePixelBudget !== undefined) maxPixels = model.imagePixelBudget + else if (model.imageDetail === 'low') maxPixels = DEFAULT_LOW_DETAIL_IMAGE_PIXEL_BUDGET + else maxPixels = DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET return { - maxPixels: model.imagePixelBudget - ?? (model.imageDetail === 'low' - ? DEFAULT_LOW_DETAIL_IMAGE_PIXEL_BUDGET - : DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET), - maxBytes: model.imageMaxBytes ?? DEFAULT_REQUEST_IMAGE_MAX_BYTES, + maxPixels, + maxBytes: model.imageMaxBytes === undefined + ? DEFAULT_REQUEST_IMAGE_MAX_BYTES + : model.imageMaxBytes, } } @@ -189,7 +198,7 @@ async function prepareRequestImages( ): Promise> { const refs = new Map() for (const message of options.messages) collectImageRefs(message.content, refs) - const policy = requestImagePolicy(model) + const policy = resolveRequestImagePolicy(model) const orderedRefs = [...refs.values()] const projected = await attachments.readImageRequests(orderedRefs, policy, signal) return new Map(orderedRefs.map((ref, index) => ( @@ -211,7 +220,7 @@ interface UsedRequestFile { function providerRejectedFileId(detail: string): boolean { const file = /\bfile(?:[_ -]?(?:id|api|not[_ -]?found|deleted|expired))?/iu.test(detail) - const missing = /(?:expired|not[_ -]?found|deleted|does not exist)/iu.test(detail) + const missing = /(?:expired|not[_ -]?found|deleted|do(?:es)? not exist|not created under (?:this|your) account)/iu.test(detail) const invalidId = /(?:invalid.{0,20}file[_ -]?(?:id|api)|file[_ -]?(?:id|api).{0,20}invalid)/iu.test(detail) return file && (missing || invalidId) } diff --git a/packages/llm/llm-deepseek/tests/adapter.spec.ts b/packages/llm/llm-deepseek/tests/adapter.spec.ts index e37a1a882e..23db2ed009 100644 --- a/packages/llm/llm-deepseek/tests/adapter.spec.ts +++ b/packages/llm/llm-deepseek/tests/adapter.spec.ts @@ -6,7 +6,7 @@ import { Context } from '@deepseek-ai/cordis' import { AttachmentId, ImageVariantId } from '@deepseek-ai/dsh-attachment' import type { AttachmentStore, ImageAttachmentRef, RequestImageAttachment } from '@deepseek-ai/dsh-attachment' import { createLaunchEnvironmentSnapshot } from '@deepseek-ai/dsh-launch-environment' -import LlmRuntime, { createUserMessage, +import LlmRuntime, { CallId, createUserMessage, CONTEXT_WINDOW_EXCEEDED_CODE, ProviderRequestId, QUOTA_EXCEEDED_CODE, @@ -18,7 +18,7 @@ import { getOrCreateAnonymousUserId, type AnonymousUserId } from '@deepseek-ai/d import { SessionId } from '@deepseek-ai/dsh-session' import * as LlmDeepSeek from '@deepseek-ai/dsh-llm-deepseek' import { DeepSeekAdapter, resolveAdapterOptions } from '@deepseek-ai/dsh-llm-deepseek' -import { httpErrorCode } from '../src/adapter.ts' +import { httpErrorCode, resolveRequestImagePolicy } from '../src/adapter.ts' import { assemble } from './assemble.ts' import { closeMockServers, mockServer, textEvents } from './mock-server.ts' import type { Behavior } from './mock-server.ts' @@ -109,6 +109,25 @@ function attachmentStoreOf( } } +describe('request image policy', () => { + it.each([ + [ + { id: 'default' }, + { maxPixels: 640_000, maxBytes: 1024 * 1024 }, + ], + [ + { id: 'low', imageDetail: 'low' as const }, + { maxPixels: 512 * 512, maxBytes: 1024 * 1024 }, + ], + [ + { id: 'custom', imagePixelBudget: 320_000, imageMaxBytes: 512_000 }, + { maxPixels: 320_000, maxBytes: 512_000 }, + ], + ])('resolves route-owned defaults and overrides for %s', (model, expected) => { + expect(resolveRequestImagePolicy(model)).toEqual(expected) + }) +}) + describe('DeepSeekAdapter against a mock server', () => { it('streams a text generation end to end through the assembler', async () => { const server = await mockServer([{ kind: 'sse', events: textEvents }]) @@ -187,6 +206,54 @@ describe('DeepSeekAdapter against a mock server', () => { expect(policies).toEqual([{ maxPixels: 640_000, maxBytes: 1024 * 1024 }]) }) + it('projects nested tool-result images with route-owned request budgets', async () => { + const server = await mockServer([ + { kind: 'sse', events: textEvents }, + { kind: 'sse', events: textEvents }, + ]) + const attachmentMocks = attachmentStoreOf(ref => Promise.resolve(requestImage(ref))) + const adapter = adapterOf({ + baseURL: server.url, + models: [ + { + id: 'vision-low', + inputModalities: ['text', 'image'], + imageDetail: 'low', + imageMaxBytes: 512_000, + }, + { + id: 'vision-custom', + inputModalities: ['text', 'image'], + imagePixelBudget: 320_000, + }, + ], + }, attachmentMocks.store) + const nested = createUserMessage({ + content: [{ + type: 'tool-result', + toolCallId: CallId('image-result'), + content: [{ type: 'image', attachment: imageRef }], + }], + source: { kind: 'plugin', plugin: 'test' }, + }) + + await drain(adapter.stream({ provider: 'deepseek-official', model: 'vision-low', messages: [nested] })) + await drain(adapter.stream({ provider: 'deepseek-official', model: 'vision-custom', messages: [nested] })) + + expect(attachmentMocks.readImageRequests).toHaveBeenNthCalledWith( + 1, + [imageRef], + { maxPixels: 512 * 512, maxBytes: 512_000 }, + expect.any(AbortSignal), + ) + expect(attachmentMocks.readImageRequests).toHaveBeenNthCalledWith( + 2, + [imageRef], + { maxPixels: 320_000, maxBytes: 1024 * 1024 }, + expect.any(AbortSignal), + ) + }) + it('reuses the exact request version between agent and compaction calls', async () => { const server = await mockServer([ { kind: 'sse', events: textEvents }, @@ -219,7 +286,7 @@ describe('DeepSeekAdapter against a mock server', () => { }) it('explains a provider rejection of a normalized image and retains the raw response as cause', async () => { - const raw = JSON.stringify({ error: { message: 'unsupported image payload' } }) + const raw = JSON.stringify({ error: { message: 'unsupported image payload for file-api-1' } }) const server = await mockServer([{ kind: 'http-error', status: 400, body: raw }]) const attachments = attachmentStoreOf(ref => Promise.resolve(requestImage(ref))).store const adapter = adapterOf({ @@ -248,11 +315,72 @@ describe('DeepSeekAdapter against a mock server', () => { cause: { message: raw }, }) expect((failure as Error).message).toContain('image/png, 8-bit sRGBA, 1x1') - expect((failure as Error).message).toContain('unsupported image payload') + expect((failure as Error).message).toContain('unsupported image payload for file-api-1') expect((failure as Error).message).not.toBe(raw) }) + it('identifies the sole image when a normalized rejection omits its file id', async () => { + const raw = JSON.stringify({ error: { message: 'unsupported image payload' } }) + const server = await mockServer([{ kind: 'http-error', status: 400, body: raw }]) + const attachments = attachmentStoreOf(ref => Promise.resolve(requestImage(ref))).store + const adapter = adapterOf({ + baseURL: server.url, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachments) + + await expect(drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [{ type: 'image', attachment: imageRef }], + source: { kind: 'plugin', plugin: 'test' }, + })], + }))).rejects.toMatchObject({ + message: expect.stringContaining(`normalized image "${imageRef.attachmentId}"`) as string, + }) + }) + + it('lists every candidate when a normalized multi-image rejection names no file id', async () => { + const secondRef: ImageAttachmentRef = { + ...imageRef, + attachmentId: AttachmentId(`sha256:${'c'.repeat(64)}`), + } + const raw = JSON.stringify({ error: { message: 'unsupported image payload' } }) + const server = await mockServer([{ kind: 'http-error', status: 400, body: raw }]) + const attachments = attachmentStoreOf((ref) => { + const first = ref.attachmentId === imageRef.attachmentId + return Promise.resolve({ + ...requestImage(ref), + variantId: ImageVariantId(`sha256:${(first ? 'b' : 'd').repeat(64)}`), + master: first ? { ...ref, name: 'diagram.png' } : ref, + hasAlpha: false, + }) + }).store + const adapter = adapterOf({ + baseURL: server.url, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachments) + + await expect(drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [ + { type: 'image', attachment: imageRef }, + { type: 'image', attachment: secondRef }, + ], + source: { kind: 'plugin', plugin: 'test' }, + })], + }))).rejects.toMatchObject({ + code: 'INVALID_REQUEST', + message: expect.stringContaining('Candidate images: "diagram.png"') as string, + cause: { message: raw }, + }) + }) + it.each([ + 'file-api-1 expired', + 'file_id file-api-10 invalid; file_id file-api-1 expired', 'file_id file-api-1 expired', 'file_not_found', 'file_id file-api-1 deleted', @@ -332,6 +460,71 @@ describe('DeepSeekAdapter against a mock server', () => { .toEqual([{ type: 'file', file_id: 'file-api-1' }, { type: 'file', file_id: 'file-api-3' }]) }) + it('invalidates every listed missing file id and preserves unlisted mappings', async () => { + const secondRef: ImageAttachmentRef = { + ...imageRef, + attachmentId: AttachmentId(`sha256:${'c'.repeat(64)}`), + } + const thirdRef: ImageAttachmentRef = { + ...imageRef, + attachmentId: AttachmentId(`sha256:${'e'.repeat(64)}`), + } + const server = await mockServer([ + { + kind: 'http-error', + status: 400, + body: JSON.stringify({ + error: { + message: 'path.to.object[index]: the following file_ids do not exist or are not created under your account: ' + + 'file-api-1, file-api-3, file-api-unknown', + }, + }), + }, + { kind: 'sse', events: textEvents }, + ]) + const attachments = attachmentStoreOf((ref) => { + let digest = 'f' + if (ref.attachmentId === imageRef.attachmentId) digest = 'b' + else if (ref.attachmentId === secondRef.attachmentId) digest = 'd' + return Promise.resolve({ + ...requestImage(ref), + variantId: ImageVariantId(`sha256:${digest.repeat(64)}`), + }) + }).store + const adapter = adapterOf({ + baseURL: server.url, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachments) + + await drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [ + { type: 'image', attachment: imageRef }, + { type: 'image', attachment: secondRef }, + { type: 'image', attachment: thirdRef }, + ], + source: { kind: 'plugin', plugin: 'test' }, + })], + })) + + expect(server.fileRequests.filter(request => request.method === 'POST')).toHaveLength(5) + const retries = server.requests as Array<{ messages: Array<{ content: Array<{ type: string; file_id?: string }> }> }> + expect(retries[0]?.messages[0]?.content.filter(block => block.type === 'file')) + .toEqual([ + { type: 'file', file_id: 'file-api-1' }, + { type: 'file', file_id: 'file-api-2' }, + { type: 'file', file_id: 'file-api-3' }, + ]) + expect(retries[1]?.messages[0]?.content.filter(block => block.type === 'file')) + .toEqual([ + { type: 'file', file_id: 'file-api-4' }, + { type: 'file', file_id: 'file-api-2' }, + { type: 'file', file_id: 'file-api-5' }, + ]) + }) + it('invalidates every used mapping when a stale-file response does not identify one file id', async () => { const secondRef: ImageAttachmentRef = { ...imageRef, @@ -644,6 +837,20 @@ describe('DeepSeekAdapter against a mock server', () => { }) }) + it('uses the HTTP status as the cause when an error response has no body', async () => { + const server = await mockServer([{ kind: 'http-error', status: 500, body: '' }]) + const adapter = adapterOf({ baseURL: server.url }) + + await expect(drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash', + messages: [], + }))).rejects.toMatchObject({ + code: 'SERVER', + cause: { message: 'DeepSeek HTTP 500' }, + }) + }) + it('classifies an HTTP context-window failure with the canonical code', async () => { const server = await mockServer([{ kind: 'http-error', From de8ea5d715ba2dc402b02765832b3bda6453a78a Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 18:52:42 +0800 Subject: [PATCH 14/28] docs: refresh image pipeline module graph --- docs/module-graph.i18n.yaml | 4 ++-- docs/module-graph.md | 5 ++++- docs/module-graph.zh.md | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/module-graph.i18n.yaml b/docs/module-graph.i18n.yaml index 4dbba75888..4c6a9ea366 100644 --- a/docs/module-graph.i18n.yaml +++ b/docs/module-graph.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/module-graph.md -module-graph.md: f17c65854dbf349adba5ff99288676a4f7eb7402 -module-graph.zh.md: 31608370c550ae7e32e1395e9f6833abf3d096c8 +module-graph.md: a7ba311ec4e7c744b970fdeec704c9c36bc81a20 +module-graph.zh.md: e3442640108559101e8ccd676a8ff2ca1fe2286c diff --git a/docs/module-graph.md b/docs/module-graph.md index f17c65854d..a7ba311ec4 100644 --- a/docs/module-graph.md +++ b/docs/module-graph.md @@ -413,8 +413,11 @@ flowchart TD pkg_settings_file --> pkg_invariants pkg_settings_file --> pkg_settings pkg_llm_deepseek --> pkg_anonymous_user_id + pkg_llm_deepseek --> pkg_atomic_write pkg_llm_deepseek --> pkg_attachment + pkg_llm_deepseek --> pkg_brand pkg_llm_deepseek --> pkg_credentials + pkg_llm_deepseek --> pkg_home_paths pkg_llm_deepseek --> pkg_invariants pkg_llm_deepseek --> pkg_launch_environment pkg_llm_deepseek --> pkg_llm @@ -1507,7 +1510,7 @@ flowchart TD | [`client-hmr`](../packages/client/hmr) | `client` | [`client-modules`](../packages/client/modules), [`host-webserver`](../packages/host/webserver), [`invariants`](../packages/runtime-diagnostics/invariants) | | [`credentials-local`](../packages/credentials/credentials-local) | `credentials` | [`atomic-write`](../packages/util/atomic-write), [`credentials`](../packages/credentials/credentials), [`home-paths`](../packages/util/home-paths), [`invariants`](../packages/runtime-diagnostics/invariants), [`launch-environment`](../packages/util/launch-environment) | | [`settings-file`](../packages/settings/settings-file) | `settings` | [`atomic-write`](../packages/util/atomic-write), [`home-paths`](../packages/util/home-paths), [`invariants`](../packages/runtime-diagnostics/invariants), [`settings`](../packages/settings/settings) | -| [`llm-deepseek`](../packages/llm/llm-deepseek) | `llm` | [`anonymous-user-id`](../packages/identity/anonymous-user-id), [`attachment`](../packages/attachment/attachment), [`credentials`](../packages/credentials/credentials), [`invariants`](../packages/runtime-diagnostics/invariants), [`launch-environment`](../packages/util/launch-environment), [`llm`](../packages/llm/llm), [`settings`](../packages/settings/settings), [`timeout`](../packages/util/timeout) | +| [`llm-deepseek`](../packages/llm/llm-deepseek) | `llm` | [`anonymous-user-id`](../packages/identity/anonymous-user-id), [`atomic-write`](../packages/util/atomic-write), [`attachment`](../packages/attachment/attachment), [`brand`](../packages/util/brand), [`credentials`](../packages/credentials/credentials), [`home-paths`](../packages/util/home-paths), [`invariants`](../packages/runtime-diagnostics/invariants), [`launch-environment`](../packages/util/launch-environment), [`llm`](../packages/llm/llm), [`settings`](../packages/settings/settings), [`timeout`](../packages/util/timeout) | | [`session`](../packages/core/session) | `core` | [`brand`](../packages/util/brand), [`invariants`](../packages/runtime-diagnostics/invariants), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope), [`typert-protocol`](../packages/typert/protocol) | | [`system-prompt`](../packages/core/system-prompt) | `core` | [`invariants`](../packages/runtime-diagnostics/invariants), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope) | | [`skill`](../packages/skill/skill) | `skill` | [`invariants`](../packages/runtime-diagnostics/invariants), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope) | diff --git a/docs/module-graph.zh.md b/docs/module-graph.zh.md index 31608370c5..e344264010 100644 --- a/docs/module-graph.zh.md +++ b/docs/module-graph.zh.md @@ -415,8 +415,11 @@ flowchart TD pkg_settings_file --> pkg_invariants pkg_settings_file --> pkg_settings pkg_llm_deepseek --> pkg_anonymous_user_id + pkg_llm_deepseek --> pkg_atomic_write pkg_llm_deepseek --> pkg_attachment + pkg_llm_deepseek --> pkg_brand pkg_llm_deepseek --> pkg_credentials + pkg_llm_deepseek --> pkg_home_paths pkg_llm_deepseek --> pkg_invariants pkg_llm_deepseek --> pkg_launch_environment pkg_llm_deepseek --> pkg_llm @@ -1509,7 +1512,7 @@ flowchart TD | [`client-hmr`](../packages/client/hmr) | `client` | [`client-modules`](../packages/client/modules), [`host-webserver`](../packages/host/webserver), [`invariants`](../packages/runtime-diagnostics/invariants) | | [`credentials-local`](../packages/credentials/credentials-local) | `credentials` | [`atomic-write`](../packages/util/atomic-write), [`credentials`](../packages/credentials/credentials), [`home-paths`](../packages/util/home-paths), [`invariants`](../packages/runtime-diagnostics/invariants), [`launch-environment`](../packages/util/launch-environment) | | [`settings-file`](../packages/settings/settings-file) | `settings` | [`atomic-write`](../packages/util/atomic-write), [`home-paths`](../packages/util/home-paths), [`invariants`](../packages/runtime-diagnostics/invariants), [`settings`](../packages/settings/settings) | -| [`llm-deepseek`](../packages/llm/llm-deepseek) | `llm` | [`anonymous-user-id`](../packages/identity/anonymous-user-id), [`attachment`](../packages/attachment/attachment), [`credentials`](../packages/credentials/credentials), [`invariants`](../packages/runtime-diagnostics/invariants), [`launch-environment`](../packages/util/launch-environment), [`llm`](../packages/llm/llm), [`settings`](../packages/settings/settings), [`timeout`](../packages/util/timeout) | +| [`llm-deepseek`](../packages/llm/llm-deepseek) | `llm` | [`anonymous-user-id`](../packages/identity/anonymous-user-id), [`atomic-write`](../packages/util/atomic-write), [`attachment`](../packages/attachment/attachment), [`brand`](../packages/util/brand), [`credentials`](../packages/credentials/credentials), [`home-paths`](../packages/util/home-paths), [`invariants`](../packages/runtime-diagnostics/invariants), [`launch-environment`](../packages/util/launch-environment), [`llm`](../packages/llm/llm), [`settings`](../packages/settings/settings), [`timeout`](../packages/util/timeout) | | [`session`](../packages/core/session) | `core` | [`brand`](../packages/util/brand), [`invariants`](../packages/runtime-diagnostics/invariants), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope), [`typert-protocol`](../packages/typert/protocol) | | [`system-prompt`](../packages/core/system-prompt) | `core` | [`invariants`](../packages/runtime-diagnostics/invariants), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope) | | [`skill`](../packages/skill/skill) | `skill` | [`invariants`](../packages/runtime-diagnostics/invariants), [`llm`](../packages/llm/llm), [`scope`](../packages/core/scope) | From 48a58b90904babd586eb5b63dc58d5d2307400ef Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 20:01:43 +0800 Subject: [PATCH 15/28] fix(images): address unified pipeline review --- ...0-unified-image-request-pipeline.i18n.yaml | 4 +- ...26-08-20-unified-image-request-pipeline.md | 10 +- ...08-20-unified-image-request-pipeline.zh.md | 10 +- apps/cli/tests/web-agent-presets.e2e.ts | 2 +- apps/web/tests/shipped-composition.e2e.ts | 1 + docs/subsystems/attachment.md | 2 +- docs/subsystems/attachment.zh.md | 2 +- .../attachment-local/src/canonical.ts | 4 +- .../attachment/attachment-local/src/index.ts | 88 +++++++--- .../attachment-local/src/request-image.ts | 6 +- .../attachment-local/tests/canonical.spec.ts | 17 ++ .../tests/request-image.spec.ts | 27 +++ .../core/tools/tests/gen-tool-catalog.spec.ts | 2 +- packages/fs/tool-fs/src/read-image.ts | 48 ++--- packages/host/apiproxy/src/api-proxy.ts | 1 - .../commands/tests/commands.spec.ts | 5 + packages/llm/llm-deepseek/README.i18n.yaml | 4 +- packages/llm/llm-deepseek/README.md | 9 +- packages/llm/llm-deepseek/README.zh.md | 9 +- packages/llm/llm-deepseek/src/adapter.ts | 19 +- packages/llm/llm-deepseek/src/file-store.ts | 95 ++++++++-- packages/llm/llm-deepseek/src/files-api.ts | 5 +- packages/llm/llm-deepseek/src/index.ts | 6 + packages/llm/llm-deepseek/src/serialize.ts | 14 +- .../llm/llm-deepseek/tests/adapter.spec.ts | 54 ++++++ .../llm/llm-deepseek/tests/file-store.spec.ts | 95 ++++++++++ .../llm/llm-deepseek/tests/files-api.spec.ts | 164 +++++++++++++++++- .../llm/llm-deepseek/tests/serialize.spec.ts | 21 +++ .../llm-deepseek/tests/upload-index.spec.ts | 109 +++++++++++- packages/llm/llm-pi-ai/README.md | 4 +- packages/llm/llm-pi-ai/README.zh.md | 4 +- packages/llm/llm-pi-ai/src/adapter.ts | 2 +- packages/llm/llm-pi-ai/src/config.ts | 6 +- packages/llm/llm-pi-ai/src/context.ts | 27 ++- packages/llm/llm-pi-ai/tests/adapter.spec.ts | 16 +- packages/llm/llm-pi-ai/tests/context.spec.ts | 60 ++++++- packages/llm/llm-pi-ai/tests/convert.spec.ts | 32 +++- packages/llm/llm/src/content.ts | 11 +- 38 files changed, 849 insertions(+), 146 deletions(-) diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml index 4f1b156e3a..07d9effb78 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md -2026-08-20-unified-image-request-pipeline.md: 72382b6130086ba5c36d386ffe7ebe413cd2243d -2026-08-20-unified-image-request-pipeline.zh.md: 15560ad475af669cc4a2d9c46354a4da08528e0b +2026-08-20-unified-image-request-pipeline.md: 07632e9e0c3aac33d89acd8aebc0f0114550ddb6 +2026-08-20-unified-image-request-pipeline.zh.md: 9d95346dab2a7747c4bcef9f213ec0fa8e5ba067 diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md index 72382b6130..07632e9e0c 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md @@ -24,19 +24,19 @@ Batch admission prepares and verifies every master once before publishing any me `AttachmentStore.readImageRequest` derives a request version under route-owned total-pixel and encoded-byte budgets. Scaling is `min(1, sqrt(maxPixels / (width * height)))`, with no enlargement, followed by inward integer rounding so the encoded raster never exceeds the total-pixel cap. DeepSeek V4 Flash Vision Exp uses 640,000 total pixels and 1MiB raw encoded bytes by default; low detail uses 512 by 512 total pixels. A 2048 by 1024 master projects to 1130 by 565 under the hard cap. Request encoding uses the same color branches, with PNG (palette only without alpha) then WebP 85 and 80 for low-color input, WebP 85 then 80 for other alpha input, and JPEG 85 then 80 for other opaque input. Each fallback runs only after the previous result exceeds 1MiB, and dimensions shrink only after both quality attempts exceed it. The same derivation is used by normal agent turns, direct `ctx.llm.stream` calls, compaction, and other auxiliary streams. -The `variantId` and cache path cover the master attachment id, transform version, route pixel and byte budgets, optional master-coordinate crop, and fixed encoder parameters. Cached output is fully decoded before reuse. DeepSeek Files and pi-ai inline base64 therefore use the same deterministic bytes for the same policy. Inline accounting uses the derived byte length after base64 expansion, not the master byte count. Equal in-process `variantId` calls share one transform and cache write; cancellation rejects only that waiter. `AttachmentStore.readImageRequests` preserves input order while the local implementation runs master and request transforms through one FIFO limiter. `imageCompressionConcurrency` is configurable from 1 through 8 and defaults to 2. Batch publication remains sequential after every master has been prepared. +The `variantId` and cache path cover the master attachment id, transform version, route pixel and byte budgets, optional master-coordinate crop, and fixed encoder parameters. A new cache entry is fully decoded before publication. Cache hits use a header probe to check format, 8-bit sRGB/sRGBA facts, dimensions, alpha, and byte limits without decoding the complete raster again; a mismatch regenerates the entry. DeepSeek Files and pi-ai inline base64 therefore use the same deterministic bytes for the same policy. Inline accounting uses the derived byte length after base64 expansion, not the master byte count. Equal in-process `variantId` calls share one transform and cache write. Each caller can cancel its own wait; the shared transform is aborted only after every waiter has cancelled. `AttachmentStore.readImageRequests` preserves input order while the local implementation runs master and request transforms through one FIFO limiter. `imageCompressionConcurrency` is configurable from 1 through 8 and defaults to 2. Batch publication remains sequential after every master has been prepared. -Request-size offload is a deterministic oldest-first projection. DeepSeek defaults to 128MiB and 600 referenced images. Its removed prefix advances past successive 64MiB byte boundaries and in 20-image count quanta, so 129 one-megabyte images remove the oldest 65, retain 64MiB, and keep that prefix stable until total history passes 192MiB. Pi-ai retains a configurable base64 request bound. A text-only route receives deterministic attachment placeholders, including nested tool-result images, while append-only session history keeps the original references. +Request-size offload is a deterministic oldest-first projection. Before reading attachments, each route uses `min(masterBytes, requestVersionMaxBytes)` as a conservative upper bound and removes the oldest over-budget prefix. Only retained masters are read and transformed, so an omitted missing or corrupt object cannot block the request. A second projection uses exact derived lengths without bringing omitted images back. DeepSeek defaults to 128MiB and 600 referenced images. Its removed prefix advances past successive 64MiB byte boundaries and in 20-image count quanta, so 129 one-megabyte images remove the oldest 65, retain 64MiB, and keep that prefix stable until total history passes 192MiB. Pi-ai retains a configurable base64 request bound. A text-only route receives deterministic attachment placeholders, including nested tool-result images, while append-only session history keeps the original references. ### Stable handles and master-coordinate crops -Every retained request image is preceded by its complete attachment id, actual request dimensions, and the preview-coordinate arguments for `read_image_region`. The tool accepts only an attachment already referenced by the calling session. It maps the supplied preview rectangle to the 2048px master with floor-at-origin and ceil-at-far-edge rounding, crops the master rather than the preview, and persists the result as a new attachment. The tool result contains the new `ImageBlock`, so model-visible output and the durable log remain equivalent. +Every retained request image is preceded by its complete attachment id and actual request dimensions. When the active request exposes `read_image_region`, the text also supplies its preview-coordinate arguments. The tool accepts only an attachment already referenced by the calling session. It maps the supplied preview rectangle to the 2048px master with floor-at-origin and ceil-at-far-edge rounding, crops the master rather than the preview, and persists the result as a new attachment. The tool result contains the new `ImageBlock`, so model-visible output and the durable log remain equivalent. ### DeepSeek Files lifecycle The direct `deepseek-official` adapter uploads every retained request version through the OpenAI-compatible Files API and sends only `file_id` content blocks. There is no inline fallback. The default catalog advertises `deepseek-v4-flash-vision-exp` as image-capable. Uploaded ids are indexed by endpoint and API-key scope plus `variantId`. Uploads request seven days by default and record the returned `expires_at`; a mapping with no more than one hour remaining is replaced without a preceding retrieve call. The index never stores the API key. -An upload is indexed only after the response returns a complete file object, matching byte count, and `expires_at`. A missing or inconsistent response leaves no local mapping, so a later request uploads again. A malformed upload index is an empty cache and is replaced on the next successful upload; filesystem I/O failures remain errors. If chat reports expired, deleted, missing, or invalid ids and names one or more ids used by the request, only those mappings are removed. A stale-file response without a specific id removes every mapping used by that chat attempt. The affected request bytes are uploaded again and chat is retried once. A second stale rejection clears the mappings identified by its response and returns the error without a third chat attempt. One upload quota error deletes the configured number of oldest harness-owned `dsh-` files and retries once. Public file operations expose list, retrieve, delete, one-variant release, and namespace-wide release. The client enforces the documented 128MiB upload limit, 32MiB chat-image limit, 10,000-file and 25GiB quotas, and one-hour to 30-day expiry range. +An upload is indexed only after the response returns a complete file object, matching byte count, and `expires_at`. A missing or inconsistent response leaves no local mapping, so a later request uploads again. Concurrent upload resolution for one scoped `variantId` shares one provider operation; one waiter cannot cancel another, and the upload stops when every waiter has cancelled. A malformed upload index is an empty cache and is replaced on the next successful upload; filesystem I/O failures remain errors. If chat reports expired, deleted, missing, or invalid ids and names one or more ids used by the request, only those mappings are removed. A stale-file response without a specific id removes every mapping used by that chat attempt. The affected request bytes are uploaded again and chat is retried once. A second stale rejection clears the mappings identified by its response and returns the error without a third chat attempt. One upload quota error first lists the configured number of oldest harness-owned `dsh-` files, then deletes that collected set and retries once; deleting after pagination keeps provider cursors valid. Public file operations expose list, retrieve, delete, one-variant release, and namespace-wide release. Every Files request carries the shared Harness `User-Agent`. The client enforces the documented 128MiB upload limit, 32MiB chat-image limit, 10,000-file and 25GiB quotas, and one-hour to 30-day expiry range. ### Diagnostics @@ -64,7 +64,7 @@ Historical attachment objects that later disappear or fail integrity verificatio ## Verification -Package tests generate 16-bit RGB and RGBA PNG fixtures, prove 8-bit conversion and clean 8-bit passthrough, retain alpha under byte pressure, distinguish high-frequency and ordinary photos from low-color graphics, stop lazy encoding after the first fitting candidate, cover square and wide 640,000-pixel projections, enforce 1MiB request bytes, singleflight equal variants, bound transform concurrency, preserve cache and upload identity, map preview crops to the master, prepare batches once, reject inconsistent Files responses, refresh near-expiry ids without retrieve, recover once from single-id, multiple-id, and ambiguous stale responses, delete quota files, normalize provider diagnostics, project text-only history, and share normal/compaction request bytes. Keyless assembled snapshots cover the real tool schemas and image request path. A credentialed test uses the built-in `deepseek-official` route and its configured endpoint, never a custom provider entry. +Package tests generate 16-bit RGB and RGBA PNG fixtures, prove 8-bit conversion and clean 8-bit passthrough, retain alpha under byte pressure, distinguish high-frequency and ordinary photos from low-color graphics, stop lazy encoding after the first fitting candidate, cover square and wide 640,000-pixel projections, enforce 1MiB request bytes, singleflight equal variants and uploads without shared-cancellation leaks, bound transform concurrency, preserve cache and upload identity, skip attachment reads for conservatively offloaded history, map preview crops to the master, prepare batches once, reject inconsistent Files responses, refresh near-expiry ids without retrieve, recover once from single-id, multiple-id, and ambiguous stale responses, paginate before quota deletion, normalize provider diagnostics, project text-only history, and share normal/compaction request bytes. Keyless assembled snapshots cover the real tool schemas and image request path. A credentialed test uses the built-in `deepseek-official` route and its configured endpoint, never a custom provider entry. ## Consequences diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md index 15560ad475..9d95346dab 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md @@ -24,19 +24,19 @@ Status: implemented `AttachmentStore.readImageRequest` 按路由拥有的总像素和编码字节预算派生请求版本。缩放公式为 `min(1, sqrt(maxPixels / (width * height)))`,不会放大小图,随后向预算内取整,确保编码光栅不超过总像素上限。DeepSeek V4 Flash Vision Exp 默认使用总像素 640,000 和原始编码字节 1MiB;low detail 使用总像素 512×512。2048×1024 主版本在这个硬上限下会投影为 1130×565。请求编码使用相同的分类分支:低色数输入先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80 的 WebP;其他透明输入依次尝试质量 85、80 的 WebP;其他非透明输入依次尝试质量 85、80 的 JPEG。只有前一结果超过 1MiB 时才执行下一个候选;两个质量档都超限后才缩小尺寸。普通 agent 轮次、直接 `ctx.llm.stream` 调用、压缩和其他辅助流都使用同一派生过程。 -`variantId` 和缓存路径覆盖主附件 ID、变换策略版本、路由像素和字节预算、可选的主版本坐标裁剪区域及固定编码参数。缓存输出会在复用前完整解码。因此,同一策略下的 DeepSeek Files 和 pi-ai 内联 base64 使用相同的确定性字节。内联计量使用派生字节经过 base64 膨胀后的长度,不使用主版本字节数。同一进程内相同 `variantId` 的调用共享一次变换和缓存写入;取消只拒绝对应等待方。`AttachmentStore.readImageRequests` 保持输入顺序,本地实现则通过一个 FIFO 限流器运行主版本和请求版本变换。`imageCompressionConcurrency` 的可配置范围为 1 至 8,默认值为 2。全部主版本准备完成后,批次仍按顺序发布。 +`variantId` 和缓存路径覆盖主附件 ID、变换策略版本、路由像素和字节预算、可选的主版本坐标裁剪区域及固定编码参数。新缓存条目在发布前会完整解码。缓存命中只探测文件头,校验格式、8-bit sRGB/sRGBA、尺寸、透明通道和字节上限,不会再次完整解码光栅;不匹配时会重新生成。因此,同一策略下的 DeepSeek Files 和 pi-ai 内联 base64 使用相同的确定性字节。内联计量使用派生字节经过 base64 膨胀后的长度,不使用主版本字节数。同一进程内相同 `variantId` 的调用共享一次变换和缓存写入。每个调用方可以取消自己的等待;只有全部等待方都取消时,共享变换才会中止。`AttachmentStore.readImageRequests` 保持输入顺序,本地实现则通过一个 FIFO 限流器运行主版本和请求版本变换。`imageCompressionConcurrency` 的可配置范围为 1 至 8,默认值为 2。全部主版本准备完成后,批次仍按顺序发布。 -请求大小 offload 是确定性的从旧到新投影。DeepSeek 默认上限为 128MiB 和 600 张引用图片。被移除前缀会越过连续的 64MiB 字节边界,并按 20 张图片数量步长递增,因此 129 张 1MiB 图片会移除最旧的 65 张并保留 64MiB;持久历史超过 192MiB 前,该前缀保持不变。Pi-ai 保留可配置的 base64 请求上限。纯文本路由会收到确定性的附件占位文本,其中包括嵌套工具结果图片;追加式会话历史继续保留原始引用。 +请求大小 offload 是确定性的从旧到新投影。读取附件前,每条路由先以 `min(主版本字节数, 请求版本字节上限)` 作为保守上界,移除超出预算的最旧前缀。系统只读取并转换保留的主版本,因此已省略的缺失或损坏对象不会阻塞请求。第二次投影使用确切派生长度,但不会重新加入已省略图片。DeepSeek 默认上限为 128MiB 和 600 张引用图片。被移除前缀会越过连续的 64MiB 字节边界,并按 20 张图片数量步长递增,因此 129 张 1MiB 图片会移除最旧的 65 张并保留 64MiB;持久历史超过 192MiB 前,该前缀保持不变。Pi-ai 保留可配置的 base64 请求上限。纯文本路由会收到确定性的附件占位文本,其中包括嵌套工具结果图片;追加式会话历史继续保留原始引用。 ### 稳定句柄与主版本坐标裁剪 -每张保留请求图片前都有完整附件 ID、实际请求尺寸和 `read_image_region` 所需的预览坐标参数。该工具只接受调用会话已经引用的附件。它按起点向下取整、远端边界向上取整,把提交的预览矩形映射到 2048px 主版本,从主版本而非预览图裁剪,并把结果保存为新附件。工具结果包含新的 `ImageBlock`,因此模型可见输出与持久日志保持一致。 +每张保留请求图片前都有完整附件 ID 和实际请求尺寸。当前请求公开 `read_image_region` 时,这段文本还会提供预览坐标参数。该工具只接受调用会话已经引用的附件。它按起点向下取整、远端边界向上取整,把提交的预览矩形映射到 2048px 主版本,从主版本而非预览图裁剪,并把结果保存为新附件。工具结果包含新的 `ImageBlock`,因此模型可见输出与持久日志保持一致。 ### DeepSeek Files 生命周期 直接 `deepseek-official` 适配器通过 OpenAI 兼容 Files API 上传每张保留的请求版本,只发送 `file_id` 内容块,不提供内联回退。默认 catalog 把 `deepseek-v4-flash-vision-exp` 公布为支持图片。上传 ID 按端点和 API key 作用域以及 `variantId` 写入索引。上传默认请求 7 天有效期,并记录返回的 `expires_at`;本地映射剩余时间不超过一小时时会直接替换,不会先查询远端文件。索引绝不存储 API key。 -只有上传响应返回完整文件对象、匹配的字节数和 `expires_at` 时,上传结果才会写入索引。缺失或不一致的响应不会留下本地映射,后续请求会重新上传。格式损坏的上传索引按空缓存处理,并在下一次成功上传时替换;文件系统 I/O 失败仍是错误。如果 chat 报告 ID 已过期、删除、缺失或无效,并指出本次请求使用的一个或多个 ID,适配器只删除这些映射。如果响应只说明文件状态失效而没有指出具体 ID,适配器会删除该次 chat 使用的全部映射。受影响的请求字节会重新上传,chat 只重试一次。第二次仍报告文件失效时,适配器会按响应清理映射并返回错误,不会发起第三次 chat。一次上传配额错误会删除配置数量的最旧 `dsh-` 文件,然后重试一次。公开文件操作提供列表、查询、删除、单个变体释放和整个作用域释放。客户端执行文档规定的 Files 单次上传 128MiB、chat 单图 32MiB、10,000 个文件、25GiB,以及一小时到 30 天有效期限制。 +只有上传响应返回完整文件对象、匹配的字节数和 `expires_at` 时,上传结果才会写入索引。缺失或不一致的响应不会留下本地映射,后续请求会重新上传。同一作用域和 `variantId` 的并发解析共享一次提供方上传;单个等待方无法取消其他等待方,全部等待方取消时才会停止上传。格式损坏的上传索引按空缓存处理,并在下一次成功上传时替换;文件系统 I/O 失败仍是错误。如果 chat 报告 ID 已过期、删除、缺失或无效,并指出本次请求使用的一个或多个 ID,适配器只删除这些映射。如果响应只说明文件状态失效而没有指出具体 ID,适配器会删除该次 chat 使用的全部映射。受影响的请求字节会重新上传,chat 只重试一次。第二次仍报告文件失效时,适配器会按响应清理映射并返回错误,不会发起第三次 chat。一次上传配额错误会先列出配置数量的最旧 `dsh-` 文件,再删除收集到的文件并重试一次;分页完成后才删除,避免游标失效。公开文件操作提供列表、查询、删除、单个变体释放和整个作用域释放。每个 Files 请求都携带 Harness 的共享 `User-Agent`。客户端执行文档规定的 Files 单次上传 128MiB、chat 单图 32MiB、10,000 个文件、25GiB,以及一小时到 30 天有效期限制。 ### 诊断 @@ -64,7 +64,7 @@ Status: implemented ## Verification -包测试会生成 16-bit RGB 和 RGBA PNG fixture,验证 8-bit 转换与干净 8-bit 字节直通、字节压力下保留透明通道、区分高频和普通照片与低色数图形、首个候选合规后停止编码、正方形和宽屏 640,000 像素投影、请求字节不超过 1MiB、相同变体 singleflight、变换并发上限、缓存与上传身份、预览到主版本坐标映射、批量只准备一次、Files 响应不一致、进入刷新余量时不查询远端并更新 ID、单个 ID、多个 ID 和模糊失效响应只恢复一次、配额删除、规范化提供方诊断、纯文本投影,以及普通请求与压缩共享请求字节。无需密钥的组装快照覆盖真实工具 schema 和图片请求路径。使用凭据的测试只使用内置 `deepseek-official` 路由及其已配置端点,不使用自定义提供方条目。 +包测试会生成 16-bit RGB 和 RGBA PNG fixture,验证 8-bit 转换与干净 8-bit 字节直通、字节压力下保留透明通道、区分高频和普通照片与低色数图形、首个候选合规后停止编码、正方形和宽屏 640,000 像素投影、请求字节不超过 1MiB、相同变体与上传 singleflight 且不会共享取消、变换并发上限、缓存与上传身份、跳过已保守 offload 的历史附件读取、预览到主版本坐标映射、批量只准备一次、Files 响应不一致、进入刷新余量时不查询远端并更新 ID、单个 ID、多个 ID 和模糊失效响应只恢复一次、删除配额文件前完成分页、规范化提供方诊断、纯文本投影,以及普通请求与压缩共享请求字节。无需密钥的组装快照覆盖真实工具 schema 和图片请求路径。使用凭据的测试只使用内置 `deepseek-official` 路由及其已配置端点,不使用自定义提供方条目。 ## Consequences diff --git a/apps/cli/tests/web-agent-presets.e2e.ts b/apps/cli/tests/web-agent-presets.e2e.ts index 0e98af0477..976381096a 100644 --- a/apps/cli/tests/web-agent-presets.e2e.ts +++ b/apps/cli/tests/web-agent-presets.e2e.ts @@ -237,7 +237,7 @@ describe('the shipped Web composition', () => { // depend on ripgrep being present on the machine. expect(toolNames(ctx, handle.agent).filter(name => name !== 'glob' && name !== 'grep')).toEqual([ 'ask_user_question', 'bash', 'create_goal', 'edit', 'exit_plan_mode', - 'get_goal', 'interrupt_agent', 'job_kill', 'job_list', 'job_output', 'list_agents', 'ralph', 'read', 'read_image', 'send_message', 'skill', + 'get_goal', 'interrupt_agent', 'job_kill', 'job_list', 'job_output', 'list_agents', 'ralph', 'read', 'read_image', 'read_image_region', 'send_message', 'skill', 'subagent', 'subagent_fork', 'todo_write', 'update_goal', 'web_search', 'workflow', 'write', ]) diff --git a/apps/web/tests/shipped-composition.e2e.ts b/apps/web/tests/shipped-composition.e2e.ts index 295e861b95..cca21dcef6 100644 --- a/apps/web/tests/shipped-composition.e2e.ts +++ b/apps/web/tests/shipped-composition.e2e.ts @@ -48,6 +48,7 @@ const EXPECTED_TOOLS = [ 'ralph', 'read', 'read_image', + 'read_image_region', 'send_message', 'skill', 'subagent', diff --git a/docs/subsystems/attachment.md b/docs/subsystems/attachment.md index ec9d1f27bd..66d00eb387 100644 --- a/docs/subsystems/attachment.md +++ b/docs/subsystems/attachment.md @@ -145,7 +145,7 @@ interface RequestImageAttachment { } ``` -`saveImage()` prepares a provider-independent 2048px, 4MiB master and atomically commits it before returning its reference. `saveImages()` prepares every validated master once before publishing the batch, so validation rejection leaves no partial objects and publication does not repeat decoding or quality selection. `admitEncodedImages()` is the wire entry for base64 uploads and delegates count, aggregate-byte, and ordered batch admission to `saveImages()`. `readImage()` verifies a master from an authorized session path. `readImageRequest()` derives and caches one request version under an exact route pixel and byte budget; `readImageRequests()` lets an implementation apply its configured bounded transform concurrency to an ordered batch. The local implementation lazily encodes preferred candidates, singleflights equal request identities, and defaults to two simultaneous transformations. `cropImage()` maps model preview coordinates back to the master and returns another durable attachment. The service is retention-neutral: resumed and forked sessions may share objects, so reference-aware garbage collection is deferred rather than tied to one session's deletion. +`saveImage()` prepares a provider-independent 2048px, 4MiB master and atomically commits it before returning its reference. `saveImages()` prepares every validated master once before publishing the batch, so validation rejection leaves no partial objects and publication does not repeat decoding or quality selection. `admitEncodedImages()` is the wire entry for base64 uploads and delegates count, aggregate-byte, and ordered batch admission to `saveImages()`. `readImage()` verifies a master from an authorized session path. `readImageRequest()` derives and caches one request version under an exact route pixel and byte budget; new entries are fully decoded before publication, while cache hits use a bounded metadata probe. `readImageRequests()` lets an implementation apply its configured transform concurrency to an ordered batch. The local implementation lazily encodes preferred candidates, singleflights equal request identities, lets each waiter cancel independently, stops shared work when no waiter remains, and defaults to two simultaneous transformations. `cropImage()` maps model preview coordinates back to the master and returns another durable attachment. The service is retention-neutral: resumed and forked sessions may share objects, so reference-aware garbage collection is deferred rather than tied to one session's deletion. diff --git a/docs/subsystems/attachment.zh.md b/docs/subsystems/attachment.zh.md index e79c2df4ca..4c3a4ce427 100644 --- a/docs/subsystems/attachment.zh.md +++ b/docs/subsystems/attachment.zh.md @@ -145,7 +145,7 @@ interface RequestImageAttachment { } ``` -`saveImage()` 准备提供方无关的 2048px、4MiB 主版本,并在返回引用前以原子方式提交。`saveImages()` 在发布批次前为每个成员各准备一次经过验证的主版本,因此校验拒绝不会留下部分对象,发布也不会重复解码或选择质量。`admitEncodedImages()` 是面向 base64 上传的 wire 入口,把张数、聚合字节和有序批量准入交给 `saveImages()`。`readImage()` 校验来自已授权会话路径的主版本。`readImageRequest()` 按确切路由的像素和字节预算派生并缓存请求版本;`readImageRequests()` 允许实现按自身配置的有界变换并发处理有序批次。本地实现按需编码首选候选、合并相同请求身份的并发任务,默认同时执行两项变换。`cropImage()` 把模型预览坐标映射回主版本,并返回另一个持久附件。该服务不规定保留策略:恢复和 fork 后的会话可能共享对象,因此基于引用的垃圾回收会延期实现,不与单个会话的删除绑定。 +`saveImage()` 准备提供方无关的 2048px、4MiB 主版本,并在返回引用前以原子方式提交。`saveImages()` 在发布批次前为每个成员各准备一次经过验证的主版本,因此校验拒绝不会留下部分对象,发布也不会重复解码或选择质量。`admitEncodedImages()` 是面向 base64 上传的 wire 入口,把张数、聚合字节和有序批量准入交给 `saveImages()`。`readImage()` 校验来自已授权会话路径的主版本。`readImageRequest()` 按确切路由的像素和字节预算派生并缓存请求版本;新条目在发布前完整解码,缓存命中只做有界元数据探测。`readImageRequests()` 允许实现按自身配置的变换并发处理有序批次。本地实现按需编码首选候选、合并相同请求身份的并发任务、允许每个等待方单独取消、没有等待方时停止共享任务,默认同时执行两项变换。`cropImage()` 把模型预览坐标映射回主版本,并返回另一个持久附件。该服务不规定保留策略:恢复和 fork 后的会话可能共享对象,因此基于引用的垃圾回收会延期实现,不与单个会话的删除绑定。 diff --git a/packages/attachment/attachment-local/src/canonical.ts b/packages/attachment/attachment-local/src/canonical.ts index 8a4193aaed..c437448f4c 100644 --- a/packages/attachment/attachment-local/src/canonical.ts +++ b/packages/attachment/attachment-local/src/canonical.ts @@ -78,8 +78,8 @@ export async function hasLowColourCount(pipeline: Sharp): Promise { const colours = new Set() for (let offset = 0; offset < data.length; offset += info.channels) { const red = data[offset] ?? 0 - const green = data[offset + 1] ?? red - const blue = data[offset + 2] ?? red + const green = info.channels < 3 ? red : data[offset + 1] ?? red + const blue = info.channels < 3 ? red : data[offset + 2] ?? red const alpha = info.channels === 2 ? data[offset + 1] ?? 255 : info.channels === 4 ? data[offset + 3] ?? 255 : 255 diff --git a/packages/attachment/attachment-local/src/index.ts b/packages/attachment/attachment-local/src/index.ts index e43153247a..46e39fb8ff 100644 --- a/packages/attachment/attachment-local/src/index.ts +++ b/packages/attachment/attachment-local/src/index.ts @@ -71,21 +71,59 @@ export interface Config { imageCompressionConcurrency?: number } -function waitForShared(operation: Promise, signal: AbortSignal | undefined): Promise { - if (signal === undefined) return operation - signal.throwIfAborted() - return new Promise((resolve, reject) => { - const abort = (): void => { - const reason: unknown = signal.reason - reject(reason instanceof Error - ? reason - : new Error('Attachment request cancelled with a non-Error reason.', { cause: reason })) - } - signal.addEventListener('abort', abort, { once: true }) - void operation.then(resolve, reject).finally(() => { - signal.removeEventListener('abort', abort) +function abortReason(signal: AbortSignal): Error { + const reason: unknown = signal.reason + return reason instanceof Error + ? reason + : new Error('Attachment request cancelled with a non-Error reason.', { cause: reason }) +} + +class SharedRequest { + readonly controller = new AbortController() + readonly promise: Promise + private settled = false + private waiters = 0 + + constructor(start: (signal: AbortSignal) => Promise) { + this.promise = start(this.controller.signal).finally(() => { + this.settled = true }) - }) + } + + wait(signal?: AbortSignal): Promise { + signal?.throwIfAborted() + this.waiters += 1 + if (signal === undefined) return this.promise.finally(() => this.release(false)) + let released = false + const release = (cancelled: boolean): void => { + if (released) return + released = true + this.release(cancelled, signal) + } + return new Promise((resolve, reject) => { + const abort = (): void => { + release(true) + reject(abortReason(signal)) + } + signal.addEventListener('abort', abort, { once: true }) + void this.promise.then((value) => { + signal.removeEventListener('abort', abort) + release(false) + resolve(value) + }, (error: unknown) => { + signal.removeEventListener('abort', abort) + release(false) + reject(error) + }) + }) + } + + private release(cancelled: boolean, signal?: AbortSignal): void { + this.waiters -= 1 + if (cancelled && this.waiters === 0 && !this.settled && signal !== undefined) { + this.controller.abort(abortReason(signal)) + } + } } /** Persistent content-addressed local attachment store. */ @@ -111,7 +149,7 @@ export class LocalAttachmentStore extends AttachmentStore { /** Resolved instance-level compression limit. */ readonly imageCompressionConcurrency: number private readonly compression: CompressionLimiter - private readonly requestInflight = new Map>() + private readonly requestInflight = new Map>() constructor(ctx: Context, config: Config) { super(ctx) @@ -191,18 +229,24 @@ export class LocalAttachmentStore extends AttachmentStore { const variantId = requestImageVariantId(ref, policy) const key = String(variantId) let operation = this.requestInflight.get(key) + if (operation?.controller.signal.aborted) { + this.requestInflight.delete(key) + operation = undefined + } if (operation === undefined) { - operation = this.compression.run(async () => readRequestImageFile( + const shared = new SharedRequest(sharedSignal => this.compression.run(async () => readRequestImageFile( this.root, - master ?? await this.readImage(ref), + master ?? await this.readImage(ref, sharedSignal), policy, - )) - this.requestInflight.set(key, operation) - void operation.finally(() => { - if (this.requestInflight.get(key) === operation) this.requestInflight.delete(key) + sharedSignal, + ))) + operation = shared + this.requestInflight.set(key, shared) + void shared.promise.finally(() => { + if (this.requestInflight.get(key) === shared) this.requestInflight.delete(key) }).catch(() => {}) } - return waitForShared(operation, signal) + return operation.wait(signal) } override async cropImage( diff --git a/packages/attachment/attachment-local/src/request-image.ts b/packages/attachment/attachment-local/src/request-image.ts index 9c92d78181..f65bc97ad5 100644 --- a/packages/attachment/attachment-local/src/request-image.ts +++ b/packages/attachment/attachment-local/src/request-image.ts @@ -19,7 +19,7 @@ import { encodeFirstWithinLimit, isExhaustedEncoding } from './encoding.ts' import { detectImage, probeImage } from './image.ts' /** Transform version included in every cache and upload-index identity. */ -export const REQUEST_IMAGE_TRANSFORM_VERSION = 'request-image-v2' +export const REQUEST_IMAGE_TRANSFORM_VERSION = 'request-image-v3' /** DeepSeek request versions normally fit at these two preferred qualities. */ export const REQUEST_IMAGE_QUALITIES = [85, 80] as const @@ -228,7 +228,7 @@ async function readCached( ): Promise { try { const data = new Uint8Array(await readFile(path, { signal })) - const detected = await detectImage(data) + const detected = await probeImage(data) const crop = policy.crop const maximum = requestImageDimensions(crop?.width ?? master.ref.width, crop?.height ?? master.ref.height, policy.maxPixels) if (data.byteLength > policy.maxBytes || detected.depth !== 'uchar' || detected.space !== 'srgb' @@ -278,7 +278,7 @@ async function writeCached(path: string, data: Uint8Array): Promise { * @param root - absolute versioned attachment storage root. * @param master - verified stored master bytes and reference. * @param policy - exact route request-image policy. - * @param signal - optional cancellation for cache I/O. + * @param signal - optional cancellation for cache I/O and image transformation. * @returns verified request bytes and deterministic variant identity. */ export async function readRequestImageFile( diff --git a/packages/attachment/attachment-local/tests/canonical.spec.ts b/packages/attachment/attachment-local/tests/canonical.spec.ts index c1307b5848..a6f489615e 100644 --- a/packages/attachment/attachment-local/tests/canonical.spec.ts +++ b/packages/attachment/attachment-local/tests/canonical.spec.ts @@ -271,6 +271,23 @@ describe('hasLowColourCount', () => { await expect(hasLowColourCount(transparent)).resolves.toBe(true) }) + it('reads grayscale-alpha samples without treating alpha or the next pixel as RGB', async () => { + const symbols: number[] = [] + for (let first = 0; first < 32; first += 1) { + for (let second = 0; second < 32; second += 1) symbols.push(first, second) + } + const pixels = new Uint8Array(symbols.length * 2) + for (const [index, symbol] of symbols.entries()) { + pixels[index * 2] = symbol * 8 + pixels[index * 2 + 1] = symbol * 8 + } + const grayscaleAlpha = sharp(pixels, { + raw: { width: 128, height: 16, channels: 2 }, + }) + + await expect(hasLowColourCount(grayscaleAlpha)).resolves.toBe(true) + }) + it('keeps an antialiased text screenshot readable on the low-colour PNG path', async () => { const source = new Uint8Array(await sharp(Buffer.from(` diff --git a/packages/attachment/attachment-local/tests/request-image.spec.ts b/packages/attachment/attachment-local/tests/request-image.spec.ts index 69bcfdf36c..3cdfadd32c 100644 --- a/packages/attachment/attachment-local/tests/request-image.spec.ts +++ b/packages/attachment/attachment-local/tests/request-image.spec.ts @@ -206,4 +206,31 @@ describe('local request-image cache', () => { expect(run).toHaveBeenCalledTimes(1) run.mockRestore() }) + + it('aborts the underlying request transform after its only waiter cancels', async () => { + const attachments = await store() + const master = (await attachments.saveImage({ + data: await image(2048, 1024), mediaType: 'image/png', name: 'cancelled.png', + })).ref + let readSignal: AbortSignal | undefined + const read = vi.spyOn(attachments, 'readImage').mockImplementation((_ref, signal) => { + readSignal = signal + return new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + }) + const controller = new AbortController() + const request = attachments.readImageRequest( + master, + { maxPixels: 640_000, maxBytes: 1024 * 1024 }, + controller.signal, + ) + await vi.waitFor(() => expect(read).toHaveBeenCalledTimes(1)) + + const reason = new Error('cancel only transform waiter') + controller.abort(reason) + + await expect(request).rejects.toBe(reason) + expect(readSignal?.reason).toBe(reason) + }) }) diff --git a/packages/core/tools/tests/gen-tool-catalog.spec.ts b/packages/core/tools/tests/gen-tool-catalog.spec.ts index ce2007c34b..50cbd49d57 100644 --- a/packages/core/tools/tests/gen-tool-catalog.spec.ts +++ b/packages/core/tools/tests/gen-tool-catalog.spec.ts @@ -31,7 +31,7 @@ describe('gen-tool-catalog collectToolCatalog', () => { 'cordis_undefine', 'create_goal', 'edit', 'exit_plan_mode', 'followup_task', 'get_goal', 'glob', 'grep', 'interrupt_agent', 'interrupt_agent', 'job_kill', 'job_list', 'job_output', 'list_agents', 'list_agents', 'lsp', 'pwsh', 'pwsh', 'ralph', - 'read', 'read_image', 'report', 'run_code', 'schedule_create', 'schedule_delete', + 'read', 'read_image', 'read_image_region', 'report', 'run_code', 'schedule_create', 'schedule_delete', 'schedule_list', 'send_message', 'send_message', 'session_event_read', 'session_event_search', 'session_event_trace', 'session_search', 'session_trace', 'skill', 'spawn_teammate', 'str_replace_editor', 'subagent', 'team_task_create', diff --git a/packages/fs/tool-fs/src/read-image.ts b/packages/fs/tool-fs/src/read-image.ts index 4766bea6ba..a900c0c720 100644 --- a/packages/fs/tool-fs/src/read-image.ts +++ b/packages/fs/tool-fs/src/read-image.ts @@ -29,6 +29,22 @@ const IMAGE_EXTENSIONS: Readonly> = { '.gif': 'image/gif', } +const IMAGE_VALUE_SCHEMA = { + type: 'object', + additionalProperties: false, + required: true, + properties: { + attachmentId: { type: 'string', required: true }, + mediaType: { type: 'string', enum: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'], required: true }, + bytes: { type: 'integer', required: true }, + width: { type: 'integer', required: true }, + height: { type: 'integer', required: true }, + name: { type: 'string' }, + sourceWidth: { type: 'integer' }, + sourceHeight: { type: 'integer' }, + }, +} as const + /** The structured outcome declared by the `read_image` output schema. */ export interface ImageReadValue { path: string @@ -214,21 +230,7 @@ export function applyReadImageTool(ctx: Context): void { additionalProperties: false, properties: { path: { type: 'string', required: true }, - image: { - type: 'object', - additionalProperties: false, - required: true, - properties: { - attachmentId: { type: 'string', required: true }, - mediaType: { type: 'string', enum: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'], required: true }, - bytes: { type: 'integer', required: true }, - width: { type: 'integer', required: true }, - height: { type: 'integer', required: true }, - name: { type: 'string' }, - sourceWidth: { type: 'integer' }, - sourceHeight: { type: 'integer' }, - }, - }, + image: IMAGE_VALUE_SCHEMA, }, }, render: (_args, value) => imageReadContent(value), @@ -370,21 +372,7 @@ export function applyReadImageTool(ctx: Context): void { height: { type: 'integer', required: true }, }, }, - image: { - type: 'object', - additionalProperties: false, - required: true, - properties: { - attachmentId: { type: 'string', required: true }, - mediaType: { type: 'string', enum: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'], required: true }, - bytes: { type: 'integer', required: true }, - width: { type: 'integer', required: true }, - height: { type: 'integer', required: true }, - name: { type: 'string' }, - sourceWidth: { type: 'integer' }, - sourceHeight: { type: 'integer' }, - }, - }, + image: IMAGE_VALUE_SCHEMA, }, }, render: (_args, value) => regionReadContent(value), diff --git a/packages/host/apiproxy/src/api-proxy.ts b/packages/host/apiproxy/src/api-proxy.ts index dd1268fe00..ce29ba52b2 100644 --- a/packages/host/apiproxy/src/api-proxy.ts +++ b/packages/host/apiproxy/src/api-proxy.ts @@ -185,7 +185,6 @@ function imageInEvent(event: SessionEvent, match: (ref: ImageAttachmentRef) => b return undefined } -/** True when the current model-visible surface contains an image. */ /** Resolve the first reference matching one opaque id. */ function referencedImage(events: readonly SessionEvent[], attachmentId: string): ImageAttachmentRef | undefined { for (const event of events) { diff --git a/packages/interaction/commands/tests/commands.spec.ts b/packages/interaction/commands/tests/commands.spec.ts index 5c80748227..c65fb49bed 100644 --- a/packages/interaction/commands/tests/commands.spec.ts +++ b/packages/interaction/commands/tests/commands.spec.ts @@ -486,6 +486,11 @@ describe('image attachments', () => { source: { mediaType: input.mediaType, bytes: 3, width: 1, height: 1 }, }) }), + validateImageBatch(inputs: readonly unknown[]) { + return (AttachmentStore.prototype as unknown as { + validateImageBatch(this: unknown, batch: readonly unknown[]): void + }).validateImageBatch.call(this, inputs) + }, // The real base-class batch method over this double's limits and members. saveImages(inputs: readonly unknown[]) { return (AttachmentStore.prototype.saveImages as (this: unknown, batch: readonly unknown[]) => Promise).call(this, inputs) diff --git a/packages/llm/llm-deepseek/README.i18n.yaml b/packages/llm/llm-deepseek/README.i18n.yaml index 5d0e91eae1..71f7f71308 100644 --- a/packages/llm/llm-deepseek/README.i18n.yaml +++ b/packages/llm/llm-deepseek/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/llm/llm-deepseek/README.md -README.md: ea82956d77f3157638aa078c56630994ccc61d75 -README.zh.md: ff8780f59a3caac7e08e5ab6e08c4b2b15d1b57d +README.md: b20d93394055e3e10dfb5a932660b6a510428492 +README.zh.md: 6e8166227d740c0431c17c091d68b5d56aea0dc5 diff --git a/packages/llm/llm-deepseek/README.md b/packages/llm/llm-deepseek/README.md index ea82956d77..b20d933940 100644 --- a/packages/llm/llm-deepseek/README.md +++ b/packages/llm/llm-deepseek/README.md @@ -23,6 +23,7 @@ The package root exposes the Cordis plugin contract and `DeepSeekAdapter`; wire maxRequestFilesBytes: 134217728 # optional positive integer; 128 MiB raw request-image default maxImagesPerRequest: 600 # provider request image-count limit imageOffloadByteQuantum: 67108864 # oldest-image removal advances in 64 MiB steps + imageOffloadCountQuantum: 20 # count overflow advances in 20-image steps fileExpiresAfterSeconds: 604800 # uploaded image lifetime; 1 hour to 30 days fileRefreshMarginSeconds: 3600 # replace ids with less lifetime remaining fileQuotaCleanupBatch: 100 # oldest harness-owned files deleted before one quota retry @@ -48,13 +49,13 @@ The package root exposes the Cordis plugin contract and `DeepSeekAdapter`; wire The plugin registers the single provider route `deepseek-official` together with its resolved `retryPolicy`; omission resolves to normal mode with five retries. A request selects it with `provider: deepseek-official`; its `model` is passed through as the wire `model` string, so changing DeepSeek models does not require lifecycle-time registration. Omitting `models` advertises `deepseek-v4-flash`, `deepseek-v4-pro`, and the image-capable `deepseek-v4-flash-vision-exp`, each with a 1,000,000-token context window; an explicit list replaces those defaults, while `models: []` advertises none. Catalog entries are exposed through `ctx.llm.listModels('deepseek-official')` for clients such as ACP editors and the Web selector, but remain advisory: unlisted model ids still pass through unchanged as text-only routes. An omitted entry name defaults to its id, and omitted `inputModalities` means `text` only. -An image-capable catalog entry declares `inputModalities: [text, image]` and may set `imagePixelBudget`, `imageMaxBytes`, or `imageDetail: low`. The ordinary default is 640,000 total pixels and 1MiB encoded bytes; low detail defaults to 512 by 512 total pixels. The attachment store scales by `min(1, sqrt(pixelBudget / (width * height)))` and rounds inward to keep the pixel count at or below the hard cap, so a 2048 by 1024 master becomes about 1130 by 565 instead of a forced square. Request encoders run lazily: low-color images try PNG (palette only without alpha) then WebP 85 and 80, other alpha images try WebP 85 then 80, and other opaque images try JPEG 85 then 80; dimensions shrink only when both quality attempts exceed 1MiB. Concurrent generation of one `variantId` shares one transform. The adapter uploads the exact derived request bytes through `POST /files` and sends `{type: "file", file_id}` blocks. It never falls back to an inline data URL. Every retained image is preceded by stable text naming the complete attachment id, actual request dimensions, and the preview-coordinate arguments for `read_image_region`. User, tool-result, agent-loop, compaction, and direct `ctx.llm.stream` requests all use this projection. Text-only routes receive stable attachment placeholders while durable history keeps its image references. +An image-capable catalog entry declares `inputModalities: [text, image]` and may set `imagePixelBudget`, `imageMaxBytes`, or `imageDetail: low`. The ordinary default is 640,000 total pixels and 1MiB encoded bytes; low detail defaults to 512 by 512 total pixels. The attachment store scales by `min(1, sqrt(pixelBudget / (width * height)))` and rounds inward to keep the pixel count at or below the hard cap, so a 2048 by 1024 master becomes about 1130 by 565 instead of a forced square. Request encoders run lazily: low-color images try PNG (palette only without alpha) then WebP 85 and 80, other alpha images try WebP 85 then 80, and other opaque images try JPEG 85 then 80; dimensions shrink only when both quality attempts exceed 1MiB. Concurrent generation of one `variantId` shares one transform. A caller can cancel its own wait without interrupting other waiters; the transform stops when no waiter remains. The adapter uploads the exact derived request bytes through `POST /files` and sends `{type: "file", file_id}` blocks. It never falls back to an inline data URL. Every retained image is preceded by stable text naming the complete attachment id and actual request dimensions. Preview-coordinate arguments are included only when the request exposes `read_image_region`. User, tool-result, agent-loop, compaction, and direct `ctx.llm.stream` requests all use this projection. Text-only routes receive stable attachment placeholders while durable history keeps its image references. -`maxRequestFilesBytes` and `maxImagesPerRequest` bound the retained request versions at 128MiB and 600 images by default. When the byte bound is crossed, the oldest prefix advances past the next 64MiB boundary; 129 one-megabyte images remove the oldest 65 and retain 64MiB, and that prefix stays unchanged until durable history exceeds 192MiB. Count overflow advances independently in `imageOffloadCountQuantum` steps. Removed images become the fixed model-visible placeholder `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`. This high-watermark projection avoids changing an old request prefix after every new image. +`maxRequestFilesBytes` and `maxImagesPerRequest` bound the retained request versions at 128MiB and 600 images by default. The byte and count quanta must not exceed their corresponding bounds. Before attachment reads, the adapter uses each route's request-version byte cap as a conservative upper bound and removes the oldest over-budget prefix; only retained masters are read and transformed. Exact derived lengths are checked again without restoring omitted images. When the byte bound is crossed, the oldest prefix advances past the next 64MiB boundary; 129 one-megabyte images remove the oldest 65 and retain 64MiB, and that prefix stays unchanged until durable history exceeds 192MiB. Count overflow advances independently in `imageOffloadCountQuantum` steps. Removed images become the fixed model-visible placeholder `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`. This high-watermark projection avoids changing an old request prefix after every new image. Uploaded ids are indexed below `DSH_HOME` by endpoint/API-key scope and request `variantId`. The variant covers the master attachment id, transform version, route pixel and byte budgets, crop, and encoder parameters, so Files API and inline-capable adapters refer to the same deterministic bytes. Uploads request a seven-day lifetime by default and store the server's `expires_at`. A local mapping with no more than one hour remaining is replaced before use; the adapter does not retrieve every remote file before chat. If chat reports expired, deleted, missing, or invalid file ids and names one or more ids used by the request, the adapter removes exactly those mappings. If the provider identifies stale file state without naming an id, it removes every file mapping used by that chat attempt. It then uploads the affected request versions again and retries chat once. A second stale-file rejection clears the mappings identified by that response and is returned without a third chat attempt. An upload response without a complete file object, matching byte count, and `expires_at` is never indexed; a later request therefore uploads again instead of trusting inconsistent local state. A malformed local upload index is treated as an empty cache and replaced by the next successful upload; permission and filesystem I/O failures still fail the request. -One quota upload failure triggers deletion of the configured number of oldest `dsh-` files and one upload retry. `DeepSeekFilesClient.delete`, `DeepSeekFileStore.release`, and `releaseAll` expose explicit remote-space reclamation. The current provider limits represented by this package are 128MiB per Files upload, 32MiB per chat-referenced image, 10,000 stored files, and 25GiB per API key; the default 1MiB request version remains below the two per-file limits. +Concurrent resolution of one scoped `variantId` shares one Files upload with waiter-local cancellation. One quota upload failure first paginates and collects the configured number of oldest `dsh-` files, then deletes that set before one upload retry. `DeepSeekFilesClient.delete`, `DeepSeekFileStore.release`, and `releaseAll` expose explicit remote-space reclamation. The current provider limits represented by this package are 128MiB per Files upload, 32MiB per chat-referenced image, 10,000 stored files, and 25GiB per API key; the default 1MiB request version remains below the two per-file limits. `contextWindow` is optional per configured model and is not exposed through the advisory catalog. `ctx.llm.resolveModelInfo('deepseek-official', model).context` returns an exact model value first, then `defaultContextWindow` for an entry without capacity or an unlisted pass-through id. The adapter default is 1,000,000; pressure-sensitive plugins therefore get deployment-owned capacity without treating the model selector as authoritative. Registering another adapter for `deepseek-official` throws `LlmError('DUPLICATE_ADAPTER')`. @@ -80,7 +81,7 @@ The plugin also declares its route in the configurable-provider directory (`ctx. ## App attribution -Every request carries the shared attribution header from dsh-llm's `attributionHeaders()` - the mandatory `User-Agent` baseline identifying the harness (see [dsh-llm § App attribution](../llm/README.md#app-attribution-attributionts)). Direct DeepSeek requests and OpenAI-compatible gateway requests get no provider-specific app-attribution headers under this adapter contract; OpenRouter app attribution is deferred to a future explicit OpenRouter adapter or mode. A request whose `GenerateOptions.purpose` is `compaction` (dsh-compaction-basic's auxiliary summarization call) additionally carries `x-deepseek-harness-compact: 1`, so the host can separate compaction traffic from conversation requests. +Every chat and Files API request carries the shared attribution header from dsh-llm's `attributionHeaders()`, the mandatory `User-Agent` baseline identifying the harness (see [dsh-llm § App attribution](../llm/README.md#app-attribution-attributionts)). Direct DeepSeek requests and OpenAI-compatible gateway requests get no provider-specific app-attribution headers under this adapter contract; OpenRouter app attribution is deferred to a future explicit OpenRouter adapter or mode. A request whose `GenerateOptions.purpose` is `compaction` (dsh-compaction-basic's auxiliary summarization call) additionally carries `x-deepseek-harness-compact: 1`, so the host can separate compaction traffic from conversation requests. DeepSeek request identity is separate from app attribution. After credential resolution, every provider request carries `x-deepseek-harness-user-id` with the stable anonymous id from [`@deepseek-ai/dsh-anonymous-user-id`](../../identity/anonymous-user-id/README.md); a request carrying `GenerateOptions.sessionId` also sends that exact value as `x-deepseek-harness-session-id`, while a direct call without a session omits the session header. Both headers go to the resolved `baseURL`, including a configured gateway, and remain outside the request body and model-visible content. diff --git a/packages/llm/llm-deepseek/README.zh.md b/packages/llm/llm-deepseek/README.zh.md index ff8780f59a..6e8166227d 100644 --- a/packages/llm/llm-deepseek/README.zh.md +++ b/packages/llm/llm-deepseek/README.zh.md @@ -23,6 +23,7 @@ harness LLM(大语言模型)seam 的 DeepSeek chat-completions 适配器: maxRequestFilesBytes: 134217728 # optional positive integer; 128 MiB raw request-image default maxImagesPerRequest: 600 # provider request image-count limit imageOffloadByteQuantum: 67108864 # oldest-image removal advances in 64 MiB steps + imageOffloadCountQuantum: 20 # count overflow advances in 20-image steps fileExpiresAfterSeconds: 604800 # uploaded image lifetime; 1 hour to 30 days fileRefreshMarginSeconds: 3600 # replace ids with less lifetime remaining fileQuotaCleanupBatch: 100 # oldest harness-owned files deleted before one quota retry @@ -48,13 +49,13 @@ harness LLM(大语言模型)seam 的 DeepSeek chat-completions 适配器: 该插件注册唯一提供方路由 `deepseek-official`,并一同注册解析后的 `retryPolicy`;省略时会解析为 normal 模式并重试五次。请求使用 `provider: deepseek-official` 选择该路由;其 `model` 会作为协议 `model` 字符串原样传递,因此更改 DeepSeek 模型不需要生命周期时注册。省略 `models` 会公布 `deepseek-v4-flash`、`deepseek-v4-pro` 与支持图片输入的 `deepseek-v4-flash-vision-exp`,三者的上下文窗口均为 1,000,000 token;显式列表会替换这些默认值,`models: []` 则不公布任何模型。Catalog 配置项通过 `ctx.llm.listModels('deepseek-official')` 公开给 ACP(Agent Client Protocol)编辑器和 Web 选择器等客户端,但仍只提供建议:未列出模型 id 仍原样传递,并按纯文本路由处理。省略配置项 name 默认为其 id,省略 `inputModalities` 则表示仅支持 `text`。 -支持图片的 catalog 配置项声明 `inputModalities: [text, image]`,并可设置 `imagePixelBudget`、`imageMaxBytes` 或 `imageDetail: low`。普通默认值为总像素 640,000、编码字节 1MiB;low detail 的默认总像素为 512×512。附件存储按 `min(1, sqrt(pixelBudget / (width * height)))` 缩放,并向预算内取整,确保总像素不超过硬上限。因此 2048×1024 主版本会得到约 1130×565 的请求版本,而不会被强制变成正方形。请求编码按需执行:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。两个质量档均超过 1MiB 时才缩小尺寸。同一 `variantId` 的并发生成共享一次变换。适配器通过 `POST /files` 上传确切的派生请求字节,再发送 `{type: "file", file_id}` 块,不会回退到内联 data URL。每张保留图片前都有稳定文本,写明完整附件 ID、实际请求尺寸,以及 `read_image_region` 所需的预览坐标参数。User、工具结果、agent loop、压缩和直接 `ctx.llm.stream` 请求都使用该投影。纯文本路由会收到稳定的附件占位文本,持久历史继续保留图片引用。 +支持图片的 catalog 配置项声明 `inputModalities: [text, image]`,并可设置 `imagePixelBudget`、`imageMaxBytes` 或 `imageDetail: low`。普通默认值为总像素 640,000、编码字节 1MiB;low detail 的默认总像素为 512×512。附件存储按 `min(1, sqrt(pixelBudget / (width * height)))` 缩放,并向预算内取整,确保总像素不超过硬上限。因此 2048×1024 主版本会得到约 1130×565 的请求版本,而不会被强制变成正方形。请求编码按需执行:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。两个质量档均超过 1MiB 时才缩小尺寸。同一 `variantId` 的并发生成共享一次变换。调用方可以单独取消等待,不会中断其他等待方;没有等待方时才会停止变换。适配器通过 `POST /files` 上传确切的派生请求字节,再发送 `{type: "file", file_id}` 块,不会回退到内联 data URL。每张保留图片前都有稳定文本,写明完整附件 ID 和实际请求尺寸。只有当前请求公开 `read_image_region` 时才会提供预览坐标参数。User、工具结果、agent loop、压缩和直接 `ctx.llm.stream` 请求都使用该投影。纯文本路由会收到稳定的附件占位文本,持久历史继续保留图片引用。 -`maxRequestFilesBytes` 和 `maxImagesPerRequest` 限制请求中保留的请求版本,默认值分别为 128MiB 和 600 张。字节数越过上限时,被移除的最旧前缀会越过下一个 64MiB 边界。由 1MiB 图片组成的历史达到 129MiB 时会移除最旧的 65 张并保留 64MiB;直到持久历史超过 192MiB,这个前缀才再次变化。图片数量超限时则按 `imageOffloadCountQuantum` 独立递增。移除的图片会变成固定模型可见占位文本 `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`。这种定量投影不会因每新增一张图片就改写较早的请求前缀。 +`maxRequestFilesBytes` 和 `maxImagesPerRequest` 限制请求中保留的请求版本,默认值分别为 128MiB 和 600 张。字节和数量步长不得超过对应上限。读取附件前,适配器以路由的请求版本字节上限作为保守上界,移除超预算的最旧前缀,只读取并转换保留的主版本。系统随后用确切派生长度再次检查,但不会重新加入已省略图片。字节数越过上限时,被移除的最旧前缀会越过下一个 64MiB 边界。由 1MiB 图片组成的历史达到 129MiB 时会移除最旧的 65 张并保留 64MiB;直到持久历史超过 192MiB,这个前缀才再次变化。图片数量超限时则按 `imageOffloadCountQuantum` 独立递增。移除的图片会变成固定模型可见占位文本 `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`。这种定量投影不会因每新增一张图片就改写较早的请求前缀。 上传 ID 按端点和 API key 作用域以及请求 `variantId` 记录在 `DSH_HOME` 下。变体身份覆盖主附件 ID、变换策略版本、路由像素和字节预算、裁剪区域及编码参数,因此 Files API 和支持内联的适配器引用同一份确定性字节。上传默认请求 7 天有效期,并保存服务端返回的 `expires_at`。本地映射剩余时间不超过一小时时会在使用前替换;适配器不会在每次 chat 前查询远端文件。如果 chat 报告文件 ID 已过期、删除、缺失或无效,并指出本次请求使用的一个或多个 ID,适配器只删除这些映射。如果响应只说明文件状态失效而没有指出 ID,适配器会删除该次 chat 使用的全部文件映射。随后重新上传受影响的请求版本,并重试一次 chat。第二次 chat 仍报告文件失效时,适配器会按该响应清理映射并返回错误,不会发起第三次 chat。上传响应若没有完整文件对象、匹配的字节数和 `expires_at`,就不会写入索引;后续请求会再次上传,而不是信任不一致的本地状态。本地上传索引格式损坏时按空缓存处理,并由下一次成功上传替换;权限和文件系统 I/O 错误仍使请求失败。 -一次上传配额错误会触发删除配置数量的最旧 `dsh-` 文件,然后重试一次上传。`DeepSeekFilesClient.delete`、`DeepSeekFileStore.release` 和 `releaseAll` 提供主动远端空间回收。本包记录的当前提供方限制为 Files 单次上传 128MiB、chat 单图引用 32MiB、每个 API key 最多 10,000 个文件和 25GiB;默认 1MiB 请求版本低于两个单文件上限。 +同一作用域和 `variantId` 的并发解析共享一次 Files 上传,每个等待方可以单独取消。一次上传配额错误会先分页收集配置数量的最旧 `dsh-` 文件,再删除这些文件并重试一次上传。`DeepSeekFilesClient.delete`、`DeepSeekFileStore.release` 和 `releaseAll` 提供主动远端空间回收。本包记录的当前提供方限制为 Files 单次上传 128MiB、chat 单图引用 32MiB、每个 API key 最多 10,000 个文件和 25GiB;默认 1MiB 请求版本低于两个单文件上限。 `contextWindow` 对每个已配置模型都可选,不会通过建议 catalog 公开。`ctx.llm.resolveModelInfo('deepseek-official', model).context` 先返回精确模型值,再对不含容量的配置项或未列出原样传递 id 返回 `defaultContextWindow`。适配器默认值为 1,000,000;因此,压力敏感插件可以获得由部署决定的容量,不会将模型 selector 视为权威。为 `deepseek-official` 注册另一个适配器会抛出 `LlmError('DUPLICATE_ADAPTER')`。 @@ -80,7 +81,7 @@ harness LLM(大语言模型)seam 的 DeepSeek chat-completions 适配器: ## 应用归因 -每个请求都携带 dsh-llm `attributionHeaders()` 的共享归因标头,即用于识别 harness 的必需 `User-Agent` 基线(见 [dsh-llm § 应用归因](../llm/README.zh.md#app-attribution-attributionts))。在该适配器约定(adapter contract)下,直接 DeepSeek 请求与 OpenAI 兼容 gateway 请求都不会获得提供方特定应用归因标头;OpenRouter 应用归因暂缓到未来的显式 OpenRouter 适配器或模式。`GenerateOptions.purpose` 为 `compaction` 的请求(dsh-compaction-basic 的辅助摘要调用)还会携带 `x-deepseek-harness-compact: 1`,让宿主可以将压缩流量与会话请求分开。 +每个 chat 和 Files API 请求都携带 dsh-llm `attributionHeaders()` 的共享归因标头,即用于识别 harness 的必需 `User-Agent` 基线(见 [dsh-llm § 应用归因](../llm/README.zh.md#app-attribution-attributionts))。在该适配器约定(adapter contract)下,直接 DeepSeek 请求与 OpenAI 兼容 gateway 请求都不会获得提供方特定应用归因标头;OpenRouter 应用归因暂缓到未来的显式 OpenRouter 适配器或模式。`GenerateOptions.purpose` 为 `compaction` 的请求(dsh-compaction-basic 的辅助摘要调用)还会携带 `x-deepseek-harness-compact: 1`,让宿主可以将压缩流量与会话请求分开。 DeepSeek 请求身份独立于应用归因。凭据解析成功后,每个提供方请求都会通过 `x-deepseek-harness-user-id` 携带来自 [`@deepseek-ai/dsh-anonymous-user-id`](../../identity/anonymous-user-id/README.zh.md) 的稳定匿名 id;携带 `GenerateOptions.sessionId` 的请求还会通过 `x-deepseek-harness-session-id` 发送该确切值,缺少会话的直接调用则省略会话标头。两个标头都会发送至解析后的 `baseURL`(包括已配置的 gateway),且不会进入请求正文或模型可见内容。 diff --git a/packages/llm/llm-deepseek/src/adapter.ts b/packages/llm/llm-deepseek/src/adapter.ts index 006882d745..3a01d424ca 100644 --- a/packages/llm/llm-deepseek/src/adapter.ts +++ b/packages/llm/llm-deepseek/src/adapter.ts @@ -8,7 +8,7 @@ * @module dsh-llm-deepseek/adapter */ -import { attributionHeaders, contentHasImage, CONTEXT_WINDOW_EXCEEDED_CODE, isContextWindowExceededError, isQuotaExceededError, LlmAdapter, LlmError, ProviderRequestId, QUOTA_EXCEEDED_CODE, ReasoningEffortId } from '@deepseek-ai/dsh-llm' +import { attributionHeaders, contentHasImage, CONTEXT_WINDOW_EXCEEDED_CODE, isContextWindowExceededError, isQuotaExceededError, LlmAdapter, LlmError, offloadRequestImagesWithPolicy, ProviderRequestId, QUOTA_EXCEEDED_CODE, ReasoningEffortId } from '@deepseek-ai/dsh-llm' import type { ContentBlock, GenerateOptions, @@ -510,14 +510,24 @@ export class DeepSeekAdapter extends LlmAdapter { const fileConnection = { baseURL: connection.baseURL, apiKey } const model = connection.models.find(entry => entry.id === options.model) + const policy = model === undefined ? undefined : resolveRequestImagePolicy(model) + const requestMessages = policy === undefined ? options.messages : offloadRequestImagesWithPolicy(options.messages, { + representation: 'raw', + maxBytes: connection.maxRequestFilesBytes, + maxImages: connection.maxImagesPerRequest, + byteQuantum: connection.imageOffloadByteQuantum, + countQuantum: connection.imageOffloadCountQuantum, + byteLength: ref => Math.min(ref.bytes, policy.maxBytes), + }) + const requestOptions = requestMessages === options.messages ? options : { ...options, messages: [...requestMessages] } const requestImages = attachments === undefined || model === undefined ? new Map() - : await prepareRequestImages(options, attachments, model, signal) + : await prepareRequestImages(requestOptions, attachments, model, signal) for (let fileAttempt = 0; fileAttempt < 2; fileAttempt += 1) { const usedFiles: UsedRequestFile[] = [] const body = attachments === undefined - ? serializeRequest(options, connection.defaults) - : await serializeRequestWithImages(options, { + ? serializeRequest(requestOptions, connection.defaults) + : await serializeRequestWithImages(requestOptions, { requestImages, resolveFileId: async (version, _block, location) => { const resolved = await this.files.ensureUploaded( @@ -533,6 +543,7 @@ export class DeepSeekAdapter extends LlmAdapter { maxImagesPerRequest: connection.maxImagesPerRequest, byteQuantum: connection.imageOffloadByteQuantum, countQuantum: connection.imageOffloadCountQuantum, + cropAvailable: options.tools?.some(tool => tool.name === 'read_image_region') ?? false, }, connection.defaults) const payload = JSON.stringify(body) diff --git a/packages/llm/llm-deepseek/src/file-store.ts b/packages/llm/llm-deepseek/src/file-store.ts index 1ec943a7f9..ddaefd1eee 100644 --- a/packages/llm/llm-deepseek/src/file-store.ts +++ b/packages/llm/llm-deepseek/src/file-store.ts @@ -36,6 +36,51 @@ interface FileStoreOptions { fetch?: typeof fetch } +interface SharedUpload { + controller: AbortController + promise: Promise + settled: boolean + waiters: number +} + +function abortReason(signal: AbortSignal): Error { + const reason: unknown = signal.reason + return reason instanceof Error + ? reason + : new Error('DeepSeek file upload cancelled with a non-Error reason.', { cause: reason }) +} + +function waitForUpload(operation: SharedUpload, signal: AbortSignal | undefined): Promise { + signal?.throwIfAborted() + operation.waiters += 1 + let released = false + const release = (cancelled: boolean): void => { + if (released) return + released = true + operation.waiters -= 1 + if (cancelled && operation.waiters === 0 && !operation.settled) { + operation.controller.abort(signal === undefined ? undefined : abortReason(signal)) + } + } + if (signal === undefined) return operation.promise.finally(() => release(false)) + return new Promise((resolve, reject) => { + const abort = (): void => { + release(true) + reject(abortReason(signal)) + } + signal.addEventListener('abort', abort, { once: true }) + void operation.promise.then((value) => { + signal.removeEventListener('abort', abort) + release(false) + resolve(value) + }, (error: unknown) => { + signal.removeEventListener('abort', abort) + release(false) + reject(error) + }) + }) +} + function extension(mediaType: RequestImageAttachment['mediaType']): 'png' | 'jpeg' | 'webp' | 'gif' { switch (mediaType) { case 'image/png': return 'png' @@ -56,7 +101,7 @@ export class DeepSeekFileStore { private readonly index: DeepSeekUploadIndex private readonly now: () => number private readonly fetchImpl: typeof fetch | undefined - private readonly inflight = new Map>() + private readonly inflight = new Map() /** * @param options - testable index, clock, and transport boundaries. @@ -76,11 +121,11 @@ export class DeepSeekFileStore { } /** - * Resolve or upload one deterministic request image. Concurrent calls in this process share one promise. + * Resolve or upload one deterministic request image. Concurrent calls share one upload while retaining independent waits. * @param version - deterministic model-request bytes and complete transformation identity. * @param connection - endpoint and API-key snapshot. * @param policy - expiry and quota-recovery policy. - * @param signal - request cancellation. + * @param signal - cancellation of this wait; shared transport stops when no waiter remains. * @returns a reusable file id and whether this call published a new upload. */ ensureUploaded( @@ -89,16 +134,34 @@ export class DeepSeekFileStore { policy: DeepSeekFilePolicy, signal?: AbortSignal, ): Promise { + signal?.throwIfAborted() const scope = deepSeekFileScope(connection.baseURL, connection.apiKey) const key = `${scope}\0${version.variantId}` - const active = this.inflight.get(key) - if (active !== undefined) return active - const operation = this.ensureUploadedOnce(version, connection, policy, signal) - this.inflight.set(key, operation) - void operation.finally(() => { - if (this.inflight.get(key) === operation) this.inflight.delete(key) + let active = this.inflight.get(key) + if (active?.controller.signal.aborted) { + this.inflight.delete(key) + active = undefined + } + if (active !== undefined) return waitForUpload(active, signal) + const controller = new AbortController() + const shared: SharedUpload = { + controller, + settled: false, + waiters: 0, + promise: undefined as never, + } + shared.promise = this.ensureUploadedOnce(version, connection, policy, controller.signal).then((value) => { + shared.settled = true + return value + }, (error: unknown) => { + shared.settled = true + throw error + }) + this.inflight.set(key, shared) + void shared.promise.finally(() => { + if (this.inflight.get(key) === shared) this.inflight.delete(key) }).catch(() => {}) - return operation + return waitForUpload(shared, signal) } private async ensureUploadedOnce( @@ -218,8 +281,8 @@ export class DeepSeekFileStore { ): Promise { const client = this.client(connection) let after: DeepSeekFileId | undefined - let deleted = 0 - while (deleted < count) { + const owned: DeepSeekFileId[] = [] + while (owned.length < count) { const page = await client.list({ ...after === undefined ? {} : { after }, limit: 1_000, @@ -228,14 +291,14 @@ export class DeepSeekFileStore { }) for (const file of page.data) { if (!file.filename.startsWith(OWNED_FILE_PREFIX)) continue - await client.delete(file.id, signal) - deleted += 1 - if (deleted === count) break + owned.push(file.id) + if (owned.length === count) break } if (!page.hasMore || page.lastId === undefined || page.lastId === after) break after = page.lastId } - return deleted + for (const fileId of owned) await client.delete(fileId, signal) + return owned.length } /** diff --git a/packages/llm/llm-deepseek/src/files-api.ts b/packages/llm/llm-deepseek/src/files-api.ts index 90ddf20b8c..f19823100e 100644 --- a/packages/llm/llm-deepseek/src/files-api.ts +++ b/packages/llm/llm-deepseek/src/files-api.ts @@ -1,6 +1,6 @@ /** OpenAI-compatible DeepSeek Files API transport. @module dsh-llm-deepseek/files-api */ -import { LlmError } from '@deepseek-ai/dsh-llm' +import { attributionHeaders, LlmError } from '@deepseek-ai/dsh-llm' import type { ImageMediaType } from '@deepseek-ai/dsh-attachment' import { DeepSeekFileId } from './file-id.ts' import type { DeepSeekFileId as DeepSeekFileIdType } from './file-id.ts' @@ -142,7 +142,8 @@ export class DeepSeekFilesClient { private async request(path: string, init: RequestInit, signal?: AbortSignal): Promise { let response: Response try { - const headers = new Headers(init.headers) + const headers = new Headers(attributionHeaders()) + for (const [name, value] of new Headers(init.headers)) headers.set(name, value) headers.set('authorization', `Bearer ${this.apiKey}`) response = await this.fetchImpl(`${this.baseURL}${path}`, { ...init, diff --git a/packages/llm/llm-deepseek/src/index.ts b/packages/llm/llm-deepseek/src/index.ts index 6def44f2d3..919168439d 100644 --- a/packages/llm/llm-deepseek/src/index.ts +++ b/packages/llm/llm-deepseek/src/index.ts @@ -291,10 +291,16 @@ export function resolveAdapterOptions(config: Config, environment?: LaunchEnviro if (!Number.isSafeInteger(imageOffloadByteQuantum) || imageOffloadByteQuantum <= 0) { throw new Error('llm-deepseek: imageOffloadByteQuantum must be a positive safe integer') } + if (imageOffloadByteQuantum > maxRequestFilesBytes) { + throw new Error('llm-deepseek: imageOffloadByteQuantum must not exceed maxRequestFilesBytes') + } const imageOffloadCountQuantum = config.imageOffloadCountQuantum ?? DEFAULT_IMAGE_OFFLOAD_COUNT_QUANTUM if (!Number.isSafeInteger(imageOffloadCountQuantum) || imageOffloadCountQuantum <= 0) { throw new Error('llm-deepseek: imageOffloadCountQuantum must be a positive safe integer') } + if (imageOffloadCountQuantum > maxImagesPerRequest) { + throw new Error('llm-deepseek: imageOffloadCountQuantum must not exceed maxImagesPerRequest') + } const fileExpiresAfterSeconds = config.fileExpiresAfterSeconds ?? DEFAULT_FILE_EXPIRY_SECONDS if (!Number.isSafeInteger(fileExpiresAfterSeconds) || fileExpiresAfterSeconds < 3_600 diff --git a/packages/llm/llm-deepseek/src/serialize.ts b/packages/llm/llm-deepseek/src/serialize.ts index f066808b99..a65c9750c3 100644 --- a/packages/llm/llm-deepseek/src/serialize.ts +++ b/packages/llm/llm-deepseek/src/serialize.ts @@ -37,7 +37,7 @@ export interface ImageSerializationOptions { block: Extract, location: ImageWireLocation, ) => Promise - /** Request versions prepared before offload selection, keyed by master attachment id. */ + /** Request versions prepared for the conservatively retained masters, keyed by attachment id. */ requestImages: ReadonlyMap /** Positive bound on accumulated referenced image bytes. */ maxRequestFilesBytes: number @@ -47,6 +47,8 @@ export interface ImageSerializationOptions { byteQuantum?: number /** Image-count removal step applied after the request exceeds its count bound. */ countQuantum?: number + /** Whether the active request exposes the region-read tool. */ + cropAvailable?: boolean } /** Durable message and image ordinal used in provider diagnostics. */ @@ -115,10 +117,14 @@ function assertSupportedImageRoles(messages: readonly Message[]): void { } /** Describe the exact request preview and its model-callable coordinate system. */ -function imageHandle(version: RequestImageAttachment, precededByContent: boolean): WireTextContentPart { +function imageHandle( + version: RequestImageAttachment, + precededByContent: boolean, + cropAvailable: boolean, +): WireTextContentPart { return { type: 'text', - text: `${precededByContent ? '\n' : ''}${requestImagePreviewText(version)}`, + text: `${precededByContent ? '\n' : ''}${requestImagePreviewText(version, cropAvailable)}`, } } @@ -137,7 +143,7 @@ async function imageParts( ) } return [ - imageHandle(version, precededByContent), + imageHandle(version, precededByContent, images.cropAvailable === true), { type: 'file', file_id: await images.resolveFileId(version, block, location) }, ] } diff --git a/packages/llm/llm-deepseek/tests/adapter.spec.ts b/packages/llm/llm-deepseek/tests/adapter.spec.ts index 23db2ed009..731df2eb0f 100644 --- a/packages/llm/llm-deepseek/tests/adapter.spec.ts +++ b/packages/llm/llm-deepseek/tests/adapter.spec.ts @@ -206,6 +206,49 @@ describe('DeepSeekAdapter against a mock server', () => { expect(policies).toEqual([{ maxPixels: 640_000, maxBytes: 1024 * 1024 }]) }) + it('does not prepare an old image removed by request offload', async () => { + const server = await mockServer([{ kind: 'sse', events: textEvents }]) + const old = { ...imageRef, attachmentId: AttachmentId(`sha256:${'c'.repeat(64)}`), bytes: 3 } + const recent = { ...imageRef, attachmentId: AttachmentId(`sha256:${'d'.repeat(64)}`), bytes: 3 } + const attachmentMocks = attachmentStoreOf((ref) => { + if (ref.attachmentId === old.attachmentId) throw new Error('old image must not be read') + return Promise.resolve(requestImage(ref)) + }) + const adapter = adapterOf({ + baseURL: server.url, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + maxRequestFilesBytes: 4, + imageOffloadByteQuantum: 2, + }, attachmentMocks.store) + + await drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [ + { type: 'image', attachment: old }, + { type: 'image', attachment: recent }, + ], + source: { kind: 'plugin', plugin: 'test' }, + })], + })) + + expect(attachmentMocks.readImageRequests).toHaveBeenCalledWith( + [recent], + { maxPixels: 640_000, maxBytes: 1024 * 1024 }, + expect.any(AbortSignal), + ) + const body = server.requests[0] as { messages: unknown[] } + expect(body.messages[0]).toMatchObject({ + role: 'user', + content: [ + { type: 'text', text: expect.stringContaining('older images are omitted first') as string }, + { type: 'text', text: expect.stringContaining(String(recent.attachmentId)) as string }, + { type: 'file', file_id: 'file-api-1' }, + ], + }) + }) + it('projects nested tool-result images with route-owned request budgets', async () => { const server = await mockServer([ { kind: 'sse', events: textEvents }, @@ -1515,6 +1558,17 @@ describe('plugin registration and config', () => { }, ) + it('rejects offload quanta larger than their request bounds', () => { + expect(() => resolveAdapterOptions({ + maxRequestFilesBytes: 10, + imageOffloadByteQuantum: 11, + })).toThrow(/imageOffloadByteQuantum must not exceed maxRequestFilesBytes/) + expect(() => resolveAdapterOptions({ + maxImagesPerRequest: 10, + imageOffloadCountQuantum: 11, + })).toThrow(/imageOffloadCountQuantum must not exceed maxImagesPerRequest/) + }) + it.each([0, 1.5, Number.MAX_SAFE_INTEGER + 1])( 'rejects invalid request file bound %s', async (maxRequestFilesBytes) => { diff --git a/packages/llm/llm-deepseek/tests/file-store.spec.ts b/packages/llm/llm-deepseek/tests/file-store.spec.ts index 041fd0a0c6..6d9e940786 100644 --- a/packages/llm/llm-deepseek/tests/file-store.spec.ts +++ b/packages/llm/llm-deepseek/tests/file-store.spec.ts @@ -81,6 +81,63 @@ describe('DeepSeekFileStore', () => { expect(remote.uploads()).toBe(1) }) + it('keeps a shared upload alive while another waiter remains', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + const index = new DeepSeekUploadIndex(join(dir, 'index.json')) + let complete: ((response: Response) => void) | undefined + let uploadSignal: AbortSignal | undefined + const fetchImpl = vi.fn((_url: string | URL | Request, init?: RequestInit) => { + uploadSignal = init?.signal ?? undefined + return new Promise((resolve, reject) => { + complete = resolve + uploadSignal?.addEventListener('abort', () => reject(uploadSignal?.reason), { once: true }) + }) + }) as typeof fetch + const store = new DeepSeekFileStore({ index, now: () => NOW, fetch: fetchImpl }) + const controller = new AbortController() + + const cancelled = store.ensureUploaded(VERSION, CONNECTION, POLICY, controller.signal) + const completed = store.ensureUploaded(VERSION, CONNECTION, POLICY) + await vi.waitFor(() => expect(fetchImpl).toHaveBeenCalledTimes(1)) + const reason = new Error('cancel one upload waiter') + controller.abort(reason) + + await expect(cancelled).rejects.toBe(reason) + expect(uploadSignal?.aborted).toBe(false) + complete?.(new Response(JSON.stringify({ + id: 'file-api-shared', + object: 'file', + bytes: 3, + created_at: NOW / 1_000, + filename: `dsh-${'a'.repeat(16)}-${'b'.repeat(8)}.png`, + purpose: 'user_data', + expires_at: NOW / 1_000 + POLICY.expiresAfterSeconds, + }), { status: 200 })) + await expect(completed).resolves.toMatchObject({ record: { fileId: 'file-api-shared' } }) + }) + + it('aborts the shared upload after its only waiter cancels', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + const index = new DeepSeekUploadIndex(join(dir, 'index.json')) + let uploadSignal: AbortSignal | undefined + const fetchImpl = vi.fn((_url: string | URL | Request, init?: RequestInit) => { + uploadSignal = init?.signal ?? undefined + return new Promise((_resolve, reject) => { + uploadSignal?.addEventListener('abort', () => reject(uploadSignal?.reason), { once: true }) + }) + }) as typeof fetch + const store = new DeepSeekFileStore({ index, now: () => NOW, fetch: fetchImpl }) + const controller = new AbortController() + const upload = store.ensureUploaded(VERSION, CONNECTION, POLICY, controller.signal) + await vi.waitFor(() => expect(fetchImpl).toHaveBeenCalledTimes(1)) + + const reason = new Error('cancel only upload waiter') + controller.abort(reason) + + await expect(upload).rejects.toBe(reason) + expect(uploadSignal?.reason).toBe(reason) + }) + it('does not persist an upload whose response is missing and retries on the next request', async () => { const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) const index = new DeepSeekUploadIndex(join(dir, 'index.json')) @@ -132,4 +189,42 @@ describe('DeepSeekFileStore', () => { await expect(store.release(VERSION, CONNECTION, POLICY)).resolves.toBe(false) expect(remote.fetchImpl).toHaveBeenCalledTimes(2) }) + + it('finishes pagination before deleting cursor files during quota recovery', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + const deleted = new Set() + const fetchImpl = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { + const target = new URL(requestUrl(input)) + if (init?.method === 'DELETE') { + const id = target.pathname.split('/').at(-1) ?? '' + deleted.add(id) + return new Response(JSON.stringify({ id, object: 'file', deleted: true }), { status: 200 }) + } + const after = target.searchParams.get('after') + if (after !== null && deleted.has(after)) throw new Error('deleted cursor cannot be reused') + const id = after === null ? 'file-api-oldest' : 'file-api-next' + return new Response(JSON.stringify({ + object: 'list', + data: [{ + id, + object: 'file', + bytes: 3, + created_at: NOW / 1_000, + filename: `dsh-${id}.png`, + purpose: 'user_data', + }], + first_id: id, + last_id: id, + has_more: after === null, + }), { status: 200 }) + }) as typeof fetch + const store = new DeepSeekFileStore({ + index: new DeepSeekUploadIndex(join(dir, 'index.json')), + now: () => NOW, + fetch: fetchImpl, + }) + + await expect(store.reclaimOldestOwned(CONNECTION, 2)).resolves.toBe(2) + expect([...deleted]).toEqual(['file-api-oldest', 'file-api-next']) + }) }) diff --git a/packages/llm/llm-deepseek/tests/files-api.spec.ts b/packages/llm/llm-deepseek/tests/files-api.spec.ts index 752c659a7f..466c9be83b 100644 --- a/packages/llm/llm-deepseek/tests/files-api.spec.ts +++ b/packages/llm/llm-deepseek/tests/files-api.spec.ts @@ -1,6 +1,12 @@ import { describe, expect, it, vi } from 'vitest' +import { userAgent } from '@deepseek-ai/dsh-llm' import { DeepSeekFileId } from '../src/file-id.ts' -import { DeepSeekFilesClient, isFilesQuotaError } from '../src/files-api.ts' +import { + DeepSeekFilesClient, + DeepSeekFilesError, + isFilesQuotaError, + MAX_FILE_UPLOAD_BYTES, +} from '../src/files-api.ts' function requestUrl(input: string | URL | Request): string { if (typeof input === 'string') return input @@ -25,7 +31,9 @@ describe('DeepSeekFilesClient', () => { const fetchImpl = vi.fn(async (url: string | URL | Request, init?: RequestInit) => { expect(requestUrl(url)).toBe('https://api.deepseek.com/files') expect(init?.method).toBe('POST') - expect(new Headers(init?.headers).get('authorization')).toBe('Bearer key') + const headers = new Headers(init?.headers) + expect(headers.get('authorization')).toBe('Bearer key') + expect(headers.get('user-agent')).toBe(userAgent()) const form = init?.body expect(form).toBeInstanceOf(FormData) if (!(form instanceof FormData)) throw new Error('expected multipart body') @@ -69,7 +77,7 @@ describe('DeepSeekFilesClient', () => { }) as typeof fetch const client = new DeepSeekFilesClient({ baseURL: 'https://api.deepseek.com', apiKey: 'key', fetch: fetchImpl }) - await expect(client.list({ limit: 20, order: 'desc' })).resolves.toMatchObject({ + await expect(client.list({ after: DeepSeekFileId('file-api-before'), limit: 20, order: 'desc' })).resolves.toMatchObject({ data: [{ id: 'file-api-one' }], firstId: 'file-api-one', lastId: 'file-api-one', hasMore: false, }) await expect(client.retrieve(DeepSeekFileId('file-api-one'))).resolves.toMatchObject({ id: 'file-api-one' }) @@ -98,5 +106,155 @@ describe('DeepSeekFilesClient', () => { data: Uint8Array.of(1), mediaType: 'image/png', filename: 'image.png', expiresAfterSeconds: 3_600, }).catch((caught: unknown) => caught) expect(isFilesQuotaError(error)).toBe(true) + expect(isFilesQuotaError(new Error('storage quota'))).toBe(false) + }) + + it.each([ + [401, 'AUTH'], + [403, 'AUTH'], + [429, 'RATE_LIMIT'], + [500, 'SERVER'], + [400, 'FILES_API'], + ] as const)('classifies HTTP %i Files failures as %s', async (status, code) => { + const client = new DeepSeekFilesClient({ + baseURL: 'https://api.deepseek.com', + apiKey: 'key', + fetch: vi.fn(() => Promise.resolve(new Response('not-json', { status }))) as typeof fetch, + }) + await expect(client.retrieve(DeepSeekFileId('missing'))).rejects.toMatchObject({ + name: 'DeepSeekFilesError', + code, + detail: '', + }) + }) + + it.each([ + null, + [], + {}, + { error: null }, + { error: [] }, + { error: { message: 1, type: 2, code: 3 } }, + ])('falls back to the HTTP status for an unstructured provider error %#', async (body) => { + const client = new DeepSeekFilesClient({ + baseURL: 'https://api.deepseek.com', + apiKey: 'key', + fetch: vi.fn(() => Promise.resolve(new Response(JSON.stringify(body), { status: 400 }))) as typeof fetch, + }) + const error = await client.retrieve(DeepSeekFileId('missing')).catch((caught: unknown) => caught) + expect(error).toBeInstanceOf(DeepSeekFilesError) + expect(error).toMatchObject({ message: 'DeepSeek Files API error (HTTP 400)', detail: '' }) + }) + + it('wraps transport failures but preserves an aborted request reason', async () => { + const transport = new Error('socket closed') + const client = new DeepSeekFilesClient({ + baseURL: 'https://api.deepseek.com', + apiKey: 'key', + fetch: vi.fn(() => Promise.reject(transport)) as typeof fetch, + }) + await expect(client.retrieve(DeepSeekFileId('one'))).rejects.toMatchObject({ + code: 'TRANSPORT', + cause: transport, + }) + + const controller = new AbortController() + const reason = new Error('cancelled') + controller.abort(reason) + await expect(client.retrieve(DeepSeekFileId('one'), controller.signal)).rejects.toBe(transport) + }) + + it.each([ + null, + [], + file({ id: 1 }), + file({ id: '' }), + file({ object: 'wrong' }), + file({ bytes: 1.5 }), + file({ bytes: -1 }), + file({ created_at: 1.5 }), + file({ created_at: -1 }), + file({ filename: 1 }), + file({ filename: '' }), + file({ purpose: 'assistants' }), + file({ expires_at: 1.5 }), + file({ expires_at: -1 }), + ])('rejects an invalid file object %#', async (body) => { + const client = new DeepSeekFilesClient({ + baseURL: 'https://api.deepseek.com', + apiKey: 'key', + fetch: vi.fn(() => Promise.resolve(new Response(JSON.stringify(body), { status: 200 }))) as typeof fetch, + }) + await expect(client.retrieve(DeepSeekFileId('one'))).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }) + }) + + it.each([ + 3_599, + 2_592_001, + 3_600.5, + ])('refuses invalid file expiry %s before transport', async (expiresAfterSeconds) => { + const fetchImpl = vi.fn() as typeof fetch + const client = new DeepSeekFilesClient({ baseURL: 'https://api.deepseek.com', apiKey: 'key', fetch: fetchImpl }) + await expect(client.upload({ + data: Uint8Array.of(1), mediaType: 'image/png', filename: 'image.png', expiresAfterSeconds, + })).rejects.toMatchObject({ code: 'INVALID_REQUEST' }) + expect(fetchImpl).not.toHaveBeenCalled() + }) + + it('refuses a file larger than the upload limit before transport', async () => { + const fetchImpl = vi.fn() as typeof fetch + const client = new DeepSeekFilesClient({ baseURL: 'https://api.deepseek.com', apiKey: 'key', fetch: fetchImpl }) + const data = { byteLength: MAX_FILE_UPLOAD_BYTES + 1 } as Uint8Array + await expect(client.upload({ + data, mediaType: 'image/png', filename: 'image.png', expiresAfterSeconds: 3_600, + })).rejects.toMatchObject({ code: 'INVALID_REQUEST' }) + expect(fetchImpl).not.toHaveBeenCalled() + }) + + it.each([ + null, + [], + {}, + { object: 'wrong', data: [], has_more: false }, + { object: 'list', data: null, has_more: false }, + { object: 'list', data: [], has_more: 0 }, + { object: 'list', data: [], has_more: false, first_id: 1 }, + { object: 'list', data: [], has_more: false, last_id: 1 }, + ])('rejects an invalid list response %#', async (body) => { + const client = new DeepSeekFilesClient({ + baseURL: 'https://api.deepseek.com', + apiKey: 'key', + fetch: vi.fn(() => Promise.resolve(new Response(JSON.stringify(body), { status: 200 }))) as typeof fetch, + }) + await expect(client.list()).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }) + }) + + it('accepts a list without cursors and uses the global fetch default', async () => { + const fetchImpl = vi.fn(() => Promise.resolve(new Response(JSON.stringify({ + object: 'list', data: [], has_more: false, + }), { status: 200 }))) + vi.stubGlobal('fetch', fetchImpl) + try { + const client = new DeepSeekFilesClient({ baseURL: 'https://api.deepseek.com///', apiKey: 'key' }) + await expect(client.list()).resolves.toEqual({ data: [], hasMore: false }) + } finally { + vi.unstubAllGlobals() + } + }) + + it.each([ + null, + [], + {}, + { id: 'wrong', object: 'file', deleted: true }, + { id: 'file-api-one', object: 'wrong', deleted: true }, + { id: 'file-api-one', object: 'file', deleted: false }, + ])('rejects an invalid delete response %#', async (body) => { + const client = new DeepSeekFilesClient({ + baseURL: 'https://api.deepseek.com', + apiKey: 'key', + fetch: vi.fn(() => Promise.resolve(new Response(JSON.stringify(body), { status: 200 }))) as typeof fetch, + }) + await expect(client.delete(DeepSeekFileId('file-api-one'))).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }) }) }) diff --git a/packages/llm/llm-deepseek/tests/serialize.spec.ts b/packages/llm/llm-deepseek/tests/serialize.spec.ts index 04717da6e0..8e04b9b3b0 100644 --- a/packages/llm/llm-deepseek/tests/serialize.spec.ts +++ b/packages/llm/llm-deepseek/tests/serialize.spec.ts @@ -60,6 +60,7 @@ function imageOptions( resolveFileId, requestImages: new Map(refs.map(ref => [ref.attachmentId, requestVersion(ref)])), maxRequestFilesBytes, + cropAvailable: true, } } @@ -378,6 +379,26 @@ describe('image serialization', () => { }]) }) + it('does not advertise region reads when the request omits that tool', async () => { + const ref = imageRef() + const images = { ...imageOptions([ref]), cropAvailable: false } + const wire = await serializeRequestWithImages(request({ + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [{ type: 'image', attachment: ref }], + source: { kind: 'plugin', plugin: 'test' }, + })], + }), images) + + expect(wire.messages[0]).toMatchObject({ + role: 'user', + content: [ + { type: 'text', text: `Image ${ref.attachmentId}; preview 1x1px.` }, + { type: 'file', file_id: 'file-api-image' }, + ], + }) + }) + it('keeps tool content textual and groups consecutive tool-result images afterward', async () => { const messages = [ createUserMessage({ diff --git a/packages/llm/llm-deepseek/tests/upload-index.spec.ts b/packages/llm/llm-deepseek/tests/upload-index.spec.ts index 6157adc06e..480772f5fb 100644 --- a/packages/llm/llm-deepseek/tests/upload-index.spec.ts +++ b/packages/llm/llm-deepseek/tests/upload-index.spec.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, writeFile } from 'node:fs/promises' +import { mkdir, mkdtemp, readFile, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { describe, expect, it } from 'vitest' @@ -10,6 +10,11 @@ const ATTACHMENT = AttachmentId(`sha256:${'a'.repeat(64)}`) const VARIANT = ImageVariantId(`sha256:${'b'.repeat(64)}`) describe('DeepSeekUploadIndex', () => { + it('normalizes trailing endpoint slashes in the credential scope', () => { + expect(deepSeekFileScope('https://api.deepseek.com///', 'key')) + .toBe(deepSeekFileScope('https://api.deepseek.com', 'key')) + }) + it('isolates API-key namespaces and reuses only records above the refresh margin', async () => { const dir = await mkdtemp(join(tmpdir(), 'dsh-upload-index-')) const index = new DeepSeekUploadIndex(join(dir, 'index.json')) @@ -70,4 +75,106 @@ describe('DeepSeekUploadIndex', () => { await expect(index.get(scope, VARIANT, 1, 1)).resolves.toEqual(record) expect(JSON.parse(await readFile(path, 'utf8'))).toMatchObject({ formatVersion: 2 }) }) + + it.each([ + 'null', + '[]', + '{}', + '{"formatVersion":1,"records":[]}', + '{"formatVersion":2,"records":null}', + '{"formatVersion":2,"records":[null]}', + '{"formatVersion":2,"records":[[]]}', + '{"formatVersion":2,"records":[{}]}', + `{"formatVersion":2,"records":[${JSON.stringify({ + scope: 'x'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + fileId: 'file-api-one', bytes: 3, createdAt: 1, expiresAt: 10_000, + })}]}`, + `{"formatVersion":2,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), masterAttachmentId: 'wrong', variantId: VARIANT, + fileId: 'file-api-one', bytes: 3, createdAt: 1, expiresAt: 10_000, + })}]}`, + `{"formatVersion":2,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: 'wrong', + fileId: 'file-api-one', bytes: 3, createdAt: 1, expiresAt: 10_000, + })}]}`, + `{"formatVersion":2,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + fileId: '', bytes: 3, createdAt: 1, expiresAt: 10_000, + })}]}`, + `{"formatVersion":2,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + fileId: 'file-api-one', bytes: -1, createdAt: 1, expiresAt: 10_000, + })}]}`, + `{"formatVersion":2,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + fileId: 'file-api-one', bytes: 1.5, createdAt: 1, expiresAt: 10_000, + })}]}`, + `{"formatVersion":2,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + fileId: 'file-api-one', bytes: 3, createdAt: -1, expiresAt: 10_000, + })}]}`, + `{"formatVersion":2,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + fileId: 'file-api-one', bytes: 3, createdAt: 1.5, expiresAt: 10_000, + })}]}`, + `{"formatVersion":2,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + fileId: 'file-api-one', bytes: 3, createdAt: 1, expiresAt: -1, + })}]}`, + `{"formatVersion":2,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + fileId: 'file-api-one', bytes: 3, createdAt: 1, expiresAt: 1.5, + })}]}`, + ])('treats an invalid persisted index as empty %#', async (text) => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-upload-index-')) + const path = join(dir, 'index.json') + await writeFile(path, text, 'utf8') + const index = new DeepSeekUploadIndex(path) + await expect(index.get( + deepSeekFileScope('https://api.deepseek.com', 'key'), VARIANT, 1, 1, + )).resolves.toBeUndefined() + }) + + it('rejects duplicate persisted mappings as a corrupt cache', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-upload-index-')) + const path = join(dir, 'index.json') + const scope = deepSeekFileScope('https://api.deepseek.com', 'key') + const record = { + scope, masterAttachmentId: ATTACHMENT, variantId: VARIANT, + fileId: DeepSeekFileId('file-api-one'), bytes: 3, createdAt: 1, expiresAt: 10_000, + } + await writeFile(path, JSON.stringify({ formatVersion: 2, records: [record, record] }), 'utf8') + const index = new DeepSeekUploadIndex(path) + await expect(index.get(scope, VARIANT, 1, 1)).resolves.toBeUndefined() + }) + + it('drops expired records on commit and clears only the selected namespace', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-upload-index-')) + const index = new DeepSeekUploadIndex(join(dir, 'index.json')) + const first = deepSeekFileScope('https://api.deepseek.com', 'first') + const second = deepSeekFileScope('https://api.deepseek.com', 'second') + const expired = { + scope: first, masterAttachmentId: ATTACHMENT, variantId: VARIANT, + fileId: DeepSeekFileId('file-api-expired'), bytes: 3, createdAt: 1, expiresAt: 2, + } + const live = { + ...expired, scope: second, fileId: DeepSeekFileId('file-api-live'), expiresAt: 10_000, + } + await index.commit(expired, 0, 0) + await index.commit(live, 3, 1) + await index.clear(first) + await index.clear(second) + await expect(index.get(second, VARIANT, 3, 1)).resolves.toBeUndefined() + await index.clear(second) + }) + + it('propagates non-cache filesystem read failures', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-upload-index-')) + const path = join(dir, 'directory') + await mkdir(path) + const index = new DeepSeekUploadIndex(path) + await expect(index.get( + deepSeekFileScope('https://api.deepseek.com', 'key'), VARIANT, 1, 1, + )).rejects.toBeInstanceOf(Error) + }) }) diff --git a/packages/llm/llm-pi-ai/README.md b/packages/llm/llm-pi-ai/README.md index 044038aa69..360da6a73b 100644 --- a/packages/llm/llm-pi-ai/README.md +++ b/packages/llm/llm-pi-ai/README.md @@ -123,7 +123,7 @@ A model that carries reasoning metadata — from the installed catalog or from i A model **without** that metadata — a hand-declared one whose entry declares no `reasoningEfforts`, and a catalog model pi-ai marks as non-reasoning — exposes no `reasoning` at all. pi-ai reports such a model as supporting the single level `off`, but `off` is translated to *omitting* the reasoning option, which is byte-for-byte the request that naming no effort already produces: selecting it could not disable anything, so a provider whose own default is to think would keep thinking with `off` shown as selected. Reporting the capability as unavailable leaves a surface offering the provider's default and nothing that misrepresents it. The profile `reasoning` value, including `off`, is the deployment default when configured; omitting it preserves the provider default. Per-request `GenerateOptions.reasoningEffort` takes precedence, and a level absent from the exact model capability fails the REQUEST with `UNSUPPORTED_REASONING_EFFORT` before network I/O instead of being clamped. Describing a model never fails that way: the models under one provider disagree about which levels they accept, so `resolveModel` reports a profile level the exact model cannot take as no default at all rather than throwing. A throw there would take the whole provider out of every model catalog built over it — one mis-set profile field hiding even the models that do support the level — so a bad configuration surfaces where it is acted on, not where it is described. pi-ai's common stream options represent `off` by omitting `reasoning`. -Supported profile fields are `apiKeyEnv`, `displayName`, `api`, `baseURL`, `models`, `modelOverrides`, `compat`, `defaultContextWindow`, `defaultMaxTokens`, `defaultInput`, `headers`, `reasoning`, `thinkingBudgets`, `cacheRetention`, `transport`, `timeoutMs`, `websocketConnectTimeoutMs`, `streamIdleTimeoutMs`, `maxRequestImageBytes`, `requestImagePixelBudget`, `requestImageMaxBytes`, and `retryPolicy`. Each resolved profile retry policy is captured with that provider route; omission uses the shared bounded normal default of five retries. The stream-idle interval is a positive finite Node timer delay, defaults to five minutes, and covers only an outstanding provider read, not consumer think time. Every image route first derives a deterministic request version from the provider-independent master under `requestImagePixelBudget` (default 2048 by 2048 total pixels) and `requestImageMaxBytes` (default 1MiB raw bytes). The same version feeds inline base64, and its stable descriptor exposes the attachment id and actual preview dimensions. `maxRequestImageBytes` then bounds the accumulated base64 length (default 20MiB): the oldest request versions are replaced by a fixed text placeholder until the request fits. Harness app attribution wins a conflicting configured header name. +Supported profile fields are `apiKeyEnv`, `displayName`, `api`, `baseURL`, `models`, `modelOverrides`, `compat`, `defaultContextWindow`, `defaultMaxTokens`, `defaultInput`, `headers`, `reasoning`, `thinkingBudgets`, `cacheRetention`, `transport`, `timeoutMs`, `websocketConnectTimeoutMs`, `streamIdleTimeoutMs`, `maxRequestImageBytes`, `requestImagePixelBudget`, `requestImageMaxBytes`, and `retryPolicy`. Each resolved profile retry policy is captured with that provider route; omission uses the shared bounded normal default of five retries. The stream-idle interval is a positive finite Node timer delay, defaults to five minutes, and covers only an outstanding provider read, not consumer think time. Every image route derives a deterministic request version from the provider-independent master under `requestImagePixelBudget` (default 2048 by 2048 total pixels) and `requestImageMaxBytes` (default 1MiB raw bytes). Before reading masters, `maxRequestImageBytes` applies to conservative request-version upper bounds and replaces the oldest over-budget images with fixed text; exact base64 lengths are checked again after retained versions are generated. The 20MiB default can retain fifteen maximum-size 1MiB versions after base64 expansion while leaving request-body headroom. The same version feeds inline base64, and its stable descriptor exposes the attachment id and actual preview dimensions. Harness app attribution wins a conflicting configured header name. The adapter forces pi-ai's SDK `maxRetries` to zero so one `stream()` call makes one provider request. The removed profile fields `maxRetries` and `maxRetryDelayMs` fail load instead of silently multiplying or hiding the separately composed agent-level retry budget. Idle expiry aborts the SDK's stable request signal and surfaces `TIMEOUT`; an earlier caller abort remains `ABORTED`. @@ -173,7 +173,7 @@ pi-ai installs several provider SDKs and lazy-loads the one selected by the cata #### What the model sees -The selected catalog model receives `GenerateOptions.system`, history, tools, and sampling fields supported by pi-ai's common streaming API. Each retained image is preceded by stable text naming its complete attachment id, actual request dimensions, and `read_image_region` preview coordinates. When accumulated base64 image payload exceeds the route's `maxRequestImageBytes`, each offloaded image (oldest first) is replaced by fixed text that tells the model to read the file again when a path is available or ask the user to attach it again. Provider-native replay metadata is restored only when the adapter validates it for the historical content. +The selected catalog model receives `GenerateOptions.system`, history, tools, and sampling fields supported by pi-ai's common streaming API. Each retained image is preceded by stable text naming its complete attachment id and actual request dimensions. The text includes `read_image_region` preview coordinates only when that tool is present in the request. When accumulated base64 image payload exceeds the route's `maxRequestImageBytes`, each offloaded image (oldest first) is replaced by fixed text that tells the model to read the file again when a path is available or ask the user to attach it again. Offloaded masters are not read or transformed. Provider-native replay metadata is restored only when the adapter validates it for the historical content. #### Token effect diff --git a/packages/llm/llm-pi-ai/README.zh.md b/packages/llm/llm-pi-ai/README.zh.md index d4b5dff10e..bf05671ee3 100644 --- a/packages/llm/llm-pi-ai/README.zh.md +++ b/packages/llm/llm-pi-ai/README.zh.md @@ -124,7 +124,7 @@ pi-ai 依据提供方 id 与 baseURL 决定每个请求的形状:系统提示 **没有**这份元数据的模型——条目未声明 `reasoningEfforts` 的手工声明模型,以及 pi-ai 标记为不具备推理能力的 catalog 模型——完全不公开 `reasoning`。pi-ai 会把这类模型报告为只支持 `off` 一档,但 `off` 会被翻译成*省略* reasoning 选项,而那与「不点名任何档位」产出的请求逐字节相同:选它关不掉任何东西,于是自身默认就在思考的提供方,会在界面显示 `off` 被选中的同时继续思考。把该能力报告为不可用,界面就只剩提供方默认这一项,不会再出现自相矛盾的控件。配置 profile 的 `reasoning` 值(包括 `off`)在存在时是部署默认值;省略它会保留提供方默认值。每次请求的 `GenerateOptions.reasoningEffort` 优先;未出现在确切模型能力中的档位会让**请求**在网络 I/O 前以 `UNSUPPORTED_REASONING_EFFORT` 失败,而不会被自动调整。**描述**一个模型则从不这样失败:同一提供方下各模型接受的档位并不一致,因此 `resolveModel` 对该模型拿不下的 profile 档位报告为「没有默认值」,而不是抛错。在那里抛错会让整个提供方从任何基于它构建的模型目录中消失——一个配错的 profile 字段连支持该档位的模型也一并藏起来——所以坏配置暴露在被执行处,而不是被描述处。pi-ai 的通用流选项通过省略 `reasoning` 表示 `off`。 -受支持的 profile 字段是 `apiKeyEnv`、`displayName`、`api`、`baseURL`、`models`、`modelOverrides`、`compat`、`defaultContextWindow`、`defaultMaxTokens`、`defaultInput`、`headers`、`reasoning`、`thinkingBudgets`、`cacheRetention`、`transport`、`timeoutMs`、`websocketConnectTimeoutMs`、`streamIdleTimeoutMs`、`maxRequestImageBytes`、`requestImagePixelBudget`、`requestImageMaxBytes` 和 `retryPolicy`。每条 profile 解析后的重试策略会随该提供方路由一同捕获;省略时使用共享的有界 normal 默认值并重试五次。流空闲间隔必须是正的有限 Node 定时器延迟,默认为五分钟,且只覆盖未完成提供方读取,不包括消费方思考时间。每条图片路由先从提供方无关的主版本派生确定性请求版本,受 `requestImagePixelBudget`(默认总像素 2048×2048)和 `requestImageMaxBytes`(默认原始字节 1MiB)约束。同一版本用于内联 base64,其稳定描述会公开附件 ID 和实际预览尺寸。`maxRequestImageBytes` 再限制累计 base64 长度(默认 20MiB);超出时从最旧请求版本开始替换为固定文本占位,直到请求可容纳。若已配置标头中有同名项,则以 Harness 应用归因为准。 +受支持的 profile 字段是 `apiKeyEnv`、`displayName`、`api`、`baseURL`、`models`、`modelOverrides`、`compat`、`defaultContextWindow`、`defaultMaxTokens`、`defaultInput`、`headers`、`reasoning`、`thinkingBudgets`、`cacheRetention`、`transport`、`timeoutMs`、`websocketConnectTimeoutMs`、`streamIdleTimeoutMs`、`maxRequestImageBytes`、`requestImagePixelBudget`、`requestImageMaxBytes` 和 `retryPolicy`。每条 profile 解析后的重试策略会随该提供方路由一同捕获;省略时使用共享的有界 normal 默认值并重试五次。流空闲间隔必须是正的有限 Node 定时器延迟,默认为五分钟,且只覆盖未完成提供方读取,不包括消费方思考时间。每条图片路由从提供方无关的主版本派生确定性请求版本,受 `requestImagePixelBudget`(默认总像素 2048×2048)和 `requestImageMaxBytes`(默认原始字节 1MiB)约束。读取主版本前,`maxRequestImageBytes` 先按请求版本的保守上界替换超预算的最旧图片;保留版本生成后再用确切 base64 长度检查。20MiB 默认值可保留十五个按 1MiB 上限生成的请求版本,并为请求正文留下余量。同一版本用于内联 base64,其稳定描述会公开附件 ID 和实际预览尺寸。若已配置标头中有同名项,则以 Harness 应用归因为准。 适配器强制 pi-ai SDK `maxRetries` 为零,因此一次 `stream()` 调用只会发起一次提供方请求。已移除 profile 字段 `maxRetries` 和 `maxRetryDelayMs` 会使加载失败,而不是静默倍增或隐藏单独组合的 agent(智能体)级重试预算。空闲超时会 abort SDK 的稳定请求信号,并以 `TIMEOUT` 呈现;较早的调用方 abort 仍为 `ABORTED`。 @@ -174,7 +174,7 @@ pi-ai 会安装多个提供方 SDK,并延迟加载 catalog 模型所选的 SDK #### 模型看到的内容 -所选 catalog 模型会收到 `GenerateOptions.system`、历史、工具,以及 pi-ai 通用流式 API 支持的采样字段。每张保留图片前都有稳定文本,写明完整附件 ID、实际请求尺寸和 `read_image_region` 使用的预览坐标。请求累积的 base64 图片载荷超过路由的 `maxRequestImageBytes` 时,被 offload 的图片会从最老开始替换为固定文本,要求模型在有路径时重新读取文件,否则请用户重新附上图片。只有当适配器验证提供方原生回放元数据与历史内容匹配时,才会恢复这些元数据。 +所选 catalog 模型会收到 `GenerateOptions.system`、历史、工具,以及 pi-ai 通用流式 API 支持的采样字段。每张保留图片前都有稳定文本,写明完整附件 ID 和实际请求尺寸。只有请求包含 `read_image_region` 时,文本才会提供该工具使用的预览坐标。请求累积的 base64 图片载荷超过路由的 `maxRequestImageBytes` 时,被 offload 的图片会从最老开始替换为固定文本,要求模型在有路径时重新读取文件,否则请用户重新附上图片。系统不会读取或转换被 offload 的主版本。只有当适配器验证提供方原生回放元数据与历史内容匹配时,才会恢复这些元数据。 #### Token 影响 diff --git a/packages/llm/llm-pi-ai/src/adapter.ts b/packages/llm/llm-pi-ai/src/adapter.ts index c5af6bff6a..f06b27a1ea 100644 --- a/packages/llm/llm-pi-ai/src/adapter.ts +++ b/packages/llm/llm-pi-ai/src/adapter.ts @@ -360,7 +360,7 @@ export class PiAiAdapter extends LlmAdapter { } const context = attachments === undefined ? toPiContext(options, undefined, onReplayDegrade) - : await toPiContext(options, attachments, onReplayDegrade, profile.maxRequestImageBytes, { + : await toPiContext({ ...options, signal: watchdog.signal }, attachments, onReplayDegrade, profile.maxRequestImageBytes, { maxPixels: profile.requestImagePixelBudget, maxBytes: profile.requestImageMaxBytes, }) diff --git a/packages/llm/llm-pi-ai/src/config.ts b/packages/llm/llm-pi-ai/src/config.ts index 2fd68d4648..5473d931de 100644 --- a/packages/llm/llm-pi-ai/src/config.ts +++ b/packages/llm/llm-pi-ai/src/config.ts @@ -46,9 +46,9 @@ export const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 300_000 * Default request-level bound on base64-encoded image payload. Every image in * history is re-encoded into every request body, so an unbounded conversation * eventually exceeds a provider or gateway request-size cap and the session - * can never complete another request. The 20MiB default admits four images at - * the attachment store's 3.5MiB raw-image default after base64 expansion and - * reserves request capacity for system prompts, history, tools, and JSON. + * can never complete another request. The 20MiB default admits fifteen 1MiB + * request versions after base64 expansion and reserves request capacity for + * system prompts, history, tools, and JSON. * Deployments behind stricter gateways lower it per route. */ export const DEFAULT_MAX_REQUEST_IMAGE_BYTES = 20 * 1024 * 1024 diff --git a/packages/llm/llm-pi-ai/src/context.ts b/packages/llm/llm-pi-ai/src/context.ts index 5cf9b7c042..4c31d2638b 100644 --- a/packages/llm/llm-pi-ai/src/context.ts +++ b/packages/llm/llm-pi-ai/src/context.ts @@ -48,6 +48,7 @@ function assertSupportedImageRoles(messages: readonly Message[]): void { async function userContent( blocks: readonly ContentBlock[], requestImages: ReadonlyMap, + cropAvailable: boolean, ): Promise { const content: (TextContent | ImageContent)[] = [] for (const block of blocks) { @@ -60,7 +61,7 @@ async function userContent( if (version === undefined) { throw new LlmError(`pi-ai request image ${block.attachment.attachmentId} was not prepared`, 'INVALID_REQUEST') } - content.push({ type: 'text', text: requestImagePreviewText(version) }) + content.push({ type: 'text', text: requestImagePreviewText(version, cropAvailable) }) content.push({ type: 'image', data: Buffer.from(version.data).toString('base64'), @@ -70,7 +71,7 @@ async function userContent( } case 'tool-result': { - const nested = await userContent(block.content, requestImages) + const nested = await userContent(block.content, requestImages, cropAvailable) if (typeof nested === 'string') { if (nested.length > 0) content.push({ type: 'text', text: nested }) } else { @@ -101,11 +102,16 @@ async function prepareRequestImages( messages: readonly Message[], attachments: AttachmentStore, policy: ImageRequestPolicy, + signal?: AbortSignal, ): Promise> { const refs = new Map() for (const message of messages) collectImageRefs(message.content, refs) + const orderedRefs = [...refs.values()] + const prepared = await attachments.readImageRequests(orderedRefs, policy, signal) const versions = new Map() - for (const [id, ref] of refs) versions.set(id, await attachments.readImageRequest(ref, policy)) + for (const [index, ref] of orderedRefs.entries()) { + versions.set(ref.attachmentId, prepared[index] as RequestImageAttachment) + } return versions } @@ -222,17 +228,24 @@ async function toPiContextWithImages( }, ): Promise { assertSupportedImageRoles(options.messages) - const requestImages = await prepareRequestImages(options.messages, attachments, requestImagePolicy) const requestMessages = offloadRequestImagesWithPolicy(options.messages, { + representation: 'base64', + ...maxRequestImageBytes === undefined ? {} : { maxBytes: maxRequestImageBytes }, + byteQuantum: 1, + byteLength: ref => Math.min(ref.bytes, requestImagePolicy.maxBytes), + }) + const requestImages = await prepareRequestImages(requestMessages, attachments, requestImagePolicy, options.signal) + const exactMessages = offloadRequestImagesWithPolicy(requestMessages, { representation: 'base64', ...maxRequestImageBytes === undefined ? {} : { maxBytes: maxRequestImageBytes }, byteQuantum: 1, byteLength: ref => requestImages.get(ref.attachmentId)?.bytes ?? ref.bytes, }) + const cropAvailable = options.tools?.some(tool => tool.name === 'read_image_region') ?? false const toolNames = new Map() const messages: PiMessage[] = [] - for (const message of requestMessages) { + for (const message of exactMessages) { if (message.role === 'system') { // pi-ai has a single systemPrompt slot; in-history system messages are // folded into user messages to preserve order (rare in practice — the @@ -250,7 +263,7 @@ async function toPiContextWithImages( } // user role: text + tool results (each result becomes its own message). const regular = message.content.filter(block => block.type !== 'tool-result') - const content = await userContent(regular, requestImages) + const content = await userContent(regular, requestImages, cropAvailable) const results = message.content.filter((block): block is Extract => ( block.type === 'tool-result' )) @@ -258,7 +271,7 @@ async function toPiContextWithImages( messages.push({ role: 'user', content, timestamp: 0 }) } for (const result of results) { - const resultContent = await userContent(result.content, requestImages) + const resultContent = await userContent(result.content, requestImages, cropAvailable) messages.push({ role: 'toolResult', toolCallId: result.toolCallId, diff --git a/packages/llm/llm-pi-ai/tests/adapter.spec.ts b/packages/llm/llm-pi-ai/tests/adapter.spec.ts index 416802e246..09befeaa4b 100644 --- a/packages/llm/llm-pi-ai/tests/adapter.spec.ts +++ b/packages/llm/llm-pi-ai/tests/adapter.spec.ts @@ -239,7 +239,11 @@ describe('PiAiAdapter provider routing', () => { } const readImage = vi.fn((_ref: ImageAttachmentRef): Promise => Promise.resolve({ ref, data: Uint8Array.of(1) })) - const readImageRequest = vi.fn((value: ImageAttachmentRef, _policy: ImageRequestPolicy): Promise => ( + const readImageRequest = vi.fn(( + value: ImageAttachmentRef, + _policy: ImageRequestPolicy, + _signal?: AbortSignal, + ): Promise => ( Promise.resolve({ variantId: ImageVariantId(`sha256:${'b'.repeat(64)}`), master: value, @@ -276,8 +280,12 @@ describe('PiAiAdapter provider routing', () => { return readImage(value) } - override readImageRequest(value: ImageAttachmentRef, policy: ImageRequestPolicy): Promise { - return readImageRequest(value, policy) + override readImageRequest( + value: ImageAttachmentRef, + policy: ImageRequestPolicy, + signal?: AbortSignal, + ): Promise { + return readImageRequest(value, policy, signal) } } @@ -301,7 +309,7 @@ describe('PiAiAdapter provider routing', () => { expect(readImageRequest).toHaveBeenCalledWith(ref, { maxPixels: 2048 * 2048, maxBytes: 1024 * 1024, - }) + }, expect.any(AbortSignal)) expect(server.paths).toEqual(['/v1/responses']) }) diff --git a/packages/llm/llm-pi-ai/tests/context.spec.ts b/packages/llm/llm-pi-ai/tests/context.spec.ts index 7163a7375d..a0fb671c95 100644 --- a/packages/llm/llm-pi-ai/tests/context.spec.ts +++ b/packages/llm/llm-pi-ai/tests/context.spec.ts @@ -1,6 +1,11 @@ import { describe, expect, it, vi } from 'vitest' import { AttachmentId, ImageVariantId } from '@deepseek-ai/dsh-attachment' -import type { AttachmentStore, ImageAttachmentRef, RequestImageAttachment } from '@deepseek-ai/dsh-attachment' +import type { + AttachmentStore, + ImageAttachmentRef, + ImageRequestPolicy, + RequestImageAttachment, +} from '@deepseek-ai/dsh-attachment' import { CallId, createMessage, createUserMessage, OFFLOADED_IMAGE_TEXT } from '@deepseek-ai/dsh-llm' import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm' import { toPiContext } from '../src/context.ts' @@ -30,11 +35,22 @@ function requestImage(value: ImageAttachmentRef, data: Uint8Array): RequestImage } function projectionStore( - readImageRequest = vi.fn((value: ImageAttachmentRef) => ( + readImageRequest: ( + value: ImageAttachmentRef, + policy: ImageRequestPolicy, + signal?: AbortSignal, + ) => Promise = vi.fn((value: ImageAttachmentRef) => ( Promise.resolve(requestImage(value, Uint8Array.of(1))) )), ): AttachmentStore { - return { readImageRequest } as unknown as AttachmentStore + return { + readImageRequest, + readImageRequests: ( + refs: readonly ImageAttachmentRef[], + policy: Parameters[1], + signal?: AbortSignal, + ) => Promise.all(refs.map(value => readImageRequest(value, policy, signal))), + } as unknown as AttachmentStore } const attachments = projectionStore() @@ -268,6 +284,42 @@ describe('pi-ai request context conversion', () => { expect(readImageRequest).toHaveBeenCalledTimes(1) }) + it('does not prepare an old image removed by the conservative request projection', async () => { + const old = { ...ref, attachmentId: AttachmentId(`sha256:${'c'.repeat(64)}`), bytes: 3 } + const recent = { ...ref, attachmentId: AttachmentId(`sha256:${'d'.repeat(64)}`), bytes: 3 } + const readImageRequest = vi.fn((value: ImageAttachmentRef) => { + if (value.attachmentId === old.attachmentId) throw new Error('old image must not be read') + return Promise.resolve(requestImage(value, Uint8Array.of(1, 2, 3))) + }) + + const context = await toPiContext(request([user([ + { type: 'image', attachment: old }, + { type: 'image', attachment: recent }, + ])]), projectionStore(readImageRequest), undefined, 4) + + expect(context.messages[0]).toMatchObject({ + role: 'user', + content: [ + { type: 'text', text: OFFLOADED_IMAGE_TEXT }, + { type: 'text', text: expect.stringContaining(String(recent.attachmentId)) as string }, + { type: 'image' }, + ], + }) + expect(readImageRequest).toHaveBeenCalledTimes(1) + expect(readImageRequest.mock.calls[0]?.[0]).toEqual(recent) + }) + + it('advertises region reads only when the request exposes the tool', async () => { + const withoutCrop = await toPiContext(request([user([{ type: 'image', attachment: ref }])]), attachments) + const withCrop = await toPiContext({ + ...request([user([{ type: 'image', attachment: ref }])]), + tools: [{ name: 'read_image_region', description: 'crop', parameters: { type: 'object' } }], + }, attachments) + + expect(JSON.stringify(withoutCrop.messages)).not.toContain('Call read_image_region') + expect(JSON.stringify(withCrop.messages)).toContain('Call read_image_region') + }) + it('keeps every image at exactly the payload bound and drops all of them when even the newest cannot fit', async () => { const sized: ImageAttachmentRef = { ...ref, bytes: 3 } const exact = await toPiContext(request([ @@ -298,7 +350,7 @@ describe('pi-ai request context conversion', () => { expect(oversized.messages).toEqual([ { role: 'user', content: OFFLOADED_IMAGE_TEXT, timestamp: 0 }, ]) - expect(readImageRequest).toHaveBeenCalledTimes(1) + expect(readImageRequest).not.toHaveBeenCalled() }) it('offloads repeated image-block occurrences by position rather than shared object identity', async () => { diff --git a/packages/llm/llm-pi-ai/tests/convert.spec.ts b/packages/llm/llm-pi-ai/tests/convert.spec.ts index e77075b0c6..ccfb321d3b 100644 --- a/packages/llm/llm-pi-ai/tests/convert.spec.ts +++ b/packages/llm/llm-pi-ai/tests/convert.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest' import { AttachmentId, ImageVariantId } from '@deepseek-ai/dsh-attachment' -import type { AttachmentStore, ImageAttachmentRef, RequestImageAttachment } from '@deepseek-ai/dsh-attachment' +import type { AttachmentStore, ImageAttachmentRef, ImageRequestPolicy, RequestImageAttachment } from '@deepseek-ai/dsh-attachment' import { createUserMessage, CallId, CONTEXT_WINDOW_EXCEEDED_CODE, EMPTY_RESPONSE_CODE, createMessage } from '@deepseek-ai/dsh-llm' import type { ContentBlock, StreamChunk } from '@deepseek-ai/dsh-llm' import type { AssistantMessage, AssistantMessageEvent, Usage } from '@earendil-works/pi-ai' @@ -58,6 +58,23 @@ function requestVersion(ref: ImageAttachmentRef): RequestImageAttachment { } } +function attachmentStore(readImageRequest: ( + ref: ImageAttachmentRef, + policy: ImageRequestPolicy, + signal?: AbortSignal, +) => Promise): AttachmentStore { + return { + readImageRequest, + readImageRequests: ( + refs: readonly ImageAttachmentRef[], + policy: ImageRequestPolicy, + signal?: AbortSignal, + ) => Promise.all( + refs.map(ref => readImageRequest(ref, policy, signal)), + ), + } as unknown as AttachmentStore +} + describe('toPiContext', () => { it('maps system prompt, user text, and tools', () => { const context = toPiContext({ @@ -91,7 +108,9 @@ describe('toPiContext', () => { width: 1, height: 1, } - const readImageRequest = vi.fn((value: ImageAttachmentRef) => Promise.resolve(requestVersion(value))) + const readImageRequest = vi.fn((value: ImageAttachmentRef, _policy: ImageRequestPolicy) => ( + Promise.resolve(requestVersion(value)) + )) const context = await toPiContext({ provider: 'openai', model: 'gpt-4.1', @@ -99,11 +118,12 @@ describe('toPiContext', () => { content: [{ type: 'text', text: 'describe' }, { type: 'image', attachment }], source: { kind: 'plugin', plugin: 'test' }, })], - }, { readImageRequest } as unknown as AttachmentStore) + }, attachmentStore(readImageRequest)) expect(readImageRequest).toHaveBeenCalledWith( attachment, { maxPixels: 2048 * 2048, maxBytes: 1024 * 1024 }, + undefined, ) expect(context.messages[0]).toEqual({ role: 'user', @@ -124,7 +144,9 @@ describe('toPiContext', () => { width: 1, height: 1, } - const readImageRequest = vi.fn((value: ImageAttachmentRef) => Promise.resolve(requestVersion(value))) + const readImageRequest = vi.fn((value: ImageAttachmentRef, _policy: ImageRequestPolicy) => ( + Promise.resolve(requestVersion(value)) + )) const context = await toPiContext({ provider: 'openai', model: 'gpt-4.1', @@ -148,7 +170,7 @@ describe('toPiContext', () => { }], source: { kind: 'plugin', plugin: 'test' }, })], - }, { readImageRequest } as unknown as AttachmentStore) + }, attachmentStore(readImageRequest)) expect(context.messages).toEqual([{ role: 'toolResult', diff --git a/packages/llm/llm/src/content.ts b/packages/llm/llm/src/content.ts index 96c39f4f9b..73a2aee889 100644 --- a/packages/llm/llm/src/content.ts +++ b/packages/llm/llm/src/content.ts @@ -21,12 +21,15 @@ export function textOnlyImageText(ref: ImageAttachmentRef): string { /** * Stable model-facing handle and coordinate description for one exact request preview. * @param version - exact request image shown beside the text. + * @param cropAvailable - whether the active request exposes `read_image_region`. * @returns attachment handle, preview dimensions, and crop-coordinate guidance. */ -export function requestImagePreviewText(version: RequestImageAttachment): string { - return `Image ${version.master.attachmentId}; preview ${version.width}x${version.height}px. ` - + 'Crop coordinates use this preview. Call read_image_region with this attachment_id, ' - + `preview_width=${version.width}, preview_height=${version.height}, x, y, width, and height.` +export function requestImagePreviewText(version: RequestImageAttachment, cropAvailable: boolean): string { + const identity = `Image ${version.master.attachmentId}; preview ${version.width}x${version.height}px.` + return cropAvailable + ? `${identity} Crop coordinates use this preview. Call read_image_region with this attachment_id, ` + + `preview_width=${version.width}, preview_height=${version.height}, x, y, width, and height.` + : identity } /** From 657ec56fbfc26cc03ae27e033f75f148796fd86c Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 20:25:35 +0800 Subject: [PATCH 16/28] test(images): cover attachment projection edges --- .../attachment-local/src/canonical.ts | 14 +- .../attachment-local/src/encoding.ts | 11 +- .../attachment-local/src/request-image.ts | 6 +- .../attachment-local/tests/canonical.spec.ts | 43 +++++ .../attachment-local/tests/encoding.spec.ts | 27 ++- .../attachment-local/tests/index.spec.ts | 24 +++ .../tests/request-image-verification.spec.ts | 47 ++++++ .../tests/request-image.spec.ts | 158 +++++++++++++++++- .../attachment-local/tests/store.spec.ts | 12 +- .../attachment/attachment/tests/index.spec.ts | 34 ++++ packages/fs/tool-fs/tests/read-image.spec.ts | 128 ++++++++++++++ 11 files changed, 483 insertions(+), 21 deletions(-) create mode 100644 packages/attachment/attachment-local/tests/request-image-verification.spec.ts diff --git a/packages/attachment/attachment-local/src/canonical.ts b/packages/attachment/attachment-local/src/canonical.ts index c437448f4c..513e5bbcc7 100644 --- a/packages/attachment/attachment-local/src/canonical.ts +++ b/packages/attachment/attachment-local/src/canonical.ts @@ -77,12 +77,10 @@ export async function hasLowColourCount(pipeline: Sharp): Promise { }).raw().toBuffer({ resolveWithObject: true }) const colours = new Set() for (let offset = 0; offset < data.length; offset += info.channels) { - const red = data[offset] ?? 0 - const green = info.channels < 3 ? red : data[offset + 1] ?? red - const blue = info.channels < 3 ? red : data[offset + 2] ?? red - const alpha = info.channels === 2 - ? data[offset + 1] ?? 255 - : info.channels === 4 ? data[offset + 3] ?? 255 : 255 + const red = data.readUInt8(offset) + const green = data.readUInt8(offset + 1) + const blue = data.readUInt8(offset + 2) + const alpha = info.channels === 4 ? data.readUInt8(offset + 3) : 255 colours.add(((red >> 3) << 15) | ((green >> 3) << 10) | ((blue >> 3) << 5) | (alpha >> 3)) if (colours.size > LOW_COLOUR_LIMIT) return false } @@ -183,8 +181,8 @@ export async function prepareMasterImage( const scale = Math.min(MIN_SCALE_STEP, sizeScale) const nextWidth = Math.max(1, Math.floor(width * scale)) const nextHeight = Math.max(1, Math.floor(height * scale)) - width = nextWidth === width && width > 1 ? width - 1 : nextWidth - height = nextHeight === height && height > 1 ? height - 1 : nextHeight + width = nextWidth + height = nextHeight } } catch (error) { if (error instanceof AttachmentError) throw error diff --git a/packages/attachment/attachment-local/src/encoding.ts b/packages/attachment/attachment-local/src/encoding.ts index 8099046c95..963edda672 100644 --- a/packages/attachment/attachment-local/src/encoding.ts +++ b/packages/attachment/attachment-local/src/encoding.ts @@ -20,16 +20,17 @@ export async function encodeFirstWithinLimit( attempts: readonly (() => Promise)[], maxBytes: number, ): Promise> { - if (attempts.length === 0) throw new Error('image encoding requires at least one candidate') - let smallest: T | undefined - for (const attempt of attempts) { + const [first, ...remaining] = attempts + if (first === undefined) throw new Error('image encoding requires at least one candidate') + let smallest = await first() + if (smallest.data.byteLength <= maxBytes) return smallest + for (const attempt of remaining) { const candidate = await attempt() if (candidate.data.byteLength <= maxBytes) return candidate - if (smallest === undefined || candidate.data.byteLength < smallest.data.byteLength) { + if (candidate.data.byteLength < smallest.data.byteLength) { smallest = candidate } } - if (smallest === undefined) throw new Error('image encoding did not execute a candidate') return { smallest } } diff --git a/packages/attachment/attachment-local/src/request-image.ts b/packages/attachment/attachment-local/src/request-image.ts index f65bc97ad5..ef7c841bed 100644 --- a/packages/attachment/attachment-local/src/request-image.ts +++ b/packages/attachment/attachment-local/src/request-image.ts @@ -263,11 +263,7 @@ async function writeCached(path: string, data: Uint8Array): Promise { const temporary = `${path}.${randomUUID()}.tmp` try { await writeFile(temporary, data, { mode: 0o600, flag: 'wx' }) - try { - await rename(temporary, path) - } catch (error: unknown) { - if ((error as NodeJS.ErrnoException | null)?.code !== 'EEXIST') throw error - } + await rename(temporary, path) } finally { await rm(temporary, { force: true }) } diff --git a/packages/attachment/attachment-local/tests/canonical.spec.ts b/packages/attachment/attachment-local/tests/canonical.spec.ts index a6f489615e..8aa30511d6 100644 --- a/packages/attachment/attachment-local/tests/canonical.spec.ts +++ b/packages/attachment/attachment-local/tests/canonical.spec.ts @@ -230,6 +230,40 @@ describe('prepareMasterImage', () => { message: 'The 16-bit PNG could not be converted to the canonical 8-bit sRGB form.', }) }) + + it.each([ + ['float PNG', { mediaType: 'image/png', depth: 'float' }], + ['uchar JPEG', { mediaType: 'image/jpeg', depth: 'uchar' }], + ] as const)('describes a failed %s conversion without exposing the encoder error', async (source, fields) => { + const detected = { + ...fields, + width: 5000, + height: 5000, + animated: false, + carriesMetadata: false, + space: 'srgb', + hasAlpha: false, + } as const + + await expect(prepareMasterImage(Uint8Array.of(1, 2, 3), detected, POLICY)) + .rejects.toMatchObject({ + code: 'ATTACHMENT_WRITE_FAILED', + message: `The ${source} could not be converted to the canonical 8-bit sRGB form.`, + }) + }) + + it('rejects a converted master whose verified alpha metadata disagrees with the source facts', async () => { + const data = await flatImage(8, 8, 'png', true) + const detected = await detectImage(data) + + await expect(prepareMasterImage(data, { ...detected, hasAlpha: false }, { + maxDimension: 4, + maxBytes: POLICY.maxBytes, + })).rejects.toMatchObject({ + code: 'ATTACHMENT_WRITE_FAILED', + message: 'Canonical image conversion did not produce a single-frame 8-bit sRGB image with matching metadata.', + }) + }) }) describe('hasLowColourCount', () => { @@ -288,6 +322,15 @@ describe('hasLowColourCount', () => { await expect(hasLowColourCount(grayscaleAlpha)).resolves.toBe(true) }) + it('reads one-channel grayscale samples as equal RGB values', async () => { + const pixels = new Uint8Array(128 * 16) + for (let index = 0; index < pixels.length; index += 1) pixels[index] = index & 0xff + + await expect(hasLowColourCount(sharp(pixels, { + raw: { width: 128, height: 16, channels: 1 }, + }))).resolves.toBe(true) + }) + it('keeps an antialiased text screenshot readable on the low-colour PNG path', async () => { const source = new Uint8Array(await sharp(Buffer.from(` diff --git a/packages/attachment/attachment-local/tests/encoding.spec.ts b/packages/attachment/attachment-local/tests/encoding.spec.ts index c95d09c43c..d75fc9b4b3 100644 --- a/packages/attachment/attachment-local/tests/encoding.spec.ts +++ b/packages/attachment/attachment-local/tests/encoding.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest' import { CompressionLimiter } from '../src/compression-limiter.ts' -import { encodeFirstWithinLimit } from '../src/encoding.ts' +import { encodeFirstWithinLimit, isExhaustedEncoding } from '../src/encoding.ts' describe('lazy image encoding', () => { it('does not execute fallback qualities after the first fitting candidate', async () => { @@ -22,6 +22,19 @@ describe('lazy image encoding', () => { expect(second).toHaveBeenCalledTimes(1) expect(third).not.toHaveBeenCalled() }) + + it('rejects an empty candidate list and reports the smallest exhausted candidate', async () => { + await expect(encodeFirstWithinLimit([], 8)).rejects.toThrow('requires at least one candidate') + const result = await encodeFirstWithinLimit([ + () => Promise.resolve({ data: new Uint8Array(12), quality: 85 }), + () => Promise.resolve({ data: new Uint8Array(9), quality: 80 }), + () => Promise.resolve({ data: new Uint8Array(10), quality: 75 }), + ], 8) + + expect(isExhaustedEncoding(result)).toBe(true) + expect(result).toMatchObject({ smallest: { quality: 80 } }) + expect(isExhaustedEncoding({ data: new Uint8Array(1) })).toBe(false) + }) }) describe('CompressionLimiter', () => { @@ -67,4 +80,16 @@ describe('CompressionLimiter', () => { await expect(failed).rejects.toThrow('synchronous setup failure') await expect(next).resolves.toBe('next') }) + + it('normalizes a non-Error rejection and releases its slot', async () => { + const limiter = new CompressionLimiter(1) + const failed = limiter.run(() => Promise.reject('native failure')) + const next = limiter.run(() => Promise.resolve('next')) + + await expect(failed).rejects.toMatchObject({ + message: 'Image compression task rejected with a non-Error value.', + cause: 'native failure', + }) + await expect(next).resolves.toBe('next') + }) }) diff --git a/packages/attachment/attachment-local/tests/index.spec.ts b/packages/attachment/attachment-local/tests/index.spec.ts index 872aa5a3f7..89ada53298 100644 --- a/packages/attachment/attachment-local/tests/index.spec.ts +++ b/packages/attachment/attachment-local/tests/index.spec.ts @@ -58,6 +58,30 @@ describe('local attachment service', () => { } }) + it('commits a fully prepared image batch in input order', async () => { + const dshHome = await mkdtemp(join(tmpdir(), 'dsh-attachment-batch-success-')) + try { + const service = new LocalAttachmentStore(new Context(), { dshHome }) + const first = new Uint8Array(await sharp({ + create: { width: 2, height: 1, channels: 3, background: { r: 1, g: 2, b: 3 } }, + }).png().toBuffer()) + const second = new Uint8Array(await sharp({ + create: { width: 1, height: 2, channels: 3, background: { r: 4, g: 5, b: 6 } }, + }).png().toBuffer()) + + const refs = await service.saveImages([ + { data: first, mediaType: 'image/png', name: 'first.png' }, + { data: second, mediaType: 'image/png', name: 'second.png' }, + ]) + + expect(refs.map(ref => ref.name)).toEqual(['first.png', 'second.png']) + await expect(Promise.all(refs.map(ref => service.readImage(ref)))) + .resolves.toHaveLength(2) + } finally { + await rm(dshHome, { recursive: true, force: true }) + } + }) + it.each([3, 4] as const)('admits a 16-bit %s-channel PNG as an 8-bit master object', async (channels) => { const dshHome = await mkdtemp(join(tmpdir(), 'dsh-attachment-16-bit-')) try { diff --git a/packages/attachment/attachment-local/tests/request-image-verification.spec.ts b/packages/attachment/attachment-local/tests/request-image-verification.spec.ts new file mode 100644 index 0000000000..aae96e0b1d --- /dev/null +++ b/packages/attachment/attachment-local/tests/request-image-verification.spec.ts @@ -0,0 +1,47 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { Context } from '@deepseek-ai/cordis' +import sharp from 'sharp' +import { afterEach, describe, expect, it, vi } from 'vitest' + +const control = vi.hoisted(() => ({ mismatch: false })) + +vi.mock('../src/image.ts', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + async detectImage(data: Uint8Array): Promise>> { + const detected = await actual.detectImage(data) + return control.mismatch ? { ...detected, width: detected.width + 1 } : detected + }, + } +}) + +import LocalAttachmentStore from '../src/index.ts' + +const homes: string[] = [] + +afterEach(async () => { + control.mismatch = false + await Promise.all(homes.splice(0).map(home => rm(home, { recursive: true, force: true }))) +}) + +describe('request image verification', () => { + it('rejects an encoded request whose decoded facts disagree with the encoder result', async () => { + const dshHome = await mkdtemp(join(tmpdir(), 'dsh-request-verification-')) + homes.push(dshHome) + const attachments = new LocalAttachmentStore(new Context(), { dshHome }) + const source = new Uint8Array(await sharp({ + create: { width: 64, height: 32, channels: 3, background: { r: 12, g: 34, b: 56 } }, + }).png().toBuffer()) + const master = (await attachments.saveImage({ data: source, mediaType: 'image/png' })).ref + control.mismatch = true + + await expect(attachments.readImageRequest(master, { maxPixels: 16 * 16, maxBytes: 1024 * 1024 })) + .rejects.toMatchObject({ + code: 'ATTACHMENT_WRITE_FAILED', + message: 'Encoded model-request image does not match its verified 8-bit sRGB metadata.', + }) + }) +}) diff --git a/packages/attachment/attachment-local/tests/request-image.spec.ts b/packages/attachment/attachment-local/tests/request-image.spec.ts index 3cdfadd32c..e33522c104 100644 --- a/packages/attachment/attachment-local/tests/request-image.spec.ts +++ b/packages/attachment/attachment-local/tests/request-image.spec.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm } from 'node:fs/promises' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { Context } from '@deepseek-ai/cordis' @@ -39,9 +39,122 @@ describe('request image dimensions', () => { }) expect(projected.width * projected.height).toBeLessThanOrEqual(640_000) }) + + it('projects a portrait within the same total-pixel budget', () => { + const projected = requestImageDimensions(2160, 3840, 640_000) + + expect(projected).toEqual({ width: 600, height: 1066 }) + expect(projected.width * projected.height).toBeLessThanOrEqual(640_000) + }) + + it('rounds a portrait inward when integer aspect rounding crosses the pixel cap', () => { + expect(requestImageDimensions(2, 4, 5)).toEqual({ width: 1, height: 2 }) + }) + + it('rejects invalid preview dimensions, origins, sizes, and bounds', () => { + expect(() => previewCropToMaster(0, 10, { + previewWidth: 10, previewHeight: 10, x: 0, y: 0, width: 1, height: 1, + })).toThrow('Master image width must be a positive integer') + expect(() => previewCropToMaster(10, 10, { + previewWidth: 0, previewHeight: 10, x: 0, y: 0, width: 1, height: 1, + })).toThrow('Preview width must be a positive integer') + expect(() => previewCropToMaster(10, 10, { + previewWidth: 10, previewHeight: 10, x: -1, y: 0, width: 1, height: 1, + })).toThrow('Preview crop origin must use non-negative integer pixels') + expect(() => previewCropToMaster(10, 10, { + previewWidth: 10, previewHeight: 10, x: 0, y: 0, width: 0, height: 1, + })).toThrow('Preview crop width must be a positive integer') + expect(() => previewCropToMaster(10, 10, { + previewWidth: 10, previewHeight: 10, x: 9, y: 0, width: 2, height: 1, + })).toThrow('Preview crop extends beyond the image shown to the model') + }) }) describe('local request-image cache', () => { + it('passes through an in-budget master and reads a request batch in input order', async () => { + const attachments = await store() + const first = (await attachments.saveImage({ data: await image(8, 4), mediaType: 'image/png' })).ref + const second = (await attachments.saveImage({ data: await image(4, 8), mediaType: 'image/png' })).ref + const firstMaster = await attachments.readImage(first) + const policy = { maxPixels: 1_000, maxBytes: 1024 * 1024 } + + const request = await attachments.readImageRequest(first, policy) + const batch = await attachments.readImageRequests([first, second], policy) + + expect(request.data).toEqual(firstMaster.data) + expect(batch.map(value => value.master.attachmentId)).toEqual([first.attachmentId, second.attachmentId]) + }) + + it('rejects invalid request policies and master crop bounds', async () => { + const attachments = await store() + const master = (await attachments.saveImage({ data: await image(8, 4), mediaType: 'image/png' })).ref + + await expect(attachments.readImageRequest(master, { maxPixels: 0, maxBytes: 100 })) + .rejects.toThrow('Image request maxPixels must be a positive integer') + await expect(attachments.readImageRequest(master, { maxPixels: 100, maxBytes: 0 })) + .rejects.toThrow('Image request maxBytes must be a positive integer') + await expect(attachments.readImageRequest(master, { + maxPixels: 100, maxBytes: 100, crop: { x: -1, y: 0, width: 1, height: 1 }, + })).rejects.toThrow('Image crop origin must use non-negative integer pixels') + await expect(attachments.readImageRequest(master, { + maxPixels: 100, maxBytes: 100, crop: { x: 0, y: 0, width: 0, height: 1 }, + })).rejects.toThrow('Image crop width must be a positive integer') + await expect(attachments.readImageRequest(master, { + maxPixels: 100, maxBytes: 100, crop: { x: 7, y: 0, width: 2, height: 1 }, + })).rejects.toThrow('Image crop extends beyond the stored master image') + }) + + it('refuses a one-pixel request that cannot meet the encoded-byte budget', async () => { + const attachments = await store() + const master = (await attachments.saveImage({ data: await image(1, 1), mediaType: 'image/png' })).ref + + await expect(attachments.readImageRequest(master, { maxPixels: 1, maxBytes: 1 })) + .rejects.toMatchObject({ code: 'IMAGE_TOO_LARGE' }) + }) + + it('regenerates invalid, oversized, incompatible, or mismatched cached variants', async () => { + const attachments = await store() + const master = (await attachments.saveImage({ data: await image(64, 32), mediaType: 'image/png' })).ref + const policy = { maxPixels: 16 * 16, maxBytes: 4_096 } + const initial = await attachments.readImageRequest(master, policy) + const hash = String(initial.variantId).slice('sha256:'.length) + const path = join(attachments.root, 'request-images', hash.slice(0, 2), hash) + const noisyPixels = new Uint8Array(64 * 64 * 3) + let state = 0x2545f491 + for (let index = 0; index < noisyPixels.length; index += 1) { + state ^= state << 13 + state ^= state >>> 17 + state ^= state << 5 + noisyPixels[index] = state & 0xff + } + const oversized = new Uint8Array(await sharp(noisyPixels, { + raw: { width: 64, height: 64, channels: 3 }, + }).png().toBuffer()) + const depth16 = new Uint8Array(await sharp({ + create: { width: 16, height: 8, channels: 3, background: { r: 1, g: 2, b: 3 } }, + }).toColourspace('rgb16').png().toBuffer()) + const cmyk = new Uint8Array(await sharp({ + create: { width: 16, height: 8, channels: 3, background: { r: 1, g: 2, b: 3 } }, + }).toColourspace('cmyk').jpeg().toBuffer()) + const tooWide = await image(23, 11) + const unexpectedAlpha = new Uint8Array(await sharp({ + create: { width: 16, height: 8, channels: 4, background: { r: 1, g: 2, b: 3, alpha: 0.5 } }, + }).png().toBuffer()) + + for (const invalid of [ + oversized, + depth16, + cmyk, + tooWide, + unexpectedAlpha, + Uint8Array.of(1, 2, 3), + ]) { + await writeFile(path, invalid) + const regenerated = await attachments.readImageRequest(master, policy) + expect(regenerated.data).toEqual(initial.data) + } + }) + it('derives stable square and wide previews and separates route budgets in the cache key', async () => { const attachments = await store() const square = (await attachments.saveImage({ @@ -99,6 +212,17 @@ describe('local request-image cache', () => { expect(pixel[1]).toBeGreaterThan(pixel[0] ?? 0) }) + it('names a crop from an unnamed attachment id', async () => { + const attachments = await store() + const master = (await attachments.saveImage({ data: await image(8, 4), mediaType: 'image/png' })).ref + + const cropped = await attachments.cropImage(master, { + previewWidth: 8, previewHeight: 4, x: 0, y: 0, width: 4, height: 4, + }) + + expect(cropped.ref.name).toMatch(/^sha256:[0-9a-f]{8}-crop\.(?:png|webp|jpg)$/u) + }) + it('classifies opaque PNG pixels and preserves alpha while enforcing the request budget', async () => { const attachments = await store() const side = 256 @@ -233,4 +357,36 @@ describe('local request-image cache', () => { await expect(request).rejects.toBe(reason) expect(readSignal?.reason).toBe(reason) }) + + it('normalizes a non-Error cancellation and replaces an aborted shared transform', async () => { + const attachments = await store() + const master = (await attachments.saveImage({ + data: await image(2048, 1024), mediaType: 'image/png', name: 'replace.png', + })).ref + const actualRead = attachments.readImage.bind(attachments) + let calls = 0 + vi.spyOn(attachments, 'readImage').mockImplementation((ref, signal) => { + calls += 1 + if (calls === 1) { + return new Promise((_resolve, reject) => { + signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + }) + } + return actualRead(ref, signal) + }) + const controller = new AbortController() + const policy = { maxPixels: 640_000, maxBytes: 1024 * 1024 } + const cancelled = attachments.readImageRequest(master, policy, controller.signal) + await vi.waitFor(() => expect(calls).toBe(1)) + + controller.abort('cancelled') + const replacement = attachments.readImageRequest(master, policy) + + await expect(cancelled).rejects.toMatchObject({ + message: 'Attachment request cancelled with a non-Error reason.', + cause: 'cancelled', + }) + await expect(replacement).resolves.toMatchObject({ width: 1130, height: 565 }) + expect(calls).toBe(2) + }) }) diff --git a/packages/attachment/attachment-local/tests/store.spec.ts b/packages/attachment/attachment-local/tests/store.spec.ts index 97445c2f85..f0c127c174 100644 --- a/packages/attachment/attachment-local/tests/store.spec.ts +++ b/packages/attachment/attachment-local/tests/store.spec.ts @@ -8,7 +8,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import sharp from 'sharp' import type { ImageAttachmentLimits } from '@deepseek-ai/dsh-attachment' import type { MasterImagePolicy } from '../src/canonical.ts' -import { readImageFile, saveImageFile } from '../src/store.ts' +import { commitPreparedImageFile, prepareImageFile, readImageFile, saveImageFile } from '../src/store.ts' const fsControl = vi.hoisted(() => ({ readSignals: [] as AbortSignal[], @@ -256,4 +256,14 @@ describe('local attachment store', () => { await expect(saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY)) .rejects.toMatchObject({ code: 'ATTACHMENT_WRITE_FAILED' }) }) + + it('rejects prepared bytes that no longer match their content-addressed reference', async () => { + const storageRoot = await root() + const prepared = await prepareImageFile({ data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) + + await expect(commitPreparedImageFile(storageRoot, { + ...prepared, + data: Uint8Array.of(...prepared.data, 0), + })).rejects.toMatchObject({ code: 'ATTACHMENT_CORRUPT' }) + }) }) diff --git a/packages/attachment/attachment/tests/index.spec.ts b/packages/attachment/attachment/tests/index.spec.ts index 3a8fa23cbe..25afbcd420 100644 --- a/packages/attachment/attachment/tests/index.spec.ts +++ b/packages/attachment/attachment/tests/index.spec.ts @@ -76,6 +76,22 @@ class RecordingStore extends AttachmentStore { } } +class UnsupportedProjectionStore extends AttachmentStore { + readonly imageLimits = LIMITS + + validateImage(): Promise { + return Promise.resolve() + } + + saveImage(): Promise { + throw new Error('not used') + } + + readImage(): Promise { + throw new Error('not used') + } +} + function image(value: number, mediaType: ImageMediaType = 'image/png'): SaveImageAttachment { return { data: Uint8Array.of(value), mediaType, name: `${value}.png` } } @@ -134,6 +150,24 @@ describe('AttachmentStore.readImageRequests', () => { expect(store.calls).toEqual(['request:1.png', 'request:2.png']) expect(versions.map(version => version.master.name)).toEqual(['1.png', '2.png']) }) + + it('reports unsupported request projection and crop operations, preserving cancellation', async () => { + const store = new UnsupportedProjectionStore(new Context()) + const ref = (await new RecordingStore(new Context()).saveImage(image(1))).ref + await expect(store.readImageRequest(ref, { maxPixels: 1, maxBytes: 1 })) + .rejects.toMatchObject({ code: 'ATTACHMENT_PROJECTION_UNSUPPORTED' }) + await expect(store.cropImage(ref, { + previewWidth: 1, previewHeight: 1, x: 0, y: 0, width: 1, height: 1, + })).rejects.toMatchObject({ code: 'ATTACHMENT_PROJECTION_UNSUPPORTED' }) + + const controller = new AbortController() + const reason = new Error('cancel unsupported projection') + controller.abort(reason) + expect(() => store.readImageRequest(ref, { maxPixels: 1, maxBytes: 1 }, controller.signal)).toThrow(reason) + expect(() => store.cropImage(ref, { + previewWidth: 1, previewHeight: 1, x: 0, y: 0, width: 1, height: 1, + }, controller.signal)).toThrow(reason) + }) }) describe('isImageAdmissionError', () => { diff --git a/packages/fs/tool-fs/tests/read-image.spec.ts b/packages/fs/tool-fs/tests/read-image.spec.ts index 2f67a464fd..3bf86d27f5 100644 --- a/packages/fs/tool-fs/tests/read-image.spec.ts +++ b/packages/fs/tool-fs/tests/read-image.spec.ts @@ -226,6 +226,134 @@ describe('read_image_region', () => { expect(result.isError).toBe(true) expect(text(result)).toContain('not referenced by the current session') }) + + it('finds images nested in tool results after skipping a non-matching nested result', async () => { + const ctx = await setup() + const source = await ctx.attachments.saveImage({ data: PNG_3X3, mediaType: 'image/png' }) + const history = [createUserMessage({ + content: [ + { type: 'tool-result', toolCallId: CallId('unrelated'), content: [{ type: 'text', text: 'none' }] }, + { type: 'tool-result', toolCallId: CallId('nested'), content: [{ type: 'image', attachment: source.ref }] }, + ], + source: { kind: 'plugin', plugin: 'test' }, + })] + + const result = await call(ctx, 'read_image_region', { + attachment_id: source.ref.attachmentId, + preview_width: 3, + preview_height: 3, + x: 0, + y: 0, + width: 1, + height: 1, + }, agentOn('vision-model', 'visual', history)) + + expect(result.isError).toBe(false) + }) + + it('rejects a missing session, empty id, and invalid coordinate arguments', async () => { + const ctx = await setup() + const base = { + attachment_id: `sha256:${'f'.repeat(64)}`, + preview_width: 1, + preview_height: 1, + x: 0, + y: 0, + width: 1, + height: 1, + } + const noSession = await call(ctx, 'read_image_region', base) + expect(text(noSession)).toContain('requires an active agent session') + + const empty = await call(ctx, 'read_image_region', { ...base, attachment_id: ' ' }, agentOn('vision-model')) + expect(text(empty)).toContain('attachment_id must be a non-empty string') + + const source = await ctx.attachments.saveImage({ data: PNG_1X1, mediaType: 'image/png' }) + const history = [createUserMessage({ + content: [{ type: 'image', attachment: source.ref }], + source: { kind: 'plugin', plugin: 'test' }, + })] + const agent = agentOn('vision-model', 'visual', history) + for (const [field, value, expected] of [ + ['preview_width', 0, 'preview_width must be a positive integer'], + ['preview_height', 0, 'preview_height must be a positive integer'], + ['x', -1, 'x must be a non-negative integer'], + ['y', -1, 'y must be a non-negative integer'], + ['width', 0, 'width must be a positive integer'], + ['height', 0, 'height must be a positive integer'], + ] as const) { + const result = await call(ctx, 'read_image_region', { + ...base, + attachment_id: source.ref.attachmentId, + [field]: value, + }, agent) + expect(text(result)).toContain(expected) + } + }) + + it('projects optional crop metadata from a provider result', async () => { + class CropMetadataStore extends AttachmentStore { + readonly imageLimits: ImageAttachmentLimits = { + maxImageBytes: 1024, + maxImagesPerMessage: 1, + maxMessageImageBytes: 1024, + maxImagePixels: 100, + maxImageDimension: 100, + mediaTypes: ['image/png'], + } + + validateImage(): Promise { return Promise.resolve() } + saveImage(): Promise { throw new Error('not used') } + readImage(): Promise { throw new Error('not used') } + override cropImage(ref: ImageAttachmentRef): Promise { + return Promise.resolve({ + ref: { ...ref, sourceWidth: 2, sourceHeight: 2 }, + source: { mediaType: ref.mediaType, bytes: ref.bytes, width: 2, height: 2 }, + }) + } + } + const ctx = await setup({ attachments: false }) + await ctx.plugin(CropMetadataStore) + const ref: ImageAttachmentRef = { + attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`), + mediaType: 'image/png', bytes: 1, width: 1, height: 1, + } + const history = [createUserMessage({ + content: [{ type: 'image', attachment: ref }], + source: { kind: 'plugin', plugin: 'test' }, + })] + + const result = await call(ctx, 'read_image_region', { + attachment_id: ref.attachmentId, + preview_width: 1, + preview_height: 1, + x: 0, + y: 0, + width: 1, + height: 1, + }, agentOn('vision-model', 'visual', history)) + + expect(result.content[1]).toMatchObject({ + type: 'image', + attachment: { sourceWidth: 2, sourceHeight: 2 }, + }) + expect(result.content[1]).not.toHaveProperty('attachment.name') + }) + + it('declares a generic read presentation for image-region calls', async () => { + const ctx = await setup() + + expect(ctx.tools.get('read_image_region')?.presentCall?.({ + attachment_id: 'sha256:abc', + preview_width: 1, + preview_height: 1, + x: 0, + y: 0, + width: 1, + height: 1, + })) + .toEqual({ card: 'generic', title: 'Read image region sha256:abc', kind: 'read' }) + }) }) describe('read_image happy path', () => { From 72b204afa1753324430df36aab2c6ae29e952510 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 20:32:50 +0800 Subject: [PATCH 17/28] feat(images): expand source upload envelope --- ...26-07-05-reconstructable-requests.i18n.yaml | 2 +- .../2026-07-05-reconstructable-requests.zh.md | 2 +- ...20-unified-image-request-pipeline.i18n.yaml | 4 ++-- ...026-08-20-unified-image-request-pipeline.md | 2 +- ...-08-20-unified-image-request-pipeline.zh.md | 4 ++-- ...-08-20-attachment-read-quarantine.i18n.yaml | 2 +- ...2026-08-20-attachment-read-quarantine.zh.md | 2 +- docs/config-catalog.i18n.yaml | 4 ++-- docs/config-catalog.md | 12 ++++++------ docs/config-catalog.zh.md | 12 ++++++------ docs/subsystems/attachment.i18n.yaml | 4 ++-- docs/subsystems/attachment.md | 2 ++ docs/subsystems/attachment.zh.md | 2 ++ .../attachment-local/README.i18n.yaml | 4 ++-- packages/attachment/attachment-local/README.md | 2 +- .../attachment/attachment-local/README.zh.md | 2 +- .../attachment/attachment-local/src/index.ts | 18 +++++++++--------- .../attachment-local/tests/index.spec.ts | 6 +++++- packages/client/connection/README.i18n.yaml | 4 ++-- packages/client/connection/README.md | 2 +- packages/client/connection/README.zh.md | 2 +- packages/client/connection/src/http-bridge.ts | 6 +++--- packages/client/connection/src/index.ts | 2 +- .../connection/tests/node-half.host.spec.ts | 6 ++++++ 24 files changed, 61 insertions(+), 47 deletions(-) diff --git a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.i18n.yaml b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.i18n.yaml index 47c4c2d198..23c4ea4142 100644 --- a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.i18n.yaml @@ -3,4 +3,4 @@ # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.md 2026-07-05-reconstructable-requests.md: 3f49ba71a6b98a84b05530c900e902b0cf9f6449 -2026-07-05-reconstructable-requests.zh.md: 8eee44449140d656a669ac506057e4fa09c2f747 +2026-07-05-reconstructable-requests.zh.md: 7b8a9df65b60f975bc3ae60b2c1b0c3a8cc22e95 diff --git a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.zh.md b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.zh.md index 8eee444491..7b8a9df65b 100644 --- a/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.zh.md +++ b/.agents/notes/implemented/architecture/2026-07-05-reconstructable-requests.zh.md @@ -51,6 +51,6 @@ Status: implemented - 在提供方处仍需全价计算的内容是固有的且已记录的:压缩(其 `compaction/*` 事件和替换条目)、真正的提示词、工具或配置变更(reason 为 `change` 的 `request/header`),或带漂移的进程边界(不同的 `resume` 快照)。提供方自身的 reasoning-content 排除由服务端管理。 - `agent/pre-step` 是当前请求的消息通道;直接修改 inbox 则是最终进入后续请求的通道。 - 工具结果裁剪无需新机制:一个已记录的单条目 surface replace(`start === end`),携带同一 `callId` 下裁剪后的 `tool/result`——属压缩家族,回放正确,缓存失效由相同的压力逻辑批量处理。 -- 无法读取的被引用附件对象仍会让模型请求失败;[附件自动隔离](../../proposed/bug-fix/2026-08-20-attachment-read-quarantine.md)记录了不削弱字节精确重建的拟议恢复方案。 +- 无法读取的被引用附件对象仍会让模型请求失败;[附件自动隔离](../../proposed/bug-fix/2026-08-20-attachment-read-quarantine.zh.md)记录了不削弱字节精确重建的拟议恢复方案。 - 会话日志每个循环实例增长一个 `request/header` 快照,并在真正变更时增加快照。它比 delta 编解码器更大,但相对分片密集型日志仍然很小,并只保留一种回放表示。`SESSION_FORMAT_VERSION` 保持 `0`;旧的 delta 事件被拒绝而非迁移。 - 快照预期输出变更一次(每个 transcript(文本记录)增加其 header 事件);写入文件系统的 fixture(测试前置数据)以规范化的撰写形式存储,工具参数使用 cwd 相对路径,因为回放只对 cwd 无关的参数路径做往返。 diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml index 07d9effb78..a721138585 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md -2026-08-20-unified-image-request-pipeline.md: 07632e9e0c3aac33d89acd8aebc0f0114550ddb6 -2026-08-20-unified-image-request-pipeline.zh.md: 9d95346dab2a7747c4bcef9f213ec0fa8e5ba067 +2026-08-20-unified-image-request-pipeline.md: c4af375d94ebf2b52fbdd0e8d3d4ee715f87f50e +2026-08-20-unified-image-request-pipeline.zh.md: a1e10c63804b42da127bd115c35587191f0a60f0 diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md index 07632e9e0c..c4af375d94 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md @@ -14,7 +14,7 @@ The image path has two explicit versions. The attachment backend owns a provider ### Provider-independent master -Admission fully decodes each source under a configurable 32MiB, 100MP, and 16384px-per-side envelope. It applies EXIF orientation, removes metadata and color profiles, converts to 8-bit sRGB/sRGBA, and preserves aspect ratio while limiting the long edge to `masterMaxDimension`, 2048px by default. `sourceWidth` and `sourceHeight` record orientation-applied dimensions when preparation reduces the raster. +Admission accepts at most 20 images and 200MiB of encoded source bytes per message. Each source is fully decoded under configurable 20MiB, 64,000,000-pixel, and 8192px-per-side limits. Preparation applies EXIF orientation, removes metadata and color profiles, converts to 8-bit sRGB/sRGBA, and preserves aspect ratio while limiting the long edge to `masterMaxDimension`, 2048px by default. `sourceWidth` and `sourceHeight` record orientation-applied dimensions when preparation reduces the raster. The master has an independent `masterMaxBytes` safety cap, 4MiB by default. Alpha is never flattened. A nearest-neighbour bounded sample classifies color complexity without averaging high-frequency pixels. Confirmed low-color input tries PNG, with palette encoding only when no alpha channel is present, followed by WebP qualities 85, 80, and 75. Other alpha input tries WebP at those qualities; other opaque input tries JPEG. Candidates execute in order and stop at the first result within the cap. Dimensions shrink only after every candidate at one size exceeds the cap. The source extension does not classify a PNG as low color. A clean, single-frame 8-bit sRGB/sRGBA PNG, JPEG, or WebP within both master limits passes through byte-identically and retains content-addressed deduplication. GIF, animation, metadata, orientation, 16-bit PNG, and incompatible color spaces force conversion. The source and a converted output are each fully decoded once; the output must match its format, dimensions, depth, color space, and alpha facts before its digest enters the reference. diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md index 9d95346dab..a1e10c6380 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md @@ -14,7 +14,7 @@ Status: implemented ### 提供方无关的主版本 -准入在可配置的 32MiB、1 亿像素和单边 16384px 源图范围内完整解码每张图片。处理会应用 EXIF 方向,删除元数据和色彩配置文件,转换为 8-bit sRGB/sRGBA,并保持宽高比把长边限制到 `masterMaxDimension`,默认 2048px。处理缩小光栅时,`sourceWidth` 和 `sourceHeight` 记录应用方向后的源尺寸。 +每条消息最多准入 20 张图片,源图编码字节总量不超过 200MiB。每张源图会在可配置的 20MiB、64,000,000 像素和单边 8192px 限制内完整解码。处理会应用 EXIF 方向,删除元数据和色彩配置文件,转换为 8-bit sRGB/sRGBA,并保持宽高比把长边限制到 `masterMaxDimension`,默认 2048px。处理缩小光栅时,`sourceWidth` 和 `sourceHeight` 记录应用方向后的源尺寸。 主版本有独立的 `masterMaxBytes` 安全上限,默认 4MiB。透明通道绝不铺平。系统通过 nearest-neighbour 对有界样本判断色彩复杂度,不会通过像素平均把高频图片误判为低色数。确认的低色数输入先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80、75 的 WebP;其他透明输入依次尝试这些质量的 WebP;其他非透明输入依次尝试这些质量的 JPEG。候选按顺序执行,首个不超过上限的结果会立即返回。同一尺寸的候选全部超限后才会缩小尺寸。源扩展名不会把 PNG 归类为低色数图片。处于两个主版本上限内的干净、单帧、8-bit sRGB/sRGBA PNG、JPEG 或 WebP 按字节原样直通,并保留内容寻址去重。GIF、动图、元数据、方向、16-bit PNG 和不兼容色彩空间都会触发转换。源图和转换输出各完整解码一次;输出的格式、尺寸、位深、色彩空间和透明通道事实通过校验后,其摘要才会进入引用。 @@ -42,7 +42,7 @@ Status: implemented 16-bit RGB 或 RGBA PNG 属于普通可接纳输入,会转换为 8-bit sRGB/sRGBA。本地转换失败时,`read_image` 会写明路径、检测到的 16-bit PNG、所需规范形式和手工转换方法。如果 DeepSeek 拒绝已规范化请求版本,主错误会写明附件 ID 或显示名称、持久消息和图片位置、规范化媒体类型、8-bit sRGB/sRGBA 位深、尺寸和提供方消息。多图片错误无法确定对象时会列出全部候选图片。原始提供方正文保留为错误 cause,不会成为唯一可见消息。 -持久附件对象之后缺失或无法通过完整性校验时,系统仍会明确失败。持久隔离和经校验恢复需要新增会话事件,由[隔离不可读历史附件](../../proposed/bug-fix/2026-08-20-attachment-read-quarantine.md)继续跟踪。 +持久附件对象之后缺失或无法通过完整性校验时,系统仍会明确失败。持久隔离和经校验恢复需要新增会话事件,由[隔离不可读历史附件](../../proposed/bug-fix/2026-08-20-attachment-read-quarantine.zh.md)继续跟踪。 ## Alternatives considered diff --git a/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.i18n.yaml b/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.i18n.yaml index ce37d7b8d3..b53d43cdc7 100644 --- a/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.i18n.yaml +++ b/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.i18n.yaml @@ -3,4 +3,4 @@ # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.md 2026-08-20-attachment-read-quarantine.md: 28e0f26cee2ec1e257fd4d43b4edc4300e2c6f23 -2026-08-20-attachment-read-quarantine.zh.md: bdc1d580a5159edcd288552e1bde9d80ea1eafd8 +2026-08-20-attachment-read-quarantine.zh.md: 7f4ceae4e1fe9e656ed762de9828e14145976dc3 diff --git a/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.zh.md b/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.zh.md index bdc1d580a5..7f4ceae4e1 100644 --- a/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.zh.md +++ b/.agents/notes/proposed/bug-fix/2026-08-20-attachment-read-quarantine.zh.md @@ -6,7 +6,7 @@ Status: proposed ## 问题 -已接纳的 `ImageAttachmentRef` 会留在持久历史中,因此在被压缩替换前都会参与之后的每次请求。引用对象丢失、完整性校验失败或无法读取时,`AttachmentStore.readImage()` 会返回 `ATTACHMENT_NOT_FOUND`、`ATTACHMENT_CORRUPT` 或 `ATTACHMENT_READ_FAILED`。未变化的历史随后会让之后每次模型请求在同一对象上失败,使会话无法继续,即使其余消息仍可使用。这是[可重建请求](../../implemented/architecture/2026-07-05-reconstructable-requests.md)保留为明确失败的对象不可用情况。 +已接纳的 `ImageAttachmentRef` 会留在持久历史中,因此在被压缩替换前都会参与之后的每次请求。引用对象丢失、完整性校验失败或无法读取时,`AttachmentStore.readImage()` 会返回 `ATTACHMENT_NOT_FOUND`、`ATTACHMENT_CORRUPT` 或 `ATTACHMENT_READ_FAILED`。未变化的历史随后会让之后每次模型请求在同一对象上失败,使会话无法继续,即使其余消息仍可使用。这是[可重建请求](../../implemented/architecture/2026-07-05-reconstructable-requests.zh.md)保留为明确失败的对象不可用情况。 ## 提案 diff --git a/docs/config-catalog.i18n.yaml b/docs/config-catalog.i18n.yaml index 3523b633cd..804d5dd86a 100644 --- a/docs/config-catalog.i18n.yaml +++ b/docs/config-catalog.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/config-catalog.md -config-catalog.md: dd91a870ecb338e784acdd1ffa0a470fa33d8813 -config-catalog.zh.md: a412a4f0afe652863cda1edad0e344b17e1697ac +config-catalog.md: 661e9a50200fd5c650c389d9bb631c04de61d228 +config-catalog.zh.md: 4299bccc1f59899bd78fd64f915c784e26eea49d diff --git a/docs/config-catalog.md b/docs/config-catalog.md index dd91a870ec..661e9a5020 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -327,15 +327,15 @@ Source: [`packages/core/agent-tool-presentation/src/index.ts:38`](../packages/co export interface Config { /** Explicit harness home; omitted follows `DSH_HOME`, then `~/.dsh`. */ dshHome?: string - /** Maximum encoded bytes accepted for one submitted image. */ + /** Maximum encoded bytes accepted for one submitted image. Default: 20 MiB. */ maxImageBytes?: number - /** Maximum image count accepted in one submitted message. */ + /** Maximum image count accepted in one submitted message. Default: 20. */ maxImagesPerMessage?: number - /** Maximum aggregate encoded image bytes accepted in one submitted message. */ + /** Maximum aggregate encoded image bytes accepted in one submitted message. Default: 200 MiB. */ maxMessageImageBytes?: number - /** Maximum intrinsic width multiplied by height accepted for one submitted image. */ + /** Maximum intrinsic width multiplied by height accepted for one submitted image. Default: 64,000,000. */ maxImagePixels?: number - /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. */ + /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. Default: 8192px. */ maxImageDimension?: number /** Long-edge pixel cap of the stored provider-independent master version. */ masterMaxDimension?: number @@ -413,7 +413,7 @@ export interface ConnectionConfig { * that is not a bare, canonical authority fails the plugin load. */ trustedHosts?: string[] - /** Maximum buffered JSON body for every `/api` request. */ + /** Maximum buffered JSON body for every `/api` request. Default: 300 MiB. */ maxRequestBodyBytes?: number } ``` diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index a412a4f0af..4299bccc1f 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -329,15 +329,15 @@ export interface Config { export interface Config { /** Explicit harness home; omitted follows `DSH_HOME`, then `~/.dsh`. */ dshHome?: string - /** Maximum encoded bytes accepted for one submitted image. */ + /** Maximum encoded bytes accepted for one submitted image. Default: 20 MiB. */ maxImageBytes?: number - /** Maximum image count accepted in one submitted message. */ + /** Maximum image count accepted in one submitted message. Default: 20. */ maxImagesPerMessage?: number - /** Maximum aggregate encoded image bytes accepted in one submitted message. */ + /** Maximum aggregate encoded image bytes accepted in one submitted message. Default: 200 MiB. */ maxMessageImageBytes?: number - /** Maximum intrinsic width multiplied by height accepted for one submitted image. */ + /** Maximum intrinsic width multiplied by height accepted for one submitted image. Default: 64,000,000. */ maxImagePixels?: number - /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. */ + /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. Default: 8192px. */ maxImageDimension?: number /** Long-edge pixel cap of the stored provider-independent master version. */ masterMaxDimension?: number @@ -415,7 +415,7 @@ export interface ConnectionConfig { * that is not a bare, canonical authority fails the plugin load. */ trustedHosts?: string[] - /** Maximum buffered JSON body for every `/api` request. */ + /** Maximum buffered JSON body for every `/api` request. Default: 300 MiB. */ maxRequestBodyBytes?: number } ``` diff --git a/docs/subsystems/attachment.i18n.yaml b/docs/subsystems/attachment.i18n.yaml index 55a43dd247..e391a3aa27 100644 --- a/docs/subsystems/attachment.i18n.yaml +++ b/docs/subsystems/attachment.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/subsystems/attachment.md -attachment.md: ec9d1f27bdde4a4d5b6e6e7328260bcb4af49948 -attachment.zh.md: e79c2df4ca168bcae4fd45e61812a4f86ce2194b +attachment.md: ea15172e3e1fafec2e09c3bedc2590fc7551eb2e +attachment.zh.md: c04114c9691fa1ba03446f903c4baf5ae021da4c diff --git a/docs/subsystems/attachment.md b/docs/subsystems/attachment.md index 66d00eb387..99d4ba7682 100644 --- a/docs/subsystems/attachment.md +++ b/docs/subsystems/attachment.md @@ -52,6 +52,8 @@ interface ImageAttachmentLimits { } ``` +The local backend admits at most 20 images and 200 MiB of encoded source data per message. One source may use up to 20 MiB, 64,000,000 pixels, and 8192 pixels on either side. These source limits precede the independent 2048-pixel, 4 MiB master preparation stage. + The reference records intrinsic dimensions and encoded length so clients can lay out history without decoding first, while every authoritative read still re-checks digest, media signature, dimensions, and metadata against the object. ## Commit and verified-read payloads diff --git a/docs/subsystems/attachment.zh.md b/docs/subsystems/attachment.zh.md index 4c3a4ce427..d235c9ed2e 100644 --- a/docs/subsystems/attachment.zh.md +++ b/docs/subsystems/attachment.zh.md @@ -52,6 +52,8 @@ interface ImageAttachmentLimits { } ``` +本地后端每条消息最多准入 20 张图片,源图编码数据总量不超过 200 MiB。单张源图不得超过 20 MiB、64,000,000 像素和单边 8192 像素。这些源文件限制先于独立的 2048 像素、4 MiB 主版本处理阶段执行。 + 引用记录固有尺寸和编码长度,使客户端无需先解码即可排布历史记录;每次权威读取仍会根据对象重新校验摘要、媒体签名、尺寸和元数据。 ## 提交与经校验读取的数据 diff --git a/packages/attachment/attachment-local/README.i18n.yaml b/packages/attachment/attachment-local/README.i18n.yaml index 0ebf6a80fe..d15a1fd01e 100644 --- a/packages/attachment/attachment-local/README.i18n.yaml +++ b/packages/attachment/attachment-local/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/attachment/attachment-local/README.md -README.md: 77b68357d5a961549bef0a015b8e48ba02fbd702 -README.zh.md: 05932c93e40d42a7f8fcdcf906f6669f6f8f7073 +README.md: 6141b7559492aa4c50831c8124a917bfdb704f4b +README.zh.md: 2a8ed6e1aef8022aba5053bf1ef0f9728340d086 diff --git a/packages/attachment/attachment-local/README.md b/packages/attachment/attachment-local/README.md index 77b68357d5..6141b75594 100644 --- a/packages/attachment/attachment-local/README.md +++ b/packages/attachment/attachment-local/README.md @@ -4,7 +4,7 @@ English | [中文](README.zh.md) The private local implementation of [`@deepseek-ai/dsh-attachment`](../attachment). Objects land at `/attachments/v1/objects//` and are addressed by an opaque `sha256:` id. Each process proves a home durable once by syncing every ancestor entry to the filesystem root. Writes use a private staging directory, owner-only files, a synced temporary file, an atomic exclusive hard-link publish, and directory syncs on the publication path (POSIX; Windows relies on filesystem metadata journaling) so the reported reference survives a crash. -Admission fully decodes the raster against a wide source envelope: 32MiB, 100MP, and 16384px per side by default. It then prepares a provider-independent master. EXIF orientation is applied, metadata and color profiles are removed, pixels become 8-bit sRGB/sRGBA, and the long edge is reduced proportionally to `masterMaxDimension` (2048px by default). The master has its own `masterMaxBytes` safety cap (4MiB by default). Alpha is retained. A nearest-neighbour bounded sample classifies color complexity without averaging high-frequency pixels. Confirmed low-color images try PNG, using a palette only when the input has no alpha channel, then WebP at qualities 85, 80, and 75. Other alpha images try WebP at those qualities; other opaque images try JPEG. Each candidate runs only after the preceding candidate exceeds the cap. Dimensions shrink only after every candidate at one size exceeds the cap. A clean, single-frame 8-bit sRGB/sRGBA PNG, JPEG, or WebP already within both master limits passes through byte-identically; 16-bit PNG, GIF, animated input, metadata, orientation, and incompatible color spaces force conversion. The source and a converted master are each fully decoded once. `saveImages` prepares and verifies every master once before publishing the batch, so validation failure leaves no partial references and commit does not repeat full image encoding. +Admission accepts at most 20 images and 200MiB of encoded source bytes per message. Each source may use up to 20MiB, 64,000,000 pixels, and 8192px per side. It then prepares a provider-independent master. EXIF orientation is applied, metadata and color profiles are removed, pixels become 8-bit sRGB/sRGBA, and the long edge is reduced proportionally to `masterMaxDimension` (2048px by default). The master has its own `masterMaxBytes` safety cap (4MiB by default). Alpha is retained. A nearest-neighbour bounded sample classifies color complexity without averaging high-frequency pixels. Confirmed low-color images try PNG, using a palette only when the input has no alpha channel, then WebP at qualities 85, 80, and 75. Other alpha images try WebP at those qualities; other opaque images try JPEG. Each candidate runs only after the preceding candidate exceeds the cap. Dimensions shrink only after every candidate at one size exceeds the cap. A clean, single-frame 8-bit sRGB/sRGBA PNG, JPEG, or WebP already within both master limits passes through byte-identically; 16-bit PNG, GIF, animated input, metadata, orientation, and incompatible color spaces force conversion. The source and a converted master are each fully decoded once. `saveImages` prepares and verifies every master once before publishing the batch, so validation failure leaves no partial references and commit does not repeat full image encoding. Request versions live below `/attachments/v1/request-images/`. `readImageRequest` scales the stored master under a total-pixel budget without enlargement, then enforces a separate encoded-byte cap. The request encoder uses the same color branches, with PNG (palette only without alpha) before WebP 85 and 80 for low-color images, WebP 85 then 80 for other alpha images, and JPEG 85 then 80 for other opaque images. It also executes candidates lazily and reduces dimensions only after both quality attempts exceed the request cap. Its cache identity includes the master id, transform version, pixel and byte budgets, optional master-coordinate crop, and fixed encoder settings. Cached bytes are fully decoded and checked as 8-bit sRGB/sRGBA before use. Concurrent calls for one identity share one transform and cache write; cancelling one waiter does not cancel the shared work. `readImageRequests` schedules batches through the service's FIFO limiter. `imageCompressionConcurrency` controls simultaneous master and request transforms from 1 through 8 and defaults to 2; file publication remains ordered after preparation. `cropImage` maps coordinates measured on a model preview back to the master, crops the master rather than the preview, and commits the crop as another durable attachment. diff --git a/packages/attachment/attachment-local/README.zh.md b/packages/attachment/attachment-local/README.zh.md index 05932c93e4..2a8ed6e1ae 100644 --- a/packages/attachment/attachment-local/README.zh.md +++ b/packages/attachment/attachment-local/README.zh.md @@ -4,7 +4,7 @@ 这是 [`@deepseek-ai/dsh-attachment`](../attachment) 的私有本地实现。对象存放在 `/attachments/v1/objects//`,并通过不透明的 `sha256:` 标识符寻址。每个进程都会把每级祖先目录项同步到文件系统根目录,以此一次性证明 home 已持久化。写入使用私有暂存目录、仅所有者可访问的文件、经过同步的临时文件、原子且排他的硬链接发布,并对发布路径执行目录同步(适用于 POSIX;Windows 依赖文件系统元数据日志),确保已报告的引用能够在崩溃后继续存在。 -准入针对宽松的源图范围完整解码光栅,默认上限为 32MiB、1 亿像素和单边 16384px。随后生成提供方无关的主版本:应用 EXIF 方向,删除元数据和色彩配置文件,转换为 8-bit sRGB/sRGBA,并保持宽高比把长边限制到 `masterMaxDimension`(默认 2048px)。主版本有独立的 `masterMaxBytes` 安全上限(默认 4MiB)。透明通道会保留。系统用 nearest-neighbour 对有界样本分类,不会通过像素平均把高频图片误判为低色数。确认的低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80、75 的 WebP;其他透明图片依次尝试这些质量的 WebP;其他非透明图片依次尝试这些质量的 JPEG。只有前一个候选超限时才会执行下一个候选;同一尺寸的候选全部超限后才缩小尺寸。已经处于两个主版本上限内的干净、单帧、8-bit sRGB/sRGBA PNG、JPEG 或 WebP 按字节原样直通;16-bit PNG、GIF、动图、元数据、方向和不兼容色彩空间都会触发转换。源图和转换后的主版本各完整解码一次。`saveImages` 在发布任何批次成员前为每张图片各准备并验证一次主版本,因此校验失败不会留下部分引用,提交阶段也不会重复执行完整图片编码。 +每条消息最多准入 20 张图片,源图编码字节总量不超过 200MiB。每张源图不得超过 20MiB、64,000,000 像素和单边 8192px。随后生成提供方无关的主版本:应用 EXIF 方向,删除元数据和色彩配置文件,转换为 8-bit sRGB/sRGBA,并保持宽高比把长边限制到 `masterMaxDimension`(默认 2048px)。主版本有独立的 `masterMaxBytes` 安全上限(默认 4MiB)。透明通道会保留。系统用 nearest-neighbour 对有界样本分类,不会通过像素平均把高频图片误判为低色数。确认的低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80、75 的 WebP;其他透明图片依次尝试这些质量的 WebP;其他非透明图片依次尝试这些质量的 JPEG。只有前一个候选超限时才会执行下一个候选;同一尺寸的候选全部超限后才缩小尺寸。已经处于两个主版本上限内的干净、单帧、8-bit sRGB/sRGBA PNG、JPEG 或 WebP 按字节原样直通;16-bit PNG、GIF、动图、元数据、方向和不兼容色彩空间都会触发转换。源图和转换后的主版本各完整解码一次。`saveImages` 在发布任何批次成员前为每张图片各准备并验证一次主版本,因此校验失败不会留下部分引用,提交阶段也不会重复执行完整图片编码。 请求版本保存在 `/attachments/v1/request-images/`。`readImageRequest` 在不放大小图的前提下,把存储的主版本缩放到总像素预算内,再执行独立的编码字节上限。请求编码器使用同一分类分支:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。候选仍按需执行,两个质量档均超限后才缩小尺寸。缓存身份包含主版本 ID、变换策略版本、像素和字节预算、可选的主版本坐标裁剪区域以及固定编码参数。缓存字节在使用前会完整解码并校验为 8-bit sRGB/sRGBA。同一身份的并发调用共享一次变换和缓存写入;取消一个等待方不会取消共享任务。`readImageRequests` 通过服务的 FIFO 限流器调度批次。`imageCompressionConcurrency` 控制同时执行的主版本和请求版本变换,范围为 1 至 8,默认值为 2;文件发布仍在准备结束后按顺序执行。`cropImage` 把模型在预览图上测得的坐标映射回主版本,从主版本而非预览图裁剪,并把裁剪结果提交为另一个持久附件。 diff --git a/packages/attachment/attachment-local/src/index.ts b/packages/attachment/attachment-local/src/index.ts index 46e39fb8ff..516280548c 100644 --- a/packages/attachment/attachment-local/src/index.ts +++ b/packages/attachment/attachment-local/src/index.ts @@ -27,15 +27,15 @@ export type { PreparedImageFile } from './store.ts' export { previewCropToMaster, readRequestImageFile, requestImageDimensions, requestImageVariantId } from './request-image.ts' /** Default maximum encoded bytes for one submitted image; oversized sources are refused, not shrunk. */ -export const DEFAULT_MAX_IMAGE_BYTES = 32 * 1024 * 1024 +export const DEFAULT_MAX_IMAGE_BYTES = 20 * 1024 * 1024 /** Default maximum images in one prompt. */ export const DEFAULT_MAX_IMAGES_PER_MESSAGE = 20 /** Default maximum aggregate image bytes in one prompt. */ -export const DEFAULT_MAX_MESSAGE_IMAGE_BYTES = 100 * 1024 * 1024 +export const DEFAULT_MAX_MESSAGE_IMAGE_BYTES = 200 * 1024 * 1024 /** Default maximum intrinsic pixels for one submitted image. */ -export const DEFAULT_MAX_IMAGE_PIXELS = 100_000_000 +export const DEFAULT_MAX_IMAGE_PIXELS = 64_000_000 /** Default per-side pixel cap for one submitted image. */ -export const DEFAULT_MAX_IMAGE_DIMENSION = 16384 +export const DEFAULT_MAX_IMAGE_DIMENSION = 8192 /** * Default long-edge target of the stored image master. A larger source * is admitted and downscaled to this edge, so admission bounds what rides @@ -53,15 +53,15 @@ export const MAX_IMAGE_COMPRESSION_CONCURRENCY = 8 export interface Config { /** Explicit harness home; omitted follows `DSH_HOME`, then `~/.dsh`. */ dshHome?: string - /** Maximum encoded bytes accepted for one submitted image. */ + /** Maximum encoded bytes accepted for one submitted image. Default: 20 MiB. */ maxImageBytes?: number - /** Maximum image count accepted in one submitted message. */ + /** Maximum image count accepted in one submitted message. Default: 20. */ maxImagesPerMessage?: number - /** Maximum aggregate encoded image bytes accepted in one submitted message. */ + /** Maximum aggregate encoded image bytes accepted in one submitted message. Default: 200 MiB. */ maxMessageImageBytes?: number - /** Maximum intrinsic width multiplied by height accepted for one submitted image. */ + /** Maximum intrinsic width multiplied by height accepted for one submitted image. Default: 64,000,000. */ maxImagePixels?: number - /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. */ + /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. Default: 8192px. */ maxImageDimension?: number /** Long-edge pixel cap of the stored provider-independent master version. */ masterMaxDimension?: number diff --git a/packages/attachment/attachment-local/tests/index.spec.ts b/packages/attachment/attachment-local/tests/index.spec.ts index 89ada53298..c3c7693614 100644 --- a/packages/attachment/attachment-local/tests/index.spec.ts +++ b/packages/attachment/attachment-local/tests/index.spec.ts @@ -19,7 +19,11 @@ import LocalAttachmentStore, { describe('local attachment service', () => { it('resolves every omitted admission limit explicitly', () => { const service = new LocalAttachmentStore(new Context(), {}) - expect(DEFAULT_MAX_IMAGE_BYTES).toBe(32 * 1024 * 1024) + expect(DEFAULT_MAX_IMAGE_BYTES).toBe(20 * 1024 * 1024) + expect(DEFAULT_MAX_IMAGES_PER_MESSAGE).toBe(20) + expect(DEFAULT_MAX_MESSAGE_IMAGE_BYTES).toBe(200 * 1024 * 1024) + expect(DEFAULT_MAX_IMAGE_PIXELS).toBe(64_000_000) + expect(DEFAULT_MAX_IMAGE_DIMENSION).toBe(8192) expect(service.imageLimits).toEqual({ maxImageBytes: DEFAULT_MAX_IMAGE_BYTES, maxImagesPerMessage: DEFAULT_MAX_IMAGES_PER_MESSAGE, diff --git a/packages/client/connection/README.i18n.yaml b/packages/client/connection/README.i18n.yaml index 44a428fa61..9d430a14a2 100644 --- a/packages/client/connection/README.i18n.yaml +++ b/packages/client/connection/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/client/connection/README.md -README.md: a7562b9dac57930b1abc0b76b9079a6865a38b35 -README.zh.md: 24c56e598ebd4b5ca39e433c5782399909f528b8 +README.md: 71ef204a589bb67c15ccab58d3cac5a13782ce27 +README.zh.md: 6d33ac3c13cdfceeba6e7472b618084267d09bbc diff --git a/packages/client/connection/README.md b/packages/client/connection/README.md index a7562b9dac..71ef204a58 100644 --- a/packages/client/connection/README.md +++ b/packages/client/connection/README.md @@ -23,4 +23,4 @@ None; this package neither assembles nor sends a provider request. ## Known Limitations and Deferred Work - **History resumes an unattached session** — opening history may create the host-side agent and add latency to the first open; there is no persistence-only read path. -- **The `/api` bridge buffers each request body in memory** — `maxRequestBodyBytes` (default 160 MiB, sized for the default 100 MiB aggregate image limit after base64 expansion plus envelope headroom) is therefore also the per-request resident bound; a streaming body path would be needed to lower it without shrinking the image limits. +- **The `/api` bridge buffers each request body in memory** — `maxRequestBodyBytes` (default 300 MiB, sized for the default 200 MiB aggregate image limit after base64 expansion plus envelope headroom) is therefore also the per-request resident bound; a streaming body path would be needed to lower it without shrinking the image limits. diff --git a/packages/client/connection/README.zh.md b/packages/client/connection/README.zh.md index 24c56e598e..6d33ac3c13 100644 --- a/packages/client/connection/README.zh.md +++ b/packages/client/connection/README.zh.md @@ -23,4 +23,4 @@ node 半侧在桥接或 upgrade 前守卫 `/api` 下的每个入口(`src/api-r ## 已知限制与暂缓事项 - **History 会恢复未附加的会话**:打开 history 可能创建宿主侧 agent,并增加首次打开的延迟;没有仅从持久化读取的路径。 -- **`/api` 桥把每个请求体整体缓冲在内存里**:`maxRequestBodyBytes`(默认 160 MiB,按默认 100 MiB 图片总量上限经 base64 膨胀加信封余量得出)因此同时是单请求的驻留内存上界;要降低它而不缩小图片限额,需要流式请求体路径。 +- **`/api` 桥把每个请求体整体缓冲在内存里**:`maxRequestBodyBytes`(默认 300 MiB,按默认 200 MiB 图片总量上限经 base64 膨胀加信封余量得出)因此同时是单请求的驻留内存上界;要降低它而不缩小图片限额,需要流式请求体路径。 diff --git a/packages/client/connection/src/http-bridge.ts b/packages/client/connection/src/http-bridge.ts index c26d83b6b7..07fc0fc5da 100644 --- a/packages/client/connection/src/http-bridge.ts +++ b/packages/client/connection/src/http-bridge.ts @@ -6,10 +6,10 @@ import type { IncomingMessage, ServerResponse } from 'node:http' /** Default carrier cap for all HTTP RPC bodies: sized for the default - * aggregate image limit (100 MiB) after base64 expansion plus envelope - * headroom (~134.3 MiB required), rounded up for slack. The bridge buffers + * aggregate image limit (200 MiB) after base64 expansion plus envelope + * headroom (~267.7 MiB required), rounded up for slack. The bridge buffers * each body in memory, so this cap is also the per-request resident bound. */ -export const DEFAULT_MAX_REQUEST_BODY_BYTES = 160 * 1024 * 1024 +export const DEFAULT_MAX_REQUEST_BODY_BYTES = 300 * 1024 * 1024 /** Transport-independent request handler consumed by the Host HTTP bridge. */ export interface FetchHandler { diff --git a/packages/client/connection/src/index.ts b/packages/client/connection/src/index.ts index 35084918e8..a1764a3d58 100644 --- a/packages/client/connection/src/index.ts +++ b/packages/client/connection/src/index.ts @@ -57,7 +57,7 @@ export interface ConnectionConfig { * that is not a bare, canonical authority fails the plugin load. */ trustedHosts?: string[] - /** Maximum buffered JSON body for every `/api` request. */ + /** Maximum buffered JSON body for every `/api` request. Default: 300 MiB. */ maxRequestBodyBytes?: number } diff --git a/packages/client/connection/tests/node-half.host.spec.ts b/packages/client/connection/tests/node-half.host.spec.ts index 0b30ce6520..022436d558 100644 --- a/packages/client/connection/tests/node-half.host.spec.ts +++ b/packages/client/connection/tests/node-half.host.spec.ts @@ -11,6 +11,7 @@ import type { AttachmentStore } from '@deepseek-ai/dsh-attachment' import { RpcId, type ClientRequest } from '@deepseek-ai/dsh-host-apiproxy/api' import type { WebServer, WebRoute, WebUpgradeRoute } from '@deepseek-ai/dsh-host-webserver' import { API_PATH, apply, HOST_EVENTS_PATH, inject, MUX_EVENTS_PATH, type HostConnectionHandle } from '../src/index.ts' +import { DEFAULT_MAX_REQUEST_BODY_BYTES } from '../src/http-bridge.ts' /** Structural webServer fake recording both route registries. */ function fakeHttpServer( @@ -90,6 +91,11 @@ async function mounted(config?: { trustedHosts?: string[] }): Promise<{ } describe('connection node half', () => { + it('reserves enough default carrier capacity for the 200 MiB image batch', () => { + expect(DEFAULT_MAX_REQUEST_BODY_BYTES).toBe(300 * 1024 * 1024) + expect(DEFAULT_MAX_REQUEST_BODY_BYTES).toBeGreaterThan(Math.ceil(200 * 1024 * 1024 * 4 / 3) + 1024 * 1024) + }) + it('fails loud when the carrier cap cannot hold the configured image batch', () => { const ctx = new Context() const routes: WebRoute[] = [] From d65e2a9e8ada278607cbf3be6a078a51d652c337 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 21:46:59 +0800 Subject: [PATCH 18/28] test(images): close unified pipeline coverage gaps --- .../attachment/attachment-local/src/index.ts | 8 +- .../attachment-local/tests/encoding.spec.ts | 1 + .../tests/request-image.spec.ts | 17 +- packages/fs/tool-fs/tests/read-image.spec.ts | 27 ++ .../commands/tests/commands.spec.ts | 5 +- packages/llm/llm-deepseek/src/file-store.ts | 37 ++- packages/llm/llm-deepseek/src/files-api.ts | 5 +- packages/llm/llm-deepseek/src/serialize.ts | 2 +- .../llm/llm-deepseek/tests/adapter.spec.ts | 37 ++- .../llm/llm-deepseek/tests/file-store.spec.ts | 278 +++++++++++++++++- .../llm/llm-deepseek/tests/files-api.spec.ts | 12 +- .../llm/llm-deepseek/tests/serialize.spec.ts | 19 ++ packages/llm/llm-pi-ai/tests/config.spec.ts | 23 ++ packages/llm/llm-pi-ai/tests/context.spec.ts | 10 + packages/llm/llm/src/content.ts | 4 +- packages/llm/llm/tests/content.spec.ts | 68 ++++- packages/llm/llm/tests/service.spec.ts | 12 + 17 files changed, 526 insertions(+), 39 deletions(-) diff --git a/packages/attachment/attachment-local/src/index.ts b/packages/attachment/attachment-local/src/index.ts index 516280548c..4fb200345b 100644 --- a/packages/attachment/attachment-local/src/index.ts +++ b/packages/attachment/attachment-local/src/index.ts @@ -93,7 +93,11 @@ class SharedRequest { wait(signal?: AbortSignal): Promise { signal?.throwIfAborted() this.waiters += 1 - if (signal === undefined) return this.promise.finally(() => this.release(false)) + if (signal === undefined) { + return this.promise.finally(() => { + this.release(false) + }) + } let released = false const release = (cancelled: boolean): void => { if (released) return @@ -113,6 +117,8 @@ class SharedRequest { }, (error: unknown) => { signal.removeEventListener('abort', abort) release(false) + // CompressionLimiter normalizes task rejections before this handler. + // oxlint-disable-next-line typescript/prefer-promise-reject-errors reject(error) }) }) diff --git a/packages/attachment/attachment-local/tests/encoding.spec.ts b/packages/attachment/attachment-local/tests/encoding.spec.ts index d75fc9b4b3..8cd7a60540 100644 --- a/packages/attachment/attachment-local/tests/encoding.spec.ts +++ b/packages/attachment/attachment-local/tests/encoding.spec.ts @@ -83,6 +83,7 @@ describe('CompressionLimiter', () => { it('normalizes a non-Error rejection and releases its slot', async () => { const limiter = new CompressionLimiter(1) + // oxlint-disable-next-line typescript/prefer-promise-reject-errors -- Native bindings can reject non-Error values. const failed = limiter.run(() => Promise.reject('native failure')) const next = limiter.run(() => Promise.resolve('next')) diff --git a/packages/attachment/attachment-local/tests/request-image.spec.ts b/packages/attachment/attachment-local/tests/request-image.spec.ts index e33522c104..726837a242 100644 --- a/packages/attachment/attachment-local/tests/request-image.spec.ts +++ b/packages/attachment/attachment-local/tests/request-image.spec.ts @@ -340,7 +340,9 @@ describe('local request-image cache', () => { const read = vi.spyOn(attachments, 'readImage').mockImplementation((_ref, signal) => { readSignal = signal return new Promise((_resolve, reject) => { - signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + signal?.addEventListener('abort', () => { + reject(new Error('request transform aborted', { cause: signal.reason })) + }, { once: true }) }) }) const controller = new AbortController() @@ -349,7 +351,9 @@ describe('local request-image cache', () => { { maxPixels: 640_000, maxBytes: 1024 * 1024 }, controller.signal, ) - await vi.waitFor(() => expect(read).toHaveBeenCalledTimes(1)) + await vi.waitFor(() => { + expect(read).toHaveBeenCalledTimes(1) + }) const reason = new Error('cancel only transform waiter') controller.abort(reason) @@ -369,7 +373,9 @@ describe('local request-image cache', () => { calls += 1 if (calls === 1) { return new Promise((_resolve, reject) => { - signal?.addEventListener('abort', () => reject(signal.reason), { once: true }) + signal?.addEventListener('abort', () => { + reject(new Error('request transform aborted', { cause: signal.reason })) + }, { once: true }) }) } return actualRead(ref, signal) @@ -377,7 +383,9 @@ describe('local request-image cache', () => { const controller = new AbortController() const policy = { maxPixels: 640_000, maxBytes: 1024 * 1024 } const cancelled = attachments.readImageRequest(master, policy, controller.signal) - await vi.waitFor(() => expect(calls).toBe(1)) + await vi.waitFor(() => { + expect(calls).toBe(1) + }) controller.abort('cancelled') const replacement = attachments.readImageRequest(master, policy) @@ -389,4 +397,5 @@ describe('local request-image cache', () => { await expect(replacement).resolves.toMatchObject({ width: 1130, height: 565 }) expect(calls).toBe(2) }) + }) diff --git a/packages/fs/tool-fs/tests/read-image.spec.ts b/packages/fs/tool-fs/tests/read-image.spec.ts index 3bf86d27f5..03616911b5 100644 --- a/packages/fs/tool-fs/tests/read-image.spec.ts +++ b/packages/fs/tool-fs/tests/read-image.spec.ts @@ -251,6 +251,33 @@ describe('read_image_region', () => { expect(result.isError).toBe(false) }) + it('continues across an earlier session message without the requested image', async () => { + const ctx = await setup() + const source = await ctx.attachments.saveImage({ data: PNG_3X3, mediaType: 'image/png' }) + const history = [ + createUserMessage({ + content: [{ type: 'text', text: 'before image' }], + source: { kind: 'plugin', plugin: 'test' }, + }), + createUserMessage({ + content: [{ type: 'image', attachment: source.ref }], + source: { kind: 'plugin', plugin: 'test' }, + }), + ] + + const result = await call(ctx, 'read_image_region', { + attachment_id: source.ref.attachmentId, + preview_width: 3, + preview_height: 3, + x: 0, + y: 0, + width: 1, + height: 1, + }, agentOn('vision-model', 'visual', history)) + + expect(result.isError).toBe(false) + }) + it('rejects a missing session, empty id, and invalid coordinate arguments', async () => { const ctx = await setup() const base = { diff --git a/packages/interaction/commands/tests/commands.spec.ts b/packages/interaction/commands/tests/commands.spec.ts index c65fb49bed..85806a1d36 100644 --- a/packages/interaction/commands/tests/commands.spec.ts +++ b/packages/interaction/commands/tests/commands.spec.ts @@ -487,9 +487,10 @@ describe('image attachments', () => { }) }), validateImageBatch(inputs: readonly unknown[]) { - return (AttachmentStore.prototype as unknown as { + const validate = AttachmentStore.prototype as unknown as { validateImageBatch(this: unknown, batch: readonly unknown[]): void - }).validateImageBatch.call(this, inputs) + } + validate.validateImageBatch.call(this, inputs) }, // The real base-class batch method over this double's limits and members. saveImages(inputs: readonly unknown[]) { diff --git a/packages/llm/llm-deepseek/src/file-store.ts b/packages/llm/llm-deepseek/src/file-store.ts index ddaefd1eee..0757b42db2 100644 --- a/packages/llm/llm-deepseek/src/file-store.ts +++ b/packages/llm/llm-deepseek/src/file-store.ts @@ -50,33 +50,44 @@ function abortReason(signal: AbortSignal): Error { : new Error('DeepSeek file upload cancelled with a non-Error reason.', { cause: reason }) } +function uploadFailure(error: unknown): Error { + return error instanceof Error + ? error + : new Error('DeepSeek file upload failed with a non-Error reason.', { cause: error }) +} + function waitForUpload(operation: SharedUpload, signal: AbortSignal | undefined): Promise { signal?.throwIfAborted() operation.waiters += 1 let released = false - const release = (cancelled: boolean): void => { + const release = (cancelledReason?: Error): void => { if (released) return released = true operation.waiters -= 1 - if (cancelled && operation.waiters === 0 && !operation.settled) { - operation.controller.abort(signal === undefined ? undefined : abortReason(signal)) + if (cancelledReason !== undefined && operation.waiters === 0 && !operation.settled) { + operation.controller.abort(cancelledReason) } } - if (signal === undefined) return operation.promise.finally(() => release(false)) + if (signal === undefined) { + return operation.promise.finally(() => { + release() + }) + } return new Promise((resolve, reject) => { const abort = (): void => { - release(true) - reject(abortReason(signal)) + const reason = abortReason(signal) + release(reason) + reject(reason) } signal.addEventListener('abort', abort, { once: true }) void operation.promise.then((value) => { signal.removeEventListener('abort', abort) - release(false) + release() resolve(value) }, (error: unknown) => { signal.removeEventListener('abort', abort) - release(false) - reject(error) + release() + reject(uploadFailure(error)) }) }) } @@ -155,7 +166,7 @@ export class DeepSeekFileStore { return value }, (error: unknown) => { shared.settled = true - throw error + throw uploadFailure(error) }) this.inflight.set(key, shared) void shared.promise.finally(() => { @@ -168,7 +179,7 @@ export class DeepSeekFileStore { version: RequestImageAttachment, connection: DeepSeekFileConnection, policy: DeepSeekFilePolicy, - signal?: AbortSignal, + signal: AbortSignal, ): Promise { if (version.bytes > MAX_CHAT_IMAGE_BYTES) { throw new LlmError('DeepSeek chat image exceeds the 32 MiB per-image limit.', 'INVALID_REQUEST') @@ -186,9 +197,9 @@ export class DeepSeekFileStore { mediaType: version.mediaType, filename: filename(version), expiresAfterSeconds: policy.expiresAfterSeconds, - ...signal === undefined ? {} : { signal }, + signal, }) - if (remote.bytes !== version.data.byteLength || remote.expiresAt === undefined) { + if (remote.bytes !== version.data.byteLength) { throw new LlmError('DeepSeek Files API upload response does not match the submitted image.', 'INVALID_RESPONSE') } return { diff --git a/packages/llm/llm-deepseek/src/files-api.ts b/packages/llm/llm-deepseek/src/files-api.ts index f19823100e..cc998b2e7e 100644 --- a/packages/llm/llm-deepseek/src/files-api.ts +++ b/packages/llm/llm-deepseek/src/files-api.ts @@ -143,7 +143,6 @@ export class DeepSeekFilesClient { let response: Response try { const headers = new Headers(attributionHeaders()) - for (const [name, value] of new Headers(init.headers)) headers.set(name, value) headers.set('authorization', `Bearer ${this.apiKey}`) response = await this.fetchImpl(`${this.baseURL}${path}`, { ...init, @@ -180,7 +179,7 @@ export class DeepSeekFilesClient { filename: string expiresAfterSeconds: number signal?: AbortSignal - }): Promise { + }): Promise { if (input.data.byteLength > MAX_FILE_UPLOAD_BYTES) { throw new LlmError('DeepSeek Files API upload exceeds 128 MiB.', 'INVALID_REQUEST') } @@ -197,7 +196,7 @@ export class DeepSeekFilesClient { const response = await this.request('/files', { method: 'POST', body: form }, input.signal) const file = parseFileObject(await response.json(), 'upload') if (file.expiresAt === undefined) throw invalidResponse('upload') - return file + return { ...file, expiresAt: file.expiresAt } } /** diff --git a/packages/llm/llm-deepseek/src/serialize.ts b/packages/llm/llm-deepseek/src/serialize.ts index a65c9750c3..b998b23a8b 100644 --- a/packages/llm/llm-deepseek/src/serialize.ts +++ b/packages/llm/llm-deepseek/src/serialize.ts @@ -317,7 +317,7 @@ export async function serializeMessagesWithImages( wire.push({ role: 'tool', tool_call_id: result.toolCallId, - content: text || (fileParts.length > 0 ? '(see attached image)' : '(no output)'), + content: text || '(no output)', }) pendingToolImages.push(...fileParts) } diff --git a/packages/llm/llm-deepseek/tests/adapter.spec.ts b/packages/llm/llm-deepseek/tests/adapter.spec.ts index 731df2eb0f..2663c4cd78 100644 --- a/packages/llm/llm-deepseek/tests/adapter.spec.ts +++ b/packages/llm/llm-deepseek/tests/adapter.spec.ts @@ -176,6 +176,7 @@ describe('DeepSeekAdapter against a mock server', () => { await drain(adapter.stream({ provider: 'deepseek-official', model: 'deepseek-v4-flash-vision-exp', + tools: [{ name: 'read_image_region', description: 'crop', parameters: { type: 'object' } }], messages: [createUserMessage({ content: [ { type: 'text', text: 'describe ' }, @@ -191,7 +192,7 @@ describe('DeepSeekAdapter against a mock server', () => { role: 'user', content: [ { type: 'text', text: 'describe ' }, - { type: 'text', text: expect.stringContaining(`Image ${imageRef.attachmentId}`) as string }, + { type: 'text', text: expect.stringContaining('Call read_image_region') as string }, { type: 'file', file_id: 'file-api-1' }, ], }], @@ -1499,6 +1500,23 @@ describe('plugin registration and config', () => { .toThrow(/maxTokens must be a positive integer/) }) + it('rejects image request limits on a text-only catalog model', () => { + expect(() => resolveAdapterOptions({ + models: [{ id: 'text-only', inputModalities: ['text'], imagePixelBudget: 1 }], + })).toThrow(/text-only catalog model .* cannot declare image request limits/) + }) + + it.each([ + ['imagePixelBudget', 0, /imagePixelBudget must be a positive safe integer/], + ['imagePixelBudget', Number.MAX_SAFE_INTEGER + 1, /imagePixelBudget must be a positive safe integer/], + ['imageMaxBytes', 0, /imageMaxBytes must be a positive safe integer/], + ['imageMaxBytes', 1.5, /imageMaxBytes must be a positive safe integer/], + ] as const)('rejects per-model %s=%s', (field, value, message) => { + expect(() => resolveAdapterOptions({ + models: [{ id: 'vision', inputModalities: ['image'], [field]: value }], + })).toThrow(message) + }) + it('prefers a model\'s own output cap over the profile default', async () => { // The profile default stays what an unlisted or uncapped model resolves // to, so adding a per-model cap changes one model rather than the route. @@ -1569,6 +1587,23 @@ describe('plugin registration and config', () => { })).toThrow(/imageOffloadCountQuantum must not exceed maxImagesPerRequest/) }) + it.each([ + ['maxImagesPerRequest', 0, /maxImagesPerRequest must be a positive safe integer/], + ['maxImagesPerRequest', 1.5, /maxImagesPerRequest must be a positive safe integer/], + ['imageOffloadByteQuantum', 0, /imageOffloadByteQuantum must be a positive safe integer/], + ['imageOffloadByteQuantum', Number.MAX_SAFE_INTEGER + 1, /imageOffloadByteQuantum must be a positive safe integer/], + ['imageOffloadCountQuantum', 0, /imageOffloadCountQuantum must be a positive safe integer/], + ['imageOffloadCountQuantum', 1.5, /imageOffloadCountQuantum must be a positive safe integer/], + ['fileExpiresAfterSeconds', 3_599, /fileExpiresAfterSeconds must be an integer from 3600 through 2592000/], + ['fileExpiresAfterSeconds', 2_592_001, /fileExpiresAfterSeconds must be an integer from 3600 through 2592000/], + ['fileRefreshMarginSeconds', -1, /fileRefreshMarginSeconds must be a non-negative integer/], + ['fileRefreshMarginSeconds', 604_800, /fileRefreshMarginSeconds must be a non-negative integer/], + ['fileQuotaCleanupBatch', 0, /fileQuotaCleanupBatch must be an integer from 1 through 1000/], + ['fileQuotaCleanupBatch', 1_001, /fileQuotaCleanupBatch must be an integer from 1 through 1000/], + ] as const)('rejects %s=%s', (field, value, message) => { + expect(() => resolveAdapterOptions({ [field]: value })).toThrow(message) + }) + it.each([0, 1.5, Number.MAX_SAFE_INTEGER + 1])( 'rejects invalid request file bound %s', async (maxRequestFilesBytes) => { diff --git a/packages/llm/llm-deepseek/tests/file-store.spec.ts b/packages/llm/llm-deepseek/tests/file-store.spec.ts index 6d9e940786..069ff47c9d 100644 --- a/packages/llm/llm-deepseek/tests/file-store.spec.ts +++ b/packages/llm/llm-deepseek/tests/file-store.spec.ts @@ -4,8 +4,9 @@ import { join } from 'node:path' import { describe, expect, it, vi } from 'vitest' import { AttachmentId, ImageVariantId } from '@deepseek-ai/dsh-attachment' import type { ImageAttachmentRef, RequestImageAttachment } from '@deepseek-ai/dsh-attachment' -import { DeepSeekFileStore } from '../src/file-store.ts' -import { DeepSeekUploadIndex } from '../src/upload-index.ts' +import { DeepSeekFileStore, MAX_CHAT_IMAGE_BYTES } from '../src/file-store.ts' +import { DeepSeekFileId } from '../src/file-id.ts' +import { deepSeekFileScope, DeepSeekUploadIndex } from '../src/upload-index.ts' const REF: ImageAttachmentRef = { attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`), @@ -90,7 +91,9 @@ describe('DeepSeekFileStore', () => { uploadSignal = init?.signal ?? undefined return new Promise((resolve, reject) => { complete = resolve - uploadSignal?.addEventListener('abort', () => reject(uploadSignal?.reason), { once: true }) + uploadSignal?.addEventListener('abort', () => { + reject(new Error('upload aborted', { cause: uploadSignal?.reason })) + }, { once: true }) }) }) as typeof fetch const store = new DeepSeekFileStore({ index, now: () => NOW, fetch: fetchImpl }) @@ -98,7 +101,9 @@ describe('DeepSeekFileStore', () => { const cancelled = store.ensureUploaded(VERSION, CONNECTION, POLICY, controller.signal) const completed = store.ensureUploaded(VERSION, CONNECTION, POLICY) - await vi.waitFor(() => expect(fetchImpl).toHaveBeenCalledTimes(1)) + await vi.waitFor(() => { + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) const reason = new Error('cancel one upload waiter') controller.abort(reason) @@ -123,13 +128,17 @@ describe('DeepSeekFileStore', () => { const fetchImpl = vi.fn((_url: string | URL | Request, init?: RequestInit) => { uploadSignal = init?.signal ?? undefined return new Promise((_resolve, reject) => { - uploadSignal?.addEventListener('abort', () => reject(uploadSignal?.reason), { once: true }) + uploadSignal?.addEventListener('abort', () => { + reject(new Error('upload aborted', { cause: uploadSignal?.reason })) + }, { once: true }) }) }) as typeof fetch const store = new DeepSeekFileStore({ index, now: () => NOW, fetch: fetchImpl }) const controller = new AbortController() const upload = store.ensureUploaded(VERSION, CONNECTION, POLICY, controller.signal) - await vi.waitFor(() => expect(fetchImpl).toHaveBeenCalledTimes(1)) + await vi.waitFor(() => { + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) const reason = new Error('cancel only upload waiter') controller.abort(reason) @@ -138,6 +147,79 @@ describe('DeepSeekFileStore', () => { expect(uploadSignal?.reason).toBe(reason) }) + it('normalizes a non-Error cancellation reason', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + const fetchImpl = vi.fn((_url: string | URL | Request, init?: RequestInit) => ( + new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => { + reject(new Error('upload aborted', { cause: init.signal?.reason })) + }, { once: true }) + }) + )) as typeof fetch + const store = new DeepSeekFileStore({ + index: new DeepSeekUploadIndex(join(dir, 'index.json')), + now: () => NOW, + fetch: fetchImpl, + }) + const controller = new AbortController() + const upload = store.ensureUploaded(VERSION, CONNECTION, POLICY, controller.signal) + await vi.waitFor(() => { + expect(fetchImpl).toHaveBeenCalledOnce() + }) + controller.abort('cancelled') + + await expect(upload).rejects.toMatchObject({ + message: 'DeepSeek file upload cancelled with a non-Error reason.', + cause: 'cancelled', + }) + }) + + it('starts a fresh upload while the cancelled transport is settling', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + let requests = 0 + const fetchImpl = vi.fn((_url: string | URL | Request, init?: RequestInit) => { + requests += 1 + if (requests === 1) { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => { + queueMicrotask(() => { + reject(new Error('upload aborted', { cause: init.signal?.reason })) + }) + }, { once: true }) + }) + } + return Promise.resolve(new Response(JSON.stringify({ + id: 'file-api-retry', object: 'file', bytes: 3, created_at: NOW / 1_000, + filename: 'dsh-retry.png', purpose: 'user_data', + expires_at: NOW / 1_000 + POLICY.expiresAfterSeconds, + }), { status: 200 })) + }) as typeof fetch + const store = new DeepSeekFileStore({ + index: new DeepSeekUploadIndex(join(dir, 'index.json')), + now: () => NOW, + fetch: fetchImpl, + }) + const controller = new AbortController() + const cancelled = store.ensureUploaded(VERSION, CONNECTION, POLICY, controller.signal) + await vi.waitFor(() => { + expect(fetchImpl).toHaveBeenCalledOnce() + }) + controller.abort(new Error('cancel first')) + const retried = store.ensureUploaded(VERSION, CONNECTION, POLICY) + + await expect(cancelled).rejects.toThrow('cancel first') + await expect(retried).resolves.toMatchObject({ record: { fileId: 'file-api-retry' } }) + }) + + it('rejects a request version above the chat per-image limit before transport', async () => { + const fetchImpl = vi.fn() as typeof fetch + const store = new DeepSeekFileStore({ now: () => NOW, fetch: fetchImpl }) + const oversized = { ...VERSION, bytes: MAX_CHAT_IMAGE_BYTES + 1 } + await expect(store.ensureUploaded(oversized, CONNECTION, POLICY)) + .rejects.toMatchObject({ code: 'INVALID_REQUEST' }) + expect(fetchImpl).not.toHaveBeenCalled() + }) + it('does not persist an upload whose response is missing and retries on the next request', async () => { const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) const index = new DeepSeekUploadIndex(join(dir, 'index.json')) @@ -158,6 +240,55 @@ describe('DeepSeekFileStore', () => { .resolves.toMatchObject({ record: { fileId: 'file-api-1' }, uploaded: true }) }) + it('rejects an upload response whose byte count differs from the request version', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + const fetchImpl = vi.fn(() => Promise.resolve(new Response(JSON.stringify({ + id: 'file-api-wrong-size', object: 'file', bytes: 2, created_at: NOW / 1_000, + filename: 'dsh-wrong.png', purpose: 'user_data', + expires_at: NOW / 1_000 + POLICY.expiresAfterSeconds, + }), { status: 200 }))) as typeof fetch + const store = new DeepSeekFileStore({ + index: new DeepSeekUploadIndex(join(dir, 'index.json')), + now: () => NOW, + fetch: fetchImpl, + }) + await expect(store.ensureUploaded(VERSION, CONNECTION, POLICY)) + .rejects.toMatchObject({ code: 'INVALID_RESPONSE' }) + }) + + it.each([ + ['image/jpeg', 'jpeg'], + ['image/webp', 'webp'], + ['image/gif', 'gif'], + ] as const)('uses the %s filename extension for uploads', async (mediaType, extension) => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + const remote = uploadFetch() + const store = new DeepSeekFileStore({ + index: new DeepSeekUploadIndex(join(dir, `${extension}.json`)), + now: () => NOW, + fetch: remote.fetchImpl, + }) + await store.ensureUploaded({ ...VERSION, mediaType }, CONNECTION, POLICY) + const form = vi.mocked(remote.fetchImpl).mock.calls[0]?.[1]?.body + expect(form).toBeInstanceOf(FormData) + const file = (form as FormData).get('file') + expect(file).toBeInstanceOf(File) + if (!(file instanceof File)) throw new Error('expected multipart file') + expect(file.name).toMatch(new RegExp(`\\.${extension}$`, 'u')) + }) + + it('normalizes a non-Error failure from the durable upload index', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + const index = new DeepSeekUploadIndex(join(dir, 'index.json')) + vi.spyOn(index, 'get').mockRejectedValue('index unavailable') + const store = new DeepSeekFileStore({ index, now: () => NOW, fetch: vi.fn() as typeof fetch }) + + await expect(store.ensureUploaded(VERSION, CONNECTION, POLICY)).rejects.toMatchObject({ + message: 'DeepSeek file upload failed with a non-Error reason.', + cause: 'index unavailable', + }) + }) + it('reuses local expires_at above the refresh margin and uploads again at the margin', async () => { const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) const index = new DeepSeekUploadIndex(join(dir, 'index.json')) @@ -190,6 +321,101 @@ describe('DeepSeekFileStore', () => { expect(remote.fetchImpl).toHaveBeenCalledTimes(2) }) + it('removes a losing upload and keeps the winning durable mapping when duplicate cleanup fails', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + const index = new DeepSeekUploadIndex(join(dir, 'index.json')) + vi.spyOn(index, 'commit').mockResolvedValue({ + accepted: false, + record: { + scope: deepSeekFileScope(CONNECTION.baseURL, CONNECTION.apiKey), + masterAttachmentId: VERSION.master.attachmentId, + variantId: VERSION.variantId, + fileId: DeepSeekFileId('file-api-winner'), + bytes: 3, + createdAt: NOW, + expiresAt: NOW + POLICY.expiresAfterSeconds * 1_000, + }, + }) + const remote = uploadFetch() + const fetchImpl = vi.fn((url: string | URL | Request, init?: RequestInit) => { + if (init?.method === 'DELETE') return Promise.resolve(new Response('failed', { status: 500 })) + return remote.fetchImpl(url, init) + }) as typeof fetch + const store = new DeepSeekFileStore({ index, now: () => NOW, fetch: fetchImpl }) + + await expect(store.ensureUploaded(VERSION, CONNECTION, POLICY)).resolves.toMatchObject({ + record: { fileId: 'file-api-winner' }, + uploaded: false, + }) + expect(fetchImpl).toHaveBeenCalledTimes(2) + }) + + it('reclaims one owned file after quota rejection and retries the upload once', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + let uploads = 0 + const fetchImpl = vi.fn((input: string | URL | Request, init?: RequestInit) => { + if (init?.method === 'POST') { + uploads += 1 + if (uploads === 1) return Promise.resolve(new Response(JSON.stringify({ + error: { message: 'stored file quota exceeded', code: 'file_quota' }, + }), { status: 400 })) + return Promise.resolve(new Response(JSON.stringify({ + id: 'file-api-recovered', object: 'file', bytes: 3, created_at: NOW / 1_000, + filename: 'dsh-recovered.png', purpose: 'user_data', + expires_at: NOW / 1_000 + POLICY.expiresAfterSeconds, + }), { status: 200 })) + } + if (init?.method === 'DELETE') { + return Promise.resolve(new Response(JSON.stringify({ + id: 'file-api-old', object: 'file', deleted: true, + }), { status: 200 })) + } + expect(new URL(requestUrl(input)).pathname).toBe('/files') + return Promise.resolve(new Response(JSON.stringify({ + object: 'list', + data: [{ + id: 'file-api-old', object: 'file', bytes: 3, created_at: NOW / 1_000, + filename: 'dsh-old.png', purpose: 'user_data', + }], + first_id: 'file-api-old', last_id: 'file-api-old', has_more: false, + }), { status: 200 })) + }) as typeof fetch + const store = new DeepSeekFileStore({ + index: new DeepSeekUploadIndex(join(dir, 'index.json')), + now: () => NOW, + fetch: fetchImpl, + }) + + await expect(store.ensureUploaded(VERSION, CONNECTION, POLICY)).resolves.toMatchObject({ + record: { fileId: 'file-api-recovered' }, uploaded: true, + }) + expect(uploads).toBe(2) + }) + + it('preserves a quota error when no harness-owned file can be reclaimed', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + const fetchImpl = vi.fn((_input: string | URL | Request, init?: RequestInit) => { + if (init?.method === 'POST') return Promise.resolve(new Response(JSON.stringify({ + error: { message: 'file count quota exceeded', code: 'file_quota' }, + }), { status: 400 })) + return Promise.resolve(new Response(JSON.stringify({ + object: 'list', + data: [{ + id: 'file-api-foreign', object: 'file', bytes: 3, created_at: NOW / 1_000, + filename: 'foreign.png', purpose: 'user_data', + }], + has_more: false, + }), { status: 200 })) + }) as typeof fetch + const store = new DeepSeekFileStore({ + index: new DeepSeekUploadIndex(join(dir, 'index.json')), + now: () => NOW, + fetch: fetchImpl, + }) + + await expect(store.ensureUploaded(VERSION, CONNECTION, POLICY)).rejects.toMatchObject({ code: 'FILES_API' }) + }) + it('finishes pagination before deleting cursor files during quota recovery', async () => { const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) const deleted = new Set() @@ -227,4 +453,44 @@ describe('DeepSeekFileStore', () => { await expect(store.reclaimOldestOwned(CONNECTION, 2)).resolves.toBe(2) expect([...deleted]).toEqual(['file-api-oldest', 'file-api-next']) }) + + it('stops pagination when a page omits or repeats its cursor', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + for (const mode of ['missing', 'repeated'] as const) { + let page = 0 + const fetchImpl = vi.fn((input: string | URL | Request, init?: RequestInit) => { + if (init?.method === 'DELETE') { + const id = requestUrl(input).split('/').at(-1) + return Promise.resolve(new Response(JSON.stringify({ id, object: 'file', deleted: true }), { status: 200 })) + } + page += 1 + const lastId = mode === 'missing' ? undefined : 'file-api-same' + return Promise.resolve(new Response(JSON.stringify({ + object: 'list', data: [], has_more: true, + ...lastId === undefined ? {} : { last_id: lastId }, + }), { status: 200 })) + }) as typeof fetch + const store = new DeepSeekFileStore({ + index: new DeepSeekUploadIndex(join(dir, `${mode}.json`)), + now: () => NOW, + fetch: fetchImpl, + }) + await expect(store.reclaimOldestOwned(CONNECTION, 1)).resolves.toBe(0) + expect(page).toBe(mode === 'missing' ? 1 : 2) + } + }) + + it('releases every batch and clears the scoped upload index', async () => { + const dir = await mkdtemp(join(tmpdir(), 'dsh-file-store-')) + const index = new DeepSeekUploadIndex(join(dir, 'index.json')) + const store = new DeepSeekFileStore({ index, now: () => NOW, fetch: vi.fn() as typeof fetch }) + const reclaim = vi.spyOn(store, 'reclaimOldestOwned') + .mockResolvedValueOnce(1_000) + .mockResolvedValueOnce(2) + const clear = vi.spyOn(index, 'clear') + + await expect(store.releaseAll(CONNECTION)).resolves.toBe(1_002) + expect(reclaim).toHaveBeenCalledTimes(2) + expect(clear).toHaveBeenCalledOnce() + }) }) diff --git a/packages/llm/llm-deepseek/tests/files-api.spec.ts b/packages/llm/llm-deepseek/tests/files-api.spec.ts index 466c9be83b..e6a1aa59ea 100644 --- a/packages/llm/llm-deepseek/tests/files-api.spec.ts +++ b/packages/llm/llm-deepseek/tests/files-api.spec.ts @@ -119,7 +119,7 @@ describe('DeepSeekFilesClient', () => { const client = new DeepSeekFilesClient({ baseURL: 'https://api.deepseek.com', apiKey: 'key', - fetch: vi.fn(() => Promise.resolve(new Response('not-json', { status }))) as typeof fetch, + fetch: vi.fn(() => Promise.resolve(new Response('not-json', { status }))), }) await expect(client.retrieve(DeepSeekFileId('missing'))).rejects.toMatchObject({ name: 'DeepSeekFilesError', @@ -139,7 +139,7 @@ describe('DeepSeekFilesClient', () => { const client = new DeepSeekFilesClient({ baseURL: 'https://api.deepseek.com', apiKey: 'key', - fetch: vi.fn(() => Promise.resolve(new Response(JSON.stringify(body), { status: 400 }))) as typeof fetch, + fetch: vi.fn(() => Promise.resolve(new Response(JSON.stringify(body), { status: 400 }))), }) const error = await client.retrieve(DeepSeekFileId('missing')).catch((caught: unknown) => caught) expect(error).toBeInstanceOf(DeepSeekFilesError) @@ -151,7 +151,7 @@ describe('DeepSeekFilesClient', () => { const client = new DeepSeekFilesClient({ baseURL: 'https://api.deepseek.com', apiKey: 'key', - fetch: vi.fn(() => Promise.reject(transport)) as typeof fetch, + fetch: vi.fn(() => Promise.reject(transport)), }) await expect(client.retrieve(DeepSeekFileId('one'))).rejects.toMatchObject({ code: 'TRANSPORT', @@ -183,7 +183,7 @@ describe('DeepSeekFilesClient', () => { const client = new DeepSeekFilesClient({ baseURL: 'https://api.deepseek.com', apiKey: 'key', - fetch: vi.fn(() => Promise.resolve(new Response(JSON.stringify(body), { status: 200 }))) as typeof fetch, + fetch: vi.fn(() => Promise.resolve(new Response(JSON.stringify(body), { status: 200 }))), }) await expect(client.retrieve(DeepSeekFileId('one'))).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }) }) @@ -224,7 +224,7 @@ describe('DeepSeekFilesClient', () => { const client = new DeepSeekFilesClient({ baseURL: 'https://api.deepseek.com', apiKey: 'key', - fetch: vi.fn(() => Promise.resolve(new Response(JSON.stringify(body), { status: 200 }))) as typeof fetch, + fetch: vi.fn(() => Promise.resolve(new Response(JSON.stringify(body), { status: 200 }))), }) await expect(client.list()).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }) }) @@ -253,7 +253,7 @@ describe('DeepSeekFilesClient', () => { const client = new DeepSeekFilesClient({ baseURL: 'https://api.deepseek.com', apiKey: 'key', - fetch: vi.fn(() => Promise.resolve(new Response(JSON.stringify(body), { status: 200 }))) as typeof fetch, + fetch: vi.fn(() => Promise.resolve(new Response(JSON.stringify(body), { status: 200 }))), }) await expect(client.delete(DeepSeekFileId('file-api-one'))).rejects.toMatchObject({ code: 'INVALID_RESPONSE' }) }) diff --git a/packages/llm/llm-deepseek/tests/serialize.spec.ts b/packages/llm/llm-deepseek/tests/serialize.spec.ts index 8e04b9b3b0..06af757c9a 100644 --- a/packages/llm/llm-deepseek/tests/serialize.spec.ts +++ b/packages/llm/llm-deepseek/tests/serialize.spec.ts @@ -399,6 +399,14 @@ describe('image serialization', () => { }) }) + it('rejects an image whose prepared request version is absent', async () => { + const ref = imageRef() + await expect(serializeMessagesWithImages([createUserMessage({ + content: [{ type: 'image', attachment: ref }], + source: { kind: 'plugin', plugin: 'test' }, + })], imageOptions([]))).rejects.toMatchObject({ code: 'INVALID_REQUEST' }) + }) + it('keeps tool content textual and groups consecutive tool-result images afterward', async () => { const messages = [ createUserMessage({ @@ -561,6 +569,17 @@ describe('image serialization', () => { expect(resolveFileId.mock.calls[0]?.[0]).toMatchObject({ master: { mediaType: 'image/jpeg' } }) }) + it('rejects an unprepared image while computing exact request bytes', async () => { + const ref = imageRef() + await expect(serializeRequestWithImages(request({ + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [{ type: 'image', attachment: ref }], + source: { kind: 'plugin', plugin: 'test' }, + })], + }), imageOptions([]))).rejects.toMatchObject({ code: 'INVALID_REQUEST' }) + }) + it.each(['system', 'assistant'] as const)('rejects an image in %s history before reading attachments', async (role) => { const resolveFileId = vi.fn() await expect(serializeMessagesWithImages([createMessage({ diff --git a/packages/llm/llm-pi-ai/tests/config.spec.ts b/packages/llm/llm-pi-ai/tests/config.spec.ts index 55444228de..61a2535a9d 100644 --- a/packages/llm/llm-pi-ai/tests/config.spec.ts +++ b/packages/llm/llm-pi-ai/tests/config.spec.ts @@ -64,3 +64,26 @@ describe('modality schema boundary', () => { expect(absent.providers['acme-gateway']?.defaultInput).toEqual(['text']) }) }) + +describe('request image policy bounds', () => { + it.each([ + ['requestImagePixelBudget', 0, /requestImagePixelBudget must be a positive safe integer/], + ['requestImagePixelBudget', Number.MAX_SAFE_INTEGER + 1, /requestImagePixelBudget must be a positive safe integer/], + ['requestImageMaxBytes', 0, /requestImageMaxBytes must be a positive safe integer/], + ['requestImageMaxBytes', 1.5, /requestImageMaxBytes must be a positive safe integer/], + ] as const)('rejects %s=%s at service resolution', (field, value, message) => { + const programmatic = { + providers: { + 'acme-gateway': { + api: 'openai-completions', + baseURL: 'https://acme.test', + models: [{ id: 'm' }], + [field]: value, + }, + }, + } as unknown as Config + expect(() => { + assertServiceable(programmatic) + }).toThrow(message) + }) +}) diff --git a/packages/llm/llm-pi-ai/tests/context.spec.ts b/packages/llm/llm-pi-ai/tests/context.spec.ts index a0fb671c95..8a3f7bd084 100644 --- a/packages/llm/llm-pi-ai/tests/context.spec.ts +++ b/packages/llm/llm-pi-ai/tests/context.spec.ts @@ -428,4 +428,14 @@ describe('pi-ai request context conversion', () => { history('assistant', [{ type: 'image', attachment: ref }]), )).toThrow(/assistant image output/) }) + + it('rejects an attachment service that omits a requested image version', async () => { + const store = { + readImageRequests: vi.fn(() => Promise.resolve([])), + } as unknown as AttachmentStore + await expect(toPiContext( + request([user([{ type: 'image', attachment: ref }])]), + store, + )).rejects.toMatchObject({ code: 'INVALID_REQUEST' }) + }) }) diff --git a/packages/llm/llm/src/content.ts b/packages/llm/llm/src/content.ts index 73a2aee889..72e452e005 100644 --- a/packages/llm/llm/src/content.ts +++ b/packages/llm/llm/src/content.ts @@ -74,7 +74,9 @@ function collectImageLengths( ): void { for (const block of blocks) { if (block.type === 'image') { - const bytes = policy.byteLength?.(block.attachment) ?? block.attachment.bytes + const bytes = policy.byteLength === undefined + ? block.attachment.bytes + : policy.byteLength(block.attachment) lengths.push(policy.representation === 'base64' ? base64Length(bytes) : bytes) } else if (block.type === 'tool-result') { collectImageLengths(block.content, lengths, policy) diff --git a/packages/llm/llm/tests/content.spec.ts b/packages/llm/llm/tests/content.spec.ts index d1b02fa011..6a0eb02c63 100644 --- a/packages/llm/llm/tests/content.spec.ts +++ b/packages/llm/llm/tests/content.spec.ts @@ -1,6 +1,13 @@ import { describe, expect, it } from 'vitest' import { AttachmentId } from '@deepseek-ai/dsh-attachment' -import { CallId, createUserMessage, OFFLOADED_IMAGE_TEXT, offloadRequestImages, offloadRequestImagesWithPolicy } from '../src/index.ts' +import { + CallId, + createUserMessage, + OFFLOADED_IMAGE_TEXT, + offloadRequestImages, + offloadRequestImagesWithPolicy, + projectImagesForTextModel, +} from '../src/index.ts' import type { ContentBlock } from '../src/index.ts' const source = { kind: 'plugin' as const, plugin: 'test' } @@ -19,6 +26,11 @@ function image(bytes: number): ContentBlock { } describe('offloadRequestImages', () => { + it('preserves every image when no payload bound is configured', () => { + const messages = [createUserMessage({ content: [image(300)], source })] + expect(offloadRequestImages(messages, undefined)).toBe(messages) + }) + it('preserves the original request when its base64 payload fits exactly', () => { const messages = [createUserMessage({ content: [image(3), image(3)], source })] expect(offloadRequestImages(messages, 8)).toBe(messages) @@ -116,4 +128,58 @@ describe('offloadRequestImagesWithPolicy', () => { expect(projected[0]?.content.filter(block => block.type === 'text')).toHaveLength(20) expect(projected[0]?.content.filter(block => block.type === 'image')).toHaveLength(581) }) + + it('uses route-owned request byte lengths when supplied', () => { + const messages = [createUserMessage({ content: [image(100), image(100)], source })] + const projected = offloadRequestImagesWithPolicy(messages, { + representation: 'raw', + maxBytes: 3, + byteLength: () => 2, + }) + expect(projected[0]?.content).toEqual([ + { type: 'text', text: OFFLOADED_IMAGE_TEXT }, + image(100), + ]) + }) +}) + +describe('projectImagesForTextModel', () => { + it('returns image-free history unchanged', () => { + const messages = [createUserMessage({ content: [{ type: 'text', text: 'plain' }], source })] + expect(projectImagesForTextModel(messages)).toBe(messages) + }) + + it('replaces direct and nested images while retaining unaffected messages and blocks', () => { + const plain = createUserMessage({ content: [{ type: 'text', text: 'plain' }], source }) + const nested = { + type: 'tool-result' as const, + toolCallId: CallId('nested-image'), + content: [{ type: 'text' as const, text: 'before' }, image(3), { type: 'text' as const, text: 'after' }], + } + const unchangedNested = { + type: 'tool-result' as const, + toolCallId: CallId('text-only'), + content: [{ type: 'text' as const, text: 'unchanged' }], + } + const visual = createUserMessage({ + content: [{ type: 'text', text: 'lead' }, image(3), unchangedNested, nested], + source, + }) + + const projected = projectImagesForTextModel([plain, visual]) + expect(projected[0]).toBe(plain) + expect(projected[1]?.content).toEqual([ + { type: 'text', text: 'lead' }, + { type: 'text', text: '[image omitted because this model accepts text only; attachment sha256:aaaaaaaa]' }, + unchangedNested, + { + ...nested, + content: [ + { type: 'text', text: 'before' }, + { type: 'text', text: '[image omitted because this model accepts text only; attachment sha256:aaaaaaaa]' }, + { type: 'text', text: 'after' }, + ], + }, + ]) + }) }) diff --git a/packages/llm/llm/tests/service.spec.ts b/packages/llm/llm/tests/service.spec.ts index b2e5399964..4c624fa862 100644 --- a/packages/llm/llm/tests/service.spec.ts +++ b/packages/llm/llm/tests/service.spec.ts @@ -986,6 +986,18 @@ describe('LlmRuntime', () => { type: 'text', text: '[image omitted because this model accepts text only; attachment sha256:aaaaaaaa]', }]) + + const frozen = Object.freeze({ + provider: 'route', + model: 'text-only', + messages: [createUserMessage({ + content: [{ type: 'image', attachment }], + source: { kind: 'plugin' as const, plugin: 'test' }, + })], + }) + await collect(ctx.llm.stream(frozen)) + expect(Object.isFrozen(seen[1])).toBe(true) + expect(Object.isFrozen(seen[1]?.messages)).toBe(true) }) it('passes cancellation through exact-model resolution', async () => { From 703ce4a3d626c2225bd85743c8a11495e4fa2a3c Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 22:00:06 +0800 Subject: [PATCH 19/28] test(deepseek): expose Files API e2e failures --- packages/llm/llm-deepseek/tests/adapter.e2e.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/llm/llm-deepseek/tests/adapter.e2e.ts b/packages/llm/llm-deepseek/tests/adapter.e2e.ts index c52195433b..06ae9f6f4d 100644 --- a/packages/llm/llm-deepseek/tests/adapter.e2e.ts +++ b/packages/llm/llm-deepseek/tests/adapter.e2e.ts @@ -178,7 +178,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('llm-deepseek e2e (real API)', () })], maxTokens: 100, }) - expect(result.finish.kind).toBe('stop') + expect(result.finish).toMatchObject({ kind: 'stop' }) expect(textOf(result).trim().length).toBeGreaterThan(0) expect(uploadedFile).toMatch(/^file-api-/u) } finally { From 0c9a664223060fc9dbcb22557f9b32e0680f5507 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Thu, 20 Aug 2026 22:05:39 +0800 Subject: [PATCH 20/28] test(deepseek): print vision failure facts --- packages/llm/llm-deepseek/tests/adapter.e2e.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/llm/llm-deepseek/tests/adapter.e2e.ts b/packages/llm/llm-deepseek/tests/adapter.e2e.ts index 06ae9f6f4d..3858f214a0 100644 --- a/packages/llm/llm-deepseek/tests/adapter.e2e.ts +++ b/packages/llm/llm-deepseek/tests/adapter.e2e.ts @@ -178,7 +178,10 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('llm-deepseek e2e (real API)', () })], maxTokens: 100, }) - expect(result.finish).toMatchObject({ kind: 'stop' }) + expect( + result.finish.kind, + `DeepSeek vision result: ${JSON.stringify(result.finish)}`, + ).toBe('stop') expect(textOf(result).trim().length).toBeGreaterThan(0) expect(uploadedFile).toMatch(/^file-api-/u) } finally { From 724783b02480e0e926e17f00d92c59f1b59228f6 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Fri, 21 Aug 2026 12:30:47 +0800 Subject: [PATCH 21/28] refactor(image): remove region reads --- ...26-08-10-minimal-read-image-tool.i18n.yaml | 4 +- .../2026-08-10-minimal-read-image-tool.md | 4 +- .../2026-08-10-minimal-read-image-tool.zh.md | 4 +- ...0-unified-image-request-pipeline.i18n.yaml | 4 +- ...26-08-20-unified-image-request-pipeline.md | 14 +- ...08-20-unified-image-request-pipeline.zh.md | 14 +- docs/config-catalog.i18n.yaml | 4 +- docs/config-catalog.md | 2 +- docs/config-catalog.zh.md | 2 +- docs/subsystems/attachment.i18n.yaml | 4 +- docs/subsystems/attachment.md | 39 +--- docs/subsystems/attachment.zh.md | 39 +--- docs/tool-catalog.i18n.yaml | 4 +- docs/tool-catalog.md | 57 +---- docs/tool-catalog.zh.md | 55 +---- examples/acp-agent/tests/acp.snapshot.ts | 7 +- .../system-prompt.expected.md | 42 +--- .../read-image/tool-schemas.expected.json | 48 +--- .../attachment-local/README.i18n.yaml | 4 +- .../attachment/attachment-local/README.md | 2 +- .../attachment/attachment-local/README.zh.md | 2 +- .../attachment/attachment-local/src/index.ts | 25 +- .../attachment-local/src/request-image.ts | 94 +------- .../tests/request-image.spec.ts | 75 +----- .../attachment/attachment/README.i18n.yaml | 4 +- packages/attachment/attachment/README.md | 4 +- packages/attachment/attachment/README.zh.md | 4 +- packages/attachment/attachment/src/index.ts | 23 -- packages/attachment/attachment/src/types.ts | 24 +- .../attachment/attachment/tests/index.spec.ts | 9 +- .../core/tools/tests/gen-tool-catalog.spec.ts | 2 +- .../extensions/tool-cordis/src/api-catalog.ts | 18 +- packages/fs/tool-fs/README.i18n.yaml | 4 +- packages/fs/tool-fs/README.md | 17 +- packages/fs/tool-fs/README.zh.md | 17 +- packages/fs/tool-fs/src/read-image.ts | 150 +----------- packages/fs/tool-fs/tests/read-image.spec.ts | 220 +----------------- packages/llm/llm-deepseek/README.i18n.yaml | 4 +- packages/llm/llm-deepseek/README.md | 6 +- packages/llm/llm-deepseek/README.zh.md | 6 +- packages/llm/llm-deepseek/src/adapter.ts | 1 - packages/llm/llm-deepseek/src/serialize.ts | 9 +- packages/llm/llm-deepseek/src/upload-index.ts | 2 +- .../llm/llm-deepseek/tests/adapter.spec.ts | 3 +- .../llm/llm-deepseek/tests/serialize.spec.ts | 31 +-- packages/llm/llm-pi-ai/README.i18n.yaml | 4 +- packages/llm/llm-pi-ai/README.md | 4 +- packages/llm/llm-pi-ai/README.zh.md | 4 +- packages/llm/llm-pi-ai/src/context.ts | 12 +- packages/llm/llm-pi-ai/tests/context.spec.ts | 11 - packages/llm/llm/src/content.ts | 13 +- scripts/gen-cordis-catalog.ts | 1 - scripts/gen-tool-catalog.ts | 6 +- scripts/type-equiv.manifest.json | 10 - 54 files changed, 132 insertions(+), 1040 deletions(-) diff --git a/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.i18n.yaml b/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.i18n.yaml index 6c37530274..b0c70c4df4 100644 --- a/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.md -2026-08-10-minimal-read-image-tool.md: 0c0c6a95fa3d8be1dbe895ecd83ff44e1e1eac17 -2026-08-10-minimal-read-image-tool.zh.md: c3c2fe1095637a19c3ebaa21cf23a501fe83c480 +2026-08-10-minimal-read-image-tool.md: 19306a35fe709a04d94090a62056575b4d51f7bc +2026-08-10-minimal-read-image-tool.zh.md: c7562c433e909d1f81361c0ced56318795e6469e diff --git a/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.md b/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.md index 0c0c6a95fa..19306a35fe 100644 --- a/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.md +++ b/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.md @@ -6,14 +6,13 @@ English | [中文](2026-08-10-minimal-read-image-tool.zh.md) ## Problem -The multimodal attachment work gave user uploads a complete durable path, but the model itself had no way to inspect an image on disk or crop a durable user upload that had no path. `read` rejects binary content by contract, so an agent asked about a screenshot or rendered chart either failed or used a lossy workaround. A standalone attempt in PR #598 combined the tool with loop-level route scoping, per-route schema visibility, and new session-log concepts. Those features were not required to publish a logged image tool result. +The multimodal attachment work gave user uploads a complete durable path, but the model itself had no way to inspect an image on disk. `read` rejects binary content by contract, so an agent asked about a screenshot or rendered chart either failed or used a lossy workaround. A standalone attempt in PR #598 combined the tool with loop-level route scoping, per-route schema visibility, and new session-log concepts. Those features were not required to publish a logged image tool result. ## Decision Both image-reading operations live in `dsh-tool-fs` and publish ordinary logged tool results over existing extension points. - **`read_image` reads a filesystem path.** Extension selects the declared PNG/JPEG/WebP/GIF media type; the attachment store's magic-byte and pixel validation stays authoritative. Bytes travel `ctx.fs.stat` → bounded `ctx.fs.readBytes` → `ctx.attachments.saveImage` → `fs/observed`. The tool result contains metadata and an `ImageBlock`. -- **`read_image_region` crops a durable session attachment.** The request names the complete attachment id, current preview dimensions, and a preview-coordinate rectangle. The tool authorizes the id against images already referenced by the calling session, maps the rectangle to the durable master, crops that master, and persists the result as a new attachment. Its result contains the cropped `ImageBlock`, so the model-visible crop is reconstructable from the log. This is the path for pasted or dragged images that have no filesystem location. - **`FileSystem.readBytes(target, signal, maxBytes)`** is a new required provider primitive: the byte bound lives at the seam so no backend can buffer an unbounded file, with the stat-size short-circuit and a one-byte-past-cap stream guard against post-stat growth (`FS_TOO_LARGE`). - **Registration is composition-conditional, execution is route-gated.** The tools register only under `ctx.inject(['attachments'], …)`. Before I/O, the strict gate resolves the calling route through `ctx.llm.resolveModelInfo` and requires `image` in `inputModalities`; unknown capability refuses. A text-only route can still consume prior durable images because the shared LLM runtime projects them to placeholders at request assembly. - **Code Mode forwards the image out-of-band**: a nested dispatch returns the canonical value (execution-local, no image block) and defers a `user`-role context message carrying the envelope and image, so the picture still reaches the next request. @@ -29,6 +28,5 @@ Both image-reading operations live in `dsh-tool-fs` and publish ordinary logged ## Consequences - The tools refuse execution on a text-only route, while existing images in session history are represented by request-local placeholders. -- Pasted and dragged images can be cropped without exposing local paths. Session reference authorization prevents access to attachments outside the current session. - Repeated image results accumulate request cost until request projection or compaction removes them; content addressing deduplicates durable bytes. - The tool-result card renders the durable reference, not pixels; inline preview is deferred to the UI packages. diff --git a/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.zh.md b/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.zh.md index c3c2fe1095..c7562c433e 100644 --- a/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.zh.md +++ b/.agents/notes/implemented/feature/2026-08-10-minimal-read-image-tool.zh.md @@ -6,14 +6,13 @@ Status: implemented ## 问题 -多模态附件工作为用户上传建立了完整的持久路径,但模型无法查看磁盘图片,也无法裁剪没有文件路径的持久用户上传。`read` 按约定拒绝二进制内容,因此被问到截图或渲染图表的 agent 要么失败,要么使用有损的变通方法。PR #598 的独立尝试把工具与循环级路由作用域、按路由控制 schema 可见性和新的会话日志概念放在一起。这些能力不是发布一条带图片且已记录的工具结果所必需的。 +多模态附件工作为用户上传建立了完整的持久路径,但模型无法查看磁盘图片。`read` 按约定拒绝二进制内容,因此被问到截图或渲染图表的 agent 要么失败,要么使用有损的变通方法。PR #598 的独立尝试把工具与循环级路由作用域、按路由控制 schema 可见性和新的会话日志概念放在一起。这些能力不是发布一条带图片且已记录的工具结果所必需的。 ## 决定 两个图片读取操作都放在 `dsh-tool-fs`,通过现有扩展点发布普通的持久工具结果。 - **`read_image` 读取文件系统路径。** 扩展名选择声明的 PNG/JPEG/WebP/GIF 媒体类型,附件存储的魔数与像素校验保持权威。字节沿 `ctx.fs.stat` → 有界 `ctx.fs.readBytes` → `ctx.attachments.saveImage` → `fs/observed` 流动。工具结果包含元数据和一个 `ImageBlock`。 -- **`read_image_region` 裁剪会话中的持久附件。** 请求给出完整附件 ID、当前预览尺寸和预览坐标矩形。工具根据当前会话已引用的图片授权该 ID,把矩形映射到持久主版本,从主版本裁剪,并把结果保存为新附件。结果包含裁剪后的 `ImageBlock`,因此模型可见裁剪可以从日志重建。这也是粘贴或拖入且没有文件路径的图片所使用的入口。 - **`FileSystem.readBytes(target, signal, maxBytes)`** 是新的必备提供方原语:字节上限放在 seam 上,任何后端都无法无界缓冲文件;stat 大小先短路,随后的流最多多读一个字节以防 stat 之后的增长(`FS_TOO_LARGE`)。 - **注册随组合条件挂载,执行按路由门禁。** 工具只在 `ctx.inject(['attachments'], …)` 作用域内注册。执行时在 I/O 之前通过 `ctx.llm.resolveModelInfo` 解析调用路由,并要求 `inputModalities` 包含 `image`;能力未知即拒绝。纯文本路由仍可使用此前的持久图片,因为共享 LLM 运行时会在请求组装时把图片投影为占位符。 - **Code Mode 以带外方式转发图像**:嵌套分派返回规范值(仅限本次执行,不含图像块),并延迟提交一条携带信封和图像的 `user` 角色上下文消息,图片仍会到达下一次请求。 @@ -29,6 +28,5 @@ Status: implemented ## 后果 - 工具在纯文本路由上拒绝执行,而会话历史中已经存在的图片会由请求期占位符表示。 -- 粘贴和拖入的图片无需暴露本地路径即可裁剪。会话引用授权会阻止访问当前会话范围外的附件。 - 重复的图片结果会累积请求成本,直到请求投影或压缩将其移除;内容寻址只去重持久字节。 - 工具结果卡片渲染持久引用而非像素;内嵌预览延后到 UI 包处理。 diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml index a721138585..e95c7faa2a 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md -2026-08-20-unified-image-request-pipeline.md: c4af375d94ebf2b52fbdd0e8d3d4ee715f87f50e -2026-08-20-unified-image-request-pipeline.zh.md: a1e10c63804b42da127bd115c35587191f0a60f0 +2026-08-20-unified-image-request-pipeline.md: f0ef01de3b22c7132e7f698d0948a0da945726ba +2026-08-20-unified-image-request-pipeline.zh.md: b1a14ac418987ab8bfee9b731ad38cb48e21753e diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md index c4af375d94..f0ef01de3b 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md @@ -6,7 +6,7 @@ English | [中文](2026-08-20-unified-image-request-pipeline.zh.md) ## Problem -Durable image history, provider resolution, inline request size, and remote file reuse have different limits. Treating an admitted image as the bytes sent on every later request forced one byte cap and one raster to serve all four concerns. Large but ordinary input was refused, clean 16-bit PNG could pass into history and fail at DeepSeek, repeated base64 expanded long requests, and a provider rejection repeated because the same durable image stayed in every future request. A model also had no stable way to crop a user upload that had no filesystem path. +Durable image history, provider resolution, inline request size, and remote file reuse have different limits. Treating an admitted image as the bytes sent on every later request forced one byte cap and one raster to serve all four concerns. Large but ordinary input was refused, clean 16-bit PNG could pass into history and fail at DeepSeek, repeated base64 expanded long requests, and a provider rejection repeated because the same durable image stayed in every future request. ## Decision @@ -24,13 +24,13 @@ Batch admission prepares and verifies every master once before publishing any me `AttachmentStore.readImageRequest` derives a request version under route-owned total-pixel and encoded-byte budgets. Scaling is `min(1, sqrt(maxPixels / (width * height)))`, with no enlargement, followed by inward integer rounding so the encoded raster never exceeds the total-pixel cap. DeepSeek V4 Flash Vision Exp uses 640,000 total pixels and 1MiB raw encoded bytes by default; low detail uses 512 by 512 total pixels. A 2048 by 1024 master projects to 1130 by 565 under the hard cap. Request encoding uses the same color branches, with PNG (palette only without alpha) then WebP 85 and 80 for low-color input, WebP 85 then 80 for other alpha input, and JPEG 85 then 80 for other opaque input. Each fallback runs only after the previous result exceeds 1MiB, and dimensions shrink only after both quality attempts exceed it. The same derivation is used by normal agent turns, direct `ctx.llm.stream` calls, compaction, and other auxiliary streams. -The `variantId` and cache path cover the master attachment id, transform version, route pixel and byte budgets, optional master-coordinate crop, and fixed encoder parameters. A new cache entry is fully decoded before publication. Cache hits use a header probe to check format, 8-bit sRGB/sRGBA facts, dimensions, alpha, and byte limits without decoding the complete raster again; a mismatch regenerates the entry. DeepSeek Files and pi-ai inline base64 therefore use the same deterministic bytes for the same policy. Inline accounting uses the derived byte length after base64 expansion, not the master byte count. Equal in-process `variantId` calls share one transform and cache write. Each caller can cancel its own wait; the shared transform is aborted only after every waiter has cancelled. `AttachmentStore.readImageRequests` preserves input order while the local implementation runs master and request transforms through one FIFO limiter. `imageCompressionConcurrency` is configurable from 1 through 8 and defaults to 2. Batch publication remains sequential after every master has been prepared. +The `variantId` and cache path cover the master attachment id, transform version, route pixel and byte budgets, and fixed encoder parameters. A new cache entry is fully decoded before publication. Cache hits use a header probe to check format, 8-bit sRGB/sRGBA facts, dimensions, alpha, and byte limits without decoding the complete raster again; a mismatch regenerates the entry. DeepSeek Files and pi-ai inline base64 therefore use the same deterministic bytes for the same policy. Inline accounting uses the derived byte length after base64 expansion, not the master byte count. Equal in-process `variantId` calls share one transform and cache write. Each caller can cancel its own wait; the shared transform is aborted only after every waiter has cancelled. `AttachmentStore.readImageRequests` preserves input order while the local implementation runs master and request transforms through one FIFO limiter. `imageCompressionConcurrency` is configurable from 1 through 8 and defaults to 2. Batch publication remains sequential after every master has been prepared. Request-size offload is a deterministic oldest-first projection. Before reading attachments, each route uses `min(masterBytes, requestVersionMaxBytes)` as a conservative upper bound and removes the oldest over-budget prefix. Only retained masters are read and transformed, so an omitted missing or corrupt object cannot block the request. A second projection uses exact derived lengths without bringing omitted images back. DeepSeek defaults to 128MiB and 600 referenced images. Its removed prefix advances past successive 64MiB byte boundaries and in 20-image count quanta, so 129 one-megabyte images remove the oldest 65, retain 64MiB, and keep that prefix stable until total history passes 192MiB. Pi-ai retains a configurable base64 request bound. A text-only route receives deterministic attachment placeholders, including nested tool-result images, while append-only session history keeps the original references. -### Stable handles and master-coordinate crops +### Stable handles -Every retained request image is preceded by its complete attachment id and actual request dimensions. When the active request exposes `read_image_region`, the text also supplies its preview-coordinate arguments. The tool accepts only an attachment already referenced by the calling session. It maps the supplied preview rectangle to the 2048px master with floor-at-origin and ceil-at-far-edge rounding, crops the master rather than the preview, and persists the result as a new attachment. The tool result contains the new `ImageBlock`, so model-visible output and the durable log remain equivalent. +Every retained request image is preceded by its complete attachment id and actual request dimensions. User messages, tool results, agent-loop requests, compaction, and direct `ctx.llm.stream` calls share this projection. ### DeepSeek Files lifecycle @@ -46,7 +46,7 @@ Historical attachment objects that later disappear or fail integrity verificatio ## Alternatives considered -**Use one 1MiB canonical image for storage and requests.** This makes model resolution determine durable quality, reduces the source for later crops, and combines local storage, inline expansion, Files quota, and model pixels into one setting. Independent master and request policies keep those responsibilities explicit. +**Use one 1MiB canonical image for storage and requests.** This makes model resolution determine durable image detail and combines local storage, inline expansion, Files quota, and model pixels into one setting. Independent master and request policies keep those responsibilities explicit. **Reject images above provider dimensions or at the encoding quality floor.** A provider limit is route-specific and future requests may use another model. Proportional master preparation and request projection accept ordinary large images while bounding each later representation. @@ -56,15 +56,13 @@ Historical attachment objects that later disappear or fail integrity verificatio **Trust a locally indexed file id indefinitely.** Remote expiry, deletion, and lost upload responses make local and provider state diverge. Response-directed invalidation and one re-upload recover without an unbounded retry loop; an ambiguous stale-file response must invalidate every file used by that attempt because it provides no safe exact target. -**Crop the request preview.** Repeated crops would compound the 640,000-pixel reduction and make coordinates depend on previous encodes. Mapping back to the master preserves the available local detail. - **Refuse text-only model selection after any image.** Durable history can outlive the model that first consumed it. Request-local placeholders keep the session usable without rewriting history. **Remove one image whenever a request crosses its limit.** That changes an early request message after nearly every new upload. Quantized removed prefixes keep cache invalidation occasional while honoring the configured high bound. ## Verification -Package tests generate 16-bit RGB and RGBA PNG fixtures, prove 8-bit conversion and clean 8-bit passthrough, retain alpha under byte pressure, distinguish high-frequency and ordinary photos from low-color graphics, stop lazy encoding after the first fitting candidate, cover square and wide 640,000-pixel projections, enforce 1MiB request bytes, singleflight equal variants and uploads without shared-cancellation leaks, bound transform concurrency, preserve cache and upload identity, skip attachment reads for conservatively offloaded history, map preview crops to the master, prepare batches once, reject inconsistent Files responses, refresh near-expiry ids without retrieve, recover once from single-id, multiple-id, and ambiguous stale responses, paginate before quota deletion, normalize provider diagnostics, project text-only history, and share normal/compaction request bytes. Keyless assembled snapshots cover the real tool schemas and image request path. A credentialed test uses the built-in `deepseek-official` route and its configured endpoint, never a custom provider entry. +Package tests generate 16-bit RGB and RGBA PNG fixtures, prove 8-bit conversion and clean 8-bit passthrough, retain alpha under byte pressure, distinguish high-frequency and ordinary photos from low-color graphics, stop lazy encoding after the first fitting candidate, cover square and wide 640,000-pixel projections, enforce 1MiB request bytes, singleflight equal variants and uploads without shared-cancellation leaks, bound transform concurrency, preserve cache and upload identity, skip attachment reads for conservatively offloaded history, prepare batches once, reject inconsistent Files responses, refresh near-expiry ids without retrieve, recover once from single-id, multiple-id, and ambiguous stale responses, paginate before quota deletion, normalize provider diagnostics, project text-only history, and share normal/compaction request bytes. Keyless assembled snapshots cover the real tool schemas and image request path. A credentialed test uses the built-in `deepseek-official` route and its configured endpoint, never a custom provider entry. ## Consequences diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md index a1e10c6380..b1a14ac418 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md @@ -6,7 +6,7 @@ Status: implemented ## Problem -持久图片历史、提供方分辨率、内联请求大小和远端文件复用有不同限制。过去把已接纳图片直接作为之后每次请求发送的字节,导致一个字节上限和一份光栅同时承担四种职责。普通大图会被拒绝;干净的 16-bit PNG 可以进入历史,之后才被 DeepSeek 拒绝;重复 base64 使长会话请求持续增长;提供方拒绝后,同一持久图片还会进入每次后续请求。模型也无法稳定裁剪没有文件系统路径的用户上传图片。 +持久图片历史、提供方分辨率、内联请求大小和远端文件复用有不同限制。过去把已接纳图片直接作为之后每次请求发送的字节,导致一个字节上限和一份光栅同时承担四种职责。普通大图会被拒绝;干净的 16-bit PNG 可以进入历史,之后才被 DeepSeek 拒绝;重复 base64 使长会话请求持续增长;提供方拒绝后,同一持久图片还会进入每次后续请求。 ## Decision @@ -24,13 +24,13 @@ Status: implemented `AttachmentStore.readImageRequest` 按路由拥有的总像素和编码字节预算派生请求版本。缩放公式为 `min(1, sqrt(maxPixels / (width * height)))`,不会放大小图,随后向预算内取整,确保编码光栅不超过总像素上限。DeepSeek V4 Flash Vision Exp 默认使用总像素 640,000 和原始编码字节 1MiB;low detail 使用总像素 512×512。2048×1024 主版本在这个硬上限下会投影为 1130×565。请求编码使用相同的分类分支:低色数输入先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80 的 WebP;其他透明输入依次尝试质量 85、80 的 WebP;其他非透明输入依次尝试质量 85、80 的 JPEG。只有前一结果超过 1MiB 时才执行下一个候选;两个质量档都超限后才缩小尺寸。普通 agent 轮次、直接 `ctx.llm.stream` 调用、压缩和其他辅助流都使用同一派生过程。 -`variantId` 和缓存路径覆盖主附件 ID、变换策略版本、路由像素和字节预算、可选的主版本坐标裁剪区域及固定编码参数。新缓存条目在发布前会完整解码。缓存命中只探测文件头,校验格式、8-bit sRGB/sRGBA、尺寸、透明通道和字节上限,不会再次完整解码光栅;不匹配时会重新生成。因此,同一策略下的 DeepSeek Files 和 pi-ai 内联 base64 使用相同的确定性字节。内联计量使用派生字节经过 base64 膨胀后的长度,不使用主版本字节数。同一进程内相同 `variantId` 的调用共享一次变换和缓存写入。每个调用方可以取消自己的等待;只有全部等待方都取消时,共享变换才会中止。`AttachmentStore.readImageRequests` 保持输入顺序,本地实现则通过一个 FIFO 限流器运行主版本和请求版本变换。`imageCompressionConcurrency` 的可配置范围为 1 至 8,默认值为 2。全部主版本准备完成后,批次仍按顺序发布。 +`variantId` 和缓存路径覆盖主附件 ID、变换策略版本、路由像素和字节预算及固定编码参数。新缓存条目在发布前会完整解码。缓存命中只探测文件头,校验格式、8-bit sRGB/sRGBA、尺寸、透明通道和字节上限,不会再次完整解码光栅;不匹配时会重新生成。因此,同一策略下的 DeepSeek Files 和 pi-ai 内联 base64 使用相同的确定性字节。内联计量使用派生字节经过 base64 膨胀后的长度,不使用主版本字节数。同一进程内相同 `variantId` 的调用共享一次变换和缓存写入。每个调用方可以取消自己的等待;只有全部等待方都取消时,共享变换才会中止。`AttachmentStore.readImageRequests` 保持输入顺序,本地实现则通过一个 FIFO 限流器运行主版本和请求版本变换。`imageCompressionConcurrency` 的可配置范围为 1 至 8,默认值为 2。全部主版本准备完成后,批次仍按顺序发布。 请求大小 offload 是确定性的从旧到新投影。读取附件前,每条路由先以 `min(主版本字节数, 请求版本字节上限)` 作为保守上界,移除超出预算的最旧前缀。系统只读取并转换保留的主版本,因此已省略的缺失或损坏对象不会阻塞请求。第二次投影使用确切派生长度,但不会重新加入已省略图片。DeepSeek 默认上限为 128MiB 和 600 张引用图片。被移除前缀会越过连续的 64MiB 字节边界,并按 20 张图片数量步长递增,因此 129 张 1MiB 图片会移除最旧的 65 张并保留 64MiB;持久历史超过 192MiB 前,该前缀保持不变。Pi-ai 保留可配置的 base64 请求上限。纯文本路由会收到确定性的附件占位文本,其中包括嵌套工具结果图片;追加式会话历史继续保留原始引用。 -### 稳定句柄与主版本坐标裁剪 +### 稳定句柄 -每张保留请求图片前都有完整附件 ID 和实际请求尺寸。当前请求公开 `read_image_region` 时,这段文本还会提供预览坐标参数。该工具只接受调用会话已经引用的附件。它按起点向下取整、远端边界向上取整,把提交的预览矩形映射到 2048px 主版本,从主版本而非预览图裁剪,并把结果保存为新附件。工具结果包含新的 `ImageBlock`,因此模型可见输出与持久日志保持一致。 +每张保留请求图片前都有完整附件 ID 和实际请求尺寸。用户消息、工具结果、agent loop 请求、压缩和直接 `ctx.llm.stream` 调用共享这套投影。 ### DeepSeek Files 生命周期 @@ -46,7 +46,7 @@ Status: implemented ## Alternatives considered -**使用一份 1MiB 规范图片同时负责存储和请求。** 这种做法让模型分辨率决定持久质量,降低之后裁剪可用的源信息,并把本地存储、内联膨胀、Files 配额和模型像素合并成一个设置。独立的主版本和请求策略会明确区分这些职责。 +**使用一份 1MiB 规范图片同时负责存储和请求。** 这种做法让模型分辨率决定持久图片细节,并把本地存储、内联膨胀、Files 配额和模型像素合并成一个设置。独立的主版本和请求策略会明确区分这些职责。 **拒绝超过提供方尺寸或达到编码质量下限的图片。** 提供方限制属于具体路由,未来请求可能改用另一个模型。按比例准备主版本和投影请求版本可以接纳普通大图,同时约束每种后续表示。 @@ -56,15 +56,13 @@ Status: implemented **永久信任本地索引中的文件 ID。** 远端过期、删除和上传响应丢失会使本地与提供方状态不一致。按响应失效和一次重新上传可以恢复,同时避免无界重试;响应没有给出可安全使用的精确目标时,必须使该次请求使用的全部文件失效。 -**从请求预览图裁剪。** 重复裁剪会叠加 640,000 像素缩小,坐标也会依赖之前的编码。映射回主版本能保留本地可用细节。 - **历史中出现图片后拒绝选择纯文本模型。** 持久历史可能比最初读取它的模型存活更久。按请求生成的占位文本可以保持会话可用,无需改写历史。 **请求每次越过上限就移除一张图片。** 这种做法会在几乎每次新增图片后改写较早的请求消息。按固定步长递增的移除前缀会降低缓存失效频率,同时遵守配置的上限。 ## Verification -包测试会生成 16-bit RGB 和 RGBA PNG fixture,验证 8-bit 转换与干净 8-bit 字节直通、字节压力下保留透明通道、区分高频和普通照片与低色数图形、首个候选合规后停止编码、正方形和宽屏 640,000 像素投影、请求字节不超过 1MiB、相同变体与上传 singleflight 且不会共享取消、变换并发上限、缓存与上传身份、跳过已保守 offload 的历史附件读取、预览到主版本坐标映射、批量只准备一次、Files 响应不一致、进入刷新余量时不查询远端并更新 ID、单个 ID、多个 ID 和模糊失效响应只恢复一次、删除配额文件前完成分页、规范化提供方诊断、纯文本投影,以及普通请求与压缩共享请求字节。无需密钥的组装快照覆盖真实工具 schema 和图片请求路径。使用凭据的测试只使用内置 `deepseek-official` 路由及其已配置端点,不使用自定义提供方条目。 +包测试会生成 16-bit RGB 和 RGBA PNG fixture,验证 8-bit 转换与干净 8-bit 字节直通、字节压力下保留透明通道、区分高频和普通照片与低色数图形、首个候选合规后停止编码、正方形和宽屏 640,000 像素投影、请求字节不超过 1MiB、相同变体与上传 singleflight 且不会共享取消、变换并发上限、缓存与上传身份、跳过已保守 offload 的历史附件读取、批量只准备一次、Files 响应不一致、进入刷新余量时不查询远端并更新 ID、单个 ID、多个 ID 和模糊失效响应只恢复一次、删除配额文件前完成分页、规范化提供方诊断、纯文本投影,以及普通请求与压缩共享请求字节。无需密钥的组装快照覆盖真实工具 schema 和图片请求路径。使用凭据的测试只使用内置 `deepseek-official` 路由及其已配置端点,不使用自定义提供方条目。 ## Consequences diff --git a/docs/config-catalog.i18n.yaml b/docs/config-catalog.i18n.yaml index 804d5dd86a..276fad138a 100644 --- a/docs/config-catalog.i18n.yaml +++ b/docs/config-catalog.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/config-catalog.md -config-catalog.md: 661e9a50200fd5c650c389d9bb631c04de61d228 -config-catalog.zh.md: 4299bccc1f59899bd78fd64f915c784e26eea49d +config-catalog.md: d288fe3b85f1599da6ecef3dcf59c04c4e8c85d5 +config-catalog.zh.md: 266465fd09312c5dde9df4453c34f3aa774db7e2 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 661e9a5020..d288fe3b85 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -346,7 +346,7 @@ export interface Config { } ``` -Source: [`packages/attachment/attachment-local/src/index.ts:53`](../packages/attachment/attachment-local/src/index.ts) +Source: [`packages/attachment/attachment-local/src/index.ts:52`](../packages/attachment/attachment-local/src/index.ts) diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 4299bccc1f..266465fd09 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -348,7 +348,7 @@ export interface Config { } ``` -来源:[`packages/attachment/attachment-local/src/index.ts:53`](../packages/attachment/attachment-local/src/index.ts) +来源:[`packages/attachment/attachment-local/src/index.ts:52`](../packages/attachment/attachment-local/src/index.ts) diff --git a/docs/subsystems/attachment.i18n.yaml b/docs/subsystems/attachment.i18n.yaml index e391a3aa27..ee14a0698f 100644 --- a/docs/subsystems/attachment.i18n.yaml +++ b/docs/subsystems/attachment.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/subsystems/attachment.md -attachment.md: ea15172e3e1fafec2e09c3bedc2590fc7551eb2e -attachment.zh.md: c04114c9691fa1ba03446f903c4baf5ae021da4c +attachment.md: 7c55bc192088f67ae7d117bc150aa0ea6fdf8b09 +attachment.zh.md: d5a140e283c1b7aa6ee5c991c2932ff65de0b88e diff --git a/docs/subsystems/attachment.md b/docs/subsystems/attachment.md index 99d4ba7682..7c55bc1920 100644 --- a/docs/subsystems/attachment.md +++ b/docs/subsystems/attachment.md @@ -89,16 +89,6 @@ interface StoredImageAttachment { } ``` -```ts type-equiv -/** Pixel rectangle in the oriented 2048px master-version coordinate system. */ -interface MasterImageCrop { - x: number - y: number - width: number - height: number -} -``` - ```ts type-equiv /** Deterministic request-image policy selected by one exact model route. */ interface ImageRequestPolicy { @@ -106,27 +96,13 @@ interface ImageRequestPolicy { maxPixels: number /** Encoded-byte cap before base64 expansion or Files API upload. */ maxBytes: number - /** Optional master-coordinate crop applied before pixel-budget scaling. */ - crop?: MasterImageCrop -} -``` - -```ts type-equiv -/** Crop coordinates measured by a model on the request preview it received. */ -interface PreviewImageCrop { - previewWidth: number - previewHeight: number - x: number - y: number - width: number - height: number } ``` ```ts type-equiv /** Cached request version derived from one provider-independent master attachment. */ interface RequestImageAttachment { - /** Cache and upload-index key over the master id, policy, crop, and fixed encoder parameters. */ + /** Cache and upload-index key over the master id, policy, and fixed encoder parameters. */ variantId: ImageVariantId /** Durable master reference from which this request version was derived. */ master: ImageAttachmentRef @@ -142,12 +118,10 @@ interface RequestImageAttachment { space: 'srgb' /** Whether the encoded request version retains an alpha channel. */ hasAlpha: boolean - /** Applied master-coordinate crop, when present. */ - crop?: MasterImageCrop } ``` -`saveImage()` prepares a provider-independent 2048px, 4MiB master and atomically commits it before returning its reference. `saveImages()` prepares every validated master once before publishing the batch, so validation rejection leaves no partial objects and publication does not repeat decoding or quality selection. `admitEncodedImages()` is the wire entry for base64 uploads and delegates count, aggregate-byte, and ordered batch admission to `saveImages()`. `readImage()` verifies a master from an authorized session path. `readImageRequest()` derives and caches one request version under an exact route pixel and byte budget; new entries are fully decoded before publication, while cache hits use a bounded metadata probe. `readImageRequests()` lets an implementation apply its configured transform concurrency to an ordered batch. The local implementation lazily encodes preferred candidates, singleflights equal request identities, lets each waiter cancel independently, stops shared work when no waiter remains, and defaults to two simultaneous transformations. `cropImage()` maps model preview coordinates back to the master and returns another durable attachment. The service is retention-neutral: resumed and forked sessions may share objects, so reference-aware garbage collection is deferred rather than tied to one session's deletion. +`saveImage()` prepares a provider-independent 2048px, 4MiB master and atomically commits it before returning its reference. `saveImages()` prepares every validated master once before publishing the batch, so validation rejection leaves no partial objects and publication does not repeat decoding or quality selection. `admitEncodedImages()` is the wire entry for base64 uploads and delegates count, aggregate-byte, and ordered batch admission to `saveImages()`. `readImage()` verifies a master from an authorized session path. `readImageRequest()` derives and caches one request version under an exact route pixel and byte budget; new entries are fully decoded before publication, while cache hits use a bounded metadata probe. `readImageRequests()` lets an implementation apply its configured transform concurrency to an ordered batch. The local implementation lazily encodes preferred candidates, singleflights equal request identities, lets each waiter cancel independently, stops shared work when no waiter remains, and defaults to two simultaneous transformations. The service is retention-neutral: resumed and forked sessions may share objects, so reference-aware garbage collection is deferred rather than tied to one session's deletion. @@ -217,15 +191,6 @@ readImageRequest( ref: ImageAttachmentRef, policy: ImageRequestPolicy, signal?: * @returns request versions in the same order as `refs`. */ async readImageRequests( refs: readonly ImageAttachmentRef[], policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise - -/** - * Crop the stored master by coordinates measured on a model request preview and persist the result. - * @param ref - session-authorized master attachment. - * @param crop - preview dimensions and preview-coordinate rectangle. - * @param signal - optional cancellation. - * @returns a new durable attachment reference suitable for a logged tool result. - */ -cropImage( ref: ImageAttachmentRef, crop: PreviewImageCrop, signal?: AbortSignal, ): Promise ``` Source: [`packages/attachment/attachment/src/index.ts`](../../packages/attachment/attachment/src/index.ts) diff --git a/docs/subsystems/attachment.zh.md b/docs/subsystems/attachment.zh.md index d235c9ed2e..d5a140e283 100644 --- a/docs/subsystems/attachment.zh.md +++ b/docs/subsystems/attachment.zh.md @@ -89,16 +89,6 @@ interface StoredImageAttachment { } ``` -```ts type-equiv -/** Pixel rectangle in the oriented 2048px master-version coordinate system. */ -interface MasterImageCrop { - x: number - y: number - width: number - height: number -} -``` - ```ts type-equiv /** Deterministic request-image policy selected by one exact model route. */ interface ImageRequestPolicy { @@ -106,27 +96,13 @@ interface ImageRequestPolicy { maxPixels: number /** Encoded-byte cap before base64 expansion or Files API upload. */ maxBytes: number - /** Optional master-coordinate crop applied before pixel-budget scaling. */ - crop?: MasterImageCrop -} -``` - -```ts type-equiv -/** Crop coordinates measured by a model on the request preview it received. */ -interface PreviewImageCrop { - previewWidth: number - previewHeight: number - x: number - y: number - width: number - height: number } ``` ```ts type-equiv /** Cached request version derived from one provider-independent master attachment. */ interface RequestImageAttachment { - /** Cache and upload-index key over the master id, policy, crop, and fixed encoder parameters. */ + /** Cache and upload-index key over the master id, policy, and fixed encoder parameters. */ variantId: ImageVariantId /** Durable master reference from which this request version was derived. */ master: ImageAttachmentRef @@ -142,12 +118,10 @@ interface RequestImageAttachment { space: 'srgb' /** Whether the encoded request version retains an alpha channel. */ hasAlpha: boolean - /** Applied master-coordinate crop, when present. */ - crop?: MasterImageCrop } ``` -`saveImage()` 准备提供方无关的 2048px、4MiB 主版本,并在返回引用前以原子方式提交。`saveImages()` 在发布批次前为每个成员各准备一次经过验证的主版本,因此校验拒绝不会留下部分对象,发布也不会重复解码或选择质量。`admitEncodedImages()` 是面向 base64 上传的 wire 入口,把张数、聚合字节和有序批量准入交给 `saveImages()`。`readImage()` 校验来自已授权会话路径的主版本。`readImageRequest()` 按确切路由的像素和字节预算派生并缓存请求版本;新条目在发布前完整解码,缓存命中只做有界元数据探测。`readImageRequests()` 允许实现按自身配置的变换并发处理有序批次。本地实现按需编码首选候选、合并相同请求身份的并发任务、允许每个等待方单独取消、没有等待方时停止共享任务,默认同时执行两项变换。`cropImage()` 把模型预览坐标映射回主版本,并返回另一个持久附件。该服务不规定保留策略:恢复和 fork 后的会话可能共享对象,因此基于引用的垃圾回收会延期实现,不与单个会话的删除绑定。 +`saveImage()` 准备提供方无关的 2048px、4MiB 主版本,并在返回引用前以原子方式提交。`saveImages()` 在发布批次前为每个成员各准备一次经过验证的主版本,因此校验拒绝不会留下部分对象,发布也不会重复解码或选择质量。`admitEncodedImages()` 是面向 base64 上传的 wire 入口,把张数、聚合字节和有序批量准入交给 `saveImages()`。`readImage()` 校验来自已授权会话路径的主版本。`readImageRequest()` 按确切路由的像素和字节预算派生并缓存请求版本;新条目在发布前完整解码,缓存命中只做有界元数据探测。`readImageRequests()` 允许实现按自身配置的变换并发处理有序批次。本地实现按需编码首选候选、合并相同请求身份的并发任务、允许每个等待方单独取消、没有等待方时停止共享任务,默认同时执行两项变换。该服务不规定保留策略:恢复和 fork 后的会话可能共享对象,因此基于引用的垃圾回收会延期实现,不与单个会话的删除绑定。 @@ -217,15 +191,6 @@ readImageRequest( ref: ImageAttachmentRef, policy: ImageRequestPolicy, signal?: * @returns request versions in the same order as `refs`. */ async readImageRequests( refs: readonly ImageAttachmentRef[], policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise - -/** - * Crop the stored master by coordinates measured on a model request preview and persist the result. - * @param ref - session-authorized master attachment. - * @param crop - preview dimensions and preview-coordinate rectangle. - * @param signal - optional cancellation. - * @returns a new durable attachment reference suitable for a logged tool result. - */ -cropImage( ref: ImageAttachmentRef, crop: PreviewImageCrop, signal?: AbortSignal, ): Promise ``` Source: [`packages/attachment/attachment/src/index.ts`](../../packages/attachment/attachment/src/index.ts) diff --git a/docs/tool-catalog.i18n.yaml b/docs/tool-catalog.i18n.yaml index d219a4c8ad..10447d6107 100644 --- a/docs/tool-catalog.i18n.yaml +++ b/docs/tool-catalog.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/tool-catalog.md -tool-catalog.md: 11a7aead7938fca40d20096e3689890258fbe31c -tool-catalog.zh.md: f29d489441b36318523e0afa2eeab9104e639fd0 +tool-catalog.md: 1fa650f1e4e025274d069f27a6522abff46af2e2 +tool-catalog.zh.md: c3209e7007e9cf05770ccee0698f9e98a32e8363 diff --git a/docs/tool-catalog.md b/docs/tool-catalog.md index 11a7aead79..1fa650f1e4 100644 --- a/docs/tool-catalog.md +++ b/docs/tool-catalog.md @@ -24,7 +24,7 @@ This table connects model-visible tool names to the plugin package and service s | `@deepseek-ai/dsh-tool-bash-persistent` | `bash` | `ctx.tools`, `ctx.terminals`, `an owning Agent at execution time` | `tool/call`, `PTY shell state`, `tool/result` | - | One owner-isolated persistent bash tool; deployment composition supplies the PTY backend and may override the model-facing environment description. | | `@deepseek-ai/dsh-tool-pwsh-persistent` | `pwsh` | `ctx.tools`, `ctx.terminals`, `an owning Agent at execution time` | `tool/call`, `PTY shell state`, `tool/result` | - | One owner-isolated persistent pwsh tool, the Windows counterpart of the persistent bash tool; deployment composition supplies a pwsh-dialect PTY backend and may override the model-facing environment description. | | `@deepseek-ai/dsh-tool-str-replace-editor` | `str_replace_editor` | `ctx.tools`, `ctx.fs` | `tool/call`, `fs/observed after view presence/absence, edit absence, or successful mutation`, `tool/result` | - | Standalone view/create/unique literal replace/line insert tool over the filesystem seam; it composes with any shell or terminal API. | -| `@deepseek-ai/dsh-tool-fs` | `edit`, `read`, `read_image`, `read_image_region`, `write` | `ctx.tools`, `ctx.fs`, `ctx.systemPrompt`, `ctx.attachments (image-tool registration)`, `ctx.llm + an image-capable route (image-tool execution)` | `tool/call`, `fs/write-intent or fs/edit-intent for mutations`, `fs/observed after read presence/absence or successful file operation`, `durable attachment (read_image and read_image_region)`, `tool/result` | - | The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-observation-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. The image tools are not registered without `ctx.attachments`; their schemas are route-independent, and execution refuses unless the exact routed model declares image input. | +| `@deepseek-ai/dsh-tool-fs` | `edit`, `read`, `read_image`, `write` | `ctx.tools`, `ctx.fs`, `ctx.systemPrompt`, `ctx.attachments (image-tool registration)`, `ctx.llm + an image-capable route (image-tool execution)` | `tool/call`, `fs/write-intent or fs/edit-intent for mutations`, `fs/observed after read presence/absence or successful file operation`, `durable attachment (read_image)`, `tool/result` | - | The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-observation-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. The image tool is not registered without `ctx.attachments`; its schema is route-independent, and execution refuses unless the exact routed model declares image input. | | `@deepseek-ai/dsh-tool-fs-search` | `glob`, `grep` | `ctx.tools`, `ctx.subprocess`, `ctx.systemPrompt` | `tool/call`, `tool/result` | - | glob and grep are unconditional discovery tools that spawn the packaged ripgrep binary (`@vscode/ripgrep`) through ctx.subprocess as ordinary foreground calls (never background jobs) — no host `rg` install and no shell layer. The catalog uses `sampleOverCapGlobResults: true`; deployments must choose that behavior explicitly. Capped results save the complete formatted list through the optional ctx.spillStore backend; returned locators are follow-up-readable/searchable when the backend exposes local paths in co-located deployments. | | `@deepseek-ai/dsh-tool-terminal` | `terminal_close`, `terminal_list`, `terminal_open`, `terminal_read`, `terminal_send`, `terminal_signal` | `ctx.tools`, `ctx.terminals`, `ctx.systemPrompt`, `ctx.jobs at call time for run_in_background` | `tool/call`, `tool/result` | - | The six terminal tools are opt-in and complement one-shot shell/filesystem tools. `terminal_send(run_in_background: true)` registers with `ctx.jobs`; TUI, named key sequences, BEL, resize, auto-start, and cross-agent sharing are absent from the schema. | | `@deepseek-ai/dsh-tool-goal` | `create_goal`, `get_goal`, `update_goal` | `ctx.tools`, `ctx.agents`, `ctx.goals`, `ctx.systemPrompt`, `a calling Agent in an authorized open turn` | `tool/call`, `goal/change for mutations`, `tool/result` | - | create, edit, pause, and resume require direct-human root authority; complete and blocked also accept the exact current goal round. The default blocked lower bound is three admitted rounds. | @@ -695,7 +695,7 @@ Source: [`packages/fs/tool-fs/src/index.ts`](../packages/fs/tool-fs/src/index.ts ### `read_image` -Read a PNG/JPEG/WebP/GIF file and return the image itself. Requires the current model to accept image input. +Read a PNG/JPEG/WebP/GIF file and return the image itself. Harness validates and downscales large supported images before the next model request, so use this tool directly instead of installing image libraries or creating thumbnails merely to inspect an image. Independent files may be read concurrently in small batches. Requires the current model to accept image input. ```json { @@ -714,57 +714,6 @@ Read a PNG/JPEG/WebP/GIF file and return the image itself. Requires the current Source: [`packages/fs/tool-fs/src/index.ts`](../packages/fs/tool-fs/src/index.ts) -### `read_image_region` - -Crop a region from an image attachment already visible in this session. Coordinates use the preview dimensions supplied beside that image. - -```json -{ - "type": "object", - "properties": { - "attachment_id": { - "type": "string", - "description": "Complete attachment id shown beside the image." - }, - "preview_width": { - "type": "integer", - "description": "Width of the preview shown to the model." - }, - "preview_height": { - "type": "integer", - "description": "Height of the preview shown to the model." - }, - "x": { - "type": "integer", - "description": "Left edge in preview pixels." - }, - "y": { - "type": "integer", - "description": "Top edge in preview pixels." - }, - "width": { - "type": "integer", - "description": "Crop width in preview pixels." - }, - "height": { - "type": "integer", - "description": "Crop height in preview pixels." - } - }, - "required": [ - "attachment_id", - "preview_width", - "preview_height", - "x", - "y", - "width", - "height" - ] -} -``` - -Source: [`packages/fs/tool-fs/src/index.ts`](../packages/fs/tool-fs/src/index.ts) - ### `write` Create or fully replace a UTF-8 text file. @@ -791,7 +740,7 @@ Create or fully replace a UTF-8 text file. Source: [`packages/fs/tool-fs/src/index.ts`](../packages/fs/tool-fs/src/index.ts) -The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-observation-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. The image tools are not registered without `ctx.attachments`; their schemas are route-independent, and execution refuses unless the exact routed model declares image input. +The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-observation-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. The image tool is not registered without `ctx.attachments`; its schema is route-independent, and execution refuses unless the exact routed model declares image input. diff --git a/docs/tool-catalog.zh.md b/docs/tool-catalog.zh.md index f29d489441..c3209e7007 100644 --- a/docs/tool-catalog.zh.md +++ b/docs/tool-catalog.zh.md @@ -28,7 +28,7 @@ | `@deepseek-ai/dsh-tool-bash-persistent` | `bash` | `ctx.tools`、`ctx.terminals`、`an owning Agent at execution time` | `tool/call`、`PTY shell state`、`tool/result` | - | 一个按所有者隔离的持久 bash 工具;部署组合提供 PTY 后端,并可覆盖面向模型的环境描述。 | | `@deepseek-ai/dsh-tool-pwsh-persistent` | `pwsh` | `ctx.tools`、`ctx.terminals`、`an owning Agent at execution time` | `tool/call`、`PTY shell state`、`tool/result` | - | 一个按所有者隔离的持久 pwsh 工具,持久 bash 工具的 Windows 对应物;部署组合提供 pwsh 方言的 PTY 后端,并可覆盖面向模型的环境描述。 | | `@deepseek-ai/dsh-tool-str-replace-editor` | `str_replace_editor` | `ctx.tools`、`ctx.fs` | `tool/call`、`fs/observed after view presence/absence, edit absence, or successful mutation`、`tool/result` | - | 基于文件系统 seam 的独立查看/创建/唯一字面量替换/按行插入工具;可与任何 shell 或终端接口组合。 | -| `@deepseek-ai/dsh-tool-fs` | `edit`、`read`、`read_image`、`read_image_region`、`write` | `ctx.tools`、`ctx.fs`、`ctx.systemPrompt`、`ctx.attachments (image-tool registration)`、`ctx.llm + an image-capable route (image-tool execution)` | `tool/call`、`fs/write-intent or fs/edit-intent for mutations`、`fs/observed after read presence/absence or successful file operation`、`durable attachment (read_image and read_image_region)`、`tool/result` | - | 先读后写/编辑策略由 `@deepseek-ai/dsh-fs-observation-policy` 添加;它是一个 `fs/*` 事件门禁插件,不会改变 schema。加载这些工具的部署按预期也应加载该插件。没有 `ctx.attachments` 时图片工具不会注册;其 schema 与路由无关,执行时除非确切路由的模型声明图片输入,否则拒绝。 | +| `@deepseek-ai/dsh-tool-fs` | `edit`、`read`、`read_image`、`write` | `ctx.tools`、`ctx.fs`、`ctx.systemPrompt`、`ctx.attachments (image-tool registration)`、`ctx.llm + an image-capable route (image-tool execution)` | `tool/call`、`fs/write-intent or fs/edit-intent for mutations`、`fs/observed after read presence/absence or successful file operation`、`durable attachment (read_image)`、`tool/result` | - | 先读后写/编辑策略由 `@deepseek-ai/dsh-fs-observation-policy` 添加;它是一个 `fs/*` 事件门禁插件,不会改变 schema。加载这些工具的部署按预期也应加载该插件。没有 `ctx.attachments` 时图片工具不会注册;其 schema 与路由无关,执行时除非确切路由的模型声明图片输入,否则拒绝。 | | `@deepseek-ai/dsh-tool-fs-search` | `glob`、`grep` | `ctx.tools`、`ctx.subprocess`、`ctx.systemPrompt` | `tool/call`、`tool/result` | - | glob 和 grep 是无条件可用的发现工具,通过 ctx.subprocess spawn 随包提供的 ripgrep 二进制文件(`@vscode/ripgrep`),并作为普通前台调用运行,绝不作为后台任务;无需在宿主机安装 `rg`,也不经过 shell 层。本目录使用 `sampleOverCapGlobResults: true`;部署必须显式选择该行为。结果超过上限时,会通过可选的 ctx.spillStore 后端保存完整的格式化列表;在共置部署中,如果后端公开本地路径,返回的定位信息可供后续读取/搜索。 | | `@deepseek-ai/dsh-tool-terminal` | `terminal_close`、`terminal_list`、`terminal_open`、`terminal_read`、`terminal_send`、`terminal_signal` | `ctx.tools`、`ctx.terminals`、`ctx.systemPrompt`、`ctx.jobs at call time for run_in_background` | `tool/call`、`tool/result` | - | 这 6 个终端工具需要选择启用,用于补充一次性 bash/文件系统工具。`terminal_send(run_in_background: true)` 会注册到 `ctx.jobs`;schema 不包含 TUI、具名按键序列、BEL、调整尺寸、自动启动和跨 agent 共享。 | | `@deepseek-ai/dsh-tool-goal` | `create_goal`、`get_goal`、`update_goal` | `ctx.tools`、`ctx.agents`、`ctx.goals`、`ctx.systemPrompt`、`a calling Agent in an authorized open turn` | `tool/call`、`goal/change for mutations`、`tool/result` | - | create、edit、pause 和 resume 要求直接来自人类的根权限;complete 和 blocked 也接受确切的当前 Goal Round。blocked 的默认下限是 3 个获准的 Round。 | @@ -701,7 +701,7 @@ pwsh 工具是 Windows 组合中 bash 执行器 seam 的 PowerShell 方言消费 ### `read_image` -读取 PNG/JPEG/WebP/GIF 文件并返回图像本身。要求当前模型接受图像输入。 +读取 PNG/JPEG/WebP/GIF 文件并返回图像本身。Harness 会在下一次模型请求前校验并缩小受支持的大图,因此仅为查看图片时应直接使用此工具,无需安装图片库或创建缩略图。可以用小批次并发读取彼此独立的文件。要求当前模型接受图像输入。 ```json { @@ -720,57 +720,6 @@ pwsh 工具是 Windows 组合中 bash 执行器 seam 的 PowerShell 方言消费 来源:[`packages/fs/tool-fs/src/index.ts`](../packages/fs/tool-fs/src/index.ts) -### `read_image_region` - -裁剪当前会话中模型已经可见的图片附件。坐标采用该图片旁给出的预览尺寸。 - -```json -{ - "type": "object", - "properties": { - "attachment_id": { - "type": "string", - "description": "Complete attachment id shown beside the image." - }, - "preview_width": { - "type": "integer", - "description": "Width of the preview shown to the model." - }, - "preview_height": { - "type": "integer", - "description": "Height of the preview shown to the model." - }, - "x": { - "type": "integer", - "description": "Left edge in preview pixels." - }, - "y": { - "type": "integer", - "description": "Top edge in preview pixels." - }, - "width": { - "type": "integer", - "description": "Crop width in preview pixels." - }, - "height": { - "type": "integer", - "description": "Crop height in preview pixels." - } - }, - "required": [ - "attachment_id", - "preview_width", - "preview_height", - "x", - "y", - "width", - "height" - ] -} -``` - -来源:[`packages/fs/tool-fs/src/index.ts`](../packages/fs/tool-fs/src/index.ts) - ### `write` 创建或完全替换 UTF-8 文本文件。 diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index 8da7ac71b1..548b4025a4 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -807,8 +807,7 @@ it('pins native DeepSeek Files image offload in the request sent by the assemble { type: 'text', text: '\nImage sha256:b1ff9c8ea3a780bad09b346c423d2d0e46815926879b18e841d928376a946640; ' - + 'preview 1x1px. Crop coordinates use this preview. Call read_image_region with this attachment_id, ' - + 'preview_width=1, preview_height=1, x, y, width, and height.', + + 'request image 1x1px.', }, { type: 'file', file_id: 'file-api-snapshot-1' }, { type: 'text', text: ', then use read_image on red.png and reply with DONE.' }, @@ -852,9 +851,7 @@ it('pins native DeepSeek Files image offload in the request sent by the assemble role: 'tool', tool_call_id: 'native-read-image', content: '{{cwd}}/red.png\nimage\n\nimage/png image, 1x1 px, 69 bytes\n' - + '\nImage sha256:b1ff9c8ea3a780bad09b346c423d2d0e46815926879b18e841d928376a946640; preview 1x1px. ' - + 'Crop coordinates use this preview. Call read_image_region with this attachment_id, preview_width=1, ' - + 'preview_height=1, x, y, width, and height.', + + '\nImage sha256:b1ff9c8ea3a780bad09b346c423d2d0e46815926879b18e841d928376a946640; request image 1x1px.', }, { role: 'user', diff --git a/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md b/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md index 7408ddb329..678de3e53f 100644 --- a/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md +++ b/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md @@ -125,28 +125,11 @@ interface ToolArgsMap { /** Maximum number of lines to return. Defaults to 2000. */ limit?: number; } & Record; - /** Read a PNG/JPEG/WebP/GIF file and return the image itself. Requires the current model to accept image input. */ + /** Read a PNG/JPEG/WebP/GIF file and return the image itself. Harness validates and downscales large supported images before the next model request, so use this tool directly instead of installing image libraries or creating thumbnails merely to inspect an image. Independent files may be read concurrently in small batches. Requires the current model to accept image input. */ read_image: { /** Path to the image file, resolved by the filesystem backend. */ file_path: string; } & Record; - /** Crop a region from an image attachment already visible in this session. Coordinates use the preview dimensions supplied beside that image. */ - read_image_region: { - /** Complete attachment id shown beside the image. */ - attachment_id: string; - /** Width of the preview shown to the model. */ - preview_width: number; - /** Height of the preview shown to the model. */ - preview_height: number; - /** Left edge in preview pixels. */ - x: number; - /** Top edge in preview pixels. */ - y: number; - /** Crop width in preview pixels. */ - width: number; - /** Crop height in preview pixels. */ - height: number; - } & Record; /** Send a message to a background subagent by its subagent id, continuing the same conversation. It becomes the subagent's next turn: if it is still working, the message waits until its current turn finishes, so it cannot redirect work already underway. This call returns no answer from the subagent — only confirmation that the message was delivered — so use it to give it more work. A failure means the message was NOT delivered. */ send_message: { /** The subagent id returned when the background subagent was started. */ @@ -384,29 +367,6 @@ interface ToolOutputMap { sourceHeight?: number; }; }; - read_image_region: { - sourceAttachmentId: string; - preview: { - width: number; - height: number; - }; - crop: { - x: number; - y: number; - width: number; - height: number; - }; - image: { - attachmentId: string; - mediaType: "image/png" | "image/jpeg" | "image/webp" | "image/gif"; - bytes: number; - width: number; - height: number; - name?: string; - sourceWidth?: number; - sourceHeight?: number; - }; - }; send_message: { messageId: string; }; diff --git a/examples/acp-agent/tests/snapshots/read-image/tool-schemas.expected.json b/examples/acp-agent/tests/snapshots/read-image/tool-schemas.expected.json index dec4bd85ab..fa8862c09a 100644 --- a/examples/acp-agent/tests/snapshots/read-image/tool-schemas.expected.json +++ b/examples/acp-agent/tests/snapshots/read-image/tool-schemas.expected.json @@ -246,7 +246,7 @@ }, { "name": "read_image", - "description": "Read a PNG/JPEG/WebP/GIF file and return the image itself. Requires the current model to accept image input.", + "description": "Read a PNG/JPEG/WebP/GIF file and return the image itself. Harness validates and downscales large supported images before the next model request, so use this tool directly instead of installing image libraries or creating thumbnails merely to inspect an image. Independent files may be read concurrently in small batches. Requires the current model to accept image input.", "parameters": { "type": "object", "properties": { @@ -260,52 +260,6 @@ ] } }, - { - "name": "read_image_region", - "description": "Crop a region from an image attachment already visible in this session. Coordinates use the preview dimensions supplied beside that image.", - "parameters": { - "type": "object", - "properties": { - "attachment_id": { - "type": "string", - "description": "Complete attachment id shown beside the image." - }, - "preview_width": { - "type": "integer", - "description": "Width of the preview shown to the model." - }, - "preview_height": { - "type": "integer", - "description": "Height of the preview shown to the model." - }, - "x": { - "type": "integer", - "description": "Left edge in preview pixels." - }, - "y": { - "type": "integer", - "description": "Top edge in preview pixels." - }, - "width": { - "type": "integer", - "description": "Crop width in preview pixels." - }, - "height": { - "type": "integer", - "description": "Crop height in preview pixels." - } - }, - "required": [ - "attachment_id", - "preview_width", - "preview_height", - "x", - "y", - "width", - "height" - ] - } - }, { "name": "send_message", "description": "Send a message to a background subagent by its subagent id, continuing the same conversation. It becomes the subagent's next turn: if it is still working, the message waits until its current turn finishes, so it cannot redirect work already underway. This call returns no answer from the subagent — only confirmation that the message was delivered — so use it to give it more work. A failure means the message was NOT delivered.", diff --git a/packages/attachment/attachment-local/README.i18n.yaml b/packages/attachment/attachment-local/README.i18n.yaml index d15a1fd01e..a8bfa1b322 100644 --- a/packages/attachment/attachment-local/README.i18n.yaml +++ b/packages/attachment/attachment-local/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/attachment/attachment-local/README.md -README.md: 6141b7559492aa4c50831c8124a917bfdb704f4b -README.zh.md: 2a8ed6e1aef8022aba5053bf1ef0f9728340d086 +README.md: d4831f864dbb061319008242395e2c8ff6d9f642 +README.zh.md: 45bddf47ea5f68c15778040de5b29817e8f62956 diff --git a/packages/attachment/attachment-local/README.md b/packages/attachment/attachment-local/README.md index 6141b75594..d4831f864d 100644 --- a/packages/attachment/attachment-local/README.md +++ b/packages/attachment/attachment-local/README.md @@ -6,7 +6,7 @@ The private local implementation of [`@deepseek-ai/dsh-attachment`](../attachmen Admission accepts at most 20 images and 200MiB of encoded source bytes per message. Each source may use up to 20MiB, 64,000,000 pixels, and 8192px per side. It then prepares a provider-independent master. EXIF orientation is applied, metadata and color profiles are removed, pixels become 8-bit sRGB/sRGBA, and the long edge is reduced proportionally to `masterMaxDimension` (2048px by default). The master has its own `masterMaxBytes` safety cap (4MiB by default). Alpha is retained. A nearest-neighbour bounded sample classifies color complexity without averaging high-frequency pixels. Confirmed low-color images try PNG, using a palette only when the input has no alpha channel, then WebP at qualities 85, 80, and 75. Other alpha images try WebP at those qualities; other opaque images try JPEG. Each candidate runs only after the preceding candidate exceeds the cap. Dimensions shrink only after every candidate at one size exceeds the cap. A clean, single-frame 8-bit sRGB/sRGBA PNG, JPEG, or WebP already within both master limits passes through byte-identically; 16-bit PNG, GIF, animated input, metadata, orientation, and incompatible color spaces force conversion. The source and a converted master are each fully decoded once. `saveImages` prepares and verifies every master once before publishing the batch, so validation failure leaves no partial references and commit does not repeat full image encoding. -Request versions live below `/attachments/v1/request-images/`. `readImageRequest` scales the stored master under a total-pixel budget without enlargement, then enforces a separate encoded-byte cap. The request encoder uses the same color branches, with PNG (palette only without alpha) before WebP 85 and 80 for low-color images, WebP 85 then 80 for other alpha images, and JPEG 85 then 80 for other opaque images. It also executes candidates lazily and reduces dimensions only after both quality attempts exceed the request cap. Its cache identity includes the master id, transform version, pixel and byte budgets, optional master-coordinate crop, and fixed encoder settings. Cached bytes are fully decoded and checked as 8-bit sRGB/sRGBA before use. Concurrent calls for one identity share one transform and cache write; cancelling one waiter does not cancel the shared work. `readImageRequests` schedules batches through the service's FIFO limiter. `imageCompressionConcurrency` controls simultaneous master and request transforms from 1 through 8 and defaults to 2; file publication remains ordered after preparation. `cropImage` maps coordinates measured on a model preview back to the master, crops the master rather than the preview, and commits the crop as another durable attachment. +Request versions live below `/attachments/v1/request-images/`. `readImageRequest` scales the stored master under a total-pixel budget without enlargement, then enforces a separate encoded-byte cap. The request encoder uses the same color branches, with PNG (palette only without alpha) before WebP 85 and 80 for low-color images, WebP 85 then 80 for other alpha images, and JPEG 85 then 80 for other opaque images. It also executes candidates lazily and reduces dimensions only after both quality attempts exceed the request cap. Its cache identity includes the master id, transform version, pixel and byte budgets, and fixed encoder settings. Cached bytes are fully decoded and checked as 8-bit sRGB/sRGBA before use. Concurrent calls for one identity share one transform and cache write; cancelling one waiter does not cancel the shared work. `readImageRequests` schedules batches through the service's FIFO limiter. `imageCompressionConcurrency` controls simultaneous master and request transforms from 1 through 8 and defaults to 2; file publication remains ordered after preparation. `DSH_HOME` resolves through the shared path policy: explicit config, `$DSH_HOME`, then `~/.dsh`. Session logs contain only the reference and verified metadata, never this host path. `readImage` forwards optional cancellation into the filesystem read, observes it around verification, and preserves it instead of wrapping it as `ATTACHMENT_READ_FAILED`. diff --git a/packages/attachment/attachment-local/README.zh.md b/packages/attachment/attachment-local/README.zh.md index 2a8ed6e1ae..45bddf47ea 100644 --- a/packages/attachment/attachment-local/README.zh.md +++ b/packages/attachment/attachment-local/README.zh.md @@ -6,7 +6,7 @@ 每条消息最多准入 20 张图片,源图编码字节总量不超过 200MiB。每张源图不得超过 20MiB、64,000,000 像素和单边 8192px。随后生成提供方无关的主版本:应用 EXIF 方向,删除元数据和色彩配置文件,转换为 8-bit sRGB/sRGBA,并保持宽高比把长边限制到 `masterMaxDimension`(默认 2048px)。主版本有独立的 `masterMaxBytes` 安全上限(默认 4MiB)。透明通道会保留。系统用 nearest-neighbour 对有界样本分类,不会通过像素平均把高频图片误判为低色数。确认的低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80、75 的 WebP;其他透明图片依次尝试这些质量的 WebP;其他非透明图片依次尝试这些质量的 JPEG。只有前一个候选超限时才会执行下一个候选;同一尺寸的候选全部超限后才缩小尺寸。已经处于两个主版本上限内的干净、单帧、8-bit sRGB/sRGBA PNG、JPEG 或 WebP 按字节原样直通;16-bit PNG、GIF、动图、元数据、方向和不兼容色彩空间都会触发转换。源图和转换后的主版本各完整解码一次。`saveImages` 在发布任何批次成员前为每张图片各准备并验证一次主版本,因此校验失败不会留下部分引用,提交阶段也不会重复执行完整图片编码。 -请求版本保存在 `/attachments/v1/request-images/`。`readImageRequest` 在不放大小图的前提下,把存储的主版本缩放到总像素预算内,再执行独立的编码字节上限。请求编码器使用同一分类分支:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。候选仍按需执行,两个质量档均超限后才缩小尺寸。缓存身份包含主版本 ID、变换策略版本、像素和字节预算、可选的主版本坐标裁剪区域以及固定编码参数。缓存字节在使用前会完整解码并校验为 8-bit sRGB/sRGBA。同一身份的并发调用共享一次变换和缓存写入;取消一个等待方不会取消共享任务。`readImageRequests` 通过服务的 FIFO 限流器调度批次。`imageCompressionConcurrency` 控制同时执行的主版本和请求版本变换,范围为 1 至 8,默认值为 2;文件发布仍在准备结束后按顺序执行。`cropImage` 把模型在预览图上测得的坐标映射回主版本,从主版本而非预览图裁剪,并把裁剪结果提交为另一个持久附件。 +请求版本保存在 `/attachments/v1/request-images/`。`readImageRequest` 在不放大小图的前提下,把存储的主版本缩放到总像素预算内,再执行独立的编码字节上限。请求编码器使用同一分类分支:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。候选仍按需执行,两个质量档均超限后才缩小尺寸。缓存身份包含主版本 ID、变换策略版本、像素和字节预算及固定编码参数。缓存字节在使用前会完整解码并校验为 8-bit sRGB/sRGBA。同一身份的并发调用共享一次变换和缓存写入;取消一个等待方不会取消共享任务。`readImageRequests` 通过服务的 FIFO 限流器调度批次。`imageCompressionConcurrency` 控制同时执行的主版本和请求版本变换,范围为 1 至 8,默认值为 2;文件发布仍在准备结束后按顺序执行。 `DSH_HOME` 按共享路径策略解析:显式配置、`$DSH_HOME`,最后是 `~/.dsh`。会话日志只包含引用和经过校验的元数据,绝不包含这个宿主路径。`readImage` 会把可选取消信号传入文件系统读取、在校验前后观察该信号,并保留取消语义,而不会将其包装成 `ATTACHMENT_READ_FAILED`。 diff --git a/packages/attachment/attachment-local/src/index.ts b/packages/attachment/attachment-local/src/index.ts index 4fb200345b..9007544047 100644 --- a/packages/attachment/attachment-local/src/index.ts +++ b/packages/attachment/attachment-local/src/index.ts @@ -8,7 +8,6 @@ import type { ImageAttachmentLimits, ImageAttachmentRef, ImageRequestPolicy, - PreviewImageCrop, RequestImageAttachment, SaveImageAttachment, SavedImageAttachment, @@ -18,13 +17,13 @@ import { resolveDshHome } from '@deepseek-ai/dsh-home-paths' import type { MasterImagePolicy } from './canonical.ts' import { CompressionLimiter } from './compression-limiter.ts' import { commitPreparedImageFile, prepareImageFile, readImageFile, validateImageFile } from './store.ts' -import { previewCropToMaster, readRequestImageFile, requestImageVariantId } from './request-image.ts' +import { readRequestImageFile, requestImageVariantId } from './request-image.ts' export { isMasterImage, prepareMasterImage } from './canonical.ts' export type { MasterImage, MasterImagePolicy } from './canonical.ts' export { commitPreparedImageFile, prepareImageFile, readImageFile, saveImageFile, validateImageFile } from './store.ts' export type { PreparedImageFile } from './store.ts' -export { previewCropToMaster, readRequestImageFile, requestImageDimensions, requestImageVariantId } from './request-image.ts' +export { readRequestImageFile, requestImageDimensions, requestImageVariantId } from './request-image.ts' /** Default maximum encoded bytes for one submitted image; oversized sources are refused, not shrunk. */ export const DEFAULT_MAX_IMAGE_BYTES = 20 * 1024 * 1024 @@ -255,26 +254,6 @@ export class LocalAttachmentStore extends AttachmentStore { return operation.wait(signal) } - override async cropImage( - ref: ImageAttachmentRef, - crop: PreviewImageCrop, - signal?: AbortSignal, - ): Promise { - const master = await this.readImage(ref, signal) - const region = previewCropToMaster(ref.width, ref.height, crop) - const version = await this.requestVersion(ref, { - maxPixels: region.width * region.height, - maxBytes: this.masterPolicy.maxBytes, - crop: region, - }, master, signal) - signal?.throwIfAborted() - const stem = ref.name?.replace(/\.[^.]+$/u, '') ?? String(ref.attachmentId).slice(0, 15) - return this.saveImage({ - data: version.data, - mediaType: version.mediaType, - name: `${stem}-crop.${version.mediaType.slice('image/'.length).replace('jpeg', 'jpg')}`, - }) - } } export default LocalAttachmentStore diff --git a/packages/attachment/attachment-local/src/request-image.ts b/packages/attachment/attachment-local/src/request-image.ts index ef7c841bed..c37a473d4c 100644 --- a/packages/attachment/attachment-local/src/request-image.ts +++ b/packages/attachment/attachment-local/src/request-image.ts @@ -1,4 +1,4 @@ -/** Deterministic cached image versions for model requests and region reads. */ +/** Deterministic cached image versions for model requests. */ import { createHash, randomUUID } from 'node:crypto' import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises' @@ -9,8 +9,6 @@ import type { ImageMediaType, ImageAttachmentRef, ImageRequestPolicy, - MasterImageCrop, - PreviewImageCrop, RequestImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' @@ -80,22 +78,6 @@ function checkedInteger(value: number, name: string): number { function validatePolicy(policy: ImageRequestPolicy): void { checkedInteger(policy.maxPixels, 'Image request maxPixels') checkedInteger(policy.maxBytes, 'Image request maxBytes') - if (policy.crop !== undefined) { - if (!Number.isSafeInteger(policy.crop.x) || policy.crop.x < 0 - || !Number.isSafeInteger(policy.crop.y) || policy.crop.y < 0) { - throw new AttachmentError('Image crop origin must use non-negative integer pixels.', 'INVALID_ATTACHMENT_REF') - } - checkedInteger(policy.crop.width, 'Image crop width') - checkedInteger(policy.crop.height, 'Image crop height') - } -} - -function checkedCrop(master: StoredImageAttachment, crop: MasterImageCrop | undefined): MasterImageCrop | undefined { - if (crop === undefined) return undefined - if (crop.x + crop.width > master.ref.width || crop.y + crop.height > master.ref.height) { - throw new AttachmentError('Image crop extends beyond the stored master image.', 'INVALID_ATTACHMENT_REF') - } - return crop } function descriptor(master: ImageAttachmentRef, policy: ImageRequestPolicy): string { @@ -104,7 +86,6 @@ function descriptor(master: ImageAttachmentRef, policy: ImageRequestPolicy): str masterAttachmentId: master.attachmentId, routePixelBudget: policy.maxPixels, encodedByteBudget: policy.maxBytes, - crop: policy.crop ?? null, encoding: { png: { compressionLevel: 9, palette: 'opaque-only' }, webpQualities: REQUEST_IMAGE_QUALITIES, @@ -118,7 +99,7 @@ function descriptor(master: ImageAttachmentRef, policy: ImageRequestPolicy): str /** * Complete deterministic identity for one master and route-owned request policy. * @param master - provider-independent durable master reference. - * @param policy - route-owned pixel, byte, and crop policy. + * @param policy - route-owned pixel and byte policy. * @returns branded digest over every request transform input. */ export function requestImageVariantId( @@ -128,20 +109,13 @@ export function requestImageVariantId( return ImageVariantId(`sha256:${digest(descriptor(master, policy))}`) } -function pipeline(master: StoredImageAttachment, crop: MasterImageCrop | undefined, width: number, height: number): Sharp { - return sourcePipeline(master, crop) +function pipeline(master: StoredImageAttachment, width: number, height: number): Sharp { + return sourcePipeline(master) .resize({ width, height, fit: 'inside', withoutEnlargement: true }) } -function sourcePipeline(master: StoredImageAttachment, crop: MasterImageCrop | undefined): Sharp { - let image = sharp(master.data, { failOn: 'error', limitInputPixels: false }).toColourspace('srgb') - if (crop !== undefined) image = image.extract({ - left: crop.x, - top: crop.y, - width: crop.width, - height: crop.height, - }) - return image +function sourcePipeline(master: StoredImageAttachment): Sharp { + return sharp(master.data, { failOn: 'error', limitInputPixels: false }).toColourspace('srgb') } async function encoded( @@ -161,13 +135,12 @@ async function encoded( function encodingAttempts( master: StoredImageAttachment, - crop: MasterImageCrop | undefined, width: number, height: number, hasAlpha: boolean, lowColour: boolean, ): Array<() => Promise> { - const prepared = pipeline(master, crop, width, height) + const prepared = pipeline(master, width, height) const webp = REQUEST_IMAGE_QUALITIES.map(quality => ( () => encoded(prepared.clone(), 'image/webp', quality) )) @@ -183,12 +156,8 @@ async function createRequestImage( policy: ImageRequestPolicy, hasAlpha: boolean, ): Promise { - const crop = checkedCrop(master, policy.crop) - const sourceWidth = crop?.width ?? master.ref.width - const sourceHeight = crop?.height ?? master.ref.height - let dimensions = requestImageDimensions(sourceWidth, sourceHeight, policy.maxPixels) - if (crop === undefined - && dimensions.width === master.ref.width + let dimensions = requestImageDimensions(master.ref.width, master.ref.height, policy.maxPixels) + if (dimensions.width === master.ref.width && dimensions.height === master.ref.height && master.data.byteLength <= policy.maxBytes) { return { @@ -198,10 +167,10 @@ async function createRequestImage( height: master.ref.height, } } - const lowColour = await hasLowColourCount(sourcePipeline(master, crop)) + const lowColour = await hasLowColourCount(sourcePipeline(master)) for (;;) { const encodedVersion = await encodeFirstWithinLimit( - encodingAttempts(master, crop, dimensions.width, dimensions.height, hasAlpha, lowColour), + encodingAttempts(master, dimensions.width, dimensions.height, hasAlpha, lowColour), policy.maxBytes, ) if (!isExhaustedEncoding(encodedVersion)) return encodedVersion @@ -229,8 +198,7 @@ async function readCached( try { const data = new Uint8Array(await readFile(path, { signal })) const detected = await probeImage(data) - const crop = policy.crop - const maximum = requestImageDimensions(crop?.width ?? master.ref.width, crop?.height ?? master.ref.height, policy.maxPixels) + const maximum = requestImageDimensions(master.ref.width, master.ref.height, policy.maxPixels) if (data.byteLength > policy.maxBytes || detected.depth !== 'uchar' || detected.space !== 'srgb' || detected.width > maximum.width || detected.height > maximum.height || detected.hasAlpha !== expectedAlpha) return undefined @@ -285,7 +253,6 @@ export async function readRequestImageFile( ): Promise { signal?.throwIfAborted() validatePolicy(policy) - checkedCrop(master, policy.crop) const source = await probeImage(master.data) const variantId = requestImageVariantId(master.ref, policy) const hash = String(variantId).slice('sha256:'.length) @@ -308,42 +275,5 @@ export async function readRequestImageFile( depth: 'uchar', space: 'srgb', hasAlpha: version.hasAlpha, - ...policy.crop === undefined ? {} : { crop: policy.crop }, - } -} - -/** - * Map a preview-coordinate rectangle to the oriented stored master. - * @param masterWidth - stored master width. - * @param masterHeight - stored master height. - * @param crop - rectangle measured on the model-visible preview. - * @returns covering integer rectangle in master coordinates. - */ -export function previewCropToMaster( - masterWidth: number, - masterHeight: number, - crop: PreviewImageCrop, -): MasterImageCrop { - checkedInteger(masterWidth, 'Master image width') - checkedInteger(masterHeight, 'Master image height') - checkedInteger(crop.previewWidth, 'Preview width') - checkedInteger(crop.previewHeight, 'Preview height') - if (!Number.isSafeInteger(crop.x) || crop.x < 0 || !Number.isSafeInteger(crop.y) || crop.y < 0) { - throw new AttachmentError('Preview crop origin must use non-negative integer pixels.', 'INVALID_ATTACHMENT_REF') - } - checkedInteger(crop.width, 'Preview crop width') - checkedInteger(crop.height, 'Preview crop height') - if (crop.x + crop.width > crop.previewWidth || crop.y + crop.height > crop.previewHeight) { - throw new AttachmentError('Preview crop extends beyond the image shown to the model.', 'INVALID_ATTACHMENT_REF') - } - const x = Math.floor(crop.x * masterWidth / crop.previewWidth) - const y = Math.floor(crop.y * masterHeight / crop.previewHeight) - const right = Math.ceil((crop.x + crop.width) * masterWidth / crop.previewWidth) - const bottom = Math.ceil((crop.y + crop.height) * masterHeight / crop.previewHeight) - return { - x, - y, - width: Math.max(1, Math.min(masterWidth, right) - x), - height: Math.max(1, Math.min(masterHeight, bottom) - y), } } diff --git a/packages/attachment/attachment-local/tests/request-image.spec.ts b/packages/attachment/attachment-local/tests/request-image.spec.ts index 726837a242..c38e8ce137 100644 --- a/packages/attachment/attachment-local/tests/request-image.spec.ts +++ b/packages/attachment/attachment-local/tests/request-image.spec.ts @@ -5,7 +5,7 @@ import { Context } from '@deepseek-ai/cordis' import sharp from 'sharp' import { afterEach, describe, expect, it, vi } from 'vitest' import { CompressionLimiter } from '../src/compression-limiter.ts' -import LocalAttachmentStore, { previewCropToMaster, requestImageDimensions } from '../src/index.ts' +import LocalAttachmentStore, { requestImageDimensions } from '../src/index.ts' const homes: string[] = [] @@ -51,23 +51,6 @@ describe('request image dimensions', () => { expect(requestImageDimensions(2, 4, 5)).toEqual({ width: 1, height: 2 }) }) - it('rejects invalid preview dimensions, origins, sizes, and bounds', () => { - expect(() => previewCropToMaster(0, 10, { - previewWidth: 10, previewHeight: 10, x: 0, y: 0, width: 1, height: 1, - })).toThrow('Master image width must be a positive integer') - expect(() => previewCropToMaster(10, 10, { - previewWidth: 0, previewHeight: 10, x: 0, y: 0, width: 1, height: 1, - })).toThrow('Preview width must be a positive integer') - expect(() => previewCropToMaster(10, 10, { - previewWidth: 10, previewHeight: 10, x: -1, y: 0, width: 1, height: 1, - })).toThrow('Preview crop origin must use non-negative integer pixels') - expect(() => previewCropToMaster(10, 10, { - previewWidth: 10, previewHeight: 10, x: 0, y: 0, width: 0, height: 1, - })).toThrow('Preview crop width must be a positive integer') - expect(() => previewCropToMaster(10, 10, { - previewWidth: 10, previewHeight: 10, x: 9, y: 0, width: 2, height: 1, - })).toThrow('Preview crop extends beyond the image shown to the model') - }) }) describe('local request-image cache', () => { @@ -85,7 +68,7 @@ describe('local request-image cache', () => { expect(batch.map(value => value.master.attachmentId)).toEqual([first.attachmentId, second.attachmentId]) }) - it('rejects invalid request policies and master crop bounds', async () => { + it('rejects invalid request policies', async () => { const attachments = await store() const master = (await attachments.saveImage({ data: await image(8, 4), mediaType: 'image/png' })).ref @@ -93,15 +76,6 @@ describe('local request-image cache', () => { .rejects.toThrow('Image request maxPixels must be a positive integer') await expect(attachments.readImageRequest(master, { maxPixels: 100, maxBytes: 0 })) .rejects.toThrow('Image request maxBytes must be a positive integer') - await expect(attachments.readImageRequest(master, { - maxPixels: 100, maxBytes: 100, crop: { x: -1, y: 0, width: 1, height: 1 }, - })).rejects.toThrow('Image crop origin must use non-negative integer pixels') - await expect(attachments.readImageRequest(master, { - maxPixels: 100, maxBytes: 100, crop: { x: 0, y: 0, width: 0, height: 1 }, - })).rejects.toThrow('Image crop width must be a positive integer') - await expect(attachments.readImageRequest(master, { - maxPixels: 100, maxBytes: 100, crop: { x: 7, y: 0, width: 2, height: 1 }, - })).rejects.toThrow('Image crop extends beyond the stored master image') }) it('refuses a one-pixel request that cannot meet the encoded-byte budget', async () => { @@ -178,51 +152,6 @@ describe('local request-image cache', () => { expect(low.width * low.height).toBeLessThanOrEqual(512 * 512 + low.width) }) - it('maps preview coordinates to the 2048px master and crops the master instead of the preview', async () => { - const attachments = await store() - const pixels = Buffer.alloc(2048 * 1024 * 3) - for (let y = 0; y < 1024; y += 1) { - for (let x = 0; x < 2048; x += 1) { - const offset = (y * 2048 + x) * 3 - pixels[offset] = x < 1024 ? 255 : 0 - pixels[offset + 1] = x < 1024 ? 0 : 255 - pixels[offset + 2] = 0 - } - } - const source = new Uint8Array(await sharp(pixels, { raw: { width: 2048, height: 1024, channels: 3 } }).png().toBuffer()) - const master = (await attachments.saveImage({ data: source, mediaType: 'image/png', name: 'halves.png' })).ref - const preview = await attachments.readImageRequest(master, { maxPixels: 640_000, maxBytes: 1024 * 1024 }) - const previewCrop = { - previewWidth: preview.width, - previewHeight: preview.height, - x: Math.floor(preview.width / 2), - y: 0, - width: preview.width - Math.floor(preview.width / 2), - height: preview.height, - } - const mapped = previewCropToMaster(master.width, master.height, previewCrop) - - const cropped = await attachments.cropImage(master, previewCrop) - const stored = await attachments.readImage(cropped.ref) - const pixel = await sharp(stored.data).resize(1, 1).removeAlpha().raw().toBuffer() - - expect(mapped).toEqual({ x: 1024, y: 0, width: 1024, height: 1024 }) - expect(cropped.ref.width).toBe(mapped.width) - expect(cropped.ref.height).toBe(mapped.height) - expect(pixel[1]).toBeGreaterThan(pixel[0] ?? 0) - }) - - it('names a crop from an unnamed attachment id', async () => { - const attachments = await store() - const master = (await attachments.saveImage({ data: await image(8, 4), mediaType: 'image/png' })).ref - - const cropped = await attachments.cropImage(master, { - previewWidth: 8, previewHeight: 4, x: 0, y: 0, width: 4, height: 4, - }) - - expect(cropped.ref.name).toMatch(/^sha256:[0-9a-f]{8}-crop\.(?:png|webp|jpg)$/u) - }) - it('classifies opaque PNG pixels and preserves alpha while enforcing the request budget', async () => { const attachments = await store() const side = 256 diff --git a/packages/attachment/attachment/README.i18n.yaml b/packages/attachment/attachment/README.i18n.yaml index 221699165d..bbccf584c6 100644 --- a/packages/attachment/attachment/README.i18n.yaml +++ b/packages/attachment/attachment/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/attachment/attachment/README.md -README.md: c4925addf079cdd65defb733e6bc40f91ed6384f -README.zh.md: 5623e0944c6f67e2cdaa90076d794cd617c46d5f +README.md: 66ce5f308cfa1ce6a028dbd248ceef1fdcc31a7c +README.zh.md: 4470956987330a451e3717d419a111def98dd6cb diff --git a/packages/attachment/attachment/README.md b/packages/attachment/attachment/README.md index c4925addf0..66ce5f308c 100644 --- a/packages/attachment/attachment/README.md +++ b/packages/attachment/attachment/README.md @@ -4,13 +4,13 @@ English | [中文](README.zh.md) The durable attachment seam. `ctx.attachments` validates and durably commits a provider-independent master image, then returns a serializable `ImageAttachmentRef`; consumers never persist browser paths, object URLs, provider URLs, or base64 in session events. -Unsent composer images remain browser-owned temporary drafts. `validateImage` runs the complete admission policy without persisting. `saveImages` owns batch count and aggregate-byte limits, prepares every validated master once before publishing any member, then commits in order and returns references only after the complete batch succeeds. A later storage failure returns no partial references, although an earlier immutable content-addressed object may remain unreachable until reference-aware garbage collection exists. `AttachmentError.code` uses the closed `AttachmentErrorCode` string union. Its `ImageAdmissionErrorCode` subset marks caller-correctable image-input failures; `isImageAdmissionError` recognizes that subset at runtime so each protocol adapter can map its own error vocabulary. `saveImage` commits one accepted image before any model-visible session event is published and resolves `SavedImageAttachment`: the returned `ref` describes the stored master while `source` (`SourceImageInfo`) preserves the submitted raster's media type, byte length, and orientation-applied dimensions. `readImage` verifies that master against its logged metadata. `readImageRequest` deterministically derives a route-sized request version whose identity covers the master id, transform version, pixel and byte budgets, crop, and encoder settings; `readImageRequests` preserves ordered results while implementations apply their own bounded concurrency. `cropImage` maps preview coordinates to the stored master and persists the result as a new attachment. Callers may cancel reads and projections; implementations preserve cancellation instead of translating it into a storage failure. +Unsent composer images remain browser-owned temporary drafts. `validateImage` runs the complete admission policy without persisting. `saveImages` owns batch count and aggregate-byte limits, prepares every validated master once before publishing any member, then commits in order and returns references only after the complete batch succeeds. A later storage failure returns no partial references, although an earlier immutable content-addressed object may remain unreachable until reference-aware garbage collection exists. `AttachmentError.code` uses the closed `AttachmentErrorCode` string union. Its `ImageAdmissionErrorCode` subset marks caller-correctable image-input failures; `isImageAdmissionError` recognizes that subset at runtime so each protocol adapter can map its own error vocabulary. `saveImage` commits one accepted image before any model-visible session event is published and resolves `SavedImageAttachment`: the returned `ref` describes the stored master while `source` (`SourceImageInfo`) preserves the submitted raster's media type, byte length, and orientation-applied dimensions. `readImage` verifies that master against its logged metadata. `readImageRequest` deterministically derives a route-sized request version whose identity covers the master id, transform version, pixel and byte budgets, and encoder settings; `readImageRequests` preserves ordered results while implementations apply their own bounded concurrency. Callers may cancel reads and projections; implementations preserve cancellation instead of translating it into a storage failure. `admitEncodedImages(attachments, images)` is the shared wire entry used by every RPC endpoint that accepts browser uploads (the session prompt endpoint and the command executor): it enforces canonical base64 on every member, then delegates batch admission — limits, validation, ordered commit — to `saveImages`. The base64 upload form is `EncodedImageAttachment`, exported from `@deepseek-ai/dsh-attachment/types` so wire contracts can reference it. ## Model Experience -Indirectly, through the role-neutral core `ImageBlock` and provider adapters that resolve its durable reference into an exact request version. Request descriptors expose the complete attachment id, actual preview dimensions, and the `read_image_region` coordinate system. +Indirectly, through the role-neutral core `ImageBlock` and provider adapters that resolve its durable reference into an exact request version. Request descriptors expose the complete attachment id and actual request dimensions. #### KV Cache effect diff --git a/packages/attachment/attachment/README.zh.md b/packages/attachment/attachment/README.zh.md index 5623e0944c..4470956987 100644 --- a/packages/attachment/attachment/README.zh.md +++ b/packages/attachment/attachment/README.zh.md @@ -4,13 +4,13 @@ 持久附件服务边界。`ctx.attachments` 校验并持久提交提供方无关的图片主版本,随后返回可序列化的 `ImageAttachmentRef`;消费方绝不会在会话事件中持久保存浏览器路径、对象 URL、提供方 URL 或 base64。 -未发送的输入区图片仍是由浏览器持有的临时草稿。`validateImage` 运行完整准入策略但不执行持久化。`saveImages` 负责批次图片数量和总字节限制,在发布任何成员前为全部成员各准备一次经过验证的主版本,然后按顺序提交,并且只在完整批次成功后返回引用。后续存储失败不会返回部分引用,但较早写入的不可变内容寻址对象可能保持不可达,直至具备按引用感知的垃圾回收。`AttachmentError.code` 使用封闭的 `AttachmentErrorCode` 字符串联合类型。其 `ImageAdmissionErrorCode` 子集标记可由调用方修正的图片输入失败;`isImageAdmissionError` 在运行时识别该子集,使每个协议适配器可以映射自己的错误词汇。`saveImage` 会在发布任何模型可见的会话事件前提交一张已接受的图片,并解析为 `SavedImageAttachment`:返回的 `ref` 描述实际存储的主版本,而 `source`(`SourceImageInfo`)保留所提交光栅的媒体类型、字节长度和应用方向后的尺寸。`readImage` 根据已记录的元数据校验该主版本。`readImageRequest` 确定性派生路由所需的请求版本,其身份覆盖主版本 ID、变换策略版本、像素和字节预算、裁剪区域及编码参数;`readImageRequests` 保持结果顺序,并由实现施加自己的有界并发。`cropImage` 把预览坐标映射到存储的主版本,并把结果保存为新附件。调用方可以取消读取和投影;实现保留取消结果,不把它转换为存储失败。 +未发送的输入区图片仍是由浏览器持有的临时草稿。`validateImage` 运行完整准入策略但不执行持久化。`saveImages` 负责批次图片数量和总字节限制,在发布任何成员前为全部成员各准备一次经过验证的主版本,然后按顺序提交,并且只在完整批次成功后返回引用。后续存储失败不会返回部分引用,但较早写入的不可变内容寻址对象可能保持不可达,直至具备按引用感知的垃圾回收。`AttachmentError.code` 使用封闭的 `AttachmentErrorCode` 字符串联合类型。其 `ImageAdmissionErrorCode` 子集标记可由调用方修正的图片输入失败;`isImageAdmissionError` 在运行时识别该子集,使每个协议适配器可以映射自己的错误词汇。`saveImage` 会在发布任何模型可见的会话事件前提交一张已接受的图片,并解析为 `SavedImageAttachment`:返回的 `ref` 描述实际存储的主版本,而 `source`(`SourceImageInfo`)保留所提交光栅的媒体类型、字节长度和应用方向后的尺寸。`readImage` 根据已记录的元数据校验该主版本。`readImageRequest` 确定性派生路由所需的请求版本,其身份覆盖主版本 ID、变换策略版本、像素和字节预算及编码参数;`readImageRequests` 保持结果顺序,并由实现施加自己的有界并发。调用方可以取消读取和投影;实现保留取消结果,不把它转换为存储失败。 `admitEncodedImages(attachments, images)` 是每个接受浏览器上传的 RPC 端点(会话 prompt 端点与命令执行器)共用的 wire 入口:它对每个成员强制执行规范 base64,随后把批量准入——限额、校验、有序提交——委托给 `saveImages`。base64 上传形式为 `EncodedImageAttachment`,从 `@deepseek-ai/dsh-attachment/types` 导出,供 wire 契约引用。 ## 模型体验 -该包通过角色无关的核心 `ImageBlock`,以及把持久引用解析为确定请求版本的提供方适配器,间接影响模型。请求描述会公开完整附件 ID、实际预览尺寸和 `read_image_region` 使用的坐标系。 +该包通过角色无关的核心 `ImageBlock`,以及把持久引用解析为确定请求版本的提供方适配器,间接影响模型。请求描述会公开完整附件 ID 和实际请求尺寸。 #### KV 缓存影响 diff --git a/packages/attachment/attachment/src/index.ts b/packages/attachment/attachment/src/index.ts index 705346d4cc..85401fad23 100644 --- a/packages/attachment/attachment/src/index.ts +++ b/packages/attachment/attachment/src/index.ts @@ -6,7 +6,6 @@ import type { ImageAttachmentLimits, ImageAttachmentRef, ImageRequestPolicy, - PreviewImageCrop, RequestImageAttachment, SaveImageAttachment, SavedImageAttachment, @@ -24,8 +23,6 @@ export type { ImageAttachmentRef, ImageRequestPolicy, ImageMediaType, - MasterImageCrop, - PreviewImageCrop, RequestImageAttachment, SaveImageAttachment, SavedImageAttachment, @@ -153,26 +150,6 @@ export abstract class AttachmentStore extends Service { return versions } - /** - * Crop the stored master by coordinates measured on a model request preview and persist the result. - * @param ref - session-authorized master attachment. - * @param crop - preview dimensions and preview-coordinate rectangle. - * @param signal - optional cancellation. - * @returns a new durable attachment reference suitable for a logged tool result. - */ - cropImage( - ref: ImageAttachmentRef, - crop: PreviewImageCrop, - signal?: AbortSignal, - ): Promise { - signal?.throwIfAborted() - void ref - void crop - return Promise.reject(new AttachmentError( - 'The mounted attachment provider cannot crop stored images.', - 'ATTACHMENT_PROJECTION_UNSUPPORTED', - )) - } } export default AttachmentStore diff --git a/packages/attachment/attachment/src/types.ts b/packages/attachment/attachment/src/types.ts index 1d83cf1afa..04f7362d38 100644 --- a/packages/attachment/attachment/src/types.ts +++ b/packages/attachment/attachment/src/types.ts @@ -63,27 +63,17 @@ export interface StoredImageAttachment { data: Uint8Array } -/** Pixel rectangle in the oriented 2048px master-version coordinate system. */ -export interface MasterImageCrop { - x: number - y: number - width: number - height: number -} - /** Deterministic request-image policy selected by one exact model route. */ export interface ImageRequestPolicy { /** Maximum width multiplied by height after aspect-preserving projection. */ maxPixels: number /** Encoded-byte cap before base64 expansion or Files API upload. */ maxBytes: number - /** Optional master-coordinate crop applied before pixel-budget scaling. */ - crop?: MasterImageCrop } /** Cached request version derived from one provider-independent master attachment. */ export interface RequestImageAttachment { - /** Cache and upload-index key over the master id, policy, crop, and fixed encoder parameters. */ + /** Cache and upload-index key over the master id, policy, and fixed encoder parameters. */ variantId: ImageVariantId /** Durable master reference from which this request version was derived. */ master: ImageAttachmentRef @@ -99,18 +89,6 @@ export interface RequestImageAttachment { space: 'srgb' /** Whether the encoded request version retains an alpha channel. */ hasAlpha: boolean - /** Applied master-coordinate crop, when present. */ - crop?: MasterImageCrop -} - -/** Crop coordinates measured by a model on the request preview it received. */ -export interface PreviewImageCrop { - previewWidth: number - previewHeight: number - x: number - y: number - width: number - height: number } /** Intrinsic facts of the submitted source raster, before master-version preparation. */ diff --git a/packages/attachment/attachment/tests/index.spec.ts b/packages/attachment/attachment/tests/index.spec.ts index 25afbcd420..589a4322d9 100644 --- a/packages/attachment/attachment/tests/index.spec.ts +++ b/packages/attachment/attachment/tests/index.spec.ts @@ -151,22 +151,15 @@ describe('AttachmentStore.readImageRequests', () => { expect(versions.map(version => version.master.name)).toEqual(['1.png', '2.png']) }) - it('reports unsupported request projection and crop operations, preserving cancellation', async () => { + it('reports unsupported request projection while preserving cancellation', async () => { const store = new UnsupportedProjectionStore(new Context()) const ref = (await new RecordingStore(new Context()).saveImage(image(1))).ref await expect(store.readImageRequest(ref, { maxPixels: 1, maxBytes: 1 })) .rejects.toMatchObject({ code: 'ATTACHMENT_PROJECTION_UNSUPPORTED' }) - await expect(store.cropImage(ref, { - previewWidth: 1, previewHeight: 1, x: 0, y: 0, width: 1, height: 1, - })).rejects.toMatchObject({ code: 'ATTACHMENT_PROJECTION_UNSUPPORTED' }) - const controller = new AbortController() const reason = new Error('cancel unsupported projection') controller.abort(reason) expect(() => store.readImageRequest(ref, { maxPixels: 1, maxBytes: 1 }, controller.signal)).toThrow(reason) - expect(() => store.cropImage(ref, { - previewWidth: 1, previewHeight: 1, x: 0, y: 0, width: 1, height: 1, - }, controller.signal)).toThrow(reason) }) }) diff --git a/packages/core/tools/tests/gen-tool-catalog.spec.ts b/packages/core/tools/tests/gen-tool-catalog.spec.ts index 50cbd49d57..ce2007c34b 100644 --- a/packages/core/tools/tests/gen-tool-catalog.spec.ts +++ b/packages/core/tools/tests/gen-tool-catalog.spec.ts @@ -31,7 +31,7 @@ describe('gen-tool-catalog collectToolCatalog', () => { 'cordis_undefine', 'create_goal', 'edit', 'exit_plan_mode', 'followup_task', 'get_goal', 'glob', 'grep', 'interrupt_agent', 'interrupt_agent', 'job_kill', 'job_list', 'job_output', 'list_agents', 'list_agents', 'lsp', 'pwsh', 'pwsh', 'ralph', - 'read', 'read_image', 'read_image_region', 'report', 'run_code', 'schedule_create', 'schedule_delete', + 'read', 'read_image', 'report', 'run_code', 'schedule_create', 'schedule_delete', 'schedule_list', 'send_message', 'send_message', 'session_event_read', 'session_event_search', 'session_event_trace', 'session_search', 'session_trace', 'skill', 'spawn_teammate', 'str_replace_editor', 'subagent', 'team_task_create', diff --git a/packages/extensions/tool-cordis/src/api-catalog.ts b/packages/extensions/tool-cordis/src/api-catalog.ts index 1602c351b5..0dbe8f0535 100644 --- a/packages/extensions/tool-cordis/src/api-catalog.ts +++ b/packages/extensions/tool-cordis/src/api-catalog.ts @@ -467,12 +467,6 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ parameters: [{ name: 'refs', description: 'durable provider-independent master references in request order.' }, { name: 'policy', description: 'exact route pixel and encoded-byte budget shared by the batch.' }, { name: 'signal', description: 'optional cancellation.' }], returns: 'request versions in the same order as `refs`.', }, - { - signature: 'cropImage( ref: ImageAttachmentRef, crop: PreviewImageCrop, signal?: AbortSignal, ): Promise', - description: 'Crop the stored master by coordinates measured on a model request preview and persist the result.', - parameters: [{ name: 'ref', description: 'session-authorized master attachment.' }, { name: 'crop', description: 'preview dimensions and preview-coordinate rectangle.' }, { name: 'signal', description: 'optional cancellation.' }], - returns: 'a new durable attachment reference suitable for a logged tool result.', - }, ], }, { @@ -3480,7 +3474,7 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'ImageRequestPolicy', - declaration: 'export interface ImageRequestPolicy {\n maxPixels: number;\n maxBytes: number;\n crop?: MasterImageCrop;\n}', + declaration: 'export interface ImageRequestPolicy {\n maxPixels: number;\n maxBytes: number;\n}', }, { name: 'ImageVariantId', @@ -3710,10 +3704,6 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'ManualCompactAgentContext', declaration: 'export interface ManualCompactAgentContext extends CompactionAgentContext {\n runMaintenance(task: (signal: AbortSignal) => Promise): Promise;\n}', }, - { - name: 'MasterImageCrop', - declaration: 'export interface MasterImageCrop {\n x: number;\n y: number;\n width: number;\n height: number;\n}', - }, { name: 'Message', declaration: 'export interface Message {\n readonly id: MessageId;\n readonly role: \'system\' | \'user\' | \'assistant\';\n readonly content: ContentBlock[];\n readonly source: MessageSource;\n}', @@ -3870,10 +3860,6 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'PreToolDecision', declaration: 'export type PreToolDecision = {\n kind: \'allow\';\n} | {\n kind: \'deny\';\n reason: string;\n} | {\n kind: \'ask\';\n reason?: string;\n};', }, - { - name: 'PreviewImageCrop', - declaration: 'export interface PreviewImageCrop {\n previewWidth: number;\n previewHeight: number;\n x: number;\n y: number;\n width: number;\n height: number;\n}', - }, { name: 'ProjectionChangeListener', declaration: 'export type ProjectionChangeListener = (session: Session, key: Extract, value: unknown, seq: number) => void;', @@ -3956,7 +3942,7 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'RequestImageAttachment', - declaration: 'export interface RequestImageAttachment {\n variantId: ImageVariantId;\n master: ImageAttachmentRef;\n data: Uint8Array;\n mediaType: ImageMediaType;\n bytes: number;\n width: number;\n height: number;\n depth: \'uchar\';\n space: \'srgb\';\n hasAlpha: boolean;\n crop?: MasterImageCrop;\n}', + declaration: 'export interface RequestImageAttachment {\n variantId: ImageVariantId;\n master: ImageAttachmentRef;\n data: Uint8Array;\n mediaType: ImageMediaType;\n bytes: number;\n width: number;\n height: number;\n depth: \'uchar\';\n space: \'srgb\';\n hasAlpha: boolean;\n}', }, { name: 'RequestRunOutcome', diff --git a/packages/fs/tool-fs/README.i18n.yaml b/packages/fs/tool-fs/README.i18n.yaml index 47084a3435..6c590d54b4 100644 --- a/packages/fs/tool-fs/README.i18n.yaml +++ b/packages/fs/tool-fs/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/fs/tool-fs/README.md -README.md: 94af10c501bcb86465d685f1f20c7d42f3b9d117 -README.zh.md: 4b8e826db3ae15b825d2f888e7d37fc3cafd1b23 +README.md: ab01840f122d6e0df2782b86840432914b27ebd0 +README.zh.md: ef738a3715b6db45d386d56ba2a776960dd341c1 diff --git a/packages/fs/tool-fs/README.md b/packages/fs/tool-fs/README.md index 94af10c501..ab01840f12 100644 --- a/packages/fs/tool-fs/README.md +++ b/packages/fs/tool-fs/README.md @@ -2,7 +2,7 @@ English | [中文](README.zh.md) -The **model-facing filesystem tools** — `read`, `read_image`, `read_image_region`, `write`, `edit` — and their **executor**. This is the consumer layer of the filesystem stack: it owns tool names, JSON schemas, argument validation, prompt sections, **read windowing**, and result formatting. It reads/writes/edits through the `ctx.fs` provider contract ([`@deepseek-ai/dsh-fs`](../fs)) **directly**. The freshness/observation policy is contributed by a separate plugin ([`@deepseek-ai/dsh-fs-observation-policy`](../fs-observation-policy)) through the `fs/*` event gate; the tool is not method-coupled to it. Under a confining provider, the shared sandbox-policy service is required for per-session execution and the tool exposes escalation for filesystem mutations. +The **model-facing filesystem tools** — `read`, `read_image`, `write`, `edit` — and their **executor**. This is the consumer layer of the filesystem stack: it owns tool names, JSON schemas, argument validation, prompt sections, **read windowing**, and result formatting. It reads/writes/edits through the `ctx.fs` provider contract ([`@deepseek-ai/dsh-fs`](../fs)) **directly**. The freshness/observation policy is contributed by a separate plugin ([`@deepseek-ai/dsh-fs-observation-policy`](../fs-observation-policy)) through the `fs/*` event gate; the tool is not method-coupled to it. Under a confining provider, the shared sandbox-policy service is required for per-session execution and the tool exposes escalation for filesystem mutations. ```ts ignore-check // Default deployment: a ctx.fs provider, the policy plugin, then the tools. @@ -14,7 +14,7 @@ await ctx.plugin(ToolFs) // this package — re `@deepseek-ai/dsh-fs-observation-policy` is **optional**: omit it and the tools run against the bare provider (unconditional write/overwrite/edit, no observed-state). A deployment that loads these tools is expected to also load it, so the behavior is read-before-write/edit. -`read_image` and `read_image_region` register only while a durable `ctx.attachments` service is mounted. Execution additionally requires the exact routed model to declare `image` input (resolved through `ctx.llm.resolveModelInfo` from the session's latest request header, falling back to agent options). `read_image_region` accepts only a complete attachment id already referenced by the calling session, so it can crop a user upload without a filesystem path but cannot cross session scope. +`read_image` registers only while a durable `ctx.attachments` service is mounted. Execution additionally requires the exact routed model to declare `image` input, resolved through `ctx.llm.resolveModelInfo` from the session's latest request header and then from agent options. ## Config @@ -32,14 +32,13 @@ All keys are optional; the defaults are the shipped read caps. | Tool | Arguments | Behavior | |---|---|---| | `read` | `file_path`, `offset?`, `limit?` | Line-numbered UTF-8 content with a pagination footer. `offset` is 1-based; `limit` defaults to and caps at the configured `readLimit` (2000). | -| `read_image` | `file_path` | Reads a PNG/JPEG/WebP/GIF file through the bounded byte seam, persists it through `ctx.attachments.saveImage`, and returns an image block beside a small metadata envelope. It succeeds only when the exact routed model declares image input. | -| `read_image_region` | `attachment_id`, `preview_width`, `preview_height`, `x`, `y`, `width`, `height` | Resolves a session-authorized image, maps the preview-coordinate rectangle to its stored master, persists the crop, and returns the new image block. | +| `read_image` | `file_path` | Reads a PNG/JPEG/WebP/GIF file through the bounded byte seam, persists it through `ctx.attachments.saveImage`, and returns an image block beside a small metadata envelope. Harness validates and downscales large supported images before the next model request, so the model can read the source directly without first creating a thumbnail. It succeeds only when the exact routed model declares image input. | | `write` | `file_path`, `content` | Create or fully replace a file. With the policy plugin: overwriting an existing file requires a prior `read` at the unchanged version; creating a new file does not. Without it: unconditional. | | `edit` | `file_path`, non-empty `old_string`, `new_string`, `replace_all?` | Literal replacement; unique match required unless `replace_all` is true. With the policy plugin: requires a prior `read` (any window) and the file unchanged since. Without it: unconditional. | Field names are snake_case to match Claude Code and existing harness tool schemas. -Structured successes are `read` → `{ path, offset, lines: [{ number, text }], totalLines }`, `read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name?, sourceWidth?, sourceHeight? } }`, `read_image_region` → `{ sourceAttachmentId, preview, crop, image }`, `write` → `{ path, operation: 'create' | 'update', before: string | null, after }`, and `edit` → `{ path, before, after }`. The image source fields appear only when master preparation downscaled the submitted raster. Native renderers preserve the line-numbered read and mutation acknowledgements below. `write`/`edit` derive replayable diff-card metadata, and `read` derives a replayable read-card window `{ path, offset, lines, totalLines, lang? }`; execution-local structured values are not added to `tool/result`, while image renderers emit the durable image blocks that the result logs. +Structured successes are `read` → `{ path, offset, lines: [{ number, text }], totalLines }`, `read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name?, sourceWidth?, sourceHeight? } }`, `write` → `{ path, operation: 'create' | 'update', before: string | null, after }`, and `edit` → `{ path, before, after }`. The image source fields appear only when master preparation downscaled the submitted raster. Native renderers preserve the line-numbered read and mutation acknowledgements below. `write`/`edit` derive replayable diff-card metadata, and `read` derives a replayable read-card window `{ path, offset, lines, totalLines, lang? }`; execution-local structured values are not added to `tool/result`, while image renderers emit the durable image blocks that the result logs. ## The tool is the executor; policy is an event gate @@ -47,7 +46,6 @@ The tools do **not** inject a policy service or inspect any cache. Each tool res - **read** — one `ctx.fs.stat` (type + size routing + version), then `readText`/`streamText`, then builds the line window, then emits `fs/observed` with a plain `ctx.emit`. (1 stat.) - **read_image** — validates the argument, extension, attachment availability, deployment media types, and the image-capable route before any I/O; then one `ctx.fs.stat` (recording an `absent` observation for a missing target, like `read`), a bounded `ctx.fs.readBytes` capped at the smaller of `imageLimits.maxImageBytes` and `imageLimits.maxMessageImageBytes` (the result is one message carrying one image), `attachments.saveImage` (content-addressed, so the image block references a durably committed object by the time `tool/result` is appended), and finally `fs/observed`. (1 stat.) -- **read_image_region** — resolves the full attachment id only from current session messages, validates integer preview coordinates, maps the rectangle to the stored master through `attachments.cropImage`, and returns the persisted crop as an image block. It performs no filesystem-path operation and emits no `fs/observed` event. - **write** — `ctx.waterfall('fs/write-intent', target, exec, () => undefined)` for the optional guard, then `ctx.fs.writeText(target, content, intent)`, then `fs/observed`. (0 stat.) - **edit** — `ctx.waterfall('fs/edit-intent', target, exec, () => undefined)` for the optional guard, then `ctx.fs.editText(target, edit, intent)`, then `fs/observed`. (0 stat.) @@ -101,7 +99,7 @@ Prefix-stable while the plugin scope and guidance text are unchanged. Tool restr #### What the model sees -The model sees the generated [`read`, `read_image`, `read_image_region`, `write`, and `edit` schemas](../../../docs/tool-catalog.md#deepseek-aidsh-tool-fs), with snake_case arguments. The image tools appear only while a durable attachment store is mounted; their schemas are route-independent, and the strict gate refuses at execution. Scoped tool restrictions can remove any definition for one agent. +The model sees the generated [`read`, `read_image`, `write`, and `edit` schemas](../../../docs/tool-catalog.md#deepseek-aidsh-tool-fs), with snake_case arguments. The image tool appears only while a durable attachment store is mounted; its schema is route-independent, and the strict gate refuses at execution. Scoped tool restrictions can remove any definition for one agent. #### Token effect @@ -129,7 +127,7 @@ Append-only; newly visible content follows the reusable request prefix and does #### What the model sees -A successful `read_image` returns ``, `image`, and a `` envelope naming the media type, master dimensions, and byte size, followed by the image itself as a native image block. A successful `read_image_region` returns an `image-region` envelope naming the source attachment, supplied preview dimensions and rectangle, and result dimensions, followed by the crop as a native image block. The result is logged with its new durable reference before the next model request. Request adapters derive previews from the master, so later region reads never crop an already reduced preview. +A successful `read_image` returns ``, `image`, and a `` envelope naming the media type, master dimensions, and byte size, followed by the image itself as a native image block. The result is logged with its durable reference before the next model request. #### Token effect @@ -157,7 +155,7 @@ Append-only; newly visible content follows the reusable request prefix and does #### What the model sees -Failures are normalized as `Error: `. This package's stable validation and read messages are `file_path must be a non-empty string`, `limit must be less than or equal to `, `old_string must be a non-empty string`, `old_string and new_string must differ`, `cannot read "": not found`, `cannot read "": not a regular file`, `offset is out of range for "" ( lines)`, `cannot read "": read_image only accepts PNG/JPEG/WebP/GIF paths`, `cannot read "" as an image: model "" does not declare image input; switch to an image-capable model to read images`, and the mismatch repair `cannot read "": the extension declares , but the bytes use a different image format; rename the file to match its actual format if it is PNG/JPEG/WebP/GIF, or convert it to one of those formats`. A failed 16-bit conversion reports `cannot read "": the 16-bit PNG could not be converted to the canonical 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`. Region reads reject empty or out-of-scope attachment ids and invalid preview rectangles before storage mutation. Provider and policy templates are quoted in their package READMEs. Guarded-mutation failures additionally carry their recovery instruction in the message, appended by this package's model-facing error wrapper: `FS_STALE_VERSION` gets `— re-read the file, then retry`, and `FS_NOT_OBSERVED` gets `— read the file, then retry`; the structured code is preserved. After that reread confirms absence, edit reports `FS_NOT_FOUND` instead of repeating a stale remedy, while write uses guarded creation. +Failures are normalized as `Error: `. This package's stable validation and read messages are `file_path must be a non-empty string`, `limit must be less than or equal to `, `old_string must be a non-empty string`, `old_string and new_string must differ`, `cannot read "": not found`, `cannot read "": not a regular file`, `offset is out of range for "" ( lines)`, `cannot read "": read_image only accepts PNG/JPEG/WebP/GIF paths`, `cannot read "" as an image: model "" does not declare image input; switch to an image-capable model to read images`, and the mismatch repair `cannot read "": the extension declares , but the bytes use a different image format; rename the file to match its actual format if it is PNG/JPEG/WebP/GIF, or convert it to one of those formats`. A failed 16-bit conversion reports `cannot read "": the 16-bit PNG could not be converted to the canonical 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`. Provider and policy templates are quoted in their package READMEs. Guarded-mutation failures additionally carry their recovery instruction in the message, appended by this package's model-facing error wrapper: `FS_STALE_VERSION` gets `— re-read the file, then retry`, and `FS_NOT_OBSERVED` gets `— read the file, then retry`; the structured code is preserved. After that reread confirms absence, edit reports `FS_NOT_FOUND` instead of repeating a stale remedy, while write uses guarded creation. #### Token effect @@ -173,4 +171,5 @@ Append-only; newly visible content follows the reusable request prefix and does - **`read` handles UTF-8 text files only** — images use the separate extension-routed `read_image` tool; PDF, audio, and video remain deferred. A directory target is `FS_NOT_REGULAR_FILE`. - **Extension-declared media type** — the extension selects the declared type and the attachment store's magic-byte validation stays authoritative; a correctly formatted image under a wrong extension is refused with the rename remedy rather than sniffed. - **No inline image preview on the tool-result card** — UI surfaces render the image result generically (the durable reference, not pixels); inline rendering is deferred to the UI packages. +- **No attachment-region tool** — an agent may crop an image through other available tools when it has a filesystem path. A pasted or dragged image without a path cannot be re-read at a higher resolution. - **No timeout surface** — `read`/`write`/`edit` take no timeout argument and declare no `timeout-policy` budget; cancellation rides `exec.signal` only ([provider rationale](../README.md#no-timeouts-on-file-io)). diff --git a/packages/fs/tool-fs/README.zh.md b/packages/fs/tool-fs/README.zh.md index 4b8e826db3..ef738a3715 100644 --- a/packages/fs/tool-fs/README.zh.md +++ b/packages/fs/tool-fs/README.zh.md @@ -2,7 +2,7 @@ [English](README.md) | 中文 -**面向模型的文件系统工具**(`read`、`read_image`、`read_image_region`、`write`、`edit`)及其**执行器**。这是文件系统栈的消费方层:拥有工具名称、JSON Schema、参数校验、提示词段、**读取窗口逻辑**和结果格式化。它**直接**通过 `ctx.fs` 提供方约定([`@deepseek-ai/dsh-fs`](../fs))读取、写入和编辑。新鲜度与观察策略由独立插件([`@deepseek-ai/dsh-fs-observation-policy`](../fs-observation-policy))通过 `fs/*` 事件门禁贡献;工具不与其方法耦合。使用施加沙箱限制的提供方时,逐会话执行需要共享沙箱策略服务,工具还会为文件系统变更提供升权路径。 +**面向模型的文件系统工具**(`read`、`read_image`、`write`、`edit`)及其**执行器**。这是文件系统栈的消费方层:拥有工具名称、JSON Schema、参数校验、提示词段、**读取窗口逻辑**和结果格式化。它**直接**通过 `ctx.fs` 提供方约定([`@deepseek-ai/dsh-fs`](../fs))读取、写入和编辑。新鲜度与观察策略由独立插件([`@deepseek-ai/dsh-fs-observation-policy`](../fs-observation-policy))通过 `fs/*` 事件门禁贡献;工具不与其方法耦合。使用施加沙箱限制的提供方时,逐会话执行需要共享沙箱策略服务,工具还会为文件系统变更提供升权路径。 ```ts ignore-check // Default deployment: a ctx.fs provider, the policy plugin, then the tools. @@ -14,7 +14,7 @@ await ctx.plugin(ToolFs) // this package — re `@deepseek-ai/dsh-fs-observation-policy` 是**可选的**:省略时,工具直接使用裸提供方(无条件写入/覆盖/编辑,无已观察状态)。加载这些工具的部署也应加载该插件,从而提供写入/编辑前读取行为。 -`read_image` 和 `read_image_region` 只在持久 `ctx.attachments` 服务已挂载时注册。执行时还要求确切路由的模型声明 `image` 输入,通过 `ctx.llm.resolveModelInfo` 从会话最新请求 header 解析,缺失时回退到 agent 选项。`read_image_region` 只接受调用会话已经引用的完整附件 ID,因此可以裁剪没有文件路径的用户上传图片,但不能越过会话范围。 +`read_image` 只在持久 `ctx.attachments` 服务已挂载时注册。执行时还要求确切路由的模型声明 `image` 输入,通过 `ctx.llm.resolveModelInfo` 依次从会话最新请求 header 和 agent 选项解析。 ## 配置 @@ -32,14 +32,13 @@ await ctx.plugin(ToolFs) // this package — re | 工具 | 参数 | 行为 | |---|---|---| | `read` | `file_path`、`offset?`、`limit?` | 带行号的 UTF-8 内容和分页 footer。`offset` 从 1 开始;`limit` 默认为配置的 `readLimit`(2000),上限也为该值。 | -| `read_image` | `file_path` | 通过有界字节 seam 读取 PNG/JPEG/WebP/GIF 文件,经 `ctx.attachments.saveImage` 持久保存,并在小型元数据信封旁返回图像块。只有确切路由的模型声明图像输入时才会成功。 | -| `read_image_region` | `attachment_id`、`preview_width`、`preview_height`、`x`、`y`、`width`、`height` | 解析会话有权访问的图片,把预览坐标矩形映射到存储主版本,持久保存裁剪结果并返回新图片块。 | +| `read_image` | `file_path` | 通过有界字节 seam 读取 PNG/JPEG/WebP/GIF 文件,经 `ctx.attachments.saveImage` 持久保存,并在小型元数据信封旁返回图像块。Harness 会在下一次模型请求前校验并缩小受支持的大图,因此模型可以直接读取源文件,无需先创建缩略图。只有确切路由的模型声明图像输入时才会成功。 | | `write` | `file_path`、`content` | 创建文件或完整替换文件。有策略插件时:覆盖现有文件要求先在未变版本上执行 `read`;创建新文件不需要。没有插件时:无条件执行。 | | `edit` | `file_path`、非空 `old_string`、`new_string`、`replace_all?` | 字面量替换;除非 `replace_all` 为 true,否则要求唯一匹配。有策略插件时:要求先执行 `read`(任何窗口),且文件此后未变。没有插件时:无条件执行。 | 字段名使用 snake_case,与 Claude Code 和现有 harness 工具 schema 一致。 -结构化成功值分别为:`read` → `{ path, offset, lines: [{ number, text }], totalLines }`,`read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name?, sourceWidth?, sourceHeight? } }`,`read_image_region` → `{ sourceAttachmentId, preview, crop, image }`,`write` → `{ path, operation: 'create' | 'update', before: string | null, after }`,`edit` → `{ path, before, after }`。图片 source 字段只在主版本准备缩小了提交光栅时出现。原生渲染器会保留下方带行号的读取结果和变更确认。`write` 和 `edit` 从这些值派生可回放的 diff 卡片元数据,`read` 派生可回放的读取卡片窗口 `{ path, offset, lines, totalLines, lang? }`;仅用于执行的结构化值不会添加到 `tool/result`,图片渲染器则会发出由结果记录的持久图片块。 +结构化成功值分别为:`read` → `{ path, offset, lines: [{ number, text }], totalLines }`,`read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name?, sourceWidth?, sourceHeight? } }`,`write` → `{ path, operation: 'create' | 'update', before: string | null, after }`,`edit` → `{ path, before, after }`。图片 source 字段只在主版本准备缩小了提交光栅时出现。原生渲染器会保留下方带行号的读取结果和变更确认。`write` 和 `edit` 从这些值派生可回放的 diff 卡片元数据,`read` 派生可回放的读取卡片窗口 `{ path, offset, lines, totalLines, lang? }`;仅用于执行的结构化值不会添加到 `tool/result`,图片渲染器则会发出由结果记录的持久图片块。 ## 工具就是执行器;策略是事件门禁 @@ -47,7 +46,6 @@ await ctx.plugin(ToolFs) // this package — re - **read**:一次 `ctx.fs.stat`(用于类型、大小路由和版本),随后调用 `readText`/`streamText`,构建行窗口,再发出 `fs/observed`,使用普通 `ctx.emit`。(1 次 stat。) - **read_image**:在任何 I/O 之前校验参数、扩展名、附件可用性、部署接受的媒体类型和图像路由;随后一次 `ctx.fs.stat`(目标缺失时与 `read` 一样记录 `absent` 观察)、以 `imageLimits.maxImageBytes` 与 `imageLimits.maxMessageImageBytes` 中较小者为上限的有界 `ctx.fs.readBytes`(结果是携带一张图像的一条消息)、`attachments.saveImage`(内容寻址,因此在 `tool/result` 事件追加时图像块引用的对象已持久提交),最后发出 `fs/observed`。(1 次 stat。) -- **read_image_region**:只从当前会话消息解析完整附件 ID,校验整数预览坐标,通过 `attachments.cropImage` 把矩形映射到存储主版本,并把持久裁剪结果作为图片块返回。它不执行文件系统路径操作,也不发出 `fs/observed` 事件。 - **write**:调用 `ctx.waterfall('fs/write-intent', target, exec, () => undefined)` 取得可选防护,然后调用 `ctx.fs.writeText(target, content, intent)`,再发出 `fs/observed`。(0 次 stat。) - **edit**:调用 `ctx.waterfall('fs/edit-intent', target, exec, () => undefined)` 取得可选防护,然后调用 `ctx.fs.editText(target, edit, intent)`,再发出 `fs/observed`。(0 次 stat。) @@ -101,7 +99,7 @@ Use the edit tool for targeted changes to existing UTF-8 text files. It replaces #### 模型看到的内容 -模型会看到已生成的 [`read`、`read_image`、`read_image_region`、`write` 和 `edit` schema](../../../docs/tool-catalog.zh.md#deepseek-aidsh-tool-fs),参数使用 snake_case。图片工具只在持久附件存储已挂载时出现;schema 本身与路由无关,严格门禁在执行时拒绝。作用域工具限制可以为某个 agent 移除任一定义。 +模型会看到已生成的 [`read`、`read_image`、`write` 和 `edit` schema](../../../docs/tool-catalog.zh.md#deepseek-aidsh-tool-fs),参数使用 snake_case。图片工具只在持久附件存储已挂载时出现;schema 本身与路由无关,严格门禁在执行时拒绝。作用域工具限制可以为某个 agent 移除任一定义。 #### Token 影响 @@ -129,7 +127,7 @@ Use the edit tool for targeted changes to existing UTF-8 text files. It replaces #### 模型看到的内容 -成功的 `read_image` 返回 ``、`image` 和写明媒体类型、主版本尺寸与字节数的 `` 信封,随后是作为原生图像块的图像本身。成功的 `read_image_region` 返回 `image-region` 信封,写明源附件、提交的预览尺寸和矩形及结果尺寸,随后是作为原生图像块的裁剪结果。新持久引用会随结果写入会话日志,然后才进入下一次模型请求。请求适配器从主版本派生预览,因此之后的局部读取不会从已经缩小的预览继续裁剪。 +成功的 `read_image` 返回 ``、`image` 和写明媒体类型、主版本尺寸与字节数的 `` 信封,随后是作为原生图像块的图像本身。结果会随持久引用写入会话日志,然后才进入下一次模型请求。 #### Token 影响 @@ -157,7 +155,7 @@ Use the edit tool for targeted changes to existing UTF-8 text files. It replaces #### 模型看到的内容 -失败会规范化为 `Error: `。本包稳定的校验和读取消息是 `file_path must be a non-empty string`、`limit must be less than or equal to `、`old_string must be a non-empty string`、`old_string and new_string must differ`、`cannot read "": not found`、`cannot read "": not a regular file`、`offset is out of range for "" ( lines)`、`cannot read "": read_image only accepts PNG/JPEG/WebP/GIF paths`、`cannot read "" as an image: model "" does not declare image input; switch to an image-capable model to read images`,以及类型不匹配的修复消息 `cannot read "": the extension declares , but the bytes use a different image format; rename the file to match its actual format if it is PNG/JPEG/WebP/GIF, or convert it to one of those formats`。16-bit 转换失败会报告 `cannot read "": the 16-bit PNG could not be converted to the canonical 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`。局部读取会在改变存储前拒绝空白或超出会话范围的附件 ID 以及无效预览矩形。提供方和策略模板在各自包的 README 中逐字列出。防护变更失败还会在消息中携带恢复指令,由本包面向模型的错误包装追加:`FS_STALE_VERSION` 追加 `re-read the file, then retry`,`FS_NOT_OBSERVED` 追加 `read the file, then retry`;结构化错误码保持不变。该次重新读取确认缺失后,edit 会报告 `FS_NOT_FOUND`,不会重复陈旧恢复指令;write 则使用带防护的创建。 +失败会规范化为 `Error: `。本包稳定的校验和读取消息是 `file_path must be a non-empty string`、`limit must be less than or equal to `、`old_string must be a non-empty string`、`old_string and new_string must differ`、`cannot read "": not found`、`cannot read "": not a regular file`、`offset is out of range for "" ( lines)`、`cannot read "": read_image only accepts PNG/JPEG/WebP/GIF paths`、`cannot read "" as an image: model "" does not declare image input; switch to an image-capable model to read images`,以及类型不匹配的修复消息 `cannot read "": the extension declares , but the bytes use a different image format; rename the file to match its actual format if it is PNG/JPEG/WebP/GIF, or convert it to one of those formats`。16-bit 转换失败会报告 `cannot read "": the 16-bit PNG could not be converted to the canonical 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`。提供方和策略模板在各自包的 README 中逐字列出。防护变更失败还会在消息中携带恢复指令,由本包面向模型的错误包装追加:`FS_STALE_VERSION` 追加 `re-read the file, then retry`,`FS_NOT_OBSERVED` 追加 `read the file, then retry`;结构化错误码保持不变。该次重新读取确认缺失后,edit 会报告 `FS_NOT_FOUND`,不会重复陈旧恢复指令;write 则使用带防护的创建。 #### Token 影响 @@ -173,4 +171,5 @@ Use the edit tool for targeted changes to existing UTF-8 text files. It replaces - **`read` 只处理 UTF-8 文本文件**:图像使用独立的、按扩展名路由的 `read_image` 工具;PDF、音频和视频仍延期处理。目录目标为 `FS_NOT_REGULAR_FILE`。 - **媒体类型按扩展名声明**:扩展名选择声明类型,附件存储的魔数校验保持权威;扩展名错误但格式正确的图像会得到改名修复提示,而不是被嗅探接受。 - **工具结果卡片没有内嵌图像预览**:UI 表面以通用形式渲染图像结果(持久引用而非像素);内嵌渲染延后到 UI 包处理。 +- **没有附件局部读取工具**:图片具有文件路径时,agent 可以用其他可用工具裁剪。粘贴或拖入但没有路径的图片无法按更高分辨率重新读取。 - **没有超时接口**:`read`/`write`/`edit` 不接受超时参数,也不声明 `timeout-policy` 预算;取消只通过 `exec.signal` 传递(见[提供方理由](../README.zh.md#no-timeouts-on-file-io))。 diff --git a/packages/fs/tool-fs/src/read-image.ts b/packages/fs/tool-fs/src/read-image.ts index a900c0c720..bbf49d568c 100644 --- a/packages/fs/tool-fs/src/read-image.ts +++ b/packages/fs/tool-fs/src/read-image.ts @@ -1,7 +1,5 @@ /** - * The model-facing image tools: `read_image` commits a PNG/JPEG/WebP/GIF file, - * while `read_image_region` crops a session-authorized durable attachment by - * coordinates measured on the exact preview shown to the model. + * The model-facing `read_image` tool commits a PNG/JPEG/WebP/GIF file. * * The route gate is deliberately stricter than the host upload preflight. An * image-reading tool is useful only when the exact calling route can inspect @@ -13,7 +11,7 @@ import { basename, extname } from 'node:path' import type { Context } from '@deepseek-ai/cordis' import { AttachmentError, AttachmentId } from '@deepseek-ai/dsh-attachment' -import type { ImageAttachmentRef, ImageMediaType, PreviewImageCrop } from '@deepseek-ai/dsh-attachment' +import type { ImageAttachmentRef, ImageMediaType } from '@deepseek-ai/dsh-attachment' import type { ContentBlock } from '@deepseek-ai/dsh-llm' import { defineTool } from '@deepseek-ai/dsh-tools' import type { GenericCallView, ToolExecution } from '@deepseek-ai/dsh-tools' @@ -62,14 +60,6 @@ export interface ImageReadValue { } } -/** Structured result of cropping a session-authorized image attachment. */ -export interface ImageRegionReadValue { - sourceAttachmentId: string - preview: { width: number; height: number } - crop: { x: number; y: number; width: number; height: number } - image: ImageReadValue['image'] -} - /** * Map a model-supplied path to its declared image media type by extension. * @param filePath - the raw `file_path` argument (not yet resolved). @@ -120,55 +110,6 @@ export function imageRefFromValue(image: ImageReadValue['image']): ImageAttachme } } -function findImageRef( - content: readonly ContentBlock[], - attachmentId: string, -): ImageAttachmentRef | undefined { - for (const block of content) { - if (block.type === 'image' && block.attachment.attachmentId === attachmentId) return block.attachment - if (block.type === 'tool-result') { - const nested = findImageRef(block.content, attachmentId) - if (nested !== undefined) return nested - } - } - return undefined -} - -function sessionImageRef(exec: ToolExecution, attachmentId: string): ImageAttachmentRef { - const session = exec.agent?.session - if (session === undefined) { - throw new Error('read_image_region requires an active agent session') - } - for (const message of session.deriveMessages()) { - const ref = findImageRef(message.content, attachmentId) - if (ref !== undefined) return ref - } - throw new Error(`attachment "${attachmentId}" is not referenced by the current session`) -} - -function positiveInteger(value: number, name: string): number { - if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`) - return value -} - -function nonNegativeInteger(value: number, name: string): number { - if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`) - return value -} - -function regionReadContent(value: ImageRegionReadValue): ContentBlock[] { - return [ - { - type: 'text', - text: `${value.sourceAttachmentId}\nimage-region\n\n` - + `preview ${value.preview.width}x${value.preview.height} px; crop ` - + `x=${value.crop.x}, y=${value.crop.y}, width=${value.crop.width}, height=${value.crop.height}; ` - + `result ${value.image.width}x${value.image.height} px\n`, - }, - { type: 'image', attachment: imageRefFromValue(value.image) }, - ] -} - /** * Format an image read as the model-facing envelope beside its image block. * A downscaled read names the on-disk dimensions and the multiplier that maps @@ -220,7 +161,9 @@ function imageReadContent(value: ImageReadValue): ContentBlock[] { export function applyReadImageTool(ctx: Context): void { ctx.tools.register(defineTool({ name: 'read_image', - description: 'Read a PNG/JPEG/WebP/GIF file and return the image itself. Requires the current model to accept image input.', + description: 'Read a PNG/JPEG/WebP/GIF file and return the image itself. ' + + 'Harness validates and downscales large supported images before the next model request, so use this tool directly instead of installing image libraries or creating thumbnails merely to inspect an image. ' + + 'Independent files may be read concurrently in small batches. Requires the current model to accept image input.', parameters: { file_path: { type: 'string', required: true, description: 'Path to the image file, resolved by the filesystem backend.' }, }, @@ -333,87 +276,4 @@ export function applyReadImageTool(ctx: Context): void { } }, })) - - ctx.tools.register(defineTool({ - name: 'read_image_region', - description: 'Crop a region from an image attachment already visible in this session. Coordinates use the preview dimensions supplied beside that image.', - parameters: { - attachment_id: { type: 'string', required: true, description: 'Complete attachment id shown beside the image.' }, - preview_width: { type: 'integer', required: true, description: 'Width of the preview shown to the model.' }, - preview_height: { type: 'integer', required: true, description: 'Height of the preview shown to the model.' }, - x: { type: 'integer', required: true, description: 'Left edge in preview pixels.' }, - y: { type: 'integer', required: true, description: 'Top edge in preview pixels.' }, - width: { type: 'integer', required: true, description: 'Crop width in preview pixels.' }, - height: { type: 'integer', required: true, description: 'Crop height in preview pixels.' }, - }, - output: { - schema: { - type: 'object', - additionalProperties: false, - properties: { - sourceAttachmentId: { type: 'string', required: true }, - preview: { - type: 'object', - additionalProperties: false, - required: true, - properties: { - width: { type: 'integer', required: true }, - height: { type: 'integer', required: true }, - }, - }, - crop: { - type: 'object', - additionalProperties: false, - required: true, - properties: { - x: { type: 'integer', required: true }, - y: { type: 'integer', required: true }, - width: { type: 'integer', required: true }, - height: { type: 'integer', required: true }, - }, - }, - image: IMAGE_VALUE_SCHEMA, - }, - }, - render: (_args, value) => regionReadContent(value), - }, - isConcurrencySafe: () => true, - async execute(args, exec) { - const attachmentId = args.attachment_id.trim() - if (attachmentId.length === 0) throw new Error('attachment_id must be a non-empty string') - const ref = sessionImageRef(exec, attachmentId) - await assertImageCapableRoute(ctx, exec, attachmentId) - const crop: PreviewImageCrop = { - previewWidth: positiveInteger(args.preview_width, 'preview_width'), - previewHeight: positiveInteger(args.preview_height, 'preview_height'), - x: nonNegativeInteger(args.x, 'x'), - y: nonNegativeInteger(args.y, 'y'), - width: positiveInteger(args.width, 'width'), - height: positiveInteger(args.height, 'height'), - } - const saved = await ctx.attachments.cropImage(ref, crop, exec.signal) - return { - sourceAttachmentId: ref.attachmentId, - preview: { width: crop.previewWidth, height: crop.previewHeight }, - crop: { x: crop.x, y: crop.y, width: crop.width, height: crop.height }, - image: { - attachmentId: saved.ref.attachmentId, - mediaType: saved.ref.mediaType, - bytes: saved.ref.bytes, - width: saved.ref.width, - height: saved.ref.height, - ...saved.ref.name === undefined ? {} : { name: saved.ref.name }, - ...saved.ref.sourceWidth === undefined ? {} : { sourceWidth: saved.ref.sourceWidth }, - ...saved.ref.sourceHeight === undefined ? {} : { sourceHeight: saved.ref.sourceHeight }, - }, - } - }, - presentCall(args): GenericCallView { - return { - card: 'generic', - title: `Read image region ${args.attachment_id}`, - kind: 'read', - } - }, - })) } diff --git a/packages/fs/tool-fs/tests/read-image.spec.ts b/packages/fs/tool-fs/tests/read-image.spec.ts index 03616911b5..16e07d93a8 100644 --- a/packages/fs/tool-fs/tests/read-image.spec.ts +++ b/packages/fs/tool-fs/tests/read-image.spec.ts @@ -12,7 +12,7 @@ import { join } from 'node:path' import { Context } from '@deepseek-ai/cordis' import { CodeRuntime } from '@deepseek-ai/dsh-code-runtime' import type { CodeRunRequest, CodeRunResult } from '@deepseek-ai/dsh-code-runtime' -import { CallId, createUserMessage, LlmAdapter, LlmRuntime } from '@deepseek-ai/dsh-llm' +import { CallId, LlmAdapter, LlmRuntime } from '@deepseek-ai/dsh-llm' import type { GenerateOptions, LlmModelInfo, LlmResolvedModelInfo, Message, StreamChunk } from '@deepseek-ai/dsh-llm' import SystemPrompt from '@deepseek-ai/dsh-system-prompt' import ToolRuntime, { RUN_CODE_NAME } from '@deepseek-ai/dsh-tools' @@ -175,214 +175,6 @@ describe('imageRefFromValue', () => { }) }) -describe('read_image_region', () => { - it('crops a session-visible attachment and returns a new logged image reference', async () => { - const ctx = await setup() - const attachments = ctx.attachments - const source = await attachments.saveImage({ data: PNG_3X3, mediaType: 'image/png', name: 'grid.png' }) - const history = [createUserMessage({ - content: [{ type: 'image', attachment: source.ref }], - source: { kind: 'plugin', plugin: 'test' }, - })] - - const result = await call(ctx, 'read_image_region', { - attachment_id: source.ref.attachmentId, - preview_width: 3, - preview_height: 3, - x: 1, - y: 0, - width: 2, - height: 2, - }, agentOn('vision-model', 'visual', history)) - - expect(result.isError).toBe(false) - expect(result.content[0]).toMatchObject({ - type: 'text', - text: expect.stringContaining('crop x=1, y=0, width=2, height=2') as string, - }) - expect(result.content[1]).toMatchObject({ - type: 'image', - attachment: { width: 2, height: 2, name: 'grid-crop.png' }, - }) - const cropped = result.content[1] - if (cropped?.type !== 'image') throw new Error('expected cropped image block') - await expect(attachments.readImage(cropped.attachment)).resolves.toMatchObject({ - ref: { attachmentId: cropped.attachment.attachmentId }, - }) - }) - - it('refuses an attachment that is absent from the current session', async () => { - const ctx = await setup() - const result = await call(ctx, 'read_image_region', { - attachment_id: `sha256:${'f'.repeat(64)}`, - preview_width: 800, - preview_height: 800, - x: 0, - y: 0, - width: 100, - height: 100, - }, agentOn('vision-model')) - - expect(result.isError).toBe(true) - expect(text(result)).toContain('not referenced by the current session') - }) - - it('finds images nested in tool results after skipping a non-matching nested result', async () => { - const ctx = await setup() - const source = await ctx.attachments.saveImage({ data: PNG_3X3, mediaType: 'image/png' }) - const history = [createUserMessage({ - content: [ - { type: 'tool-result', toolCallId: CallId('unrelated'), content: [{ type: 'text', text: 'none' }] }, - { type: 'tool-result', toolCallId: CallId('nested'), content: [{ type: 'image', attachment: source.ref }] }, - ], - source: { kind: 'plugin', plugin: 'test' }, - })] - - const result = await call(ctx, 'read_image_region', { - attachment_id: source.ref.attachmentId, - preview_width: 3, - preview_height: 3, - x: 0, - y: 0, - width: 1, - height: 1, - }, agentOn('vision-model', 'visual', history)) - - expect(result.isError).toBe(false) - }) - - it('continues across an earlier session message without the requested image', async () => { - const ctx = await setup() - const source = await ctx.attachments.saveImage({ data: PNG_3X3, mediaType: 'image/png' }) - const history = [ - createUserMessage({ - content: [{ type: 'text', text: 'before image' }], - source: { kind: 'plugin', plugin: 'test' }, - }), - createUserMessage({ - content: [{ type: 'image', attachment: source.ref }], - source: { kind: 'plugin', plugin: 'test' }, - }), - ] - - const result = await call(ctx, 'read_image_region', { - attachment_id: source.ref.attachmentId, - preview_width: 3, - preview_height: 3, - x: 0, - y: 0, - width: 1, - height: 1, - }, agentOn('vision-model', 'visual', history)) - - expect(result.isError).toBe(false) - }) - - it('rejects a missing session, empty id, and invalid coordinate arguments', async () => { - const ctx = await setup() - const base = { - attachment_id: `sha256:${'f'.repeat(64)}`, - preview_width: 1, - preview_height: 1, - x: 0, - y: 0, - width: 1, - height: 1, - } - const noSession = await call(ctx, 'read_image_region', base) - expect(text(noSession)).toContain('requires an active agent session') - - const empty = await call(ctx, 'read_image_region', { ...base, attachment_id: ' ' }, agentOn('vision-model')) - expect(text(empty)).toContain('attachment_id must be a non-empty string') - - const source = await ctx.attachments.saveImage({ data: PNG_1X1, mediaType: 'image/png' }) - const history = [createUserMessage({ - content: [{ type: 'image', attachment: source.ref }], - source: { kind: 'plugin', plugin: 'test' }, - })] - const agent = agentOn('vision-model', 'visual', history) - for (const [field, value, expected] of [ - ['preview_width', 0, 'preview_width must be a positive integer'], - ['preview_height', 0, 'preview_height must be a positive integer'], - ['x', -1, 'x must be a non-negative integer'], - ['y', -1, 'y must be a non-negative integer'], - ['width', 0, 'width must be a positive integer'], - ['height', 0, 'height must be a positive integer'], - ] as const) { - const result = await call(ctx, 'read_image_region', { - ...base, - attachment_id: source.ref.attachmentId, - [field]: value, - }, agent) - expect(text(result)).toContain(expected) - } - }) - - it('projects optional crop metadata from a provider result', async () => { - class CropMetadataStore extends AttachmentStore { - readonly imageLimits: ImageAttachmentLimits = { - maxImageBytes: 1024, - maxImagesPerMessage: 1, - maxMessageImageBytes: 1024, - maxImagePixels: 100, - maxImageDimension: 100, - mediaTypes: ['image/png'], - } - - validateImage(): Promise { return Promise.resolve() } - saveImage(): Promise { throw new Error('not used') } - readImage(): Promise { throw new Error('not used') } - override cropImage(ref: ImageAttachmentRef): Promise { - return Promise.resolve({ - ref: { ...ref, sourceWidth: 2, sourceHeight: 2 }, - source: { mediaType: ref.mediaType, bytes: ref.bytes, width: 2, height: 2 }, - }) - } - } - const ctx = await setup({ attachments: false }) - await ctx.plugin(CropMetadataStore) - const ref: ImageAttachmentRef = { - attachmentId: AttachmentId(`sha256:${'a'.repeat(64)}`), - mediaType: 'image/png', bytes: 1, width: 1, height: 1, - } - const history = [createUserMessage({ - content: [{ type: 'image', attachment: ref }], - source: { kind: 'plugin', plugin: 'test' }, - })] - - const result = await call(ctx, 'read_image_region', { - attachment_id: ref.attachmentId, - preview_width: 1, - preview_height: 1, - x: 0, - y: 0, - width: 1, - height: 1, - }, agentOn('vision-model', 'visual', history)) - - expect(result.content[1]).toMatchObject({ - type: 'image', - attachment: { sourceWidth: 2, sourceHeight: 2 }, - }) - expect(result.content[1]).not.toHaveProperty('attachment.name') - }) - - it('declares a generic read presentation for image-region calls', async () => { - const ctx = await setup() - - expect(ctx.tools.get('read_image_region')?.presentCall?.({ - attachment_id: 'sha256:abc', - preview_width: 1, - preview_height: 1, - x: 0, - y: 0, - width: 1, - height: 1, - })) - .toEqual({ card: 'generic', title: 'Read image region sha256:abc', kind: 'read' }) - }) -}) - describe('read_image happy path', () => { it('commits the bytes durably and renders the envelope beside an image block', async () => { await writeFile(join(dir, 'red.png'), PNG_1X1) @@ -773,7 +565,7 @@ describe('registration surface', () => { const attachmentsFiber = await ctx.plugin(LocalAttachmentStore, { dshHome: home }) const toolFsFiber = await ctx.plugin(ToolFs) const names = () => ctx.tools.schemas().map(schema => schema.name).sort() - expect(names()).toEqual(['edit', 'read', 'read_image', 'read_image_region', 'write']) + expect(names()).toEqual(['edit', 'read', 'read_image', 'write']) // Disposing only the attachment store tears down the scoped inject fiber: // read_image withdraws while the unconditional tools stay registered. @@ -782,7 +574,7 @@ describe('registration surface', () => { // Remounting the store restores the conditional registration. const remounted = await ctx.plugin(LocalAttachmentStore, { dshHome: home }) - expect(names()).toEqual(['edit', 'read', 'read_image', 'read_image_region', 'write']) + expect(names()).toEqual(['edit', 'read', 'read_image', 'write']) // Disposing the whole plugin withdraws every tool, read_image included. await toolFsFiber.dispose() @@ -801,12 +593,6 @@ describe('registration surface', () => { kind: 'read', locations: [{ path: 'shot.png' }], }) - expect(ctx.tools.executionMode({ - signal: testToolSignal, - callId: CallId('region-parallel'), - name: 'read_image_region', - arguments: { attachment_id: 'sha256:a', preview_width: 1, preview_height: 1, x: 0, y: 0, width: 1, height: 1 }, - })).toEqual({ kind: 'parallel' }) }) }) diff --git a/packages/llm/llm-deepseek/README.i18n.yaml b/packages/llm/llm-deepseek/README.i18n.yaml index 71f7f71308..bea18ff3ac 100644 --- a/packages/llm/llm-deepseek/README.i18n.yaml +++ b/packages/llm/llm-deepseek/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/llm/llm-deepseek/README.md -README.md: b20d93394055e3e10dfb5a932660b6a510428492 -README.zh.md: 6e8166227d740c0431c17c091d68b5d56aea0dc5 +README.md: bb7f6a520701134cd43ff6223ef4efbf82d02eb4 +README.zh.md: 934c189232711655aa785a7497f5bb6dff1cbb46 diff --git a/packages/llm/llm-deepseek/README.md b/packages/llm/llm-deepseek/README.md index b20d933940..bb7f6a5207 100644 --- a/packages/llm/llm-deepseek/README.md +++ b/packages/llm/llm-deepseek/README.md @@ -49,11 +49,11 @@ The package root exposes the Cordis plugin contract and `DeepSeekAdapter`; wire The plugin registers the single provider route `deepseek-official` together with its resolved `retryPolicy`; omission resolves to normal mode with five retries. A request selects it with `provider: deepseek-official`; its `model` is passed through as the wire `model` string, so changing DeepSeek models does not require lifecycle-time registration. Omitting `models` advertises `deepseek-v4-flash`, `deepseek-v4-pro`, and the image-capable `deepseek-v4-flash-vision-exp`, each with a 1,000,000-token context window; an explicit list replaces those defaults, while `models: []` advertises none. Catalog entries are exposed through `ctx.llm.listModels('deepseek-official')` for clients such as ACP editors and the Web selector, but remain advisory: unlisted model ids still pass through unchanged as text-only routes. An omitted entry name defaults to its id, and omitted `inputModalities` means `text` only. -An image-capable catalog entry declares `inputModalities: [text, image]` and may set `imagePixelBudget`, `imageMaxBytes`, or `imageDetail: low`. The ordinary default is 640,000 total pixels and 1MiB encoded bytes; low detail defaults to 512 by 512 total pixels. The attachment store scales by `min(1, sqrt(pixelBudget / (width * height)))` and rounds inward to keep the pixel count at or below the hard cap, so a 2048 by 1024 master becomes about 1130 by 565 instead of a forced square. Request encoders run lazily: low-color images try PNG (palette only without alpha) then WebP 85 and 80, other alpha images try WebP 85 then 80, and other opaque images try JPEG 85 then 80; dimensions shrink only when both quality attempts exceed 1MiB. Concurrent generation of one `variantId` shares one transform. A caller can cancel its own wait without interrupting other waiters; the transform stops when no waiter remains. The adapter uploads the exact derived request bytes through `POST /files` and sends `{type: "file", file_id}` blocks. It never falls back to an inline data URL. Every retained image is preceded by stable text naming the complete attachment id and actual request dimensions. Preview-coordinate arguments are included only when the request exposes `read_image_region`. User, tool-result, agent-loop, compaction, and direct `ctx.llm.stream` requests all use this projection. Text-only routes receive stable attachment placeholders while durable history keeps its image references. +An image-capable catalog entry declares `inputModalities: [text, image]` and may set `imagePixelBudget`, `imageMaxBytes`, or `imageDetail: low`. The ordinary default is 640,000 total pixels and 1MiB encoded bytes; low detail defaults to 512 by 512 total pixels. The attachment store scales by `min(1, sqrt(pixelBudget / (width * height)))` and rounds inward to keep the pixel count at or below the hard cap, so a 2048 by 1024 master becomes about 1130 by 565 instead of a forced square. Request encoders run lazily: low-color images try PNG (palette only without alpha) then WebP 85 and 80, other alpha images try WebP 85 then 80, and other opaque images try JPEG 85 then 80; dimensions shrink only when both quality attempts exceed 1MiB. Concurrent generation of one `variantId` shares one transform. A caller can cancel its own wait without interrupting other waiters; the transform stops when no waiter remains. The adapter uploads the exact derived request bytes through `POST /files` and sends `{type: "file", file_id}` blocks. It never falls back to an inline data URL. Every retained image is preceded by stable text naming the complete attachment id and actual request dimensions. User, tool-result, agent-loop, compaction, and direct `ctx.llm.stream` requests all use this projection. Text-only routes receive stable attachment placeholders while durable history keeps its image references. `maxRequestFilesBytes` and `maxImagesPerRequest` bound the retained request versions at 128MiB and 600 images by default. The byte and count quanta must not exceed their corresponding bounds. Before attachment reads, the adapter uses each route's request-version byte cap as a conservative upper bound and removes the oldest over-budget prefix; only retained masters are read and transformed. Exact derived lengths are checked again without restoring omitted images. When the byte bound is crossed, the oldest prefix advances past the next 64MiB boundary; 129 one-megabyte images remove the oldest 65 and retain 64MiB, and that prefix stays unchanged until durable history exceeds 192MiB. Count overflow advances independently in `imageOffloadCountQuantum` steps. Removed images become the fixed model-visible placeholder `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`. This high-watermark projection avoids changing an old request prefix after every new image. -Uploaded ids are indexed below `DSH_HOME` by endpoint/API-key scope and request `variantId`. The variant covers the master attachment id, transform version, route pixel and byte budgets, crop, and encoder parameters, so Files API and inline-capable adapters refer to the same deterministic bytes. Uploads request a seven-day lifetime by default and store the server's `expires_at`. A local mapping with no more than one hour remaining is replaced before use; the adapter does not retrieve every remote file before chat. If chat reports expired, deleted, missing, or invalid file ids and names one or more ids used by the request, the adapter removes exactly those mappings. If the provider identifies stale file state without naming an id, it removes every file mapping used by that chat attempt. It then uploads the affected request versions again and retries chat once. A second stale-file rejection clears the mappings identified by that response and is returned without a third chat attempt. An upload response without a complete file object, matching byte count, and `expires_at` is never indexed; a later request therefore uploads again instead of trusting inconsistent local state. A malformed local upload index is treated as an empty cache and replaced by the next successful upload; permission and filesystem I/O failures still fail the request. +Uploaded ids are indexed below `DSH_HOME` by endpoint/API-key scope and request `variantId`. The variant covers the master attachment id, transform version, route pixel and byte budgets, and encoder parameters, so Files API and inline-capable adapters refer to the same deterministic bytes. Uploads request a seven-day lifetime by default and store the server's `expires_at`. A local mapping with no more than one hour remaining is replaced before use; the adapter does not retrieve every remote file before chat. If chat reports expired, deleted, missing, or invalid file ids and names one or more ids used by the request, the adapter removes exactly those mappings. If the provider identifies stale file state without naming an id, it removes every file mapping used by that chat attempt. It then uploads the affected request versions again and retries chat once. A second stale-file rejection clears the mappings identified by that response and is returned without a third chat attempt. An upload response without a complete file object, matching byte count, and `expires_at` is never indexed; a later request therefore uploads again instead of trusting inconsistent local state. A malformed local upload index is treated as an empty cache and replaced by the next successful upload; permission and filesystem I/O failures still fail the request. Concurrent resolution of one scoped `variantId` shares one Files upload with waiter-local cancellation. One quota upload failure first paginates and collects the configured number of oldest `dsh-` files, then deletes that set before one upload retry. `DeepSeekFilesClient.delete`, `DeepSeekFileStore.release`, and `releaseAll` expose explicit remote-space reclamation. The current provider limits represented by this package are 128MiB per Files upload, 32MiB per chat-referenced image, 10,000 stored files, and 25GiB per API key; the default 1MiB request version remains below the two per-file limits. @@ -104,7 +104,7 @@ Non-2xx responses throw `LlmError` with stable codes: `AUTH` (401/403), `QUOTA` #### What the model sees -The selected DeepSeek model receives the harness system prompt, message history, tool schemas, stop sequences, and call config. The vision model receives retained user and tool-result images as Files API references beside stable attachment handles and preview dimensions; an over-budget older image is represented by the documented placeholder. Reasoning content from a prior assistant turn is passed back verbatim, whether or not that turn called a tool. +The selected DeepSeek model receives the harness system prompt, message history, tool schemas, stop sequences, and call config. The vision model receives retained user and tool-result images as Files API references beside stable attachment handles and request-image dimensions; an over-budget older image is represented by the documented placeholder. Reasoning content from a prior assistant turn is passed back verbatim, whether or not that turn called a tool. #### Token effect diff --git a/packages/llm/llm-deepseek/README.zh.md b/packages/llm/llm-deepseek/README.zh.md index 6e8166227d..934c189232 100644 --- a/packages/llm/llm-deepseek/README.zh.md +++ b/packages/llm/llm-deepseek/README.zh.md @@ -49,11 +49,11 @@ harness LLM(大语言模型)seam 的 DeepSeek chat-completions 适配器: 该插件注册唯一提供方路由 `deepseek-official`,并一同注册解析后的 `retryPolicy`;省略时会解析为 normal 模式并重试五次。请求使用 `provider: deepseek-official` 选择该路由;其 `model` 会作为协议 `model` 字符串原样传递,因此更改 DeepSeek 模型不需要生命周期时注册。省略 `models` 会公布 `deepseek-v4-flash`、`deepseek-v4-pro` 与支持图片输入的 `deepseek-v4-flash-vision-exp`,三者的上下文窗口均为 1,000,000 token;显式列表会替换这些默认值,`models: []` 则不公布任何模型。Catalog 配置项通过 `ctx.llm.listModels('deepseek-official')` 公开给 ACP(Agent Client Protocol)编辑器和 Web 选择器等客户端,但仍只提供建议:未列出模型 id 仍原样传递,并按纯文本路由处理。省略配置项 name 默认为其 id,省略 `inputModalities` 则表示仅支持 `text`。 -支持图片的 catalog 配置项声明 `inputModalities: [text, image]`,并可设置 `imagePixelBudget`、`imageMaxBytes` 或 `imageDetail: low`。普通默认值为总像素 640,000、编码字节 1MiB;low detail 的默认总像素为 512×512。附件存储按 `min(1, sqrt(pixelBudget / (width * height)))` 缩放,并向预算内取整,确保总像素不超过硬上限。因此 2048×1024 主版本会得到约 1130×565 的请求版本,而不会被强制变成正方形。请求编码按需执行:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。两个质量档均超过 1MiB 时才缩小尺寸。同一 `variantId` 的并发生成共享一次变换。调用方可以单独取消等待,不会中断其他等待方;没有等待方时才会停止变换。适配器通过 `POST /files` 上传确切的派生请求字节,再发送 `{type: "file", file_id}` 块,不会回退到内联 data URL。每张保留图片前都有稳定文本,写明完整附件 ID 和实际请求尺寸。只有当前请求公开 `read_image_region` 时才会提供预览坐标参数。User、工具结果、agent loop、压缩和直接 `ctx.llm.stream` 请求都使用该投影。纯文本路由会收到稳定的附件占位文本,持久历史继续保留图片引用。 +支持图片的 catalog 配置项声明 `inputModalities: [text, image]`,并可设置 `imagePixelBudget`、`imageMaxBytes` 或 `imageDetail: low`。普通默认值为总像素 640,000、编码字节 1MiB;low detail 的默认总像素为 512×512。附件存储按 `min(1, sqrt(pixelBudget / (width * height)))` 缩放,并向预算内取整,确保总像素不超过硬上限。因此 2048×1024 主版本会得到约 1130×565 的请求版本,而不会被强制变成正方形。请求编码按需执行:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。两个质量档均超过 1MiB 时才缩小尺寸。同一 `variantId` 的并发生成共享一次变换。调用方可以单独取消等待,不会中断其他等待方;没有等待方时才会停止变换。适配器通过 `POST /files` 上传确切的派生请求字节,再发送 `{type: "file", file_id}` 块,不会回退到内联 data URL。每张保留图片前都有稳定文本,写明完整附件 ID 和实际请求尺寸。User、工具结果、agent loop、压缩和直接 `ctx.llm.stream` 请求都使用该投影。纯文本路由会收到稳定的附件占位文本,持久历史继续保留图片引用。 `maxRequestFilesBytes` 和 `maxImagesPerRequest` 限制请求中保留的请求版本,默认值分别为 128MiB 和 600 张。字节和数量步长不得超过对应上限。读取附件前,适配器以路由的请求版本字节上限作为保守上界,移除超预算的最旧前缀,只读取并转换保留的主版本。系统随后用确切派生长度再次检查,但不会重新加入已省略图片。字节数越过上限时,被移除的最旧前缀会越过下一个 64MiB 边界。由 1MiB 图片组成的历史达到 129MiB 时会移除最旧的 65 张并保留 64MiB;直到持久历史超过 192MiB,这个前缀才再次变化。图片数量超限时则按 `imageOffloadCountQuantum` 独立递增。移除的图片会变成固定模型可见占位文本 `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`。这种定量投影不会因每新增一张图片就改写较早的请求前缀。 -上传 ID 按端点和 API key 作用域以及请求 `variantId` 记录在 `DSH_HOME` 下。变体身份覆盖主附件 ID、变换策略版本、路由像素和字节预算、裁剪区域及编码参数,因此 Files API 和支持内联的适配器引用同一份确定性字节。上传默认请求 7 天有效期,并保存服务端返回的 `expires_at`。本地映射剩余时间不超过一小时时会在使用前替换;适配器不会在每次 chat 前查询远端文件。如果 chat 报告文件 ID 已过期、删除、缺失或无效,并指出本次请求使用的一个或多个 ID,适配器只删除这些映射。如果响应只说明文件状态失效而没有指出 ID,适配器会删除该次 chat 使用的全部文件映射。随后重新上传受影响的请求版本,并重试一次 chat。第二次 chat 仍报告文件失效时,适配器会按该响应清理映射并返回错误,不会发起第三次 chat。上传响应若没有完整文件对象、匹配的字节数和 `expires_at`,就不会写入索引;后续请求会再次上传,而不是信任不一致的本地状态。本地上传索引格式损坏时按空缓存处理,并由下一次成功上传替换;权限和文件系统 I/O 错误仍使请求失败。 +上传 ID 按端点和 API key 作用域以及请求 `variantId` 记录在 `DSH_HOME` 下。变体身份覆盖主附件 ID、变换策略版本、路由像素和字节预算及编码参数,因此 Files API 和支持内联的适配器引用同一份确定性字节。上传默认请求 7 天有效期,并保存服务端返回的 `expires_at`。本地映射剩余时间不超过一小时时会在使用前替换;适配器不会在每次 chat 前查询远端文件。如果 chat 报告文件 ID 已过期、删除、缺失或无效,并指出本次请求使用的一个或多个 ID,适配器只删除这些映射。如果响应只说明文件状态失效而没有指出 ID,适配器会删除该次 chat 使用的全部文件映射。随后重新上传受影响的请求版本,并重试一次 chat。第二次 chat 仍报告文件失效时,适配器会按该响应清理映射并返回错误,不会发起第三次 chat。上传响应若没有完整文件对象、匹配的字节数和 `expires_at`,就不会写入索引;后续请求会再次上传,而不是信任不一致的本地状态。本地上传索引格式损坏时按空缓存处理,并由下一次成功上传替换;权限和文件系统 I/O 错误仍使请求失败。 同一作用域和 `variantId` 的并发解析共享一次 Files 上传,每个等待方可以单独取消。一次上传配额错误会先分页收集配置数量的最旧 `dsh-` 文件,再删除这些文件并重试一次上传。`DeepSeekFilesClient.delete`、`DeepSeekFileStore.release` 和 `releaseAll` 提供主动远端空间回收。本包记录的当前提供方限制为 Files 单次上传 128MiB、chat 单图引用 32MiB、每个 API key 最多 10,000 个文件和 25GiB;默认 1MiB 请求版本低于两个单文件上限。 @@ -104,7 +104,7 @@ DeepSeek 请求身份独立于应用归因。凭据解析成功后,每个提 #### 模型看到的内容 -所选 DeepSeek 模型会收到 harness 系统提示词、消息历史、工具 schema、stop sequence 和调用配置。视觉模型会通过 Files API 引用收到保留的 user 与工具结果图片,旁边带有稳定附件句柄和预览尺寸;超出上限的较旧图片由已记录的占位文本表示。之前 assistant 轮次的推理内容会原文回传,无论该轮次是否调用了工具。 +所选 DeepSeek 模型会收到 harness 系统提示词、消息历史、工具 schema、stop sequence 和调用配置。视觉模型会通过 Files API 引用收到保留的 user 与工具结果图片,旁边带有稳定附件句柄和请求图片尺寸;超出上限的较旧图片由已记录的占位文本表示。之前 assistant 轮次的推理内容会原文回传,无论该轮次是否调用了工具。 #### Token 影响 diff --git a/packages/llm/llm-deepseek/src/adapter.ts b/packages/llm/llm-deepseek/src/adapter.ts index 3a01d424ca..30817c738e 100644 --- a/packages/llm/llm-deepseek/src/adapter.ts +++ b/packages/llm/llm-deepseek/src/adapter.ts @@ -543,7 +543,6 @@ export class DeepSeekAdapter extends LlmAdapter { maxImagesPerRequest: connection.maxImagesPerRequest, byteQuantum: connection.imageOffloadByteQuantum, countQuantum: connection.imageOffloadCountQuantum, - cropAvailable: options.tools?.some(tool => tool.name === 'read_image_region') ?? false, }, connection.defaults) const payload = JSON.stringify(body) diff --git a/packages/llm/llm-deepseek/src/serialize.ts b/packages/llm/llm-deepseek/src/serialize.ts index b998b23a8b..4ac6280cd4 100644 --- a/packages/llm/llm-deepseek/src/serialize.ts +++ b/packages/llm/llm-deepseek/src/serialize.ts @@ -6,7 +6,7 @@ * @module dsh-llm-deepseek/serialize */ -import { contentHasImage, LlmError, offloadRequestImagesWithPolicy, requestImagePreviewText } from '@deepseek-ai/dsh-llm' +import { contentHasImage, LlmError, offloadRequestImagesWithPolicy, requestImageHandleText } from '@deepseek-ai/dsh-llm' import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm' import type { ImageAttachmentRef, RequestImageAttachment } from '@deepseek-ai/dsh-attachment' import type { @@ -47,8 +47,6 @@ export interface ImageSerializationOptions { byteQuantum?: number /** Image-count removal step applied after the request exceeds its count bound. */ countQuantum?: number - /** Whether the active request exposes the region-read tool. */ - cropAvailable?: boolean } /** Durable message and image ordinal used in provider diagnostics. */ @@ -120,11 +118,10 @@ function assertSupportedImageRoles(messages: readonly Message[]): void { function imageHandle( version: RequestImageAttachment, precededByContent: boolean, - cropAvailable: boolean, ): WireTextContentPart { return { type: 'text', - text: `${precededByContent ? '\n' : ''}${requestImagePreviewText(version, cropAvailable)}`, + text: `${precededByContent ? '\n' : ''}${requestImageHandleText(version)}`, } } @@ -143,7 +140,7 @@ async function imageParts( ) } return [ - imageHandle(version, precededByContent, images.cropAvailable === true), + imageHandle(version, precededByContent), { type: 'file', file_id: await images.resolveFileId(version, block, location) }, ] } diff --git a/packages/llm/llm-deepseek/src/upload-index.ts b/packages/llm/llm-deepseek/src/upload-index.ts index 12d433b760..297e1021c1 100644 --- a/packages/llm/llm-deepseek/src/upload-index.ts +++ b/packages/llm/llm-deepseek/src/upload-index.ts @@ -15,7 +15,7 @@ export interface DeepSeekUploadRecord { scope: DeepSeekFileScopeType /** Provider-independent master attachment from which the uploaded request version was derived. */ masterAttachmentId: AttachmentId - /** Complete request transformation identity, including crop and encoder parameters. */ + /** Complete request transformation identity, including route budgets and encoder parameters. */ variantId: ImageVariantIdType fileId: DeepSeekFileIdType bytes: number diff --git a/packages/llm/llm-deepseek/tests/adapter.spec.ts b/packages/llm/llm-deepseek/tests/adapter.spec.ts index 2663c4cd78..08b3706758 100644 --- a/packages/llm/llm-deepseek/tests/adapter.spec.ts +++ b/packages/llm/llm-deepseek/tests/adapter.spec.ts @@ -176,7 +176,6 @@ describe('DeepSeekAdapter against a mock server', () => { await drain(adapter.stream({ provider: 'deepseek-official', model: 'deepseek-v4-flash-vision-exp', - tools: [{ name: 'read_image_region', description: 'crop', parameters: { type: 'object' } }], messages: [createUserMessage({ content: [ { type: 'text', text: 'describe ' }, @@ -192,7 +191,7 @@ describe('DeepSeekAdapter against a mock server', () => { role: 'user', content: [ { type: 'text', text: 'describe ' }, - { type: 'text', text: expect.stringContaining('Call read_image_region') as string }, + { type: 'text', text: expect.stringContaining(`Image ${imageRef.attachmentId}; request image 1x1px.`) as string }, { type: 'file', file_id: 'file-api-1' }, ], }], diff --git a/packages/llm/llm-deepseek/tests/serialize.spec.ts b/packages/llm/llm-deepseek/tests/serialize.spec.ts index 06af757c9a..1b14a0c320 100644 --- a/packages/llm/llm-deepseek/tests/serialize.spec.ts +++ b/packages/llm/llm-deepseek/tests/serialize.spec.ts @@ -60,7 +60,6 @@ function imageOptions( resolveFileId, requestImages: new Map(refs.map(ref => [ref.attachmentId, requestVersion(ref)])), maxRequestFilesBytes, - cropAvailable: true, } } @@ -353,14 +352,14 @@ describe('image serialization', () => { role: 'user', content: [ { type: 'text', text: 'before' }, - { type: 'text', text: expect.stringContaining(`Image ${ref.attachmentId}; preview 1x1px`) as string }, + { type: 'text', text: expect.stringContaining(`Image ${ref.attachmentId}; request image 1x1px`) as string }, { type: 'file', file_id: 'file-api-image' }, { type: 'text', text: 'after' }, ], }]) }) - it('gives image-only input a stable handle and preview coordinate system', async () => { + it('gives image-only input a stable handle and request dimensions', async () => { const ref = imageRef() const wire = await serializeRequestWithImages(request({ model: 'deepseek-v4-flash-vision-exp', @@ -373,32 +372,12 @@ describe('image serialization', () => { expect(wire.messages).toEqual([{ role: 'user', content: [ - { type: 'text', text: expect.stringContaining('Call read_image_region') as string }, + { type: 'text', text: `Image ${ref.attachmentId}; request image 1x1px.` }, { type: 'file', file_id: 'file-api-image' }, ], }]) }) - it('does not advertise region reads when the request omits that tool', async () => { - const ref = imageRef() - const images = { ...imageOptions([ref]), cropAvailable: false } - const wire = await serializeRequestWithImages(request({ - model: 'deepseek-v4-flash-vision-exp', - messages: [createUserMessage({ - content: [{ type: 'image', attachment: ref }], - source: { kind: 'plugin', plugin: 'test' }, - })], - }), images) - - expect(wire.messages[0]).toMatchObject({ - role: 'user', - content: [ - { type: 'text', text: `Image ${ref.attachmentId}; preview 1x1px.` }, - { type: 'file', file_id: 'file-api-image' }, - ], - }) - }) - it('rejects an image whose prepared request version is absent', async () => { const ref = imageRef() await expect(serializeMessagesWithImages([createUserMessage({ @@ -528,14 +507,14 @@ describe('image serialization', () => { { role: 'tool', tool_call_id: 'before-system', - content: expect.stringContaining('Call read_image_region') as string, + content: expect.stringContaining('request image 1x1px') as string, }, expect.objectContaining({ role: 'user' }), { role: 'system', content: 'system history' }, { role: 'tool', tool_call_id: 'before-assistant', - content: expect.stringContaining('Call read_image_region') as string, + content: expect.stringContaining('request image 1x1px') as string, }, expect.objectContaining({ role: 'user' }), { role: 'assistant', content: 'assistant history' }, diff --git a/packages/llm/llm-pi-ai/README.i18n.yaml b/packages/llm/llm-pi-ai/README.i18n.yaml index b951aad76e..038224198d 100644 --- a/packages/llm/llm-pi-ai/README.i18n.yaml +++ b/packages/llm/llm-pi-ai/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/llm/llm-pi-ai/README.md -README.md: 044038aa69535ad90c9dc59ad63f05ab68560d28 -README.zh.md: d4b5dff10ea0f3668038cc4d3a6876f52ae273cb +README.md: 8f4d1537d8ccec3e89c0553f877541d11b285f66 +README.zh.md: 354851018de0ea79b82215c3d970266cd2be5763 diff --git a/packages/llm/llm-pi-ai/README.md b/packages/llm/llm-pi-ai/README.md index 360da6a73b..8f4d1537d8 100644 --- a/packages/llm/llm-pi-ai/README.md +++ b/packages/llm/llm-pi-ai/README.md @@ -123,7 +123,7 @@ A model that carries reasoning metadata — from the installed catalog or from i A model **without** that metadata — a hand-declared one whose entry declares no `reasoningEfforts`, and a catalog model pi-ai marks as non-reasoning — exposes no `reasoning` at all. pi-ai reports such a model as supporting the single level `off`, but `off` is translated to *omitting* the reasoning option, which is byte-for-byte the request that naming no effort already produces: selecting it could not disable anything, so a provider whose own default is to think would keep thinking with `off` shown as selected. Reporting the capability as unavailable leaves a surface offering the provider's default and nothing that misrepresents it. The profile `reasoning` value, including `off`, is the deployment default when configured; omitting it preserves the provider default. Per-request `GenerateOptions.reasoningEffort` takes precedence, and a level absent from the exact model capability fails the REQUEST with `UNSUPPORTED_REASONING_EFFORT` before network I/O instead of being clamped. Describing a model never fails that way: the models under one provider disagree about which levels they accept, so `resolveModel` reports a profile level the exact model cannot take as no default at all rather than throwing. A throw there would take the whole provider out of every model catalog built over it — one mis-set profile field hiding even the models that do support the level — so a bad configuration surfaces where it is acted on, not where it is described. pi-ai's common stream options represent `off` by omitting `reasoning`. -Supported profile fields are `apiKeyEnv`, `displayName`, `api`, `baseURL`, `models`, `modelOverrides`, `compat`, `defaultContextWindow`, `defaultMaxTokens`, `defaultInput`, `headers`, `reasoning`, `thinkingBudgets`, `cacheRetention`, `transport`, `timeoutMs`, `websocketConnectTimeoutMs`, `streamIdleTimeoutMs`, `maxRequestImageBytes`, `requestImagePixelBudget`, `requestImageMaxBytes`, and `retryPolicy`. Each resolved profile retry policy is captured with that provider route; omission uses the shared bounded normal default of five retries. The stream-idle interval is a positive finite Node timer delay, defaults to five minutes, and covers only an outstanding provider read, not consumer think time. Every image route derives a deterministic request version from the provider-independent master under `requestImagePixelBudget` (default 2048 by 2048 total pixels) and `requestImageMaxBytes` (default 1MiB raw bytes). Before reading masters, `maxRequestImageBytes` applies to conservative request-version upper bounds and replaces the oldest over-budget images with fixed text; exact base64 lengths are checked again after retained versions are generated. The 20MiB default can retain fifteen maximum-size 1MiB versions after base64 expansion while leaving request-body headroom. The same version feeds inline base64, and its stable descriptor exposes the attachment id and actual preview dimensions. Harness app attribution wins a conflicting configured header name. +Supported profile fields are `apiKeyEnv`, `displayName`, `api`, `baseURL`, `models`, `modelOverrides`, `compat`, `defaultContextWindow`, `defaultMaxTokens`, `defaultInput`, `headers`, `reasoning`, `thinkingBudgets`, `cacheRetention`, `transport`, `timeoutMs`, `websocketConnectTimeoutMs`, `streamIdleTimeoutMs`, `maxRequestImageBytes`, `requestImagePixelBudget`, `requestImageMaxBytes`, and `retryPolicy`. Each resolved profile retry policy is captured with that provider route; omission uses the shared bounded normal default of five retries. The stream-idle interval is a positive finite Node timer delay, defaults to five minutes, and covers only an outstanding provider read, not consumer think time. Every image route derives a deterministic request version from the provider-independent master under `requestImagePixelBudget` (default 2048 by 2048 total pixels) and `requestImageMaxBytes` (default 1MiB raw bytes). Before reading masters, `maxRequestImageBytes` applies to conservative request-version upper bounds and replaces the oldest over-budget images with fixed text; exact base64 lengths are checked again after retained versions are generated. The 20MiB default can retain fifteen maximum-size 1MiB versions after base64 expansion while leaving request-body headroom. The same version feeds inline base64, and its stable descriptor exposes the attachment id and actual request-image dimensions. Harness app attribution wins a conflicting configured header name. The adapter forces pi-ai's SDK `maxRetries` to zero so one `stream()` call makes one provider request. The removed profile fields `maxRetries` and `maxRetryDelayMs` fail load instead of silently multiplying or hiding the separately composed agent-level retry budget. Idle expiry aborts the SDK's stable request signal and surfaces `TIMEOUT`; an earlier caller abort remains `ABORTED`. @@ -173,7 +173,7 @@ pi-ai installs several provider SDKs and lazy-loads the one selected by the cata #### What the model sees -The selected catalog model receives `GenerateOptions.system`, history, tools, and sampling fields supported by pi-ai's common streaming API. Each retained image is preceded by stable text naming its complete attachment id and actual request dimensions. The text includes `read_image_region` preview coordinates only when that tool is present in the request. When accumulated base64 image payload exceeds the route's `maxRequestImageBytes`, each offloaded image (oldest first) is replaced by fixed text that tells the model to read the file again when a path is available or ask the user to attach it again. Offloaded masters are not read or transformed. Provider-native replay metadata is restored only when the adapter validates it for the historical content. +The selected catalog model receives `GenerateOptions.system`, history, tools, and sampling fields supported by pi-ai's common streaming API. Each retained image is preceded by stable text naming its complete attachment id and actual request dimensions. When accumulated base64 image payload exceeds the route's `maxRequestImageBytes`, each offloaded image (oldest first) is replaced by fixed text that tells the model to read the file again when a path is available or ask the user to attach it again. Offloaded masters are not read or transformed. Provider-native replay metadata is restored only when the adapter validates it for the historical content. #### Token effect diff --git a/packages/llm/llm-pi-ai/README.zh.md b/packages/llm/llm-pi-ai/README.zh.md index bf05671ee3..354851018d 100644 --- a/packages/llm/llm-pi-ai/README.zh.md +++ b/packages/llm/llm-pi-ai/README.zh.md @@ -124,7 +124,7 @@ pi-ai 依据提供方 id 与 baseURL 决定每个请求的形状:系统提示 **没有**这份元数据的模型——条目未声明 `reasoningEfforts` 的手工声明模型,以及 pi-ai 标记为不具备推理能力的 catalog 模型——完全不公开 `reasoning`。pi-ai 会把这类模型报告为只支持 `off` 一档,但 `off` 会被翻译成*省略* reasoning 选项,而那与「不点名任何档位」产出的请求逐字节相同:选它关不掉任何东西,于是自身默认就在思考的提供方,会在界面显示 `off` 被选中的同时继续思考。把该能力报告为不可用,界面就只剩提供方默认这一项,不会再出现自相矛盾的控件。配置 profile 的 `reasoning` 值(包括 `off`)在存在时是部署默认值;省略它会保留提供方默认值。每次请求的 `GenerateOptions.reasoningEffort` 优先;未出现在确切模型能力中的档位会让**请求**在网络 I/O 前以 `UNSUPPORTED_REASONING_EFFORT` 失败,而不会被自动调整。**描述**一个模型则从不这样失败:同一提供方下各模型接受的档位并不一致,因此 `resolveModel` 对该模型拿不下的 profile 档位报告为「没有默认值」,而不是抛错。在那里抛错会让整个提供方从任何基于它构建的模型目录中消失——一个配错的 profile 字段连支持该档位的模型也一并藏起来——所以坏配置暴露在被执行处,而不是被描述处。pi-ai 的通用流选项通过省略 `reasoning` 表示 `off`。 -受支持的 profile 字段是 `apiKeyEnv`、`displayName`、`api`、`baseURL`、`models`、`modelOverrides`、`compat`、`defaultContextWindow`、`defaultMaxTokens`、`defaultInput`、`headers`、`reasoning`、`thinkingBudgets`、`cacheRetention`、`transport`、`timeoutMs`、`websocketConnectTimeoutMs`、`streamIdleTimeoutMs`、`maxRequestImageBytes`、`requestImagePixelBudget`、`requestImageMaxBytes` 和 `retryPolicy`。每条 profile 解析后的重试策略会随该提供方路由一同捕获;省略时使用共享的有界 normal 默认值并重试五次。流空闲间隔必须是正的有限 Node 定时器延迟,默认为五分钟,且只覆盖未完成提供方读取,不包括消费方思考时间。每条图片路由从提供方无关的主版本派生确定性请求版本,受 `requestImagePixelBudget`(默认总像素 2048×2048)和 `requestImageMaxBytes`(默认原始字节 1MiB)约束。读取主版本前,`maxRequestImageBytes` 先按请求版本的保守上界替换超预算的最旧图片;保留版本生成后再用确切 base64 长度检查。20MiB 默认值可保留十五个按 1MiB 上限生成的请求版本,并为请求正文留下余量。同一版本用于内联 base64,其稳定描述会公开附件 ID 和实际预览尺寸。若已配置标头中有同名项,则以 Harness 应用归因为准。 +受支持的 profile 字段是 `apiKeyEnv`、`displayName`、`api`、`baseURL`、`models`、`modelOverrides`、`compat`、`defaultContextWindow`、`defaultMaxTokens`、`defaultInput`、`headers`、`reasoning`、`thinkingBudgets`、`cacheRetention`、`transport`、`timeoutMs`、`websocketConnectTimeoutMs`、`streamIdleTimeoutMs`、`maxRequestImageBytes`、`requestImagePixelBudget`、`requestImageMaxBytes` 和 `retryPolicy`。每条 profile 解析后的重试策略会随该提供方路由一同捕获;省略时使用共享的有界 normal 默认值并重试五次。流空闲间隔必须是正的有限 Node 定时器延迟,默认为五分钟,且只覆盖未完成提供方读取,不包括消费方思考时间。每条图片路由从提供方无关的主版本派生确定性请求版本,受 `requestImagePixelBudget`(默认总像素 2048×2048)和 `requestImageMaxBytes`(默认原始字节 1MiB)约束。读取主版本前,`maxRequestImageBytes` 先按请求版本的保守上界替换超预算的最旧图片;保留版本生成后再用确切 base64 长度检查。20MiB 默认值可保留十五个按 1MiB 上限生成的请求版本,并为请求正文留下余量。同一版本用于内联 base64,其稳定描述会公开附件 ID 和实际请求图片尺寸。若已配置标头中有同名项,则以 Harness 应用归因为准。 适配器强制 pi-ai SDK `maxRetries` 为零,因此一次 `stream()` 调用只会发起一次提供方请求。已移除 profile 字段 `maxRetries` 和 `maxRetryDelayMs` 会使加载失败,而不是静默倍增或隐藏单独组合的 agent(智能体)级重试预算。空闲超时会 abort SDK 的稳定请求信号,并以 `TIMEOUT` 呈现;较早的调用方 abort 仍为 `ABORTED`。 @@ -174,7 +174,7 @@ pi-ai 会安装多个提供方 SDK,并延迟加载 catalog 模型所选的 SDK #### 模型看到的内容 -所选 catalog 模型会收到 `GenerateOptions.system`、历史、工具,以及 pi-ai 通用流式 API 支持的采样字段。每张保留图片前都有稳定文本,写明完整附件 ID 和实际请求尺寸。只有请求包含 `read_image_region` 时,文本才会提供该工具使用的预览坐标。请求累积的 base64 图片载荷超过路由的 `maxRequestImageBytes` 时,被 offload 的图片会从最老开始替换为固定文本,要求模型在有路径时重新读取文件,否则请用户重新附上图片。系统不会读取或转换被 offload 的主版本。只有当适配器验证提供方原生回放元数据与历史内容匹配时,才会恢复这些元数据。 +所选 catalog 模型会收到 `GenerateOptions.system`、历史、工具,以及 pi-ai 通用流式 API 支持的采样字段。每张保留图片前都有稳定文本,写明完整附件 ID 和实际请求尺寸。请求累积的 base64 图片载荷超过路由的 `maxRequestImageBytes` 时,被 offload 的图片会从最老开始替换为固定文本,要求模型在有路径时重新读取文件,否则请用户重新附上图片。系统不会读取或转换被 offload 的主版本。只有当适配器验证提供方原生回放元数据与历史内容匹配时,才会恢复这些元数据。 #### Token 影响 diff --git a/packages/llm/llm-pi-ai/src/context.ts b/packages/llm/llm-pi-ai/src/context.ts index 4c31d2638b..5d2df24d18 100644 --- a/packages/llm/llm-pi-ai/src/context.ts +++ b/packages/llm/llm-pi-ai/src/context.ts @@ -4,7 +4,7 @@ * @module dsh-llm-pi-ai/context */ -import { CallId, contentHasImage, LlmError, offloadRequestImagesWithPolicy, requestImagePreviewText } from '@deepseek-ai/dsh-llm' +import { CallId, contentHasImage, LlmError, offloadRequestImagesWithPolicy, requestImageHandleText } from '@deepseek-ai/dsh-llm' import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm' import type { AttachmentId, @@ -48,7 +48,6 @@ function assertSupportedImageRoles(messages: readonly Message[]): void { async function userContent( blocks: readonly ContentBlock[], requestImages: ReadonlyMap, - cropAvailable: boolean, ): Promise { const content: (TextContent | ImageContent)[] = [] for (const block of blocks) { @@ -61,7 +60,7 @@ async function userContent( if (version === undefined) { throw new LlmError(`pi-ai request image ${block.attachment.attachmentId} was not prepared`, 'INVALID_REQUEST') } - content.push({ type: 'text', text: requestImagePreviewText(version, cropAvailable) }) + content.push({ type: 'text', text: requestImageHandleText(version) }) content.push({ type: 'image', data: Buffer.from(version.data).toString('base64'), @@ -71,7 +70,7 @@ async function userContent( } case 'tool-result': { - const nested = await userContent(block.content, requestImages, cropAvailable) + const nested = await userContent(block.content, requestImages) if (typeof nested === 'string') { if (nested.length > 0) content.push({ type: 'text', text: nested }) } else { @@ -241,7 +240,6 @@ async function toPiContextWithImages( byteQuantum: 1, byteLength: ref => requestImages.get(ref.attachmentId)?.bytes ?? ref.bytes, }) - const cropAvailable = options.tools?.some(tool => tool.name === 'read_image_region') ?? false const toolNames = new Map() const messages: PiMessage[] = [] @@ -263,7 +261,7 @@ async function toPiContextWithImages( } // user role: text + tool results (each result becomes its own message). const regular = message.content.filter(block => block.type !== 'tool-result') - const content = await userContent(regular, requestImages, cropAvailable) + const content = await userContent(regular, requestImages) const results = message.content.filter((block): block is Extract => ( block.type === 'tool-result' )) @@ -271,7 +269,7 @@ async function toPiContextWithImages( messages.push({ role: 'user', content, timestamp: 0 }) } for (const result of results) { - const resultContent = await userContent(result.content, requestImages, cropAvailable) + const resultContent = await userContent(result.content, requestImages) messages.push({ role: 'toolResult', toolCallId: result.toolCallId, diff --git a/packages/llm/llm-pi-ai/tests/context.spec.ts b/packages/llm/llm-pi-ai/tests/context.spec.ts index 8a3f7bd084..da1dcaf28b 100644 --- a/packages/llm/llm-pi-ai/tests/context.spec.ts +++ b/packages/llm/llm-pi-ai/tests/context.spec.ts @@ -309,17 +309,6 @@ describe('pi-ai request context conversion', () => { expect(readImageRequest.mock.calls[0]?.[0]).toEqual(recent) }) - it('advertises region reads only when the request exposes the tool', async () => { - const withoutCrop = await toPiContext(request([user([{ type: 'image', attachment: ref }])]), attachments) - const withCrop = await toPiContext({ - ...request([user([{ type: 'image', attachment: ref }])]), - tools: [{ name: 'read_image_region', description: 'crop', parameters: { type: 'object' } }], - }, attachments) - - expect(JSON.stringify(withoutCrop.messages)).not.toContain('Call read_image_region') - expect(JSON.stringify(withCrop.messages)).toContain('Call read_image_region') - }) - it('keeps every image at exactly the payload bound and drops all of them when even the newest cannot fit', async () => { const sized: ImageAttachmentRef = { ...ref, bytes: 3 } const exact = await toPiContext(request([ diff --git a/packages/llm/llm/src/content.ts b/packages/llm/llm/src/content.ts index 72e452e005..c30a62dccb 100644 --- a/packages/llm/llm/src/content.ts +++ b/packages/llm/llm/src/content.ts @@ -19,17 +19,12 @@ export function textOnlyImageText(ref: ImageAttachmentRef): string { } /** - * Stable model-facing handle and coordinate description for one exact request preview. + * Stable model-facing handle for one exact request image. * @param version - exact request image shown beside the text. - * @param cropAvailable - whether the active request exposes `read_image_region`. - * @returns attachment handle, preview dimensions, and crop-coordinate guidance. + * @returns attachment handle and request-image dimensions. */ -export function requestImagePreviewText(version: RequestImageAttachment, cropAvailable: boolean): string { - const identity = `Image ${version.master.attachmentId}; preview ${version.width}x${version.height}px.` - return cropAvailable - ? `${identity} Crop coordinates use this preview. Call read_image_region with this attachment_id, ` - + `preview_width=${version.width}, preview_height=${version.height}, x, y, width, and height.` - : identity +export function requestImageHandleText(version: RequestImageAttachment): string { + return `Image ${version.master.attachmentId}; request image ${version.width}x${version.height}px.` } /** diff --git a/scripts/gen-cordis-catalog.ts b/scripts/gen-cordis-catalog.ts index 558dafca6b..a5ed9feff9 100644 --- a/scripts/gen-cordis-catalog.ts +++ b/scripts/gen-cordis-catalog.ts @@ -294,7 +294,6 @@ export const LINK_MAP: Readonly> = { EncodedImageAttachment: 'attachment.md', ImageAttachmentRef: 'attachment.md', ImageRequestPolicy: 'attachment.md', - PreviewImageCrop: 'attachment.md', RequestImageAttachment: 'attachment.md', SaveImageAttachment: 'attachment.md', SavedImageAttachment: 'attachment.md', diff --git a/scripts/gen-tool-catalog.ts b/scripts/gen-tool-catalog.ts index 48ba2255a5..805316352b 100644 --- a/scripts/gen-tool-catalog.ts +++ b/scripts/gen-tool-catalog.ts @@ -315,17 +315,17 @@ const TOOL_PACKAGES: ToolPackage[] = [ dir: 'tool-fs', source: 'packages/fs/tool-fs/src/index.ts', requires: ['ctx.tools', 'ctx.fs', 'ctx.systemPrompt', 'ctx.attachments (image-tool registration)', 'ctx.llm + an image-capable route (image-tool execution)'], - writes: ['tool/call', 'fs/write-intent or fs/edit-intent for mutations', 'fs/observed after read presence/absence or successful file operation', 'durable attachment (read_image and read_image_region)', 'tool/result'], + writes: ['tool/call', 'fs/write-intent or fs/edit-intent for mutations', 'fs/observed after read presence/absence or successful file operation', 'durable attachment (read_image)', 'tool/result'], async mount(ctx) { // The tool needs `fs`; the bare provider is sufficient because policy // changes behavior, not schema shape. The catalog seam marker opts into - // both attachments-conditional image schemas without attachment I/O. + // the attachments-conditional image schema without attachment I/O. await ctx.plugin(LocalFileSystem) await ctx.plugin(CatalogAttachmentStore) await ctx.plugin(ToolFs) }, note: - 'The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-observation-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. The image tools are not registered without `ctx.attachments`; their schemas are route-independent, and execution refuses unless the exact routed model declares image input.', + 'The read-before-write/edit policy is added by `@deepseek-ai/dsh-fs-observation-policy` (an `fs/*` event-gate plugin, no schema change); a deployment that loads these tools is expected to also load it. The image tool is not registered without `ctx.attachments`; its schema is route-independent, and execution refuses unless the exact routed model declares image input.', }, { pkg: '@deepseek-ai/dsh-tool-fs-search', diff --git a/scripts/type-equiv.manifest.json b/scripts/type-equiv.manifest.json index 946015d483..a83e2fc8e7 100644 --- a/scripts/type-equiv.manifest.json +++ b/scripts/type-equiv.manifest.json @@ -930,21 +930,11 @@ "symbol": "StoredImageAttachment", "source": "packages/attachment/attachment/src/types.ts" }, - { - "doc": "docs/subsystems/attachment.md", - "symbol": "MasterImageCrop", - "source": "packages/attachment/attachment/src/types.ts" - }, { "doc": "docs/subsystems/attachment.md", "symbol": "ImageRequestPolicy", "source": "packages/attachment/attachment/src/types.ts" }, - { - "doc": "docs/subsystems/attachment.md", - "symbol": "PreviewImageCrop", - "source": "packages/attachment/attachment/src/types.ts" - }, { "doc": "docs/subsystems/attachment.md", "symbol": "RequestImageAttachment", From 2491e12fd81f0bcd0d8ed18f28878a5742cd1897 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Fri, 21 Aug 2026 13:19:50 +0800 Subject: [PATCH 22/28] refactor(attachment): normalize image storage API --- ...0-unified-image-request-pipeline.i18n.yaml | 4 +- ...26-08-20-unified-image-request-pipeline.md | 26 ++-- ...08-20-unified-image-request-pipeline.zh.md | 24 ++-- docs/config-catalog.i18n.yaml | 4 +- docs/config-catalog.md | 12 +- docs/config-catalog.zh.md | 12 +- docs/subsystems/attachment.i18n.yaml | 4 +- docs/subsystems/attachment.md | 55 ++++--- docs/subsystems/attachment.zh.md | 55 ++++--- .../system-prompt.expected.md | 6 +- packages/acp/acp/tests/dispose.spec.ts | 2 +- packages/acp/acp/tests/harness.ts | 9 +- packages/acp/acp/tests/turns.spec.ts | 6 +- .../attachment-local/README.i18n.yaml | 4 +- .../attachment/attachment-local/README.md | 8 +- .../attachment/attachment-local/README.zh.md | 8 +- .../attachment-local/src/encoding.ts | 2 +- .../attachment/attachment-local/src/index.ts | 61 ++++---- .../src/{canonical.ts => normalization.ts} | 63 ++++---- .../attachment-local/src/request-image.ts | 72 +++++----- .../attachment/attachment-local/src/store.ts | 80 +++++------ .../attachment-local/tests/index.spec.ts | 18 +-- ...anonical.spec.ts => normalization.spec.ts} | 136 +++++++++--------- .../tests/request-image-verification.spec.ts | 4 +- .../tests/request-image.spec.ts | 78 +++++----- .../attachment-local/tests/store.spec.ts | 42 +++--- .../attachment/attachment/README.i18n.yaml | 4 +- packages/attachment/attachment/README.md | 4 +- packages/attachment/attachment/README.zh.md | 4 +- packages/attachment/attachment/src/index.ts | 42 ++---- packages/attachment/attachment/src/types.ts | 42 ++---- .../attachment/attachment/tests/index.spec.ts | 37 ++--- .../extensions/tool-cordis/src/api-catalog.ts | 32 ++--- packages/fs/tool-fs/README.i18n.yaml | 4 +- packages/fs/tool-fs/README.md | 6 +- packages/fs/tool-fs/README.zh.md | 6 +- packages/fs/tool-fs/src/read-image.ts | 44 +++--- packages/fs/tool-fs/tests/read-image.spec.ts | 38 ++--- .../command-goal/tests/command-goal.spec.ts | 9 +- .../apiproxy/tests/api-proxy-models.spec.ts | 15 +- .../commands/tests/commands.spec.ts | 10 +- packages/llm/llm-deepseek/README.i18n.yaml | 4 +- packages/llm/llm-deepseek/README.md | 6 +- packages/llm/llm-deepseek/README.zh.md | 6 +- packages/llm/llm-deepseek/src/adapter.ts | 6 +- packages/llm/llm-deepseek/src/file-store.ts | 6 +- packages/llm/llm-deepseek/src/upload-index.ts | 26 ++-- .../llm/llm-deepseek/tests/adapter.e2e.ts | 15 +- .../llm/llm-deepseek/tests/adapter.spec.ts | 25 ++-- .../llm-deepseek/tests/dynamic-config.spec.ts | 10 +- .../llm/llm-deepseek/tests/file-store.spec.ts | 4 +- .../llm/llm-deepseek/tests/serialize.spec.ts | 4 +- .../llm-deepseek/tests/upload-index.spec.ts | 63 ++++---- packages/llm/llm-pi-ai/README.i18n.yaml | 4 +- packages/llm/llm-pi-ai/README.md | 4 +- packages/llm/llm-pi-ai/README.zh.md | 4 +- packages/llm/llm-pi-ai/src/config.ts | 2 +- packages/llm/llm-pi-ai/src/context.ts | 11 +- packages/llm/llm-pi-ai/tests/adapter.spec.ts | 5 +- packages/llm/llm-pi-ai/tests/context.spec.ts | 20 +-- packages/llm/llm-pi-ai/tests/convert.spec.ts | 13 +- .../llm/llm-pi-ai/tests/provider-apis.e2e.ts | 5 +- packages/llm/llm/src/content.ts | 2 +- .../mcp/mcp-client/tests/mcp-client.spec.ts | 9 +- .../plan/plan-mode/tests/plan-mode.spec.ts | 5 +- scripts/gen-cordis-catalog.ts | 2 - scripts/gen-tool-catalog.ts | 4 +- scripts/test-invariants.ts | 3 +- 68 files changed, 612 insertions(+), 748 deletions(-) rename packages/attachment/attachment-local/src/{canonical.ts => normalization.ts} (77%) rename packages/attachment/attachment-local/tests/{canonical.spec.ts => normalization.spec.ts} (63%) diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml index e95c7faa2a..1c6145c359 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md -2026-08-20-unified-image-request-pipeline.md: f0ef01de3b22c7132e7f698d0948a0da945726ba -2026-08-20-unified-image-request-pipeline.zh.md: b1a14ac418987ab8bfee9b731ad38cb48e21753e +2026-08-20-unified-image-request-pipeline.md: 6a3bae8a970677c32bbfb7966d2bc13d4e504804 +2026-08-20-unified-image-request-pipeline.zh.md: 10a4aed0b5ca9168c6a6ee4ec0258a210b50d531 diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md index f0ef01de3b..6a3bae8a97 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md @@ -1,4 +1,4 @@ -# Agent Note: Unified image masters, request versions, and provider files +# Agent Note: Unified normalized attachments, request versions, and provider files Status: implemented @@ -10,23 +10,23 @@ Durable image history, provider resolution, inline request size, and remote file ## Decision -The image path has two explicit versions. The attachment backend owns a provider-independent durable master. Each image-capable model route owns a deterministic request policy, and the attachment backend derives and caches the exact request version from the master. Session history contains only the master reference; inline bytes and provider file ids remain transient request projections. +The image path has two explicit versions. The attachment backend owns a provider-independent durable normalized attachment. Each image-capable model route owns a deterministic request policy, and the attachment backend derives and caches the exact request version from that attachment. Session history contains only the normalized attachment reference; inline bytes and provider file ids remain transient request projections. -### Provider-independent master +### Provider-independent normalized attachment -Admission accepts at most 20 images and 200MiB of encoded source bytes per message. Each source is fully decoded under configurable 20MiB, 64,000,000-pixel, and 8192px-per-side limits. Preparation applies EXIF orientation, removes metadata and color profiles, converts to 8-bit sRGB/sRGBA, and preserves aspect ratio while limiting the long edge to `masterMaxDimension`, 2048px by default. `sourceWidth` and `sourceHeight` record orientation-applied dimensions when preparation reduces the raster. +Admission accepts at most 20 images and 200MiB of encoded source bytes per message. Each source is fully decoded under configurable 20MiB, 64,000,000-pixel, and 8192px-per-side limits. Normalization applies EXIF orientation, removes metadata and color profiles, converts to 8-bit sRGB/sRGBA, and preserves aspect ratio while limiting the long edge to `normalizedImageMaxDimension`, 2048px by default. When scaling reduces the raster, `originalDimensions` records its orientation-applied width and height before normalization. -The master has an independent `masterMaxBytes` safety cap, 4MiB by default. Alpha is never flattened. A nearest-neighbour bounded sample classifies color complexity without averaging high-frequency pixels. Confirmed low-color input tries PNG, with palette encoding only when no alpha channel is present, followed by WebP qualities 85, 80, and 75. Other alpha input tries WebP at those qualities; other opaque input tries JPEG. Candidates execute in order and stop at the first result within the cap. Dimensions shrink only after every candidate at one size exceeds the cap. The source extension does not classify a PNG as low color. A clean, single-frame 8-bit sRGB/sRGBA PNG, JPEG, or WebP within both master limits passes through byte-identically and retains content-addressed deduplication. GIF, animation, metadata, orientation, 16-bit PNG, and incompatible color spaces force conversion. The source and a converted output are each fully decoded once; the output must match its format, dimensions, depth, color space, and alpha facts before its digest enters the reference. +The normalized attachment has an independent `normalizedImageMaxBytes` safety cap, 4MiB by default. Alpha is never flattened. A nearest-neighbour bounded sample classifies color complexity without averaging high-frequency pixels. Confirmed low-color input tries PNG, with palette encoding only when no alpha channel is present, followed by WebP qualities 85, 80, and 75. Other alpha input tries WebP at those qualities; other opaque input tries JPEG. Candidates execute in order and stop at the first result within the cap. Dimensions shrink only after every candidate at one size exceeds the cap. The source extension does not classify a PNG as low color. A clean, single-frame 8-bit sRGB/sRGBA PNG, JPEG, or WebP within both normalization limits passes through byte-identically and retains content-addressed deduplication. GIF, animation, metadata, orientation, 16-bit PNG, and incompatible color spaces force conversion. The source and a converted output are each fully decoded once; the output must match its format, dimensions, depth, color space, and alpha facts before its digest enters the reference. -Batch admission prepares and verifies every master once before publishing any member. Validation failure starts no writes. Publication uses those prepared bytes directly, so a large batch does not repeat full decoding and encoding during commit. A later storage failure returns no partial references; already published immutable objects may remain unreachable under the existing storage rule. +Batch admission prepares and verifies every normalized attachment once before publishing any member. Validation failure starts no writes. Publication uses those prepared bytes directly, so a large batch does not repeat full decoding and encoding during commit. A later storage failure returns no partial references; already published immutable objects may remain unreachable under the existing storage rule. ### Deterministic request versions -`AttachmentStore.readImageRequest` derives a request version under route-owned total-pixel and encoded-byte budgets. Scaling is `min(1, sqrt(maxPixels / (width * height)))`, with no enlargement, followed by inward integer rounding so the encoded raster never exceeds the total-pixel cap. DeepSeek V4 Flash Vision Exp uses 640,000 total pixels and 1MiB raw encoded bytes by default; low detail uses 512 by 512 total pixels. A 2048 by 1024 master projects to 1130 by 565 under the hard cap. Request encoding uses the same color branches, with PNG (palette only without alpha) then WebP 85 and 80 for low-color input, WebP 85 then 80 for other alpha input, and JPEG 85 then 80 for other opaque input. Each fallback runs only after the previous result exceeds 1MiB, and dimensions shrink only after both quality attempts exceed it. The same derivation is used by normal agent turns, direct `ctx.llm.stream` calls, compaction, and other auxiliary streams. +`AttachmentStore.readImageRequest` derives a request version under route-owned total-pixel and encoded-byte budgets. Scaling is `min(1, sqrt(maxPixels / (width * height)))`, with no enlargement, followed by inward integer rounding so the encoded raster never exceeds the total-pixel cap. DeepSeek V4 Flash Vision Exp uses 640,000 total pixels and 1MiB raw encoded bytes by default; low detail uses 512 by 512 total pixels. A 2048 by 1024 normalized attachment projects to 1130 by 565 under the hard cap. Request encoding uses the same color branches, with PNG (palette only without alpha) then WebP 85 and 80 for low-color input, WebP 85 then 80 for other alpha input, and JPEG 85 then 80 for other opaque input. Each fallback runs only after the previous result exceeds 1MiB, and dimensions shrink only after both quality attempts exceed it. The same derivation is used by normal agent turns, direct `ctx.llm.stream` calls, compaction, and other auxiliary streams. -The `variantId` and cache path cover the master attachment id, transform version, route pixel and byte budgets, and fixed encoder parameters. A new cache entry is fully decoded before publication. Cache hits use a header probe to check format, 8-bit sRGB/sRGBA facts, dimensions, alpha, and byte limits without decoding the complete raster again; a mismatch regenerates the entry. DeepSeek Files and pi-ai inline base64 therefore use the same deterministic bytes for the same policy. Inline accounting uses the derived byte length after base64 expansion, not the master byte count. Equal in-process `variantId` calls share one transform and cache write. Each caller can cancel its own wait; the shared transform is aborted only after every waiter has cancelled. `AttachmentStore.readImageRequests` preserves input order while the local implementation runs master and request transforms through one FIFO limiter. `imageCompressionConcurrency` is configurable from 1 through 8 and defaults to 2. Batch publication remains sequential after every master has been prepared. +The `variantId` and cache path cover the normalized attachment id, transform version, route pixel and byte budgets, and fixed encoder parameters. A new cache entry is fully decoded before publication. Cache hits use a header probe to check format, 8-bit sRGB/sRGBA facts, dimensions, alpha, and byte limits without decoding the complete raster again; a mismatch regenerates the entry. DeepSeek Files and pi-ai inline base64 therefore use the same deterministic bytes for the same policy. Inline accounting uses the derived byte length after base64 expansion, not the normalized attachment byte count. Equal in-process `variantId` calls share one transform and cache write. Each caller can cancel its own wait; the shared transform is aborted only after every waiter has cancelled. Callers preserve order by applying `Promise.all` to singular `readImageRequest` calls. The local implementation runs normalization and request transforms through one FIFO limiter; `imageCompressionConcurrency` is configurable from 1 through 8 and defaults to 2. Batch publication remains sequential after every normalized attachment has been prepared. -Request-size offload is a deterministic oldest-first projection. Before reading attachments, each route uses `min(masterBytes, requestVersionMaxBytes)` as a conservative upper bound and removes the oldest over-budget prefix. Only retained masters are read and transformed, so an omitted missing or corrupt object cannot block the request. A second projection uses exact derived lengths without bringing omitted images back. DeepSeek defaults to 128MiB and 600 referenced images. Its removed prefix advances past successive 64MiB byte boundaries and in 20-image count quanta, so 129 one-megabyte images remove the oldest 65, retain 64MiB, and keep that prefix stable until total history passes 192MiB. Pi-ai retains a configurable base64 request bound. A text-only route receives deterministic attachment placeholders, including nested tool-result images, while append-only session history keeps the original references. +Request-size offload is a deterministic oldest-first projection. Before reading attachments, each route uses `min(attachmentBytes, requestVersionMaxBytes)` as a conservative upper bound and removes the oldest over-budget prefix. Only retained attachments are read and transformed, so an omitted missing or corrupt object cannot block the request. A second projection uses exact derived lengths without bringing omitted images back. DeepSeek defaults to 128MiB and 600 referenced images. Its removed prefix advances past successive 64MiB byte boundaries and in 20-image count quanta, so 129 one-megabyte images remove the oldest 65, retain 64MiB, and keep that prefix stable until total history passes 192MiB. Pi-ai retains a configurable base64 request bound. A text-only route receives deterministic attachment placeholders, including nested tool-result images, while append-only session history keeps the original references. ### Stable handles @@ -40,15 +40,15 @@ An upload is indexed only after the response returns a complete file object, mat ### Diagnostics -A 16-bit RGB or RGBA PNG is normal admitted input and converts to 8-bit sRGB/sRGBA. If local conversion fails, `read_image` names the path, detected 16-bit PNG, required canonical form, and manual conversion remedy. If DeepSeek rejects a normalized request version, the primary error names the attachment or display name, durable message and image position, normalized media type, 8-bit sRGB/sRGBA depth, dimensions, and provider message. An ambiguous multi-image rejection lists every candidate. The raw provider body remains the error cause rather than the only visible message. +A 16-bit RGB or RGBA PNG is normal admitted input and converts to 8-bit sRGB/sRGBA. If local conversion fails, `read_image` names the path, detected 16-bit PNG, required normalized form, and manual conversion remedy. If DeepSeek rejects a normalized request version, the primary error names the attachment or display name, durable message and image position, normalized media type, 8-bit sRGB/sRGBA depth, dimensions, and provider message. An ambiguous multi-image rejection lists every candidate. The raw provider body remains the error cause rather than the only visible message. Historical attachment objects that later disappear or fail integrity verification remain fail-loud. Durable quarantine and verified recovery require session events and are tracked by [Quarantine unreadable historical attachments](../../proposed/bug-fix/2026-08-20-attachment-read-quarantine.md). ## Alternatives considered -**Use one 1MiB canonical image for storage and requests.** This makes model resolution determine durable image detail and combines local storage, inline expansion, Files quota, and model pixels into one setting. Independent master and request policies keep those responsibilities explicit. +**Use one 1MiB normalized attachment for storage and requests.** This makes model resolution determine durable image detail and combines local storage, inline expansion, Files quota, and model pixels into one setting. Independent normalization and request policies keep those responsibilities explicit. -**Reject images above provider dimensions or at the encoding quality floor.** A provider limit is route-specific and future requests may use another model. Proportional master preparation and request projection accept ordinary large images while bounding each later representation. +**Reject images above provider dimensions or at the encoding quality floor.** A provider limit is route-specific and future requests may use another model. Proportional normalization and request projection accept ordinary large images while bounding each later representation. **Treat PNG as a screenshot and reject 16-bit PNG.** File format does not reveal pixel complexity, and 16-bit RGB/RGBA is a convertible sample depth rather than an unsupported image type. Pixel sampling and post-conversion probes give the required facts. @@ -66,4 +66,4 @@ Package tests generate 16-bit RGB and RGBA PNG fixtures, prove 8-bit conversion ## Consequences -Durable masters consume up to the independent local safety cap, while request caches and remote Files consume additional derived storage. Deterministic identities and singleflight make that work reusable across turns and sessions sharing the same DSH home. Two simultaneous transforms reduce batch latency while increasing peak RSS relative to serial execution; deployments with tighter memory can set the limit to one. Encoder or transform-version changes create new future identities without rewriting existing history. DeepSeek image requests now depend on Files API availability; bounded stale-id recovery handles inconsistent remote state, while a general Files outage remains a visible request failure. Missing or corrupt durable masters still require the separate quarantine design. +Normalized attachments consume up to the independent local safety cap, while request caches and remote Files consume additional derived storage. Deterministic identities and singleflight make that work reusable across turns and sessions sharing the same DSH home. Two simultaneous transforms reduce batch latency while increasing peak RSS relative to serial execution; deployments with tighter memory can set the limit to one. Encoder or transform-version changes create new future identities without rewriting existing history. DeepSeek image requests now depend on Files API availability; bounded stale-id recovery handles inconsistent remote state, while a general Files outage remains a visible request failure. Missing or corrupt durable attachments still require the separate quarantine design. diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md index b1a14ac418..10a4aed0b5 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md @@ -1,4 +1,4 @@ -# Agent Note: 统一图片主版本、请求版本与提供方文件 +# Agent Note: 统一规范化附件、请求版本与提供方文件 Status: implemented @@ -10,23 +10,23 @@ Status: implemented ## Decision -图片路径有两个显式版本。附件后端拥有提供方无关的持久主版本。每条支持图片的模型路由拥有确定性请求策略,附件后端从主版本派生并缓存确切请求版本。会话历史只包含主版本引用;内联字节和提供方文件 ID 都是瞬时请求投影。 +图片路径有两个显式版本。附件后端拥有提供方无关的持久规范化附件。每条支持图片的模型路由拥有确定性请求策略,附件后端从该附件派生并缓存确切请求版本。会话历史只包含规范化附件引用;内联字节和提供方文件 ID 都是瞬时请求投影。 -### 提供方无关的主版本 +### 提供方无关的规范化附件 -每条消息最多准入 20 张图片,源图编码字节总量不超过 200MiB。每张源图会在可配置的 20MiB、64,000,000 像素和单边 8192px 限制内完整解码。处理会应用 EXIF 方向,删除元数据和色彩配置文件,转换为 8-bit sRGB/sRGBA,并保持宽高比把长边限制到 `masterMaxDimension`,默认 2048px。处理缩小光栅时,`sourceWidth` 和 `sourceHeight` 记录应用方向后的源尺寸。 +每条消息最多准入 20 张图片,源图编码字节总量不超过 200MiB。每张源图会在可配置的 20MiB、64,000,000 像素和单边 8192px 限制内完整解码。规范化过程会应用 EXIF 方向,删除元数据和色彩配置文件,转换为 8-bit sRGB/sRGBA,并保持宽高比把长边限制到 `normalizedImageMaxDimension`,默认 2048px。缩放减小光栅时,`originalDimensions` 记录规范化之前、应用方向之后的输入宽高。 -主版本有独立的 `masterMaxBytes` 安全上限,默认 4MiB。透明通道绝不铺平。系统通过 nearest-neighbour 对有界样本判断色彩复杂度,不会通过像素平均把高频图片误判为低色数。确认的低色数输入先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80、75 的 WebP;其他透明输入依次尝试这些质量的 WebP;其他非透明输入依次尝试这些质量的 JPEG。候选按顺序执行,首个不超过上限的结果会立即返回。同一尺寸的候选全部超限后才会缩小尺寸。源扩展名不会把 PNG 归类为低色数图片。处于两个主版本上限内的干净、单帧、8-bit sRGB/sRGBA PNG、JPEG 或 WebP 按字节原样直通,并保留内容寻址去重。GIF、动图、元数据、方向、16-bit PNG 和不兼容色彩空间都会触发转换。源图和转换输出各完整解码一次;输出的格式、尺寸、位深、色彩空间和透明通道事实通过校验后,其摘要才会进入引用。 +规范化附件有独立的 `normalizedImageMaxBytes` 安全上限,默认 4MiB。透明通道绝不铺平。系统通过 nearest-neighbour 对有界样本判断色彩复杂度,不会通过像素平均把高频图片误判为低色数。确认的低色数输入先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80、75 的 WebP;其他透明输入依次尝试这些质量的 WebP;其他非透明输入依次尝试这些质量的 JPEG。候选按顺序执行,首个不超过上限的结果会立即返回。同一尺寸的候选全部超限后才会缩小尺寸。源扩展名不会把 PNG 归类为低色数图片。处于两个规范化上限内的干净、单帧、8-bit sRGB/sRGBA PNG、JPEG 或 WebP 按字节原样直通,并保留内容寻址去重。GIF、动图、元数据、方向、16-bit PNG 和不兼容色彩空间都会触发转换。源图和转换输出各完整解码一次;输出的格式、尺寸、位深、色彩空间和透明通道事实通过校验后,其摘要才会进入引用。 -批量准入在发布任何成员前,为每张图片各准备并验证一次主版本。校验失败不会开始写入。发布直接使用这些已准备字节,因此大批次不会在提交时重复完整解码和编码。之后发生的存储失败不会返回部分引用;按现有存储规则,已经发布的不可变对象可能保持不可达。 +批量准入在发布任何成员前,为每张图片各准备并验证一次规范化附件。校验失败不会开始写入。发布直接使用这些已准备字节,因此大批次不会在提交时重复完整解码和编码。之后发生的存储失败不会返回部分引用;按现有存储规则,已经发布的不可变对象可能保持不可达。 ### 确定性请求版本 -`AttachmentStore.readImageRequest` 按路由拥有的总像素和编码字节预算派生请求版本。缩放公式为 `min(1, sqrt(maxPixels / (width * height)))`,不会放大小图,随后向预算内取整,确保编码光栅不超过总像素上限。DeepSeek V4 Flash Vision Exp 默认使用总像素 640,000 和原始编码字节 1MiB;low detail 使用总像素 512×512。2048×1024 主版本在这个硬上限下会投影为 1130×565。请求编码使用相同的分类分支:低色数输入先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80 的 WebP;其他透明输入依次尝试质量 85、80 的 WebP;其他非透明输入依次尝试质量 85、80 的 JPEG。只有前一结果超过 1MiB 时才执行下一个候选;两个质量档都超限后才缩小尺寸。普通 agent 轮次、直接 `ctx.llm.stream` 调用、压缩和其他辅助流都使用同一派生过程。 +`AttachmentStore.readImageRequest` 按路由拥有的总像素和编码字节预算派生请求版本。缩放公式为 `min(1, sqrt(maxPixels / (width * height)))`,不会放大小图,随后向预算内取整,确保编码光栅不超过总像素上限。DeepSeek V4 Flash Vision Exp 默认使用总像素 640,000 和原始编码字节 1MiB;low detail 使用总像素 512×512。2048×1024 规范化附件在这个硬上限下会投影为 1130×565。请求编码使用相同的分类分支:低色数输入先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80 的 WebP;其他透明输入依次尝试质量 85、80 的 WebP;其他非透明输入依次尝试质量 85、80 的 JPEG。只有前一结果超过 1MiB 时才执行下一个候选;两个质量档都超限后才缩小尺寸。普通 agent 轮次、直接 `ctx.llm.stream` 调用、压缩和其他辅助流都使用同一派生过程。 -`variantId` 和缓存路径覆盖主附件 ID、变换策略版本、路由像素和字节预算及固定编码参数。新缓存条目在发布前会完整解码。缓存命中只探测文件头,校验格式、8-bit sRGB/sRGBA、尺寸、透明通道和字节上限,不会再次完整解码光栅;不匹配时会重新生成。因此,同一策略下的 DeepSeek Files 和 pi-ai 内联 base64 使用相同的确定性字节。内联计量使用派生字节经过 base64 膨胀后的长度,不使用主版本字节数。同一进程内相同 `variantId` 的调用共享一次变换和缓存写入。每个调用方可以取消自己的等待;只有全部等待方都取消时,共享变换才会中止。`AttachmentStore.readImageRequests` 保持输入顺序,本地实现则通过一个 FIFO 限流器运行主版本和请求版本变换。`imageCompressionConcurrency` 的可配置范围为 1 至 8,默认值为 2。全部主版本准备完成后,批次仍按顺序发布。 +`variantId` 和缓存路径覆盖规范化附件 ID、变换策略版本、路由像素和字节预算及固定编码参数。新缓存条目在发布前会完整解码。缓存命中只探测文件头,校验格式、8-bit sRGB/sRGBA、尺寸、透明通道和字节上限,不会再次完整解码光栅;不匹配时会重新生成。因此,同一策略下的 DeepSeek Files 和 pi-ai 内联 base64 使用相同的确定性字节。内联计量使用派生字节经过 base64 膨胀后的长度,不使用规范化附件字节数。同一进程内相同 `variantId` 的调用共享一次变换和缓存写入。每个调用方可以取消自己的等待;只有全部等待方都取消时,共享变换才会中止。调用方对单数 `readImageRequest` 使用 `Promise.all` 保持结果顺序。本地实现通过一个 FIFO 限流器运行规范化和请求变换,`imageCompressionConcurrency` 的可配置范围为 1 至 8,默认值为 2。全部规范化附件准备完成后,批次仍按顺序发布。 -请求大小 offload 是确定性的从旧到新投影。读取附件前,每条路由先以 `min(主版本字节数, 请求版本字节上限)` 作为保守上界,移除超出预算的最旧前缀。系统只读取并转换保留的主版本,因此已省略的缺失或损坏对象不会阻塞请求。第二次投影使用确切派生长度,但不会重新加入已省略图片。DeepSeek 默认上限为 128MiB 和 600 张引用图片。被移除前缀会越过连续的 64MiB 字节边界,并按 20 张图片数量步长递增,因此 129 张 1MiB 图片会移除最旧的 65 张并保留 64MiB;持久历史超过 192MiB 前,该前缀保持不变。Pi-ai 保留可配置的 base64 请求上限。纯文本路由会收到确定性的附件占位文本,其中包括嵌套工具结果图片;追加式会话历史继续保留原始引用。 +请求大小 offload 是确定性的从旧到新投影。读取附件前,每条路由先以 `min(附件字节数, 请求版本字节上限)` 作为保守上界,移除超出预算的最旧前缀。系统只读取并转换保留的附件,因此已省略的缺失或损坏对象不会阻塞请求。第二次投影使用确切派生长度,但不会重新加入已省略图片。DeepSeek 默认上限为 128MiB 和 600 张引用图片。被移除前缀会越过连续的 64MiB 字节边界,并按 20 张图片数量步长递增,因此 129 张 1MiB 图片会移除最旧的 65 张并保留 64MiB;持久历史超过 192MiB 前,该前缀保持不变。Pi-ai 保留可配置的 base64 请求上限。纯文本路由会收到确定性的附件占位文本,其中包括嵌套工具结果图片;追加式会话历史继续保留原始引用。 ### 稳定句柄 @@ -46,9 +46,9 @@ Status: implemented ## Alternatives considered -**使用一份 1MiB 规范图片同时负责存储和请求。** 这种做法让模型分辨率决定持久图片细节,并把本地存储、内联膨胀、Files 配额和模型像素合并成一个设置。独立的主版本和请求策略会明确区分这些职责。 +**使用一份 1MiB 规范化附件同时负责存储和请求。** 这种做法让模型分辨率决定持久图片细节,并把本地存储、内联膨胀、Files 配额和模型像素合并成一个设置。独立的规范化和请求策略会明确区分这些职责。 -**拒绝超过提供方尺寸或达到编码质量下限的图片。** 提供方限制属于具体路由,未来请求可能改用另一个模型。按比例准备主版本和投影请求版本可以接纳普通大图,同时约束每种后续表示。 +**拒绝超过提供方尺寸或达到编码质量下限的图片。** 提供方限制属于具体路由,未来请求可能改用另一个模型。按比例规范化和投影请求版本可以接纳普通大图,同时约束每种后续表示。 **把 PNG 当作截图,并拒绝 16-bit PNG。** 文件格式不能说明像素复杂度,16-bit RGB/RGBA 是可转换位深,不是不支持的图片类型。像素采样和转换后探测能提供所需事实。 @@ -66,4 +66,4 @@ Status: implemented ## Consequences -持久主版本最多占用独立的本地安全上限,请求缓存和远端 Files 还会占用额外派生存储。确定性身份和 singleflight 使这些成本可以被共享同一 DSH home 的轮次和会话复用。同时执行两个变换会降低批次延迟,但峰值 RSS 高于串行执行;内存更紧张的部署可以把上限设为 1。编码器或变换策略版本变化会为未来内容产生新身份,不会改写已有历史。DeepSeek 图片请求现在依赖 Files API 可用性;有界的陈旧 ID 恢复会处理远端状态不一致,一般 Files 故障仍会成为可见请求失败。缺失或损坏的持久主版本仍需要单独的隔离设计。 +持久规范化附件最多占用独立的本地安全上限,请求缓存和远端 Files 还会占用额外派生存储。确定性身份和 singleflight 使这些成本可以被共享同一 DSH home 的轮次和会话复用。同时执行两个变换会降低批次延迟,但峰值 RSS 高于串行执行;内存更紧张的部署可以把上限设为 1。编码器或变换策略版本变化会为未来内容产生新身份,不会改写已有历史。DeepSeek 图片请求现在依赖 Files API 可用性;有界的陈旧 ID 恢复会处理远端状态不一致,一般 Files 故障仍会成为可见请求失败。缺失或损坏的持久附件仍需要单独的隔离设计。 diff --git a/docs/config-catalog.i18n.yaml b/docs/config-catalog.i18n.yaml index 276fad138a..a7d0db5eec 100644 --- a/docs/config-catalog.i18n.yaml +++ b/docs/config-catalog.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/config-catalog.md -config-catalog.md: d288fe3b85f1599da6ecef3dcf59c04c4e8c85d5 -config-catalog.zh.md: 266465fd09312c5dde9df4453c34f3aa774db7e2 +config-catalog.md: 8152b3c3280a7b85543d6cfeec850c8b3a25ca47 +config-catalog.zh.md: 95fd49e380e0cc9322fe8d12d0bdaf8dd310efac diff --git a/docs/config-catalog.md b/docs/config-catalog.md index d288fe3b85..8152b3c328 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -337,16 +337,16 @@ export interface Config { maxImagePixels?: number /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. Default: 8192px. */ maxImageDimension?: number - /** Long-edge pixel cap of the stored provider-independent master version. */ - masterMaxDimension?: number - /** Encoded-byte safety cap of the stored provider-independent master version. */ - masterMaxBytes?: number - /** Maximum simultaneous master or request-image transformations in this service instance. */ + /** Long-edge pixel cap of the stored provider-independent normalized image. */ + normalizedImageMaxDimension?: number + /** Encoded-byte safety cap of the stored provider-independent normalized image. */ + normalizedImageMaxBytes?: number + /** Maximum simultaneous normalization or request-image transformations in this service instance. */ imageCompressionConcurrency?: number } ``` -Source: [`packages/attachment/attachment-local/src/index.ts:52`](../packages/attachment/attachment-local/src/index.ts) +Source: [`packages/attachment/attachment-local/src/index.ts:51`](../packages/attachment/attachment-local/src/index.ts) diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index 266465fd09..95fd49e380 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -339,16 +339,16 @@ export interface Config { maxImagePixels?: number /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. Default: 8192px. */ maxImageDimension?: number - /** Long-edge pixel cap of the stored provider-independent master version. */ - masterMaxDimension?: number - /** Encoded-byte safety cap of the stored provider-independent master version. */ - masterMaxBytes?: number - /** Maximum simultaneous master or request-image transformations in this service instance. */ + /** Long-edge pixel cap of the stored provider-independent normalized image. */ + normalizedImageMaxDimension?: number + /** Encoded-byte safety cap of the stored provider-independent normalized image. */ + normalizedImageMaxBytes?: number + /** Maximum simultaneous normalization or request-image transformations in this service instance. */ imageCompressionConcurrency?: number } ``` -来源:[`packages/attachment/attachment-local/src/index.ts:52`](../packages/attachment/attachment-local/src/index.ts) +来源:[`packages/attachment/attachment-local/src/index.ts:51`](../packages/attachment/attachment-local/src/index.ts) diff --git a/docs/subsystems/attachment.i18n.yaml b/docs/subsystems/attachment.i18n.yaml index ee14a0698f..b93c9ef1ca 100644 --- a/docs/subsystems/attachment.i18n.yaml +++ b/docs/subsystems/attachment.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/subsystems/attachment.md -attachment.md: 7c55bc192088f67ae7d117bc150aa0ea6fdf8b09 -attachment.zh.md: d5a140e283c1b7aa6ee5c991c2932ff65de0b88e +attachment.md: e6d0a53db2827a38a1535380319b6220aa37f0a4 +attachment.zh.md: 8328ec610d4d68624f75f00d6a397b13fdf31c4e diff --git a/docs/subsystems/attachment.md b/docs/subsystems/attachment.md index 7c55bc1920..e6d0a53db2 100644 --- a/docs/subsystems/attachment.md +++ b/docs/subsystems/attachment.md @@ -18,7 +18,7 @@ type ImageMediaType = 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif' ``` ```ts type-equiv -/** Durable, serializable metadata for one immutable image object. */ +/** Durable, serializable reference to one immutable normalized image. */ interface ImageAttachmentRef { /** Opaque storage identifier; never a filesystem path or bearer URL. */ attachmentId: AttachmentId @@ -32,10 +32,14 @@ interface ImageAttachmentRef { height: number /** Optional display name stripped of local path information. */ name?: string - /** Perceived source width before master-version downscaling; present only when it differs from {@link width}. */ - sourceWidth?: number - /** Perceived source height before master-version downscaling; present only when it differs from {@link height}. */ - sourceHeight?: number + /** + * Input dimensions after applying EXIF orientation and before normalization + * scaling. Present only when normalization reduced the image. + */ + originalDimensions?: { + width: number + height: number + } } ``` @@ -52,7 +56,7 @@ interface ImageAttachmentLimits { } ``` -The local backend admits at most 20 images and 200 MiB of encoded source data per message. One source may use up to 20 MiB, 64,000,000 pixels, and 8192 pixels on either side. These source limits precede the independent 2048-pixel, 4 MiB master preparation stage. +The local backend admits at most 20 images and 200 MiB of encoded source data per message. One source may use up to 20 MiB, 64,000,000 pixels, and 8192 pixels on either side. These source limits precede the independent normalization stage, which limits the long edge to 2048 pixels and encoded data to 4 MiB by default. The reference records intrinsic dimensions and encoded length so clients can lay out history without decoding first, while every authoritative read still re-checks digest, media signature, dimensions, and metadata against the object. @@ -100,12 +104,12 @@ interface ImageRequestPolicy { ``` ```ts type-equiv -/** Cached request version derived from one provider-independent master attachment. */ +/** Cached request version derived from one provider-independent normalized attachment. */ interface RequestImageAttachment { - /** Cache and upload-index key over the master id, policy, and fixed encoder parameters. */ + /** Cache and upload-index key over the attachment id, policy, and fixed encoder parameters. */ variantId: ImageVariantId - /** Durable master reference from which this request version was derived. */ - master: ImageAttachmentRef + /** Durable normalized attachment from which this request version was derived. */ + attachment: ImageAttachmentRef /** Encoded request bytes. */ data: Uint8Array mediaType: ImageMediaType @@ -121,7 +125,7 @@ interface RequestImageAttachment { } ``` -`saveImage()` prepares a provider-independent 2048px, 4MiB master and atomically commits it before returning its reference. `saveImages()` prepares every validated master once before publishing the batch, so validation rejection leaves no partial objects and publication does not repeat decoding or quality selection. `admitEncodedImages()` is the wire entry for base64 uploads and delegates count, aggregate-byte, and ordered batch admission to `saveImages()`. `readImage()` verifies a master from an authorized session path. `readImageRequest()` derives and caches one request version under an exact route pixel and byte budget; new entries are fully decoded before publication, while cache hits use a bounded metadata probe. `readImageRequests()` lets an implementation apply its configured transform concurrency to an ordered batch. The local implementation lazily encodes preferred candidates, singleflights equal request identities, lets each waiter cancel independently, stops shared work when no waiter remains, and defaults to two simultaneous transformations. The service is retention-neutral: resumed and forked sessions may share objects, so reference-aware garbage collection is deferred rather than tied to one session's deletion. +`saveImage()` prepares and atomically commits a provider-independent normalized attachment before returning its `ImageAttachmentRef`. `saveImages()` prepares every validated attachment once before publishing the batch, so validation rejection leaves no partial objects and publication does not repeat decoding or quality selection. `admitEncodedImages()` is the wire entry for base64 uploads and delegates count, aggregate-byte, and ordered batch admission to `saveImages()`. `readImage()` verifies a normalized attachment from an authorized session path. `readImageRequest()` derives and caches one request version under an exact route pixel and byte budget; new entries are fully decoded before publication, while cache hits use a bounded metadata probe. Callers use `Promise.all` over the singular method when they need an ordered batch. The local implementation lazily encodes preferred candidates, singleflights equal request identities, lets each waiter cancel independently, stops shared work when no waiter remains, and bounds all transforms with its instance-level limiter, which defaults to two simultaneous transformations. The service is retention-neutral: resumed and forked sessions may share objects, so reference-aware garbage collection is deferred rather than tied to one session's deletion. @@ -149,48 +153,37 @@ abstract validateImage(input: SaveImageAttachment): Promise /** * Validate and durably commit one ordered image batch. * @param inputs - encoded images in owning-message order. - * @returns durable master references in the same order after every member succeeds. + * @returns durable normalized attachment references in the same order after every member succeeds. */ async saveImages(inputs: readonly SaveImageAttachment[]): Promise /** * Validate and durably commit one image before its owning session event is appended. - * Implementations may store a prepared master version of the submitted raster; - * the returned reference always describes the stored bytes, while `source` - * preserves the submitted raster's intrinsic facts for callers that report - * or map coordinates against the original. + * The returned reference describes the persisted normalized image. When + * normalization reduces the raster, its `originalDimensions` records the + * orientation-applied input dimensions. * @param input - encoded bytes, declared media type, and optional display name. - * @returns the durable content-addressed reference beside the submitted source facts. + * @returns the durable content-addressed normalized image reference. */ -abstract saveImage(input: SaveImageAttachment): Promise +abstract saveImage(input: SaveImageAttachment): Promise /** * Read one image and verify that bytes still match the recorded reference. * @param ref - durable reference from the session log. * @param signal - optional cancellation for backend read and verification work. - * @returns the verified bytes and master reference. + * @returns the verified bytes and normalized attachment reference. * @throws the signal reason when aborted, or a storage error when verification fails. */ abstract readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise /** - * Generate or read one deterministic model-request version from the stored master image. - * @param ref - durable provider-independent master reference. + * Generate or read one deterministic model-request version from the stored normalized image. + * @param ref - durable provider-independent normalized attachment reference. * @param policy - exact route pixel and encoded-byte budget. * @param signal - optional cancellation. * @returns request bytes and the cache/upload identity covering every transform input. */ readImageRequest( ref: ImageAttachmentRef, policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise - -/** - * Generate or read an ordered batch of deterministic model-request versions. - * Implementations may use their own bounded transform concurrency while preserving input order. - * @param refs - durable provider-independent master references in request order. - * @param policy - exact route pixel and encoded-byte budget shared by the batch. - * @param signal - optional cancellation. - * @returns request versions in the same order as `refs`. - */ -async readImageRequests( refs: readonly ImageAttachmentRef[], policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise ``` Source: [`packages/attachment/attachment/src/index.ts`](../../packages/attachment/attachment/src/index.ts) diff --git a/docs/subsystems/attachment.zh.md b/docs/subsystems/attachment.zh.md index d5a140e283..8328ec610d 100644 --- a/docs/subsystems/attachment.zh.md +++ b/docs/subsystems/attachment.zh.md @@ -18,7 +18,7 @@ type ImageMediaType = 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif' ``` ```ts type-equiv -/** Durable, serializable metadata for one immutable image object. */ +/** Durable, serializable reference to one immutable normalized image. */ interface ImageAttachmentRef { /** Opaque storage identifier; never a filesystem path or bearer URL. */ attachmentId: AttachmentId @@ -32,10 +32,14 @@ interface ImageAttachmentRef { height: number /** Optional display name stripped of local path information. */ name?: string - /** Perceived source width before master-version downscaling; present only when it differs from {@link width}. */ - sourceWidth?: number - /** Perceived source height before master-version downscaling; present only when it differs from {@link height}. */ - sourceHeight?: number + /** + * Input dimensions after applying EXIF orientation and before normalization + * scaling. Present only when normalization reduced the image. + */ + originalDimensions?: { + width: number + height: number + } } ``` @@ -52,7 +56,7 @@ interface ImageAttachmentLimits { } ``` -本地后端每条消息最多准入 20 张图片,源图编码数据总量不超过 200 MiB。单张源图不得超过 20 MiB、64,000,000 像素和单边 8192 像素。这些源文件限制先于独立的 2048 像素、4 MiB 主版本处理阶段执行。 +本地后端每条消息最多准入 20 张图片,源图编码数据总量不超过 200 MiB。单张源图不得超过 20 MiB、64,000,000 像素和单边 8192 像素。这些源文件限制先于独立的规范化阶段执行;该阶段默认把长边限制为 2048 像素,把编码数据限制为 4 MiB。 引用记录固有尺寸和编码长度,使客户端无需先解码即可排布历史记录;每次权威读取仍会根据对象重新校验摘要、媒体签名、尺寸和元数据。 @@ -100,12 +104,12 @@ interface ImageRequestPolicy { ``` ```ts type-equiv -/** Cached request version derived from one provider-independent master attachment. */ +/** Cached request version derived from one provider-independent normalized attachment. */ interface RequestImageAttachment { - /** Cache and upload-index key over the master id, policy, and fixed encoder parameters. */ + /** Cache and upload-index key over the attachment id, policy, and fixed encoder parameters. */ variantId: ImageVariantId - /** Durable master reference from which this request version was derived. */ - master: ImageAttachmentRef + /** Durable normalized attachment from which this request version was derived. */ + attachment: ImageAttachmentRef /** Encoded request bytes. */ data: Uint8Array mediaType: ImageMediaType @@ -121,7 +125,7 @@ interface RequestImageAttachment { } ``` -`saveImage()` 准备提供方无关的 2048px、4MiB 主版本,并在返回引用前以原子方式提交。`saveImages()` 在发布批次前为每个成员各准备一次经过验证的主版本,因此校验拒绝不会留下部分对象,发布也不会重复解码或选择质量。`admitEncodedImages()` 是面向 base64 上传的 wire 入口,把张数、聚合字节和有序批量准入交给 `saveImages()`。`readImage()` 校验来自已授权会话路径的主版本。`readImageRequest()` 按确切路由的像素和字节预算派生并缓存请求版本;新条目在发布前完整解码,缓存命中只做有界元数据探测。`readImageRequests()` 允许实现按自身配置的变换并发处理有序批次。本地实现按需编码首选候选、合并相同请求身份的并发任务、允许每个等待方单独取消、没有等待方时停止共享任务,默认同时执行两项变换。该服务不规定保留策略:恢复和 fork 后的会话可能共享对象,因此基于引用的垃圾回收会延期实现,不与单个会话的删除绑定。 +`saveImage()` 准备并原子提交提供方无关的规范化附件,然后直接返回 `ImageAttachmentRef`。`saveImages()` 在发布批次前为每个成员各准备一次经过验证的附件,因此校验拒绝不会留下部分对象,发布也不会重复解码或选择质量。`admitEncodedImages()` 是面向 base64 上传的 wire 入口,把张数、聚合字节和有序批量准入交给 `saveImages()`。`readImage()` 校验来自已授权会话路径的规范化附件。`readImageRequest()` 按确切路由的像素和字节预算派生并缓存请求版本;新条目在发布前完整解码,缓存命中只做有界元数据探测。调用方需要有序批次时,对单数方法使用 `Promise.all`。本地实现按需编码首选候选、合并相同请求身份的并发任务、允许每个等待方单独取消、没有等待方时停止共享任务,并通过实例级限流器限制全部变换,默认同时执行两项。该服务不规定保留策略:恢复和 fork 后的会话可能共享对象,因此基于引用的垃圾回收会延期实现,不与单个会话的删除绑定。 @@ -149,48 +153,37 @@ abstract validateImage(input: SaveImageAttachment): Promise /** * Validate and durably commit one ordered image batch. * @param inputs - encoded images in owning-message order. - * @returns durable master references in the same order after every member succeeds. + * @returns durable normalized attachment references in the same order after every member succeeds. */ async saveImages(inputs: readonly SaveImageAttachment[]): Promise /** * Validate and durably commit one image before its owning session event is appended. - * Implementations may store a prepared master version of the submitted raster; - * the returned reference always describes the stored bytes, while `source` - * preserves the submitted raster's intrinsic facts for callers that report - * or map coordinates against the original. + * The returned reference describes the persisted normalized image. When + * normalization reduces the raster, its `originalDimensions` records the + * orientation-applied input dimensions. * @param input - encoded bytes, declared media type, and optional display name. - * @returns the durable content-addressed reference beside the submitted source facts. + * @returns the durable content-addressed normalized image reference. */ -abstract saveImage(input: SaveImageAttachment): Promise +abstract saveImage(input: SaveImageAttachment): Promise /** * Read one image and verify that bytes still match the recorded reference. * @param ref - durable reference from the session log. * @param signal - optional cancellation for backend read and verification work. - * @returns the verified bytes and master reference. + * @returns the verified bytes and normalized attachment reference. * @throws the signal reason when aborted, or a storage error when verification fails. */ abstract readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise /** - * Generate or read one deterministic model-request version from the stored master image. - * @param ref - durable provider-independent master reference. + * Generate or read one deterministic model-request version from the stored normalized image. + * @param ref - durable provider-independent normalized attachment reference. * @param policy - exact route pixel and encoded-byte budget. * @param signal - optional cancellation. * @returns request bytes and the cache/upload identity covering every transform input. */ readImageRequest( ref: ImageAttachmentRef, policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise - -/** - * Generate or read an ordered batch of deterministic model-request versions. - * Implementations may use their own bounded transform concurrency while preserving input order. - * @param refs - durable provider-independent master references in request order. - * @param policy - exact route pixel and encoded-byte budget shared by the batch. - * @param signal - optional cancellation. - * @returns request versions in the same order as `refs`. - */ -async readImageRequests( refs: readonly ImageAttachmentRef[], policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise ``` Source: [`packages/attachment/attachment/src/index.ts`](../../packages/attachment/attachment/src/index.ts) diff --git a/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md b/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md index 678de3e53f..0d2c35c8c6 100644 --- a/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md +++ b/examples/acp-agent/tests/snapshots/code-mode-read-image/system-prompt.expected.md @@ -363,8 +363,10 @@ interface ToolOutputMap { width: number; height: number; name?: string; - sourceWidth?: number; - sourceHeight?: number; + originalDimensions?: { + width: number; + height: number; + }; }; }; send_message: { diff --git a/packages/acp/acp/tests/dispose.spec.ts b/packages/acp/acp/tests/dispose.spec.ts index e5a4a66a3b..4aa32f078c 100644 --- a/packages/acp/acp/tests/dispose.spec.ts +++ b/packages/acp/acp/tests/dispose.spec.ts @@ -30,7 +30,7 @@ describe('ACP connection ownership', () => { it('disposal drains asynchronous assistant image delivery before releasing sessions', async () => { const script: StreamChunk[][] = [] harness = await makeBridgeHarness({ script }) - const { ref } = await harness.attachments!.saveImage({ data: Uint8Array.of(4), mediaType: 'image/png' }) + const ref = await harness.attachments!.saveImage({ data: Uint8Array.of(4), mediaType: 'image/png' }) script.push([ { type: 'block-start', index: 0, blockType: 'image' }, { type: 'block-end', index: 0, block: { type: 'image', attachment: ref } }, diff --git a/packages/acp/acp/tests/harness.ts b/packages/acp/acp/tests/harness.ts index 7c0532e92d..ce6e93794f 100644 --- a/packages/acp/acp/tests/harness.ts +++ b/packages/acp/acp/tests/harness.ts @@ -13,7 +13,7 @@ import { type Stream, } from '@agentclientprotocol/sdk' import AttachmentStore, { AttachmentError, AttachmentId } from '@deepseek-ai/dsh-attachment' -import type { ImageAttachmentLimits, ImageAttachmentRef, SavedImageAttachment, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' +import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' import { type GenerateOptions, LlmAdapter, type LlmResolvedModelInfo, type StreamChunk } from '@deepseek-ai/dsh-llm' import AgentLoop from '@deepseek-ai/dsh-agent-loop' import { mountAgentLoopTestDependencies } from '@deepseek-ai/dsh-agent-loop-testkit' @@ -99,7 +99,7 @@ class MemoryAttachmentStore extends AttachmentStore { if (input.data.byteLength === 0) throw new AttachmentError('Image is empty.', 'INVALID_IMAGE') } - saveImage(input: SaveImageAttachment): Promise { + saveImage(input: SaveImageAttachment): Promise { this.saved.push(input) const digest = createHash('sha256').update(input.data).digest('hex') const ref: ImageAttachmentRef = { @@ -110,10 +110,7 @@ class MemoryAttachmentStore extends AttachmentStore { height: 1, } this.objects.set(ref.attachmentId, { ref, data: Uint8Array.from(input.data) }) - return Promise.resolve({ - ref, - source: { mediaType: ref.mediaType, bytes: ref.bytes, width: ref.width, height: ref.height }, - }) + return Promise.resolve(ref) } async readImage(ref: ImageAttachmentRef): Promise { diff --git a/packages/acp/acp/tests/turns.spec.ts b/packages/acp/acp/tests/turns.spec.ts index c9b229caf1..71e2a21e43 100644 --- a/packages/acp/acp/tests/turns.spec.ts +++ b/packages/acp/acp/tests/turns.spec.ts @@ -44,7 +44,7 @@ describe('ACP prompt lifecycle', () => { it('delivers a committed assistant image as verified ACP base64', async () => { const script: StreamChunk[][] = [] harness = await makeBridgeHarness({ script }) - const { ref } = await harness.attachments!.saveImage({ data: Uint8Array.of(1), mediaType: 'image/png' }) + const ref = await harness.attachments!.saveImage({ data: Uint8Array.of(1), mediaType: 'image/png' }) script.push([ { type: 'block-start', index: 0, blockType: 'image' }, { @@ -68,7 +68,7 @@ describe('ACP prompt lifecycle', () => { it('preserves committed text/image/text order on the ACP wire', async () => { const script: StreamChunk[][] = [] harness = await makeBridgeHarness({ script }) - const { ref } = await harness.attachments!.saveImage({ data: Uint8Array.of(2), mediaType: 'image/jpeg' }) + const ref = await harness.attachments!.saveImage({ data: Uint8Array.of(2), mediaType: 'image/jpeg' }) script.push([ { type: 'block-start', index: 0, blockType: 'text' }, { type: 'block-end', index: 0, block: { type: 'text', text: 'before' } }, @@ -92,7 +92,7 @@ describe('ACP prompt lifecycle', () => { it('does not settle a prompt before ordered output delivery drains', async () => { const script: StreamChunk[][] = [] harness = await makeBridgeHarness({ script }) - const { ref } = await harness.attachments!.saveImage({ data: Uint8Array.of(3), mediaType: 'image/png' }) + const ref = await harness.attachments!.saveImage({ data: Uint8Array.of(3), mediaType: 'image/png' }) script.push([ { type: 'block-start', index: 0, blockType: 'image' }, { type: 'block-end', index: 0, block: { type: 'image', attachment: ref } }, diff --git a/packages/attachment/attachment-local/README.i18n.yaml b/packages/attachment/attachment-local/README.i18n.yaml index a8bfa1b322..412e9a4cb6 100644 --- a/packages/attachment/attachment-local/README.i18n.yaml +++ b/packages/attachment/attachment-local/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/attachment/attachment-local/README.md -README.md: d4831f864dbb061319008242395e2c8ff6d9f642 -README.zh.md: 45bddf47ea5f68c15778040de5b29817e8f62956 +README.md: 849363ce53c6186359ecad34aecb1c2a48f07441 +README.zh.md: f0fe90c2569f60df48998e46d5b05a0d024959df diff --git a/packages/attachment/attachment-local/README.md b/packages/attachment/attachment-local/README.md index d4831f864d..849363ce53 100644 --- a/packages/attachment/attachment-local/README.md +++ b/packages/attachment/attachment-local/README.md @@ -4,9 +4,9 @@ English | [中文](README.zh.md) The private local implementation of [`@deepseek-ai/dsh-attachment`](../attachment). Objects land at `/attachments/v1/objects//` and are addressed by an opaque `sha256:` id. Each process proves a home durable once by syncing every ancestor entry to the filesystem root. Writes use a private staging directory, owner-only files, a synced temporary file, an atomic exclusive hard-link publish, and directory syncs on the publication path (POSIX; Windows relies on filesystem metadata journaling) so the reported reference survives a crash. -Admission accepts at most 20 images and 200MiB of encoded source bytes per message. Each source may use up to 20MiB, 64,000,000 pixels, and 8192px per side. It then prepares a provider-independent master. EXIF orientation is applied, metadata and color profiles are removed, pixels become 8-bit sRGB/sRGBA, and the long edge is reduced proportionally to `masterMaxDimension` (2048px by default). The master has its own `masterMaxBytes` safety cap (4MiB by default). Alpha is retained. A nearest-neighbour bounded sample classifies color complexity without averaging high-frequency pixels. Confirmed low-color images try PNG, using a palette only when the input has no alpha channel, then WebP at qualities 85, 80, and 75. Other alpha images try WebP at those qualities; other opaque images try JPEG. Each candidate runs only after the preceding candidate exceeds the cap. Dimensions shrink only after every candidate at one size exceeds the cap. A clean, single-frame 8-bit sRGB/sRGBA PNG, JPEG, or WebP already within both master limits passes through byte-identically; 16-bit PNG, GIF, animated input, metadata, orientation, and incompatible color spaces force conversion. The source and a converted master are each fully decoded once. `saveImages` prepares and verifies every master once before publishing the batch, so validation failure leaves no partial references and commit does not repeat full image encoding. +Admission accepts at most 20 images and 200MiB of encoded source bytes per message. Each source may use up to 20MiB, 64,000,000 pixels, and 8192px per side. It then prepares a provider-independent normalized attachment. EXIF orientation is applied, metadata and color profiles are removed, pixels become 8-bit sRGB/sRGBA, and the long edge is reduced proportionally to `normalizedImageMaxDimension` (2048px by default). The normalized attachment has its own `normalizedImageMaxBytes` safety cap (4MiB by default). Alpha is retained. A nearest-neighbour bounded sample classifies color complexity without averaging high-frequency pixels. Confirmed low-color images try PNG, using a palette only when the input has no alpha channel, then WebP at qualities 85, 80, and 75. Other alpha images try WebP at those qualities; other opaque images try JPEG. Each candidate runs only after the preceding candidate exceeds the cap. Dimensions shrink only after every candidate at one size exceeds the cap. A clean, single-frame 8-bit sRGB/sRGBA PNG, JPEG, or WebP already within both normalization limits passes through byte-identically; 16-bit PNG, GIF, animated input, metadata, orientation, and incompatible color spaces force conversion. The source and converted attachment are each fully decoded once. `saveImages` prepares and verifies every normalized attachment once before publishing the batch, so validation failure leaves no partial references and commit does not repeat full image encoding. -Request versions live below `/attachments/v1/request-images/`. `readImageRequest` scales the stored master under a total-pixel budget without enlargement, then enforces a separate encoded-byte cap. The request encoder uses the same color branches, with PNG (palette only without alpha) before WebP 85 and 80 for low-color images, WebP 85 then 80 for other alpha images, and JPEG 85 then 80 for other opaque images. It also executes candidates lazily and reduces dimensions only after both quality attempts exceed the request cap. Its cache identity includes the master id, transform version, pixel and byte budgets, and fixed encoder settings. Cached bytes are fully decoded and checked as 8-bit sRGB/sRGBA before use. Concurrent calls for one identity share one transform and cache write; cancelling one waiter does not cancel the shared work. `readImageRequests` schedules batches through the service's FIFO limiter. `imageCompressionConcurrency` controls simultaneous master and request transforms from 1 through 8 and defaults to 2; file publication remains ordered after preparation. +Request versions live below `/attachments/v1/request-images/`. `readImageRequest` scales the stored normalized attachment under a total-pixel budget without enlargement, then enforces a separate encoded-byte cap. The request encoder uses the same color branches, with PNG (palette only without alpha) before WebP 85 and 80 for low-color images, WebP 85 then 80 for other alpha images, and JPEG 85 then 80 for other opaque images. It executes candidates lazily and reduces dimensions only after both quality attempts exceed the request cap. Its cache identity includes the attachment id, transform version, pixel and byte budgets, and fixed encoder settings. Cached bytes are fully decoded and checked as 8-bit sRGB/sRGBA before use. Concurrent calls for one identity share one transform and cache write; cancelling one waiter does not cancel the shared work. Callers compose ordered batches from singular reads, while the service's FIFO limiter applies `imageCompressionConcurrency` to simultaneous normalization and request transforms. The setting ranges from 1 through 8 and defaults to 2; file publication remains ordered after preparation. `DSH_HOME` resolves through the shared path policy: explicit config, `$DSH_HOME`, then `~/.dsh`. Session logs contain only the reference and verified metadata, never this host path. `readImage` forwards optional cancellation into the filesystem read, observes it around verification, and preserves it instead of wrapping it as `ATTACHMENT_READ_FAILED`. @@ -16,11 +16,11 @@ Indirectly, through durable replay of historical user images and structured mode #### KV Cache effect -Master preparation and request projection are deterministic. An unchanged master and route policy reuse identical cached request bytes on later turns. +Normalization and request projection are deterministic. An unchanged attachment and route policy reuse identical cached request bytes on later turns. ## Known Limitations and Deferred Work - Objects are retained indefinitely; reference-aware garbage collection is deferred. - The local backend assumes the host and provider adapter share this filesystem service. - Animated GIF sources keep only their first frame; animation is outside the version-one image contract. -- The master and request encoders are pinned by the installed sharp/libvips build; an encoder or transform-version upgrade re-addresses future masters or request variants while existing objects stay valid. +- The normalization and request encoders are pinned by the installed sharp/libvips build; an encoder or transform-version upgrade re-addresses future normalized attachments or request variants while existing objects stay valid. diff --git a/packages/attachment/attachment-local/README.zh.md b/packages/attachment/attachment-local/README.zh.md index 45bddf47ea..f0fe90c256 100644 --- a/packages/attachment/attachment-local/README.zh.md +++ b/packages/attachment/attachment-local/README.zh.md @@ -4,9 +4,9 @@ 这是 [`@deepseek-ai/dsh-attachment`](../attachment) 的私有本地实现。对象存放在 `/attachments/v1/objects//`,并通过不透明的 `sha256:` 标识符寻址。每个进程都会把每级祖先目录项同步到文件系统根目录,以此一次性证明 home 已持久化。写入使用私有暂存目录、仅所有者可访问的文件、经过同步的临时文件、原子且排他的硬链接发布,并对发布路径执行目录同步(适用于 POSIX;Windows 依赖文件系统元数据日志),确保已报告的引用能够在崩溃后继续存在。 -每条消息最多准入 20 张图片,源图编码字节总量不超过 200MiB。每张源图不得超过 20MiB、64,000,000 像素和单边 8192px。随后生成提供方无关的主版本:应用 EXIF 方向,删除元数据和色彩配置文件,转换为 8-bit sRGB/sRGBA,并保持宽高比把长边限制到 `masterMaxDimension`(默认 2048px)。主版本有独立的 `masterMaxBytes` 安全上限(默认 4MiB)。透明通道会保留。系统用 nearest-neighbour 对有界样本分类,不会通过像素平均把高频图片误判为低色数。确认的低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80、75 的 WebP;其他透明图片依次尝试这些质量的 WebP;其他非透明图片依次尝试这些质量的 JPEG。只有前一个候选超限时才会执行下一个候选;同一尺寸的候选全部超限后才缩小尺寸。已经处于两个主版本上限内的干净、单帧、8-bit sRGB/sRGBA PNG、JPEG 或 WebP 按字节原样直通;16-bit PNG、GIF、动图、元数据、方向和不兼容色彩空间都会触发转换。源图和转换后的主版本各完整解码一次。`saveImages` 在发布任何批次成员前为每张图片各准备并验证一次主版本,因此校验失败不会留下部分引用,提交阶段也不会重复执行完整图片编码。 +每条消息最多准入 20 张图片,源图编码字节总量不超过 200MiB。每张源图不得超过 20MiB、64,000,000 像素和单边 8192px。随后生成提供方无关的规范化附件:应用 EXIF 方向,删除元数据和色彩配置文件,转换为 8-bit sRGB/sRGBA,并保持宽高比把长边限制到 `normalizedImageMaxDimension`(默认 2048px)。规范化附件有独立的 `normalizedImageMaxBytes` 安全上限(默认 4MiB)。透明通道会保留。系统用 nearest-neighbour 对有界样本分类,不会通过像素平均把高频图片误判为低色数。确认的低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80、75 的 WebP;其他透明图片依次尝试这些质量的 WebP;其他非透明图片依次尝试这些质量的 JPEG。只有前一个候选超限时才会执行下一个候选;同一尺寸的候选全部超限后才缩小尺寸。已经处于两个规范化上限内的干净、单帧、8-bit sRGB/sRGBA PNG、JPEG 或 WebP 按字节原样直通;16-bit PNG、GIF、动图、元数据、方向和不兼容色彩空间都会触发转换。源图和转换后的附件各完整解码一次。`saveImages` 在发布任何批次成员前为每张图片各准备并验证一次规范化附件,因此校验失败不会留下部分引用,提交阶段也不会重复执行完整图片编码。 -请求版本保存在 `/attachments/v1/request-images/`。`readImageRequest` 在不放大小图的前提下,把存储的主版本缩放到总像素预算内,再执行独立的编码字节上限。请求编码器使用同一分类分支:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。候选仍按需执行,两个质量档均超限后才缩小尺寸。缓存身份包含主版本 ID、变换策略版本、像素和字节预算及固定编码参数。缓存字节在使用前会完整解码并校验为 8-bit sRGB/sRGBA。同一身份的并发调用共享一次变换和缓存写入;取消一个等待方不会取消共享任务。`readImageRequests` 通过服务的 FIFO 限流器调度批次。`imageCompressionConcurrency` 控制同时执行的主版本和请求版本变换,范围为 1 至 8,默认值为 2;文件发布仍在准备结束后按顺序执行。 +请求版本保存在 `/attachments/v1/request-images/`。`readImageRequest` 在不放大小图的前提下,把存储的规范化附件缩放到总像素预算内,再执行独立的编码字节上限。请求编码器使用同一分类分支:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。候选按需执行,两个质量档均超限后才缩小尺寸。缓存身份包含附件 ID、变换策略版本、像素和字节预算及固定编码参数。缓存字节在使用前会完整解码并校验为 8-bit sRGB/sRGBA。同一身份的并发调用共享一次变换和缓存写入;取消一个等待方不会取消共享任务。调用方组合单数读取得到有序批次,服务的 FIFO 限流器通过 `imageCompressionConcurrency` 限制同时执行的规范化和请求变换。该配置范围为 1 至 8,默认值为 2;文件发布仍在准备结束后按顺序执行。 `DSH_HOME` 按共享路径策略解析:显式配置、`$DSH_HOME`,最后是 `~/.dsh`。会话日志只包含引用和经过校验的元数据,绝不包含这个宿主路径。`readImage` 会把可选取消信号传入文件系统读取、在校验前后观察该信号,并保留取消语义,而不会将其包装成 `ATTACHMENT_READ_FAILED`。 @@ -16,11 +16,11 @@ #### KV 缓存影响 -主版本准备和请求投影都是确定性的。主版本和路由策略不变时,之后各轮会复用相同的缓存请求字节。 +规范化和请求投影都是确定性的。附件和路由策略不变时,之后各轮会复用相同的缓存请求字节。 ## 已知限制与待完成工作 - 对象会无限期保留;基于引用的垃圾回收尚未实现。 - 本地后端假定宿主与提供方适配器共享同一个文件系统服务。 - 动态 GIF 源图只保留首帧;动画在版本一图片契约之外。 -- 主版本和请求版本编码器由安装的 sharp/libvips 构建钉定;编码器或变换策略版本升级会让未来的主版本或请求变体产生新地址,已有对象保持有效。 +- 规范化和请求版本编码器由安装的 sharp/libvips 构建钉定;编码器或变换策略版本升级会让未来的规范化附件或请求变体产生新地址,已有对象保持有效。 diff --git a/packages/attachment/attachment-local/src/encoding.ts b/packages/attachment/attachment-local/src/encoding.ts index 963edda672..bf83d48cf9 100644 --- a/packages/attachment/attachment-local/src/encoding.ts +++ b/packages/attachment/attachment-local/src/encoding.ts @@ -1,4 +1,4 @@ -/** Shared lazy candidate execution for master and request-image encoders. */ +/** Shared lazy candidate execution for normalization and request-image encoders. */ /** One encoded candidate carrying its complete bytes. */ export interface EncodedCandidate { diff --git a/packages/attachment/attachment-local/src/index.ts b/packages/attachment/attachment-local/src/index.ts index 9007544047..e9a1145ba5 100644 --- a/packages/attachment/attachment-local/src/index.ts +++ b/packages/attachment/attachment-local/src/index.ts @@ -10,17 +10,16 @@ import type { ImageRequestPolicy, RequestImageAttachment, SaveImageAttachment, - SavedImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' import { resolveDshHome } from '@deepseek-ai/dsh-home-paths' -import type { MasterImagePolicy } from './canonical.ts' +import type { NormalizationPolicy } from './normalization.ts' import { CompressionLimiter } from './compression-limiter.ts' import { commitPreparedImageFile, prepareImageFile, readImageFile, validateImageFile } from './store.ts' import { readRequestImageFile, requestImageVariantId } from './request-image.ts' -export { isMasterImage, prepareMasterImage } from './canonical.ts' -export type { MasterImage, MasterImagePolicy } from './canonical.ts' +export { canPassThroughNormalization, normalizeImage } from './normalization.ts' +export type { NormalizedImage, NormalizationPolicy } from './normalization.ts' export { commitPreparedImageFile, prepareImageFile, readImageFile, saveImageFile, validateImageFile } from './store.ts' export type { PreparedImageFile } from './store.ts' export { readRequestImageFile, requestImageDimensions, requestImageVariantId } from './request-image.ts' @@ -36,13 +35,13 @@ export const DEFAULT_MAX_IMAGE_PIXELS = 64_000_000 /** Default per-side pixel cap for one submitted image. */ export const DEFAULT_MAX_IMAGE_DIMENSION = 8192 /** - * Default long-edge target of the stored image master. A larger source + * Default long-edge target of the stored normalized image. A larger source * is admitted and downscaled to this edge, so admission bounds what rides * every later model request without refusing ordinary large sources. */ -export const DEFAULT_MASTER_MAX_DIMENSION = 2048 -/** Default independent safety cap for one stored master version. */ -export const DEFAULT_MASTER_MAX_BYTES = 4 * 1024 * 1024 +export const DEFAULT_NORMALIZED_IMAGE_MAX_DIMENSION = 2048 +/** Default independent safety cap for one stored normalized image. */ +export const DEFAULT_NORMALIZED_IMAGE_MAX_BYTES = 4 * 1024 * 1024 /** Conservative default number of simultaneous native image transformations per store. */ export const DEFAULT_IMAGE_COMPRESSION_CONCURRENCY = 2 /** Maximum configurable native image transformations per store. */ @@ -62,11 +61,11 @@ export interface Config { maxImagePixels?: number /** Maximum intrinsic width and maximum intrinsic height accepted for one submitted image. Default: 8192px. */ maxImageDimension?: number - /** Long-edge pixel cap of the stored provider-independent master version. */ - masterMaxDimension?: number - /** Encoded-byte safety cap of the stored provider-independent master version. */ - masterMaxBytes?: number - /** Maximum simultaneous master or request-image transformations in this service instance. */ + /** Long-edge pixel cap of the stored provider-independent normalized image. */ + normalizedImageMaxDimension?: number + /** Encoded-byte safety cap of the stored provider-independent normalized image. */ + normalizedImageMaxBytes?: number + /** Maximum simultaneous normalization or request-image transformations in this service instance. */ imageCompressionConcurrency?: number } @@ -140,8 +139,8 @@ export class LocalAttachmentStore extends AttachmentStore { maxMessageImageBytes: z.number().step(1).min(1).default(DEFAULT_MAX_MESSAGE_IMAGE_BYTES), maxImagePixels: z.number().step(1).min(1).default(DEFAULT_MAX_IMAGE_PIXELS), maxImageDimension: z.number().step(1).min(1).default(DEFAULT_MAX_IMAGE_DIMENSION), - masterMaxDimension: z.number().step(1).min(1).default(DEFAULT_MASTER_MAX_DIMENSION), - masterMaxBytes: z.number().step(1).min(1).default(DEFAULT_MASTER_MAX_BYTES), + normalizedImageMaxDimension: z.number().step(1).min(1).default(DEFAULT_NORMALIZED_IMAGE_MAX_DIMENSION), + normalizedImageMaxBytes: z.number().step(1).min(1).default(DEFAULT_NORMALIZED_IMAGE_MAX_BYTES), imageCompressionConcurrency: z.number().step(1).min(1).max(MAX_IMAGE_COMPRESSION_CONCURRENCY) .default(DEFAULT_IMAGE_COMPRESSION_CONCURRENCY), }) @@ -149,8 +148,8 @@ export class LocalAttachmentStore extends AttachmentStore { /** Absolute versioned storage root. */ readonly root: string readonly imageLimits: ImageAttachmentLimits - /** Resolved provider-independent master-version storage policy. */ - readonly masterPolicy: Readonly + /** Resolved provider-independent normalization policy. */ + readonly normalizationPolicy: Readonly /** Resolved instance-level compression limit. */ readonly imageCompressionConcurrency: number private readonly compression: CompressionLimiter @@ -167,9 +166,9 @@ export class LocalAttachmentStore extends AttachmentStore { maxImageDimension: config.maxImageDimension ?? DEFAULT_MAX_IMAGE_DIMENSION, mediaTypes: Object.freeze(['image/png', 'image/jpeg', 'image/webp', 'image/gif'] as const), }) - this.masterPolicy = Object.freeze({ - maxDimension: config.masterMaxDimension ?? DEFAULT_MASTER_MAX_DIMENSION, - maxBytes: config.masterMaxBytes ?? DEFAULT_MASTER_MAX_BYTES, + this.normalizationPolicy = Object.freeze({ + maxDimension: config.normalizedImageMaxDimension ?? DEFAULT_NORMALIZED_IMAGE_MAX_DIMENSION, + maxBytes: config.normalizedImageMaxBytes ?? DEFAULT_NORMALIZED_IMAGE_MAX_BYTES, }) const compressionConcurrency = config.imageCompressionConcurrency ?? DEFAULT_IMAGE_COMPRESSION_CONCURRENCY if (!Number.isSafeInteger(compressionConcurrency) @@ -184,22 +183,22 @@ export class LocalAttachmentStore extends AttachmentStore { } async validateImage(input: SaveImageAttachment): Promise { - await this.compression.run(() => validateImageFile(input, this.imageLimits, this.masterPolicy)) + await this.compression.run(() => validateImageFile(input, this.imageLimits, this.normalizationPolicy)) } override async saveImages(inputs: readonly SaveImageAttachment[]): Promise { this.validateImageBatch(inputs) const prepared = await Promise.all(inputs.map(input => this.compression.run( - () => prepareImageFile(input, this.imageLimits, this.masterPolicy), + () => prepareImageFile(input, this.imageLimits, this.normalizationPolicy), ))) const refs: ImageAttachmentRef[] = [] - for (const image of prepared) refs.push((await commitPreparedImageFile(this.root, image)).ref) + for (const image of prepared) refs.push(await commitPreparedImageFile(this.root, image)) return refs } - async saveImage(input: SaveImageAttachment): Promise { + async saveImage(input: SaveImageAttachment): Promise { const prepared = await this.compression.run( - () => prepareImageFile(input, this.imageLimits, this.masterPolicy), + () => prepareImageFile(input, this.imageLimits, this.normalizationPolicy), ) return commitPreparedImageFile(this.root, prepared) } @@ -216,18 +215,10 @@ export class LocalAttachmentStore extends AttachmentStore { return this.requestVersion(ref, policy, undefined, signal) } - override async readImageRequests( - refs: readonly ImageAttachmentRef[], - policy: ImageRequestPolicy, - signal?: AbortSignal, - ): Promise { - return Promise.all(refs.map(ref => this.requestVersion(ref, policy, undefined, signal))) - } - private requestVersion( ref: ImageAttachmentRef, policy: ImageRequestPolicy, - master: StoredImageAttachment | undefined, + stored: StoredImageAttachment | undefined, signal: AbortSignal | undefined, ): Promise { signal?.throwIfAborted() @@ -241,7 +232,7 @@ export class LocalAttachmentStore extends AttachmentStore { if (operation === undefined) { const shared = new SharedRequest(sharedSignal => this.compression.run(async () => readRequestImageFile( this.root, - master ?? await this.readImage(ref, sharedSignal), + stored ?? await this.readImage(ref, sharedSignal), policy, sharedSignal, ))) diff --git a/packages/attachment/attachment-local/src/canonical.ts b/packages/attachment/attachment-local/src/normalization.ts similarity index 77% rename from packages/attachment/attachment-local/src/canonical.ts rename to packages/attachment/attachment-local/src/normalization.ts index 513e5bbcc7..acfec63c0f 100644 --- a/packages/attachment/attachment-local/src/canonical.ts +++ b/packages/attachment/attachment-local/src/normalization.ts @@ -1,4 +1,4 @@ -/** Deterministic provider-independent master-image encoding. */ +/** Deterministic provider-independent image normalization. */ import sharp, { type Sharp } from 'sharp' import { AttachmentError } from '@deepseek-ai/dsh-attachment' @@ -7,23 +7,23 @@ import { encodeFirstWithinLimit, isExhaustedEncoding } from './encoding.ts' import { detectImage } from './image.ts' import type { DetectedImage } from './image.ts' -/** Deployment-resolved storage policy for the provider-independent master version. */ -export interface MasterImagePolicy { +/** Deployment-resolved policy for the persisted normalized attachment. */ +export interface NormalizationPolicy { /** Long-edge cap in pixels; larger sources are downscaled proportionally. */ maxDimension: number - /** Independent safety cap for encoded master bytes. */ + /** Independent safety cap for encoded normalized image bytes. */ maxBytes: number } -/** Master bytes beside the facts recorded by a durable reference. */ -export interface MasterImage { +/** Normalized bytes beside the facts recorded by a durable reference. */ +export interface NormalizedImage { data: Uint8Array mediaType: ImageMediaType width: number height: number } -const MASTER_QUALITIES = [85, 80, 75] as const +const NORMALIZATION_QUALITIES = [85, 80, 75] as const const LOW_COLOUR_SAMPLE_EDGE = 128 const LOW_COLOUR_LIMIT = 256 const MIN_SCALE_STEP = 0.9 @@ -34,7 +34,7 @@ async function encode( mediaType: 'image/png' | 'image/jpeg' | 'image/webp', quality?: number, palette = true, -): Promise { +): Promise { const encoded = mediaType === 'image/png' ? pipeline.png({ compressionLevel: 9, palette }) : mediaType === 'image/webp' @@ -45,13 +45,17 @@ async function encode( } /** - * Whether bytes already satisfy the master-version storage contract. + * Whether bytes already satisfy the normalization requirements. * @param detected - fully decoded source facts. * @param bytes - encoded source length. - * @param policy - resolved master limits. + * @param policy - resolved normalization limits. * @returns whether the source can pass through byte-identically. */ -export function isMasterImage(detected: DetectedImage, bytes: number, policy: MasterImagePolicy): boolean { +export function canPassThroughNormalization( + detected: DetectedImage, + bytes: number, + policy: NormalizationPolicy, +): boolean { return detected.mediaType !== 'image/gif' && !detected.animated && !detected.carriesMetadata @@ -87,8 +91,11 @@ export async function hasLowColourCount(pipeline: Sharp): Promise { return true } -/** Assert that a re-encoded master is an 8-bit sRGB/sRGBA single-frame image with matching facts. */ -async function verifyMaster(image: MasterImage, expectedAlpha: boolean | undefined): Promise { +/** Assert that a normalized output is an 8-bit sRGB/sRGBA single-frame image with matching facts. */ +async function verifyNormalizedImage( + image: NormalizedImage, + expectedAlpha: boolean | undefined, +): Promise { const detected = await detectImage(image.data) if (detected.mediaType !== image.mediaType || detected.width !== image.width @@ -99,7 +106,7 @@ async function verifyMaster(image: MasterImage, expectedAlpha: boolean | undefin || detected.space !== 'srgb' || (expectedAlpha !== undefined && detected.hasAlpha !== expectedAlpha)) { throw new AttachmentError( - 'Canonical image conversion did not produce a single-frame 8-bit sRGB image with matching metadata.', + 'Image normalization did not produce a single-frame 8-bit sRGB image with matching metadata.', 'ATTACHMENT_WRITE_FAILED', ) } @@ -130,36 +137,36 @@ function encodingAttemptsAtSize( height: number, hasAlpha: boolean, lowColour: boolean, -): Array<() => Promise> { +): Array<() => Promise> { const prepared = preparedPipeline(data, width, height) - const webp = MASTER_QUALITIES.map(quality => ( + const webp = NORMALIZATION_QUALITIES.map(quality => ( () => encode(prepared.clone(), 'image/webp', quality) )) if (lowColour) { return [() => encode(prepared.clone(), 'image/png', undefined, !hasAlpha), ...webp] } if (hasAlpha) return webp - return MASTER_QUALITIES.map(quality => ( + return NORMALIZATION_QUALITIES.map(quality => ( () => encode(prepared.clone(), 'image/jpeg', quality) )) } /** - * Produce the 2048px provider-independent master version of one fully decoded source. + * Produce the persisted provider-independent normalized version of one fully decoded source. * The source is passed through only when it is already clean, single-frame, 8-bit sRGB/sRGBA, - * and inside both master limits. Re-encoding never removes transparency. After the fixed + * and inside both normalization limits. Re-encoding never removes transparency. After the fixed * quality floor is reached, dimensions continue shrinking until the independent byte cap holds. * @param data - complete admitted source bytes. * @param detected - fully decoded source facts. - * @param policy - resolved independent master limits. - * @returns verified provider-independent master bytes and metadata. + * @param policy - resolved independent normalization limits. + * @returns verified provider-independent normalized bytes and metadata. */ -export async function prepareMasterImage( +export async function normalizeImage( data: Uint8Array, detected: DetectedImage, - policy: MasterImagePolicy, -): Promise { - if (isMasterImage(detected, data.byteLength, policy)) { + policy: NormalizationPolicy, +): Promise { + if (canPassThroughNormalization(detected, data.byteLength, policy)) { return { data, mediaType: detected.mediaType, width: detected.width, height: detected.height } } try { @@ -174,7 +181,7 @@ export async function prepareMasterImage( policy.maxBytes, ) if (!isExhaustedEncoding(encoded)) { - return await verifyMaster(encoded, detected.mediaType === 'image/gif' ? undefined : detected.hasAlpha) + return await verifyNormalizedImage(encoded, detected.mediaType === 'image/gif' ? undefined : detected.hasAlpha) } if (width === 1 && height === 1) break const sizeScale = Math.sqrt(policy.maxBytes / encoded.smallest.data.byteLength) * 0.95 @@ -190,10 +197,10 @@ export async function prepareMasterImage( ? `${detected.depth === 'ushort' ? '16-bit' : detected.depth} PNG` : `${detected.depth} ${detected.mediaType.slice('image/'.length).toUpperCase()}` throw new AttachmentError( - `The ${source} could not be converted to the canonical 8-bit sRGB form.`, + `The ${source} could not be converted to the normalized 8-bit sRGB form.`, 'ATTACHMENT_WRITE_FAILED', { cause: error }, ) } - throw new AttachmentError('Image cannot be encoded within the configured master-image byte cap.', 'IMAGE_TOO_LARGE') + throw new AttachmentError('Image cannot be encoded within the configured normalized-image byte cap.', 'IMAGE_TOO_LARGE') } diff --git a/packages/attachment/attachment-local/src/request-image.ts b/packages/attachment/attachment-local/src/request-image.ts index c37a473d4c..b7c9068bfb 100644 --- a/packages/attachment/attachment-local/src/request-image.ts +++ b/packages/attachment/attachment-local/src/request-image.ts @@ -12,12 +12,12 @@ import type { RequestImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' -import { hasLowColourCount } from './canonical.ts' +import { hasLowColourCount } from './normalization.ts' import { encodeFirstWithinLimit, isExhaustedEncoding } from './encoding.ts' import { detectImage, probeImage } from './image.ts' /** Transform version included in every cache and upload-index identity. */ -export const REQUEST_IMAGE_TRANSFORM_VERSION = 'request-image-v3' +export const REQUEST_IMAGE_TRANSFORM_VERSION = 'request-image-v4' /** DeepSeek request versions normally fit at these two preferred qualities. */ export const REQUEST_IMAGE_QUALITIES = [85, 80] as const @@ -80,10 +80,10 @@ function validatePolicy(policy: ImageRequestPolicy): void { checkedInteger(policy.maxBytes, 'Image request maxBytes') } -function descriptor(master: ImageAttachmentRef, policy: ImageRequestPolicy): string { +function descriptor(attachment: ImageAttachmentRef, policy: ImageRequestPolicy): string { return JSON.stringify({ transformVersion: REQUEST_IMAGE_TRANSFORM_VERSION, - masterAttachmentId: master.attachmentId, + attachmentId: attachment.attachmentId, routePixelBudget: policy.maxPixels, encodedByteBudget: policy.maxBytes, encoding: { @@ -97,25 +97,25 @@ function descriptor(master: ImageAttachmentRef, policy: ImageRequestPolicy): str } /** - * Complete deterministic identity for one master and route-owned request policy. - * @param master - provider-independent durable master reference. + * Complete deterministic identity for one attachment and route-owned request policy. + * @param attachment - provider-independent durable normalized attachment reference. * @param policy - route-owned pixel and byte policy. * @returns branded digest over every request transform input. */ export function requestImageVariantId( - master: ImageAttachmentRef, + attachment: ImageAttachmentRef, policy: ImageRequestPolicy, ): ReturnType { - return ImageVariantId(`sha256:${digest(descriptor(master, policy))}`) + return ImageVariantId(`sha256:${digest(descriptor(attachment, policy))}`) } -function pipeline(master: StoredImageAttachment, width: number, height: number): Sharp { - return sourcePipeline(master) +function pipeline(attachment: StoredImageAttachment, width: number, height: number): Sharp { + return sourcePipeline(attachment) .resize({ width, height, fit: 'inside', withoutEnlargement: true }) } -function sourcePipeline(master: StoredImageAttachment): Sharp { - return sharp(master.data, { failOn: 'error', limitInputPixels: false }).toColourspace('srgb') +function sourcePipeline(attachment: StoredImageAttachment): Sharp { + return sharp(attachment.data, { failOn: 'error', limitInputPixels: false }).toColourspace('srgb') } async function encoded( @@ -134,13 +134,13 @@ async function encoded( } function encodingAttempts( - master: StoredImageAttachment, + attachment: StoredImageAttachment, width: number, height: number, hasAlpha: boolean, lowColour: boolean, ): Array<() => Promise> { - const prepared = pipeline(master, width, height) + const prepared = pipeline(attachment, width, height) const webp = REQUEST_IMAGE_QUALITIES.map(quality => ( () => encoded(prepared.clone(), 'image/webp', quality) )) @@ -152,25 +152,25 @@ function encodingAttempts( } async function createRequestImage( - master: StoredImageAttachment, + attachment: StoredImageAttachment, policy: ImageRequestPolicy, hasAlpha: boolean, ): Promise { - let dimensions = requestImageDimensions(master.ref.width, master.ref.height, policy.maxPixels) - if (dimensions.width === master.ref.width - && dimensions.height === master.ref.height - && master.data.byteLength <= policy.maxBytes) { + let dimensions = requestImageDimensions(attachment.ref.width, attachment.ref.height, policy.maxPixels) + if (dimensions.width === attachment.ref.width + && dimensions.height === attachment.ref.height + && attachment.data.byteLength <= policy.maxBytes) { return { - data: master.data, - mediaType: master.ref.mediaType, - width: master.ref.width, - height: master.ref.height, + data: attachment.data, + mediaType: attachment.ref.mediaType, + width: attachment.ref.width, + height: attachment.ref.height, } } - const lowColour = await hasLowColourCount(sourcePipeline(master)) + const lowColour = await hasLowColourCount(sourcePipeline(attachment)) for (;;) { const encodedVersion = await encodeFirstWithinLimit( - encodingAttempts(master, dimensions.width, dimensions.height, hasAlpha, lowColour), + encodingAttempts(attachment, dimensions.width, dimensions.height, hasAlpha, lowColour), policy.maxBytes, ) if (!isExhaustedEncoding(encodedVersion)) return encodedVersion @@ -190,7 +190,7 @@ function cachePath(root: string, hash: string): string { async function readCached( path: string, - master: StoredImageAttachment, + attachment: StoredImageAttachment, policy: ImageRequestPolicy, expectedAlpha: boolean, signal?: AbortSignal, @@ -198,7 +198,7 @@ async function readCached( try { const data = new Uint8Array(await readFile(path, { signal })) const detected = await probeImage(data) - const maximum = requestImageDimensions(master.ref.width, master.ref.height, policy.maxPixels) + const maximum = requestImageDimensions(attachment.ref.width, attachment.ref.height, policy.maxPixels) if (data.byteLength > policy.maxBytes || detected.depth !== 'uchar' || detected.space !== 'srgb' || detected.width > maximum.width || detected.height > maximum.height || detected.hasAlpha !== expectedAlpha) return undefined @@ -240,33 +240,33 @@ async function writeCached(path: string, data: Uint8Array): Promise { /** * Generate or reuse one request image below the local attachment root. * @param root - absolute versioned attachment storage root. - * @param master - verified stored master bytes and reference. + * @param attachment - verified normalized attachment bytes and reference. * @param policy - exact route request-image policy. * @param signal - optional cancellation for cache I/O and image transformation. * @returns verified request bytes and deterministic variant identity. */ export async function readRequestImageFile( root: string, - master: StoredImageAttachment, + attachment: StoredImageAttachment, policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise { signal?.throwIfAborted() validatePolicy(policy) - const source = await probeImage(master.data) - const variantId = requestImageVariantId(master.ref, policy) + const source = await probeImage(attachment.data) + const variantId = requestImageVariantId(attachment.ref, policy) const hash = String(variantId).slice('sha256:'.length) const path = cachePath(root, hash) - const cached = await readCached(path, master, policy, source.hasAlpha, signal) - const created = cached ?? await createRequestImage(master, policy, source.hasAlpha) - const version = cached ?? (created.data === master.data + const cached = await readCached(path, attachment, policy, source.hasAlpha, signal) + const created = cached ?? await createRequestImage(attachment, policy, source.hasAlpha) + const version = cached ?? (created.data === attachment.data ? { ...created, hasAlpha: source.hasAlpha } : await verifyRequestImage(created, source.hasAlpha)) signal?.throwIfAborted() - if (cached === undefined && version.data !== master.data) await writeCached(path, version.data) + if (cached === undefined && version.data !== attachment.data) await writeCached(path, version.data) return { variantId, - master: master.ref, + attachment: attachment.ref, data: version.data, mediaType: version.mediaType, bytes: version.data.byteLength, diff --git a/packages/attachment/attachment-local/src/store.ts b/packages/attachment/attachment-local/src/store.ts index ba45256416..5fbb8e9201 100644 --- a/packages/attachment/attachment-local/src/store.ts +++ b/packages/attachment/attachment-local/src/store.ts @@ -12,12 +12,10 @@ import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, - SavedImageAttachment, - SourceImageInfo, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' -import { prepareMasterImage } from './canonical.ts' -import type { MasterImagePolicy } from './canonical.ts' +import { normalizeImage } from './normalization.ts' +import type { NormalizationPolicy } from './normalization.ts' import { detectImage, probeImage } from './image.ts' import type { DetectedImage } from './image.ts' @@ -52,71 +50,69 @@ async function inspectMetadata( data: Uint8Array, declaredMediaType: ImageAttachmentRef['mediaType'], limits: ImageAttachmentLimits, -): Promise<{ detected: DetectedImage; source: SourceImageInfo }> { +): Promise { if (data.byteLength === 0) throw new AttachmentError('Image is empty.', 'INVALID_IMAGE') const detected = await detectImage(data, { maxPixels: limits.maxImagePixels, maxDimension: limits.maxImageDimension }) if (detected.mediaType !== declaredMediaType) throw new AttachmentError('Declared image type does not match its bytes.', 'IMAGE_TYPE_MISMATCH') - return { - detected, - source: { mediaType: detected.mediaType, bytes: data.byteLength, width: detected.width, height: detected.height }, - } + return detected } /** * Run the full admission policy for one image without touching storage, - * including master-version preparation: a batch whose members all validate - * cannot later be refused by the master byte cap during publication. + * including normalization: a batch whose members all validate cannot later + * be refused by the normalized image byte cap during publication. * @param input - encoded bytes and declared metadata. * @param limits - resolved source admission policy. - * @param policy - resolved master-version storage policy. - * @returns completion after the raster has been decoded and its master version proven to fit. + * @param policy - resolved normalization policy. + * @returns completion after the raster has been decoded and its normalized version proven to fit. */ export async function validateImageFile( input: SaveImageAttachment, limits: ImageAttachmentLimits, - policy: MasterImagePolicy, + policy: NormalizationPolicy, ): Promise { await prepareImageFile(input, limits, policy) } -/** Fully prepared master object, verified before any batch member is persisted. */ -export interface PreparedImageFile extends SavedImageAttachment { - /** Deterministic master bytes whose digest is {@link ref.attachmentId}. */ +/** Fully prepared normalized object, verified before any batch member is persisted. */ +export interface PreparedImageFile { + /** Deterministic normalized bytes whose digest is {@link ref.attachmentId}. */ data: Uint8Array + /** Durable reference describing {@link data}. */ + ref: ImageAttachmentRef } /** * Decode, normalize, and verify one submitted image without touching storage. * @param input - submitted encoded bytes and declared media type. * @param limits - source admission policy. - * @param policy - independent master-version storage policy. + * @param policy - independent normalization policy. * @returns immutable reference facts beside bytes ready for atomic publication. */ export async function prepareImageFile( input: SaveImageAttachment, limits: ImageAttachmentLimits, - policy: MasterImagePolicy, + policy: NormalizationPolicy, ): Promise { if (input.data.byteLength > limits.maxImageBytes) { throw new AttachmentError('Image exceeds the configured byte limit.', 'IMAGE_TOO_LARGE') } - const { detected, source } = await inspectMetadata(input.data, input.mediaType, limits) - const master = await prepareMasterImage(input.data, detected, policy) - const sha256 = digest(master.data) + const detected = await inspectMetadata(input.data, input.mediaType, limits) + const normalized = await normalizeImage(input.data, detected, policy) + const sha256 = digest(normalized.data) const name = displayName(input.name) - const downscaled = source.width !== master.width || source.height !== master.height + const downscaled = detected.width !== normalized.width || detected.height !== normalized.height return { - data: master.data, + data: normalized.data, ref: { attachmentId: AttachmentId(`sha256:${sha256}`), - mediaType: master.mediaType, - width: master.width, - height: master.height, - bytes: master.data.byteLength, + mediaType: normalized.mediaType, + width: normalized.width, + height: normalized.height, + bytes: normalized.data.byteLength, ...(name !== undefined ? { name } : {}), - ...downscaled ? { sourceWidth: source.width, sourceHeight: source.height } : {}, + ...downscaled ? { originalDimensions: { width: detected.width, height: detected.height } } : {}, }, - source, } } @@ -180,18 +176,18 @@ async function ensureDurableHome(path: string): Promise { } /** - * Publish one already verified master below a versioned attachment root. + * Publish one already verified normalized image below a versioned attachment root. * @param root - absolute `DSH_HOME/attachments/v1` root. - * @param prepared - deterministic master bytes, reference, and source facts. - * @returns durable content-addressed reference beside the submitted source facts. + * @param prepared - deterministic normalized bytes and reference. + * @returns durable content-addressed normalized image reference. */ export async function commitPreparedImageFile( root: string, prepared: PreparedImageFile, -): Promise { - const master = prepared.data +): Promise { + const normalized = prepared.data const sha256 = ensureReference(prepared.ref) - if (digest(master) !== sha256 || master.byteLength !== prepared.ref.bytes) { + if (digest(normalized) !== sha256 || normalized.byteLength !== prepared.ref.bytes) { throw new AttachmentError('Prepared attachment bytes do not match their reference.', 'ATTACHMENT_CORRUPT') } const bucket = join(root, 'objects', sha256.slice(0, 2)) @@ -207,7 +203,7 @@ export async function commitPreparedImageFile( let handle try { handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600) - await handle.writeFile(master) + await handle.writeFile(normalized) await handle.sync() await handle.close() handle = undefined @@ -242,7 +238,7 @@ export async function commitPreparedImageFile( if (error instanceof AttachmentError) throw error throw new AttachmentError('Unable to persist image attachment.', 'ATTACHMENT_WRITE_FAILED', { cause: error }) } - return { ref: prepared.ref, source: prepared.source } + return prepared.ref } /** @@ -250,15 +246,15 @@ export async function commitPreparedImageFile( * @param root - absolute `DSH_HOME/attachments/v1` root. * @param input - submitted encoded bytes and declared media type. * @param limits - resolved source admission policy. - * @param policy - resolved master-version storage policy. - * @returns durable content-addressed reference beside submitted source facts. + * @param policy - resolved normalization policy. + * @returns durable content-addressed normalized image reference. */ export async function saveImageFile( root: string, input: SaveImageAttachment, limits: ImageAttachmentLimits, - policy: MasterImagePolicy, -): Promise { + policy: NormalizationPolicy, +): Promise { return commitPreparedImageFile(root, await prepareImageFile(input, limits, policy)) } diff --git a/packages/attachment/attachment-local/tests/index.spec.ts b/packages/attachment/attachment-local/tests/index.spec.ts index c3c7693614..f8deea3c5c 100644 --- a/packages/attachment/attachment-local/tests/index.spec.ts +++ b/packages/attachment/attachment-local/tests/index.spec.ts @@ -6,8 +6,8 @@ import { join } from 'node:path' import { describe, expect, it } from 'vitest' import sharp from 'sharp' import LocalAttachmentStore, { - DEFAULT_MASTER_MAX_BYTES, - DEFAULT_MASTER_MAX_DIMENSION, + DEFAULT_NORMALIZED_IMAGE_MAX_BYTES, + DEFAULT_NORMALIZED_IMAGE_MAX_DIMENSION, DEFAULT_IMAGE_COMPRESSION_CONCURRENCY, DEFAULT_MAX_IMAGE_BYTES, DEFAULT_MAX_IMAGE_DIMENSION, @@ -32,9 +32,9 @@ describe('local attachment service', () => { maxImageDimension: DEFAULT_MAX_IMAGE_DIMENSION, mediaTypes: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'], }) - expect(service.masterPolicy).toEqual({ - maxDimension: DEFAULT_MASTER_MAX_DIMENSION, - maxBytes: DEFAULT_MASTER_MAX_BYTES, + expect(service.normalizationPolicy).toEqual({ + maxDimension: DEFAULT_NORMALIZED_IMAGE_MAX_DIMENSION, + maxBytes: DEFAULT_NORMALIZED_IMAGE_MAX_BYTES, }) expect(service.imageCompressionConcurrency).toBe(DEFAULT_IMAGE_COMPRESSION_CONCURRENCY) }) @@ -55,7 +55,7 @@ describe('local attachment service', () => { 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC', 'base64', )) - const { ref } = await service.saveImage({ data, mediaType: 'image/png' }) + const ref = await service.saveImage({ data, mediaType: 'image/png' }) await expect(service.readImage(ref)).resolves.toEqual({ ref, data }) } finally { await rm(dshHome, { recursive: true, force: true }) @@ -86,7 +86,7 @@ describe('local attachment service', () => { } }) - it.each([3, 4] as const)('admits a 16-bit %s-channel PNG as an 8-bit master object', async (channels) => { + it.each([3, 4] as const)('admits a 16-bit %s-channel PNG as an 8-bit normalized object', async (channels) => { const dshHome = await mkdtemp(join(tmpdir(), 'dsh-attachment-16-bit-')) try { const service = new LocalAttachmentStore(new Context(), { dshHome }) @@ -95,7 +95,7 @@ describe('local attachment service', () => { }).toColourspace('rgb16').png().toBuffer()) const saved = await service.saveImage({ data: source, mediaType: 'image/png' }) - const stored = await service.readImage(saved.ref) + const stored = await service.readImage(saved) const metadata = await sharp(stored.data).metadata() expect(stored.data).not.toEqual(source) @@ -108,7 +108,7 @@ describe('local attachment service', () => { it('prepares every batch member before any write', async () => { const dshHome = await mkdtemp(join(tmpdir(), 'dsh-attachment-batch-')) try { - const service = new LocalAttachmentStore(new Context(), { dshHome, masterMaxBytes: 1 }) + const service = new LocalAttachmentStore(new Context(), { dshHome, normalizedImageMaxBytes: 1 }) const valid = Uint8Array.from(Buffer.from( 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACXBIWXMAAAPoAAAD6AG1e1JrAAAADElEQVQImWNgZGIGAAAOAAeCcsnOAAAAAElFTkSuQmCC', 'base64', diff --git a/packages/attachment/attachment-local/tests/canonical.spec.ts b/packages/attachment/attachment-local/tests/normalization.spec.ts similarity index 63% rename from packages/attachment/attachment-local/tests/canonical.spec.ts rename to packages/attachment/attachment-local/tests/normalization.spec.ts index 8aa30511d6..4b530988d1 100644 --- a/packages/attachment/attachment-local/tests/canonical.spec.ts +++ b/packages/attachment/attachment-local/tests/normalization.spec.ts @@ -1,10 +1,10 @@ import { describe, expect, it } from 'vitest' import sharp from 'sharp' -import { hasLowColourCount, isMasterImage, prepareMasterImage } from '../src/canonical.ts' -import type { MasterImagePolicy } from '../src/canonical.ts' +import { hasLowColourCount, canPassThroughNormalization, normalizeImage } from '../src/normalization.ts' +import type { NormalizationPolicy } from '../src/normalization.ts' import { detectImage } from '../src/image.ts' -const POLICY: MasterImagePolicy = { maxDimension: 2048, maxBytes: 4 * 1024 * 1024 } +const POLICY: NormalizationPolicy = { maxDimension: 2048, maxBytes: 4 * 1024 * 1024 } /** Deterministic pseudo-random RGB noise; PNG cannot compress it below raw size. */ function noisePixels(width: number, height: number): Uint8Array { @@ -31,29 +31,29 @@ async function flatImage(width: number, height: number, format: 'png' | 'jpeg' | return new Uint8Array(await image.toFormat(format, format === 'webp' && alpha ? { lossless: true } : {}).toBuffer()) } -describe('isMasterImage', () => { +describe('canPassThroughNormalization', () => { it('accepts an in-budget clean PNG/JPEG/WebP and refuses GIF, animation, metadata, oversized edges, and oversized bytes', () => { const clean = { animated: false, carriesMetadata: false, depth: 'uchar', space: 'srgb', hasAlpha: false } - expect(isMasterImage({ mediaType: 'image/png', width: 2048, height: 4, ...clean }, 100, POLICY)).toBe(true) - expect(isMasterImage({ mediaType: 'image/gif', width: 4, height: 4, ...clean }, 100, POLICY)).toBe(false) - expect(isMasterImage({ mediaType: 'image/webp', width: 4, height: 4, animated: true, carriesMetadata: false, depth: 'uchar', space: 'srgb', hasAlpha: false }, 100, POLICY)).toBe(false) - expect(isMasterImage({ mediaType: 'image/jpeg', width: 4, height: 4, animated: false, carriesMetadata: true, depth: 'uchar', space: 'srgb', hasAlpha: false }, 100, POLICY)).toBe(false) - expect(isMasterImage({ mediaType: 'image/png', width: 4, height: 4, ...clean, depth: 'ushort' }, 100, POLICY)).toBe(false) - expect(isMasterImage({ mediaType: 'image/png', width: 4, height: 4, ...clean, space: 'rgb16' }, 100, POLICY)).toBe(false) - expect(isMasterImage({ mediaType: 'image/jpeg', width: 2049, height: 4, ...clean }, 100, POLICY)).toBe(false) - expect(isMasterImage({ mediaType: 'image/webp', width: 4, height: 4, ...clean }, POLICY.maxBytes + 1, POLICY)).toBe(false) + expect(canPassThroughNormalization({ mediaType: 'image/png', width: 2048, height: 4, ...clean }, 100, POLICY)).toBe(true) + expect(canPassThroughNormalization({ mediaType: 'image/gif', width: 4, height: 4, ...clean }, 100, POLICY)).toBe(false) + expect(canPassThroughNormalization({ mediaType: 'image/webp', width: 4, height: 4, animated: true, carriesMetadata: false, depth: 'uchar', space: 'srgb', hasAlpha: false }, 100, POLICY)).toBe(false) + expect(canPassThroughNormalization({ mediaType: 'image/jpeg', width: 4, height: 4, animated: false, carriesMetadata: true, depth: 'uchar', space: 'srgb', hasAlpha: false }, 100, POLICY)).toBe(false) + expect(canPassThroughNormalization({ mediaType: 'image/png', width: 4, height: 4, ...clean, depth: 'ushort' }, 100, POLICY)).toBe(false) + expect(canPassThroughNormalization({ mediaType: 'image/png', width: 4, height: 4, ...clean, space: 'rgb16' }, 100, POLICY)).toBe(false) + expect(canPassThroughNormalization({ mediaType: 'image/jpeg', width: 2049, height: 4, ...clean }, 100, POLICY)).toBe(false) + expect(canPassThroughNormalization({ mediaType: 'image/webp', width: 4, height: 4, ...clean }, POLICY.maxBytes + 1, POLICY)).toBe(false) }) }) -describe('prepareMasterImage', () => { - it('passes an already-canonical source through byte-identically', async () => { +describe('normalizeImage', () => { + it('passes an already-normalized source through byte-identically', async () => { const data = await flatImage(6, 4, 'webp') const detected = await detectImage(data) - const canonical = await prepareMasterImage(data, detected, POLICY) + const normalized = await normalizeImage(data, detected, POLICY) - expect(canonical.data).toBe(data) - expect(canonical).toMatchObject({ mediaType: 'image/webp', width: 6, height: 4 }) + expect(normalized.data).toBe(data) + expect(normalized).toMatchObject({ mediaType: 'image/webp', width: 6, height: 4 }) }) it.each([3, 4] as const)('converts a 16-bit %s-channel PNG to 8-bit sRGB without passthrough', async (channels) => { @@ -63,11 +63,11 @@ describe('prepareMasterImage', () => { const detected = await detectImage(data) expect(detected).toMatchObject({ depth: 'ushort', space: 'rgb16', hasAlpha: channels === 4 }) - const canonical = await prepareMasterImage(data, detected, POLICY) + const normalized = await normalizeImage(data, detected, POLICY) - expect(canonical.data).not.toBe(data) - expect(canonical.data).not.toEqual(data) - await expect(detectImage(canonical.data)).resolves.toMatchObject({ + expect(normalized.data).not.toBe(data) + expect(normalized.data).not.toEqual(data) + await expect(detectImage(normalized.data)).resolves.toMatchObject({ depth: 'uchar', space: 'srgb', hasAlpha: channels === 4, width: 7, height: 5, }) }) @@ -76,19 +76,19 @@ describe('prepareMasterImage', () => { const data = await flatImage(10, 6, 'png') const detected = await detectImage(data) - const canonical = await prepareMasterImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes }) + const normalized = await normalizeImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes }) - expect(canonical).toMatchObject({ mediaType: 'image/png', width: 5, height: 3 }) - await expect(detectImage(canonical.data)).resolves.toMatchObject({ mediaType: 'image/png', width: 5, height: 3, animated: false, carriesMetadata: false, depth: 'uchar', space: 'srgb' }) - const again = await prepareMasterImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes }) - expect(again.data).toEqual(canonical.data) + expect(normalized).toMatchObject({ mediaType: 'image/png', width: 5, height: 3 }) + await expect(detectImage(normalized.data)).resolves.toMatchObject({ mediaType: 'image/png', width: 5, height: 3, animated: false, carriesMetadata: false, depth: 'uchar', space: 'srgb' }) + const again = await normalizeImage(data, detected, { maxDimension: 5, maxBytes: POLICY.maxBytes }) + expect(again.data).toEqual(normalized.data) }) - it('re-encodes the canonical output of a resize into itself (idempotence)', async () => { + it('re-encodes the normalized output of a resize into itself (idempotence)', async () => { const data = await flatImage(10, 6, 'png') - const first = await prepareMasterImage(data, await detectImage(data), { maxDimension: 5, maxBytes: POLICY.maxBytes }) + const first = await normalizeImage(data, await detectImage(data), { maxDimension: 5, maxBytes: POLICY.maxBytes }) - const second = await prepareMasterImage(first.data, await detectImage(first.data), { maxDimension: 5, maxBytes: POLICY.maxBytes }) + const second = await normalizeImage(first.data, await detectImage(first.data), { maxDimension: 5, maxBytes: POLICY.maxBytes }) expect(second.data).toBe(first.data) }) @@ -97,19 +97,19 @@ describe('prepareMasterImage', () => { const data = await flatImage(6, 4, 'gif') const detected = await detectImage(data) - const canonical = await prepareMasterImage(data, detected, POLICY) + const normalized = await normalizeImage(data, detected, POLICY) - expect(canonical.mediaType).toBe('image/png') - await expect(detectImage(canonical.data)).resolves.toMatchObject({ mediaType: 'image/png', width: 6, height: 4, animated: false, carriesMetadata: false, depth: 'uchar', space: 'srgb' }) + expect(normalized.mediaType).toBe('image/png') + await expect(detectImage(normalized.data)).resolves.toMatchObject({ mediaType: 'image/png', width: 6, height: 4, animated: false, carriesMetadata: false, depth: 'uchar', space: 'srgb' }) }) it('keeps a low-colour alpha source on PNG when the budget holds', async () => { const data = await flatImage(9, 5, 'webp', true) const detected = await detectImage(data) - const canonical = await prepareMasterImage(data, detected, { maxDimension: 4, maxBytes: POLICY.maxBytes }) + const normalized = await normalizeImage(data, detected, { maxDimension: 4, maxBytes: POLICY.maxBytes }) - expect(canonical).toMatchObject({ mediaType: 'image/png', width: 4, height: 2 }) + expect(normalized).toMatchObject({ mediaType: 'image/png', width: 4, height: 2 }) }) it('retains an all-opaque alpha channel while converting a low-colour image', async () => { @@ -117,13 +117,13 @@ describe('prepareMasterImage', () => { create: { width: 10, height: 6, channels: 4, background: { r: 12, g: 200, b: 64, alpha: 1 } }, }).png().toBuffer()) - const canonical = await prepareMasterImage(data, await detectImage(data), { + const normalized = await normalizeImage(data, await detectImage(data), { maxDimension: 5, maxBytes: POLICY.maxBytes, }) - expect(canonical).toMatchObject({ mediaType: 'image/png', width: 5, height: 3 }) - await expect(detectImage(canonical.data)).resolves.toMatchObject({ hasAlpha: true }) + expect(normalized).toMatchObject({ mediaType: 'image/png', width: 5, height: 3 }) + await expect(detectImage(normalized.data)).resolves.toMatchObject({ hasAlpha: true }) }) it('keeps transparency when the byte cap requires another encoding and smaller dimensions', async () => { @@ -140,20 +140,20 @@ describe('prepareMasterImage', () => { } const data = new Uint8Array(await sharp(pixels, { raw: { width: side, height: side, channels: 4 } }).png().toBuffer()) - const canonical = await prepareMasterImage(data, await detectImage(data), { maxDimension: side, maxBytes: 1_024 }) + const normalized = await normalizeImage(data, await detectImage(data), { maxDimension: side, maxBytes: 1_024 }) - expect(canonical.data.byteLength).toBeLessThanOrEqual(1_024) - expect(canonical.width).toBeLessThan(side) - await expect(detectImage(canonical.data)).resolves.toMatchObject({ hasAlpha: true, depth: 'uchar', space: 'srgb' }) + expect(normalized.data.byteLength).toBeLessThanOrEqual(1_024) + expect(normalized.width).toBeLessThan(side) + await expect(detectImage(normalized.data)).resolves.toMatchObject({ hasAlpha: true, depth: 'uchar', space: 'srgb' }) }) it('re-encodes an oversized photographic JPEG as JPEG', async () => { const data = await noiseImage(64, 32, 'jpeg') const detected = await detectImage(data) - const canonical = await prepareMasterImage(data, detected, { maxDimension: 32, maxBytes: POLICY.maxBytes }) + const normalized = await normalizeImage(data, detected, { maxDimension: 32, maxBytes: POLICY.maxBytes }) - expect(canonical).toMatchObject({ mediaType: 'image/jpeg', width: 32, height: 16 }) + expect(normalized).toMatchObject({ mediaType: 'image/jpeg', width: 32, height: 16 }) }) it('classifies a photographic PNG by pixels and uses an opaque photographic encoding', async () => { @@ -174,21 +174,21 @@ describe('prepareMasterImage', () => { const detected = await detectImage(data) const budget = { maxDimension: 128, maxBytes: POLICY.maxBytes } - const canonical = await prepareMasterImage(data, detected, budget) + const normalized = await normalizeImage(data, detected, budget) - expect(canonical.mediaType).toBe('image/jpeg') - expect(canonical).toMatchObject({ width: 128, height: 128 }) - expect(canonical.data.byteLength).toBeLessThanOrEqual(budget.maxBytes) + expect(normalized.mediaType).toBe('image/jpeg') + expect(normalized).toMatchObject({ width: 128, height: 128 }) + expect(normalized.data.byteLength).toBeLessThanOrEqual(budget.maxBytes) }) it('shrinks dimensions after the quality floor instead of refusing an oversized encoding', async () => { const data = await noiseImage(64, 64, 'png') - const canonical = await prepareMasterImage(data, await detectImage(data), { maxDimension: 2048, maxBytes: 512 }) + const normalized = await normalizeImage(data, await detectImage(data), { maxDimension: 2048, maxBytes: 512 }) - expect(canonical.data.byteLength).toBeLessThanOrEqual(512) - expect(canonical.width).toBeLessThan(64) - expect(canonical.height).toBeLessThan(64) + expect(normalized.data.byteLength).toBeLessThanOrEqual(512) + expect(normalized.width).toBeLessThan(64) + expect(normalized.height).toBeLessThan(64) }) it('re-encodes an in-budget oriented JPEG, baking rotation and stripping metadata', async () => { @@ -199,11 +199,11 @@ describe('prepareMasterImage', () => { // Orientation 6 rotates 90°: the perceived source is 2x4. expect(detected).toMatchObject({ width: 2, height: 4, carriesMetadata: true }) - const canonical = await prepareMasterImage(data, detected, POLICY) + const normalized = await normalizeImage(data, detected, POLICY) - expect(canonical.data).not.toBe(data) - expect(canonical).toMatchObject({ width: 2, height: 4 }) - await expect(detectImage(canonical.data)).resolves.toMatchObject({ width: 2, height: 4, carriesMetadata: false }) + expect(normalized.data).not.toBe(data) + expect(normalized).toMatchObject({ width: 2, height: 4 }) + await expect(detectImage(normalized.data)).resolves.toMatchObject({ width: 2, height: 4, carriesMetadata: false }) }) it('re-encodes an in-budget image with an ICC profile and strips the profile', async () => { @@ -213,10 +213,10 @@ describe('prepareMasterImage', () => { const detected = await detectImage(data) expect(detected.carriesMetadata).toBe(true) - const canonical = await prepareMasterImage(data, detected, POLICY) + const normalized = await normalizeImage(data, detected, POLICY) - expect(canonical.data).not.toBe(data) - await expect(detectImage(canonical.data)).resolves.toMatchObject({ carriesMetadata: false }) + expect(normalized.data).not.toBe(data) + await expect(detectImage(normalized.data)).resolves.toMatchObject({ carriesMetadata: false }) }) it('maps an encoder fault on undecodable bytes to a storage failure', async () => { @@ -224,10 +224,10 @@ describe('prepareMasterImage', () => { mediaType: 'image/png', width: 5000, height: 5000, animated: false, carriesMetadata: false, depth: 'ushort', space: 'rgb16', hasAlpha: true, } as const - await expect(prepareMasterImage(Uint8Array.of(1, 2, 3), detected, POLICY)) + await expect(normalizeImage(Uint8Array.of(1, 2, 3), detected, POLICY)) .rejects.toMatchObject({ code: 'ATTACHMENT_WRITE_FAILED', - message: 'The 16-bit PNG could not be converted to the canonical 8-bit sRGB form.', + message: 'The 16-bit PNG could not be converted to the normalized 8-bit sRGB form.', }) }) @@ -245,23 +245,23 @@ describe('prepareMasterImage', () => { hasAlpha: false, } as const - await expect(prepareMasterImage(Uint8Array.of(1, 2, 3), detected, POLICY)) + await expect(normalizeImage(Uint8Array.of(1, 2, 3), detected, POLICY)) .rejects.toMatchObject({ code: 'ATTACHMENT_WRITE_FAILED', - message: `The ${source} could not be converted to the canonical 8-bit sRGB form.`, + message: `The ${source} could not be converted to the normalized 8-bit sRGB form.`, }) }) - it('rejects a converted master whose verified alpha metadata disagrees with the source facts', async () => { + it('rejects a converted normalized image whose verified alpha metadata disagrees with the source facts', async () => { const data = await flatImage(8, 8, 'png', true) const detected = await detectImage(data) - await expect(prepareMasterImage(data, { ...detected, hasAlpha: false }, { + await expect(normalizeImage(data, { ...detected, hasAlpha: false }, { maxDimension: 4, maxBytes: POLICY.maxBytes, })).rejects.toMatchObject({ code: 'ATTACHMENT_WRITE_FAILED', - message: 'Canonical image conversion did not produce a single-frame 8-bit sRGB image with matching metadata.', + message: 'Image normalization did not produce a single-frame 8-bit sRGB image with matching metadata.', }) }) }) @@ -339,13 +339,13 @@ describe('hasLowColourCount', () => { `)).removeAlpha().png().toBuffer()) - const master = await prepareMasterImage(source, await detectImage(source), { + const normalized = await normalizeImage(source, await detectImage(source), { maxDimension: 512, maxBytes: POLICY.maxBytes, }) - const stats = await sharp(master.data).greyscale().stats() + const stats = await sharp(normalized.data).greyscale().stats() - expect(master).toMatchObject({ mediaType: 'image/png', width: 512, height: 256 }) + expect(normalized).toMatchObject({ mediaType: 'image/png', width: 512, height: 256 }) expect(stats.channels[0]?.min).toBeLessThan(80) expect(stats.channels[0]?.max).toBeGreaterThan(240) }) diff --git a/packages/attachment/attachment-local/tests/request-image-verification.spec.ts b/packages/attachment/attachment-local/tests/request-image-verification.spec.ts index aae96e0b1d..32bc005c94 100644 --- a/packages/attachment/attachment-local/tests/request-image-verification.spec.ts +++ b/packages/attachment/attachment-local/tests/request-image-verification.spec.ts @@ -35,10 +35,10 @@ describe('request image verification', () => { const source = new Uint8Array(await sharp({ create: { width: 64, height: 32, channels: 3, background: { r: 12, g: 34, b: 56 } }, }).png().toBuffer()) - const master = (await attachments.saveImage({ data: source, mediaType: 'image/png' })).ref + const attachment = await attachments.saveImage({ data: source, mediaType: 'image/png' }) control.mismatch = true - await expect(attachments.readImageRequest(master, { maxPixels: 16 * 16, maxBytes: 1024 * 1024 })) + await expect(attachments.readImageRequest(attachment, { maxPixels: 16 * 16, maxBytes: 1024 * 1024 })) .rejects.toMatchObject({ code: 'ATTACHMENT_WRITE_FAILED', message: 'Encoded model-request image does not match its verified 8-bit sRGB metadata.', diff --git a/packages/attachment/attachment-local/tests/request-image.spec.ts b/packages/attachment/attachment-local/tests/request-image.spec.ts index c38e8ce137..7052da89e8 100644 --- a/packages/attachment/attachment-local/tests/request-image.spec.ts +++ b/packages/attachment/attachment-local/tests/request-image.spec.ts @@ -54,43 +54,45 @@ describe('request image dimensions', () => { }) describe('local request-image cache', () => { - it('passes through an in-budget master and reads a request batch in input order', async () => { + it('passes through an in-budget attachment and composes ordered request reads', async () => { const attachments = await store() - const first = (await attachments.saveImage({ data: await image(8, 4), mediaType: 'image/png' })).ref - const second = (await attachments.saveImage({ data: await image(4, 8), mediaType: 'image/png' })).ref - const firstMaster = await attachments.readImage(first) + const first = await attachments.saveImage({ data: await image(8, 4), mediaType: 'image/png' }) + const second = await attachments.saveImage({ data: await image(4, 8), mediaType: 'image/png' }) + const firstStored = await attachments.readImage(first) const policy = { maxPixels: 1_000, maxBytes: 1024 * 1024 } const request = await attachments.readImageRequest(first, policy) - const batch = await attachments.readImageRequests([first, second], policy) + const batch = await Promise.all([first, second].map( + attachment => attachments.readImageRequest(attachment, policy), + )) - expect(request.data).toEqual(firstMaster.data) - expect(batch.map(value => value.master.attachmentId)).toEqual([first.attachmentId, second.attachmentId]) + expect(request.data).toEqual(firstStored.data) + expect(batch.map(value => value.attachment.attachmentId)).toEqual([first.attachmentId, second.attachmentId]) }) it('rejects invalid request policies', async () => { const attachments = await store() - const master = (await attachments.saveImage({ data: await image(8, 4), mediaType: 'image/png' })).ref + const attachment = await attachments.saveImage({ data: await image(8, 4), mediaType: 'image/png' }) - await expect(attachments.readImageRequest(master, { maxPixels: 0, maxBytes: 100 })) + await expect(attachments.readImageRequest(attachment, { maxPixels: 0, maxBytes: 100 })) .rejects.toThrow('Image request maxPixels must be a positive integer') - await expect(attachments.readImageRequest(master, { maxPixels: 100, maxBytes: 0 })) + await expect(attachments.readImageRequest(attachment, { maxPixels: 100, maxBytes: 0 })) .rejects.toThrow('Image request maxBytes must be a positive integer') }) it('refuses a one-pixel request that cannot meet the encoded-byte budget', async () => { const attachments = await store() - const master = (await attachments.saveImage({ data: await image(1, 1), mediaType: 'image/png' })).ref + const attachment = await attachments.saveImage({ data: await image(1, 1), mediaType: 'image/png' }) - await expect(attachments.readImageRequest(master, { maxPixels: 1, maxBytes: 1 })) + await expect(attachments.readImageRequest(attachment, { maxPixels: 1, maxBytes: 1 })) .rejects.toMatchObject({ code: 'IMAGE_TOO_LARGE' }) }) it('regenerates invalid, oversized, incompatible, or mismatched cached variants', async () => { const attachments = await store() - const master = (await attachments.saveImage({ data: await image(64, 32), mediaType: 'image/png' })).ref + const attachment = await attachments.saveImage({ data: await image(64, 32), mediaType: 'image/png' }) const policy = { maxPixels: 16 * 16, maxBytes: 4_096 } - const initial = await attachments.readImageRequest(master, policy) + const initial = await attachments.readImageRequest(attachment, policy) const hash = String(initial.variantId).slice('sha256:'.length) const path = join(attachments.root, 'request-images', hash.slice(0, 2), hash) const noisyPixels = new Uint8Array(64 * 64 * 3) @@ -124,19 +126,19 @@ describe('local request-image cache', () => { Uint8Array.of(1, 2, 3), ]) { await writeFile(path, invalid) - const regenerated = await attachments.readImageRequest(master, policy) + const regenerated = await attachments.readImageRequest(attachment, policy) expect(regenerated.data).toEqual(initial.data) } }) it('derives stable square and wide previews and separates route budgets in the cache key', async () => { const attachments = await store() - const square = (await attachments.saveImage({ + const square = await attachments.saveImage({ data: await image(2048, 2048), mediaType: 'image/png', name: 'square.png', - })).ref - const wide = (await attachments.saveImage({ + }) + const wide = await attachments.saveImage({ data: await image(2048, 1024), mediaType: 'image/png', name: 'wide.png', - })).ref + }) const squareRequest = await attachments.readImageRequest(square, { maxPixels: 640_000, maxBytes: 1024 * 1024 }) const wideRequest = await attachments.readImageRequest(wide, { maxPixels: 640_000, maxBytes: 1024 * 1024 }) @@ -178,8 +180,8 @@ describe('local request-image cache', () => { const alphaSource = new Uint8Array(await sharp(alphaPixels, { raw: { width: side, height: side, channels: 4 }, }).png().toBuffer()) - const photo = (await attachments.saveImage({ data: photoSource, mediaType: 'image/png' })).ref - const alpha = (await attachments.saveImage({ data: alphaSource, mediaType: 'image/png' })).ref + const photo = await attachments.saveImage({ data: photoSource, mediaType: 'image/png' }) + const alpha = await attachments.saveImage({ data: alphaSource, mediaType: 'image/png' }) const photoRequest = await attachments.readImageRequest(photo, { maxPixels: 128 * 128, maxBytes: 1024 * 1024 }) const alphaRequest = await attachments.readImageRequest(alpha, { maxPixels: 128 * 128, maxBytes: 4_096 }) @@ -195,9 +197,9 @@ describe('local request-image cache', () => { const source = new Uint8Array(await sharp({ create: { width: 64, height: 32, channels, background: { r: 12, g: 34, b: 56, alpha: 0.5 } }, }).toColourspace('rgb16').png().toBuffer()) - const master = (await attachments.saveImage({ data: source, mediaType: 'image/png' })).ref + const attachment = await attachments.saveImage({ data: source, mediaType: 'image/png' }) - const request = await attachments.readImageRequest(master, { maxPixels: 16 * 16, maxBytes: 1024 * 1024 }) + const request = await attachments.readImageRequest(attachment, { maxPixels: 16 * 16, maxBytes: 1024 * 1024 }) expect(request.bytes).toBeLessThanOrEqual(1024 * 1024) expect(request.width * request.height).toBeLessThanOrEqual(16 * 16) @@ -211,9 +213,9 @@ describe('local request-image cache', () => { const source = new Uint8Array(await sharp({ create: { width: 64, height: 32, channels: 4, background: { r: 12, g: 34, b: 56, alpha: 1 } }, }).png().toBuffer()) - const master = (await attachments.saveImage({ data: source, mediaType: 'image/png' })).ref + const attachment = await attachments.saveImage({ data: source, mediaType: 'image/png' }) - const request = await attachments.readImageRequest(master, { maxPixels: 16 * 16, maxBytes: 1024 * 1024 }) + const request = await attachments.readImageRequest(attachment, { maxPixels: 16 * 16, maxBytes: 1024 * 1024 }) await expect(sharp(request.data).metadata()).resolves.toMatchObject({ hasAlpha: true }) }) @@ -232,9 +234,9 @@ describe('local request-image cache', () => { const source = new Uint8Array(await sharp(pixels, { raw: { width: side, height: side, channels: 3 }, }).png().toBuffer()) - const master = (await attachments.saveImage({ data: source, mediaType: 'image/png' })).ref + const attachment = await attachments.saveImage({ data: source, mediaType: 'image/png' }) - const request = await attachments.readImageRequest(master, { maxPixels: 640_000, maxBytes: 1024 * 1024 }) + const request = await attachments.readImageRequest(attachment, { maxPixels: 640_000, maxBytes: 1024 * 1024 }) expect(request).toMatchObject({ width: 800, height: 800 }) expect(request.bytes).toBeLessThanOrEqual(1024 * 1024) @@ -242,15 +244,15 @@ describe('local request-image cache', () => { it('shares one request transform between concurrent callers without sharing cancellation', async () => { const attachments = await store() - const master = (await attachments.saveImage({ + const attachment = await attachments.saveImage({ data: await image(2048, 1024), mediaType: 'image/png', name: 'shared.png', - })).ref + }) const run = vi.spyOn(CompressionLimiter.prototype, 'run') const controller = new AbortController() const policy = { maxPixels: 640_000, maxBytes: 1024 * 1024 } - const cancelled = attachments.readImageRequest(master, policy, controller.signal) - const completed = attachments.readImageRequest(master, policy) + const cancelled = attachments.readImageRequest(attachment, policy, controller.signal) + const completed = attachments.readImageRequest(attachment, policy) const reason = new Error('cancel one waiter') controller.abort(reason) @@ -262,9 +264,9 @@ describe('local request-image cache', () => { it('aborts the underlying request transform after its only waiter cancels', async () => { const attachments = await store() - const master = (await attachments.saveImage({ + const attachment = await attachments.saveImage({ data: await image(2048, 1024), mediaType: 'image/png', name: 'cancelled.png', - })).ref + }) let readSignal: AbortSignal | undefined const read = vi.spyOn(attachments, 'readImage').mockImplementation((_ref, signal) => { readSignal = signal @@ -276,7 +278,7 @@ describe('local request-image cache', () => { }) const controller = new AbortController() const request = attachments.readImageRequest( - master, + attachment, { maxPixels: 640_000, maxBytes: 1024 * 1024 }, controller.signal, ) @@ -293,9 +295,9 @@ describe('local request-image cache', () => { it('normalizes a non-Error cancellation and replaces an aborted shared transform', async () => { const attachments = await store() - const master = (await attachments.saveImage({ + const attachment = await attachments.saveImage({ data: await image(2048, 1024), mediaType: 'image/png', name: 'replace.png', - })).ref + }) const actualRead = attachments.readImage.bind(attachments) let calls = 0 vi.spyOn(attachments, 'readImage').mockImplementation((ref, signal) => { @@ -311,13 +313,13 @@ describe('local request-image cache', () => { }) const controller = new AbortController() const policy = { maxPixels: 640_000, maxBytes: 1024 * 1024 } - const cancelled = attachments.readImageRequest(master, policy, controller.signal) + const cancelled = attachments.readImageRequest(attachment, policy, controller.signal) await vi.waitFor(() => { expect(calls).toBe(1) }) controller.abort('cancelled') - const replacement = attachments.readImageRequest(master, policy) + const replacement = attachments.readImageRequest(attachment, policy) await expect(cancelled).rejects.toMatchObject({ message: 'Attachment request cancelled with a non-Error reason.', diff --git a/packages/attachment/attachment-local/tests/store.spec.ts b/packages/attachment/attachment-local/tests/store.spec.ts index f0c127c174..ad29f856ec 100644 --- a/packages/attachment/attachment-local/tests/store.spec.ts +++ b/packages/attachment/attachment-local/tests/store.spec.ts @@ -7,7 +7,7 @@ import { mkdtemp, rm } from 'node:fs/promises' import { afterEach, describe, expect, it, vi } from 'vitest' import sharp from 'sharp' import type { ImageAttachmentLimits } from '@deepseek-ai/dsh-attachment' -import type { MasterImagePolicy } from '../src/canonical.ts' +import type { NormalizationPolicy } from '../src/normalization.ts' import { commitPreparedImageFile, prepareImageFile, readImageFile, saveImageFile } from '../src/store.ts' const fsControl = vi.hoisted(() => ({ @@ -39,7 +39,7 @@ const PNG = Uint8Array.from(Buffer.from( 'base64', )) -const POLICY: MasterImagePolicy = { maxDimension: 2048, maxBytes: 1024 * 1024 } +const POLICY: NormalizationPolicy = { maxDimension: 2048, maxBytes: 1024 * 1024 } const LIMITS: ImageAttachmentLimits = { maxImageBytes: 1024, @@ -107,7 +107,7 @@ describe('local attachment store', () => { it('creates and persists a missing nested home directory against the filesystem root', async () => { const storageRoot = join(await root(), 'home', 'attachments', 'v1') - const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) + const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) await expect(readImageFile(storageRoot, ref)).resolves.toEqual({ ref, data: PNG }) }) @@ -121,7 +121,7 @@ describe('local attachment store', () => { const sha256 = createHash('sha256').update(PNG).digest('hex') const object = join(storageRoot, 'objects', sha256.slice(0, 2), sha256) - expect(first.ref).toEqual({ + expect(first).toEqual({ attachmentId: `sha256:${sha256}`, mediaType: 'image/png', bytes: PNG.byteLength, @@ -129,17 +129,16 @@ describe('local attachment store', () => { height: 1, name: 'pixel.png', }) - expect(first.source).toEqual({ mediaType: 'image/png', bytes: PNG.byteLength, width: 1, height: 1 }) - expect(second.ref.attachmentId).toBe(first.ref.attachmentId) + expect(second.attachmentId).toBe(first.attachmentId) expect(new Uint8Array(await readFile(object))).toEqual(PNG) if (process.platform !== 'win32') { expect((await stat(object)).mode & 0o777).toBe(0o600) expect((await stat(join(storageRoot, 'objects', sha256.slice(0, 2)))).mode & 0o777).toBe(0o700) } - await expect(readImageFile(storageRoot, first.ref)).resolves.toEqual({ ref: first.ref, data: PNG }) + await expect(readImageFile(storageRoot, first)).resolves.toEqual({ ref: first, data: PNG }) }) - it('stores the image master of an oversized source and reads it back verified', async () => { + it('stores the normalized image of an oversized source and reads it back verified', async () => { const storageRoot = await root() const oversized = new Uint8Array(await sharp({ create: { width: 4, height: 4, channels: 3, background: { r: 9, g: 9, b: 9 } }, @@ -149,24 +148,29 @@ describe('local attachment store', () => { data: oversized, mediaType: 'image/png', name: 'big.png', }, { ...LIMITS, maxImagePixels: 64 }, { maxDimension: 2, maxBytes: 1024 * 1024 }) - expect(saved.source).toEqual({ mediaType: 'image/png', bytes: oversized.byteLength, width: 4, height: 4 }) - expect(saved.ref).toMatchObject({ mediaType: 'image/png', width: 2, height: 2, name: 'big.png' }) - expect(saved.ref.bytes).not.toBe(oversized.byteLength) - const read = await readImageFile(storageRoot, saved.ref) - expect(read.data.byteLength).toBe(saved.ref.bytes) - expect(String(saved.ref.attachmentId)).toBe(`sha256:${createHash('sha256').update(read.data).digest('hex')}`) + expect(saved).toMatchObject({ + mediaType: 'image/png', + width: 2, + height: 2, + name: 'big.png', + originalDimensions: { width: 4, height: 4 }, + }) + expect(saved.bytes).not.toBe(oversized.byteLength) + const read = await readImageFile(storageRoot, saved) + expect(read.data.byteLength).toBe(saved.bytes) + expect(String(saved.attachmentId)).toBe(`sha256:${createHash('sha256').update(read.data).digest('hex')}`) }) it('keeps admitted history readable after deployment limits become stricter', async () => { const storageRoot = await root() - const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) + const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) await expect(readImageFile(storageRoot, ref)).resolves.toEqual({ ref, data: PNG }) }) it('forwards read cancellation to the filesystem and preserves its reason', async () => { const storageRoot = await root() - const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) + const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) const controller = new AbortController() fsControl.readSignals.length = 0 @@ -205,12 +209,12 @@ describe('local attachment store', () => { const unnamed = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png', name: '\u0000', }, LIMITS, POLICY) - expect(unnamed.ref).not.toHaveProperty('name') + expect(unnamed).not.toHaveProperty('name') }) it('fails closed when an object is missing, corrupted, or addressed by an invalid reference', async () => { const storageRoot = await root() - const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) + const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) const sha256 = String(ref.attachmentId).slice('sha256:'.length) const object = join(storageRoot, 'objects', sha256.slice(0, 2), sha256) await chmod(object, 0o600) @@ -242,7 +246,7 @@ describe('local attachment store', () => { .rejects.toMatchObject({ code: 'ATTACHMENT_CORRUPT' }) await writeFile(target, PNG) - const { ref } = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) + const ref = await saveImageFile(storageRoot, { data: PNG, mediaType: 'image/png' }, LIMITS, POLICY) await expect(readImageFile(storageRoot, { ...ref, width: ref.width + 1 })) .rejects.toMatchObject({ code: 'ATTACHMENT_CORRUPT' }) }) diff --git a/packages/attachment/attachment/README.i18n.yaml b/packages/attachment/attachment/README.i18n.yaml index bbccf584c6..e27f25e933 100644 --- a/packages/attachment/attachment/README.i18n.yaml +++ b/packages/attachment/attachment/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/attachment/attachment/README.md -README.md: 66ce5f308cfa1ce6a028dbd248ceef1fdcc31a7c -README.zh.md: 4470956987330a451e3717d419a111def98dd6cb +README.md: 3ad568c7308f1ab85cb4af3fcc2afd3cba9a611a +README.zh.md: fadbb1c5bbf097c599da651055d63a1ed64cd579 diff --git a/packages/attachment/attachment/README.md b/packages/attachment/attachment/README.md index 66ce5f308c..3ad568c730 100644 --- a/packages/attachment/attachment/README.md +++ b/packages/attachment/attachment/README.md @@ -2,9 +2,9 @@ English | [中文](README.zh.md) -The durable attachment seam. `ctx.attachments` validates and durably commits a provider-independent master image, then returns a serializable `ImageAttachmentRef`; consumers never persist browser paths, object URLs, provider URLs, or base64 in session events. +The durable attachment seam. `ctx.attachments` validates and durably commits a provider-independent normalized image, then returns a serializable `ImageAttachmentRef`; consumers never persist browser paths, object URLs, provider URLs, or base64 in session events. -Unsent composer images remain browser-owned temporary drafts. `validateImage` runs the complete admission policy without persisting. `saveImages` owns batch count and aggregate-byte limits, prepares every validated master once before publishing any member, then commits in order and returns references only after the complete batch succeeds. A later storage failure returns no partial references, although an earlier immutable content-addressed object may remain unreachable until reference-aware garbage collection exists. `AttachmentError.code` uses the closed `AttachmentErrorCode` string union. Its `ImageAdmissionErrorCode` subset marks caller-correctable image-input failures; `isImageAdmissionError` recognizes that subset at runtime so each protocol adapter can map its own error vocabulary. `saveImage` commits one accepted image before any model-visible session event is published and resolves `SavedImageAttachment`: the returned `ref` describes the stored master while `source` (`SourceImageInfo`) preserves the submitted raster's media type, byte length, and orientation-applied dimensions. `readImage` verifies that master against its logged metadata. `readImageRequest` deterministically derives a route-sized request version whose identity covers the master id, transform version, pixel and byte budgets, and encoder settings; `readImageRequests` preserves ordered results while implementations apply their own bounded concurrency. Callers may cancel reads and projections; implementations preserve cancellation instead of translating it into a storage failure. +Unsent composer images remain browser-owned temporary drafts. `validateImage` runs the complete admission policy without persisting. `saveImages` owns batch count and aggregate-byte limits, prepares every normalized attachment before publishing any member, then commits in order and returns references only after the complete batch succeeds. A later storage failure returns no partial references, although an earlier immutable content-addressed object may remain unreachable until reference-aware garbage collection exists. `AttachmentError.code` uses the closed `AttachmentErrorCode` string union. Its `ImageAdmissionErrorCode` subset marks caller-correctable image-input failures; `isImageAdmissionError` recognizes that subset at runtime so each protocol adapter can map its own error vocabulary. `saveImage` commits one accepted image before any model-visible session event is published and returns its `ImageAttachmentRef`. When normalization reduces the raster, the reference records the orientation-applied input size in `originalDimensions`. `readImage` verifies the normalized attachment against its logged metadata. `readImageRequest` deterministically derives a route-sized request version whose identity covers the attachment id, transform version, pixel and byte budgets, and encoder settings. Callers compose ordered batches with `Promise.all(refs.map(...))`; the local implementation still bounds compression through its instance limiter, cache, and singleflight. Callers may cancel reads and projections; implementations preserve cancellation instead of translating it into a storage failure. `admitEncodedImages(attachments, images)` is the shared wire entry used by every RPC endpoint that accepts browser uploads (the session prompt endpoint and the command executor): it enforces canonical base64 on every member, then delegates batch admission — limits, validation, ordered commit — to `saveImages`. The base64 upload form is `EncodedImageAttachment`, exported from `@deepseek-ai/dsh-attachment/types` so wire contracts can reference it. diff --git a/packages/attachment/attachment/README.zh.md b/packages/attachment/attachment/README.zh.md index 4470956987..fadbb1c5bb 100644 --- a/packages/attachment/attachment/README.zh.md +++ b/packages/attachment/attachment/README.zh.md @@ -2,9 +2,9 @@ [English](README.md) | 中文 -持久附件服务边界。`ctx.attachments` 校验并持久提交提供方无关的图片主版本,随后返回可序列化的 `ImageAttachmentRef`;消费方绝不会在会话事件中持久保存浏览器路径、对象 URL、提供方 URL 或 base64。 +持久附件服务边界。`ctx.attachments` 校验并持久提交提供方无关的规范化图片,随后返回可序列化的 `ImageAttachmentRef`;消费方绝不会在会话事件中持久保存浏览器路径、对象 URL、提供方 URL 或 base64。 -未发送的输入区图片仍是由浏览器持有的临时草稿。`validateImage` 运行完整准入策略但不执行持久化。`saveImages` 负责批次图片数量和总字节限制,在发布任何成员前为全部成员各准备一次经过验证的主版本,然后按顺序提交,并且只在完整批次成功后返回引用。后续存储失败不会返回部分引用,但较早写入的不可变内容寻址对象可能保持不可达,直至具备按引用感知的垃圾回收。`AttachmentError.code` 使用封闭的 `AttachmentErrorCode` 字符串联合类型。其 `ImageAdmissionErrorCode` 子集标记可由调用方修正的图片输入失败;`isImageAdmissionError` 在运行时识别该子集,使每个协议适配器可以映射自己的错误词汇。`saveImage` 会在发布任何模型可见的会话事件前提交一张已接受的图片,并解析为 `SavedImageAttachment`:返回的 `ref` 描述实际存储的主版本,而 `source`(`SourceImageInfo`)保留所提交光栅的媒体类型、字节长度和应用方向后的尺寸。`readImage` 根据已记录的元数据校验该主版本。`readImageRequest` 确定性派生路由所需的请求版本,其身份覆盖主版本 ID、变换策略版本、像素和字节预算及编码参数;`readImageRequests` 保持结果顺序,并由实现施加自己的有界并发。调用方可以取消读取和投影;实现保留取消结果,不把它转换为存储失败。 +未发送的输入区图片仍是由浏览器持有的临时草稿。`validateImage` 运行完整准入策略但不执行持久化。`saveImages` 负责批次图片数量和总字节限制,在发布任何成员前准备全部规范化附件,然后按顺序提交,并且只在完整批次成功后返回引用。后续存储失败不会返回部分引用,但较早写入的不可变内容寻址对象可能保持不可达,直至具备按引用感知的垃圾回收。`AttachmentError.code` 使用封闭的 `AttachmentErrorCode` 字符串联合类型。其 `ImageAdmissionErrorCode` 子集标记可由调用方修正的图片输入失败;`isImageAdmissionError` 在运行时识别该子集,使每个协议适配器可以映射自己的错误词汇。`saveImage` 会在发布任何模型可见的会话事件前提交一张已接受的图片,并直接返回 `ImageAttachmentRef`。规范化过程缩小图片时,引用会通过 `originalDimensions` 记录应用方向后的输入尺寸。`readImage` 根据已记录的元数据校验规范化附件。`readImageRequest` 确定性派生路由所需的请求版本,其身份覆盖附件 ID、变换策略版本、像素和字节预算及编码参数。调用方通过 `Promise.all(refs.map(...))` 组合有序批次,本地实现仍通过实例级限流器、缓存和 singleflight 限制压缩并发。调用方可以取消读取和投影;实现保留取消结果,不把它转换为存储失败。 `admitEncodedImages(attachments, images)` 是每个接受浏览器上传的 RPC 端点(会话 prompt 端点与命令执行器)共用的 wire 入口:它对每个成员强制执行规范 base64,随后把批量准入——限额、校验、有序提交——委托给 `saveImages`。base64 上传形式为 `EncodedImageAttachment`,从 `@deepseek-ai/dsh-attachment/types` 导出,供 wire 契约引用。 diff --git a/packages/attachment/attachment/src/index.ts b/packages/attachment/attachment/src/index.ts index 85401fad23..8b54926efa 100644 --- a/packages/attachment/attachment/src/index.ts +++ b/packages/attachment/attachment/src/index.ts @@ -8,7 +8,6 @@ import type { ImageRequestPolicy, RequestImageAttachment, SaveImageAttachment, - SavedImageAttachment, StoredImageAttachment, } from './types.ts' @@ -25,8 +24,6 @@ export type { ImageMediaType, RequestImageAttachment, SaveImageAttachment, - SavedImageAttachment, - SourceImageInfo, StoredImageAttachment, } from './types.ts' @@ -80,40 +77,39 @@ export abstract class AttachmentStore extends Service { /** * Validate and durably commit one ordered image batch. * @param inputs - encoded images in owning-message order. - * @returns durable master references in the same order after every member succeeds. + * @returns durable normalized attachment references in the same order after every member succeeds. */ async saveImages(inputs: readonly SaveImageAttachment[]): Promise { this.validateImageBatch(inputs) for (const input of inputs) await this.validateImage(input) const refs: ImageAttachmentRef[] = [] - for (const input of inputs) refs.push((await this.saveImage(input)).ref) + for (const input of inputs) refs.push(await this.saveImage(input)) return refs } /** * Validate and durably commit one image before its owning session event is appended. - * Implementations may store a prepared master version of the submitted raster; - * the returned reference always describes the stored bytes, while `source` - * preserves the submitted raster's intrinsic facts for callers that report - * or map coordinates against the original. + * The returned reference describes the persisted normalized image. When + * normalization reduces the raster, its `originalDimensions` records the + * orientation-applied input dimensions. * @param input - encoded bytes, declared media type, and optional display name. - * @returns the durable content-addressed reference beside the submitted source facts. + * @returns the durable content-addressed normalized image reference. */ - abstract saveImage(input: SaveImageAttachment): Promise + abstract saveImage(input: SaveImageAttachment): Promise /** * Read one image and verify that bytes still match the recorded reference. * @param ref - durable reference from the session log. * @param signal - optional cancellation for backend read and verification work. - * @returns the verified bytes and master reference. + * @returns the verified bytes and normalized attachment reference. * @throws the signal reason when aborted, or a storage error when verification fails. */ abstract readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise /** - * Generate or read one deterministic model-request version from the stored master image. - * @param ref - durable provider-independent master reference. + * Generate or read one deterministic model-request version from the stored normalized image. + * @param ref - durable provider-independent normalized attachment reference. * @param policy - exact route pixel and encoded-byte budget. * @param signal - optional cancellation. * @returns request bytes and the cache/upload identity covering every transform input. @@ -132,24 +128,6 @@ export abstract class AttachmentStore extends Service { )) } - /** - * Generate or read an ordered batch of deterministic model-request versions. - * Implementations may use their own bounded transform concurrency while preserving input order. - * @param refs - durable provider-independent master references in request order. - * @param policy - exact route pixel and encoded-byte budget shared by the batch. - * @param signal - optional cancellation. - * @returns request versions in the same order as `refs`. - */ - async readImageRequests( - refs: readonly ImageAttachmentRef[], - policy: ImageRequestPolicy, - signal?: AbortSignal, - ): Promise { - const versions: RequestImageAttachment[] = [] - for (const ref of refs) versions.push(await this.readImageRequest(ref, policy, signal)) - return versions - } - } export default AttachmentStore diff --git a/packages/attachment/attachment/src/types.ts b/packages/attachment/attachment/src/types.ts index 04f7362d38..e23a7a7d4c 100644 --- a/packages/attachment/attachment/src/types.ts +++ b/packages/attachment/attachment/src/types.ts @@ -7,7 +7,7 @@ export type { AttachmentId } from './brand.ts' /** Raster image formats accepted by the version-one attachment path. */ export type ImageMediaType = 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif' -/** Durable, serializable metadata for one immutable image object. */ +/** Durable, serializable reference to one immutable normalized image. */ export interface ImageAttachmentRef { /** Opaque storage identifier; never a filesystem path or bearer URL. */ attachmentId: AttachmentId @@ -21,10 +21,14 @@ export interface ImageAttachmentRef { height: number /** Optional display name stripped of local path information. */ name?: string - /** Perceived source width before master-version downscaling; present only when it differs from {@link width}. */ - sourceWidth?: number - /** Perceived source height before master-version downscaling; present only when it differs from {@link height}. */ - sourceHeight?: number + /** + * Input dimensions after applying EXIF orientation and before normalization + * scaling. Present only when normalization reduced the image. + */ + originalDimensions?: { + width: number + height: number + } } /** Deployment-resolved limits used by upload admission and request buffering. */ @@ -71,12 +75,12 @@ export interface ImageRequestPolicy { maxBytes: number } -/** Cached request version derived from one provider-independent master attachment. */ +/** Cached request version derived from one provider-independent normalized attachment. */ export interface RequestImageAttachment { - /** Cache and upload-index key over the master id, policy, and fixed encoder parameters. */ + /** Cache and upload-index key over the attachment id, policy, and fixed encoder parameters. */ variantId: ImageVariantId - /** Durable master reference from which this request version was derived. */ - master: ImageAttachmentRef + /** Durable normalized attachment from which this request version was derived. */ + attachment: ImageAttachmentRef /** Encoded request bytes. */ data: Uint8Array mediaType: ImageMediaType @@ -90,23 +94,3 @@ export interface RequestImageAttachment { /** Whether the encoded request version retains an alpha channel. */ hasAlpha: boolean } - -/** Intrinsic facts of the submitted source raster, before master-version preparation. */ -export interface SourceImageInfo { - /** Media type verified from the submitted bytes. */ - mediaType: ImageMediaType - /** Exact submitted encoded byte length. */ - bytes: number - /** Perceived source width in pixels, with any EXIF orientation applied, so it shares axes with the stored raster. */ - width: number - /** Perceived source height in pixels, with any EXIF orientation applied, so it shares axes with the stored raster. */ - height: number -} - -/** Commit result pairing the durable reference with the submitted source raster it was derived from. */ -export interface SavedImageAttachment { - /** Durable reference describing the stored bytes. */ - ref: ImageAttachmentRef - /** Submitted source raster facts; equals the `ref` fields when the store kept the submitted bytes. */ - source: SourceImageInfo -} diff --git a/packages/attachment/attachment/tests/index.spec.ts b/packages/attachment/attachment/tests/index.spec.ts index 589a4322d9..be784f0276 100644 --- a/packages/attachment/attachment/tests/index.spec.ts +++ b/packages/attachment/attachment/tests/index.spec.ts @@ -10,7 +10,6 @@ import AttachmentStore, { type ImageRequestPolicy, type RequestImageAttachment, type SaveImageAttachment, - type SavedImageAttachment, type StoredImageAttachment, } from '../src/index.ts' @@ -35,20 +34,17 @@ class RecordingStore extends AttachmentStore { if (value === this.rejectValidationAt) throw new Error(`invalid:${value}`) } - async saveImage(input: SaveImageAttachment): Promise { + async saveImage(input: SaveImageAttachment): Promise { const value = input.data[0] ?? 0 this.calls.push(`save:${value}`) if (value === this.rejectSaveAt) throw new Error(`write:${value}`) return { - ref: { - attachmentId: AttachmentId(`sha256:${String(value).padStart(64, '0')}`), - mediaType: input.mediaType, - bytes: input.data.byteLength, - width: 1, - height: 1, - ...input.name === undefined ? {} : { name: input.name }, - }, - source: { mediaType: input.mediaType, bytes: input.data.byteLength, width: 1, height: 1 }, + attachmentId: AttachmentId(`sha256:${String(value).padStart(64, '0')}`), + mediaType: input.mediaType, + bytes: input.data.byteLength, + width: 1, + height: 1, + ...input.name === undefined ? {} : { name: input.name }, } } @@ -63,7 +59,7 @@ class RecordingStore extends AttachmentStore { this.calls.push(`request:${ref.name}`) return Promise.resolve({ variantId: ImageVariantId(`sha256:${String(ref.bytes).padStart(64, '0')}`), - master: ref, + attachment: ref, data: Uint8Array.of(ref.bytes), mediaType: ref.mediaType, bytes: 1, @@ -83,7 +79,7 @@ class UnsupportedProjectionStore extends AttachmentStore { return Promise.resolve() } - saveImage(): Promise { + saveImage(): Promise { throw new Error('not used') } @@ -139,21 +135,10 @@ describe('AttachmentStore.saveImages', () => { }) }) -describe('AttachmentStore.readImageRequests', () => { - it('uses the default serial projection and preserves input order', async () => { - const store = new RecordingStore(new Context()) - const refs = await store.saveImages([image(1), image(2)]) - store.calls.length = 0 - - const versions = await store.readImageRequests(refs, { maxPixels: 1, maxBytes: 1 }) - - expect(store.calls).toEqual(['request:1.png', 'request:2.png']) - expect(versions.map(version => version.master.name)).toEqual(['1.png', '2.png']) - }) - +describe('AttachmentStore.readImageRequest', () => { it('reports unsupported request projection while preserving cancellation', async () => { const store = new UnsupportedProjectionStore(new Context()) - const ref = (await new RecordingStore(new Context()).saveImage(image(1))).ref + const ref = await new RecordingStore(new Context()).saveImage(image(1)) await expect(store.readImageRequest(ref, { maxPixels: 1, maxBytes: 1 })) .rejects.toMatchObject({ code: 'ATTACHMENT_PROJECTION_UNSUPPORTED' }) const controller = new AbortController() diff --git a/packages/extensions/tool-cordis/src/api-catalog.ts b/packages/extensions/tool-cordis/src/api-catalog.ts index 0dbe8f0535..464a3d8f5c 100644 --- a/packages/extensions/tool-cordis/src/api-catalog.ts +++ b/packages/extensions/tool-cordis/src/api-catalog.ts @@ -440,33 +440,27 @@ export const SERVICE_API: readonly ServiceApiEntry[] = [ signature: 'async saveImages(inputs: readonly SaveImageAttachment[]): Promise', description: 'Validate and durably commit one ordered image batch.', parameters: [{ name: 'inputs', description: 'encoded images in owning-message order.' }], - returns: 'durable master references in the same order after every member succeeds.', + returns: 'durable normalized attachment references in the same order after every member succeeds.', }, { - signature: 'abstract saveImage(input: SaveImageAttachment): Promise', - description: 'Validate and durably commit one image before its owning session event is appended. Implementations may store a prepared master version of the submitted raster; the returned reference always describes the stored bytes, while `source` preserves the submitted raster\'s intrinsic facts for callers that report or map coordinates against the original.', + signature: 'abstract saveImage(input: SaveImageAttachment): Promise', + description: 'Validate and durably commit one image before its owning session event is appended. The returned reference describes the persisted normalized image. When normalization reduces the raster, its `originalDimensions` records the orientation-applied input dimensions.', parameters: [{ name: 'input', description: 'encoded bytes, declared media type, and optional display name.' }], - returns: 'the durable content-addressed reference beside the submitted source facts.', + returns: 'the durable content-addressed normalized image reference.', }, { signature: 'abstract readImage(ref: ImageAttachmentRef, signal?: AbortSignal): Promise', description: 'Read one image and verify that bytes still match the recorded reference.', parameters: [{ name: 'ref', description: 'durable reference from the session log.' }, { name: 'signal', description: 'optional cancellation for backend read and verification work.' }], - returns: 'the verified bytes and master reference.', + returns: 'the verified bytes and normalized attachment reference.', throws: ['the signal reason when aborted, or a storage error when verification fails.'], }, { signature: 'readImageRequest( ref: ImageAttachmentRef, policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise', - description: 'Generate or read one deterministic model-request version from the stored master image.', - parameters: [{ name: 'ref', description: 'durable provider-independent master reference.' }, { name: 'policy', description: 'exact route pixel and encoded-byte budget.' }, { name: 'signal', description: 'optional cancellation.' }], + description: 'Generate or read one deterministic model-request version from the stored normalized image.', + parameters: [{ name: 'ref', description: 'durable provider-independent normalized attachment reference.' }, { name: 'policy', description: 'exact route pixel and encoded-byte budget.' }, { name: 'signal', description: 'optional cancellation.' }], returns: 'request bytes and the cache/upload identity covering every transform input.', }, - { - signature: 'async readImageRequests( refs: readonly ImageAttachmentRef[], policy: ImageRequestPolicy, signal?: AbortSignal, ): Promise', - description: 'Generate or read an ordered batch of deterministic model-request versions. Implementations may use their own bounded transform concurrency while preserving input order.', - parameters: [{ name: 'refs', description: 'durable provider-independent master references in request order.' }, { name: 'policy', description: 'exact route pixel and encoded-byte budget shared by the batch.' }, { name: 'signal', description: 'optional cancellation.' }], - returns: 'request versions in the same order as `refs`.', - }, ], }, { @@ -3462,7 +3456,7 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'ImageAttachmentRef', - declaration: 'export interface ImageAttachmentRef {\n attachmentId: AttachmentId;\n mediaType: ImageMediaType;\n bytes: number;\n width: number;\n height: number;\n name?: string;\n sourceWidth?: number;\n sourceHeight?: number;\n}', + declaration: 'export interface ImageAttachmentRef {\n attachmentId: AttachmentId;\n mediaType: ImageMediaType;\n bytes: number;\n width: number;\n height: number;\n name?: string;\n originalDimensions?: {\n width: number;\n height: number;\n };\n}', }, { name: 'ImageBlock', @@ -3942,7 +3936,7 @@ export const TYPE_API: readonly TypeApiEntry[] = [ }, { name: 'RequestImageAttachment', - declaration: 'export interface RequestImageAttachment {\n variantId: ImageVariantId;\n master: ImageAttachmentRef;\n data: Uint8Array;\n mediaType: ImageMediaType;\n bytes: number;\n width: number;\n height: number;\n depth: \'uchar\';\n space: \'srgb\';\n hasAlpha: boolean;\n}', + declaration: 'export interface RequestImageAttachment {\n variantId: ImageVariantId;\n attachment: ImageAttachmentRef;\n data: Uint8Array;\n mediaType: ImageMediaType;\n bytes: number;\n width: number;\n height: number;\n depth: \'uchar\';\n space: \'srgb\';\n hasAlpha: boolean;\n}', }, { name: 'RequestRunOutcome', @@ -4028,10 +4022,6 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'SandboxPolicyRequest', declaration: 'export interface SandboxPolicyRequest {\n session?: Session;\n mode?: SandboxMode;\n}', }, - { - name: 'SavedImageAttachment', - declaration: 'export interface SavedImageAttachment {\n ref: ImageAttachmentRef;\n source: SourceImageInfo;\n}', - }, { name: 'SaveImageAttachment', declaration: 'export interface SaveImageAttachment {\n data: Uint8Array;\n mediaType: ImageMediaType;\n name?: string;\n}', @@ -4436,10 +4426,6 @@ export const TYPE_API: readonly TypeApiEntry[] = [ name: 'SkillViewOptions', declaration: 'export interface SkillViewOptions extends SkillLookupOptions {\n readonly scope?: ScopeKey | undefined;\n}', }, - { - name: 'SourceImageInfo', - declaration: 'export interface SourceImageInfo {\n mediaType: ImageMediaType;\n bytes: number;\n width: number;\n height: number;\n}', - }, { name: 'SpawnTeammateRequest', declaration: 'export interface SpawnTeammateRequest {\n readonly name: string;\n readonly description: string;\n readonly prompt: ContentBlock[];\n readonly context: \'fresh\' | \'fork\';\n readonly provider: string;\n readonly signal: AbortSignal;\n}', diff --git a/packages/fs/tool-fs/README.i18n.yaml b/packages/fs/tool-fs/README.i18n.yaml index 6c590d54b4..3ef67e88c0 100644 --- a/packages/fs/tool-fs/README.i18n.yaml +++ b/packages/fs/tool-fs/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/fs/tool-fs/README.md -README.md: ab01840f122d6e0df2782b86840432914b27ebd0 -README.zh.md: ef738a3715b6db45d386d56ba2a776960dd341c1 +README.md: 763cb831233da5b1f14c73e353920e9d6a87ced9 +README.zh.md: aa55de59452e7779ef278748abac17837800ba02 diff --git a/packages/fs/tool-fs/README.md b/packages/fs/tool-fs/README.md index ab01840f12..763cb83123 100644 --- a/packages/fs/tool-fs/README.md +++ b/packages/fs/tool-fs/README.md @@ -38,7 +38,7 @@ All keys are optional; the defaults are the shipped read caps. Field names are snake_case to match Claude Code and existing harness tool schemas. -Structured successes are `read` → `{ path, offset, lines: [{ number, text }], totalLines }`, `read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name?, sourceWidth?, sourceHeight? } }`, `write` → `{ path, operation: 'create' | 'update', before: string | null, after }`, and `edit` → `{ path, before, after }`. The image source fields appear only when master preparation downscaled the submitted raster. Native renderers preserve the line-numbered read and mutation acknowledgements below. `write`/`edit` derive replayable diff-card metadata, and `read` derives a replayable read-card window `{ path, offset, lines, totalLines, lang? }`; execution-local structured values are not added to `tool/result`, while image renderers emit the durable image blocks that the result logs. +Structured successes are `read` → `{ path, offset, lines: [{ number, text }], totalLines }`, `read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name?, originalDimensions?: { width, height } } }`, `write` → `{ path, operation: 'create' | 'update', before: string | null, after }`, and `edit` → `{ path, before, after }`. `originalDimensions` appears only when normalization downscaled the submitted raster and records its orientation-applied input size. Native renderers preserve the line-numbered read and mutation acknowledgements below. `write`/`edit` derive replayable diff-card metadata, and `read` derives a replayable read-card window `{ path, offset, lines, totalLines, lang? }`; execution-local structured values are not added to `tool/result`, while image renderers emit the durable image blocks that the result logs. ## The tool is the executor; policy is an event gate @@ -127,7 +127,7 @@ Append-only; newly visible content follows the reusable request prefix and does #### What the model sees -A successful `read_image` returns ``, `image`, and a `` envelope naming the media type, master dimensions, and byte size, followed by the image itself as a native image block. The result is logged with its durable reference before the next model request. +A successful `read_image` returns ``, `image`, and a `` envelope naming the media type, normalized dimensions, and byte size, followed by the image itself as a native image block. The result is logged with its durable reference before the next model request. #### Token effect @@ -155,7 +155,7 @@ Append-only; newly visible content follows the reusable request prefix and does #### What the model sees -Failures are normalized as `Error: `. This package's stable validation and read messages are `file_path must be a non-empty string`, `limit must be less than or equal to `, `old_string must be a non-empty string`, `old_string and new_string must differ`, `cannot read "": not found`, `cannot read "": not a regular file`, `offset is out of range for "" ( lines)`, `cannot read "": read_image only accepts PNG/JPEG/WebP/GIF paths`, `cannot read "" as an image: model "" does not declare image input; switch to an image-capable model to read images`, and the mismatch repair `cannot read "": the extension declares , but the bytes use a different image format; rename the file to match its actual format if it is PNG/JPEG/WebP/GIF, or convert it to one of those formats`. A failed 16-bit conversion reports `cannot read "": the 16-bit PNG could not be converted to the canonical 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`. Provider and policy templates are quoted in their package READMEs. Guarded-mutation failures additionally carry their recovery instruction in the message, appended by this package's model-facing error wrapper: `FS_STALE_VERSION` gets `— re-read the file, then retry`, and `FS_NOT_OBSERVED` gets `— read the file, then retry`; the structured code is preserved. After that reread confirms absence, edit reports `FS_NOT_FOUND` instead of repeating a stale remedy, while write uses guarded creation. +Failures are normalized as `Error: `. This package's stable validation and read messages are `file_path must be a non-empty string`, `limit must be less than or equal to `, `old_string must be a non-empty string`, `old_string and new_string must differ`, `cannot read "": not found`, `cannot read "": not a regular file`, `offset is out of range for "" ( lines)`, `cannot read "": read_image only accepts PNG/JPEG/WebP/GIF paths`, `cannot read "" as an image: model "" does not declare image input; switch to an image-capable model to read images`, and the mismatch repair `cannot read "": the extension declares , but the bytes use a different image format; rename the file to match its actual format if it is PNG/JPEG/WebP/GIF, or convert it to one of those formats`. A failed 16-bit conversion reports `cannot read "": the 16-bit PNG could not be converted to the normalized 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`. Provider and policy templates are quoted in their package READMEs. Guarded-mutation failures additionally carry their recovery instruction in the message, appended by this package's model-facing error wrapper: `FS_STALE_VERSION` gets `— re-read the file, then retry`, and `FS_NOT_OBSERVED` gets `— read the file, then retry`; the structured code is preserved. After that reread confirms absence, edit reports `FS_NOT_FOUND` instead of repeating a stale remedy, while write uses guarded creation. #### Token effect diff --git a/packages/fs/tool-fs/README.zh.md b/packages/fs/tool-fs/README.zh.md index ef738a3715..aa55de5945 100644 --- a/packages/fs/tool-fs/README.zh.md +++ b/packages/fs/tool-fs/README.zh.md @@ -38,7 +38,7 @@ await ctx.plugin(ToolFs) // this package — re 字段名使用 snake_case,与 Claude Code 和现有 harness 工具 schema 一致。 -结构化成功值分别为:`read` → `{ path, offset, lines: [{ number, text }], totalLines }`,`read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name?, sourceWidth?, sourceHeight? } }`,`write` → `{ path, operation: 'create' | 'update', before: string | null, after }`,`edit` → `{ path, before, after }`。图片 source 字段只在主版本准备缩小了提交光栅时出现。原生渲染器会保留下方带行号的读取结果和变更确认。`write` 和 `edit` 从这些值派生可回放的 diff 卡片元数据,`read` 派生可回放的读取卡片窗口 `{ path, offset, lines, totalLines, lang? }`;仅用于执行的结构化值不会添加到 `tool/result`,图片渲染器则会发出由结果记录的持久图片块。 +结构化成功值分别为:`read` → `{ path, offset, lines: [{ number, text }], totalLines }`,`read_image` → `{ path, image: { attachmentId, mediaType, bytes, width, height, name?, originalDimensions?: { width, height } } }`,`write` → `{ path, operation: 'create' | 'update', before: string | null, after }`,`edit` → `{ path, before, after }`。`originalDimensions` 只在规范化过程缩小提交光栅时出现,并记录应用方向后的输入尺寸。原生渲染器会保留下方带行号的读取结果和变更确认。`write` 和 `edit` 从这些值派生可回放的 diff 卡片元数据,`read` 派生可回放的读取卡片窗口 `{ path, offset, lines, totalLines, lang? }`;仅用于执行的结构化值不会添加到 `tool/result`,图片渲染器则会发出由结果记录的持久图片块。 ## 工具就是执行器;策略是事件门禁 @@ -127,7 +127,7 @@ Use the edit tool for targeted changes to existing UTF-8 text files. It replaces #### 模型看到的内容 -成功的 `read_image` 返回 ``、`image` 和写明媒体类型、主版本尺寸与字节数的 `` 信封,随后是作为原生图像块的图像本身。结果会随持久引用写入会话日志,然后才进入下一次模型请求。 +成功的 `read_image` 返回 ``、`image` 和写明媒体类型、规范化尺寸与字节数的 `` 信封,随后是作为原生图像块的图像本身。结果会随持久引用写入会话日志,然后才进入下一次模型请求。 #### Token 影响 @@ -155,7 +155,7 @@ Use the edit tool for targeted changes to existing UTF-8 text files. It replaces #### 模型看到的内容 -失败会规范化为 `Error: `。本包稳定的校验和读取消息是 `file_path must be a non-empty string`、`limit must be less than or equal to `、`old_string must be a non-empty string`、`old_string and new_string must differ`、`cannot read "": not found`、`cannot read "": not a regular file`、`offset is out of range for "" ( lines)`、`cannot read "": read_image only accepts PNG/JPEG/WebP/GIF paths`、`cannot read "" as an image: model "" does not declare image input; switch to an image-capable model to read images`,以及类型不匹配的修复消息 `cannot read "": the extension declares , but the bytes use a different image format; rename the file to match its actual format if it is PNG/JPEG/WebP/GIF, or convert it to one of those formats`。16-bit 转换失败会报告 `cannot read "": the 16-bit PNG could not be converted to the canonical 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`。提供方和策略模板在各自包的 README 中逐字列出。防护变更失败还会在消息中携带恢复指令,由本包面向模型的错误包装追加:`FS_STALE_VERSION` 追加 `re-read the file, then retry`,`FS_NOT_OBSERVED` 追加 `read the file, then retry`;结构化错误码保持不变。该次重新读取确认缺失后,edit 会报告 `FS_NOT_FOUND`,不会重复陈旧恢复指令;write 则使用带防护的创建。 +失败会规范化为 `Error: `。本包稳定的校验和读取消息是 `file_path must be a non-empty string`、`limit must be less than or equal to `、`old_string must be a non-empty string`、`old_string and new_string must differ`、`cannot read "": not found`、`cannot read "": not a regular file`、`offset is out of range for "" ( lines)`、`cannot read "": read_image only accepts PNG/JPEG/WebP/GIF paths`、`cannot read "" as an image: model "" does not declare image input; switch to an image-capable model to read images`,以及类型不匹配的修复消息 `cannot read "": the extension declares , but the bytes use a different image format; rename the file to match its actual format if it is PNG/JPEG/WebP/GIF, or convert it to one of those formats`。16-bit 转换失败会报告 `cannot read "": the 16-bit PNG could not be converted to the normalized 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`。提供方和策略模板在各自包的 README 中逐字列出。防护变更失败还会在消息中携带恢复指令,由本包面向模型的错误包装追加:`FS_STALE_VERSION` 追加 `re-read the file, then retry`,`FS_NOT_OBSERVED` 追加 `read the file, then retry`;结构化错误码保持不变。该次重新读取确认缺失后,edit 会报告 `FS_NOT_FOUND`,不会重复陈旧恢复指令;write 则使用带防护的创建。 #### Token 影响 diff --git a/packages/fs/tool-fs/src/read-image.ts b/packages/fs/tool-fs/src/read-image.ts index bbf49d568c..b1cbad9bb9 100644 --- a/packages/fs/tool-fs/src/read-image.ts +++ b/packages/fs/tool-fs/src/read-image.ts @@ -38,8 +38,14 @@ const IMAGE_VALUE_SCHEMA = { width: { type: 'integer', required: true }, height: { type: 'integer', required: true }, name: { type: 'string' }, - sourceWidth: { type: 'integer' }, - sourceHeight: { type: 'integer' }, + originalDimensions: { + type: 'object', + additionalProperties: false, + properties: { + width: { type: 'integer', required: true }, + height: { type: 'integer', required: true }, + }, + }, }, } as const @@ -53,10 +59,11 @@ export interface ImageReadValue { width: number height: number name?: string - /** Intrinsic width of the file on disk; present only when storage downscaled it. */ - sourceWidth?: number - /** Intrinsic height of the file on disk; present only when storage downscaled it. */ - sourceHeight?: number + /** Orientation-applied file dimensions before normalization; present only when storage reduced it. */ + originalDimensions?: { + width: number + height: number + } } } @@ -105,8 +112,9 @@ export function imageRefFromValue(image: ImageReadValue['image']): ImageAttachme width: image.width, height: image.height, ...image.name === undefined ? {} : { name: image.name }, - ...image.sourceWidth === undefined ? {} : { sourceWidth: image.sourceWidth }, - ...image.sourceHeight === undefined ? {} : { sourceHeight: image.sourceHeight }, + ...image.originalDimensions === undefined ? {} : { + originalDimensions: { ...image.originalDimensions }, + }, } } @@ -120,15 +128,15 @@ export function imageRefFromValue(image: ImageReadValue['image']): ImageAttachme */ export function formatImageReadOutput(displayPath: string, image: ImageReadValue['image']): string { let scaled = '' - if (image.sourceWidth !== undefined && image.sourceHeight !== undefined) { + if (image.originalDimensions !== undefined) { // Integer rounding can give the two axes slightly different ratios, so the // advice names one multiplier only when both round to the same value. - const x = (image.sourceWidth / image.width).toFixed(2) - const y = (image.sourceHeight / image.height).toFixed(2) + const x = (image.originalDimensions.width / image.width).toFixed(2) + const y = (image.originalDimensions.height / image.height).toFixed(2) const advice = x === y ? `multiply coordinates by ${x}` : `multiply x coordinates by ${x} and y coordinates by ${y}` - scaled = ` (downscaled from ${image.sourceWidth}x${image.sourceHeight} px; ${advice} to locate features in the original file)` + scaled = ` (downscaled from ${image.originalDimensions.width}x${image.originalDimensions.height} px; ${advice} to locate features in the original file)` } return `${displayPath} image @@ -208,11 +216,8 @@ export function applyReadImageTool(ctx: Context): void { // Persist before returning: the image block must reference a durably // committed object by the time the tool/result event is appended. let ref: ImageAttachmentRef - let source: { width: number; height: number } try { - const saved = await attachments.saveImage({ data, mediaType, name: basename(target.displayPath) }) - ref = saved.ref - source = saved.source + ref = await attachments.saveImage({ data, mediaType, name: basename(target.displayPath) }) } catch (error: unknown) { if (!(error instanceof AttachmentError)) throw error // Dimension refusals stay recoverable tool errors: an oversized image @@ -238,7 +243,7 @@ export function applyReadImageTool(ctx: Context): void { } if (error.code === 'ATTACHMENT_WRITE_FAILED' && /16-bit PNG/iu.test(error.message)) { throw new Error( - `cannot read "${target.displayPath}": the 16-bit PNG could not be converted to the canonical 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`, + `cannot read "${target.displayPath}": the 16-bit PNG could not be converted to the normalized 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`, { cause: error }, ) } @@ -250,7 +255,6 @@ export function applyReadImageTool(ctx: Context): void { ) } ctx.emit('fs/observed', target, { kind: 'present', version: info.version }, exec) - const downscaled = source.width !== ref.width || source.height !== ref.height const value: ImageReadValue = { path: target.displayPath, image: { @@ -260,7 +264,9 @@ export function applyReadImageTool(ctx: Context): void { width: ref.width, height: ref.height, ...ref.name === undefined ? {} : { name: ref.name }, - ...downscaled ? { sourceWidth: source.width, sourceHeight: source.height } : {}, + ...ref.originalDimensions === undefined ? {} : { + originalDimensions: { ...ref.originalDimensions }, + }, }, } return value diff --git a/packages/fs/tool-fs/tests/read-image.spec.ts b/packages/fs/tool-fs/tests/read-image.spec.ts index 16e07d93a8..6b33b3dcb3 100644 --- a/packages/fs/tool-fs/tests/read-image.spec.ts +++ b/packages/fs/tool-fs/tests/read-image.spec.ts @@ -21,7 +21,7 @@ import LocalFileSystem from '@deepseek-ai/dsh-fs-local' import * as FsPolicy from '@deepseek-ai/dsh-fs-observation-policy' import LocalAttachmentStore from '@deepseek-ai/dsh-attachment-local' import { AttachmentError, AttachmentId, AttachmentStore } from '@deepseek-ai/dsh-attachment' -import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, SavedImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' +import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' import * as ToolFs from '@deepseek-ai/dsh-tool-fs' import { applyReadImageTool, @@ -170,8 +170,8 @@ describe('imageRefFromValue', () => { const base = { attachmentId: 'sha256:00', mediaType: 'image/png' as const, bytes: 1, width: 1, height: 1 } expect(imageRefFromValue(base)).toEqual(base) expect(imageRefFromValue({ ...base, name: 'a.png' })).toEqual({ ...base, name: 'a.png' }) - expect(imageRefFromValue({ ...base, sourceWidth: 4, sourceHeight: 2 })) - .toEqual({ ...base, sourceWidth: 4, sourceHeight: 2 }) + expect(imageRefFromValue({ ...base, originalDimensions: { width: 4, height: 2 } })) + .toEqual({ ...base, originalDimensions: { width: 4, height: 2 } }) }) }) @@ -347,7 +347,7 @@ describe('argument and service preconditions', () => { throw new Error('unreachable: admission refuses before validation') } - saveImage(_input: SaveImageAttachment): Promise { + saveImage(_input: SaveImageAttachment): Promise { throw new Error('unreachable: admission refuses before save') } @@ -424,7 +424,7 @@ describe('image admission failures', () => { return Promise.resolve() } - async saveImage(_input: SaveImageAttachment): Promise { + async saveImage(_input: SaveImageAttachment): Promise { throw FailingStore.failure } @@ -442,15 +442,15 @@ describe('image admission failures', () => { expect(text(storageFault)).toContain('Unable to persist image attachment.') FailingStore.failure = new AttachmentError( - 'The 16-bit PNG could not be converted to the canonical 8-bit sRGB form.', + 'The 16-bit PNG could not be converted to the normalized 8-bit sRGB form.', 'ATTACHMENT_WRITE_FAILED', ) const sixteenBit = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model')) expect(text(sixteenBit)).toContain( - `cannot read "${join(dir, 'red.png')}": the 16-bit PNG could not be converted to the canonical 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`, + `cannot read "${join(dir, 'red.png')}": the 16-bit PNG could not be converted to the normalized 8-bit sRGB form; convert it to an 8-bit PNG/JPEG/WebP and retry`, ) - FailingStore.failure = new AttachmentError('Image cannot be encoded within the configured canonical byte target.', 'IMAGE_TOO_LARGE') + FailingStore.failure = new AttachmentError('Image cannot be encoded within the configured normalized-image byte cap.', 'IMAGE_TOO_LARGE') const overBudget = await readImage(ctx, { file_path: 'red.png' }, agentOn('vision-model')) expect(overBudget.isError).toBe(true) expect(text(overBudget)).toContain('cannot be stored within the deployment\'s byte limits; downscale the image and read the smaller copy') @@ -492,11 +492,8 @@ describe('image admission failures', () => { return Promise.resolve() } - async saveImage(input: SaveImageAttachment): Promise { - return { - ref: { attachmentId: AttachmentId('sha256:feed'), mediaType: input.mediaType, bytes: input.data.length, width: 1, height: 1 }, - source: { mediaType: input.mediaType, bytes: input.data.length, width: 1, height: 1 }, - } + async saveImage(input: SaveImageAttachment): Promise { + return { attachmentId: AttachmentId('sha256:feed'), mediaType: input.mediaType, bytes: input.data.length, width: 1, height: 1 } } readImage(_ref: ImageAttachmentRef): Promise { @@ -513,7 +510,7 @@ describe('image admission failures', () => { }) it('names the on-disk dimensions and coordinate multiplier when storage downscales', async () => { - /** Store whose image master halves the source on both sides. */ + /** Store whose normalized image halves the input on both sides. */ class DownscalingStore extends AttachmentStore { readonly imageLimits: ImageAttachmentLimits = Object.freeze({ maxImageBytes: 1024, @@ -528,10 +525,14 @@ describe('image admission failures', () => { return Promise.resolve() } - async saveImage(input: SaveImageAttachment): Promise { + async saveImage(input: SaveImageAttachment): Promise { return { - ref: { attachmentId: AttachmentId('sha256:feed'), mediaType: input.mediaType, bytes: 7, width: 2, height: 1 }, - source: { mediaType: input.mediaType, bytes: input.data.length, width: 4, height: 2 }, + attachmentId: AttachmentId('sha256:feed'), + mediaType: input.mediaType, + bytes: 7, + width: 2, + height: 1, + originalDimensions: { width: 4, height: 2 }, } } @@ -549,7 +550,8 @@ describe('image admission failures', () => { it('names per-axis multipliers when integer rounding makes the ratios differ', () => { const envelope = formatImageReadOutput('/img/photo.jpg', { - attachmentId: 'sha256:feed', mediaType: 'image/jpeg', bytes: 9, width: 2, height: 1, sourceWidth: 5, sourceHeight: 2, + attachmentId: 'sha256:feed', mediaType: 'image/jpeg', bytes: 9, width: 2, height: 1, + originalDimensions: { width: 5, height: 2 }, }) expect(envelope).toContain('downscaled from 5x2 px; multiply x coordinates by 2.50 and y coordinates by 2.00 to locate features in the original file') }) diff --git a/packages/goal/command-goal/tests/command-goal.spec.ts b/packages/goal/command-goal/tests/command-goal.spec.ts index aa163784df..2127844646 100644 --- a/packages/goal/command-goal/tests/command-goal.spec.ts +++ b/packages/goal/command-goal/tests/command-goal.spec.ts @@ -243,11 +243,8 @@ describe('/goal image attachments', () => { const saveImage = (input: { mediaType: string; name?: string }) => { saved += 1 return Promise.resolve({ - ref: { - attachmentId: `att-${saved}`, mediaType: input.mediaType, bytes: 3, width: 1, height: 1, - ...input.name === undefined ? {} : { name: input.name }, - }, - source: { mediaType: input.mediaType, bytes: 3, width: 1, height: 1 }, + attachmentId: `att-${saved}`, mediaType: input.mediaType, bytes: 3, width: 1, height: 1, + ...input.name === undefined ? {} : { name: input.name }, }) } test.ctx.provide('attachments', { @@ -259,7 +256,7 @@ describe('/goal image attachments', () => { saveImage, async saveImages(inputs: readonly { mediaType: string; name?: string }[]) { const refs = [] - for (const input of inputs) refs.push((await saveImage(input)).ref) + for (const input of inputs) refs.push(await saveImage(input)) return refs }, }) diff --git a/packages/host/apiproxy/tests/api-proxy-models.spec.ts b/packages/host/apiproxy/tests/api-proxy-models.spec.ts index 99f99c3432..1317220ef3 100644 --- a/packages/host/apiproxy/tests/api-proxy-models.spec.ts +++ b/packages/host/apiproxy/tests/api-proxy-models.spec.ts @@ -134,15 +134,12 @@ describe('Web session model selection', () => { const { ctx, agent, sessionId } = await harness() const validateImage = vi.fn((_input: { data: Uint8Array }) => Promise.resolve()) const saveImage = vi.fn((input: { data: Uint8Array; mediaType: 'image/png'; name?: string }) => Promise.resolve({ - ref: { - attachmentId: `att-${String(input.data[0])}`, - mediaType: input.mediaType, - bytes: input.data.byteLength, - width: 1, - height: 1, - ...input.name === undefined ? {} : { name: input.name }, - }, - source: { mediaType: input.mediaType, bytes: input.data.byteLength, width: 1, height: 1 }, + attachmentId: `att-${String(input.data[0])}`, + mediaType: input.mediaType, + bytes: input.data.byteLength, + width: 1, + height: 1, + ...input.name === undefined ? {} : { name: input.name }, })) const attachments = { imageLimits: { diff --git a/packages/interaction/commands/tests/commands.spec.ts b/packages/interaction/commands/tests/commands.spec.ts index 85806a1d36..a95ee024dc 100644 --- a/packages/interaction/commands/tests/commands.spec.ts +++ b/packages/interaction/commands/tests/commands.spec.ts @@ -479,11 +479,8 @@ describe('image attachments', () => { saveImage: vi.fn((input: { mediaType: string; name?: string }) => { saved += 1 return Promise.resolve({ - ref: { - attachmentId: `att-${saved}`, mediaType: input.mediaType, bytes: 3, width: 1, height: 1, - ...input.name === undefined ? {} : { name: input.name }, - }, - source: { mediaType: input.mediaType, bytes: 3, width: 1, height: 1 }, + attachmentId: `att-${saved}`, mediaType: input.mediaType, bytes: 3, width: 1, height: 1, + ...input.name === undefined ? {} : { name: input.name }, }) }), validateImageBatch(inputs: readonly unknown[]) { @@ -595,8 +592,7 @@ describe('image attachments', () => { store.saveImage.mockImplementationOnce((input: { mediaType: string }) => { controller.abort('operator cancelled during admission') return Promise.resolve({ - ref: { attachmentId: 'att-late', mediaType: input.mediaType, bytes: 3, width: 1, height: 1 }, - source: { mediaType: input.mediaType, bytes: 3, width: 1, height: 1 }, + attachmentId: 'att-late', mediaType: input.mediaType, bytes: 3, width: 1, height: 1, }) }) ctx.provide('attachments', store) diff --git a/packages/llm/llm-deepseek/README.i18n.yaml b/packages/llm/llm-deepseek/README.i18n.yaml index bea18ff3ac..c4db847155 100644 --- a/packages/llm/llm-deepseek/README.i18n.yaml +++ b/packages/llm/llm-deepseek/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/llm/llm-deepseek/README.md -README.md: bb7f6a520701134cd43ff6223ef4efbf82d02eb4 -README.zh.md: 934c189232711655aa785a7497f5bb6dff1cbb46 +README.md: d17d520c2444d8a0195d997f4df4ff5e0f05befd +README.zh.md: cc823897894102df0dc1da17478eee6ba7ebd21d diff --git a/packages/llm/llm-deepseek/README.md b/packages/llm/llm-deepseek/README.md index bb7f6a5207..d17d520c24 100644 --- a/packages/llm/llm-deepseek/README.md +++ b/packages/llm/llm-deepseek/README.md @@ -49,11 +49,11 @@ The package root exposes the Cordis plugin contract and `DeepSeekAdapter`; wire The plugin registers the single provider route `deepseek-official` together with its resolved `retryPolicy`; omission resolves to normal mode with five retries. A request selects it with `provider: deepseek-official`; its `model` is passed through as the wire `model` string, so changing DeepSeek models does not require lifecycle-time registration. Omitting `models` advertises `deepseek-v4-flash`, `deepseek-v4-pro`, and the image-capable `deepseek-v4-flash-vision-exp`, each with a 1,000,000-token context window; an explicit list replaces those defaults, while `models: []` advertises none. Catalog entries are exposed through `ctx.llm.listModels('deepseek-official')` for clients such as ACP editors and the Web selector, but remain advisory: unlisted model ids still pass through unchanged as text-only routes. An omitted entry name defaults to its id, and omitted `inputModalities` means `text` only. -An image-capable catalog entry declares `inputModalities: [text, image]` and may set `imagePixelBudget`, `imageMaxBytes`, or `imageDetail: low`. The ordinary default is 640,000 total pixels and 1MiB encoded bytes; low detail defaults to 512 by 512 total pixels. The attachment store scales by `min(1, sqrt(pixelBudget / (width * height)))` and rounds inward to keep the pixel count at or below the hard cap, so a 2048 by 1024 master becomes about 1130 by 565 instead of a forced square. Request encoders run lazily: low-color images try PNG (palette only without alpha) then WebP 85 and 80, other alpha images try WebP 85 then 80, and other opaque images try JPEG 85 then 80; dimensions shrink only when both quality attempts exceed 1MiB. Concurrent generation of one `variantId` shares one transform. A caller can cancel its own wait without interrupting other waiters; the transform stops when no waiter remains. The adapter uploads the exact derived request bytes through `POST /files` and sends `{type: "file", file_id}` blocks. It never falls back to an inline data URL. Every retained image is preceded by stable text naming the complete attachment id and actual request dimensions. User, tool-result, agent-loop, compaction, and direct `ctx.llm.stream` requests all use this projection. Text-only routes receive stable attachment placeholders while durable history keeps its image references. +An image-capable catalog entry declares `inputModalities: [text, image]` and may set `imagePixelBudget`, `imageMaxBytes`, or `imageDetail: low`. The ordinary default is 640,000 total pixels and 1MiB encoded bytes; low detail defaults to 512 by 512 total pixels. The attachment store scales by `min(1, sqrt(pixelBudget / (width * height)))` and rounds inward to keep the pixel count at or below the hard cap, so a 2048 by 1024 normalized attachment becomes about 1130 by 565 instead of a forced square. Request encoders run lazily: low-color images try PNG (palette only without alpha) then WebP 85 and 80, other alpha images try WebP 85 then 80, and other opaque images try JPEG 85 then 80; dimensions shrink only when both quality attempts exceed 1MiB. Concurrent generation of one `variantId` shares one transform. A caller can cancel its own wait without interrupting other waiters; the transform stops when no waiter remains. The adapter uploads the exact derived request bytes through `POST /files` and sends `{type: "file", file_id}` blocks. It never falls back to an inline data URL. Every retained image is preceded by stable text naming the complete attachment id and actual request dimensions. User, tool-result, agent-loop, compaction, and direct `ctx.llm.stream` requests all use this projection. Text-only routes receive stable attachment placeholders while durable history keeps its image references. -`maxRequestFilesBytes` and `maxImagesPerRequest` bound the retained request versions at 128MiB and 600 images by default. The byte and count quanta must not exceed their corresponding bounds. Before attachment reads, the adapter uses each route's request-version byte cap as a conservative upper bound and removes the oldest over-budget prefix; only retained masters are read and transformed. Exact derived lengths are checked again without restoring omitted images. When the byte bound is crossed, the oldest prefix advances past the next 64MiB boundary; 129 one-megabyte images remove the oldest 65 and retain 64MiB, and that prefix stays unchanged until durable history exceeds 192MiB. Count overflow advances independently in `imageOffloadCountQuantum` steps. Removed images become the fixed model-visible placeholder `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`. This high-watermark projection avoids changing an old request prefix after every new image. +`maxRequestFilesBytes` and `maxImagesPerRequest` bound the retained request versions at 128MiB and 600 images by default. The byte and count quanta must not exceed their corresponding bounds. Before attachment reads, the adapter uses each route's request-version byte cap as a conservative upper bound and removes the oldest over-budget prefix; only retained normalized attachments are read and transformed. Exact derived lengths are checked again without restoring omitted images. When the byte bound is crossed, the oldest prefix advances past the next 64MiB boundary; 129 one-megabyte images remove the oldest 65 and retain 64MiB, and that prefix stays unchanged until durable history exceeds 192MiB. Count overflow advances independently in `imageOffloadCountQuantum` steps. Removed images become the fixed model-visible placeholder `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`. This high-watermark projection avoids changing an old request prefix after every new image. -Uploaded ids are indexed below `DSH_HOME` by endpoint/API-key scope and request `variantId`. The variant covers the master attachment id, transform version, route pixel and byte budgets, and encoder parameters, so Files API and inline-capable adapters refer to the same deterministic bytes. Uploads request a seven-day lifetime by default and store the server's `expires_at`. A local mapping with no more than one hour remaining is replaced before use; the adapter does not retrieve every remote file before chat. If chat reports expired, deleted, missing, or invalid file ids and names one or more ids used by the request, the adapter removes exactly those mappings. If the provider identifies stale file state without naming an id, it removes every file mapping used by that chat attempt. It then uploads the affected request versions again and retries chat once. A second stale-file rejection clears the mappings identified by that response and is returned without a third chat attempt. An upload response without a complete file object, matching byte count, and `expires_at` is never indexed; a later request therefore uploads again instead of trusting inconsistent local state. A malformed local upload index is treated as an empty cache and replaced by the next successful upload; permission and filesystem I/O failures still fail the request. +Uploaded ids are indexed below `DSH_HOME` by endpoint/API-key scope and request `variantId`. The variant covers the normalized attachment id, transform version, route pixel and byte budgets, and encoder parameters, so Files API and inline-capable adapters refer to the same deterministic bytes. Uploads request a seven-day lifetime by default and store the server's `expires_at`. A local mapping with no more than one hour remaining is replaced before use; the adapter does not retrieve every remote file before chat. If chat reports expired, deleted, missing, or invalid file ids and names one or more ids used by the request, the adapter removes exactly those mappings. If the provider identifies stale file state without naming an id, it removes every file mapping used by that chat attempt. It then uploads the affected request versions again and retries chat once. A second stale-file rejection clears the mappings identified by that response and is returned without a third chat attempt. An upload response without a complete file object, matching byte count, and `expires_at` is never indexed; a later request therefore uploads again instead of trusting inconsistent local state. A malformed local upload index is treated as an empty cache and replaced by the next successful upload; permission and filesystem I/O failures still fail the request. Concurrent resolution of one scoped `variantId` shares one Files upload with waiter-local cancellation. One quota upload failure first paginates and collects the configured number of oldest `dsh-` files, then deletes that set before one upload retry. `DeepSeekFilesClient.delete`, `DeepSeekFileStore.release`, and `releaseAll` expose explicit remote-space reclamation. The current provider limits represented by this package are 128MiB per Files upload, 32MiB per chat-referenced image, 10,000 stored files, and 25GiB per API key; the default 1MiB request version remains below the two per-file limits. diff --git a/packages/llm/llm-deepseek/README.zh.md b/packages/llm/llm-deepseek/README.zh.md index 934c189232..cc82389789 100644 --- a/packages/llm/llm-deepseek/README.zh.md +++ b/packages/llm/llm-deepseek/README.zh.md @@ -49,11 +49,11 @@ harness LLM(大语言模型)seam 的 DeepSeek chat-completions 适配器: 该插件注册唯一提供方路由 `deepseek-official`,并一同注册解析后的 `retryPolicy`;省略时会解析为 normal 模式并重试五次。请求使用 `provider: deepseek-official` 选择该路由;其 `model` 会作为协议 `model` 字符串原样传递,因此更改 DeepSeek 模型不需要生命周期时注册。省略 `models` 会公布 `deepseek-v4-flash`、`deepseek-v4-pro` 与支持图片输入的 `deepseek-v4-flash-vision-exp`,三者的上下文窗口均为 1,000,000 token;显式列表会替换这些默认值,`models: []` 则不公布任何模型。Catalog 配置项通过 `ctx.llm.listModels('deepseek-official')` 公开给 ACP(Agent Client Protocol)编辑器和 Web 选择器等客户端,但仍只提供建议:未列出模型 id 仍原样传递,并按纯文本路由处理。省略配置项 name 默认为其 id,省略 `inputModalities` 则表示仅支持 `text`。 -支持图片的 catalog 配置项声明 `inputModalities: [text, image]`,并可设置 `imagePixelBudget`、`imageMaxBytes` 或 `imageDetail: low`。普通默认值为总像素 640,000、编码字节 1MiB;low detail 的默认总像素为 512×512。附件存储按 `min(1, sqrt(pixelBudget / (width * height)))` 缩放,并向预算内取整,确保总像素不超过硬上限。因此 2048×1024 主版本会得到约 1130×565 的请求版本,而不会被强制变成正方形。请求编码按需执行:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。两个质量档均超过 1MiB 时才缩小尺寸。同一 `variantId` 的并发生成共享一次变换。调用方可以单独取消等待,不会中断其他等待方;没有等待方时才会停止变换。适配器通过 `POST /files` 上传确切的派生请求字节,再发送 `{type: "file", file_id}` 块,不会回退到内联 data URL。每张保留图片前都有稳定文本,写明完整附件 ID 和实际请求尺寸。User、工具结果、agent loop、压缩和直接 `ctx.llm.stream` 请求都使用该投影。纯文本路由会收到稳定的附件占位文本,持久历史继续保留图片引用。 +支持图片的 catalog 配置项声明 `inputModalities: [text, image]`,并可设置 `imagePixelBudget`、`imageMaxBytes` 或 `imageDetail: low`。普通默认值为总像素 640,000、编码字节 1MiB;low detail 的默认总像素为 512×512。附件存储按 `min(1, sqrt(pixelBudget / (width * height)))` 缩放,并向预算内取整,确保总像素不超过硬上限。因此 2048×1024 规范化附件会得到约 1130×565 的请求版本,而不会被强制变成正方形。请求编码按需执行:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。两个质量档均超过 1MiB 时才缩小尺寸。同一 `variantId` 的并发生成共享一次变换。调用方可以单独取消等待,不会中断其他等待方;没有等待方时才会停止变换。适配器通过 `POST /files` 上传确切的派生请求字节,再发送 `{type: "file", file_id}` 块,不会回退到内联 data URL。每张保留图片前都有稳定文本,写明完整附件 ID 和实际请求尺寸。User、工具结果、agent loop、压缩和直接 `ctx.llm.stream` 请求都使用该投影。纯文本路由会收到稳定的附件占位文本,持久历史继续保留图片引用。 -`maxRequestFilesBytes` 和 `maxImagesPerRequest` 限制请求中保留的请求版本,默认值分别为 128MiB 和 600 张。字节和数量步长不得超过对应上限。读取附件前,适配器以路由的请求版本字节上限作为保守上界,移除超预算的最旧前缀,只读取并转换保留的主版本。系统随后用确切派生长度再次检查,但不会重新加入已省略图片。字节数越过上限时,被移除的最旧前缀会越过下一个 64MiB 边界。由 1MiB 图片组成的历史达到 129MiB 时会移除最旧的 65 张并保留 64MiB;直到持久历史超过 192MiB,这个前缀才再次变化。图片数量超限时则按 `imageOffloadCountQuantum` 独立递增。移除的图片会变成固定模型可见占位文本 `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`。这种定量投影不会因每新增一张图片就改写较早的请求前缀。 +`maxRequestFilesBytes` 和 `maxImagesPerRequest` 限制请求中保留的请求版本,默认值分别为 128MiB 和 600 张。字节和数量步长不得超过对应上限。读取附件前,适配器以路由的请求版本字节上限作为保守上界,移除超预算的最旧前缀,只读取并转换保留的规范化附件。系统随后用确切派生长度再次检查,但不会重新加入已省略图片。字节数越过上限时,被移除的最旧前缀会越过下一个 64MiB 边界。由 1MiB 图片组成的历史达到 129MiB 时会移除最旧的 65 张并保留 64MiB;直到持久历史超过 192MiB,这个前缀才再次变化。图片数量超限时则按 `imageOffloadCountQuantum` 独立递增。移除的图片会变成固定模型可见占位文本 `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`。这种定量投影不会因每新增一张图片就改写较早的请求前缀。 -上传 ID 按端点和 API key 作用域以及请求 `variantId` 记录在 `DSH_HOME` 下。变体身份覆盖主附件 ID、变换策略版本、路由像素和字节预算及编码参数,因此 Files API 和支持内联的适配器引用同一份确定性字节。上传默认请求 7 天有效期,并保存服务端返回的 `expires_at`。本地映射剩余时间不超过一小时时会在使用前替换;适配器不会在每次 chat 前查询远端文件。如果 chat 报告文件 ID 已过期、删除、缺失或无效,并指出本次请求使用的一个或多个 ID,适配器只删除这些映射。如果响应只说明文件状态失效而没有指出 ID,适配器会删除该次 chat 使用的全部文件映射。随后重新上传受影响的请求版本,并重试一次 chat。第二次 chat 仍报告文件失效时,适配器会按该响应清理映射并返回错误,不会发起第三次 chat。上传响应若没有完整文件对象、匹配的字节数和 `expires_at`,就不会写入索引;后续请求会再次上传,而不是信任不一致的本地状态。本地上传索引格式损坏时按空缓存处理,并由下一次成功上传替换;权限和文件系统 I/O 错误仍使请求失败。 +上传 ID 按端点和 API key 作用域以及请求 `variantId` 记录在 `DSH_HOME` 下。变体身份覆盖规范化附件 ID、变换策略版本、路由像素和字节预算及编码参数,因此 Files API 和支持内联的适配器引用同一份确定性字节。上传默认请求 7 天有效期,并保存服务端返回的 `expires_at`。本地映射剩余时间不超过一小时时会在使用前替换;适配器不会在每次 chat 前查询远端文件。如果 chat 报告文件 ID 已过期、删除、缺失或无效,并指出本次请求使用的一个或多个 ID,适配器只删除这些映射。如果响应只说明文件状态失效而没有指出 ID,适配器会删除该次 chat 使用的全部文件映射。随后重新上传受影响的请求版本,并重试一次 chat。第二次 chat 仍报告文件失效时,适配器会按该响应清理映射并返回错误,不会发起第三次 chat。上传响应若没有完整文件对象、匹配的字节数和 `expires_at`,就不会写入索引;后续请求会再次上传,而不是信任不一致的本地状态。本地上传索引格式损坏时按空缓存处理,并由下一次成功上传替换;权限和文件系统 I/O 错误仍使请求失败。 同一作用域和 `variantId` 的并发解析共享一次 Files 上传,每个等待方可以单独取消。一次上传配额错误会先分页收集配置数量的最旧 `dsh-` 文件,再删除这些文件并重试一次上传。`DeepSeekFilesClient.delete`、`DeepSeekFileStore.release` 和 `releaseAll` 提供主动远端空间回收。本包记录的当前提供方限制为 Files 单次上传 128MiB、chat 单图引用 32MiB、每个 API key 最多 10,000 个文件和 25GiB;默认 1MiB 请求版本低于两个单文件上限。 diff --git a/packages/llm/llm-deepseek/src/adapter.ts b/packages/llm/llm-deepseek/src/adapter.ts index 30817c738e..9c4756f3d3 100644 --- a/packages/llm/llm-deepseek/src/adapter.ts +++ b/packages/llm/llm-deepseek/src/adapter.ts @@ -200,7 +200,9 @@ async function prepareRequestImages( for (const message of options.messages) collectImageRefs(message.content, refs) const policy = resolveRequestImagePolicy(model) const orderedRefs = [...refs.values()] - const projected = await attachments.readImageRequests(orderedRefs, policy, signal) + const projected = await Promise.all(orderedRefs.map( + ref => attachments.readImageRequest(ref, policy, signal), + )) return new Map(orderedRefs.map((ref, index) => ( [ref.attachmentId, projected[index] as RequestImageAttachment] ))) @@ -250,7 +252,7 @@ function normalizedImageFacts( file: { version: RequestImageAttachment; location: ImageWireLocation }, ): string { const version = file.version - const name = version.master.name ?? version.master.attachmentId + const name = version.attachment.name ?? version.attachment.attachmentId const colour = version.hasAlpha ? 'sRGBA' : 'sRGB' return `"${name}" at message ${file.location.message}, image ${file.location.image} ` + `(${version.mediaType}, 8-bit ${colour}, ${version.width}x${version.height})` diff --git a/packages/llm/llm-deepseek/src/file-store.ts b/packages/llm/llm-deepseek/src/file-store.ts index 0757b42db2..fde86fa44e 100644 --- a/packages/llm/llm-deepseek/src/file-store.ts +++ b/packages/llm/llm-deepseek/src/file-store.ts @@ -102,9 +102,9 @@ function extension(mediaType: RequestImageAttachment['mediaType']): 'png' | 'jpe } function filename(version: RequestImageAttachment): string { - const master = String(version.master.attachmentId).slice('sha256:'.length, 'sha256:'.length + 16) + const attachment = String(version.attachment.attachmentId).slice('sha256:'.length, 'sha256:'.length + 16) const variant = String(version.variantId).slice('sha256:'.length, 'sha256:'.length + 8) - return `${OWNED_FILE_PREFIX}${master}-${variant}.${extension(version.mediaType)}` + return `${OWNED_FILE_PREFIX}${attachment}-${variant}.${extension(version.mediaType)}` } /** User-scoped durable file-id reuse for the DeepSeek route. */ @@ -204,7 +204,7 @@ export class DeepSeekFileStore { } return { scope, - masterAttachmentId: version.master.attachmentId, + attachmentId: version.attachment.attachmentId, variantId: version.variantId, fileId: remote.id, bytes: remote.bytes, diff --git a/packages/llm/llm-deepseek/src/upload-index.ts b/packages/llm/llm-deepseek/src/upload-index.ts index 297e1021c1..d442bb55fe 100644 --- a/packages/llm/llm-deepseek/src/upload-index.ts +++ b/packages/llm/llm-deepseek/src/upload-index.ts @@ -13,8 +13,8 @@ import type { DeepSeekFileId as DeepSeekFileIdType, DeepSeekFileScope as DeepSee /** One durable remote upload mapping. Unix times are milliseconds. */ export interface DeepSeekUploadRecord { scope: DeepSeekFileScopeType - /** Provider-independent master attachment from which the uploaded request version was derived. */ - masterAttachmentId: AttachmentId + /** Provider-independent normalized attachment from which the uploaded request version was derived. */ + attachmentId: AttachmentId /** Complete request transformation identity, including route budgets and encoder parameters. */ variantId: ImageVariantIdType fileId: DeepSeekFileIdType @@ -24,7 +24,7 @@ export interface DeepSeekUploadRecord { } interface StoredIndex { - formatVersion: 2 + formatVersion: 3 records: DeepSeekUploadRecord[] } @@ -61,7 +61,7 @@ function parseRecord(value: unknown): DeepSeekUploadRecord { } const record = value as Record if (typeof record.scope !== 'string' || !/^[0-9a-f]{64}$/u.test(record.scope) - || typeof record.masterAttachmentId !== 'string' || !/^sha256:[0-9a-f]{64}$/u.test(record.masterAttachmentId) + || typeof record.attachmentId !== 'string' || !/^sha256:[0-9a-f]{64}$/u.test(record.attachmentId) || typeof record.variantId !== 'string' || !/^sha256:[0-9a-f]{64}$/u.test(record.variantId) || typeof record.fileId !== 'string' || record.fileId.length === 0 || !Number.isSafeInteger(record.bytes) || (record.bytes as number) < 0 @@ -71,7 +71,7 @@ function parseRecord(value: unknown): DeepSeekUploadRecord { } return { scope: DeepSeekFileScope(record.scope), - masterAttachmentId: record.masterAttachmentId as AttachmentId, + attachmentId: record.attachmentId as AttachmentId, variantId: ImageVariantId(record.variantId), fileId: DeepSeekFileId(record.fileId), bytes: record.bytes as number, @@ -91,7 +91,7 @@ function parseIndex(text: string): StoredIndex { throw new InvalidUploadIndexError('llm-deepseek: upload index is not an object') } const index = value as { formatVersion?: unknown; records?: unknown } - if (index.formatVersion !== 2 || !Array.isArray(index.records)) { + if (index.formatVersion !== 3 || !Array.isArray(index.records)) { throw new InvalidUploadIndexError('llm-deepseek: unsupported upload index format') } const records = index.records.map(parseRecord) @@ -101,7 +101,7 @@ function parseIndex(text: string): StoredIndex { if (keys.has(key)) throw new InvalidUploadIndexError('llm-deepseek: upload index contains duplicate mappings') keys.add(key) } - return { formatVersion: 2, records } + return { formatVersion: 3, records } } function reusable(record: DeepSeekUploadRecord, now: number, refreshMarginMs: number): boolean { @@ -114,9 +114,9 @@ export class DeepSeekUploadIndex { readonly path: string /** - * @param path - explicit test path; omission uses `DSH_HOME/llm-deepseek/files-v2.json`. + * @param path - explicit test path; omission uses `DSH_HOME/llm-deepseek/files-v3.json`. */ - constructor(path = join(resolveDshHome(), 'llm-deepseek', 'files-v2.json')) { + constructor(path = join(resolveDshHome(), 'llm-deepseek', 'files-v3.json')) { this.path = path } @@ -125,7 +125,7 @@ export class DeepSeekUploadIndex { return parseIndex(await readFile(this.path, 'utf8')) } catch (error: unknown) { if (absent(error) || error instanceof InvalidUploadIndexError) { - return { formatVersion: 2, records: [] } + return { formatVersion: 3, records: [] } } throw error } @@ -184,7 +184,7 @@ export class DeepSeekUploadIndex { && !(record.scope === candidate.scope && record.variantId === candidate.variantId) )) records.push(candidate) - await this.save({ formatVersion: 2, records }) + await this.save({ formatVersion: 3, records }) return { record: candidate, accepted: true } }) } @@ -206,7 +206,7 @@ export class DeepSeekUploadIndex { const records = index.records.filter(record => !( record.scope === scope && record.variantId === variantId && record.fileId === fileId )) - if (records.length !== index.records.length) await this.save({ formatVersion: 2, records }) + if (records.length !== index.records.length) await this.save({ formatVersion: 3, records }) }) } @@ -219,7 +219,7 @@ export class DeepSeekUploadIndex { await withFileLock(this.path, async () => { const index = await this.load() const records = index.records.filter(record => record.scope !== scope) - if (records.length !== index.records.length) await this.save({ formatVersion: 2, records }) + if (records.length !== index.records.length) await this.save({ formatVersion: 3, records }) }) } } diff --git a/packages/llm/llm-deepseek/tests/adapter.e2e.ts b/packages/llm/llm-deepseek/tests/adapter.e2e.ts index 3858f214a0..ee3bea8435 100644 --- a/packages/llm/llm-deepseek/tests/adapter.e2e.ts +++ b/packages/llm/llm-deepseek/tests/adapter.e2e.ts @@ -13,7 +13,6 @@ import type { ImageAttachmentRef, ImageRequestPolicy, RequestImageAttachment, - SavedImageAttachment, SaveImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' @@ -58,7 +57,7 @@ class E2eAttachmentStore extends AttachmentStore { } readonly version: RequestImageAttachment = { variantId: ImageVariantId(`sha256:${randomBytes(32).toString('hex')}`), - master: this.ref, + attachment: this.ref, data: TEST_PNG, mediaType: 'image/png', bytes: TEST_PNG.byteLength, @@ -73,16 +72,8 @@ class E2eAttachmentStore extends AttachmentStore { return Promise.resolve() } - saveImage(_input: SaveImageAttachment): Promise { - return Promise.resolve({ - ref: this.ref, - source: { - mediaType: this.ref.mediaType, - bytes: this.ref.bytes, - width: this.ref.width, - height: this.ref.height, - }, - }) + saveImage(_input: SaveImageAttachment): Promise { + return Promise.resolve(this.ref) } readImage(ref: ImageAttachmentRef, _signal?: AbortSignal): Promise { diff --git a/packages/llm/llm-deepseek/tests/adapter.spec.ts b/packages/llm/llm-deepseek/tests/adapter.spec.ts index 08b3706758..baac1ee39e 100644 --- a/packages/llm/llm-deepseek/tests/adapter.spec.ts +++ b/packages/llm/llm-deepseek/tests/adapter.spec.ts @@ -77,7 +77,7 @@ const imageRef: ImageAttachmentRef = { function requestImage(ref = imageRef): RequestImageAttachment { return { variantId: ImageVariantId(`sha256:${'b'.repeat(64)}`), - master: ref, + attachment: ref, data: Uint8Array.of(1, 2, 3), mediaType: 'image/png', bytes: 3, @@ -94,18 +94,11 @@ function attachmentStoreOf( ): { store: AttachmentStore readImageRequest: ReturnType> - readImageRequests: ReturnType } { const readImageRequest = vi.fn(project) - const readImageRequests = vi.fn(async ( - refs: readonly ImageAttachmentRef[], - policy: unknown, - signal?: AbortSignal, - ) => Promise.all(refs.map(ref => readImageRequest(ref, policy, signal)))) return { - store: { readImageRequest, readImageRequests } as unknown as AttachmentStore, + store: { readImageRequest } as unknown as AttachmentStore, readImageRequest, - readImageRequests, } } @@ -233,8 +226,8 @@ describe('DeepSeekAdapter against a mock server', () => { })], })) - expect(attachmentMocks.readImageRequests).toHaveBeenCalledWith( - [recent], + expect(attachmentMocks.readImageRequest).toHaveBeenCalledWith( + recent, { maxPixels: 640_000, maxBytes: 1024 * 1024 }, expect.any(AbortSignal), ) @@ -283,15 +276,15 @@ describe('DeepSeekAdapter against a mock server', () => { await drain(adapter.stream({ provider: 'deepseek-official', model: 'vision-low', messages: [nested] })) await drain(adapter.stream({ provider: 'deepseek-official', model: 'vision-custom', messages: [nested] })) - expect(attachmentMocks.readImageRequests).toHaveBeenNthCalledWith( + expect(attachmentMocks.readImageRequest).toHaveBeenNthCalledWith( 1, - [imageRef], + imageRef, { maxPixels: 512 * 512, maxBytes: 512_000 }, expect.any(AbortSignal), ) - expect(attachmentMocks.readImageRequests).toHaveBeenNthCalledWith( + expect(attachmentMocks.readImageRequest).toHaveBeenNthCalledWith( 2, - [imageRef], + imageRef, { maxPixels: 320_000, maxBytes: 1024 * 1024 }, expect.any(AbortSignal), ) @@ -395,7 +388,7 @@ describe('DeepSeekAdapter against a mock server', () => { return Promise.resolve({ ...requestImage(ref), variantId: ImageVariantId(`sha256:${(first ? 'b' : 'd').repeat(64)}`), - master: first ? { ...ref, name: 'diagram.png' } : ref, + attachment: first ? { ...ref, name: 'diagram.png' } : ref, hasAlpha: false, }) }).store diff --git a/packages/llm/llm-deepseek/tests/dynamic-config.spec.ts b/packages/llm/llm-deepseek/tests/dynamic-config.spec.ts index c0a2e29750..4617ebdfed 100644 --- a/packages/llm/llm-deepseek/tests/dynamic-config.spec.ts +++ b/packages/llm/llm-deepseek/tests/dynamic-config.spec.ts @@ -10,7 +10,6 @@ import type { ImageAttachmentRef, ImageRequestPolicy, RequestImageAttachment, - SavedImageAttachment, SaveImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' @@ -46,11 +45,8 @@ class StaticAttachmentStore extends AttachmentStore { return Promise.resolve() } - saveImage(_input: SaveImageAttachment): Promise { - return Promise.resolve({ - ref: IMAGE_REF, - source: { mediaType: IMAGE_REF.mediaType, bytes: IMAGE_REF.bytes, width: IMAGE_REF.width, height: IMAGE_REF.height }, - }) + saveImage(_input: SaveImageAttachment): Promise { + return Promise.resolve(IMAGE_REF) } readImage(ref: ImageAttachmentRef, _signal?: AbortSignal): Promise { @@ -64,7 +60,7 @@ class StaticAttachmentStore extends AttachmentStore { ): Promise { return Promise.resolve({ variantId: ImageVariantId(`sha256:${'b'.repeat(64)}`), - master: ref, + attachment: ref, data: Uint8Array.of(1, 2, 3), mediaType: ref.mediaType, bytes: 3, diff --git a/packages/llm/llm-deepseek/tests/file-store.spec.ts b/packages/llm/llm-deepseek/tests/file-store.spec.ts index 069ff47c9d..d6d154033b 100644 --- a/packages/llm/llm-deepseek/tests/file-store.spec.ts +++ b/packages/llm/llm-deepseek/tests/file-store.spec.ts @@ -17,7 +17,7 @@ const REF: ImageAttachmentRef = { } const VERSION: RequestImageAttachment = { variantId: ImageVariantId(`sha256:${'b'.repeat(64)}`), - master: REF, + attachment: REF, data: Uint8Array.of(1, 2, 3), mediaType: 'image/png', bytes: 3, @@ -328,7 +328,7 @@ describe('DeepSeekFileStore', () => { accepted: false, record: { scope: deepSeekFileScope(CONNECTION.baseURL, CONNECTION.apiKey), - masterAttachmentId: VERSION.master.attachmentId, + attachmentId: VERSION.attachment.attachmentId, variantId: VERSION.variantId, fileId: DeepSeekFileId('file-api-winner'), bytes: 3, diff --git a/packages/llm/llm-deepseek/tests/serialize.spec.ts b/packages/llm/llm-deepseek/tests/serialize.spec.ts index 1b14a0c320..547713a74c 100644 --- a/packages/llm/llm-deepseek/tests/serialize.spec.ts +++ b/packages/llm/llm-deepseek/tests/serialize.spec.ts @@ -39,7 +39,7 @@ function requestVersion(ref: ImageAttachmentRef): RequestImageAttachment { const hash = String(ref.attachmentId).slice('sha256:'.length) return { variantId: ImageVariantId(`sha256:${hash}`), - master: ref, + attachment: ref, data: new Uint8Array(ref.bytes), mediaType: ref.mediaType, bytes: ref.bytes, @@ -545,7 +545,7 @@ describe('image serialization', () => { ], }) expect(resolveFileId).toHaveBeenCalledTimes(1) - expect(resolveFileId.mock.calls[0]?.[0]).toMatchObject({ master: { mediaType: 'image/jpeg' } }) + expect(resolveFileId.mock.calls[0]?.[0]).toMatchObject({ attachment: { mediaType: 'image/jpeg' } }) }) it('rejects an unprepared image while computing exact request bytes', async () => { diff --git a/packages/llm/llm-deepseek/tests/upload-index.spec.ts b/packages/llm/llm-deepseek/tests/upload-index.spec.ts index 480772f5fb..2cad8a22be 100644 --- a/packages/llm/llm-deepseek/tests/upload-index.spec.ts +++ b/packages/llm/llm-deepseek/tests/upload-index.spec.ts @@ -22,7 +22,7 @@ describe('DeepSeekUploadIndex', () => { const second = deepSeekFileScope('https://api.deepseek.com', 'second-key') const record = { scope: first, - masterAttachmentId: ATTACHMENT, + attachmentId: ATTACHMENT, variantId: VARIANT, fileId: DeepSeekFileId('file-api-one'), bytes: 3, @@ -41,7 +41,7 @@ describe('DeepSeekUploadIndex', () => { const index = new DeepSeekUploadIndex(join(dir, 'index.json')) const scope = deepSeekFileScope('https://api.deepseek.com', 'key') const first = { - scope, masterAttachmentId: ATTACHMENT, variantId: VARIANT, + scope, attachmentId: ATTACHMENT, variantId: VARIANT, fileId: DeepSeekFileId('file-api-first'), bytes: 3, createdAt: 1, expiresAt: 10_000, } const duplicate = { ...first, fileId: DeepSeekFileId('file-api-duplicate') } @@ -62,7 +62,7 @@ describe('DeepSeekUploadIndex', () => { const scope = deepSeekFileScope('https://api.deepseek.com', 'key') const record = { scope, - masterAttachmentId: ATTACHMENT, + attachmentId: ATTACHMENT, variantId: VARIANT, fileId: DeepSeekFileId('file-api-repaired'), bytes: 3, @@ -73,7 +73,7 @@ describe('DeepSeekUploadIndex', () => { await expect(index.get(scope, VARIANT, 1, 1)).resolves.toBeUndefined() await expect(index.commit(record, 1, 1)).resolves.toEqual({ record, accepted: true }) await expect(index.get(scope, VARIANT, 1, 1)).resolves.toEqual(record) - expect(JSON.parse(await readFile(path, 'utf8'))).toMatchObject({ formatVersion: 2 }) + expect(JSON.parse(await readFile(path, 'utf8'))).toMatchObject({ formatVersion: 3 }) }) it.each([ @@ -81,48 +81,49 @@ describe('DeepSeekUploadIndex', () => { '[]', '{}', '{"formatVersion":1,"records":[]}', - '{"formatVersion":2,"records":null}', - '{"formatVersion":2,"records":[null]}', - '{"formatVersion":2,"records":[[]]}', - '{"formatVersion":2,"records":[{}]}', - `{"formatVersion":2,"records":[${JSON.stringify({ - scope: 'x'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + '{"formatVersion":2,"records":[]}', + '{"formatVersion":3,"records":null}', + '{"formatVersion":3,"records":[null]}', + '{"formatVersion":3,"records":[[]]}', + '{"formatVersion":3,"records":[{}]}', + `{"formatVersion":3,"records":[${JSON.stringify({ + scope: 'x'.repeat(64), attachmentId: ATTACHMENT, variantId: VARIANT, fileId: 'file-api-one', bytes: 3, createdAt: 1, expiresAt: 10_000, })}]}`, - `{"formatVersion":2,"records":[${JSON.stringify({ - scope: 'a'.repeat(64), masterAttachmentId: 'wrong', variantId: VARIANT, + `{"formatVersion":3,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), attachmentId: 'wrong', variantId: VARIANT, fileId: 'file-api-one', bytes: 3, createdAt: 1, expiresAt: 10_000, })}]}`, - `{"formatVersion":2,"records":[${JSON.stringify({ - scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: 'wrong', + `{"formatVersion":3,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), attachmentId: ATTACHMENT, variantId: 'wrong', fileId: 'file-api-one', bytes: 3, createdAt: 1, expiresAt: 10_000, })}]}`, - `{"formatVersion":2,"records":[${JSON.stringify({ - scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + `{"formatVersion":3,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), attachmentId: ATTACHMENT, variantId: VARIANT, fileId: '', bytes: 3, createdAt: 1, expiresAt: 10_000, })}]}`, - `{"formatVersion":2,"records":[${JSON.stringify({ - scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + `{"formatVersion":3,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), attachmentId: ATTACHMENT, variantId: VARIANT, fileId: 'file-api-one', bytes: -1, createdAt: 1, expiresAt: 10_000, })}]}`, - `{"formatVersion":2,"records":[${JSON.stringify({ - scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + `{"formatVersion":3,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), attachmentId: ATTACHMENT, variantId: VARIANT, fileId: 'file-api-one', bytes: 1.5, createdAt: 1, expiresAt: 10_000, })}]}`, - `{"formatVersion":2,"records":[${JSON.stringify({ - scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + `{"formatVersion":3,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), attachmentId: ATTACHMENT, variantId: VARIANT, fileId: 'file-api-one', bytes: 3, createdAt: -1, expiresAt: 10_000, })}]}`, - `{"formatVersion":2,"records":[${JSON.stringify({ - scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + `{"formatVersion":3,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), attachmentId: ATTACHMENT, variantId: VARIANT, fileId: 'file-api-one', bytes: 3, createdAt: 1.5, expiresAt: 10_000, })}]}`, - `{"formatVersion":2,"records":[${JSON.stringify({ - scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + `{"formatVersion":3,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), attachmentId: ATTACHMENT, variantId: VARIANT, fileId: 'file-api-one', bytes: 3, createdAt: 1, expiresAt: -1, })}]}`, - `{"formatVersion":2,"records":[${JSON.stringify({ - scope: 'a'.repeat(64), masterAttachmentId: ATTACHMENT, variantId: VARIANT, + `{"formatVersion":3,"records":[${JSON.stringify({ + scope: 'a'.repeat(64), attachmentId: ATTACHMENT, variantId: VARIANT, fileId: 'file-api-one', bytes: 3, createdAt: 1, expiresAt: 1.5, })}]}`, ])('treats an invalid persisted index as empty %#', async (text) => { @@ -140,10 +141,10 @@ describe('DeepSeekUploadIndex', () => { const path = join(dir, 'index.json') const scope = deepSeekFileScope('https://api.deepseek.com', 'key') const record = { - scope, masterAttachmentId: ATTACHMENT, variantId: VARIANT, + scope, attachmentId: ATTACHMENT, variantId: VARIANT, fileId: DeepSeekFileId('file-api-one'), bytes: 3, createdAt: 1, expiresAt: 10_000, } - await writeFile(path, JSON.stringify({ formatVersion: 2, records: [record, record] }), 'utf8') + await writeFile(path, JSON.stringify({ formatVersion: 3, records: [record, record] }), 'utf8') const index = new DeepSeekUploadIndex(path) await expect(index.get(scope, VARIANT, 1, 1)).resolves.toBeUndefined() }) @@ -154,7 +155,7 @@ describe('DeepSeekUploadIndex', () => { const first = deepSeekFileScope('https://api.deepseek.com', 'first') const second = deepSeekFileScope('https://api.deepseek.com', 'second') const expired = { - scope: first, masterAttachmentId: ATTACHMENT, variantId: VARIANT, + scope: first, attachmentId: ATTACHMENT, variantId: VARIANT, fileId: DeepSeekFileId('file-api-expired'), bytes: 3, createdAt: 1, expiresAt: 2, } const live = { diff --git a/packages/llm/llm-pi-ai/README.i18n.yaml b/packages/llm/llm-pi-ai/README.i18n.yaml index 038224198d..42364c1d5c 100644 --- a/packages/llm/llm-pi-ai/README.i18n.yaml +++ b/packages/llm/llm-pi-ai/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/llm/llm-pi-ai/README.md -README.md: 8f4d1537d8ccec3e89c0553f877541d11b285f66 -README.zh.md: 354851018de0ea79b82215c3d970266cd2be5763 +README.md: 43472de90803481deebb9bc91586a4b85e443db5 +README.zh.md: 76b3dc9dec2a4bf83933319aa065954a191e595b diff --git a/packages/llm/llm-pi-ai/README.md b/packages/llm/llm-pi-ai/README.md index 8f4d1537d8..43472de908 100644 --- a/packages/llm/llm-pi-ai/README.md +++ b/packages/llm/llm-pi-ai/README.md @@ -123,7 +123,7 @@ A model that carries reasoning metadata — from the installed catalog or from i A model **without** that metadata — a hand-declared one whose entry declares no `reasoningEfforts`, and a catalog model pi-ai marks as non-reasoning — exposes no `reasoning` at all. pi-ai reports such a model as supporting the single level `off`, but `off` is translated to *omitting* the reasoning option, which is byte-for-byte the request that naming no effort already produces: selecting it could not disable anything, so a provider whose own default is to think would keep thinking with `off` shown as selected. Reporting the capability as unavailable leaves a surface offering the provider's default and nothing that misrepresents it. The profile `reasoning` value, including `off`, is the deployment default when configured; omitting it preserves the provider default. Per-request `GenerateOptions.reasoningEffort` takes precedence, and a level absent from the exact model capability fails the REQUEST with `UNSUPPORTED_REASONING_EFFORT` before network I/O instead of being clamped. Describing a model never fails that way: the models under one provider disagree about which levels they accept, so `resolveModel` reports a profile level the exact model cannot take as no default at all rather than throwing. A throw there would take the whole provider out of every model catalog built over it — one mis-set profile field hiding even the models that do support the level — so a bad configuration surfaces where it is acted on, not where it is described. pi-ai's common stream options represent `off` by omitting `reasoning`. -Supported profile fields are `apiKeyEnv`, `displayName`, `api`, `baseURL`, `models`, `modelOverrides`, `compat`, `defaultContextWindow`, `defaultMaxTokens`, `defaultInput`, `headers`, `reasoning`, `thinkingBudgets`, `cacheRetention`, `transport`, `timeoutMs`, `websocketConnectTimeoutMs`, `streamIdleTimeoutMs`, `maxRequestImageBytes`, `requestImagePixelBudget`, `requestImageMaxBytes`, and `retryPolicy`. Each resolved profile retry policy is captured with that provider route; omission uses the shared bounded normal default of five retries. The stream-idle interval is a positive finite Node timer delay, defaults to five minutes, and covers only an outstanding provider read, not consumer think time. Every image route derives a deterministic request version from the provider-independent master under `requestImagePixelBudget` (default 2048 by 2048 total pixels) and `requestImageMaxBytes` (default 1MiB raw bytes). Before reading masters, `maxRequestImageBytes` applies to conservative request-version upper bounds and replaces the oldest over-budget images with fixed text; exact base64 lengths are checked again after retained versions are generated. The 20MiB default can retain fifteen maximum-size 1MiB versions after base64 expansion while leaving request-body headroom. The same version feeds inline base64, and its stable descriptor exposes the attachment id and actual request-image dimensions. Harness app attribution wins a conflicting configured header name. +Supported profile fields are `apiKeyEnv`, `displayName`, `api`, `baseURL`, `models`, `modelOverrides`, `compat`, `defaultContextWindow`, `defaultMaxTokens`, `defaultInput`, `headers`, `reasoning`, `thinkingBudgets`, `cacheRetention`, `transport`, `timeoutMs`, `websocketConnectTimeoutMs`, `streamIdleTimeoutMs`, `maxRequestImageBytes`, `requestImagePixelBudget`, `requestImageMaxBytes`, and `retryPolicy`. Each resolved profile retry policy is captured with that provider route; omission uses the shared bounded normal default of five retries. The stream-idle interval is a positive finite Node timer delay, defaults to five minutes, and covers only an outstanding provider read, not consumer think time. Every image route derives a deterministic request version from the provider-independent normalized attachment under `requestImagePixelBudget` (default 2048 by 2048 total pixels) and `requestImageMaxBytes` (default 1MiB raw bytes). Before reading attachments, `maxRequestImageBytes` applies to conservative request-version upper bounds and replaces the oldest over-budget images with fixed text; exact base64 lengths are checked again after retained versions are generated. The 20MiB default can retain fifteen maximum-size 1MiB versions after base64 expansion while leaving request-body headroom. The same version feeds inline base64, and its stable descriptor exposes the attachment id and actual request-image dimensions. Harness app attribution wins a conflicting configured header name. The adapter forces pi-ai's SDK `maxRetries` to zero so one `stream()` call makes one provider request. The removed profile fields `maxRetries` and `maxRetryDelayMs` fail load instead of silently multiplying or hiding the separately composed agent-level retry budget. Idle expiry aborts the SDK's stable request signal and surfaces `TIMEOUT`; an earlier caller abort remains `ABORTED`. @@ -173,7 +173,7 @@ pi-ai installs several provider SDKs and lazy-loads the one selected by the cata #### What the model sees -The selected catalog model receives `GenerateOptions.system`, history, tools, and sampling fields supported by pi-ai's common streaming API. Each retained image is preceded by stable text naming its complete attachment id and actual request dimensions. When accumulated base64 image payload exceeds the route's `maxRequestImageBytes`, each offloaded image (oldest first) is replaced by fixed text that tells the model to read the file again when a path is available or ask the user to attach it again. Offloaded masters are not read or transformed. Provider-native replay metadata is restored only when the adapter validates it for the historical content. +The selected catalog model receives `GenerateOptions.system`, history, tools, and sampling fields supported by pi-ai's common streaming API. Each retained image is preceded by stable text naming its complete attachment id and actual request dimensions. When accumulated base64 image payload exceeds the route's `maxRequestImageBytes`, each offloaded image (oldest first) is replaced by fixed text that tells the model to read the file again when a path is available or ask the user to attach it again. Offloaded normalized attachments are not read or transformed. Provider-native replay metadata is restored only when the adapter validates it for the historical content. #### Token effect diff --git a/packages/llm/llm-pi-ai/README.zh.md b/packages/llm/llm-pi-ai/README.zh.md index 354851018d..76b3dc9dec 100644 --- a/packages/llm/llm-pi-ai/README.zh.md +++ b/packages/llm/llm-pi-ai/README.zh.md @@ -124,7 +124,7 @@ pi-ai 依据提供方 id 与 baseURL 决定每个请求的形状:系统提示 **没有**这份元数据的模型——条目未声明 `reasoningEfforts` 的手工声明模型,以及 pi-ai 标记为不具备推理能力的 catalog 模型——完全不公开 `reasoning`。pi-ai 会把这类模型报告为只支持 `off` 一档,但 `off` 会被翻译成*省略* reasoning 选项,而那与「不点名任何档位」产出的请求逐字节相同:选它关不掉任何东西,于是自身默认就在思考的提供方,会在界面显示 `off` 被选中的同时继续思考。把该能力报告为不可用,界面就只剩提供方默认这一项,不会再出现自相矛盾的控件。配置 profile 的 `reasoning` 值(包括 `off`)在存在时是部署默认值;省略它会保留提供方默认值。每次请求的 `GenerateOptions.reasoningEffort` 优先;未出现在确切模型能力中的档位会让**请求**在网络 I/O 前以 `UNSUPPORTED_REASONING_EFFORT` 失败,而不会被自动调整。**描述**一个模型则从不这样失败:同一提供方下各模型接受的档位并不一致,因此 `resolveModel` 对该模型拿不下的 profile 档位报告为「没有默认值」,而不是抛错。在那里抛错会让整个提供方从任何基于它构建的模型目录中消失——一个配错的 profile 字段连支持该档位的模型也一并藏起来——所以坏配置暴露在被执行处,而不是被描述处。pi-ai 的通用流选项通过省略 `reasoning` 表示 `off`。 -受支持的 profile 字段是 `apiKeyEnv`、`displayName`、`api`、`baseURL`、`models`、`modelOverrides`、`compat`、`defaultContextWindow`、`defaultMaxTokens`、`defaultInput`、`headers`、`reasoning`、`thinkingBudgets`、`cacheRetention`、`transport`、`timeoutMs`、`websocketConnectTimeoutMs`、`streamIdleTimeoutMs`、`maxRequestImageBytes`、`requestImagePixelBudget`、`requestImageMaxBytes` 和 `retryPolicy`。每条 profile 解析后的重试策略会随该提供方路由一同捕获;省略时使用共享的有界 normal 默认值并重试五次。流空闲间隔必须是正的有限 Node 定时器延迟,默认为五分钟,且只覆盖未完成提供方读取,不包括消费方思考时间。每条图片路由从提供方无关的主版本派生确定性请求版本,受 `requestImagePixelBudget`(默认总像素 2048×2048)和 `requestImageMaxBytes`(默认原始字节 1MiB)约束。读取主版本前,`maxRequestImageBytes` 先按请求版本的保守上界替换超预算的最旧图片;保留版本生成后再用确切 base64 长度检查。20MiB 默认值可保留十五个按 1MiB 上限生成的请求版本,并为请求正文留下余量。同一版本用于内联 base64,其稳定描述会公开附件 ID 和实际请求图片尺寸。若已配置标头中有同名项,则以 Harness 应用归因为准。 +受支持的 profile 字段是 `apiKeyEnv`、`displayName`、`api`、`baseURL`、`models`、`modelOverrides`、`compat`、`defaultContextWindow`、`defaultMaxTokens`、`defaultInput`、`headers`、`reasoning`、`thinkingBudgets`、`cacheRetention`、`transport`、`timeoutMs`、`websocketConnectTimeoutMs`、`streamIdleTimeoutMs`、`maxRequestImageBytes`、`requestImagePixelBudget`、`requestImageMaxBytes` 和 `retryPolicy`。每条 profile 解析后的重试策略会随该提供方路由一同捕获;省略时使用共享的有界 normal 默认值并重试五次。流空闲间隔必须是正的有限 Node 定时器延迟,默认为五分钟,且只覆盖未完成提供方读取,不包括消费方思考时间。每条图片路由从提供方无关的规范化附件派生确定性请求版本,受 `requestImagePixelBudget`(默认总像素 2048×2048)和 `requestImageMaxBytes`(默认原始字节 1MiB)约束。读取附件前,`maxRequestImageBytes` 先按请求版本的保守上界替换超预算的最旧图片;保留版本生成后再用确切 base64 长度检查。20MiB 默认值可保留十五个按 1MiB 上限生成的请求版本,并为请求正文留下余量。同一版本用于内联 base64,其稳定描述会公开附件 ID 和实际请求图片尺寸。若已配置标头中有同名项,则以 Harness 应用归因为准。 适配器强制 pi-ai SDK `maxRetries` 为零,因此一次 `stream()` 调用只会发起一次提供方请求。已移除 profile 字段 `maxRetries` 和 `maxRetryDelayMs` 会使加载失败,而不是静默倍增或隐藏单独组合的 agent(智能体)级重试预算。空闲超时会 abort SDK 的稳定请求信号,并以 `TIMEOUT` 呈现;较早的调用方 abort 仍为 `ABORTED`。 @@ -174,7 +174,7 @@ pi-ai 会安装多个提供方 SDK,并延迟加载 catalog 模型所选的 SDK #### 模型看到的内容 -所选 catalog 模型会收到 `GenerateOptions.system`、历史、工具,以及 pi-ai 通用流式 API 支持的采样字段。每张保留图片前都有稳定文本,写明完整附件 ID 和实际请求尺寸。请求累积的 base64 图片载荷超过路由的 `maxRequestImageBytes` 时,被 offload 的图片会从最老开始替换为固定文本,要求模型在有路径时重新读取文件,否则请用户重新附上图片。系统不会读取或转换被 offload 的主版本。只有当适配器验证提供方原生回放元数据与历史内容匹配时,才会恢复这些元数据。 +所选 catalog 模型会收到 `GenerateOptions.system`、历史、工具,以及 pi-ai 通用流式 API 支持的采样字段。每张保留图片前都有稳定文本,写明完整附件 ID 和实际请求尺寸。请求累积的 base64 图片载荷超过路由的 `maxRequestImageBytes` 时,被 offload 的图片会从最老开始替换为固定文本,要求模型在有路径时重新读取文件,否则请用户重新附上图片。系统不会读取或转换被 offload 的规范化附件。只有当适配器验证提供方原生回放元数据与历史内容匹配时,才会恢复这些元数据。 #### Token 影响 diff --git a/packages/llm/llm-pi-ai/src/config.ts b/packages/llm/llm-pi-ai/src/config.ts index 5473d931de..5fdcc2cdb2 100644 --- a/packages/llm/llm-pi-ai/src/config.ts +++ b/packages/llm/llm-pi-ai/src/config.ts @@ -52,7 +52,7 @@ export const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 300_000 * Deployments behind stricter gateways lower it per route. */ export const DEFAULT_MAX_REQUEST_IMAGE_BYTES = 20 * 1024 * 1024 -/** Default total-pixel budget preserves the complete 2048px local master. */ +/** Default total-pixel budget preserves the complete 2048px normalized attachment. */ export const DEFAULT_REQUEST_IMAGE_PIXEL_BUDGET = 2048 * 2048 /** Default raw encoded-byte cap before inline base64 expansion. */ export const DEFAULT_REQUEST_IMAGE_MAX_BYTES = 1024 * 1024 diff --git a/packages/llm/llm-pi-ai/src/context.ts b/packages/llm/llm-pi-ai/src/context.ts index 5d2df24d18..9faf457c9a 100644 --- a/packages/llm/llm-pi-ai/src/context.ts +++ b/packages/llm/llm-pi-ai/src/context.ts @@ -56,10 +56,7 @@ async function userContent( if (block.text.length > 0) content.push({ type: 'text', text: block.text }) break case 'image': { - const version = requestImages.get(block.attachment.attachmentId) - if (version === undefined) { - throw new LlmError(`pi-ai request image ${block.attachment.attachmentId} was not prepared`, 'INVALID_REQUEST') - } + const version = requestImages.get(block.attachment.attachmentId) as RequestImageAttachment content.push({ type: 'text', text: requestImageHandleText(version) }) content.push({ type: 'image', @@ -106,7 +103,9 @@ async function prepareRequestImages( const refs = new Map() for (const message of messages) collectImageRefs(message.content, refs) const orderedRefs = [...refs.values()] - const prepared = await attachments.readImageRequests(orderedRefs, policy, signal) + const prepared = await Promise.all(orderedRefs.map( + ref => attachments.readImageRequest(ref, policy, signal), + )) const versions = new Map() for (const [index, ref] of orderedRefs.entries()) { versions.set(ref.attachmentId, prepared[index] as RequestImageAttachment) @@ -238,7 +237,7 @@ async function toPiContextWithImages( representation: 'base64', ...maxRequestImageBytes === undefined ? {} : { maxBytes: maxRequestImageBytes }, byteQuantum: 1, - byteLength: ref => requestImages.get(ref.attachmentId)?.bytes ?? ref.bytes, + byteLength: ref => (requestImages.get(ref.attachmentId) as RequestImageAttachment).bytes, }) const toolNames = new Map() const messages: PiMessage[] = [] diff --git a/packages/llm/llm-pi-ai/tests/adapter.spec.ts b/packages/llm/llm-pi-ai/tests/adapter.spec.ts index 09befeaa4b..e2ed0233d9 100644 --- a/packages/llm/llm-pi-ai/tests/adapter.spec.ts +++ b/packages/llm/llm-pi-ai/tests/adapter.spec.ts @@ -6,7 +6,6 @@ import type { ImageAttachmentRef, ImageRequestPolicy, RequestImageAttachment, - SavedImageAttachment, SaveImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' @@ -246,7 +245,7 @@ describe('PiAiAdapter provider routing', () => { ): Promise => ( Promise.resolve({ variantId: ImageVariantId(`sha256:${'b'.repeat(64)}`), - master: value, + attachment: value, data: Uint8Array.of(1), mediaType: value.mediaType, bytes: 1, @@ -272,7 +271,7 @@ describe('PiAiAdapter provider routing', () => { return Promise.reject(new Error('not used')) } - saveImage(_input: SaveImageAttachment): Promise { + saveImage(_input: SaveImageAttachment): Promise { return Promise.reject(new Error('not used')) } diff --git a/packages/llm/llm-pi-ai/tests/context.spec.ts b/packages/llm/llm-pi-ai/tests/context.spec.ts index da1dcaf28b..026ca2c608 100644 --- a/packages/llm/llm-pi-ai/tests/context.spec.ts +++ b/packages/llm/llm-pi-ai/tests/context.spec.ts @@ -22,7 +22,7 @@ const ref: ImageAttachmentRef = { function requestImage(value: ImageAttachmentRef, data: Uint8Array): RequestImageAttachment { return { variantId: ImageVariantId(`sha256:${'b'.repeat(64)}`), - master: value, + attachment: value, data, mediaType: value.mediaType, bytes: data.byteLength, @@ -43,14 +43,7 @@ function projectionStore( Promise.resolve(requestImage(value, Uint8Array.of(1))) )), ): AttachmentStore { - return { - readImageRequest, - readImageRequests: ( - refs: readonly ImageAttachmentRef[], - policy: Parameters[1], - signal?: AbortSignal, - ) => Promise.all(refs.map(value => readImageRequest(value, policy, signal))), - } as unknown as AttachmentStore + return { readImageRequest } as unknown as AttachmentStore } const attachments = projectionStore() @@ -418,13 +411,4 @@ describe('pi-ai request context conversion', () => { )).toThrow(/assistant image output/) }) - it('rejects an attachment service that omits a requested image version', async () => { - const store = { - readImageRequests: vi.fn(() => Promise.resolve([])), - } as unknown as AttachmentStore - await expect(toPiContext( - request([user([{ type: 'image', attachment: ref }])]), - store, - )).rejects.toMatchObject({ code: 'INVALID_REQUEST' }) - }) }) diff --git a/packages/llm/llm-pi-ai/tests/convert.spec.ts b/packages/llm/llm-pi-ai/tests/convert.spec.ts index ccfb321d3b..c540f2d50b 100644 --- a/packages/llm/llm-pi-ai/tests/convert.spec.ts +++ b/packages/llm/llm-pi-ai/tests/convert.spec.ts @@ -46,7 +46,7 @@ async function collect(stream: AsyncIterable): Promise Promise): AttachmentStore { - return { - readImageRequest, - readImageRequests: ( - refs: readonly ImageAttachmentRef[], - policy: ImageRequestPolicy, - signal?: AbortSignal, - ) => Promise.all( - refs.map(ref => readImageRequest(ref, policy, signal)), - ), - } as unknown as AttachmentStore + return { readImageRequest } as unknown as AttachmentStore } describe('toPiContext', () => { diff --git a/packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts b/packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts index 93732e75d3..f651aa1f9a 100644 --- a/packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts +++ b/packages/llm/llm-pi-ai/tests/provider-apis.e2e.ts @@ -7,7 +7,6 @@ import type { ImageAttachmentRef, ImageRequestPolicy, RequestImageAttachment, - SavedImageAttachment, SaveImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' @@ -81,7 +80,7 @@ async function harness(image?: StoredImageAttachment): Promise { return Promise.reject(new Error('e2e attachment fixture is read-only')) } - saveImage(_input: SaveImageAttachment): Promise { + saveImage(_input: SaveImageAttachment): Promise { return Promise.reject(new Error('e2e attachment fixture is read-only')) } @@ -98,7 +97,7 @@ async function harness(image?: StoredImageAttachment): Promise { } return Promise.resolve({ variantId: ImageVariantId(`sha256:${'f'.repeat(64)}`), - master: fixture.ref, + attachment: fixture.ref, data: fixture.data, mediaType: fixture.ref.mediaType, bytes: fixture.data.byteLength, diff --git a/packages/llm/llm/src/content.ts b/packages/llm/llm/src/content.ts index c30a62dccb..4620275429 100644 --- a/packages/llm/llm/src/content.ts +++ b/packages/llm/llm/src/content.ts @@ -24,7 +24,7 @@ export function textOnlyImageText(ref: ImageAttachmentRef): string { * @returns attachment handle and request-image dimensions. */ export function requestImageHandleText(version: RequestImageAttachment): string { - return `Image ${version.master.attachmentId}; request image ${version.width}x${version.height}px.` + return `Image ${version.attachment.attachmentId}; request image ${version.width}x${version.height}px.` } /** diff --git a/packages/mcp/mcp-client/tests/mcp-client.spec.ts b/packages/mcp/mcp-client/tests/mcp-client.spec.ts index 9f4854e2d8..b72b8faad4 100644 --- a/packages/mcp/mcp-client/tests/mcp-client.spec.ts +++ b/packages/mcp/mcp-client/tests/mcp-client.spec.ts @@ -3,7 +3,7 @@ import { Client } from '@modelcontextprotocol/sdk/client/index.js' import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js' import { Context } from '@deepseek-ai/cordis' import AttachmentStore, { AttachmentError, AttachmentId } from '@deepseek-ai/dsh-attachment' -import type { ImageAttachmentLimits, ImageAttachmentRef, SavedImageAttachment, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' +import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' import { CallId, LlmAdapter, LlmRuntime } from '@deepseek-ai/dsh-llm' import type { ContentBlock } from '@deepseek-ai/dsh-llm' import type { GenerateOptions, LlmResolvedModelInfo, StreamChunk } from '@deepseek-ai/dsh-llm' @@ -86,7 +86,7 @@ class RecordingAttachmentStore extends AttachmentStore { return Promise.resolve() } - saveImage(input: SaveImageAttachment): Promise { + saveImage(input: SaveImageAttachment): Promise { this.saved.push(input) const marker = input.data[0] ?? 0 const ref: ImageAttachmentRef = { @@ -96,10 +96,7 @@ class RecordingAttachmentStore extends AttachmentStore { width: 1, height: 1, } - return Promise.resolve({ - ref, - source: { mediaType: ref.mediaType, bytes: ref.bytes, width: ref.width, height: ref.height }, - }) + return Promise.resolve(ref) } readImage(_ref: ImageAttachmentRef): Promise { diff --git a/packages/plan/plan-mode/tests/plan-mode.spec.ts b/packages/plan/plan-mode/tests/plan-mode.spec.ts index d1b4be3058..8285147953 100644 --- a/packages/plan/plan-mode/tests/plan-mode.spec.ts +++ b/packages/plan/plan-mode/tests/plan-mode.spec.ts @@ -653,8 +653,7 @@ describe('/plan', () => { const saveImage = (input: { mediaType: string }) => { saved += 1 return Promise.resolve({ - ref: { attachmentId: `att-${saved}`, mediaType: input.mediaType, bytes: 3, width: 1, height: 1 }, - source: { mediaType: input.mediaType, bytes: 3, width: 1, height: 1 }, + attachmentId: `att-${saved}`, mediaType: input.mediaType, bytes: 3, width: 1, height: 1, }) } ctx.provide('attachments', { @@ -666,7 +665,7 @@ describe('/plan', () => { saveImage, async saveImages(inputs: readonly { mediaType: string }[]) { const refs = [] - for (const input of inputs) refs.push((await saveImage(input)).ref) + for (const input of inputs) refs.push(await saveImage(input)) return refs }, }) diff --git a/scripts/gen-cordis-catalog.ts b/scripts/gen-cordis-catalog.ts index a5ed9feff9..522e98df96 100644 --- a/scripts/gen-cordis-catalog.ts +++ b/scripts/gen-cordis-catalog.ts @@ -296,8 +296,6 @@ export const LINK_MAP: Readonly> = { ImageRequestPolicy: 'attachment.md', RequestImageAttachment: 'attachment.md', SaveImageAttachment: 'attachment.md', - SavedImageAttachment: 'attachment.md', - SourceImageInfo: 'attachment.md', StoredImageAttachment: 'attachment.md', ShellExecRequest: 'shell.md', ShellExecSpec: 'shell.md', diff --git a/scripts/gen-tool-catalog.ts b/scripts/gen-tool-catalog.ts index 805316352b..874f564483 100644 --- a/scripts/gen-tool-catalog.ts +++ b/scripts/gen-tool-catalog.ts @@ -25,7 +25,7 @@ import { PwshLocalExecutor } from '@deepseek-ai/dsh-pwsh-local' import LocalSubprocessRuntime from '@deepseek-ai/dsh-subprocess-local' import LocalFileSystem from '@deepseek-ai/dsh-fs-local' import { AttachmentStore } from '@deepseek-ai/dsh-attachment' -import type { ImageAttachmentLimits, ImageAttachmentRef, SavedImageAttachment, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' +import type { ImageAttachmentLimits, ImageAttachmentRef, SaveImageAttachment, StoredImageAttachment } from '@deepseek-ai/dsh-attachment' import UserQuestionService from '@deepseek-ai/dsh-user-questions' import PlanModeController from '@deepseek-ai/dsh-plan-mode' import WebRuntime from '@deepseek-ai/dsh-web' @@ -83,7 +83,7 @@ class CatalogAttachmentStore extends AttachmentStore { return Promise.reject(new Error('gen-tool-catalog: attachment validation is unreachable during schema harvest')) } - override saveImage(_input: SaveImageAttachment): Promise { + override saveImage(_input: SaveImageAttachment): Promise { return Promise.reject(new Error('gen-tool-catalog: attachment writes are unreachable during schema harvest')) } diff --git a/scripts/test-invariants.ts b/scripts/test-invariants.ts index 4f57edc2f8..a3b96a90a3 100644 --- a/scripts/test-invariants.ts +++ b/scripts/test-invariants.ts @@ -12,7 +12,6 @@ import { AttachmentStore } from '@deepseek-ai/dsh-attachment' import type { ImageAttachmentLimits, ImageAttachmentRef, - SavedImageAttachment, SaveImageAttachment, StoredImageAttachment, } from '@deepseek-ai/dsh-attachment' @@ -126,7 +125,7 @@ class TestAttachmentStore extends AttachmentStore { return Promise.reject(new Error('test invariant attachment store does not validate images')) } - saveImage(_input: SaveImageAttachment): Promise { + saveImage(_input: SaveImageAttachment): Promise { return Promise.reject(new Error('test invariant attachment store does not save images')) } From cbc830adeddf67c707168041cf9cdbb21e9b55a0 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Fri, 21 Aug 2026 13:35:20 +0800 Subject: [PATCH 23/28] test(composition): remove retired image-region tool --- apps/cli/tests/web-agent-presets.e2e.ts | 2 +- apps/web/tests/shipped-composition.e2e.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/cli/tests/web-agent-presets.e2e.ts b/apps/cli/tests/web-agent-presets.e2e.ts index 976381096a..0e98af0477 100644 --- a/apps/cli/tests/web-agent-presets.e2e.ts +++ b/apps/cli/tests/web-agent-presets.e2e.ts @@ -237,7 +237,7 @@ describe('the shipped Web composition', () => { // depend on ripgrep being present on the machine. expect(toolNames(ctx, handle.agent).filter(name => name !== 'glob' && name !== 'grep')).toEqual([ 'ask_user_question', 'bash', 'create_goal', 'edit', 'exit_plan_mode', - 'get_goal', 'interrupt_agent', 'job_kill', 'job_list', 'job_output', 'list_agents', 'ralph', 'read', 'read_image', 'read_image_region', 'send_message', 'skill', + 'get_goal', 'interrupt_agent', 'job_kill', 'job_list', 'job_output', 'list_agents', 'ralph', 'read', 'read_image', 'send_message', 'skill', 'subagent', 'subagent_fork', 'todo_write', 'update_goal', 'web_search', 'workflow', 'write', ]) diff --git a/apps/web/tests/shipped-composition.e2e.ts b/apps/web/tests/shipped-composition.e2e.ts index cca21dcef6..295e861b95 100644 --- a/apps/web/tests/shipped-composition.e2e.ts +++ b/apps/web/tests/shipped-composition.e2e.ts @@ -48,7 +48,6 @@ const EXPECTED_TOOLS = [ 'ralph', 'read', 'read_image', - 'read_image_region', 'send_message', 'skill', 'subagent', From 6a27286e440d96d30916dd77bffa133e00f0d665 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Fri, 21 Aug 2026 15:09:14 +0800 Subject: [PATCH 24/28] docs(i18n): fix rebased image note links --- ...b-multimodal-image-input-and-durable-attachments.i18n.yaml | 2 +- ...2-web-multimodal-image-input-and-durable-attachments.zh.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.i18n.yaml b/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.i18n.yaml index be716462f5..0702b19390 100644 --- a/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.i18n.yaml @@ -3,4 +3,4 @@ # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.md 2026-07-22-web-multimodal-image-input-and-durable-attachments.md: 30ac1dcff9e6400a3bcf58f7b8e5237e20bd5c04 -2026-07-22-web-multimodal-image-input-and-durable-attachments.zh.md: 68c370c3dd2234e67717429bed417755ed20305d +2026-07-22-web-multimodal-image-input-and-durable-attachments.zh.md: 359bb9048632222518d87aadd348bca217c8f7c4 diff --git a/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.zh.md b/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.zh.md index 68c370c3dd..359bb90486 100644 --- a/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.zh.md +++ b/.agents/notes/implemented/feature/2026-07-22-web-multimodal-image-input-and-durable-attachments.zh.md @@ -69,7 +69,7 @@ interface ComposerAttachment { 这一拆分把会话 provide 通道的输入 hook 与 actions 用作实时输入区状态的唯一订阅路径,同时避免把不可序列化的浏览器对象写进持久 JSON。只有纯文本草稿镜像使用 `localStorage`;附件标识符、浏览器 `File` 对象和对象 URL 都限定在实时会话输入外壳的 scope 内。未发送图片因此无法跨重载或会话 scope 释放保留。切换 Workspace 时,只有目标外壳接受完整图片批次,图文混合草稿才会移动;拒绝时,文本和图片都留在来源外壳。原生客户端可以在操作系统临时目录中暂存输入,但必须像对待浏览器对象 URL 一样对待该路径:不再需要时删除,并在消息被接受前把字节复制进持久存储。 -本地附件后端依次解析显式 `dshHome`、`$DSH_HOME` 和 `~/.dsh`。它把内容寻址对象存储在 `$DSH_HOME/attachments/v1/objects//` 下,并为目录和文件设置仅所有者可访问的权限。每个进程首次为某个 home 保存对象时,都会创建该 home,并逐级同步每个祖先目录项直至文件系统根目录;不能把存在视为持久性,因为另一个进程可能仍处于 `mkdir` 与父目录 `fsync` 之间。随后,服务写入并同步临时文件,再以原子方式发布,并对发布路径执行目录同步使其持久(POSIX;Windows 依赖文件系统元数据日志),之后才返回引用。内容摘要编码在不透明的 `sha256:` 标识符中。准入会应用方向、删除元数据、转换为 8-bit sRGB/sRGBA,并在独立尺寸和字节上限内保持宽高比,生成与提供方无关的主版本。读取会校验摘要、字节长度和已记录元数据。路由专用的确定性请求版本单独缓存,完整策略见[统一图片主版本、请求版本和提供方文件](2026-08-20-unified-image-request-pipeline.md)。 +本地附件后端依次解析显式 `dshHome`、`$DSH_HOME` 和 `~/.dsh`。它把内容寻址对象存储在 `$DSH_HOME/attachments/v1/objects//` 下,并为目录和文件设置仅所有者可访问的权限。每个进程首次为某个 home 保存对象时,都会创建该 home,并逐级同步每个祖先目录项直至文件系统根目录;不能把存在视为持久性,因为另一个进程可能仍处于 `mkdir` 与父目录 `fsync` 之间。随后,服务写入并同步临时文件,再以原子方式发布,并对发布路径执行目录同步使其持久(POSIX;Windows 依赖文件系统元数据日志),之后才返回引用。内容摘要编码在不透明的 `sha256:` 标识符中。准入会应用方向、删除元数据、转换为 8-bit sRGB/sRGBA,并在独立尺寸和字节上限内保持宽高比,生成与提供方无关的主版本。读取会校验摘要、字节长度和已记录元数据。路由专用的确定性请求版本单独缓存,完整策略见[统一图片主版本、请求版本和提供方文件](2026-08-20-unified-image-request-pipeline.zh.md)。 第一版不对存储执行自动删除。已发送的用户图片和模型生成图片会一直保留,以供历史记录、恢复和 fork 使用。按引用感知的垃圾回收需要单独设计,因为仅按时间清理可能删除仍被持久会话引用的数据。部署的字节和像素限制是写入时的准入策略;读取时会校验摘要和已记录的元数据,但不重新应用当前准入限制,因此收紧策略不会导致旧历史记录失效。 @@ -122,7 +122,7 @@ Base64 只跨越一次协议边界,并在持久化后丢弃。每个入口都 模型目录项增加可选且可合并扩展的输入模态声明。缺少声明表示未知;声明存在但不含 `image`,则明确表示不支持图片。 -宿主是权威的前置检查点。它会解析会话最新路由到的提供方和模型,并在缺失时依次回退到 agent 选项和宿主默认值;如果模型明确排除图片输入,宿主会在写入附件或事件前拒绝新的图片提示词,客户端则恢复草稿。包含图片的提示词准入与模型选择共用一条逐 agent 串行链([顺序决策](../bug-fix/2026-07-29-atomic-web-image-admission.md)),也包括不进入排队 UI 镜像的 steering。这会为提示词和并发选择提供确定顺序。图片进入持久历史后仍可选择纯文本模型;共享 LLM 运行时会在该请求中把保留的图片块替换为确定的文本占位符。`session.updateQueue` 只接受文本内容,因此队列编辑无法绕过准入注入图片。能力未知时继续进入适配器强制检查,使未收录的模型标识符仍然可用。浏览器会在分配预览 URL 前拒绝声明不支持的图片媒体类型,但不会为部署限制或模型能力保留快照。宿主会根据当前的单张字节数、图片数量、总字节数、媒体类型、尺寸、像素数和路由模型策略校验整个批次,再写入附件或事件;拒绝会通过 composer 的短时 toast 显示。 +宿主是权威的前置检查点。它会解析会话最新路由到的提供方和模型,并在缺失时依次回退到 agent 选项和宿主默认值;如果模型明确排除图片输入,宿主会在写入附件或事件前拒绝新的图片提示词,客户端则恢复草稿。包含图片的提示词准入与模型选择共用一条逐 agent 串行链([顺序决策](../bug-fix/2026-07-29-atomic-web-image-admission.zh.md)),也包括不进入排队 UI 镜像的 steering。这会为提示词和并发选择提供确定顺序。图片进入持久历史后仍可选择纯文本模型;共享 LLM 运行时会在该请求中把保留的图片块替换为确定的文本占位符。`session.updateQueue` 只接受文本内容,因此队列编辑无法绕过准入注入图片。能力未知时继续进入适配器强制检查,使未收录的模型标识符仍然可用。浏览器会在分配预览 URL 前拒绝声明不支持的图片媒体类型,但不会为部署限制或模型能力保留快照。宿主会根据当前的单张字节数、图片数量、总字节数、媒体类型、尺寸、像素数和路由模型策略校验整个批次,再写入附件或事件;拒绝会通过 composer 的短时 toast 显示。 Pi-AI 与直接 DeepSeek 适配器都会在请求时解析 `ctx.attachments`,递归转换每个保留的图片引用,包括嵌套在工具结果中的引用,并且仅为声明支持图片输入的模型生成提供方原生图片内容。两个适配器都从持久主版本请求同一个确定性路由版本。Pi-AI 在考虑 base64 扩张的请求预算内内联携带它。内置 DeepSeek 路由公布 `deepseek-v4-flash-vision-exp`,把每个保留的版本上传到 Files API,并通过索引复用、过期处理、有界陈旧 ID 重试、配额清理和显式删除发送 `file_id` 块。DeepSeek 纯文本模型、未声明图片能力的自定义模型和未列出的透传 ID 保持纯文本。在请求时解析服务,可避免 Cordis 加载顺序将可选附件服务的可用性固化。适配器不得展平或静默跳过保留图片;不支持的角色与模型会以类型化的 `UNSUPPORTED_CONTENT` 失败。 From 6816cc0b04b95a874d2686fa2fd390c38828a737 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Fri, 21 Aug 2026 15:25:27 +0800 Subject: [PATCH 25/28] test(snapshot): stabilize persisted-turn coverage --- .../test-support/acp-snapshot/tests/harness.spec.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/test-support/acp-snapshot/tests/harness.spec.ts b/packages/test-support/acp-snapshot/tests/harness.spec.ts index 5bd97b101d..005456e7c0 100644 --- a/packages/test-support/acp-snapshot/tests/harness.spec.ts +++ b/packages/test-support/acp-snapshot/tests/harness.spec.ts @@ -705,9 +705,9 @@ describe('runScenario', () => { it('waitForTurnStart rejects missing, earlier, and malformed durable turns', { timeout: 20_000 }, async () => { const missing = await scenario({}) await expect(runScenario( - { steps: [...boot, { op: 'waitForTurnStart', timeoutMs: 20 }] }, + { steps: [...boot, { op: 'waitForTurnStart', timeoutMs: 200 }] }, { agent: AGENT, mode: 'replay', fixtureFile: missing.fixtureFile }, - )).rejects.toThrow(/did not persist turn\/start within 20ms/) + )).rejects.toThrow(/did not persist turn\/start within 200ms/) const earlier = await scenario({ prompt: 'hang-until-cancel', @@ -725,11 +725,11 @@ describe('runScenario', () => { steps: [ ...boot, { op: 'promptAndCancel', text: 'hang' }, - { op: 'waitForTurnStart', minimumTurn: 3, timeoutMs: 20 }, + { op: 'waitForTurnStart', minimumTurn: 3, timeoutMs: 200 }, ], }, { agent: AGENT, mode: 'replay', fixtureFile: earlier.fixtureFile }, - )).rejects.toThrow(/turn\/start at or beyond turn 3 within 20ms/) + )).rejects.toThrow(/turn\/start at or beyond turn 3 within 200ms/) const closed = await scenario({ prompt: 'hang-until-cancel', @@ -748,11 +748,11 @@ describe('runScenario', () => { steps: [ ...boot, { op: 'promptAndCancel', text: 'hang' }, - { op: 'waitForTurnStart', timeoutMs: 20 }, + { op: 'waitForTurnStart', timeoutMs: 200 }, ], }, { agent: AGENT, mode: 'replay', fixtureFile: closed.fixtureFile }, - )).rejects.toThrow(/did not persist turn\/start within 20ms/) + )).rejects.toThrow(/did not persist turn\/start within 200ms/) for (const turn of [undefined, 0]) { const malformed = await scenario({ From e30d92a03e990ad4f92863b72061a363af0269b4 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Fri, 21 Aug 2026 17:27:08 +0800 Subject: [PATCH 26/28] fix(attachment): accept opaque WebP alpha omission --- .../attachment-local/README.i18n.yaml | 4 +-- .../attachment/attachment-local/README.md | 2 +- .../attachment/attachment-local/README.zh.md | 2 +- .../attachment/attachment-local/src/image.ts | 18 +++++++++++++ .../attachment-local/src/normalization.ts | 4 +-- .../attachment-local/src/request-image.ts | 6 ++--- .../tests/normalization.spec.ts | 23 +++++++++++----- .../tests/request-image.spec.ts | 26 +++++++++++++++---- 8 files changed, 65 insertions(+), 20 deletions(-) diff --git a/packages/attachment/attachment-local/README.i18n.yaml b/packages/attachment/attachment-local/README.i18n.yaml index 412e9a4cb6..3698abdcb2 100644 --- a/packages/attachment/attachment-local/README.i18n.yaml +++ b/packages/attachment/attachment-local/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/attachment/attachment-local/README.md -README.md: 849363ce53c6186359ecad34aecb1c2a48f07441 -README.zh.md: f0fe90c2569f60df48998e46d5b05a0d024959df +README.md: 3ed4ab3251b0a609807c76930226bec63f0164cd +README.zh.md: 85abd10389acc46c2d89dd85628f5d201b089710 diff --git a/packages/attachment/attachment-local/README.md b/packages/attachment/attachment-local/README.md index 849363ce53..3ed4ab3251 100644 --- a/packages/attachment/attachment-local/README.md +++ b/packages/attachment/attachment-local/README.md @@ -4,7 +4,7 @@ English | [中文](README.zh.md) The private local implementation of [`@deepseek-ai/dsh-attachment`](../attachment). Objects land at `/attachments/v1/objects//` and are addressed by an opaque `sha256:` id. Each process proves a home durable once by syncing every ancestor entry to the filesystem root. Writes use a private staging directory, owner-only files, a synced temporary file, an atomic exclusive hard-link publish, and directory syncs on the publication path (POSIX; Windows relies on filesystem metadata journaling) so the reported reference survives a crash. -Admission accepts at most 20 images and 200MiB of encoded source bytes per message. Each source may use up to 20MiB, 64,000,000 pixels, and 8192px per side. It then prepares a provider-independent normalized attachment. EXIF orientation is applied, metadata and color profiles are removed, pixels become 8-bit sRGB/sRGBA, and the long edge is reduced proportionally to `normalizedImageMaxDimension` (2048px by default). The normalized attachment has its own `normalizedImageMaxBytes` safety cap (4MiB by default). Alpha is retained. A nearest-neighbour bounded sample classifies color complexity without averaging high-frequency pixels. Confirmed low-color images try PNG, using a palette only when the input has no alpha channel, then WebP at qualities 85, 80, and 75. Other alpha images try WebP at those qualities; other opaque images try JPEG. Each candidate runs only after the preceding candidate exceeds the cap. Dimensions shrink only after every candidate at one size exceeds the cap. A clean, single-frame 8-bit sRGB/sRGBA PNG, JPEG, or WebP already within both normalization limits passes through byte-identically; 16-bit PNG, GIF, animated input, metadata, orientation, and incompatible color spaces force conversion. The source and converted attachment are each fully decoded once. `saveImages` prepares and verifies every normalized attachment once before publishing the batch, so validation failure leaves no partial references and commit does not repeat full image encoding. +Admission accepts at most 20 images and 200MiB of encoded source bytes per message. Each source may use up to 20MiB, 64,000,000 pixels, and 8192px per side. It then prepares a provider-independent normalized attachment. EXIF orientation is applied, metadata and color profiles are removed, pixels become 8-bit sRGB/sRGBA, and the long edge is reduced proportionally to `normalizedImageMaxDimension` (2048px by default). The normalized attachment has its own `normalizedImageMaxBytes` safety cap (4MiB by default). Transparent pixels are retained; Sharp/libvips may omit an alpha plane whose samples are all opaque. A nearest-neighbour bounded sample classifies color complexity without averaging high-frequency pixels. Confirmed low-color images try PNG, using a palette only when the input has no alpha channel, then WebP at qualities 85, 80, and 75. Other alpha images try WebP at those qualities; other opaque images try JPEG. Each candidate runs only after the preceding candidate exceeds the cap. Dimensions shrink only after every candidate at one size exceeds the cap. A clean, single-frame 8-bit sRGB/sRGBA PNG, JPEG, or WebP already within both normalization limits passes through byte-identically; 16-bit PNG, GIF, animated input, metadata, orientation, and incompatible color spaces force conversion. The source and converted attachment are each fully decoded once. `saveImages` prepares and verifies every normalized attachment once before publishing the batch, so validation failure leaves no partial references and commit does not repeat full image encoding. Request versions live below `/attachments/v1/request-images/`. `readImageRequest` scales the stored normalized attachment under a total-pixel budget without enlargement, then enforces a separate encoded-byte cap. The request encoder uses the same color branches, with PNG (palette only without alpha) before WebP 85 and 80 for low-color images, WebP 85 then 80 for other alpha images, and JPEG 85 then 80 for other opaque images. It executes candidates lazily and reduces dimensions only after both quality attempts exceed the request cap. Its cache identity includes the attachment id, transform version, pixel and byte budgets, and fixed encoder settings. Cached bytes are fully decoded and checked as 8-bit sRGB/sRGBA before use. Concurrent calls for one identity share one transform and cache write; cancelling one waiter does not cancel the shared work. Callers compose ordered batches from singular reads, while the service's FIFO limiter applies `imageCompressionConcurrency` to simultaneous normalization and request transforms. The setting ranges from 1 through 8 and defaults to 2; file publication remains ordered after preparation. diff --git a/packages/attachment/attachment-local/README.zh.md b/packages/attachment/attachment-local/README.zh.md index f0fe90c256..85abd10389 100644 --- a/packages/attachment/attachment-local/README.zh.md +++ b/packages/attachment/attachment-local/README.zh.md @@ -4,7 +4,7 @@ 这是 [`@deepseek-ai/dsh-attachment`](../attachment) 的私有本地实现。对象存放在 `/attachments/v1/objects//`,并通过不透明的 `sha256:` 标识符寻址。每个进程都会把每级祖先目录项同步到文件系统根目录,以此一次性证明 home 已持久化。写入使用私有暂存目录、仅所有者可访问的文件、经过同步的临时文件、原子且排他的硬链接发布,并对发布路径执行目录同步(适用于 POSIX;Windows 依赖文件系统元数据日志),确保已报告的引用能够在崩溃后继续存在。 -每条消息最多准入 20 张图片,源图编码字节总量不超过 200MiB。每张源图不得超过 20MiB、64,000,000 像素和单边 8192px。随后生成提供方无关的规范化附件:应用 EXIF 方向,删除元数据和色彩配置文件,转换为 8-bit sRGB/sRGBA,并保持宽高比把长边限制到 `normalizedImageMaxDimension`(默认 2048px)。规范化附件有独立的 `normalizedImageMaxBytes` 安全上限(默认 4MiB)。透明通道会保留。系统用 nearest-neighbour 对有界样本分类,不会通过像素平均把高频图片误判为低色数。确认的低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80、75 的 WebP;其他透明图片依次尝试这些质量的 WebP;其他非透明图片依次尝试这些质量的 JPEG。只有前一个候选超限时才会执行下一个候选;同一尺寸的候选全部超限后才缩小尺寸。已经处于两个规范化上限内的干净、单帧、8-bit sRGB/sRGBA PNG、JPEG 或 WebP 按字节原样直通;16-bit PNG、GIF、动图、元数据、方向和不兼容色彩空间都会触发转换。源图和转换后的附件各完整解码一次。`saveImages` 在发布任何批次成员前为每张图片各准备并验证一次规范化附件,因此校验失败不会留下部分引用,提交阶段也不会重复执行完整图片编码。 +每条消息最多准入 20 张图片,源图编码字节总量不超过 200MiB。每张源图不得超过 20MiB、64,000,000 像素和单边 8192px。随后生成提供方无关的规范化附件:应用 EXIF 方向,删除元数据和色彩配置文件,转换为 8-bit sRGB/sRGBA,并保持宽高比把长边限制到 `normalizedImageMaxDimension`(默认 2048px)。规范化附件有独立的 `normalizedImageMaxBytes` 安全上限(默认 4MiB)。透明像素会保留;当所有 alpha 样本均为不透明时,Sharp/libvips 可能省略没有实际作用的 alpha 平面。系统用 nearest-neighbour 对有界样本分类,不会通过像素平均把高频图片误判为低色数。确认的低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,随后依次尝试质量 85、80、75 的 WebP;其他透明图片依次尝试这些质量的 WebP;其他非透明图片依次尝试这些质量的 JPEG。只有前一个候选超限时才会执行下一个候选;同一尺寸的候选全部超限后才缩小尺寸。已经处于两个规范化上限内的干净、单帧、8-bit sRGB/sRGBA PNG、JPEG 或 WebP 按字节原样直通;16-bit PNG、GIF、动图、元数据、方向和不兼容色彩空间都会触发转换。源图和转换后的附件各完整解码一次。`saveImages` 在发布任何批次成员前为每张图片各准备并验证一次规范化附件,因此校验失败不会留下部分引用,提交阶段也不会重复执行完整图片编码。 请求版本保存在 `/attachments/v1/request-images/`。`readImageRequest` 在不放大小图的前提下,把存储的规范化附件缩放到总像素预算内,再执行独立的编码字节上限。请求编码器使用同一分类分支:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。候选按需执行,两个质量档均超限后才缩小尺寸。缓存身份包含附件 ID、变换策略版本、像素和字节预算及固定编码参数。缓存字节在使用前会完整解码并校验为 8-bit sRGB/sRGBA。同一身份的并发调用共享一次变换和缓存写入;取消一个等待方不会取消共享任务。调用方组合单数读取得到有序批次,服务的 FIFO 限流器通过 `imageCompressionConcurrency` 限制同时执行的规范化和请求变换。该配置范围为 1 至 8,默认值为 2;文件发布仍在准备结束后按顺序执行。 diff --git a/packages/attachment/attachment-local/src/image.ts b/packages/attachment/attachment-local/src/image.ts index beedd3b8c0..c34944f676 100644 --- a/packages/attachment/attachment-local/src/image.ts +++ b/packages/attachment/attachment-local/src/image.ts @@ -23,6 +23,24 @@ export interface DetectedImage { hasAlpha: boolean } +/** + * Check alpha metadata for bytes produced by this package's encoders. + * Sharp/libvips may omit an all-opaque alpha plane from WebP output; every + * other addition or removal indicates that the encoded result is incompatible + * with its source facts. + * @param sourceHasAlpha - whether the source bytes declare an alpha plane, or undefined when the source frame is unspecified. + * @param output - decoded media type and alpha metadata from the encoded result. + * @returns whether the output alpha metadata is compatible with the source. + */ +export function encodedAlphaIsCompatible( + sourceHasAlpha: boolean | undefined, + output: Pick, +): boolean { + return sourceHasAlpha === undefined + || output.hasAlpha === sourceHasAlpha + || (sourceHasAlpha && !output.hasAlpha && output.mediaType === 'image/webp') +} + const MEDIA_TYPES: Readonly> = { png: 'image/png', jpeg: 'image/jpeg', diff --git a/packages/attachment/attachment-local/src/normalization.ts b/packages/attachment/attachment-local/src/normalization.ts index acfec63c0f..e9ecd8d3e7 100644 --- a/packages/attachment/attachment-local/src/normalization.ts +++ b/packages/attachment/attachment-local/src/normalization.ts @@ -4,7 +4,7 @@ import sharp, { type Sharp } from 'sharp' import { AttachmentError } from '@deepseek-ai/dsh-attachment' import type { ImageMediaType } from '@deepseek-ai/dsh-attachment' import { encodeFirstWithinLimit, isExhaustedEncoding } from './encoding.ts' -import { detectImage } from './image.ts' +import { detectImage, encodedAlphaIsCompatible } from './image.ts' import type { DetectedImage } from './image.ts' /** Deployment-resolved policy for the persisted normalized attachment. */ @@ -104,7 +104,7 @@ async function verifyNormalizedImage( || detected.carriesMetadata || detected.depth !== 'uchar' || detected.space !== 'srgb' - || (expectedAlpha !== undefined && detected.hasAlpha !== expectedAlpha)) { + || !encodedAlphaIsCompatible(expectedAlpha, detected)) { throw new AttachmentError( 'Image normalization did not produce a single-frame 8-bit sRGB image with matching metadata.', 'ATTACHMENT_WRITE_FAILED', diff --git a/packages/attachment/attachment-local/src/request-image.ts b/packages/attachment/attachment-local/src/request-image.ts index b7c9068bfb..66c427480b 100644 --- a/packages/attachment/attachment-local/src/request-image.ts +++ b/packages/attachment/attachment-local/src/request-image.ts @@ -14,7 +14,7 @@ import type { } from '@deepseek-ai/dsh-attachment' import { hasLowColourCount } from './normalization.ts' import { encodeFirstWithinLimit, isExhaustedEncoding } from './encoding.ts' -import { detectImage, probeImage } from './image.ts' +import { detectImage, encodedAlphaIsCompatible, probeImage } from './image.ts' /** Transform version included in every cache and upload-index identity. */ export const REQUEST_IMAGE_TRANSFORM_VERSION = 'request-image-v4' @@ -201,7 +201,7 @@ async function readCached( const maximum = requestImageDimensions(attachment.ref.width, attachment.ref.height, policy.maxPixels) if (data.byteLength > policy.maxBytes || detected.depth !== 'uchar' || detected.space !== 'srgb' || detected.width > maximum.width || detected.height > maximum.height - || detected.hasAlpha !== expectedAlpha) return undefined + || !encodedAlphaIsCompatible(expectedAlpha, detected)) return undefined return { data, mediaType: detected.mediaType, width: detected.width, height: detected.height, hasAlpha: detected.hasAlpha } } catch (error: unknown) { if ((error as NodeJS.ErrnoException | null)?.code === 'ENOENT') return undefined @@ -217,7 +217,7 @@ async function verifyRequestImage( const detected = await detectImage(image.data) if (detected.depth !== 'uchar' || detected.space !== 'srgb' || detected.width !== image.width || detected.height !== image.height - || detected.mediaType !== image.mediaType || detected.hasAlpha !== expectedAlpha) { + || detected.mediaType !== image.mediaType || !encodedAlphaIsCompatible(expectedAlpha, detected)) { throw new AttachmentError( 'Encoded model-request image does not match its verified 8-bit sRGB metadata.', 'ATTACHMENT_WRITE_FAILED', diff --git a/packages/attachment/attachment-local/tests/normalization.spec.ts b/packages/attachment/attachment-local/tests/normalization.spec.ts index 4b530988d1..d43ae45bcd 100644 --- a/packages/attachment/attachment-local/tests/normalization.spec.ts +++ b/packages/attachment/attachment-local/tests/normalization.spec.ts @@ -112,18 +112,29 @@ describe('normalizeImage', () => { expect(normalized).toMatchObject({ mediaType: 'image/png', width: 4, height: 2 }) }) - it('retains an all-opaque alpha channel while converting a low-colour image', async () => { - const data = new Uint8Array(await sharp({ - create: { width: 10, height: 6, channels: 4, background: { r: 12, g: 200, b: 64, alpha: 1 } }, + it('accepts WebP output that omits an all-opaque source alpha plane', async () => { + const width = 64 + const height = 32 + const rgb = noisePixels(width, height) + const rgba = new Uint8Array(width * height * 4) + for (let pixel = 0; pixel < width * height; pixel += 1) { + rgba[pixel * 4] = rgb[pixel * 3] ?? 0 + rgba[pixel * 4 + 1] = rgb[pixel * 3 + 1] ?? 0 + rgba[pixel * 4 + 2] = rgb[pixel * 3 + 2] ?? 0 + rgba[pixel * 4 + 3] = 255 + } + const data = new Uint8Array(await sharp(rgba, { + raw: { width, height, channels: 4 }, }).png().toBuffer()) + await expect(detectImage(data)).resolves.toMatchObject({ hasAlpha: true }) const normalized = await normalizeImage(data, await detectImage(data), { - maxDimension: 5, + maxDimension: 32, maxBytes: POLICY.maxBytes, }) - expect(normalized).toMatchObject({ mediaType: 'image/png', width: 5, height: 3 }) - await expect(detectImage(normalized.data)).resolves.toMatchObject({ hasAlpha: true }) + expect(normalized).toMatchObject({ mediaType: 'image/webp', width: 32, height: 16 }) + await expect(detectImage(normalized.data)).resolves.toMatchObject({ hasAlpha: false }) }) it('keeps transparency when the byte cap requires another encoding and smaller dimensions', async () => { diff --git a/packages/attachment/attachment-local/tests/request-image.spec.ts b/packages/attachment/attachment-local/tests/request-image.spec.ts index 7052da89e8..66932b5e52 100644 --- a/packages/attachment/attachment-local/tests/request-image.spec.ts +++ b/packages/attachment/attachment-local/tests/request-image.spec.ts @@ -21,6 +21,23 @@ async function image(width: number, height: number): Promise { }).png().toBuffer()) } +async function complexOpaqueAlphaImage(width: number, height: number): Promise { + const pixels = new Uint8Array(width * height * 4) + let state = 0x2545f491 + for (let offset = 0; offset < pixels.length; offset += 4) { + for (let channel = 0; channel < 3; channel += 1) { + state ^= state << 13 + state ^= state >>> 17 + state ^= state << 5 + pixels[offset + channel] = state & 0xff + } + pixels[offset + 3] = 255 + } + return new Uint8Array(await sharp(pixels, { + raw: { width, height, channels: 4 }, + }).png().toBuffer()) +} + afterEach(async () => { await Promise.all(homes.splice(0).map(home => rm(home, { recursive: true, force: true }))) }) @@ -208,16 +225,15 @@ describe('local request-image cache', () => { }) }) - it('retains an all-opaque alpha channel in a resized request version', async () => { + it('accepts a resized WebP request version that omits an all-opaque alpha plane', async () => { const attachments = await store() - const source = new Uint8Array(await sharp({ - create: { width: 64, height: 32, channels: 4, background: { r: 12, g: 34, b: 56, alpha: 1 } }, - }).png().toBuffer()) + const source = await complexOpaqueAlphaImage(64, 32) const attachment = await attachments.saveImage({ data: source, mediaType: 'image/png' }) const request = await attachments.readImageRequest(attachment, { maxPixels: 16 * 16, maxBytes: 1024 * 1024 }) - await expect(sharp(request.data).metadata()).resolves.toMatchObject({ hasAlpha: true }) + expect(request.mediaType).toBe('image/webp') + await expect(sharp(request.data).metadata()).resolves.toMatchObject({ hasAlpha: false }) }) it('keeps a complex 640,000-pixel request version below 1 MiB', async () => { From 1b389798dcab65d2a29f673aa25ab4e68ca7876f Mon Sep 17 00:00:00 2001 From: creatixchu Date: Fri, 21 Aug 2026 18:14:22 +0800 Subject: [PATCH 27/28] fix(llm-deepseek): fall back when Files resolution fails --- ...1-deepseek-files-inline-fallback.i18n.yaml | 6 + ...26-08-21-deepseek-files-inline-fallback.md | 37 +++ ...08-21-deepseek-files-inline-fallback.zh.md | 37 +++ ...0-unified-image-request-pipeline.i18n.yaml | 4 +- ...26-08-20-unified-image-request-pipeline.md | 8 +- ...08-20-unified-image-request-pipeline.zh.md | 8 +- docs/config-catalog.i18n.yaml | 4 +- docs/config-catalog.md | 10 +- docs/config-catalog.zh.md | 10 +- examples/acp-agent/tests/acp.snapshot.ts | 42 ++- packages/llm/llm-deepseek/README.i18n.yaml | 4 +- packages/llm/llm-deepseek/README.md | 15 +- packages/llm/llm-deepseek/README.zh.md | 15 +- packages/llm/llm-deepseek/src/adapter.ts | 99 ++++-- packages/llm/llm-deepseek/src/index.ts | 43 ++- packages/llm/llm-deepseek/src/serialize.ts | 66 ++-- packages/llm/llm-deepseek/src/types.ts | 11 +- .../llm/llm-deepseek/tests/adapter.spec.ts | 299 +++++++++++++++++- .../llm/llm-deepseek/tests/serialize.spec.ts | 64 +++- 19 files changed, 695 insertions(+), 87 deletions(-) create mode 100644 .agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.i18n.yaml create mode 100644 .agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.md create mode 100644 .agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.zh.md diff --git a/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.i18n.yaml new file mode 100644 index 0000000000..ed4af5577d --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write .agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.md +2026-08-21-deepseek-files-inline-fallback.md: c58b3e2257b426f1b5df8a4d6952e890a2bd2982 +2026-08-21-deepseek-files-inline-fallback.zh.md: 34625c6250d52a73ccaac3e33adbd2ed099aab5b diff --git a/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.md b/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.md new file mode 100644 index 0000000000..c58b3e2257 --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.md @@ -0,0 +1,37 @@ +# Agent Note: Recover DeepSeek image requests from Files resolution failures + +Status: implemented + +English | [中文](2026-08-21-deepseek-files-inline-fallback.zh.md) + +## Problem + +The direct DeepSeek vision route uses provider file ids so repeated requests do not resend image bytes. An unavailable, unsupported, or stalled Files endpoint can prevent chat before the model request begins even though the same endpoint still accepts inline image data. A fallback that retains the 128MiB Files budget would exceed the inline request-body limit, while a fallback that independently transforms images could send different pixels from the failed file-id attempt. + +## Decision + +Files remains the preferred transport. Each request-image file resolution has the configurable `filesApiTimeoutMs` deadline, one minute by default and always below `streamIdleTimeoutMs`. Successful resolutions refresh the outer idle watchdog. Caller cancellation and the outer stream deadline remain terminal outcomes. + +A file resolution failure discards the transient file parts assembled for that chat attempt and rebuilds the complete image request with base64 data URLs. Every retained image uses the already prepared deterministic `RequestImageAttachment`; the fallback performs no additional decode, resize, or encode, and a chat request never mixes file ids with inline images. Upload mappings committed before a later image fails remain available to later requests. The next request tries Files again, so recovery requires no process-wide outage state. + +Inline fallback has a separate base64-expanded high watermark, `maxInlineRequestImageBytes`, of 20MiB by default. `inlineImageOffloadByteQuantum` defaults to 10MiB, so crossing the high watermark advances the deterministic oldest-image prefix to the next 10MiB removal boundary. The existing 600-image bound and count quantum still apply. File mode retains its 128MiB high watermark and 64MiB removal quantum. + +Provider chat errors keep their existing classifications. A stale file id is invalidated, re-uploaded, and retried once. If that replacement resolution fails, the permitted retry uses the inline representation. A generic chat failure does not switch transports because it does not establish that Files resolution failed. + +## Alternatives considered + +**Send inline images first.** Rejected because successful Files uploads allow deterministic request bytes to be reused across turns without repeating base64 in every request. + +**Mix resolved file ids with inline images after one upload fails.** Rejected because the request would still depend on the failing Files service and would have two independent image budgets. + +**Apply the 128MiB Files bound to inline fallback.** Rejected because base64 expands the payload and can exceed the chat request-body limit. The 20MiB budget leaves space for JSON, text history, and tools. + +**Remember an outage and bypass Files on later requests.** Rejected because a process-local circuit state introduces recovery timing and shared failure state. Retrying Files on the next request detects service recovery without another timer. + +## Verification + +Serializer tests cover file and data-URL representations over the same request versions, all supported media types, tool-result placement, and 20-to-10 base64 offload. Adapter tests cover immediate resolution failure, failure after a partial set of file ids, deadline-triggered fallback, stale-id replacement failure, all-inline request bodies, caller cancellation without fallback, and generic chat failure without a transport switch. Configuration tests cover both inline bounds and the Files deadline relationship. + +## Consequences + +A Files outage no longer prevents an image chat that fits the inline budget. Fallback repeats image bytes and may omit more history than file mode because its limit is lower. A request can leave successful uploads behind when a later image fails, but their indexed mappings are reusable and do not change the chat body sent by the fallback. Explicit file-management operations continue to expose their own failures. diff --git a/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.zh.md b/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.zh.md new file mode 100644 index 0000000000..34625c6250 --- /dev/null +++ b/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.zh.md @@ -0,0 +1,37 @@ +# Agent Note: DeepSeek Files 解析失败时恢复图片请求 + +Status: implemented + +[English](2026-08-21-deepseek-files-inline-fallback.md) | 中文 + +## Problem + +DeepSeek 官方视觉路由使用提供方文件 ID,使重复请求不必再次发送图片字节。如果 Files 端点不可用、不受支持或一直不返回,chat 会在模型请求开始前失败,即使同一端点仍接受内联图片数据。沿用 128MiB Files 预算的回退会超过内联请求体上限,独立转换图片的回退则可能发送与失败 file ID 尝试不同的像素。 + +## Decision + +Files 仍是首选传输方式。每张请求图片的文件解析都有可配置的 `filesApiTimeoutMs` 时限,默认一分钟,且始终小于 `streamIdleTimeoutMs`。每次成功解析都会刷新外层 idle watchdog。调用方取消和外层流时限仍直接终止请求。 + +文件解析失败后,适配器会丢弃为该次 chat 尝试组装的临时文件块,并用 base64 data URL 重新组装完整图片请求。每张保留图片都复用已经准备好的确定性 `RequestImageAttachment`;回退不会再次解码、缩放或编码,同一个 chat 请求也不会混用 file ID 和内联图片。较早图片在后续图片失败前已经提交的上传映射会保留,供之后请求使用。下一次请求会重新尝试 Files,因此不需要保存进程级故障状态。 + +内联回退使用独立的 base64 膨胀后高水位,`maxInlineRequestImageBytes` 默认为 20MiB。`inlineImageOffloadByteQuantum` 默认为 10MiB,因此越过高水位时,确定性的最旧图片前缀会推进到下一个 10MiB 移除边界。现有 600 张图片上限和数量步长继续生效。文件模式继续使用 128MiB 高水位和 64MiB 移除步长。 + +提供方 chat 错误继续使用现有分类。失效 file ID 会被清除、重新上传并重试一次。如果替换解析失败,这次允许的重试会使用内联表示。普通 chat 错误不能证明 Files 解析失败,因此不会切换传输方式。 + +## Alternatives considered + +**优先发送内联图片。** 不采用,因为 Files 上传成功后可以跨轮次复用确定性的请求字节,不必在每次请求中重复 base64。 + +**某次上传失败后混用已解析 file ID 和内联图片。** 不采用,因为请求仍依赖发生故障的 Files 服务,而且需要同时处理两套图片预算。 + +**把 128MiB Files 上限用于内联回退。** 不采用,因为 base64 会扩大负载,并可能超过 chat 请求体上限。20MiB 预算会为 JSON、文本历史和工具留下空间。 + +**记住故障,并在后续请求中跳过 Files。** 不采用,因为进程级状态会引入恢复时间和共享故障状态。下一次请求重新尝试 Files,可以在无需新增计时器的情况下发现服务恢复。 + +## Verification + +序列化测试覆盖相同请求版本的文件和 data URL 表示、全部支持的媒体类型、工具结果位置,以及 20MiB 到 10MiB 的 base64 offload。适配器测试覆盖立即解析失败、部分 file ID 成功后的失败、时限触发的回退、失效 ID 替换失败、全内联请求体、调用方取消时不回退,以及普通 chat 错误不切换传输方式。配置测试覆盖两项内联预算和 Files 时限关系。 + +## Consequences + +符合内联预算的图片 chat 不会再因 Files 故障而失败。回退会重复发送图片字节,而且由于上限更低,可能比文件模式省略更多历史。后续图片失败时,请求可能留下较早图片的成功上传,但这些索引映射可以复用,也不会改变回退发送的 chat 请求体。显式文件管理操作继续暴露自身错误。 diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml index 1c6145c359..6a379a3532 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md -2026-08-20-unified-image-request-pipeline.md: 6a3bae8a970677c32bbfb7966d2bc13d4e504804 -2026-08-20-unified-image-request-pipeline.zh.md: 10a4aed0b5ca9168c6a6ee4ec0258a210b50d531 +2026-08-20-unified-image-request-pipeline.md: ada15d540539977c631e359ffdc7baa4fa84c78e +2026-08-20-unified-image-request-pipeline.zh.md: 85c9a1f837d82cba2bc62b30402433f50c873cbe diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md index 6a3bae8a97..ada15d5405 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.md @@ -34,7 +34,7 @@ Every retained request image is preceded by its complete attachment id and actua ### DeepSeek Files lifecycle -The direct `deepseek-official` adapter uploads every retained request version through the OpenAI-compatible Files API and sends only `file_id` content blocks. There is no inline fallback. The default catalog advertises `deepseek-v4-flash-vision-exp` as image-capable. Uploaded ids are indexed by endpoint and API-key scope plus `variantId`. Uploads request seven days by default and record the returned `expires_at`; a mapping with no more than one hour remaining is replaced without a preceding retrieve call. The index never stores the API key. +The direct `deepseek-official` adapter normally uploads every retained request version through the OpenAI-compatible Files API and sends `file_id` content blocks. A [bounded inline fallback](../bug-fix/2026-08-21-deepseek-files-inline-fallback.md) sends the same deterministic request versions when file resolution fails. The default catalog advertises `deepseek-v4-flash-vision-exp` as image-capable. Uploaded ids are indexed by endpoint and API-key scope plus `variantId`. Uploads request seven days by default and record the returned `expires_at`; a mapping with no more than one hour remaining is replaced without a preceding retrieve call. The index never stores the API key. An upload is indexed only after the response returns a complete file object, matching byte count, and `expires_at`. A missing or inconsistent response leaves no local mapping, so a later request uploads again. Concurrent upload resolution for one scoped `variantId` shares one provider operation; one waiter cannot cancel another, and the upload stops when every waiter has cancelled. A malformed upload index is an empty cache and is replaced on the next successful upload; filesystem I/O failures remain errors. If chat reports expired, deleted, missing, or invalid ids and names one or more ids used by the request, only those mappings are removed. A stale-file response without a specific id removes every mapping used by that chat attempt. The affected request bytes are uploaded again and chat is retried once. A second stale rejection clears the mappings identified by its response and returns the error without a third chat attempt. One upload quota error first lists the configured number of oldest harness-owned `dsh-` files, then deletes that collected set and retries once; deleting after pagination keeps provider cursors valid. Public file operations expose list, retrieve, delete, one-variant release, and namespace-wide release. Every Files request carries the shared Harness `User-Agent`. The client enforces the documented 128MiB upload limit, 32MiB chat-image limit, 10,000-file and 25GiB quotas, and one-hour to 30-day expiry range. @@ -52,7 +52,7 @@ Historical attachment objects that later disappear or fail integrity verificatio **Treat PNG as a screenshot and reject 16-bit PNG.** File format does not reveal pixel complexity, and 16-bit RGB/RGBA is a convertible sample depth rather than an unsupported image type. Pixel sampling and post-conversion probes give the required facts. -**Keep DeepSeek data URLs.** Inline base64 repeats bytes on every request and caps usable image history by request-body size. Files API references reuse uploaded deterministic request bytes and provide explicit expiry and deletion. +**Keep DeepSeek data URLs as the primary transport.** Inline base64 repeats bytes on every request and caps usable image history by request-body size. Files API references reuse uploaded deterministic request bytes and provide explicit expiry and deletion; the bounded fallback uses data URLs only when file resolution fails. **Trust a locally indexed file id indefinitely.** Remote expiry, deletion, and lost upload responses make local and provider state diverge. Response-directed invalidation and one re-upload recover without an unbounded retry loop; an ambiguous stale-file response must invalidate every file used by that attempt because it provides no safe exact target. @@ -62,8 +62,8 @@ Historical attachment objects that later disappear or fail integrity verificatio ## Verification -Package tests generate 16-bit RGB and RGBA PNG fixtures, prove 8-bit conversion and clean 8-bit passthrough, retain alpha under byte pressure, distinguish high-frequency and ordinary photos from low-color graphics, stop lazy encoding after the first fitting candidate, cover square and wide 640,000-pixel projections, enforce 1MiB request bytes, singleflight equal variants and uploads without shared-cancellation leaks, bound transform concurrency, preserve cache and upload identity, skip attachment reads for conservatively offloaded history, prepare batches once, reject inconsistent Files responses, refresh near-expiry ids without retrieve, recover once from single-id, multiple-id, and ambiguous stale responses, paginate before quota deletion, normalize provider diagnostics, project text-only history, and share normal/compaction request bytes. Keyless assembled snapshots cover the real tool schemas and image request path. A credentialed test uses the built-in `deepseek-official` route and its configured endpoint, never a custom provider entry. +Package tests generate 16-bit RGB and RGBA PNG fixtures, prove 8-bit conversion and clean 8-bit passthrough, retain alpha under byte pressure, distinguish high-frequency and ordinary photos from low-color graphics, stop lazy encoding after the first fitting candidate, cover square and wide 640,000-pixel projections, enforce 1MiB request bytes, singleflight equal variants and uploads without shared-cancellation leaks, bound transform concurrency, preserve cache and upload identity, skip attachment reads for conservatively offloaded history, prepare batches once, reject inconsistent Files responses, refresh near-expiry ids without retrieve, recover once from single-id, multiple-id, and ambiguous stale responses, fall back to bounded all-inline requests after file resolution failure, paginate before quota deletion, normalize provider diagnostics, project text-only history, and share normal/compaction request bytes. Keyless assembled snapshots cover the real tool schemas and image request path. A credentialed test uses the built-in `deepseek-official` route and its configured endpoint, never a custom provider entry. ## Consequences -Normalized attachments consume up to the independent local safety cap, while request caches and remote Files consume additional derived storage. Deterministic identities and singleflight make that work reusable across turns and sessions sharing the same DSH home. Two simultaneous transforms reduce batch latency while increasing peak RSS relative to serial execution; deployments with tighter memory can set the limit to one. Encoder or transform-version changes create new future identities without rewriting existing history. DeepSeek image requests now depend on Files API availability; bounded stale-id recovery handles inconsistent remote state, while a general Files outage remains a visible request failure. Missing or corrupt durable attachments still require the separate quarantine design. +Normalized attachments consume up to the independent local safety cap, while request caches and remote Files consume additional derived storage. Deterministic identities and singleflight make that work reusable across turns and sessions sharing the same DSH home. Two simultaneous transforms reduce batch latency while increasing peak RSS relative to serial execution; deployments with tighter memory can set the limit to one. Encoder or transform-version changes create new future identities without rewriting existing history. DeepSeek image requests prefer Files reuse; bounded stale-id recovery handles inconsistent remote state, while file-resolution failures use the smaller inline budget. Missing or corrupt durable attachments still require the separate quarantine design. diff --git a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md index 10a4aed0b5..85c9a1f837 100644 --- a/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md +++ b/.agents/notes/implemented/feature/2026-08-20-unified-image-request-pipeline.zh.md @@ -34,7 +34,7 @@ Status: implemented ### DeepSeek Files 生命周期 -直接 `deepseek-official` 适配器通过 OpenAI 兼容 Files API 上传每张保留的请求版本,只发送 `file_id` 内容块,不提供内联回退。默认 catalog 把 `deepseek-v4-flash-vision-exp` 公布为支持图片。上传 ID 按端点和 API key 作用域以及 `variantId` 写入索引。上传默认请求 7 天有效期,并记录返回的 `expires_at`;本地映射剩余时间不超过一小时时会直接替换,不会先查询远端文件。索引绝不存储 API key。 +直接 `deepseek-official` 适配器通常通过 OpenAI 兼容 Files API 上传每张保留的请求版本,并发送 `file_id` 内容块。文件解析失败时,[有界内联回退](../bug-fix/2026-08-21-deepseek-files-inline-fallback.zh.md)会发送相同的确定性请求版本。默认 catalog 把 `deepseek-v4-flash-vision-exp` 公布为支持图片。上传 ID 按端点和 API key 作用域以及 `variantId` 写入索引。上传默认请求 7 天有效期,并记录返回的 `expires_at`;本地映射剩余时间不超过一小时时会直接替换,不会先查询远端文件。索引绝不存储 API key。 只有上传响应返回完整文件对象、匹配的字节数和 `expires_at` 时,上传结果才会写入索引。缺失或不一致的响应不会留下本地映射,后续请求会重新上传。同一作用域和 `variantId` 的并发解析共享一次提供方上传;单个等待方无法取消其他等待方,全部等待方取消时才会停止上传。格式损坏的上传索引按空缓存处理,并在下一次成功上传时替换;文件系统 I/O 失败仍是错误。如果 chat 报告 ID 已过期、删除、缺失或无效,并指出本次请求使用的一个或多个 ID,适配器只删除这些映射。如果响应只说明文件状态失效而没有指出具体 ID,适配器会删除该次 chat 使用的全部映射。受影响的请求字节会重新上传,chat 只重试一次。第二次仍报告文件失效时,适配器会按响应清理映射并返回错误,不会发起第三次 chat。一次上传配额错误会先列出配置数量的最旧 `dsh-` 文件,再删除收集到的文件并重试一次;分页完成后才删除,避免游标失效。公开文件操作提供列表、查询、删除、单个变体释放和整个作用域释放。每个 Files 请求都携带 Harness 的共享 `User-Agent`。客户端执行文档规定的 Files 单次上传 128MiB、chat 单图 32MiB、10,000 个文件、25GiB,以及一小时到 30 天有效期限制。 @@ -52,7 +52,7 @@ Status: implemented **把 PNG 当作截图,并拒绝 16-bit PNG。** 文件格式不能说明像素复杂度,16-bit RGB/RGBA 是可转换位深,不是不支持的图片类型。像素采样和转换后探测能提供所需事实。 -**继续向 DeepSeek 发送 data URL。** 内联 base64 会在每次请求中重复字节,并按请求正文大小限制可用图片历史。Files API 引用会复用上传后的确定性请求字节,并提供显式有效期和删除操作。 +**把 DeepSeek data URL 作为首选传输方式。** 内联 base64 会在每次请求中重复字节,并按请求正文大小限制可用图片历史。Files API 引用会复用上传后的确定性请求字节,并提供显式有效期和删除操作;有界回退只在文件解析失败时使用 data URL。 **永久信任本地索引中的文件 ID。** 远端过期、删除和上传响应丢失会使本地与提供方状态不一致。按响应失效和一次重新上传可以恢复,同时避免无界重试;响应没有给出可安全使用的精确目标时,必须使该次请求使用的全部文件失效。 @@ -62,8 +62,8 @@ Status: implemented ## Verification -包测试会生成 16-bit RGB 和 RGBA PNG fixture,验证 8-bit 转换与干净 8-bit 字节直通、字节压力下保留透明通道、区分高频和普通照片与低色数图形、首个候选合规后停止编码、正方形和宽屏 640,000 像素投影、请求字节不超过 1MiB、相同变体与上传 singleflight 且不会共享取消、变换并发上限、缓存与上传身份、跳过已保守 offload 的历史附件读取、批量只准备一次、Files 响应不一致、进入刷新余量时不查询远端并更新 ID、单个 ID、多个 ID 和模糊失效响应只恢复一次、删除配额文件前完成分页、规范化提供方诊断、纯文本投影,以及普通请求与压缩共享请求字节。无需密钥的组装快照覆盖真实工具 schema 和图片请求路径。使用凭据的测试只使用内置 `deepseek-official` 路由及其已配置端点,不使用自定义提供方条目。 +包测试会生成 16-bit RGB 和 RGBA PNG fixture,验证 8-bit 转换与干净 8-bit 字节直通、字节压力下保留透明通道、区分高频和普通照片与低色数图形、首个候选合规后停止编码、正方形和宽屏 640,000 像素投影、请求字节不超过 1MiB、相同变体与上传 singleflight 且不会共享取消、变换并发上限、缓存与上传身份、跳过已保守 offload 的历史附件读取、批量只准备一次、Files 响应不一致、进入刷新余量时不查询远端并更新 ID、单个 ID、多个 ID 和模糊失效响应只恢复一次、文件解析失败后回退到有界全内联请求、删除配额文件前完成分页、规范化提供方诊断、纯文本投影,以及普通请求与压缩共享请求字节。无需密钥的组装快照覆盖真实工具 schema 和图片请求路径。使用凭据的测试只使用内置 `deepseek-official` 路由及其已配置端点,不使用自定义提供方条目。 ## Consequences -持久规范化附件最多占用独立的本地安全上限,请求缓存和远端 Files 还会占用额外派生存储。确定性身份和 singleflight 使这些成本可以被共享同一 DSH home 的轮次和会话复用。同时执行两个变换会降低批次延迟,但峰值 RSS 高于串行执行;内存更紧张的部署可以把上限设为 1。编码器或变换策略版本变化会为未来内容产生新身份,不会改写已有历史。DeepSeek 图片请求现在依赖 Files API 可用性;有界的陈旧 ID 恢复会处理远端状态不一致,一般 Files 故障仍会成为可见请求失败。缺失或损坏的持久附件仍需要单独的隔离设计。 +持久规范化附件最多占用独立的本地安全上限,请求缓存和远端 Files 还会占用额外派生存储。确定性身份和 singleflight 使这些成本可以被共享同一 DSH home 的轮次和会话复用。同时执行两个变换会降低批次延迟,但峰值 RSS 高于串行执行;内存更紧张的部署可以把上限设为 1。编码器或变换策略版本变化会为未来内容产生新身份,不会改写已有历史。DeepSeek 图片请求优先复用 Files;有界的陈旧 ID 恢复会处理远端状态不一致,文件解析失败则使用较小的内联预算。缺失或损坏的持久附件仍需要单独的隔离设计。 diff --git a/docs/config-catalog.i18n.yaml b/docs/config-catalog.i18n.yaml index 7b1abc90d1..d0665a143e 100644 --- a/docs/config-catalog.i18n.yaml +++ b/docs/config-catalog.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/config-catalog.md -config-catalog.md: 552c09c08abef2cab957d2a8caab9412cb4522e5 -config-catalog.zh.md: fc1993bf4c4f21ec6ec85341f4ce09a04b6a8b66 +config-catalog.md: de340b7ffade528301b4538b0553bc11ec969985 +config-catalog.zh.md: eb17ee89fd7860cc0774073bea542aca315ce652 diff --git a/docs/config-catalog.md b/docs/config-catalog.md index 552c09c08a..de340b7ffa 100644 --- a/docs/config-catalog.md +++ b/docs/config-catalog.md @@ -940,12 +940,18 @@ export interface Config { streamIdleTimeoutMs?: number /** Maximum accumulated file-referenced image bytes per chat request (default 128 MiB). */ maxRequestFilesBytes?: number - /** Maximum number of file-referenced images per chat request (default 600). */ + /** Maximum accumulated base64 image payload after Files API fallback (default 20 MiB). */ + maxInlineRequestImageBytes?: number + /** Maximum number of represented images per chat request (default 600). */ maxImagesPerRequest?: number /** Raw-byte removal step after the request exceeds its file bound (default 64 MiB). */ imageOffloadByteQuantum?: number + /** Base64-byte removal step after inline fallback exceeds its bound (default 10 MiB). */ + inlineImageOffloadByteQuantum?: number /** Image-count removal step after the request exceeds its count bound (default 20). */ imageOffloadCountQuantum?: number + /** Maximum duration of one request-image Files API resolution (default one minute). */ + filesApiTimeoutMs?: number /** Explicit lifetime assigned to each uploaded image (default seven days). */ fileExpiresAfterSeconds?: number /** Remaining lifetime below which an indexed file is replaced (default one hour). */ @@ -981,7 +987,7 @@ export interface DeepSeekCatalogModel { Depends on: [`ModelModality`](../packages/llm/llm/src/index.ts) · [`RetryPolicyConfig`](../packages/llm/llm/src/index.ts) -Source: [`packages/llm/llm-deepseek/src/index.ts:100`](../packages/llm/llm-deepseek/src/index.ts) +Source: [`packages/llm/llm-deepseek/src/index.ts:106`](../packages/llm/llm-deepseek/src/index.ts) diff --git a/docs/config-catalog.zh.md b/docs/config-catalog.zh.md index fc1993bf4c..eb17ee89fd 100644 --- a/docs/config-catalog.zh.md +++ b/docs/config-catalog.zh.md @@ -942,12 +942,18 @@ export interface Config { streamIdleTimeoutMs?: number /** Maximum accumulated file-referenced image bytes per chat request (default 128 MiB). */ maxRequestFilesBytes?: number - /** Maximum number of file-referenced images per chat request (default 600). */ + /** Maximum accumulated base64 image payload after Files API fallback (default 20 MiB). */ + maxInlineRequestImageBytes?: number + /** Maximum number of represented images per chat request (default 600). */ maxImagesPerRequest?: number /** Raw-byte removal step after the request exceeds its file bound (default 64 MiB). */ imageOffloadByteQuantum?: number + /** Base64-byte removal step after inline fallback exceeds its bound (default 10 MiB). */ + inlineImageOffloadByteQuantum?: number /** Image-count removal step after the request exceeds its count bound (default 20). */ imageOffloadCountQuantum?: number + /** Maximum duration of one request-image Files API resolution (default one minute). */ + filesApiTimeoutMs?: number /** Explicit lifetime assigned to each uploaded image (default seven days). */ fileExpiresAfterSeconds?: number /** Remaining lifetime below which an indexed file is replaced (default one hour). */ @@ -983,7 +989,7 @@ export interface DeepSeekCatalogModel { 依赖:[`ModelModality`](../packages/llm/llm/src/index.ts) · [`RetryPolicyConfig`](../packages/llm/llm/src/index.ts) -来源:[`packages/llm/llm-deepseek/src/index.ts:100`](../packages/llm/llm-deepseek/src/index.ts) +来源:[`packages/llm/llm-deepseek/src/index.ts:106`](../packages/llm/llm-deepseek/src/index.ts) diff --git a/examples/acp-agent/tests/acp.snapshot.ts b/examples/acp-agent/tests/acp.snapshot.ts index 548b4025a4..0bfbe4e3ec 100644 --- a/examples/acp-agent/tests/acp.snapshot.ts +++ b/examples/acp-agent/tests/acp.snapshot.ts @@ -702,9 +702,10 @@ defineAcpSnapshotSuite({ hasPwsh, }) -it('pins native DeepSeek Files image offload in the request sent by the assembled app', async () => { +it('pins native DeepSeek Files offload and inline fallback in assembled requests', async () => { const requests: Record[] = [] const fileRequests: Array<{ method: string; path: string; bytes: number }> = [] + let rejectFiles = false const server = createServer((request: IncomingMessage, response: ServerResponse) => { const chunks: Buffer[] = [] request.on('data', (chunk: Buffer) => { chunks.push(chunk) }) @@ -723,6 +724,12 @@ it('pins native DeepSeek Files image offload in the request sent by the assemble const file = form.get('file') if (!(file instanceof Blob)) throw new Error('snapshot Files upload omitted file') fileRequests.push({ method: 'POST', path: url.pathname, bytes: file.size }) + if (rejectFiles) { + response.writeHead(503, { 'content-type': 'application/json' }).end(JSON.stringify({ + error: { message: 'Files temporarily unavailable' }, + })) + return + } const createdAt = Math.floor(Date.now() / 1_000) response.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ id: 'file-api-snapshot-1', @@ -861,6 +868,39 @@ it('pins native DeepSeek Files image offload in the request sent by the assemble ], }, ]) + + rejectFiles = true + const fallback = await runScenario(input, { + agent: AGENT, + mode: 'record', + configPath: IMAGE_OFFLOAD_CONFIG, + fixtureFile: join(SNAPSHOTS_DIR, 'image-offload-request', 'session.jsonl'), + workspaceDir: join(SNAPSHOTS_DIR, 'read-image', 'workspace'), + env: { + DSH_SNAPSHOT_API_KEY: 'snapshot-fallback-key', + DSH_SNAPSHOT_BASE_URL: `http://127.0.0.1:${address.port}`, + }, + }) + expect(fallback.stderr).toBe('') + expect(fileRequests).toEqual([ + { method: 'POST', path: '/files', bytes: 69 }, + { method: 'POST', path: '/files', bytes: 69 }, + ]) + expect(requests).toHaveLength(3) + const fallbackMessages = requests[2]?.messages as { content?: unknown }[] | undefined + const fallbackInput = fallbackMessages?.find(message => JSON.stringify(message.content).includes('[image omitted')) + expect(fallbackInput?.content).toEqual([ + { type: 'text', text: 'Compare the older image ' }, + { type: 'text', text: OFFLOADED_IMAGE_TEXT }, + { type: 'text', text: ' with the newer image ' }, + { + type: 'text', + text: '\nImage sha256:b1ff9c8ea3a780bad09b346c423d2d0e46815926879b18e841d928376a946640; ' + + 'request image 1x1px.', + }, + { type: 'image_url', image_url: { url: `data:image/png;base64,${image}` } }, + { type: 'text', text: ', then use read_image on red.png and reply with DONE.' }, + ]) } finally { await new Promise(resolve => server.close(() => { resolve() })) } diff --git a/packages/llm/llm-deepseek/README.i18n.yaml b/packages/llm/llm-deepseek/README.i18n.yaml index c4db847155..e254c6267d 100644 --- a/packages/llm/llm-deepseek/README.i18n.yaml +++ b/packages/llm/llm-deepseek/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/llm/llm-deepseek/README.md -README.md: d17d520c2444d8a0195d997f4df4ff5e0f05befd -README.zh.md: cc823897894102df0dc1da17478eee6ba7ebd21d +README.md: 7a22955565027b30677e46a80a8b719bc7e61917 +README.zh.md: db1669509956d651dcb8948e1191a17cf9a0bfee diff --git a/packages/llm/llm-deepseek/README.md b/packages/llm/llm-deepseek/README.md index d17d520c24..7a22955565 100644 --- a/packages/llm/llm-deepseek/README.md +++ b/packages/llm/llm-deepseek/README.md @@ -21,9 +21,12 @@ The package root exposes the Cordis plugin contract and `DeepSeekAdapter`; wire maxTokens: 256000 # optional positive per-request output cap; this is the default streamIdleTimeoutMs: 300000 # optional; positive finite Node timer delay; five-minute default maxRequestFilesBytes: 134217728 # optional positive integer; 128 MiB raw request-image default + maxInlineRequestImageBytes: 20971520 # base64 fallback high watermark; 20 MiB default maxImagesPerRequest: 600 # provider request image-count limit imageOffloadByteQuantum: 67108864 # oldest-image removal advances in 64 MiB steps + inlineImageOffloadByteQuantum: 10485760 # fallback removal advances in 10 MiB steps imageOffloadCountQuantum: 20 # count overflow advances in 20-image steps + filesApiTimeoutMs: 60000 # per-image Files resolution deadline; below streamIdleTimeoutMs fileExpiresAfterSeconds: 604800 # uploaded image lifetime; 1 hour to 30 days fileRefreshMarginSeconds: 3600 # replace ids with less lifetime remaining fileQuotaCleanupBatch: 100 # oldest harness-owned files deleted before one quota retry @@ -49,11 +52,13 @@ The package root exposes the Cordis plugin contract and `DeepSeekAdapter`; wire The plugin registers the single provider route `deepseek-official` together with its resolved `retryPolicy`; omission resolves to normal mode with five retries. A request selects it with `provider: deepseek-official`; its `model` is passed through as the wire `model` string, so changing DeepSeek models does not require lifecycle-time registration. Omitting `models` advertises `deepseek-v4-flash`, `deepseek-v4-pro`, and the image-capable `deepseek-v4-flash-vision-exp`, each with a 1,000,000-token context window; an explicit list replaces those defaults, while `models: []` advertises none. Catalog entries are exposed through `ctx.llm.listModels('deepseek-official')` for clients such as ACP editors and the Web selector, but remain advisory: unlisted model ids still pass through unchanged as text-only routes. An omitted entry name defaults to its id, and omitted `inputModalities` means `text` only. -An image-capable catalog entry declares `inputModalities: [text, image]` and may set `imagePixelBudget`, `imageMaxBytes`, or `imageDetail: low`. The ordinary default is 640,000 total pixels and 1MiB encoded bytes; low detail defaults to 512 by 512 total pixels. The attachment store scales by `min(1, sqrt(pixelBudget / (width * height)))` and rounds inward to keep the pixel count at or below the hard cap, so a 2048 by 1024 normalized attachment becomes about 1130 by 565 instead of a forced square. Request encoders run lazily: low-color images try PNG (palette only without alpha) then WebP 85 and 80, other alpha images try WebP 85 then 80, and other opaque images try JPEG 85 then 80; dimensions shrink only when both quality attempts exceed 1MiB. Concurrent generation of one `variantId` shares one transform. A caller can cancel its own wait without interrupting other waiters; the transform stops when no waiter remains. The adapter uploads the exact derived request bytes through `POST /files` and sends `{type: "file", file_id}` blocks. It never falls back to an inline data URL. Every retained image is preceded by stable text naming the complete attachment id and actual request dimensions. User, tool-result, agent-loop, compaction, and direct `ctx.llm.stream` requests all use this projection. Text-only routes receive stable attachment placeholders while durable history keeps its image references. +An image-capable catalog entry declares `inputModalities: [text, image]` and may set `imagePixelBudget`, `imageMaxBytes`, or `imageDetail: low`. The ordinary default is 640,000 total pixels and 1MiB encoded bytes; low detail defaults to 512 by 512 total pixels. The attachment store scales by `min(1, sqrt(pixelBudget / (width * height)))` and rounds inward to keep the pixel count at or below the hard cap, so a 2048 by 1024 normalized attachment becomes about 1130 by 565 instead of a forced square. Request encoders run lazily: low-color images try PNG (palette only without alpha) then WebP 85 and 80, other alpha images try WebP 85 then 80, and other opaque images try JPEG 85 then 80; dimensions shrink only when both quality attempts exceed 1MiB. Concurrent generation of one `variantId` shares one transform. A caller can cancel its own wait without interrupting other waiters; the transform stops when no waiter remains. The adapter normally uploads the exact derived request bytes through `POST /files` and sends `{type: "file", file_id}` blocks. A failed or timed-out file-id resolution rebuilds the whole chat request with those same request versions as base64 data URLs; one request never mixes file ids and inline images. Every retained image is preceded by stable text naming the complete attachment id and actual request dimensions. User, tool-result, agent-loop, compaction, and direct `ctx.llm.stream` requests all use this projection. Text-only routes receive stable attachment placeholders while durable history keeps its image references. `maxRequestFilesBytes` and `maxImagesPerRequest` bound the retained request versions at 128MiB and 600 images by default. The byte and count quanta must not exceed their corresponding bounds. Before attachment reads, the adapter uses each route's request-version byte cap as a conservative upper bound and removes the oldest over-budget prefix; only retained normalized attachments are read and transformed. Exact derived lengths are checked again without restoring omitted images. When the byte bound is crossed, the oldest prefix advances past the next 64MiB boundary; 129 one-megabyte images remove the oldest 65 and retain 64MiB, and that prefix stays unchanged until durable history exceeds 192MiB. Count overflow advances independently in `imageOffloadCountQuantum` steps. Removed images become the fixed model-visible placeholder `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`. This high-watermark projection avoids changing an old request prefix after every new image. -Uploaded ids are indexed below `DSH_HOME` by endpoint/API-key scope and request `variantId`. The variant covers the normalized attachment id, transform version, route pixel and byte budgets, and encoder parameters, so Files API and inline-capable adapters refer to the same deterministic bytes. Uploads request a seven-day lifetime by default and store the server's `expires_at`. A local mapping with no more than one hour remaining is replaced before use; the adapter does not retrieve every remote file before chat. If chat reports expired, deleted, missing, or invalid file ids and names one or more ids used by the request, the adapter removes exactly those mappings. If the provider identifies stale file state without naming an id, it removes every file mapping used by that chat attempt. It then uploads the affected request versions again and retries chat once. A second stale-file rejection clears the mappings identified by that response and is returned without a third chat attempt. An upload response without a complete file object, matching byte count, and `expires_at` is never indexed; a later request therefore uploads again instead of trusting inconsistent local state. A malformed local upload index is treated as an empty cache and replaced by the next successful upload; permission and filesystem I/O failures still fail the request. +Inline fallback has an independent base64 budget. `maxInlineRequestImageBytes` defaults to 20MiB and `inlineImageOffloadByteQuantum` to 10MiB, so a history of 21 one-megabyte base64 payloads removes the oldest 11 and retains 10MiB. The calculation uses base64-expanded lengths. The prepared request versions are reused byte-for-byte; fallback does not decode or compress an image again. Successful mappings created before a later image fails remain indexed for future requests. + +Uploaded ids are indexed below `DSH_HOME` by endpoint/API-key scope and request `variantId`. The variant covers the normalized attachment id, transform version, route pixel and byte budgets, and encoder parameters, so Files API and inline fallback refer to the same deterministic bytes. Uploads request a seven-day lifetime by default and store the server's `expires_at`. A local mapping with no more than one hour remaining is replaced before use; the adapter does not retrieve every remote file before chat. If chat reports expired, deleted, missing, or invalid file ids and names one or more ids used by the request, the adapter removes exactly those mappings. If the provider identifies stale file state without naming an id, it removes every file mapping used by that chat attempt. It then uploads the affected request versions again and retries chat once. A second stale-file rejection clears the mappings identified by that response and is returned without a third chat attempt. An upload response without a complete file object, matching byte count, and `expires_at` is never indexed; a later request therefore uploads again instead of trusting inconsistent local state. A malformed local upload index is treated as an empty cache and replaced by the next successful upload. File resolution, including local index access and remote upload, has a per-image one-minute deadline by default; it must remain below `streamIdleTimeoutMs`. Each successful resolution refreshes the outer idle watchdog. Any resolution failure switches that request to inline mode, while explicit public file-management operations continue to report their own failures. Concurrent resolution of one scoped `variantId` shares one Files upload with waiter-local cancellation. One quota upload failure first paginates and collects the configured number of oldest `dsh-` files, then deletes that set before one upload retry. `DeepSeekFilesClient.delete`, `DeepSeekFileStore.release`, and `releaseAll` expose explicit remote-space reclamation. The current provider limits represented by this package are 128MiB per Files upload, 32MiB per chat-referenced image, 10,000 stored files, and 25GiB per API key; the default 1MiB request version remains below the two per-file limits. @@ -65,7 +70,7 @@ The same exact-model result exposes ordered `off`, `low`, `high`, and `max` effo `thinking: disabled` is a deployment lock that publishes only `off` with `off` as its default. Omitting `reasoningEffort` or configuring it as `off` is valid; configuring `low`, `high`, or `max` fails plugin loading, and a direct per-request attempt to enable thinking fails before network I/O. A request with `GenerateOptions.purpose: 'session-title'` also forces thinking disabled and omits the already-resolved effort, reserving its bounded output for visible title text without changing conversation or compaction defaults. -`streamIdleTimeoutMs` bounds each outstanding provider read, including the initial `fetch`, without counting time the consumer spends between chunks. DeepSeek SSE comments rearm an outstanding read as transport activity but never become `StreamChunk` values or session-log events. One stable abort signal reaches the request and body reader for the whole call; expiry stops the transport and throws `LlmError('TIMEOUT')`, while an earlier caller abort throws `LlmError('ABORTED')`. The adapter normally makes one chat request per `stream()` call and makes a second only for the stale-file recovery described above. It registers the configured retry policy as provider metadata, and `dsh-llm-retry` separately executes that policy at durable agent-step boundaries. +`streamIdleTimeoutMs` bounds each outstanding provider read, including the initial `fetch`, without counting time the consumer spends between chunks. DeepSeek SSE comments and successful file resolutions rearm an outstanding read as transport activity but never become `StreamChunk` values or session-log events. One stable abort signal reaches the request and body reader for the whole call; expiry stops the transport and throws `LlmError('TIMEOUT')`, while an earlier caller abort throws `LlmError('ABORTED')`. The adapter normally makes one chat request per `stream()` call and makes a second only for stale-file recovery. A file-resolution failure before the first chat sends one inline request. If replacement resolution fails after a stale-file response, the inline request is the one permitted retry. It registers the configured retry policy as provider metadata, and `dsh-llm-retry` separately executes that policy at durable agent-step boundaries. ## Dynamic configuration (settings + credentials) @@ -104,7 +109,7 @@ Non-2xx responses throw `LlmError` with stable codes: `AUTH` (401/403), `QUOTA` #### What the model sees -The selected DeepSeek model receives the harness system prompt, message history, tool schemas, stop sequences, and call config. The vision model receives retained user and tool-result images as Files API references beside stable attachment handles and request-image dimensions; an over-budget older image is represented by the documented placeholder. Reasoning content from a prior assistant turn is passed back verbatim, whether or not that turn called a tool. +The selected DeepSeek model receives the harness system prompt, message history, tool schemas, stop sequences, and call config. The vision model normally receives retained user and tool-result images as Files API references beside stable attachment handles and request-image dimensions; a Files resolution failure sends all retained images as inline data URLs instead. An over-budget older image is represented by the documented placeholder. Reasoning content from a prior assistant turn is passed back verbatim, whether or not that turn called a tool. #### Token effect @@ -134,4 +139,4 @@ Loop-retained response blocks append to the next request and preserve its earlie - **`tool_choice` is not mapped** — not part of the core vocabulary (MVP cut, shared with the pi-ai twin). - **Requests use raw `fetch`, not `@cordisjs/plugin-http`** — no shared proxy/interception configuration; adoption is deferred until a second adapter wants it (`TODO(http)`). - **Plugin-added content block types are skipped** — core text and supported image blocks are serialized, and empty tool output crosses the wire as the literal `(no output)`. -- **Images are input-only durable attachments** — direct external URLs and assistant image output are not supported; DeepSeek input uses the Files API. +- **Images are input-only durable attachments** — direct external URLs and assistant image output are not supported; DeepSeek input normally uses the Files API and uses inline base64 only for per-request recovery. diff --git a/packages/llm/llm-deepseek/README.zh.md b/packages/llm/llm-deepseek/README.zh.md index cc82389789..db16695099 100644 --- a/packages/llm/llm-deepseek/README.zh.md +++ b/packages/llm/llm-deepseek/README.zh.md @@ -21,9 +21,12 @@ harness LLM(大语言模型)seam 的 DeepSeek chat-completions 适配器: maxTokens: 256000 # optional positive per-request output cap; this is the default streamIdleTimeoutMs: 300000 # optional; positive finite Node timer delay; five-minute default maxRequestFilesBytes: 134217728 # optional positive integer; 128 MiB raw request-image default + maxInlineRequestImageBytes: 20971520 # base64 fallback high watermark; 20 MiB default maxImagesPerRequest: 600 # provider request image-count limit imageOffloadByteQuantum: 67108864 # oldest-image removal advances in 64 MiB steps + inlineImageOffloadByteQuantum: 10485760 # fallback removal advances in 10 MiB steps imageOffloadCountQuantum: 20 # count overflow advances in 20-image steps + filesApiTimeoutMs: 60000 # per-image Files resolution deadline; below streamIdleTimeoutMs fileExpiresAfterSeconds: 604800 # uploaded image lifetime; 1 hour to 30 days fileRefreshMarginSeconds: 3600 # replace ids with less lifetime remaining fileQuotaCleanupBatch: 100 # oldest harness-owned files deleted before one quota retry @@ -49,11 +52,13 @@ harness LLM(大语言模型)seam 的 DeepSeek chat-completions 适配器: 该插件注册唯一提供方路由 `deepseek-official`,并一同注册解析后的 `retryPolicy`;省略时会解析为 normal 模式并重试五次。请求使用 `provider: deepseek-official` 选择该路由;其 `model` 会作为协议 `model` 字符串原样传递,因此更改 DeepSeek 模型不需要生命周期时注册。省略 `models` 会公布 `deepseek-v4-flash`、`deepseek-v4-pro` 与支持图片输入的 `deepseek-v4-flash-vision-exp`,三者的上下文窗口均为 1,000,000 token;显式列表会替换这些默认值,`models: []` 则不公布任何模型。Catalog 配置项通过 `ctx.llm.listModels('deepseek-official')` 公开给 ACP(Agent Client Protocol)编辑器和 Web 选择器等客户端,但仍只提供建议:未列出模型 id 仍原样传递,并按纯文本路由处理。省略配置项 name 默认为其 id,省略 `inputModalities` 则表示仅支持 `text`。 -支持图片的 catalog 配置项声明 `inputModalities: [text, image]`,并可设置 `imagePixelBudget`、`imageMaxBytes` 或 `imageDetail: low`。普通默认值为总像素 640,000、编码字节 1MiB;low detail 的默认总像素为 512×512。附件存储按 `min(1, sqrt(pixelBudget / (width * height)))` 缩放,并向预算内取整,确保总像素不超过硬上限。因此 2048×1024 规范化附件会得到约 1130×565 的请求版本,而不会被强制变成正方形。请求编码按需执行:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。两个质量档均超过 1MiB 时才缩小尺寸。同一 `variantId` 的并发生成共享一次变换。调用方可以单独取消等待,不会中断其他等待方;没有等待方时才会停止变换。适配器通过 `POST /files` 上传确切的派生请求字节,再发送 `{type: "file", file_id}` 块,不会回退到内联 data URL。每张保留图片前都有稳定文本,写明完整附件 ID 和实际请求尺寸。User、工具结果、agent loop、压缩和直接 `ctx.llm.stream` 请求都使用该投影。纯文本路由会收到稳定的附件占位文本,持久历史继续保留图片引用。 +支持图片的 catalog 配置项声明 `inputModalities: [text, image]`,并可设置 `imagePixelBudget`、`imageMaxBytes` 或 `imageDetail: low`。普通默认值为总像素 640,000、编码字节 1MiB;low detail 的默认总像素为 512×512。附件存储按 `min(1, sqrt(pixelBudget / (width * height)))` 缩放,并向预算内取整,确保总像素不超过硬上限。因此 2048×1024 规范化附件会得到约 1130×565 的请求版本,而不会被强制变成正方形。请求编码按需执行:低色数图片先尝试 PNG,只有不带 alpha 通道时才使用 palette,再尝试质量 85 和 80 的 WebP;其他透明图片依次尝试质量 85 和 80 的 WebP;其他非透明图片依次尝试质量 85 和 80 的 JPEG。两个质量档均超过 1MiB 时才缩小尺寸。同一 `variantId` 的并发生成共享一次变换。调用方可以单独取消等待,不会中断其他等待方;没有等待方时才会停止变换。适配器通常通过 `POST /files` 上传确切的派生请求字节,再发送 `{type: "file", file_id}` 块。File ID 解析失败或超时后,适配器会用相同请求版本的 base64 data URL 重新组装整个 chat 请求;同一请求不会混用 file ID 和内联图片。每张保留图片前都有稳定文本,写明完整附件 ID 和实际请求尺寸。User、工具结果、agent loop、压缩和直接 `ctx.llm.stream` 请求都使用该投影。纯文本路由会收到稳定的附件占位文本,持久历史继续保留图片引用。 `maxRequestFilesBytes` 和 `maxImagesPerRequest` 限制请求中保留的请求版本,默认值分别为 128MiB 和 600 张。字节和数量步长不得超过对应上限。读取附件前,适配器以路由的请求版本字节上限作为保守上界,移除超预算的最旧前缀,只读取并转换保留的规范化附件。系统随后用确切派生长度再次检查,但不会重新加入已省略图片。字节数越过上限时,被移除的最旧前缀会越过下一个 64MiB 边界。由 1MiB 图片组成的历史达到 129MiB 时会移除最旧的 65 张并保留 64MiB;直到持久历史超过 192MiB,这个前缀才再次变化。图片数量超限时则按 `imageOffloadCountQuantum` 独立递增。移除的图片会变成固定模型可见占位文本 `[image omitted to keep the request within its image limit; older images are omitted first. If this image is still needed, read its file again when a path is available; otherwise ask the user to attach it again.]`。这种定量投影不会因每新增一张图片就改写较早的请求前缀。 -上传 ID 按端点和 API key 作用域以及请求 `variantId` 记录在 `DSH_HOME` 下。变体身份覆盖规范化附件 ID、变换策略版本、路由像素和字节预算及编码参数,因此 Files API 和支持内联的适配器引用同一份确定性字节。上传默认请求 7 天有效期,并保存服务端返回的 `expires_at`。本地映射剩余时间不超过一小时时会在使用前替换;适配器不会在每次 chat 前查询远端文件。如果 chat 报告文件 ID 已过期、删除、缺失或无效,并指出本次请求使用的一个或多个 ID,适配器只删除这些映射。如果响应只说明文件状态失效而没有指出 ID,适配器会删除该次 chat 使用的全部文件映射。随后重新上传受影响的请求版本,并重试一次 chat。第二次 chat 仍报告文件失效时,适配器会按该响应清理映射并返回错误,不会发起第三次 chat。上传响应若没有完整文件对象、匹配的字节数和 `expires_at`,就不会写入索引;后续请求会再次上传,而不是信任不一致的本地状态。本地上传索引格式损坏时按空缓存处理,并由下一次成功上传替换;权限和文件系统 I/O 错误仍使请求失败。 +内联回退使用独立的 base64 预算。`maxInlineRequestImageBytes` 默认为 20MiB,`inlineImageOffloadByteQuantum` 默认为 10MiB,因此由 21 个 1MiB base64 负载组成的历史会移除最旧的 11 个并保留 10MiB。计算使用 base64 膨胀后的长度。系统逐字节复用已经准备好的请求版本;回退不会再次解码或压缩图片。前面图片已经成功写入的上传映射会保留,供后续请求复用。 + +上传 ID 按端点和 API key 作用域以及请求 `variantId` 记录在 `DSH_HOME` 下。变体身份覆盖规范化附件 ID、变换策略版本、路由像素和字节预算及编码参数,因此 Files API 和内联回退引用同一份确定性字节。上传默认请求 7 天有效期,并保存服务端返回的 `expires_at`。本地映射剩余时间不超过一小时时会在使用前替换;适配器不会在每次 chat 前查询远端文件。如果 chat 报告文件 ID 已过期、删除、缺失或无效,并指出本次请求使用的一个或多个 ID,适配器只删除这些映射。如果响应只说明文件状态失效而没有指出 ID,适配器会删除该次 chat 使用的全部文件映射。随后重新上传受影响的请求版本,并重试一次 chat。第二次 chat 仍报告文件失效时,适配器会按该响应清理映射并返回错误,不会发起第三次 chat。上传响应若没有完整文件对象、匹配的字节数和 `expires_at`,就不会写入索引;后续请求会再次上传,而不是信任不一致的本地状态。本地上传索引格式损坏时按空缓存处理,并由下一次成功上传替换。文件解析包括本地索引访问和远端上传,默认每张图片的时限为一分钟,且必须小于 `streamIdleTimeoutMs`。每次成功解析都会刷新外层 idle watchdog。任何解析失败都会把该请求切换到内联模式;显式公共文件管理操作仍会报告自身错误。 同一作用域和 `variantId` 的并发解析共享一次 Files 上传,每个等待方可以单独取消。一次上传配额错误会先分页收集配置数量的最旧 `dsh-` 文件,再删除这些文件并重试一次上传。`DeepSeekFilesClient.delete`、`DeepSeekFileStore.release` 和 `releaseAll` 提供主动远端空间回收。本包记录的当前提供方限制为 Files 单次上传 128MiB、chat 单图引用 32MiB、每个 API key 最多 10,000 个文件和 25GiB;默认 1MiB 请求版本低于两个单文件上限。 @@ -65,7 +70,7 @@ harness LLM(大语言模型)seam 的 DeepSeek chat-completions 适配器: `thinking: disabled` 是部署锁定:它只公布 `off`,并以 `off` 为默认值。省略 `reasoningEffort` 或将其配置为 `off` 均有效;配置 `low`、`high` 或 `max` 会使插件加载失败,直接按请求启用思考也会在网络 I/O 前失败。携带 `GenerateOptions.purpose: 'session-title'` 的请求也会强制禁用思考并省略已解析的推理强度,将有界输出保留给可见标题文本,不改变会话或压缩(compaction)默认值。 -`streamIdleTimeoutMs` 会限制每次未完成提供方读取,包括初始 `fetch`,但不计入消费方在分片间花费的时间。DeepSeek SSE 注释会作为传输活动使尚未完成的读取重新布防,但绝不会成为 `StreamChunk` 值或会话日志事件。同一个稳定的 abort 信号会在整个调用期间传递给请求与 body reader;过期会停止传输并抛出 `LlmError('TIMEOUT')`,较早的调用方 abort 则抛出 `LlmError('ABORTED')`。适配器通常每次 `stream()` 调用发起一次 chat 请求,只有上述失效文件恢复会发起第二次。适配器把已配置重试策略注册为提供方元数据,再由 `dsh-llm-retry` 在持久化的 agent(智能体)步骤边界单独执行该策略。 +`streamIdleTimeoutMs` 会限制每次未完成提供方读取,包括初始 `fetch`,但不计入消费方在分片间花费的时间。DeepSeek SSE 注释和成功的文件解析会作为传输活动使尚未完成的读取重新计时,但绝不会成为 `StreamChunk` 值或会话日志事件。同一个稳定的 abort 信号会在整个调用期间传递给请求与 body reader;过期会停止传输并抛出 `LlmError('TIMEOUT')`,较早的调用方 abort 则抛出 `LlmError('ABORTED')`。适配器通常每次 `stream()` 调用发起一次 chat 请求,只有失效文件恢复会发起第二次。首次 chat 前的文件解析失败会发送一次内联请求。如果失效文件响应后的替换解析失败,该内联请求就是唯一允许的重试。适配器把已配置重试策略注册为提供方元数据,再由 `dsh-llm-retry` 在持久化的 agent(智能体)步骤边界单独执行该策略。 ## 动态配置(settings + credentials) @@ -104,7 +109,7 @@ DeepSeek 请求身份独立于应用归因。凭据解析成功后,每个提 #### 模型看到的内容 -所选 DeepSeek 模型会收到 harness 系统提示词、消息历史、工具 schema、stop sequence 和调用配置。视觉模型会通过 Files API 引用收到保留的 user 与工具结果图片,旁边带有稳定附件句柄和请求图片尺寸;超出上限的较旧图片由已记录的占位文本表示。之前 assistant 轮次的推理内容会原文回传,无论该轮次是否调用了工具。 +所选 DeepSeek 模型会收到 harness 系统提示词、消息历史、工具 schema、stop sequence 和调用配置。视觉模型通常通过 Files API 引用收到保留的 user 与工具结果图片,旁边带有稳定附件句柄和请求图片尺寸;Files 解析失败时,所有保留图片改用内联 data URL。超出上限的较旧图片由已记录的占位文本表示。之前 assistant 轮次的推理内容会原文回传,无论该轮次是否调用了工具。 #### Token 影响 @@ -134,4 +139,4 @@ loop 保留的响应块会追加到下一个请求,并保留其较早可复用 - **未映射 `tool_choice`**:它不属于核心词汇(MVP 取舍,与 pi-ai twin 共享)。 - **请求使用原始 `fetch`,而非 `@cordisjs/plugin-http`**:没有共享 proxy/拦截配置;采用暂缓到第二个适配器需要该功能时(`TODO(http)`)。 - **会跳过插件添加的内容块类型**:核心文本与支持的图片块会被序列化,空工具输出会以字面 `(no output)` 通过协议发送。 -- **图片是仅输入的持久附件**:不支持直接外部 URL 和 assistant 图片输出;DeepSeek 图片输入使用 Files API。 +- **图片是仅输入的持久附件**:不支持直接外部 URL 和 assistant 图片输出;DeepSeek 图片输入通常使用 Files API,仅在单次请求恢复时使用内联 base64。 diff --git a/packages/llm/llm-deepseek/src/adapter.ts b/packages/llm/llm-deepseek/src/adapter.ts index 9c4756f3d3..8c30333131 100644 --- a/packages/llm/llm-deepseek/src/adapter.ts +++ b/packages/llm/llm-deepseek/src/adapter.ts @@ -28,7 +28,7 @@ import type { RequestImageAttachment, } from '@deepseek-ai/dsh-attachment' import type { CredentialRef } from '@deepseek-ai/dsh-credentials' -import { idleWatchdog, timeoutOf } from '@deepseek-ai/dsh-timeout' +import { deadline, idleWatchdog, timeoutOf } from '@deepseek-ai/dsh-timeout' import type { AnonymousUserId } from '@deepseek-ai/dsh-anonymous-user-id' import { serializeRequest, serializeRequestWithImages } from './serialize.ts' import type { ImageWireLocation, RequestDefaults } from './serialize.ts' @@ -37,7 +37,7 @@ import type { DeepSeekFilePolicy } from './file-store.ts' import type { DeepSeekFileId } from './file-id.ts' import { parseSse } from './sse.ts' import { translate } from './translate.ts' -import type { WireError } from './types.ts' +import type { WireError, WireRequest } from './types.ts' /** One optional model entry advertised by the direct-fetch adapter. */ export interface DeepSeekCatalogModel { @@ -89,12 +89,18 @@ export interface DeepSeekConnectionOptions { streamIdleTimeoutMs: number /** Maximum accumulated file-referenced image bytes in one request. */ maxRequestFilesBytes: number - /** Maximum number of file-referenced images in one request. */ + /** Maximum accumulated base64 image payload after Files API fallback. */ + maxInlineRequestImageBytes: number + /** Maximum number of represented images in one request. */ maxImagesPerRequest: number /** Raw-byte removal step after the file-reference bound is exceeded. */ imageOffloadByteQuantum: number + /** Base64-byte removal step after the inline fallback bound is exceeded. */ + inlineImageOffloadByteQuantum: number /** Image-count removal step after the count bound is exceeded. */ imageOffloadCountQuantum: number + /** Maximum duration of one request-image Files API resolution. */ + filesApiTimeoutMs: number /** Upload expiry, refresh, and quota-recovery policy. */ filePolicy: DeepSeekFilePolicy /** Provider-owned model-request retry policy, already resolved. */ @@ -128,6 +134,8 @@ export const DEFAULT_CONTEXT_WINDOW = 1_000_000 export const DEFAULT_MAX_TOKENS = 256_000 /** Default bound on accumulated file-referenced image bytes per request. */ export const DEFAULT_MAX_REQUEST_FILES_BYTES = 128 * 1024 * 1024 +/** Default bound on accumulated base64 image payload after Files API fallback. */ +export const DEFAULT_MAX_INLINE_REQUEST_IMAGE_BYTES = 20 * 1024 * 1024 /** Provider request image-count limit. */ export const DEFAULT_MAX_IMAGES_PER_REQUEST = 600 /** Total-pixel budget matching DeepSeek's normal vision projection. */ @@ -138,6 +146,8 @@ export const DEFAULT_LOW_DETAIL_IMAGE_PIXEL_BUDGET = 512 * 512 export const DEFAULT_REQUEST_IMAGE_MAX_BYTES = 1024 * 1024 /** Deterministic raw-byte removal step. */ export const DEFAULT_IMAGE_OFFLOAD_BYTE_QUANTUM = 64 * 1024 * 1024 +/** Deterministic base64-byte removal step after Files API fallback. */ +export const DEFAULT_INLINE_IMAGE_OFFLOAD_BYTE_QUANTUM = 10 * 1024 * 1024 /** Deterministic image-count removal step. */ export const DEFAULT_IMAGE_OFFLOAD_COUNT_QUANTUM = 20 /** Default explicit lifetime for uploaded images. */ @@ -146,7 +156,10 @@ export const DEFAULT_FILE_EXPIRY_SECONDS = 7 * 24 * 60 * 60 export const DEFAULT_FILE_REFRESH_MARGIN_SECONDS = 60 * 60 /** Default number of oldest harness-owned files removed on quota recovery. */ export const DEFAULT_FILE_QUOTA_CLEANUP_BATCH = 100 +/** Default deadline for resolving one request image through the Files API. */ +export const DEFAULT_FILES_API_TIMEOUT_MS = 60_000 const STREAM_IDLE_TIMEOUT_CODE = 'LLM_STREAM_IDLE_TIMEOUT' +const FILES_API_TIMEOUT_CODE = 'DEEPSEEK_FILES_API_TIMEOUT' const OFF_REASONING_EFFORT = ReasoningEffortId('off') const LOW_REASONING_EFFORT = ReasoningEffortId('low') const HIGH_REASONING_EFFORT = ReasoningEffortId('high') @@ -161,6 +174,14 @@ const OFF_ONLY_REASONING_EFFORTS = [ { id: OFF_REASONING_EFFORT, name: 'Off' }, ] as const +/** Marks a failed file-id resolution that may be retried as an inline request. */ +class FileResolutionFailure extends Error { + constructor(cause: unknown) { + super('DeepSeek Files API could not resolve a request image.', { cause }) + this.name = 'FileResolutionFailure' + } +} + function collectImageRefs( content: readonly ContentBlock[], refs: Map, @@ -494,7 +515,7 @@ export class DeepSeekAdapter extends LlmAdapter { apiKey: string, userId: AnonymousUserId, attachments: AttachmentStore | undefined, - onComment: () => void, + onActivity: () => void, ): AsyncIterable { const headers = { 'authorization': `Bearer ${apiKey}`, @@ -525,27 +546,58 @@ export class DeepSeekAdapter extends LlmAdapter { const requestImages = attachments === undefined || model === undefined ? new Map() : await prepareRequestImages(requestOptions, attachments, model, signal) - for (let fileAttempt = 0; fileAttempt < 2; fileAttempt += 1) { + let representation: 'file' | 'base64' = 'file' + let fileAttempt = 0 + while (true) { const usedFiles: UsedRequestFile[] = [] - const body = attachments === undefined - ? serializeRequest(requestOptions, connection.defaults) - : await serializeRequestWithImages(requestOptions, { + let body: WireRequest + if (attachments === undefined) { + body = serializeRequest(requestOptions, connection.defaults) + } else if (representation === 'base64') { + body = await serializeRequestWithImages(requestOptions, { + representation: { kind: 'base64' }, requestImages, - resolveFileId: async (version, _block, location) => { - const resolved = await this.files.ensureUploaded( - version, - fileConnection, - connection.filePolicy, - signal, - ) - usedFiles.push({ version, fileId: resolved.record.fileId, location }) - return resolved.record.fileId - }, - maxRequestFilesBytes: connection.maxRequestFilesBytes, + maxRequestImageBytes: connection.maxInlineRequestImageBytes, maxImagesPerRequest: connection.maxImagesPerRequest, - byteQuantum: connection.imageOffloadByteQuantum, + byteQuantum: connection.inlineImageOffloadByteQuantum, countQuantum: connection.imageOffloadCountQuantum, }, connection.defaults) + } else { + try { + body = await serializeRequestWithImages(requestOptions, { + representation: { + kind: 'file', + resolveFileId: async (version, _block, location) => { + using filesDeadline = deadline(signal, connection.filesApiTimeoutMs, FILES_API_TIMEOUT_CODE) + let resolved: Awaited> + try { + resolved = await this.files.ensureUploaded( + version, + fileConnection, + connection.filePolicy, + filesDeadline.signal, + ) + } catch (error: unknown) { + if (signal.aborted) throw error + throw new FileResolutionFailure(error) + } + onActivity() + usedFiles.push({ version, fileId: resolved.record.fileId, location }) + return resolved.record.fileId + }, + }, + requestImages, + maxRequestImageBytes: connection.maxRequestFilesBytes, + maxImagesPerRequest: connection.maxImagesPerRequest, + byteQuantum: connection.imageOffloadByteQuantum, + countQuantum: connection.imageOffloadCountQuantum, + }, connection.defaults) + } catch (error: unknown) { + if (!(error instanceof FileResolutionFailure)) throw error + representation = 'base64' + continue + } + } const payload = JSON.stringify(body) // TODO(http): adopt the Cordis HTTP service when shared transport configuration @@ -586,7 +638,10 @@ export class DeepSeekAdapter extends LlmAdapter { await Promise.all(staleMappings(usedFiles, detail).map(file => ( this.files.invalidate(file.version, file.fileId, fileConnection) ))) - if (fileAttempt === 0) continue + if (fileAttempt === 0) { + fileAttempt += 1 + continue + } } if (response.status === 400 && usedFiles.length > 0 && providerRejectedNormalizedImage(detail)) { message = normalizedImageDiagnostic(usedFiles, message, detail) @@ -604,7 +659,7 @@ export class DeepSeekAdapter extends LlmAdapter { throw new LlmError('DeepSeek API returned no response body', 'EMPTY_RESPONSE') } - yield* translate(parseSse(response.body, onComment)) + yield* translate(parseSse(response.body, onActivity)) return } } diff --git a/packages/llm/llm-deepseek/src/index.ts b/packages/llm/llm-deepseek/src/index.ts index 919168439d..e8632c22da 100644 --- a/packages/llm/llm-deepseek/src/index.ts +++ b/packages/llm/llm-deepseek/src/index.ts @@ -25,9 +25,12 @@ import { DEFAULT_FILE_EXPIRY_SECONDS, DEFAULT_FILE_QUOTA_CLEANUP_BATCH, DEFAULT_FILE_REFRESH_MARGIN_SECONDS, + DEFAULT_FILES_API_TIMEOUT_MS, DEFAULT_IMAGE_OFFLOAD_BYTE_QUANTUM, DEFAULT_IMAGE_OFFLOAD_COUNT_QUANTUM, + DEFAULT_INLINE_IMAGE_OFFLOAD_BYTE_QUANTUM, DEFAULT_LOW_DETAIL_IMAGE_PIXEL_BUDGET, + DEFAULT_MAX_INLINE_REQUEST_IMAGE_BYTES, DEFAULT_MAX_IMAGES_PER_REQUEST, DEFAULT_MAX_REQUEST_FILES_BYTES, DEFAULT_MAX_TOKENS, @@ -43,9 +46,12 @@ export { DEFAULT_FILE_EXPIRY_SECONDS, DEFAULT_FILE_QUOTA_CLEANUP_BATCH, DEFAULT_FILE_REFRESH_MARGIN_SECONDS, + DEFAULT_FILES_API_TIMEOUT_MS, DEFAULT_IMAGE_OFFLOAD_BYTE_QUANTUM, DEFAULT_IMAGE_OFFLOAD_COUNT_QUANTUM, + DEFAULT_INLINE_IMAGE_OFFLOAD_BYTE_QUANTUM, DEFAULT_LOW_DETAIL_IMAGE_PIXEL_BUDGET, + DEFAULT_MAX_INLINE_REQUEST_IMAGE_BYTES, DEFAULT_MAX_IMAGES_PER_REQUEST, DEFAULT_MAX_REQUEST_FILES_BYTES, DEFAULT_MAX_TOKENS, @@ -116,12 +122,18 @@ export interface Config { streamIdleTimeoutMs?: number /** Maximum accumulated file-referenced image bytes per chat request (default 128 MiB). */ maxRequestFilesBytes?: number - /** Maximum number of file-referenced images per chat request (default 600). */ + /** Maximum accumulated base64 image payload after Files API fallback (default 20 MiB). */ + maxInlineRequestImageBytes?: number + /** Maximum number of represented images per chat request (default 600). */ maxImagesPerRequest?: number /** Raw-byte removal step after the request exceeds its file bound (default 64 MiB). */ imageOffloadByteQuantum?: number + /** Base64-byte removal step after inline fallback exceeds its bound (default 10 MiB). */ + inlineImageOffloadByteQuantum?: number /** Image-count removal step after the request exceeds its count bound (default 20). */ imageOffloadCountQuantum?: number + /** Maximum duration of one request-image Files API resolution (default one minute). */ + filesApiTimeoutMs?: number /** Explicit lifetime assigned to each uploaded image (default seven days). */ fileExpiresAfterSeconds?: number /** Remaining lifetime below which an indexed file is replaced (default one hour). */ @@ -154,9 +166,12 @@ export const Config: z = z.object({ models: z.array(catalogModel).default(DEFAULT_MODELS), streamIdleTimeoutMs: z.number().min(Number.MIN_VALUE).max(MAX_TIMER_DELAY_MS).default(DEFAULT_STREAM_IDLE_TIMEOUT_MS), maxRequestFilesBytes: z.number().step(1).min(1).default(DEFAULT_MAX_REQUEST_FILES_BYTES), + maxInlineRequestImageBytes: z.number().step(1).min(1).default(DEFAULT_MAX_INLINE_REQUEST_IMAGE_BYTES), maxImagesPerRequest: z.number().step(1).min(1).default(DEFAULT_MAX_IMAGES_PER_REQUEST), imageOffloadByteQuantum: z.number().step(1).min(1).default(DEFAULT_IMAGE_OFFLOAD_BYTE_QUANTUM), + inlineImageOffloadByteQuantum: z.number().step(1).min(1).default(DEFAULT_INLINE_IMAGE_OFFLOAD_BYTE_QUANTUM), imageOffloadCountQuantum: z.number().step(1).min(1).default(DEFAULT_IMAGE_OFFLOAD_COUNT_QUANTUM), + filesApiTimeoutMs: z.number().min(Number.MIN_VALUE).max(MAX_TIMER_DELAY_MS).default(DEFAULT_FILES_API_TIMEOUT_MS), fileExpiresAfterSeconds: z.number().step(1).min(3_600).max(2_592_000).default(DEFAULT_FILE_EXPIRY_SECONDS), fileRefreshMarginSeconds: z.number().step(1).min(0).default(DEFAULT_FILE_REFRESH_MARGIN_SECONDS), fileQuotaCleanupBatch: z.number().step(1).min(1).max(1_000).default(DEFAULT_FILE_QUOTA_CLEANUP_BATCH), @@ -283,6 +298,10 @@ export function resolveAdapterOptions(config: Config, environment?: LaunchEnviro if (!Number.isSafeInteger(maxRequestFilesBytes) || maxRequestFilesBytes <= 0) { throw new Error('llm-deepseek: maxRequestFilesBytes must be a positive safe integer') } + const maxInlineRequestImageBytes = config.maxInlineRequestImageBytes ?? DEFAULT_MAX_INLINE_REQUEST_IMAGE_BYTES + if (!Number.isSafeInteger(maxInlineRequestImageBytes) || maxInlineRequestImageBytes <= 0) { + throw new Error('llm-deepseek: maxInlineRequestImageBytes must be a positive safe integer') + } const maxImagesPerRequest = config.maxImagesPerRequest ?? DEFAULT_MAX_IMAGES_PER_REQUEST if (!Number.isSafeInteger(maxImagesPerRequest) || maxImagesPerRequest <= 0) { throw new Error('llm-deepseek: maxImagesPerRequest must be a positive safe integer') @@ -294,6 +313,14 @@ export function resolveAdapterOptions(config: Config, environment?: LaunchEnviro if (imageOffloadByteQuantum > maxRequestFilesBytes) { throw new Error('llm-deepseek: imageOffloadByteQuantum must not exceed maxRequestFilesBytes') } + const inlineImageOffloadByteQuantum = config.inlineImageOffloadByteQuantum + ?? DEFAULT_INLINE_IMAGE_OFFLOAD_BYTE_QUANTUM + if (!Number.isSafeInteger(inlineImageOffloadByteQuantum) || inlineImageOffloadByteQuantum <= 0) { + throw new Error('llm-deepseek: inlineImageOffloadByteQuantum must be a positive safe integer') + } + if (inlineImageOffloadByteQuantum > maxInlineRequestImageBytes) { + throw new Error('llm-deepseek: inlineImageOffloadByteQuantum must not exceed maxInlineRequestImageBytes') + } const imageOffloadCountQuantum = config.imageOffloadCountQuantum ?? DEFAULT_IMAGE_OFFLOAD_COUNT_QUANTUM if (!Number.isSafeInteger(imageOffloadCountQuantum) || imageOffloadCountQuantum <= 0) { throw new Error('llm-deepseek: imageOffloadCountQuantum must be a positive safe integer') @@ -301,6 +328,17 @@ export function resolveAdapterOptions(config: Config, environment?: LaunchEnviro if (imageOffloadCountQuantum > maxImagesPerRequest) { throw new Error('llm-deepseek: imageOffloadCountQuantum must not exceed maxImagesPerRequest') } + const filesApiTimeoutMs = config.filesApiTimeoutMs ?? DEFAULT_FILES_API_TIMEOUT_MS + if (!Number.isFinite(filesApiTimeoutMs) + || filesApiTimeoutMs <= 0 + || filesApiTimeoutMs > MAX_TIMER_DELAY_MS) { + throw new Error( + `llm-deepseek: filesApiTimeoutMs must be a positive finite number no greater than ${MAX_TIMER_DELAY_MS}`, + ) + } + if (filesApiTimeoutMs >= streamIdleTimeoutMs) { + throw new Error('llm-deepseek: filesApiTimeoutMs must be below streamIdleTimeoutMs') + } const fileExpiresAfterSeconds = config.fileExpiresAfterSeconds ?? DEFAULT_FILE_EXPIRY_SECONDS if (!Number.isSafeInteger(fileExpiresAfterSeconds) || fileExpiresAfterSeconds < 3_600 @@ -333,9 +371,12 @@ export function resolveAdapterOptions(config: Config, environment?: LaunchEnviro models: resolveModels(config.models), streamIdleTimeoutMs, maxRequestFilesBytes, + maxInlineRequestImageBytes, maxImagesPerRequest, imageOffloadByteQuantum, + inlineImageOffloadByteQuantum, imageOffloadCountQuantum, + filesApiTimeoutMs, filePolicy: { expiresAfterSeconds: fileExpiresAfterSeconds, refreshMarginSeconds: fileRefreshMarginSeconds, diff --git a/packages/llm/llm-deepseek/src/serialize.ts b/packages/llm/llm-deepseek/src/serialize.ts index 4ac6280cd4..3b22967d96 100644 --- a/packages/llm/llm-deepseek/src/serialize.ts +++ b/packages/llm/llm-deepseek/src/serialize.ts @@ -1,7 +1,7 @@ /** * Serialize harness messages into DeepSeek chat completions. Text-only * requests retain string user content; the image path resolves durable - * attachments into ordered Files API parts. Tool-result images follow their + * attachments into ordered file-id or inline parts. Tool-result images follow their * string-only tool messages in a separate user message. * @module dsh-llm-deepseek/serialize */ @@ -10,7 +10,7 @@ import { contentHasImage, LlmError, offloadRequestImagesWithPolicy, requestImage import type { ContentBlock, GenerateOptions, Message } from '@deepseek-ai/dsh-llm' import type { ImageAttachmentRef, RequestImageAttachment } from '@deepseek-ai/dsh-attachment' import type { - WireFileContentPart, + WireImageContentPart, WireMessage, WireRequest, WireTextContentPart, @@ -29,21 +29,30 @@ interface ResolvedThinking { reasoningEffort?: 'low' | 'high' | 'max' } +/** Provider representation for every retained image in one request. */ +export type ImageRequestRepresentation = + | { + kind: 'file' + /** Resolve a retained request version to a reusable DeepSeek file id. */ + resolveFileId: ( + version: RequestImageAttachment, + block: Extract, + location: ImageWireLocation, + ) => Promise + } + | { kind: 'base64' } + /** Dependencies required only when the request contains image input. */ export interface ImageSerializationOptions { - /** Resolve a retained request version to a reusable DeepSeek file id. */ - resolveFileId: ( - version: RequestImageAttachment, - block: Extract, - location: ImageWireLocation, - ) => Promise - /** Request versions prepared for the conservatively retained masters, keyed by attachment id. */ + /** One representation used for every retained image in this request. */ + representation: ImageRequestRepresentation + /** Request versions prepared for the conservatively retained normalized attachments, keyed by attachment id. */ requestImages: ReadonlyMap - /** Positive bound on accumulated referenced image bytes. */ - maxRequestFilesBytes: number - /** Maximum referenced images in one request. */ + /** Positive bound on accumulated represented image bytes. */ + maxRequestImageBytes: number + /** Maximum represented images in one request. */ maxImagesPerRequest?: number - /** Raw-byte removal step applied after the request exceeds its byte bound. */ + /** Represented-byte removal step applied after the request exceeds its byte bound. */ byteQuantum?: number /** Image-count removal step applied after the request exceeds its count bound. */ countQuantum?: number @@ -125,13 +134,13 @@ function imageHandle( } } -/** Resolve one durable image into its descriptor and transient DeepSeek file-id part. */ +/** Resolve one durable image into its descriptor and transient DeepSeek image part. */ async function imageParts( block: Extract, images: ImageSerializationOptions, location: ImageWireLocation, precededByContent: boolean, -): Promise<[WireTextContentPart, WireFileContentPart]> { +): Promise<[WireTextContentPart, WireImageContentPart]> { const version = images.requestImages.get(block.attachment.attachmentId) if (version === undefined) { throw new LlmError( @@ -139,10 +148,13 @@ async function imageParts( 'INVALID_REQUEST', ) } - return [ - imageHandle(version, precededByContent), - { type: 'file', file_id: await images.resolveFileId(version, block, location) }, - ] + const image: WireImageContentPart = images.representation.kind === 'file' + ? { type: 'file', file_id: await images.representation.resolveFileId(version, block, location) } + : { + type: 'image_url', + image_url: { url: `data:${version.mediaType};base64,${Buffer.from(version.data).toString('base64')}` }, + } + return [imageHandle(version, precededByContent), image] } /** Convert user or nested tool-result blocks into ordered wire parts. */ @@ -177,7 +189,7 @@ async function contentParts( function userContent(parts: readonly WireUserContentPart[]): string | WireUserContentPart[] { const text: string[] = [] for (const part of parts) { - if (part.type === 'file') return [...parts] + if (part.type !== 'text') return [...parts] text.push(part.text) } return text.join('') @@ -263,7 +275,7 @@ export function serializeMessages(messages: Message[]): WireMessage[] { * Consecutive tool results keep string `tool` messages and share one following * user message containing their images. * @param messages - transient request history after request-size offloading. - * @param images - prepared request versions and reusable provider file-id resolver. + * @param images - prepared request versions, one provider representation, and its budget. * @returns ordered DeepSeek wire messages. */ export async function serializeMessagesWithImages( @@ -272,7 +284,7 @@ export async function serializeMessagesWithImages( ): Promise { assertSupportedImageRoles(messages) const wire: WireMessage[] = [] - let pendingToolImages: WireFileContentPart[] = [] + let pendingToolImages: WireImageContentPart[] = [] const flushToolImages = (): void => { if (pendingToolImages.length === 0) return wire.push({ @@ -309,14 +321,14 @@ export async function serializeMessagesWithImages( } for (const result of toolResults) { const parts = await contentParts(result.content, images, messageIndex + 1, nextImage) - const fileParts = parts.filter((part): part is WireFileContentPart => part.type === 'file') + const imageParts = parts.filter((part): part is WireImageContentPart => part.type !== 'text') const text = parts.filter(part => part.type === 'text').map(part => part.text).join('') wire.push({ role: 'tool', tool_call_id: result.toolCallId, content: text || '(no output)', }) - pendingToolImages.push(...fileParts) + pendingToolImages.push(...imageParts) } } flushToolImages() @@ -378,7 +390,7 @@ export function serializeRequest( /** * Build one image-capable request while keeping durable bytes out of session * messages. Oversized oldest images become deterministic text after their - * exact request-version byte lengths are known and before provider upload. + * exact request-version byte lengths are known and before provider serialization. * @param options - harness request containing image-capable user content. * @param images - attachment resolver, request bound, and cancellation. * @param defaults - adapter-level thinking defaults. @@ -391,7 +403,7 @@ export async function serializeRequestWithImages( ): Promise { assertSupportedImageRoles(options.messages) const requestMessages = offloadRequestImagesWithPolicy(options.messages, { - representation: 'raw', + representation: images.representation.kind === 'file' ? 'raw' : 'base64', byteLength: (ref) => { const version = images.requestImages.get(ref.attachmentId) if (version === undefined) { @@ -399,7 +411,7 @@ export async function serializeRequestWithImages( } return version.bytes }, - maxBytes: images.maxRequestFilesBytes, + maxBytes: images.maxRequestImageBytes, ...images.maxImagesPerRequest === undefined ? {} : { maxImages: images.maxImagesPerRequest }, ...images.byteQuantum === undefined ? {} : { byteQuantum: images.byteQuantum }, ...images.countQuantum === undefined ? {} : { countQuantum: images.countQuantum }, diff --git a/packages/llm/llm-deepseek/src/types.ts b/packages/llm/llm-deepseek/src/types.ts index 54f39b095b..f5dd5df0aa 100644 --- a/packages/llm/llm-deepseek/src/types.ts +++ b/packages/llm/llm-deepseek/src/types.ts @@ -47,8 +47,17 @@ export interface WireFileContentPart { file_id: string } +/** Inline base64 data URL inside a multimodal user message. */ +export interface WireImageUrlContentPart { + type: 'image_url' + image_url: { url: string } +} + +/** One image representation accepted by a multimodal user message. */ +export type WireImageContentPart = WireFileContentPart | WireImageUrlContentPart + /** Ordered input part accepted by a multimodal user message. */ -export type WireUserContentPart = WireTextContentPart | WireFileContentPart +export type WireUserContentPart = WireTextContentPart | WireImageContentPart /** User-role message: text-only string or ordered multimodal input. */ export interface WireUserMessage { diff --git a/packages/llm/llm-deepseek/tests/adapter.spec.ts b/packages/llm/llm-deepseek/tests/adapter.spec.ts index baac1ee39e..5ef87231bb 100644 --- a/packages/llm/llm-deepseek/tests/adapter.spec.ts +++ b/packages/llm/llm-deepseek/tests/adapter.spec.ts @@ -8,6 +8,7 @@ import type { AttachmentStore, ImageAttachmentRef, RequestImageAttachment } from import { createLaunchEnvironmentSnapshot } from '@deepseek-ai/dsh-launch-environment' import LlmRuntime, { CallId, createUserMessage, CONTEXT_WINDOW_EXCEEDED_CODE, + LlmError, ProviderRequestId, QUOTA_EXCEEDED_CODE, ReasoningEffortId, @@ -52,6 +53,7 @@ async function harness(baseURL: string, config: object = {}) { function adapterOf( config: Partial & { apiKey?: string } = {}, attachments?: AttachmentStore, + files?: LlmDeepSeek.DeepSeekFileStore, ): DeepSeekAdapter { const { apiKey, ...rest } = config return new DeepSeekAdapter({ @@ -59,6 +61,7 @@ function adapterOf( resolveApiKey: () => Promise.resolve(apiKey ?? 'k'), resolveUserId: () => TEST_USER_ID, resolveAttachments: () => attachments, + ...files === undefined ? {} : { resolveFiles: () => files }, }) } @@ -102,6 +105,32 @@ function attachmentStoreOf( } } +function fileStoreOf( + implementation: (...args: Parameters) => ReturnType, +) { + const ensureUploaded = vi.fn(implementation) + const invalidate = vi.fn(() => Promise.resolve()) + return { + store: { ensureUploaded, invalidate } as unknown as LlmDeepSeek.DeepSeekFileStore, + ensureUploaded, + invalidate, + } +} + +function fileReference(fileId: string): Awaited> { + return { + record: { fileId: LlmDeepSeek.DeepSeekFileId(fileId) }, + uploaded: true, + } as Awaited> +} + +function successfulSseResponse(): Response { + return new Response(textEvents.map(event => `data: ${event}\n\n`).join(''), { + status: 200, + headers: { 'content-type': 'text/event-stream' }, + }) +} + describe('request image policy', () => { it.each([ [ @@ -199,6 +228,188 @@ describe('DeepSeekAdapter against a mock server', () => { expect(policies).toEqual([{ maxPixels: 640_000, maxBytes: 1024 * 1024 }]) }) + it('falls back to one all-base64 request when Files API resolution fails', async () => { + const server = await mockServer([{ kind: 'sse', events: textEvents }]) + const secondRef = { ...imageRef, attachmentId: AttachmentId(`sha256:${'c'.repeat(64)}`) } + const attachments = attachmentStoreOf(ref => Promise.resolve({ + ...requestImage(ref), + variantId: ImageVariantId(`sha256:${(ref.attachmentId === imageRef.attachmentId ? 'b' : 'd').repeat(64)}`), + })).store + const files = fileStoreOf(() => Promise.reject(new LlmError('Files unavailable', 'SERVER'))) + const adapter = adapterOf({ + baseURL: server.url, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachments, files.store) + + await drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [ + { type: 'image', attachment: imageRef }, + { type: 'image', attachment: secondRef }, + ], + source: { kind: 'plugin', plugin: 'test' }, + })], + })) + + const body = server.requests[0] as { messages: Array<{ content: unknown }> } + expect(JSON.stringify(body.messages[0]?.content).match(/"type":"image_url"/g)).toHaveLength(2) + expect(JSON.stringify(body)).not.toContain('file_id') + expect(files.ensureUploaded).toHaveBeenCalledTimes(1) + }) + + it('reduces base64 fallback history from the configured high watermark to its half-size quantum', async () => { + const server = await mockServer([{ kind: 'sse', events: textEvents }]) + const attachments = attachmentStoreOf(ref => Promise.resolve(requestImage(ref))).store + const files = fileStoreOf(() => Promise.reject(new LlmError('Files unavailable', 'SERVER'))) + const adapter = adapterOf({ + baseURL: server.url, + maxInlineRequestImageBytes: 80, + inlineImageOffloadByteQuantum: 40, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachments, files.store) + + await drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: Array.from({ length: 21 }, () => ({ type: 'image' as const, attachment: imageRef })), + source: { kind: 'plugin', plugin: 'test' }, + })], + })) + + const body = JSON.stringify(server.requests[0]) + expect(body.match(/older images are omitted first/g)).toHaveLength(11) + expect(body.match(/"type":"image_url"/g)).toHaveLength(10) + }) + + it('discards partially resolved file ids and falls back with every retained image inline', async () => { + const server = await mockServer([{ kind: 'sse', events: textEvents }]) + const secondRef = { ...imageRef, attachmentId: AttachmentId(`sha256:${'c'.repeat(64)}`) } + const attachments = attachmentStoreOf(ref => Promise.resolve({ + ...requestImage(ref), + variantId: ImageVariantId(`sha256:${(ref.attachmentId === imageRef.attachmentId ? 'b' : 'd').repeat(64)}`), + })).store + const files = fileStoreOf(() => Promise.reject(new Error('unused'))) + files.ensureUploaded + .mockResolvedValueOnce(fileReference('file-api-partial')) + .mockRejectedValueOnce(new LlmError('Files unavailable', 'TRANSPORT')) + const adapter = adapterOf({ + baseURL: server.url, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachments, files.store) + + await drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [ + { type: 'image', attachment: imageRef }, + { type: 'image', attachment: secondRef }, + ], + source: { kind: 'plugin', plugin: 'test' }, + })], + })) + + const body = server.requests[0] as { messages: Array<{ content: unknown }> } + expect(JSON.stringify(body.messages[0]?.content).match(/"type":"image_url"/g)).toHaveLength(2) + expect(JSON.stringify(body)).not.toContain('file-api-partial') + }) + + it('falls back after the configured Files API deadline without aborting chat', async () => { + vi.useFakeTimers() + const started = Promise.withResolvers() + const files = fileStoreOf((_version, _connection, _policy, signal) => new Promise((_resolve, reject) => { + started.resolve(undefined) + signal?.addEventListener('abort', () => { + const reason: unknown = signal.reason + reject(reason instanceof Error ? reason : new Error('files operation aborted')) + }, { once: true }) + })) + const attachments = attachmentStoreOf(ref => Promise.resolve(requestImage(ref))).store + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue(successfulSseResponse()) + const adapter = adapterOf({ + baseURL: 'https://deepseek.invalid', + filesApiTimeoutMs: 50, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachments, files.store) + + const pending = drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [{ type: 'image', attachment: imageRef }], + source: { kind: 'plugin', plugin: 'test' }, + })], + })) + await started.promise + await vi.advanceTimersByTimeAsync(50) + await pending + + expect(fetchSpy).toHaveBeenCalledTimes(1) + expect(String(fetchSpy.mock.calls[0]?.[1]?.body)).toContain('image_url') + fetchSpy.mockRestore() + }) + + it('does not turn caller cancellation during file resolution into base64 fallback', async () => { + const started = Promise.withResolvers() + const files = fileStoreOf((_version, _connection, _policy, signal) => new Promise((_resolve, reject) => { + started.resolve(undefined) + signal?.addEventListener('abort', () => { reject(new Error('cancelled')) }, { once: true }) + })) + const attachments = attachmentStoreOf(ref => Promise.resolve(requestImage(ref))).store + const fetchSpy = vi.spyOn(globalThis, 'fetch') + const controller = new AbortController() + const adapter = adapterOf({ + baseURL: 'https://deepseek.invalid', + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachments, files.store) + + const pending = drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + signal: controller.signal, + messages: [createUserMessage({ + content: [{ type: 'image', attachment: imageRef }], + source: { kind: 'plugin', plugin: 'test' }, + })], + })) + await started.promise + controller.abort() + + await expect(pending).rejects.toMatchObject({ code: 'ABORTED' }) + expect(fetchSpy).not.toHaveBeenCalled() + fetchSpy.mockRestore() + }) + + it('does not retry a generic chat failure through base64 fallback', async () => { + const server = await mockServer([{ + kind: 'http-error', + status: 503, + body: JSON.stringify({ error: { message: 'chat unavailable' } }), + }]) + const attachments = attachmentStoreOf(ref => Promise.resolve(requestImage(ref))).store + const files = fileStoreOf(() => Promise.resolve(fileReference('file-api-ready'))) + const adapter = adapterOf({ + baseURL: server.url, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachments, files.store) + + await expect(drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [{ type: 'image', attachment: imageRef }], + source: { kind: 'plugin', plugin: 'test' }, + })], + }))).rejects.toMatchObject({ code: 'SERVER', message: 'chat unavailable' }) + + expect(server.requests).toHaveLength(1) + expect(JSON.stringify(server.requests[0])).toContain('file-api-ready') + expect(JSON.stringify(server.requests[0])).not.toContain('image_url') + }) + it('does not prepare an old image removed by request offload', async () => { const server = await mockServer([{ kind: 'sse', events: textEvents }]) const old = { ...imageRef, attachmentId: AttachmentId(`sha256:${'c'.repeat(64)}`), bytes: 3 } @@ -454,6 +665,40 @@ describe('DeepSeekAdapter against a mock server', () => { expect(attachmentMocks.readImageRequest).toHaveBeenCalledTimes(1) }) + it('uses inline fallback when stale-id recovery cannot resolve a replacement file', async () => { + const server = await mockServer([ + { + kind: 'http-error', + status: 400, + body: JSON.stringify({ error: { message: 'file_id file-api-stale expired' } }), + }, + { kind: 'sse', events: textEvents }, + ]) + const attachments = attachmentStoreOf(ref => Promise.resolve(requestImage(ref))).store + const files = fileStoreOf(() => Promise.reject(new Error('unused'))) + files.ensureUploaded + .mockResolvedValueOnce(fileReference('file-api-stale')) + .mockRejectedValueOnce(new LlmError('Files unavailable', 'SERVER')) + const adapter = adapterOf({ + baseURL: server.url, + models: [{ id: 'deepseek-v4-flash-vision-exp', inputModalities: ['text', 'image'] }], + }, attachments, files.store) + + await drain(adapter.stream({ + provider: 'deepseek-official', + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [{ type: 'image', attachment: imageRef }], + source: { kind: 'plugin', plugin: 'test' }, + })], + })) + + expect(files.invalidate).toHaveBeenCalledTimes(1) + expect(server.requests).toHaveLength(2) + expect(JSON.stringify(server.requests[0])).toContain('file-api-stale') + expect(JSON.stringify(server.requests[1])).toContain('image_url') + }) + it('invalidates only the identified mapping when a multi-image request names one stale file id', async () => { const secondRef: ImageAttachmentRef = { ...imageRef, @@ -1150,7 +1395,11 @@ describe('DeepSeekAdapter against a mock server', () => { }) return Promise.resolve(new Response(body, { status: 200 })) }) - const adapter = adapterOf({ baseURL: 'https://example.invalid', streamIdleTimeoutMs: 100 }) + const adapter = adapterOf({ + baseURL: 'https://example.invalid', + filesApiTimeoutMs: 50, + streamIdleTimeoutMs: 100, + }) try { const drain = (async () => { for await (const _chunk of adapter.stream({ provider: 'deepseek-official', model: 'm', messages: [] })) { /* drain */ } @@ -1181,7 +1430,11 @@ describe('DeepSeekAdapter against a mock server', () => { }) return Promise.resolve(new Response(body, { status: 200 })) }) - const adapter = adapterOf({ baseURL: 'https://example.invalid', streamIdleTimeoutMs: 100 }) + const adapter = adapterOf({ + baseURL: 'https://example.invalid', + filesApiTimeoutMs: 50, + streamIdleTimeoutMs: 100, + }) try { const chunks: string[] = [] const drain = (async () => { @@ -1573,6 +1826,10 @@ describe('plugin registration and config', () => { maxRequestFilesBytes: 10, imageOffloadByteQuantum: 11, })).toThrow(/imageOffloadByteQuantum must not exceed maxRequestFilesBytes/) + expect(() => resolveAdapterOptions({ + maxInlineRequestImageBytes: 10, + inlineImageOffloadByteQuantum: 11, + })).toThrow(/inlineImageOffloadByteQuantum must not exceed maxInlineRequestImageBytes/) expect(() => resolveAdapterOptions({ maxImagesPerRequest: 10, imageOffloadCountQuantum: 11, @@ -1584,6 +1841,8 @@ describe('plugin registration and config', () => { ['maxImagesPerRequest', 1.5, /maxImagesPerRequest must be a positive safe integer/], ['imageOffloadByteQuantum', 0, /imageOffloadByteQuantum must be a positive safe integer/], ['imageOffloadByteQuantum', Number.MAX_SAFE_INTEGER + 1, /imageOffloadByteQuantum must be a positive safe integer/], + ['inlineImageOffloadByteQuantum', 0, /inlineImageOffloadByteQuantum must be a positive safe integer/], + ['inlineImageOffloadByteQuantum', Number.MAX_SAFE_INTEGER + 1, /inlineImageOffloadByteQuantum must be a positive safe integer/], ['imageOffloadCountQuantum', 0, /imageOffloadCountQuantum must be a positive safe integer/], ['imageOffloadCountQuantum', 1.5, /imageOffloadCountQuantum must be a positive safe integer/], ['fileExpiresAfterSeconds', 3_599, /fileExpiresAfterSeconds must be an integer from 3600 through 2592000/], @@ -1612,6 +1871,22 @@ describe('plugin registration and config', () => { }, ) + it.each([0, 1.5, Number.MAX_SAFE_INTEGER + 1])( + 'rejects invalid inline request image bound %s', + async (maxInlineRequestImageBytes) => { + expect(() => resolveAdapterOptions({ maxInlineRequestImageBytes })) + .toThrow(/maxInlineRequestImageBytes must be a positive safe integer/) + + const ctx = new Context() + await ctx.plugin(LlmRuntime) + await expect(ctx.plugin(LlmDeepSeek, { + baseURL: 'http://127.0.0.1:1', + maxInlineRequestImageBytes, + })).rejects.toThrow(/maxInlineRequestImageBytes/) + expect(ctx.llm.listProviders()).toEqual([]) + }, + ) + it('falls back to DEEPSEEK_API_KEY and DEEPSEEK_BASE_URL env vars', async () => { vi.stubEnv('DEEPSEEK_API_KEY', 'env-key') vi.stubEnv('DEEPSEEK_BASE_URL', 'http://127.0.0.1:1') @@ -1753,6 +2028,26 @@ describe('plugin registration and config', () => { })).rejects.toThrow(/streamIdleTimeoutMs/) }) + it('rejects invalid Files API timeout bounds for direct and plugin composition', async () => { + expect(() => resolveAdapterOptions({ filesApiTimeoutMs: Number.POSITIVE_INFINITY })) + .toThrow(/filesApiTimeoutMs.*positive finite/) + expect(() => resolveAdapterOptions({ filesApiTimeoutMs: MAX_TIMER_DELAY_MS + 1 })) + .toThrow(/filesApiTimeoutMs.*no greater/) + + const ctx = new Context() + await ctx.plugin(LlmRuntime) + await expect(ctx.plugin(LlmDeepSeek, { + baseURL: 'http://127.0.0.1:1', + filesApiTimeoutMs: 0, + })).rejects.toThrow(/filesApiTimeoutMs/) + await expect(ctx.plugin(LlmDeepSeek, { + baseURL: 'http://127.0.0.1:1', + filesApiTimeoutMs: MAX_TIMER_DELAY_MS + 1, + })).rejects.toThrow(/filesApiTimeoutMs/) + expect(() => resolveAdapterOptions({ filesApiTimeoutMs: 100, streamIdleTimeoutMs: 100 })) + .toThrow(/filesApiTimeoutMs must be below streamIdleTimeoutMs/) + }) + it('rejects invalid nested retryPolicy before registering the provider', async () => { const ctx = new Context() await ctx.plugin(LlmRuntime) diff --git a/packages/llm/llm-deepseek/tests/serialize.spec.ts b/packages/llm/llm-deepseek/tests/serialize.spec.ts index 547713a74c..968ceabdaf 100644 --- a/packages/llm/llm-deepseek/tests/serialize.spec.ts +++ b/packages/llm/llm-deepseek/tests/serialize.spec.ts @@ -11,6 +11,8 @@ import { } from '../src/serialize.ts' import type { ImageSerializationOptions } from '../src/serialize.ts' +type FileResolver = Extract['resolveFileId'] + function request(overrides: Partial = {}): GenerateOptions { return { provider: 'deepseek-official', model: 'deepseek-v4-flash', messages: [], ...overrides } } @@ -32,7 +34,7 @@ function imageRef(mediaType: ImageMediaType = 'image/png', bytes = 3): ImageAtta } function fileResolver(id = 'file-api-image') { - return vi.fn(() => Promise.resolve(id)) + return vi.fn(() => Promise.resolve(id)) } function requestVersion(ref: ImageAttachmentRef): RequestImageAttachment { @@ -53,13 +55,26 @@ function requestVersion(ref: ImageAttachmentRef): RequestImageAttachment { function imageOptions( refs: readonly ImageAttachmentRef[], - resolveFileId: ImageSerializationOptions['resolveFileId'] = fileResolver(), - maxRequestFilesBytes = 20 * 1024 * 1024, + resolveFileId: FileResolver = fileResolver(), + maxRequestImageBytes = 20 * 1024 * 1024, ) { return { - resolveFileId, + representation: { kind: 'file' as const, resolveFileId }, requestImages: new Map(refs.map(ref => [ref.attachmentId, requestVersion(ref)])), - maxRequestFilesBytes, + maxRequestImageBytes, + } +} + +function inlineImageOptions( + refs: readonly ImageAttachmentRef[], + maxRequestImageBytes = 20 * 1024 * 1024, + byteQuantum = 10 * 1024 * 1024, +): ImageSerializationOptions { + return { + representation: { kind: 'base64' }, + requestImages: new Map(refs.map(ref => [ref.attachmentId, requestVersion(ref)])), + maxRequestImageBytes, + byteQuantum, } } @@ -359,6 +374,30 @@ describe('image serialization', () => { }]) }) + it.each([ + ['image/png', 'data:image/png;base64,AAAA'], + ['image/jpeg', 'data:image/jpeg;base64,AAAA'], + ['image/webp', 'data:image/webp;base64,AAAA'], + ['image/gif', 'data:image/gif;base64,AAAA'], + ] as const)('serializes every retained %s request version as an inline data URL', async (mediaType, url) => { + const ref = imageRef(mediaType) + const wire = await serializeRequestWithImages(request({ + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: [{ type: 'image', attachment: ref }], + source: { kind: 'plugin', plugin: 'test' }, + })], + }), inlineImageOptions([ref])) + + expect(wire.messages).toEqual([{ + role: 'user', + content: [ + { type: 'text', text: `Image ${ref.attachmentId}; request image 1x1px.` }, + { type: 'image_url', image_url: { url } }, + ], + }]) + }) + it('gives image-only input a stable handle and request dimensions', async () => { const ref = imageRef() const wire = await serializeRequestWithImages(request({ @@ -548,6 +587,21 @@ describe('image serialization', () => { expect(resolveFileId.mock.calls[0]?.[0]).toMatchObject({ attachment: { mediaType: 'image/jpeg' } }) }) + it('drops base64 history from a 20-unit high watermark to a 10-unit low watermark', async () => { + const ref = imageRef('image/png', 3) + const wire = await serializeRequestWithImages(request({ + model: 'deepseek-v4-flash-vision-exp', + messages: [createUserMessage({ + content: Array.from({ length: 21 }, () => ({ type: 'image' as const, attachment: ref })), + source: { kind: 'plugin', plugin: 'test' }, + })], + }), inlineImageOptions([ref], 80, 40)) + + const content = wire.messages[0]?.content + expect(JSON.stringify(content).match(/older images are omitted first/g)).toHaveLength(11) + expect(JSON.stringify(content).match(/"type":"image_url"/g)).toHaveLength(10) + }) + it('rejects an unprepared image while computing exact request bytes', async () => { const ref = imageRef() await expect(serializeRequestWithImages(request({ From d618bfebb4411b5af36e4f9203bd0457a962d496 Mon Sep 17 00:00:00 2001 From: creatixchu Date: Fri, 21 Aug 2026 18:34:16 +0800 Subject: [PATCH 28/28] fix(deepseek): decouple files and stream timeouts --- ...2026-08-21-deepseek-files-inline-fallback.i18n.yaml | 4 ++-- .../2026-08-21-deepseek-files-inline-fallback.md | 4 ++-- .../2026-08-21-deepseek-files-inline-fallback.zh.md | 4 ++-- packages/llm/llm-deepseek/README.i18n.yaml | 4 ++-- packages/llm/llm-deepseek/README.md | 4 ++-- packages/llm/llm-deepseek/README.zh.md | 4 ++-- packages/llm/llm-deepseek/src/index.ts | 3 --- packages/llm/llm-deepseek/tests/adapter.spec.ts | 10 ++++------ 8 files changed, 16 insertions(+), 21 deletions(-) diff --git a/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.i18n.yaml index ed4af5577d..c4f148394e 100644 --- a/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.i18n.yaml +++ b/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.md -2026-08-21-deepseek-files-inline-fallback.md: c58b3e2257b426f1b5df8a4d6952e890a2bd2982 -2026-08-21-deepseek-files-inline-fallback.zh.md: 34625c6250d52a73ccaac3e33adbd2ed099aab5b +2026-08-21-deepseek-files-inline-fallback.md: 7442089038e2cf47f37661c0f098054d03f67aef +2026-08-21-deepseek-files-inline-fallback.zh.md: 0534514f9bc52f7871f43c288466643cebc7d69c diff --git a/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.md b/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.md index c58b3e2257..7442089038 100644 --- a/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.md +++ b/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.md @@ -10,7 +10,7 @@ The direct DeepSeek vision route uses provider file ids so repeated requests do ## Decision -Files remains the preferred transport. Each request-image file resolution has the configurable `filesApiTimeoutMs` deadline, one minute by default and always below `streamIdleTimeoutMs`. Successful resolutions refresh the outer idle watchdog. Caller cancellation and the outer stream deadline remain terminal outcomes. +Files remains the preferred transport. Each request-image file resolution has the configurable `filesApiTimeoutMs` deadline, one minute by default. The stream idle deadline defaults to five minutes, so the Files deadline normally leaves time for inline fallback. A deployment may configure the stream idle deadline to expire first. Successful resolutions refresh the outer idle watchdog. Caller cancellation and the outer stream deadline remain terminal outcomes. A file resolution failure discards the transient file parts assembled for that chat attempt and rebuilds the complete image request with base64 data URLs. Every retained image uses the already prepared deterministic `RequestImageAttachment`; the fallback performs no additional decode, resize, or encode, and a chat request never mixes file ids with inline images. Upload mappings committed before a later image fails remain available to later requests. The next request tries Files again, so recovery requires no process-wide outage state. @@ -30,7 +30,7 @@ Provider chat errors keep their existing classifications. A stale file id is inv ## Verification -Serializer tests cover file and data-URL representations over the same request versions, all supported media types, tool-result placement, and 20-to-10 base64 offload. Adapter tests cover immediate resolution failure, failure after a partial set of file ids, deadline-triggered fallback, stale-id replacement failure, all-inline request bodies, caller cancellation without fallback, and generic chat failure without a transport switch. Configuration tests cover both inline bounds and the Files deadline relationship. +Serializer tests cover file and data-URL representations over the same request versions, all supported media types, tool-result placement, and 20-to-10 base64 offload. Adapter tests cover immediate resolution failure, failure after a partial set of file ids, deadline-triggered fallback, stale-id replacement failure, all-inline request bodies, caller cancellation without fallback, and generic chat failure without a transport switch. Configuration tests cover both inline bounds and independent Files and stream idle deadlines. ## Consequences diff --git a/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.zh.md b/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.zh.md index 34625c6250..0534514f9b 100644 --- a/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.zh.md +++ b/.agents/notes/implemented/bug-fix/2026-08-21-deepseek-files-inline-fallback.zh.md @@ -10,7 +10,7 @@ DeepSeek 官方视觉路由使用提供方文件 ID,使重复请求不必再 ## Decision -Files 仍是首选传输方式。每张请求图片的文件解析都有可配置的 `filesApiTimeoutMs` 时限,默认一分钟,且始终小于 `streamIdleTimeoutMs`。每次成功解析都会刷新外层 idle watchdog。调用方取消和外层流时限仍直接终止请求。 +Files 仍是首选传输方式。每张请求图片的文件解析都有可配置的 `filesApiTimeoutMs` 时限,默认一分钟。stream idle 时限默认为五分钟,因此 Files 时限通常会为内联回退留出时间。部署也可以把 stream idle 时限设得更短,让它先终止请求。每次成功解析都会刷新外层 idle watchdog。调用方取消和外层流时限仍直接终止请求。 文件解析失败后,适配器会丢弃为该次 chat 尝试组装的临时文件块,并用 base64 data URL 重新组装完整图片请求。每张保留图片都复用已经准备好的确定性 `RequestImageAttachment`;回退不会再次解码、缩放或编码,同一个 chat 请求也不会混用 file ID 和内联图片。较早图片在后续图片失败前已经提交的上传映射会保留,供之后请求使用。下一次请求会重新尝试 Files,因此不需要保存进程级故障状态。 @@ -30,7 +30,7 @@ Files 仍是首选传输方式。每张请求图片的文件解析都有可配 ## Verification -序列化测试覆盖相同请求版本的文件和 data URL 表示、全部支持的媒体类型、工具结果位置,以及 20MiB 到 10MiB 的 base64 offload。适配器测试覆盖立即解析失败、部分 file ID 成功后的失败、时限触发的回退、失效 ID 替换失败、全内联请求体、调用方取消时不回退,以及普通 chat 错误不切换传输方式。配置测试覆盖两项内联预算和 Files 时限关系。 +序列化测试覆盖相同请求版本的文件和 data URL 表示、全部支持的媒体类型、工具结果位置,以及 20MiB 到 10MiB 的 base64 offload。适配器测试覆盖立即解析失败、部分 file ID 成功后的失败、时限触发的回退、失效 ID 替换失败、全内联请求体、调用方取消时不回退,以及普通 chat 错误不切换传输方式。配置测试覆盖两项内联预算,以及相互独立的 Files 和 stream idle 时限。 ## Consequences diff --git a/packages/llm/llm-deepseek/README.i18n.yaml b/packages/llm/llm-deepseek/README.i18n.yaml index e254c6267d..e434fefafc 100644 --- a/packages/llm/llm-deepseek/README.i18n.yaml +++ b/packages/llm/llm-deepseek/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/llm/llm-deepseek/README.md -README.md: 7a22955565027b30677e46a80a8b719bc7e61917 -README.zh.md: db1669509956d651dcb8948e1191a17cf9a0bfee +README.md: 8a62b7b587323de152ea3322ce310d48a41247cc +README.zh.md: 009732f4256c49d7ae8d702c41df532236f77c5f diff --git a/packages/llm/llm-deepseek/README.md b/packages/llm/llm-deepseek/README.md index 7a22955565..8a62b7b587 100644 --- a/packages/llm/llm-deepseek/README.md +++ b/packages/llm/llm-deepseek/README.md @@ -26,7 +26,7 @@ The package root exposes the Cordis plugin contract and `DeepSeekAdapter`; wire imageOffloadByteQuantum: 67108864 # oldest-image removal advances in 64 MiB steps inlineImageOffloadByteQuantum: 10485760 # fallback removal advances in 10 MiB steps imageOffloadCountQuantum: 20 # count overflow advances in 20-image steps - filesApiTimeoutMs: 60000 # per-image Files resolution deadline; below streamIdleTimeoutMs + filesApiTimeoutMs: 60000 # per-image Files resolution deadline; one-minute default fileExpiresAfterSeconds: 604800 # uploaded image lifetime; 1 hour to 30 days fileRefreshMarginSeconds: 3600 # replace ids with less lifetime remaining fileQuotaCleanupBatch: 100 # oldest harness-owned files deleted before one quota retry @@ -58,7 +58,7 @@ An image-capable catalog entry declares `inputModalities: [text, image]` and may Inline fallback has an independent base64 budget. `maxInlineRequestImageBytes` defaults to 20MiB and `inlineImageOffloadByteQuantum` to 10MiB, so a history of 21 one-megabyte base64 payloads removes the oldest 11 and retains 10MiB. The calculation uses base64-expanded lengths. The prepared request versions are reused byte-for-byte; fallback does not decode or compress an image again. Successful mappings created before a later image fails remain indexed for future requests. -Uploaded ids are indexed below `DSH_HOME` by endpoint/API-key scope and request `variantId`. The variant covers the normalized attachment id, transform version, route pixel and byte budgets, and encoder parameters, so Files API and inline fallback refer to the same deterministic bytes. Uploads request a seven-day lifetime by default and store the server's `expires_at`. A local mapping with no more than one hour remaining is replaced before use; the adapter does not retrieve every remote file before chat. If chat reports expired, deleted, missing, or invalid file ids and names one or more ids used by the request, the adapter removes exactly those mappings. If the provider identifies stale file state without naming an id, it removes every file mapping used by that chat attempt. It then uploads the affected request versions again and retries chat once. A second stale-file rejection clears the mappings identified by that response and is returned without a third chat attempt. An upload response without a complete file object, matching byte count, and `expires_at` is never indexed; a later request therefore uploads again instead of trusting inconsistent local state. A malformed local upload index is treated as an empty cache and replaced by the next successful upload. File resolution, including local index access and remote upload, has a per-image one-minute deadline by default; it must remain below `streamIdleTimeoutMs`. Each successful resolution refreshes the outer idle watchdog. Any resolution failure switches that request to inline mode, while explicit public file-management operations continue to report their own failures. +Uploaded ids are indexed below `DSH_HOME` by endpoint/API-key scope and request `variantId`. The variant covers the normalized attachment id, transform version, route pixel and byte budgets, and encoder parameters, so Files API and inline fallback refer to the same deterministic bytes. Uploads request a seven-day lifetime by default and store the server's `expires_at`. A local mapping with no more than one hour remaining is replaced before use; the adapter does not retrieve every remote file before chat. If chat reports expired, deleted, missing, or invalid file ids and names one or more ids used by the request, the adapter removes exactly those mappings. If the provider identifies stale file state without naming an id, it removes every file mapping used by that chat attempt. It then uploads the affected request versions again and retries chat once. A second stale-file rejection clears the mappings identified by that response and is returned without a third chat attempt. An upload response without a complete file object, matching byte count, and `expires_at` is never indexed; a later request therefore uploads again instead of trusting inconsistent local state. A malformed local upload index is treated as an empty cache and replaced by the next successful upload. File resolution, including local index access and remote upload, has a per-image one-minute deadline by default. The default five-minute stream idle deadline therefore leaves time for inline fallback; a deployment may configure a shorter stream idle deadline when it wants that outer deadline to terminate the request first. Each successful resolution refreshes the outer idle watchdog. Any resolution failure switches that request to inline mode, while explicit public file-management operations continue to report their own failures. Concurrent resolution of one scoped `variantId` shares one Files upload with waiter-local cancellation. One quota upload failure first paginates and collects the configured number of oldest `dsh-` files, then deletes that set before one upload retry. `DeepSeekFilesClient.delete`, `DeepSeekFileStore.release`, and `releaseAll` expose explicit remote-space reclamation. The current provider limits represented by this package are 128MiB per Files upload, 32MiB per chat-referenced image, 10,000 stored files, and 25GiB per API key; the default 1MiB request version remains below the two per-file limits. diff --git a/packages/llm/llm-deepseek/README.zh.md b/packages/llm/llm-deepseek/README.zh.md index db16695099..009732f425 100644 --- a/packages/llm/llm-deepseek/README.zh.md +++ b/packages/llm/llm-deepseek/README.zh.md @@ -26,7 +26,7 @@ harness LLM(大语言模型)seam 的 DeepSeek chat-completions 适配器: imageOffloadByteQuantum: 67108864 # oldest-image removal advances in 64 MiB steps inlineImageOffloadByteQuantum: 10485760 # fallback removal advances in 10 MiB steps imageOffloadCountQuantum: 20 # count overflow advances in 20-image steps - filesApiTimeoutMs: 60000 # per-image Files resolution deadline; below streamIdleTimeoutMs + filesApiTimeoutMs: 60000 # per-image Files resolution deadline; one-minute default fileExpiresAfterSeconds: 604800 # uploaded image lifetime; 1 hour to 30 days fileRefreshMarginSeconds: 3600 # replace ids with less lifetime remaining fileQuotaCleanupBatch: 100 # oldest harness-owned files deleted before one quota retry @@ -58,7 +58,7 @@ harness LLM(大语言模型)seam 的 DeepSeek chat-completions 适配器: 内联回退使用独立的 base64 预算。`maxInlineRequestImageBytes` 默认为 20MiB,`inlineImageOffloadByteQuantum` 默认为 10MiB,因此由 21 个 1MiB base64 负载组成的历史会移除最旧的 11 个并保留 10MiB。计算使用 base64 膨胀后的长度。系统逐字节复用已经准备好的请求版本;回退不会再次解码或压缩图片。前面图片已经成功写入的上传映射会保留,供后续请求复用。 -上传 ID 按端点和 API key 作用域以及请求 `variantId` 记录在 `DSH_HOME` 下。变体身份覆盖规范化附件 ID、变换策略版本、路由像素和字节预算及编码参数,因此 Files API 和内联回退引用同一份确定性字节。上传默认请求 7 天有效期,并保存服务端返回的 `expires_at`。本地映射剩余时间不超过一小时时会在使用前替换;适配器不会在每次 chat 前查询远端文件。如果 chat 报告文件 ID 已过期、删除、缺失或无效,并指出本次请求使用的一个或多个 ID,适配器只删除这些映射。如果响应只说明文件状态失效而没有指出 ID,适配器会删除该次 chat 使用的全部文件映射。随后重新上传受影响的请求版本,并重试一次 chat。第二次 chat 仍报告文件失效时,适配器会按该响应清理映射并返回错误,不会发起第三次 chat。上传响应若没有完整文件对象、匹配的字节数和 `expires_at`,就不会写入索引;后续请求会再次上传,而不是信任不一致的本地状态。本地上传索引格式损坏时按空缓存处理,并由下一次成功上传替换。文件解析包括本地索引访问和远端上传,默认每张图片的时限为一分钟,且必须小于 `streamIdleTimeoutMs`。每次成功解析都会刷新外层 idle watchdog。任何解析失败都会把该请求切换到内联模式;显式公共文件管理操作仍会报告自身错误。 +上传 ID 按端点和 API key 作用域以及请求 `variantId` 记录在 `DSH_HOME` 下。变体身份覆盖规范化附件 ID、变换策略版本、路由像素和字节预算及编码参数,因此 Files API 和内联回退引用同一份确定性字节。上传默认请求 7 天有效期,并保存服务端返回的 `expires_at`。本地映射剩余时间不超过一小时时会在使用前替换;适配器不会在每次 chat 前查询远端文件。如果 chat 报告文件 ID 已过期、删除、缺失或无效,并指出本次请求使用的一个或多个 ID,适配器只删除这些映射。如果响应只说明文件状态失效而没有指出 ID,适配器会删除该次 chat 使用的全部文件映射。随后重新上传受影响的请求版本,并重试一次 chat。第二次 chat 仍报告文件失效时,适配器会按该响应清理映射并返回错误,不会发起第三次 chat。上传响应若没有完整文件对象、匹配的字节数和 `expires_at`,就不会写入索引;后续请求会再次上传,而不是信任不一致的本地状态。本地上传索引格式损坏时按空缓存处理,并由下一次成功上传替换。文件解析包括本地索引访问和远端上传,默认每张图片的时限为一分钟。默认的 stream idle 时限为五分钟,因此通常有时间执行内联回退;部署可以设置更短的 stream idle 时限,让外层时限先终止请求。每次成功解析都会刷新外层 idle watchdog。任何解析失败都会把该请求切换到内联模式;显式公共文件管理操作仍会报告自身错误。 同一作用域和 `variantId` 的并发解析共享一次 Files 上传,每个等待方可以单独取消。一次上传配额错误会先分页收集配置数量的最旧 `dsh-` 文件,再删除这些文件并重试一次上传。`DeepSeekFilesClient.delete`、`DeepSeekFileStore.release` 和 `releaseAll` 提供主动远端空间回收。本包记录的当前提供方限制为 Files 单次上传 128MiB、chat 单图引用 32MiB、每个 API key 最多 10,000 个文件和 25GiB;默认 1MiB 请求版本低于两个单文件上限。 diff --git a/packages/llm/llm-deepseek/src/index.ts b/packages/llm/llm-deepseek/src/index.ts index e8632c22da..3af0236d29 100644 --- a/packages/llm/llm-deepseek/src/index.ts +++ b/packages/llm/llm-deepseek/src/index.ts @@ -336,9 +336,6 @@ export function resolveAdapterOptions(config: Config, environment?: LaunchEnviro `llm-deepseek: filesApiTimeoutMs must be a positive finite number no greater than ${MAX_TIMER_DELAY_MS}`, ) } - if (filesApiTimeoutMs >= streamIdleTimeoutMs) { - throw new Error('llm-deepseek: filesApiTimeoutMs must be below streamIdleTimeoutMs') - } const fileExpiresAfterSeconds = config.fileExpiresAfterSeconds ?? DEFAULT_FILE_EXPIRY_SECONDS if (!Number.isSafeInteger(fileExpiresAfterSeconds) || fileExpiresAfterSeconds < 3_600 diff --git a/packages/llm/llm-deepseek/tests/adapter.spec.ts b/packages/llm/llm-deepseek/tests/adapter.spec.ts index 5ef87231bb..085b35063a 100644 --- a/packages/llm/llm-deepseek/tests/adapter.spec.ts +++ b/packages/llm/llm-deepseek/tests/adapter.spec.ts @@ -348,7 +348,7 @@ describe('DeepSeekAdapter against a mock server', () => { await pending expect(fetchSpy).toHaveBeenCalledTimes(1) - expect(String(fetchSpy.mock.calls[0]?.[1]?.body)).toContain('image_url') + expect(fetchSpy.mock.calls[0]?.[1]?.body).toEqual(expect.stringContaining('image_url')) fetchSpy.mockRestore() }) @@ -1397,7 +1397,6 @@ describe('DeepSeekAdapter against a mock server', () => { }) const adapter = adapterOf({ baseURL: 'https://example.invalid', - filesApiTimeoutMs: 50, streamIdleTimeoutMs: 100, }) try { @@ -1432,7 +1431,6 @@ describe('DeepSeekAdapter against a mock server', () => { }) const adapter = adapterOf({ baseURL: 'https://example.invalid', - filesApiTimeoutMs: 50, streamIdleTimeoutMs: 100, }) try { @@ -2028,7 +2026,7 @@ describe('plugin registration and config', () => { })).rejects.toThrow(/streamIdleTimeoutMs/) }) - it('rejects invalid Files API timeout bounds for direct and plugin composition', async () => { + it('validates Files API timeout bounds independently of the stream idle deadline', async () => { expect(() => resolveAdapterOptions({ filesApiTimeoutMs: Number.POSITIVE_INFINITY })) .toThrow(/filesApiTimeoutMs.*positive finite/) expect(() => resolveAdapterOptions({ filesApiTimeoutMs: MAX_TIMER_DELAY_MS + 1 })) @@ -2044,8 +2042,8 @@ describe('plugin registration and config', () => { baseURL: 'http://127.0.0.1:1', filesApiTimeoutMs: MAX_TIMER_DELAY_MS + 1, })).rejects.toThrow(/filesApiTimeoutMs/) - expect(() => resolveAdapterOptions({ filesApiTimeoutMs: 100, streamIdleTimeoutMs: 100 })) - .toThrow(/filesApiTimeoutMs must be below streamIdleTimeoutMs/) + expect(resolveAdapterOptions({ filesApiTimeoutMs: 100, streamIdleTimeoutMs: 100 })) + .toMatchObject({ filesApiTimeoutMs: 100, streamIdleTimeoutMs: 100 }) }) it('rejects invalid nested retryPolicy before registering the provider', async () => {