From da1cb2c06ede6512cca2ce34df37e907d45aa9c5 Mon Sep 17 00:00:00 2001 From: Tianyi Cui <53024+tianyicui@users.noreply.github.com> Date: Mon, 24 Aug 2026 02:56:43 +0800 Subject: [PATCH] test(snapshot): centralize SDK session corpus --- docs/testing.i18n.yaml | 4 ++-- docs/testing.md | 2 +- docs/testing.zh.md | 2 +- examples/python-sdk-agent/tests/sdk.snapshot.ts | 10 ++++++++-- .../sdk}/bash-tool/notifications.expected.jsonl | 0 .../sdk}/bash-tool/result.expected.json | 0 .../sdk}/bash-tool/session.jsonl | 0 snapshots/sdk/bash-tool/snapshot.yml | 2 ++ .../sdk}/persistent-tools/notifications.expected.jsonl | 0 .../sdk}/persistent-tools/result.expected.json | 0 .../sdk}/persistent-tools/session.jsonl | 0 snapshots/sdk/persistent-tools/snapshot.yml | 2 ++ .../notifications.expected.jsonl | 0 .../subagent-spawn-in-process/result.expected.json | 0 .../sdk}/subagent-spawn-in-process/session.1.jsonl | 0 .../sdk}/subagent-spawn-in-process/session.jsonl | 0 snapshots/sdk/subagent-spawn-in-process/snapshot.yml | 2 ++ .../sdk}/text-turn/notifications.expected.jsonl | 0 .../sdk}/text-turn/result.expected.json | 0 .../sdk}/text-turn/session.jsonl | 0 snapshots/sdk/text-turn/snapshot.yml | 2 ++ 21 files changed, 20 insertions(+), 6 deletions(-) rename {examples/python-sdk-agent/tests/snapshots => snapshots/sdk}/bash-tool/notifications.expected.jsonl (100%) rename {examples/python-sdk-agent/tests/snapshots => snapshots/sdk}/bash-tool/result.expected.json (100%) rename {examples/python-sdk-agent/tests/snapshots => snapshots/sdk}/bash-tool/session.jsonl (100%) create mode 100644 snapshots/sdk/bash-tool/snapshot.yml rename {examples/python-sdk-agent/tests/snapshots => snapshots/sdk}/persistent-tools/notifications.expected.jsonl (100%) rename {examples/python-sdk-agent/tests/snapshots => snapshots/sdk}/persistent-tools/result.expected.json (100%) rename {examples/python-sdk-agent/tests/snapshots => snapshots/sdk}/persistent-tools/session.jsonl (100%) create mode 100644 snapshots/sdk/persistent-tools/snapshot.yml rename {examples/python-sdk-agent/tests/snapshots => snapshots/sdk}/subagent-spawn-in-process/notifications.expected.jsonl (100%) rename {examples/python-sdk-agent/tests/snapshots => snapshots/sdk}/subagent-spawn-in-process/result.expected.json (100%) rename {examples/python-sdk-agent/tests/snapshots => snapshots/sdk}/subagent-spawn-in-process/session.1.jsonl (100%) rename {examples/python-sdk-agent/tests/snapshots => snapshots/sdk}/subagent-spawn-in-process/session.jsonl (100%) create mode 100644 snapshots/sdk/subagent-spawn-in-process/snapshot.yml rename {examples/python-sdk-agent/tests/snapshots => snapshots/sdk}/text-turn/notifications.expected.jsonl (100%) rename {examples/python-sdk-agent/tests/snapshots => snapshots/sdk}/text-turn/result.expected.json (100%) rename {examples/python-sdk-agent/tests/snapshots => snapshots/sdk}/text-turn/session.jsonl (100%) create mode 100644 snapshots/sdk/text-turn/snapshot.yml diff --git a/docs/testing.i18n.yaml b/docs/testing.i18n.yaml index 64cf0d584f..952aa991bf 100644 --- a/docs/testing.i18n.yaml +++ b/docs/testing.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write docs/testing.md -testing.md: 9b0ed0a103b1ca9f149d289cabcc2b0ba9bc909f -testing.zh.md: 18e72deed19b7ecd981123e933f23cbbf7ba2c87 +testing.md: 1d9f50e3d593e41ca0823e9b6a8182b573db5feb +testing.zh.md: dbe6bd4fedb779584998003587c71f5846d52c1c diff --git a/docs/testing.md b/docs/testing.md index 9b0ed0a103..1d9f50e3d5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -46,4 +46,4 @@ An e2e assertion re-runs the command or re-reads the file externally; a keyword ## When a snapshot test is required -Every non-trivial model-, protocol-, or human-visible change adds or updates a keyless scenario in the same PR through its owning recorded-session suite. Package tests, e2e assertions, mock/test-only compositions, and PR rationale do not replace the assembled transcript; extend the harness when needed. ACP automation scenarios live under `snapshots/acp/`, driven by the [`dsh-session-snapshot`](../packages/test-support/session-snapshot/README.md) ACP adapter; `examples/headless-agent` owns the internal canonical-event JSONL snapshots and replay fixtures. The `pwsh-tool-turn` ACP scenario boots real `pwsh` and skips where it is absent. Completed interactive-terminal journeys use JSONL-driven scenarios under `apps/cli/tests/snapshots/`; transient presentation uses the package-local semantic matrix, with a PTY case when input, Loader selection, or terminal teardown changes. Browser-rendered web GUI journeys use `apps/web/tests/snapshots/`. The two SDKs project the agent loop, session lifecycle, and `SessionEventMap` independently, so changing any of those updates both: `examples/python-sdk-agent/tests/snapshots/` owns the TypeScript client; `scripts/snapshots/python-sdk-single-exe/` owns the Python client, which only the required `python-runtime` CI job runs. New capability seams, lifecycle variants, or transcript surfaces name every coverage tier at plan time and verify the harness can express it before implementation. +Every non-trivial model-, protocol-, or human-visible change adds or updates a keyless scenario in the same PR through its owning recorded-session suite. Package tests, e2e assertions, mock/test-only compositions, and PR rationale do not replace the assembled transcript; extend the harness when needed. ACP automation scenarios live under `snapshots/acp/`, driven by the [`dsh-session-snapshot`](../packages/test-support/session-snapshot/README.md) ACP adapter; `examples/headless-agent` owns the internal canonical-event JSONL snapshots and replay fixtures. The `pwsh-tool-turn` ACP scenario boots real `pwsh` and skips where it is absent. Completed interactive-terminal journeys use JSONL-driven scenarios under `apps/cli/tests/snapshots/`; transient presentation uses the package-local semantic matrix, with a PTY case when input, Loader selection, or terminal teardown changes. Browser-rendered web GUI journeys use `apps/web/tests/snapshots/`. The two SDKs project the agent loop, session lifecycle, and `SessionEventMap` independently, so changing any of those updates both: `snapshots/sdk/` owns the TypeScript client; `scripts/snapshots/python-sdk-single-exe/` owns the Python client, which only the required `python-runtime` CI job runs. New capability seams, lifecycle variants, or transcript surfaces name every coverage tier at plan time and verify the harness can express it before implementation. diff --git a/docs/testing.zh.md b/docs/testing.zh.md index 18e72deed1..dbe6bd4fed 100644 --- a/docs/testing.zh.md +++ b/docs/testing.zh.md @@ -46,4 +46,4 @@ e2e 断言应重新运行命令或从外部重新读取文件；对 agent 自身 ## 何时需要快照测试 -每项非平凡的模型可见、协议可见或人类可见变更，都必须在同一 PR 中，通过所属录制会话套件添加或更新无密钥场景。包测试、e2e 断言、mock 与仅测试组合、PR 理由都不能取代组装后的 transcript；必要时应扩展 harness。ACP 自动化场景位于 `snapshots/acp/`，由 [`dsh-session-snapshot`](../packages/test-support/session-snapshot/README.zh.md) ACP 适配器驱动；`examples/headless-agent` 拥有内部规范事件 JSONL 快照与回放 fixture。`pwsh-tool-turn` ACP 场景启动真实 `pwsh`，在无 `pwsh` 的主机上跳过。已完成的交互式终端旅程使用 `apps/cli/tests/snapshots/` 下由 JSONL 驱动的场景；瞬态呈现使用包内语义矩阵，输入、Loader 选择或终端清理发生变化时还要添加 PTY 用例。浏览器渲染的 Web GUI 旅程使用上述 Web 应用快照套件。两个 SDK 各自独立地投影 agent loop、会话生命周期与 `SessionEventMap`，因此改动其中任何一项都要同时更新两者：`examples/python-sdk-agent/tests/snapshots/` 拥有 TypeScript 客户端；`scripts/snapshots/python-sdk-single-exe/` 拥有 Python 客户端，且只有必需的 `python-runtime` CI 作业会运行它。新的能力 seam、生命周期变体或 transcript 呈现接口在计划阶段就要列出每个覆盖层级，并在实现前验证 harness 能够表达它们。 +每项非平凡的模型可见、协议可见或人类可见变更，都必须在同一 PR 中，通过所属录制会话套件添加或更新无密钥场景。包测试、e2e 断言、mock 与仅测试组合、PR 理由都不能取代组装后的 transcript；必要时应扩展 harness。ACP 自动化场景位于 `snapshots/acp/`，由 [`dsh-session-snapshot`](../packages/test-support/session-snapshot/README.zh.md) ACP 适配器驱动；`examples/headless-agent` 拥有内部规范事件 JSONL 快照与回放 fixture。`pwsh-tool-turn` ACP 场景启动真实 `pwsh`，在无 `pwsh` 的主机上跳过。已完成的交互式终端旅程使用 `apps/cli/tests/snapshots/` 下由 JSONL 驱动的场景；瞬态呈现使用包内语义矩阵，输入、Loader 选择或终端清理发生变化时还要添加 PTY 用例。浏览器渲染的 Web GUI 旅程使用上述 Web 应用快照套件。两个 SDK 各自独立地投影 agent loop、会话生命周期与 `SessionEventMap`，因此改动其中任何一项都要同时更新两者：`snapshots/sdk/` 拥有 TypeScript 客户端；`scripts/snapshots/python-sdk-single-exe/` 拥有 Python 客户端，且只有必需的 `python-runtime` CI 作业会运行它。新的能力 seam、生命周期变体或 transcript 呈现接口在计划阶段就要列出每个覆盖层级，并在实现前验证 harness 能够表达它们。 diff --git a/examples/python-sdk-agent/tests/sdk.snapshot.ts b/examples/python-sdk-agent/tests/sdk.snapshot.ts index 8c9e0b3cf5..ba9da1f006 100644 --- a/examples/python-sdk-agent/tests/sdk.snapshot.ts +++ b/examples/python-sdk-agent/tests/sdk.snapshot.ts @@ -19,6 +19,7 @@ import { normalizeSessionLog, normalizeSessionSnapshot, normalizeStdout, + parseSnapshotManifest, refreshFixtureReplacements, scrubRequestHeaders, scrubSessionSnapshot, @@ -31,7 +32,7 @@ import { import { DeepSeekHarness, type HarnessNotification, type RunResult } from '@deepseek-ai/dsh-sdk-client' const testsDir = dirOf(import.meta.url) -const snapshotsDir = join(testsDir, 'snapshots') +const snapshotsDir = fileURLToPath(new URL('../../../snapshots/sdk/', import.meta.url)) const liveConfig = join(testsDir, '..', 'typescript-sdk.cordis.yml') const replayConfig = join(testsDir, '..', 'typescript-sdk.cordis.snapshot.yml') const minimalLiveConfig = join(testsDir, '..', 'typescript-sdk-minimal.cordis.yml') @@ -65,7 +66,7 @@ function dirOf(url: string): string { } interface SdkScenario { - /** Scenario name; the snapshots/ fixture directory. */ + /** Scenario name; the `snapshots/sdk/` fixture directory. */ name: string /** The user prompt for the single SDK turn. */ prompt: string @@ -370,6 +371,11 @@ describe('TypeScript SDK snapshots over the jsonrpc runtime', () => { for (const scenario of SCENARIOS) { it(`replays ${scenario.name} through the SDK`, async () => { const scenarioDir = join(snapshotsDir, scenario.name) + const manifest = parseSnapshotManifest( + await readFile(join(scenarioDir, 'snapshot.yml'), 'utf8'), + join(scenarioDir, 'snapshot.yml'), + ) + expect(manifest).toEqual({ version: 1, profile: 'sdk' }) const notificationsExpectedPath = join(scenarioDir, 'notifications.expected.jsonl') const resultExpectedPath = join(scenarioDir, 'result.expected.json') diff --git a/examples/python-sdk-agent/tests/snapshots/bash-tool/notifications.expected.jsonl b/snapshots/sdk/bash-tool/notifications.expected.jsonl similarity index 100% rename from examples/python-sdk-agent/tests/snapshots/bash-tool/notifications.expected.jsonl rename to snapshots/sdk/bash-tool/notifications.expected.jsonl diff --git a/examples/python-sdk-agent/tests/snapshots/bash-tool/result.expected.json b/snapshots/sdk/bash-tool/result.expected.json similarity index 100% rename from examples/python-sdk-agent/tests/snapshots/bash-tool/result.expected.json rename to snapshots/sdk/bash-tool/result.expected.json diff --git a/examples/python-sdk-agent/tests/snapshots/bash-tool/session.jsonl b/snapshots/sdk/bash-tool/session.jsonl similarity index 100% rename from examples/python-sdk-agent/tests/snapshots/bash-tool/session.jsonl rename to snapshots/sdk/bash-tool/session.jsonl diff --git a/snapshots/sdk/bash-tool/snapshot.yml b/snapshots/sdk/bash-tool/snapshot.yml new file mode 100644 index 0000000000..95e0fb7247 --- /dev/null +++ b/snapshots/sdk/bash-tool/snapshot.yml @@ -0,0 +1,2 @@ +version: 1 +profile: sdk diff --git a/examples/python-sdk-agent/tests/snapshots/persistent-tools/notifications.expected.jsonl b/snapshots/sdk/persistent-tools/notifications.expected.jsonl similarity index 100% rename from examples/python-sdk-agent/tests/snapshots/persistent-tools/notifications.expected.jsonl rename to snapshots/sdk/persistent-tools/notifications.expected.jsonl diff --git a/examples/python-sdk-agent/tests/snapshots/persistent-tools/result.expected.json b/snapshots/sdk/persistent-tools/result.expected.json similarity index 100% rename from examples/python-sdk-agent/tests/snapshots/persistent-tools/result.expected.json rename to snapshots/sdk/persistent-tools/result.expected.json diff --git a/examples/python-sdk-agent/tests/snapshots/persistent-tools/session.jsonl b/snapshots/sdk/persistent-tools/session.jsonl similarity index 100% rename from examples/python-sdk-agent/tests/snapshots/persistent-tools/session.jsonl rename to snapshots/sdk/persistent-tools/session.jsonl diff --git a/snapshots/sdk/persistent-tools/snapshot.yml b/snapshots/sdk/persistent-tools/snapshot.yml new file mode 100644 index 0000000000..95e0fb7247 --- /dev/null +++ b/snapshots/sdk/persistent-tools/snapshot.yml @@ -0,0 +1,2 @@ +version: 1 +profile: sdk diff --git a/examples/python-sdk-agent/tests/snapshots/subagent-spawn-in-process/notifications.expected.jsonl b/snapshots/sdk/subagent-spawn-in-process/notifications.expected.jsonl similarity index 100% rename from examples/python-sdk-agent/tests/snapshots/subagent-spawn-in-process/notifications.expected.jsonl rename to snapshots/sdk/subagent-spawn-in-process/notifications.expected.jsonl diff --git a/examples/python-sdk-agent/tests/snapshots/subagent-spawn-in-process/result.expected.json b/snapshots/sdk/subagent-spawn-in-process/result.expected.json similarity index 100% rename from examples/python-sdk-agent/tests/snapshots/subagent-spawn-in-process/result.expected.json rename to snapshots/sdk/subagent-spawn-in-process/result.expected.json diff --git a/examples/python-sdk-agent/tests/snapshots/subagent-spawn-in-process/session.1.jsonl b/snapshots/sdk/subagent-spawn-in-process/session.1.jsonl similarity index 100% rename from examples/python-sdk-agent/tests/snapshots/subagent-spawn-in-process/session.1.jsonl rename to snapshots/sdk/subagent-spawn-in-process/session.1.jsonl diff --git a/examples/python-sdk-agent/tests/snapshots/subagent-spawn-in-process/session.jsonl b/snapshots/sdk/subagent-spawn-in-process/session.jsonl similarity index 100% rename from examples/python-sdk-agent/tests/snapshots/subagent-spawn-in-process/session.jsonl rename to snapshots/sdk/subagent-spawn-in-process/session.jsonl diff --git a/snapshots/sdk/subagent-spawn-in-process/snapshot.yml b/snapshots/sdk/subagent-spawn-in-process/snapshot.yml new file mode 100644 index 0000000000..95e0fb7247 --- /dev/null +++ b/snapshots/sdk/subagent-spawn-in-process/snapshot.yml @@ -0,0 +1,2 @@ +version: 1 +profile: sdk diff --git a/examples/python-sdk-agent/tests/snapshots/text-turn/notifications.expected.jsonl b/snapshots/sdk/text-turn/notifications.expected.jsonl similarity index 100% rename from examples/python-sdk-agent/tests/snapshots/text-turn/notifications.expected.jsonl rename to snapshots/sdk/text-turn/notifications.expected.jsonl diff --git a/examples/python-sdk-agent/tests/snapshots/text-turn/result.expected.json b/snapshots/sdk/text-turn/result.expected.json similarity index 100% rename from examples/python-sdk-agent/tests/snapshots/text-turn/result.expected.json rename to snapshots/sdk/text-turn/result.expected.json diff --git a/examples/python-sdk-agent/tests/snapshots/text-turn/session.jsonl b/snapshots/sdk/text-turn/session.jsonl similarity index 100% rename from examples/python-sdk-agent/tests/snapshots/text-turn/session.jsonl rename to snapshots/sdk/text-turn/session.jsonl diff --git a/snapshots/sdk/text-turn/snapshot.yml b/snapshots/sdk/text-turn/snapshot.yml new file mode 100644 index 0000000000..95e0fb7247 --- /dev/null +++ b/snapshots/sdk/text-turn/snapshot.yml @@ -0,0 +1,2 @@ +version: 1 +profile: sdk