mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-08-29 04:26:38 +00:00
test(snapshot): drive ordinary turns through headless dsh
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { existsSync, readFileSync } from 'node:fs'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { createServer } from 'node:http'
|
||||
import type { IncomingMessage, ServerResponse } from 'node:http'
|
||||
@@ -731,7 +731,7 @@ const hasPwsh = spawnSync(resolvePwshPath(), ['-NoLogo', '-NoProfile', '-NonInte
|
||||
defineAcpSnapshotSuite({
|
||||
agent: AGENT,
|
||||
snapshotsDir: SNAPSHOTS_DIR,
|
||||
scenarios: SCENARIOS,
|
||||
scenarios: SCENARIOS.filter(scenario => existsSync(join(SNAPSHOTS_DIR, scenario.name))),
|
||||
mode: snapshotModeFromEnv(process.env.DSH_SNAPSHOT),
|
||||
hasPwsh,
|
||||
})
|
||||
|
||||
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write packages/test-support/session-snapshot/README.md
|
||||
README.md: dfd298448d3373d0966d87963be2f1d8425f5961
|
||||
README.zh.md: fa6c960feee202a893b263b913e7ec178da46ac2
|
||||
README.md: 4a2de1738b082cb2a860f7d611ad8bef318aabe6
|
||||
README.zh.md: 72e0b8f7e0ab7f5d64bd7a87c235285b2aeba310
|
||||
|
||||
@@ -4,7 +4,7 @@ English | [中文](README.zh.md)
|
||||
|
||||
Session-log snapshot support for the keyless snapshot tier (`pnpm run test:snapshot`, [testing policy](../../../docs/testing.md)). Transport-neutral normalization and fixture invariants live beside the existing ACP protocol adapter so additional `dsh` profile adapters can reuse the recorded-session format without copying its guards.
|
||||
|
||||
Every recorded-session directory carries a closed `snapshot.yml` manifest. `profile` names the shipped `dsh` controller; a directory owns its local `session.jsonl` unless `session.source` names another scenario's read-only canonical recording. Unknown fields, JavaScript YAML tags, absolute paths, and platform-specific separators fail during collection.
|
||||
Every recorded-session directory carries a closed `snapshot.yml` manifest. `profile` names the shipped `dsh` controller, `composition` groups scenarios under one profile-patch and request-header pin, `recording` distinguishes live-recordable sessions from deliberately authored scripts, and `header` records pin and sidecar ownership. `replay.override` declares the exceptional sidecar needed for a failure or hang that successful chunks cannot reconstruct. A directory owns its local `session.jsonl` unless `session.source` names another scenario's read-only canonical recording. Unknown fields, JavaScript YAML tags, malformed names and indexes, absolute paths, and platform-specific separators fail during collection.
|
||||
|
||||
The current ACP adapter has four importable layers:
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
无密钥快照层(`pnpm run test:snapshot`,见[测试策略](../../../docs/testing.zh.md))的会话日志快照支持。与传输无关的规范化和 fixture(测试前置数据)不变量与现有 ACP(Agent Client Protocol)协议适配器位于同一包中,使其他 `dsh` profile 适配器可以复用录制会话格式,而不复制其保护机制。
|
||||
|
||||
每个录制会话目录都包含一个封闭的 `snapshot.yml` manifest。`profile` 指名随附的 `dsh` 控制器;除非 `session.source` 指向另一个场景的只读规范录制,否则目录拥有本地 `session.jsonl`。收集期间会拒绝未知字段、JavaScript YAML tag、绝对路径和平台专用分隔符。
|
||||
每个录制会话目录都包含一个封闭的 `snapshot.yml` manifest。`profile` 指名随附的 `dsh` 控制器,`composition` 把场景归入同一组 profile patch 与请求头 pin,`recording` 区分可通过真实模型录制的会话和特意手写的脚本,`header` 记录 pin 与 sidecar 的所有权。失败或挂起无法由成功 chunk 重建时,`replay.override` 声明所需的例外 sidecar。除非 `session.source` 指向另一个场景的只读规范录制,否则目录拥有本地 `session.jsonl`。收集期间会拒绝未知字段、JavaScript YAML tag、格式错误的名称和索引、绝对路径及平台专用分隔符。
|
||||
|
||||
当前 ACP 适配器包含四个可单独导入的层:
|
||||
|
||||
|
||||
@@ -48,8 +48,11 @@ export {
|
||||
} from './normalize.ts'
|
||||
export {
|
||||
parseSnapshotManifest,
|
||||
type SnapshotHeaderManifest,
|
||||
type SnapshotManifest,
|
||||
type SnapshotProfile,
|
||||
type SnapshotRecording,
|
||||
type SnapshotReplayManifest,
|
||||
type SnapshotSessionReference,
|
||||
} from './manifest.ts'
|
||||
export {
|
||||
|
||||
@@ -6,6 +6,33 @@ import * as yaml from 'js-yaml'
|
||||
/** Public `dsh` profile used to control a recorded-session scenario. */
|
||||
export type SnapshotProfile = 'headless' | 'sdk' | 'acp' | 'web'
|
||||
|
||||
/** How a canonical session may be regenerated. */
|
||||
export type SnapshotRecording = 'live' | 'authored'
|
||||
|
||||
/** Request-header ownership metadata for one composition. */
|
||||
export interface SnapshotHeaderManifest {
|
||||
/** Stable class name shared only by byte-identical request headers. */
|
||||
class: string
|
||||
/** Whether this scenario owns the class's tokenized header sequence. */
|
||||
pin?: true
|
||||
/** Scenario that owns the readable system-prompt sidecar. */
|
||||
systemPromptSource?: string
|
||||
/** Scenario that owns the readable tool-schema sidecar. */
|
||||
toolSchemasSource?: string
|
||||
/** Child fixture indexes that own distinct system-prompt sidecars. */
|
||||
childSystemPrompts?: number[]
|
||||
/** Child fixture indexes that own distinct tool-schema sidecars. */
|
||||
childToolSchemas?: number[]
|
||||
/** Legitimate changed-header count after the initial request header. */
|
||||
changes?: number
|
||||
}
|
||||
|
||||
/** Replay facts that cannot be reconstructed from successful model chunks. */
|
||||
export interface SnapshotReplayManifest {
|
||||
/** A scenario-local `replay.override.json` replaces or patches the recorded model script. */
|
||||
override: true
|
||||
}
|
||||
|
||||
/** Optional reference to another scenario's canonical session. */
|
||||
export interface SnapshotSessionReference {
|
||||
/** Repository-relative POSIX path from this scenario directory to the owning `session.jsonl`. */
|
||||
@@ -18,11 +45,21 @@ export interface SnapshotManifest {
|
||||
version: 1
|
||||
/** Shipped profile whose public interface controls the scenario. */
|
||||
profile: SnapshotProfile
|
||||
/** Composition id whose sole pin owns its profile patches. */
|
||||
composition?: string
|
||||
/** Whether the session is live-recordable or deliberately authored. */
|
||||
recording?: SnapshotRecording
|
||||
/** Request-header class and sidecar ownership. */
|
||||
header?: SnapshotHeaderManifest
|
||||
/** Exceptional replay metadata absent for ordinary successful recordings. */
|
||||
replay?: SnapshotReplayManifest
|
||||
/** Absent when this directory owns `session.jsonl`; present for a read-only borrower. */
|
||||
session?: SnapshotSessionReference
|
||||
}
|
||||
|
||||
const PROFILES = new Set<SnapshotProfile>(['headless', 'sdk', 'acp', 'web'])
|
||||
const RECORDINGS = new Set<SnapshotRecording>(['live', 'authored'])
|
||||
const NAME_RE = /^[a-z0-9]+(?:-[a-z0-9]+)*$/
|
||||
|
||||
function record(value: unknown, label: string): Record<string, unknown> {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
||||
@@ -36,6 +73,22 @@ function exactKeys(value: Record<string, unknown>, allowed: readonly string[], l
|
||||
if (unknown.length > 0) throw new Error(`${label} has unknown field(s): ${unknown.join(', ')}`)
|
||||
}
|
||||
|
||||
function name(value: unknown, label: string): string {
|
||||
if (typeof value !== 'string' || !NAME_RE.test(value)) {
|
||||
throw new Error(`${label} must be a lower-kebab-case name`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function positiveIndexes(value: unknown, label: string): number[] {
|
||||
if (!Array.isArray(value)
|
||||
|| value.some(item => !Number.isInteger(item) || Number(item) < 1)
|
||||
|| new Set(value).size !== value.length) {
|
||||
throw new Error(`${label} must be an array of unique positive integers`)
|
||||
}
|
||||
return [...value as number[]]
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one `snapshot.yml` without admitting JavaScript YAML tags or unknown fields.
|
||||
* @param source - complete manifest text.
|
||||
@@ -52,12 +105,68 @@ export function parseSnapshotManifest(source: string, path = 'snapshot.yml'): Sn
|
||||
|
||||
try {
|
||||
const root = record(parsed, 'manifest')
|
||||
exactKeys(root, ['version', 'profile', 'session'], 'manifest')
|
||||
exactKeys(root, ['version', 'profile', 'composition', 'recording', 'header', 'replay', 'session'], 'manifest')
|
||||
if (root.version !== 1) throw new Error('manifest.version must equal 1')
|
||||
if (typeof root.profile !== 'string' || !PROFILES.has(root.profile as SnapshotProfile)) {
|
||||
throw new Error('manifest.profile must be headless, sdk, acp, or web')
|
||||
}
|
||||
|
||||
const composition = root.composition === undefined
|
||||
? undefined
|
||||
: name(root.composition, 'manifest.composition')
|
||||
let recording: SnapshotRecording | undefined
|
||||
if (root.recording !== undefined) {
|
||||
if (typeof root.recording !== 'string' || !RECORDINGS.has(root.recording as SnapshotRecording)) {
|
||||
throw new Error('manifest.recording must be live or authored')
|
||||
}
|
||||
recording = root.recording as SnapshotRecording
|
||||
}
|
||||
|
||||
let header: SnapshotHeaderManifest | undefined
|
||||
if (root.header !== undefined) {
|
||||
const value = record(root.header, 'manifest.header')
|
||||
exactKeys(value, [
|
||||
'class',
|
||||
'pin',
|
||||
'systemPromptSource',
|
||||
'toolSchemasSource',
|
||||
'childSystemPrompts',
|
||||
'childToolSchemas',
|
||||
'changes',
|
||||
], 'manifest.header')
|
||||
if (value.pin !== undefined && value.pin !== true) {
|
||||
throw new Error('manifest.header.pin must equal true when present')
|
||||
}
|
||||
if (value.changes !== undefined && (!Number.isInteger(value.changes) || Number(value.changes) < 0)) {
|
||||
throw new Error('manifest.header.changes must be a non-negative integer')
|
||||
}
|
||||
header = {
|
||||
class: name(value.class, 'manifest.header.class'),
|
||||
...(value.pin === true ? { pin: true as const } : {}),
|
||||
...(value.systemPromptSource === undefined
|
||||
? {}
|
||||
: { systemPromptSource: name(value.systemPromptSource, 'manifest.header.systemPromptSource') }),
|
||||
...(value.toolSchemasSource === undefined
|
||||
? {}
|
||||
: { toolSchemasSource: name(value.toolSchemasSource, 'manifest.header.toolSchemasSource') }),
|
||||
...(value.childSystemPrompts === undefined
|
||||
? {}
|
||||
: { childSystemPrompts: positiveIndexes(value.childSystemPrompts, 'manifest.header.childSystemPrompts') }),
|
||||
...(value.childToolSchemas === undefined
|
||||
? {}
|
||||
: { childToolSchemas: positiveIndexes(value.childToolSchemas, 'manifest.header.childToolSchemas') }),
|
||||
...(value.changes === undefined ? {} : { changes: Number(value.changes) }),
|
||||
}
|
||||
}
|
||||
|
||||
let replay: SnapshotReplayManifest | undefined
|
||||
if (root.replay !== undefined) {
|
||||
const value = record(root.replay, 'manifest.replay')
|
||||
exactKeys(value, ['override'], 'manifest.replay')
|
||||
if (value.override !== true) throw new Error('manifest.replay.override must equal true')
|
||||
replay = { override: true }
|
||||
}
|
||||
|
||||
let session: SnapshotSessionReference | undefined
|
||||
if (root.session !== undefined) {
|
||||
const value = record(root.session, 'manifest.session')
|
||||
@@ -74,6 +183,10 @@ export function parseSnapshotManifest(source: string, path = 'snapshot.yml'): Sn
|
||||
return {
|
||||
version: 1,
|
||||
profile: root.profile as SnapshotProfile,
|
||||
...(composition === undefined ? {} : { composition }),
|
||||
...(recording === undefined ? {} : { recording }),
|
||||
...(header === undefined ? {} : { header }),
|
||||
...(replay === undefined ? {} : { replay }),
|
||||
...(session === undefined ? {} : { session }),
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
@@ -23,11 +23,53 @@ describe('snapshot manifest', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('parses composition, recording, header, and exceptional replay metadata', () => {
|
||||
expect(parseSnapshotManifest([
|
||||
'version: 1',
|
||||
'profile: sdk',
|
||||
'composition: continuable-subagent',
|
||||
'recording: authored',
|
||||
'header:',
|
||||
' class: continuable-subagent',
|
||||
' pin: true',
|
||||
' systemPromptSource: text-turn',
|
||||
' toolSchemasSource: text-turn',
|
||||
' childSystemPrompts: [1]',
|
||||
' childToolSchemas: [1, 2]',
|
||||
' changes: 1',
|
||||
'replay:',
|
||||
' override: true',
|
||||
'',
|
||||
].join('\n'))).toEqual({
|
||||
version: 1,
|
||||
profile: 'sdk',
|
||||
composition: 'continuable-subagent',
|
||||
recording: 'authored',
|
||||
header: {
|
||||
class: 'continuable-subagent',
|
||||
pin: true,
|
||||
systemPromptSource: 'text-turn',
|
||||
toolSchemasSource: 'text-turn',
|
||||
childSystemPrompts: [1],
|
||||
childToolSchemas: [1, 2],
|
||||
changes: 1,
|
||||
},
|
||||
replay: { override: true },
|
||||
})
|
||||
})
|
||||
|
||||
it.each([
|
||||
['', 'manifest must be a mapping'],
|
||||
['version: 2\nprofile: acp\n', 'manifest.version must equal 1'],
|
||||
['version: 1\nprofile: private\n', 'manifest.profile must be headless, sdk, acp, or web'],
|
||||
['version: 1\nprofile: acp\nextra: true\n', 'manifest has unknown field(s): extra'],
|
||||
['version: 1\nprofile: acp\ncomposition: Not_Safe\n', 'manifest.composition must be a lower-kebab-case name'],
|
||||
['version: 1\nprofile: acp\nrecording: maybe\n', 'manifest.recording must be live or authored'],
|
||||
['version: 1\nprofile: acp\nheader: {}\n', 'manifest.header.class must be a lower-kebab-case name'],
|
||||
['version: 1\nprofile: acp\nheader:\n class: base\n pin: false\n', 'manifest.header.pin must equal true when present'],
|
||||
['version: 1\nprofile: acp\nheader:\n class: base\n childToolSchemas: [1, 1]\n', 'manifest.header.childToolSchemas must be an array of unique positive integers'],
|
||||
['version: 1\nprofile: acp\nheader:\n class: base\n changes: -1\n', 'manifest.header.changes must be a non-negative integer'],
|
||||
['version: 1\nprofile: acp\nreplay:\n override: false\n', 'manifest.replay.override must equal true'],
|
||||
['version: 1\nprofile: acp\nsession: {}\n', 'manifest.session.source must be a non-empty string'],
|
||||
['version: 1\nprofile: acp\nsession:\n source: /tmp/session.jsonl\n', 'manifest.session.source must be a relative POSIX path'],
|
||||
['version: 1\nprofile: acp\nsession:\n source: ..\\session.jsonl\n', 'manifest.session.source must be a relative POSIX path'],
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ This tree contains only tests whose committed session JSONL is replay input and
|
||||
|
||||
Every process under test starts through the `dsh` CLI with a shipped profile and optional scenario patches. Test clients may drive a public protocol or browser interface; do not add another application entrypoint, hidden CLI mode, or executable scenario driver.
|
||||
|
||||
Each scenario owns or explicitly references one primary `session.jsonl` plus contiguous child files. The owner alone records or refreshes it. Shared references are read-only, acyclic, and used only when another interface intentionally renders the same recorded behavior.
|
||||
Each scenario owns or explicitly references one primary `session.jsonl` plus contiguous child files. The owner alone records or refreshes it. For an ordinary one-shot case, derive the user task and replay script from that JSONL; do not duplicate them in an `input.json`. Shared references are read-only, acyclic, and used only when another interface intentionally renders the same recorded behavior.
|
||||
|
||||
Committed sessions are normalization fixed points. Replace volatile identities with typed relationship-preserving tokens, replace request system prompts and tool schemas with tokens, and keep exactly one readable sidecar owner per header class. Never redact arbitrary user or tool text merely because it resembles an identifier.
|
||||
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
{
|
||||
"steps": [
|
||||
{ "op": "initialize" },
|
||||
{ "op": "newSession" },
|
||||
{ "op": "prompt", "text": "Reply with exactly the word: PONG. Do not use any tools." }
|
||||
]
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
version: 1
|
||||
profile: acp
|
||||
@@ -1,5 +0,0 @@
|
||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"mcpCapabilities":{"http":true},"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false},"sessionCapabilities":{"close":{},"list":{},"resume":{}}},"authMethods":[]}}
|
||||
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}","configOptions":[{"id":"model","name":"Model","category":"model","type":"select","currentValue":"[\"deepseek-official\",\"deepseek-v4-flash\"]","options":[{"group":"deepseek-official","name":"DeepSeek","options":[{"value":"[\"deepseek-official\",\"deepseek-v4-flash\"]","name":"deepseek-v4-flash"},{"value":"[\"deepseek-official\",\"deepseek-v4-pro\"]","name":"deepseek-v4-pro"}]}]}]}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","messageId":"{{messageId}}","content":{"type":"text","text":"The user wants me to reply with exactly the word \"PONG\" and not use any tools."}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","messageId":"{{messageId}}","content":{"type":"text","text":"PONG"}}}}
|
||||
{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}}
|
||||
@@ -1,7 +0,0 @@
|
||||
{
|
||||
"steps": [
|
||||
{ "op": "initialize" },
|
||||
{ "op": "newSession" },
|
||||
{ "op": "prompt", "text": "Use the bash tool to run exactly: echo SNAPSHOT_OK. Then reply with the single word DONE and stop." }
|
||||
]
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
version: 1
|
||||
profile: acp
|
||||
@@ -1,8 +0,0 @@
|
||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"mcpCapabilities":{"http":true},"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false},"sessionCapabilities":{"close":{},"list":{},"resume":{}}},"authMethods":[]}}
|
||||
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}","configOptions":[{"id":"model","name":"Model","category":"model","type":"select","currentValue":"[\"deepseek-official\",\"deepseek-v4-flash\"]","options":[{"group":"deepseek-official","name":"DeepSeek","options":[{"value":"[\"deepseek-official\",\"deepseek-v4-flash\"]","name":"deepseek-v4-flash"},{"value":"[\"deepseek-official\",\"deepseek-v4-pro\"]","name":"deepseek-v4-pro"}]}]}]}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","messageId":"{{messageId}}","content":{"type":"text","text":"The user wants me to run a specific bash command and then reply with DONE."}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_00_Rn2Mz1y8uZN62ukEXiNO2077","title":"bash","kind":"other","status":"in_progress","rawInput":{"command":"echo SNAPSHOT_OK","description":"Run echo SNAPSHOT_OK"}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_00_Rn2Mz1y8uZN62ukEXiNO2077","status":"completed","content":[{"type":"content","content":{"type":"text","text":"SNAPSHOT_OK\n"}}]}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","messageId":"{{messageId}}","content":{"type":"text","text":"The command executed successfully and printed SNAPSHOT_OK. Now I need to reply with the single word DONE."}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","messageId":"{{messageId}}","content":{"type":"text","text":"DONE"}}}}
|
||||
{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}}
|
||||
@@ -0,0 +1,111 @@
|
||||
/** Recorded-session replay through the shipped headless `dsh` profile. */
|
||||
|
||||
import { readFile, readdir } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { normalizeSessionSnapshot, parseSnapshotManifest, type NormalizeContext } from '@deepseek-ai/dsh-session-snapshot'
|
||||
import { LOADER_SMOKE_TEST_TIMEOUT_MS, runLoaderSmoke } from '@deepseek-ai/dsh-loader-smoke'
|
||||
|
||||
const repoRoot = fileURLToPath(new URL('../../', import.meta.url))
|
||||
const snapshotsRoot = fileURLToPath(new URL('./', import.meta.url))
|
||||
const dshBin = join(repoRoot, 'apps/cli/src/bin.ts')
|
||||
const tsconfigPath = join(repoRoot, 'tsconfig.json')
|
||||
const defaultCompositionDir = join(snapshotsRoot, 'text-turn')
|
||||
const basePatch = join(defaultCompositionDir, 'cordis.yml')
|
||||
const replayPatch = join(defaultCompositionDir, 'cordis.snapshot.yml')
|
||||
|
||||
interface JsonObject {
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
function contextOf(log: string): NormalizeContext {
|
||||
const header = JSON.parse(log.split('\n').find(line => line.trim() !== '') ?? '{}') as JsonObject
|
||||
return {
|
||||
sessionIds: typeof header.id === 'string' ? [header.id] : [],
|
||||
cwd: typeof header.cwd === 'string' ? header.cwd : '\0missing-cwd\0',
|
||||
}
|
||||
}
|
||||
|
||||
async function persistedSession(cwd: string): Promise<string> {
|
||||
const root = join(cwd, '.dsh', 'sessions')
|
||||
const files = (await readdir(root, { recursive: true }))
|
||||
.filter(file => file.endsWith('session.jsonl'))
|
||||
expect(files).toHaveLength(1)
|
||||
return readFile(join(root, files[0] as string), 'utf8')
|
||||
}
|
||||
|
||||
function records(log: string): JsonObject[] {
|
||||
return log.split(/\r?\n/)
|
||||
.filter(line => line.trim() !== '')
|
||||
.map(line => JSON.parse(line) as JsonObject)
|
||||
}
|
||||
|
||||
function taskFromSession(log: string): string {
|
||||
for (const record of records(log)) {
|
||||
if (record.type !== 'user/message') continue
|
||||
const data = record.data as JsonObject | undefined
|
||||
const source = data?.source as JsonObject | undefined
|
||||
if (source?.kind !== 'user' || !Array.isArray(data?.content)) continue
|
||||
const blocks = data.content as JsonObject[]
|
||||
if (blocks.length === 1 && blocks[0]?.type === 'text' && typeof blocks[0].text === 'string') {
|
||||
return blocks[0].text
|
||||
}
|
||||
}
|
||||
throw new Error('headless snapshot session has no single-text user task')
|
||||
}
|
||||
|
||||
function finalTextFromSession(log: string): string {
|
||||
const messages = records(log).flatMap((record) => {
|
||||
if (record.type !== 'assistant/message') return []
|
||||
const data = record.data as JsonObject | undefined
|
||||
const message = data?.message as JsonObject | undefined
|
||||
return message === undefined ? [] : [message]
|
||||
})
|
||||
const content = messages.at(-1)?.content
|
||||
if (!Array.isArray(content)) throw new Error('headless snapshot session has no final assistant message')
|
||||
return (content as JsonObject[])
|
||||
.flatMap(block => block.type === 'text' && typeof block.text === 'string' ? [block.text] : [])
|
||||
.join('')
|
||||
}
|
||||
|
||||
describe('headless recorded-session snapshots', () => {
|
||||
it.each(['text-turn', 'tool-call-turn'])('replays %s through dsh --profile headless', async (name) => {
|
||||
const scenarioDir = join(snapshotsRoot, name)
|
||||
const manifestPath = join(scenarioDir, 'snapshot.yml')
|
||||
expect(parseSnapshotManifest(await readFile(manifestPath, 'utf8'), manifestPath)).toMatchObject({
|
||||
version: 1,
|
||||
profile: 'headless',
|
||||
composition: 'default',
|
||||
recording: 'live',
|
||||
})
|
||||
const fixture = await readFile(join(scenarioDir, 'session.jsonl'), 'utf8')
|
||||
const task = taskFromSession(fixture)
|
||||
|
||||
let actual = ''
|
||||
const result = await runLoaderSmoke({
|
||||
label: 'tool-call-turn headless snapshot',
|
||||
tempDirPrefix: 'dsh-headless-session-snapshot-',
|
||||
binScript: dshBin,
|
||||
configPath: basePatch,
|
||||
binArgs: [
|
||||
'--profile', 'headless',
|
||||
'--patch', basePatch,
|
||||
'--patch', replayPatch,
|
||||
task,
|
||||
],
|
||||
tsconfigPath,
|
||||
env: {
|
||||
DSH_SNAPSHOT: 'replay',
|
||||
DSH_SNAPSHOT_FILE: join(scenarioDir, 'session.jsonl'),
|
||||
DSH_TELEMETRY_DISABLED: '1',
|
||||
},
|
||||
inspect: async (cwd) => { actual = await persistedSession(cwd) },
|
||||
})
|
||||
|
||||
expect(result.stdout).toBe(`${finalTextFromSession(fixture)}\n`)
|
||||
expect(result.stderr).toBe('')
|
||||
expect(normalizeSessionSnapshot(actual, contextOf(actual)))
|
||||
.toBe(normalizeSessionSnapshot(fixture, contextOf(fixture)))
|
||||
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
|
||||
})
|
||||
@@ -0,0 +1,40 @@
|
||||
# Replay patch shared by the ordinary headless snapshot composition. The model
|
||||
# script comes from the scenario's committed session JSONL.
|
||||
|
||||
- id: llm-deepseek
|
||||
name: '@deepseek-ai/dsh-llm-deepseek'
|
||||
disabled: true
|
||||
|
||||
- id: plugin-package-inventory-deepseek
|
||||
disabled: true
|
||||
|
||||
- id: session-title-llm
|
||||
disabled: true
|
||||
|
||||
- id: session-persistence-jsonl
|
||||
name: '@deepseek-ai/dsh-session-persistence-jsonl'
|
||||
config:
|
||||
root: !!js dshHomePath('sessions')
|
||||
compression: none
|
||||
|
||||
- id: sandbox
|
||||
name: '@deepseek-ai/dsh-sandbox-local'
|
||||
config:
|
||||
runnerCommand:
|
||||
- bash
|
||||
- -c
|
||||
- while [ "$1" != "--" ]; do shift; done; shift; exec "$@"
|
||||
- passthrough-runner
|
||||
runnerFailureSignatures:
|
||||
- 'passthrough-runner: profile rejected'
|
||||
|
||||
- insert:
|
||||
- id: llm-replay
|
||||
name: '@deepseek-ai/dsh-llm-replay'
|
||||
config:
|
||||
providers:
|
||||
- id: deepseek-official
|
||||
name: DeepSeek
|
||||
models:
|
||||
- id: deepseek-v4-flash
|
||||
- id: deepseek-v4-pro
|
||||
@@ -0,0 +1,80 @@
|
||||
# Live-recording patch shared by the ordinary headless snapshot composition.
|
||||
# The shipped profile owns application startup; this patch fixes only the
|
||||
# deterministic test composition and raw persistence needed for recording.
|
||||
|
||||
- id: llm-deepseek
|
||||
name: '@deepseek-ai/dsh-llm-deepseek'
|
||||
config:
|
||||
thinking: enabled
|
||||
reasoningEffort: max
|
||||
models:
|
||||
- id: deepseek-v4-flash
|
||||
- id: deepseek-v4-pro
|
||||
- id: deepseek-v4-flash-vision-exp
|
||||
inputModalities: [text, image]
|
||||
|
||||
- id: sandbox-policy
|
||||
name: '@deepseek-ai/dsh-sandbox-policy'
|
||||
config:
|
||||
mode: !!js "process.env.DSH_PERMISSION_MODE ?? (process.env.DSH_SNAPSHOT === undefined ? 'workspace-write' : 'danger-full-access')"
|
||||
workspaceRoot: !!js process.cwd()
|
||||
|
||||
- id: approval
|
||||
name: '@deepseek-ai/dsh-user-approval'
|
||||
config:
|
||||
policy: !!js "(process.env.DSH_PERMISSION_MODE ?? (process.env.DSH_SNAPSHOT === undefined ? 'workspace-write' : 'danger-full-access')) === 'danger-full-access' ? 'never' : 'ask'"
|
||||
|
||||
- id: session-persistence-jsonl
|
||||
name: '@deepseek-ai/dsh-session-persistence-jsonl'
|
||||
config:
|
||||
root: !!js dshHomePath('sessions')
|
||||
compression: none
|
||||
|
||||
- id: session-title-llm
|
||||
disabled: true
|
||||
|
||||
- id: system-prompt
|
||||
name: '@deepseek-ai/dsh-system-prompt'
|
||||
config:
|
||||
persona: |
|
||||
You are a coding assistant powered by the {{model}} model. Your working directory is {{cwd}}. Your bash tool runs under a file sandbox — a `[sandbox: file access denied …]` result is policy, not a command bug.
|
||||
|
||||
Verify your work by running the code or tests. Keep answers brief and factual.
|
||||
|
||||
- id: agent-instructions
|
||||
name: '@deepseek-ai/dsh-agent-instructions'
|
||||
config:
|
||||
maxBytes: 65536
|
||||
|
||||
- id: tool-subagent
|
||||
name: '@deepseek-ai/dsh-tool-subagent'
|
||||
config:
|
||||
provider: spawn
|
||||
toolName: subagent
|
||||
backgroundMode: continuable
|
||||
maxDepth: 1
|
||||
|
||||
- id: tool-subagent-fork
|
||||
name: '@deepseek-ai/dsh-tool-subagent'
|
||||
config:
|
||||
provider: fork
|
||||
toolName: subagent_fork
|
||||
backgroundMode: one-shot
|
||||
enableRunInBackground: false
|
||||
maxDepth: 1
|
||||
|
||||
- id: fs-sandbox
|
||||
name: '@deepseek-ai/dsh-fs-sandbox'
|
||||
config:
|
||||
cwd: !!js process.cwd()
|
||||
|
||||
- insert:
|
||||
- id: hooks-claude-code
|
||||
name: '@deepseek-ai/dsh-hooks-claude-code'
|
||||
config:
|
||||
configPath: ./hooks.json
|
||||
|
||||
- id: hooks-codex
|
||||
name: '@deepseek-ai/dsh-hooks-codex'
|
||||
config:
|
||||
configPath: ./codex-hooks.json
|
||||
@@ -0,0 +1,7 @@
|
||||
version: 1
|
||||
profile: headless
|
||||
composition: default
|
||||
recording: live
|
||||
header:
|
||||
class: default
|
||||
pin: true
|
||||
@@ -0,0 +1,4 @@
|
||||
version: 1
|
||||
profile: headless
|
||||
composition: default
|
||||
recording: live
|
||||
@@ -51,6 +51,7 @@ export default defineConfig({
|
||||
...(process.env.DSH_EXAMPLE_MODE === 'lib' ? ['apps/web/tests/**/*.snapshot.ts'] : []),
|
||||
'apps/cli/tests/**/*.snapshot.ts',
|
||||
'examples/*/tests/**/*.snapshot.ts',
|
||||
'snapshots/**/*.snapshot.ts',
|
||||
],
|
||||
// Replay never writes committed outputs and every scenario owns its
|
||||
// mutable runtime state (the subprocess suites use a unique temp dir and
|
||||
|
||||
Reference in New Issue
Block a user