mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-08-29 04:26:38 +00:00
test(acp): align assembled automation coverage
This commit is contained in:
@@ -53,10 +53,24 @@ const hasSeatbelt = process.platform === 'darwin' && spawnSync('sandbox-exec', [
|
||||
}).status === 0
|
||||
const hasRunner = hasBwrap || hasSeatbelt
|
||||
|
||||
const STANDARD_EXECUTION_UPDATES = new Set([
|
||||
'agent_message_chunk',
|
||||
'agent_thought_chunk',
|
||||
'tool_call',
|
||||
'tool_call_update',
|
||||
'usage_update',
|
||||
])
|
||||
|
||||
interface Spawned extends LaunchedAcpTestAgent {
|
||||
permissionRequests: RequestPermissionRequest[]
|
||||
}
|
||||
|
||||
/** Require a model answer while allowing every standard semantic execution update. */
|
||||
function expectStandardExecutionUpdates(updates: LaunchedAcpTestAgent['updates']): void {
|
||||
expect(updates.some(update => update.sessionUpdate === 'agent_message_chunk')).toBe(true)
|
||||
expect(updates.every(update => STANDARD_EXECUTION_UPDATES.has(update.sessionUpdate))).toBe(true)
|
||||
}
|
||||
|
||||
/** Boot the example with an optional sandbox override; the scripted client answers every permission prompt with `answer`. */
|
||||
function launchExampleAcpAgent(
|
||||
cwd: string,
|
||||
@@ -112,7 +126,9 @@ describe('default sandbox composition keyless smoke (real cordis.yml via the Loa
|
||||
const init = await client.initialize({ protocolVersion: PROTOCOL_VERSION, clientCapabilities: {} })
|
||||
expect(init.protocolVersion).toBe(PROTOCOL_VERSION)
|
||||
expect(init.agentCapabilities).toEqual({
|
||||
mcpCapabilities: { http: true },
|
||||
promptCapabilities: { image: false, audio: false, embeddedContext: false },
|
||||
sessionCapabilities: { close: {}, list: {}, resume: {} },
|
||||
})
|
||||
const { sessionId } = await client.newSession({ cwd: workdir, mcpServers: [] })
|
||||
expect(sessionId.length).toBeGreaterThan(0)
|
||||
@@ -136,7 +152,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || !hasRunner)('default sandbox co
|
||||
}],
|
||||
})
|
||||
expect(['end_turn', 'max_tokens']).toContain(res.stopReason)
|
||||
expect(updates.every(update => update.sessionUpdate === 'agent_message_chunk')).toBe(true)
|
||||
expectStandardExecutionUpdates(updates)
|
||||
|
||||
// The WORLD: the approved escalated retry landed the write.
|
||||
const proof = await readFile(join(workdir, 'escalated.txt'), 'utf8')
|
||||
@@ -168,7 +184,7 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY || !hasRunner)('default sandbox co
|
||||
}],
|
||||
})
|
||||
expect(['end_turn', 'max_tokens']).toContain(res.stopReason)
|
||||
expect(updates.every(update => update.sessionUpdate === 'agent_message_chunk')).toBe(true)
|
||||
expectStandardExecutionUpdates(updates)
|
||||
|
||||
// The WORLD: rejected means the file never appeared.
|
||||
await expect(readFile(join(workdir, 'refused.txt'), 'utf8')).rejects.toThrow()
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}}
|
||||
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"GOAL READY"}}}}
|
||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"mcpCapabilities":{"http":true},"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false},"sessionCapabilities":{"close":{},"list":{},"resume":{}}},"authMethods":[]}}
|
||||
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}","configOptions":[{"id":"model","name":"Model","category":"model","type":"select","currentValue":"[\"deepseek-official\",\"deepseek-v4-flash\"]","options":[{"group":"deepseek-official","name":"DeepSeek","options":[{"value":"[\"deepseek-official\",\"deepseek-v4-flash\"]","name":"deepseek-v4-flash"},{"value":"[\"deepseek-official\",\"deepseek-v4-pro\"]","name":"deepseek-v4-pro"}]}]}]}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_goal_create","title":"create_goal","kind":"other","status":"in_progress","rawInput":{"objective":"Finish the ACP goal-round-driver snapshot proof","max_goal_rounds":2}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_goal_create","status":"completed","content":[{"type":"content","content":{"type":"text","text":"{\"goal\":{\"id\":\"goal-{{sessionId}}\",\"revision\":1,\"objective\":\"Finish the ACP goal-round-driver snapshot proof\",\"phase\":\"active\",\"roundsStarted\":0,\"maxGoalRounds\":2},\"activation\":\"armed\"}"}}]}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_goal_get","title":"get_goal","kind":"other","status":"in_progress","rawInput":{}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_goal_get","status":"completed","content":[{"type":"content","content":{"type":"text","text":"{\"goal\":{\"id\":\"goal-{{sessionId}}\",\"revision\":1,\"objective\":\"Finish the ACP goal-round-driver snapshot proof\",\"phase\":\"active\",\"roundsStarted\":0,\"maxGoalRounds\":2},\"activation\":\"armed\"}"}}]}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","messageId":"{{messageId}}","content":{"type":"text","text":"GOAL READY"}}}}
|
||||
{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"GOAL ROUND ONE"}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"partial"}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","messageId":"{{messageId}}","content":{"type":"text","text":"GOAL ROUND ONE"}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","messageId":"{{messageId}}","content":{"type":"text","text":"partial"}}}}
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false}},"authMethods":[]}}
|
||||
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}"}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"GOAL READY"}}}}
|
||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"mcpCapabilities":{"http":true},"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false},"sessionCapabilities":{"close":{},"list":{},"resume":{}}},"authMethods":[]}}
|
||||
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}","configOptions":[{"id":"model","name":"Model","category":"model","type":"select","currentValue":"[\"deepseek-official\",\"deepseek-v4-flash\"]","options":[{"group":"deepseek-official","name":"DeepSeek","options":[{"value":"[\"deepseek-official\",\"deepseek-v4-flash\"]","name":"deepseek-v4-flash"},{"value":"[\"deepseek-official\",\"deepseek-v4-pro\"]","name":"deepseek-v4-pro"}]}]}]}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_goal_create","title":"create_goal","kind":"other","status":"in_progress","rawInput":{"objective":"Finish the ACP goal wrap-up snapshot proof","max_goal_rounds":2}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_goal_create","status":"completed","content":[{"type":"content","content":{"type":"text","text":"{\"goal\":{\"id\":\"goal-{{sessionId}}\",\"revision\":1,\"objective\":\"Finish the ACP goal wrap-up snapshot proof\",\"phase\":\"active\",\"roundsStarted\":0,\"maxGoalRounds\":2},\"activation\":\"armed\"}"}}]}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","messageId":"{{messageId}}","content":{"type":"text","text":"GOAL READY"}}}}
|
||||
{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"GOAL WRAP-UP: the snapshot objective is achieved and this closing message reaches the user."}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_goal_complete","title":"update_goal","kind":"other","status":"in_progress","rawInput":{"goal_id":"goal-{{sessionId}}","revision":1,"action":"complete"}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_goal_complete","status":"completed","content":[{"type":"content","content":{"type":"text","text":"{\"goal\":{\"id\":\"goal-{{sessionId}}\",\"revision\":2,\"objective\":\"Finish the ACP goal wrap-up snapshot proof\",\"phase\":\"complete\",\"roundsStarted\":1,\"maxGoalRounds\":2},\"activation\":\"disarmed\"}"}}]}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","messageId":"{{messageId}}","content":{"type":"text","text":"GOAL WRAP-UP: the snapshot objective is achieved and this closing message reaches the user."}}}}
|
||||
|
||||
@@ -24,6 +24,14 @@ const AGENT: AgentUnderTest = {
|
||||
tsconfigPath: fileURLToPath(new URL('../../../tsconfig.json', import.meta.url)),
|
||||
}
|
||||
|
||||
const STANDARD_EXECUTION_UPDATES = new Set([
|
||||
'agent_message_chunk',
|
||||
'agent_thought_chunk',
|
||||
'tool_call',
|
||||
'tool_call_update',
|
||||
'usage_update',
|
||||
])
|
||||
|
||||
let spawned: LaunchedAcpTestAgent | undefined
|
||||
let workdir: string | undefined
|
||||
|
||||
@@ -65,8 +73,8 @@ describe.skipIf(!process.env.DEEPSEEK_API_KEY)('acp-agent e2e: a PreToolUse hook
|
||||
// Assert the denied operation independently of the model response.
|
||||
await expect(access(join(workdir, 'proof.txt'))).rejects.toThrow()
|
||||
|
||||
// ACP publishes only the committed answer; hook/tool trace stays in the session log.
|
||||
expect(updates.length).toBeGreaterThan(0)
|
||||
expect(updates.every(update => update.sessionUpdate === 'agent_message_chunk')).toBe(true)
|
||||
// ACP publishes committed semantic execution facts, never hook internals or UI projections.
|
||||
expect(updates.some(update => update.sessionUpdate === 'agent_message_chunk')).toBe(true)
|
||||
expect(updates.every(update => STANDARD_EXECUTION_UPDATES.has(update.sessionUpdate))).toBe(true)
|
||||
}, 180_000)
|
||||
})
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":1,"agentInfo":{"name":"deepseek-harness-acp","version":"0.0.1"},"agentCapabilities":{"mcpCapabilities":{"http":true},"promptCapabilities":{"image":false,"audio":false,"embeddedContext":false},"sessionCapabilities":{"close":{},"list":{},"resume":{}}},"authMethods":[]}}
|
||||
{"jsonrpc":"2.0","id":2,"result":{"sessionId":"{{sessionId}}","configOptions":[{"id":"model","name":"Model","category":"model","type":"select","currentValue":"[\"deepseek-official\",\"deepseek-v4-flash\"]","options":[{"group":"deepseek-official","name":"DeepSeek","options":[{"value":"[\"deepseek-official\",\"deepseek-v4-flash\"]","name":"deepseek-v4-flash"},{"value":"[\"deepseek-official\",\"deepseek-v4-pro\"]","name":"deepseek-v4-pro"}]}]},{"id":"reasoning_effort","name":"Reasoning effort","category":"thought_level","type":"select","currentValue":"max","options":[{"value":"off","name":"off"},{"value":"low","name":"low"},{"value":"high","name":"high"},{"value":"max","name":"max"}]}]}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_thought_chunk","messageId":"{{messageId}}","content":{"type":"text","text":"The user wants me to read data.txt first, then write to replace its contents with the exact line, then reply DONE."}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"usage_update","used":6311,"size":1000000}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"usage_update","used":"{{usedTokens}}","size":1000000}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_00_Jxz49JNt6i4oaDnzes2I0794","title":"read","kind":"other","status":"in_progress","rawInput":{"file_path":"data.txt"}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_00_Jxz49JNt6i4oaDnzes2I0794","status":"completed","content":[{"type":"content","content":{"type":"text","text":"<path>{{cwd}}/data.txt</path>\n<type>file</type>\n<content>\n1: original contents\n\n(End of file - total 1 lines)\n</content>"}}]}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"usage_update","used":6414,"size":1000000}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"usage_update","used":"{{usedTokens}}","size":1000000}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call","toolCallId":"call_00_ET_7mLiYX652hJA9GW6d1bl4653","title":"write","kind":"other","status":"in_progress","rawInput":{"file_path":"data.txt","content":"The replacement line is deliberately longer than the configured sixty-four byte diff-basis bound."}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"tool_call_update","toolCallId":"call_00_ET_7mLiYX652hJA9GW6d1bl4653","status":"completed","content":[{"type":"content","content":{"type":"text","text":"<path>{{cwd}}/data.txt</path>\n<type>file</type>\n<content>\nUpdated file\n</content>"}}]}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"agent_message_chunk","messageId":"{{messageId}}","content":{"type":"text","text":"DONE"}}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"usage_update","used":6473,"size":1000000}}}
|
||||
{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"{{sessionId}}","update":{"sessionUpdate":"usage_update","used":"{{usedTokens}}","size":1000000}}}
|
||||
{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}}
|
||||
|
||||
@@ -65,12 +65,12 @@
|
||||
"@deepseek-ai/dsh-agent-spine-demo": "workspace:^",
|
||||
"@deepseek-ai/dsh-app-boot": "workspace:^",
|
||||
"@deepseek-ai/dsh-invariants": "workspace:^",
|
||||
"@deepseek-ai/dsh-llm": "workspace:^",
|
||||
"@deepseek-ai/dsh-session-checkpoint-policy": "workspace:^",
|
||||
"@deepseek-ai/dsh-session-persistence-jsonl": "workspace:^",
|
||||
"@deepseek-ai/dsh-session-query": "workspace:^",
|
||||
"@deepseek-ai/dsh-session-query-sqlite": "workspace:^",
|
||||
"@deepseek-ai/dsh-system-prompt": "workspace:^",
|
||||
"@deepseek-ai/dsh-token-meter": "workspace:^",
|
||||
"@deepseek-ai/dsh-tools": "workspace:^",
|
||||
"@deepseek-ai/dsh-agent-instructions": "workspace:^",
|
||||
"@deepseek-ai/cordis": "workspace:^",
|
||||
|
||||
@@ -12,4 +12,4 @@
|
||||
workspaceContext: false
|
||||
|
||||
- id: token-meter
|
||||
name: '@deepseek-ai/dsh-token-meter'
|
||||
name: '../../../llm/token-meter/src/index.ts'
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
const SESSION_ID = '{{sessionId}}'
|
||||
const MESSAGE_ID = '{{messageId}}'
|
||||
const USED_TOKENS = '{{usedTokens}}'
|
||||
const CWD = '{{cwd}}'
|
||||
const SYSTEM = '{{system}}'
|
||||
const TOOLS = '{{tools}}'
|
||||
@@ -193,6 +194,10 @@ function scrubValue(value: unknown, ctx: NormalizeContext, cwdPathMode: CwdPathM
|
||||
if (value !== null && typeof value === 'object') {
|
||||
const out: Record<string, unknown> = {}
|
||||
for (const [k, v] of Object.entries(value)) out[k] = scrubValue(v, ctx, cwdPathMode, k)
|
||||
if (
|
||||
(value as { sessionUpdate?: unknown }).sessionUpdate === 'usage_update'
|
||||
&& typeof (value as { used?: unknown }).used === 'number'
|
||||
) out.used = USED_TOKENS
|
||||
return out
|
||||
}
|
||||
return value
|
||||
|
||||
@@ -68,6 +68,27 @@ describe('normalizeStdout', () => {
|
||||
expect(out).toContain('"messageId":"{{messageId}}"')
|
||||
})
|
||||
|
||||
it('stabilizes path-dependent context occupancy without hiding capacity', () => {
|
||||
const raw = JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
method: 'session/update',
|
||||
params: {
|
||||
sessionId: ctx.sessionIds[0],
|
||||
update: { sessionUpdate: 'usage_update', used: 6_438, size: 1_000_000 },
|
||||
},
|
||||
})
|
||||
|
||||
const frame = JSON.parse(normalizeStdout(raw, ctx)) as {
|
||||
params: { update: { used: string; size: number } }
|
||||
}
|
||||
|
||||
expect(frame.params.update).toEqual({
|
||||
sessionUpdate: 'usage_update',
|
||||
used: '{{usedTokens}}',
|
||||
size: 1_000_000,
|
||||
})
|
||||
})
|
||||
|
||||
it('scrubs cwd at file URI and chained-punctuation boundaries', () => {
|
||||
const raw = JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
|
||||
Generated
+3
-3
@@ -3991,6 +3991,9 @@ importers:
|
||||
'@deepseek-ai/dsh-invariants':
|
||||
specifier: workspace:^
|
||||
version: link:../../runtime-diagnostics/invariants
|
||||
'@deepseek-ai/dsh-llm':
|
||||
specifier: workspace:^
|
||||
version: link:../../llm/llm
|
||||
'@deepseek-ai/dsh-session-checkpoint-policy':
|
||||
specifier: workspace:^
|
||||
version: link:../../session/session-checkpoint-policy
|
||||
@@ -4006,9 +4009,6 @@ importers:
|
||||
'@deepseek-ai/dsh-system-prompt':
|
||||
specifier: workspace:^
|
||||
version: link:../../core/system-prompt
|
||||
'@deepseek-ai/dsh-token-meter':
|
||||
specifier: workspace:^
|
||||
version: link:../../llm/token-meter
|
||||
'@deepseek-ai/dsh-tools':
|
||||
specifier: workspace:^
|
||||
version: link:../../core/tools
|
||||
|
||||
Reference in New Issue
Block a user