mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-08-29 04:26:38 +00:00
test(ci): stabilize cross-platform consumer gates
This commit is contained in:
pku-xht
@@ -46,6 +46,9 @@ describe.skipIf(!hasPwsh)('tool-pwsh through a real Loader composition', () => {
|
||||
libBinScript: driver,
|
||||
configPath,
|
||||
tsconfigPath: repoTsconfig,
|
||||
// The self-hosted Windows pool can take roughly 40 seconds to boot this
|
||||
// real Loader composition under the full CI load.
|
||||
processTimeoutMs: 90_000,
|
||||
inspect: async (cwd) => {
|
||||
report = JSON.parse(await readFile(join(cwd, 'pwsh-loader-report.json'), 'utf8')) as PwshLoaderReport
|
||||
},
|
||||
@@ -59,5 +62,5 @@ describe.skipIf(!hasPwsh)('tool-pwsh through a real Loader composition', () => {
|
||||
expect(report?.foregroundText).toBe('loader-ok\n')
|
||||
expect(report?.backgroundText).toContain('loader-bg-ok')
|
||||
expect(report?.backgroundText).toContain('[status: completed, exit code: 0]')
|
||||
}, LOADER_SMOKE_TEST_TIMEOUT_MS)
|
||||
}, LOADER_SMOKE_TEST_TIMEOUT_MS + 75_000)
|
||||
})
|
||||
|
||||
@@ -4,6 +4,7 @@ import Loader from '@deepseek-ai/cordis-plugin-loader'
|
||||
import { chmodSync, existsSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { PassThrough, type Readable } from 'node:stream'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import SubagentRuntime from '@deepseek-ai/dsh-subagent'
|
||||
import type { Agent } from '@deepseek-ai/dsh-agent'
|
||||
@@ -121,6 +122,63 @@ function tapBoundedExitWait(child: SubprocessHandle, onWait: () => void): Subpro
|
||||
}
|
||||
}
|
||||
|
||||
function replaceProtocolStreams(
|
||||
child: SubprocessHandle,
|
||||
stdin: PassThrough,
|
||||
stdout: Readable,
|
||||
): SubprocessHandle {
|
||||
if (child.stdin === undefined) throw new Error('expected piped child stdin')
|
||||
stdin.pipe(child.stdin)
|
||||
return {
|
||||
pid: child.pid,
|
||||
stdin,
|
||||
stdout,
|
||||
stderr: child.stderr,
|
||||
collected: child.collected,
|
||||
done: child.done,
|
||||
terminate: () => { child.terminate() },
|
||||
waitForExit: (signal?: AbortSignal) => child.waitForExit(signal),
|
||||
}
|
||||
}
|
||||
|
||||
function closeProtocolImmediately(child: SubprocessHandle): SubprocessHandle {
|
||||
const stdout = new PassThrough()
|
||||
stdout.end()
|
||||
return replaceProtocolStreams(child, new PassThrough(), stdout)
|
||||
}
|
||||
|
||||
function closeProtocolOnPrompt(child: SubprocessHandle, onClose: () => void = () => {}): SubprocessHandle {
|
||||
if (child.stdout === undefined) throw new Error('expected piped child stdout')
|
||||
const stdin = new PassThrough()
|
||||
const stdout = new PassThrough()
|
||||
child.stdout.pipe(stdout)
|
||||
let requestText = ''
|
||||
let closed = false
|
||||
stdin.on('data', (chunk: Buffer) => {
|
||||
if (closed) return
|
||||
requestText += chunk.toString('utf8')
|
||||
if (!requestText.includes('"session/prompt"')) return
|
||||
closed = true
|
||||
child.stdout?.unpipe(stdout)
|
||||
stdout.end()
|
||||
onClose()
|
||||
})
|
||||
return replaceProtocolStreams(child, stdin, stdout)
|
||||
}
|
||||
|
||||
function replaceProcessOutcome(child: SubprocessHandle, outcome: SubprocessOutcome): SubprocessHandle {
|
||||
return {
|
||||
pid: child.pid,
|
||||
stdin: child.stdin,
|
||||
stdout: child.stdout,
|
||||
stderr: child.stderr,
|
||||
collected: child.collected,
|
||||
done: child.done.then(() => outcome),
|
||||
terminate: () => { child.terminate() },
|
||||
waitForExit: (signal?: AbortSignal) => child.waitForExit(signal),
|
||||
}
|
||||
}
|
||||
|
||||
describe('acpStopReason', () => {
|
||||
it('maps each ACP stop reason to the harness vocabulary', () => {
|
||||
expect(acpStopReason('end_turn')).toBe('completed')
|
||||
@@ -589,18 +647,16 @@ describe('dsh-subagent-acp', () => {
|
||||
)
|
||||
})
|
||||
|
||||
// Windows anonymous pipes do not surface a child stdout half-close while
|
||||
// the child stays alive.
|
||||
it.skipIf(process.platform === 'win32')('reports initialize-stage transport when the child closes the protocol but stays alive', async () => {
|
||||
it('reports initialize-stage transport when the child closes the protocol but stays alive', async () => {
|
||||
const error = await startAcpRun(request(), {
|
||||
command: process.execPath,
|
||||
args: [mockServer],
|
||||
cwd: process.cwd(),
|
||||
permission: 'reject',
|
||||
env: { MOCK_CLOSE_PROTOCOL_ON_INITIALIZE: '1' },
|
||||
env: {},
|
||||
disposeEofGraceMs: 50,
|
||||
disposeGraceMs: 50,
|
||||
spawn: spawnSubprocess,
|
||||
spawn: spec => closeProtocolImmediately(spawnSubprocess(spec)),
|
||||
}).catch((cause: unknown) => cause)
|
||||
expect(error).toBeInstanceOf(Error)
|
||||
expect((error as Error).message).toBe(
|
||||
@@ -938,18 +994,16 @@ describe('dsh-subagent-acp', () => {
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
// Windows anonymous pipes do not surface a child stdout half-close while
|
||||
// the child stays alive.
|
||||
it.skipIf(process.platform === 'win32')('classifies a prompt transport failure without copying SDK text', async () => {
|
||||
it('classifies a prompt transport failure without copying SDK text', async () => {
|
||||
const run = await startAcpRun(request('private prompt text'), {
|
||||
command: process.execPath,
|
||||
args: [mockServer],
|
||||
cwd: process.cwd(),
|
||||
permission: 'reject',
|
||||
env: { MOCK_CLOSE_PROTOCOL_ON_PROMPT: '1' },
|
||||
env: { MOCK_HANG: '1' },
|
||||
disposeEofGraceMs: 100,
|
||||
disposeGraceMs: 100,
|
||||
spawn: spawnSubprocess,
|
||||
spawn: spec => closeProtocolOnPrompt(spawnSubprocess(spec)),
|
||||
})
|
||||
const result = await run.result
|
||||
expect(result).toEqual({
|
||||
@@ -961,9 +1015,7 @@ describe('dsh-subagent-acp', () => {
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
// Windows anonymous pipes do not surface a child stdout half-close while
|
||||
// the child stays alive.
|
||||
it.skipIf(process.platform === 'win32')('lets local cancellation interrupt prompt-failure process observation', async () => {
|
||||
it('lets local cancellation interrupt prompt-failure process observation', async () => {
|
||||
const controller = new AbortController()
|
||||
const protocolEnded = Promise.withResolvers<undefined>()
|
||||
let boundedExitWaits = 0
|
||||
@@ -972,13 +1024,15 @@ describe('dsh-subagent-acp', () => {
|
||||
args: [mockServer],
|
||||
cwd: process.cwd(),
|
||||
permission: 'reject',
|
||||
env: { MOCK_CLOSE_PROTOCOL_ON_PROMPT: '1' },
|
||||
env: { MOCK_HANG: '1' },
|
||||
disposeEofGraceMs: 100,
|
||||
disposeGraceMs: 5000,
|
||||
spawn: (spec) => {
|
||||
const child = spawnSubprocess(spec)
|
||||
child.stdout?.once('end', () => { protocolEnded.resolve(undefined) })
|
||||
return tapBoundedExitWait(child, () => { boundedExitWaits += 1 })
|
||||
return closeProtocolOnPrompt(
|
||||
tapBoundedExitWait(child, () => { boundedExitWaits += 1 }),
|
||||
() => { protocolEnded.resolve(undefined) },
|
||||
)
|
||||
},
|
||||
})
|
||||
await protocolEnded.promise
|
||||
@@ -1006,6 +1060,29 @@ describe('dsh-subagent-acp', () => {
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
it('reports a signal-only process outcome', async () => {
|
||||
const run = await startAcpRun(request(), {
|
||||
command: process.execPath,
|
||||
args: [mockServer],
|
||||
cwd: process.cwd(),
|
||||
permission: 'reject',
|
||||
env: { MOCK_CRASH_AFTER_CHUNK: '1' },
|
||||
disposeEofGraceMs: DEFAULT_DISPOSE_EOF_GRACE_MS,
|
||||
disposeGraceMs: DEFAULT_DISPOSE_GRACE_MS,
|
||||
spawn: spec => replaceProcessOutcome(
|
||||
spawnSubprocess(spec),
|
||||
{ exitCode: null, signal: 'SIGTERM' },
|
||||
),
|
||||
})
|
||||
const result = await run.result
|
||||
expect(result).toEqual({
|
||||
output: [{ type: 'text', text: 'mock child answer' }],
|
||||
diagnostic: expectedFailure('stage: process; category: process-exit; signal: SIGTERM'),
|
||||
stopReason: 'error',
|
||||
})
|
||||
await run.dispose()
|
||||
})
|
||||
|
||||
it('rejects a spawn failure after provider-owned cleanup', async () => {
|
||||
const privateCommand = '/nonexistent/private/SECRET_TOKEN/acp-agent'
|
||||
const error = await startAcpRun(
|
||||
|
||||
@@ -1405,7 +1405,10 @@ describe('dsh-workflow-worker-thread', () => {
|
||||
const worker = (handle as unknown as { worker: Worker }).worker
|
||||
const logs: string[] = []
|
||||
ctx.on('workflow/log', (_info, message) => { logs.push(message) })
|
||||
await waitFor(() => { expect(logs).toContain('armed') })
|
||||
await waitFor(
|
||||
() => { expect(logs).toContain('armed') },
|
||||
process.platform === 'win32' ? 20_000 : 10_000,
|
||||
)
|
||||
handle.cancel('stop it')
|
||||
// The grace is deliberately huge: only the host-triggered worker death,
|
||||
// not the cancellation timer, settles this.
|
||||
@@ -1414,7 +1417,7 @@ describe('dsh-workflow-worker-thread', () => {
|
||||
expect(result.stopReason).toBe('cancelled')
|
||||
expect(result.error).toContain('stop it')
|
||||
await handle.dispose()
|
||||
}, 15_000)
|
||||
}, process.platform === 'win32' ? 30_000 : 15_000)
|
||||
})
|
||||
|
||||
describe('service API', () => {
|
||||
|
||||
@@ -437,6 +437,15 @@ describe('Node 24 lane ownership', () => {
|
||||
expect(subject.find(item => item.id === 'web-snapshot')).toMatchObject({
|
||||
displayCommand: 'DSH_SNAPSHOT=replay pnpm run test:web:built',
|
||||
env: { DSH_SNAPSHOT: 'replay' },
|
||||
after: [
|
||||
'publint',
|
||||
'lint-and-duplication',
|
||||
'snapshot',
|
||||
'expected-output',
|
||||
'doc-typecheck',
|
||||
'node-next-types',
|
||||
'built-bin-smoke',
|
||||
],
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
+17
-4
@@ -415,6 +415,18 @@ function ciArtifactGates(): Gate[] {
|
||||
function ciConsumerGates(): Gate[] {
|
||||
const builtTree = ['build']
|
||||
const validatedBuild = ['built-package-invariants']
|
||||
// The HMR web test starts `dev:web`, which rewrites the shared `lib/` and
|
||||
// `apps/web/dist/` trees. Let every build-artifact reader settle before that
|
||||
// writer starts; `after` preserves the web diagnostic even if a reader fails.
|
||||
const buildArtifactReaders = [
|
||||
'publint',
|
||||
'lint-and-duplication',
|
||||
'snapshot',
|
||||
'expected-output',
|
||||
'doc-typecheck',
|
||||
'node-next-types',
|
||||
'built-bin-smoke',
|
||||
]
|
||||
return [
|
||||
ciBuildGate(),
|
||||
pnpmScript('node-compat', 'check:node-compat', {
|
||||
@@ -429,7 +441,7 @@ function ciConsumerGates(): Gate[] {
|
||||
}),
|
||||
snapshotGate(validatedBuild),
|
||||
expectedOutputGate(validatedBuild),
|
||||
webSnapshotGate(validatedBuild),
|
||||
webSnapshotGate(validatedBuild, buildArtifactReaders),
|
||||
pnpmScript('doc-typecheck', 'doc-typecheck:contracts-ready', {
|
||||
needs: validatedBuild,
|
||||
env: { DSH_DOC_TYPECHECK_USE_BUILD_OUTPUT: '1' },
|
||||
@@ -442,7 +454,8 @@ function ciConsumerGates(): Gate[] {
|
||||
]
|
||||
}
|
||||
|
||||
function webSnapshotGate(needs: string[]): Gate {
|
||||
function webSnapshotGate(needs: string[], after?: string[]): Gate {
|
||||
const order = after === undefined ? { needs } : { needs, after }
|
||||
const workerRaw = process.env.DSH_WEB_SNAPSHOT_WORKERS
|
||||
if (workerRaw !== undefined && workerRaw !== '') {
|
||||
const workers = Number.parseInt(workerRaw, 10)
|
||||
@@ -453,7 +466,7 @@ function webSnapshotGate(needs: string[]): Gate {
|
||||
label: 'web browser snapshot',
|
||||
displayCommand: `DSH_SNAPSHOT=replay DSH_WEB_SNAPSHOT_WORKERS=${workers} pnpm run test:web:ci`,
|
||||
env: { DSH_SNAPSHOT: 'replay' },
|
||||
needs,
|
||||
...order,
|
||||
streamOutput: true,
|
||||
})
|
||||
}
|
||||
@@ -461,7 +474,7 @@ function webSnapshotGate(needs: string[]): Gate {
|
||||
label: 'web browser snapshot',
|
||||
displayCommand: 'DSH_SNAPSHOT=replay pnpm run test:web:built',
|
||||
env: { DSH_SNAPSHOT: 'replay' },
|
||||
needs,
|
||||
...order,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -376,7 +376,7 @@
|
||||
},
|
||||
{
|
||||
"name": "str_replace_editor",
|
||||
"description": "Custom editing tool for viewing, creating and editing files\n* State is persistent across command calls and discussions with the user\n* If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep\n* The `create` command cannot be used if the specified `path` already exists as a file\n* If a `command` generates a long output, it will be truncated and marked with `<response clipped>`\n\nNotes for using the `str_replace` command:\n* The `old_str` parameter should match EXACTLY one or more consecutive lines from the original file. Be mindful of whitespaces!\n* If the `old_str` parameter is not unique in the file, the replacement will not be performed. Make sure to include enough context in `old_str` to make it unique\n* The `new_str` parameter should contain the edited lines that should replace the `old_str`",
|
||||
"description": "Custom editing tool for viewing, creating and editing files\n* State is persistent across command calls and discussions with the user\n* If `path` is a file, `view` displays the result of applying `cat -n`. If `path` is a directory, `view` lists non-hidden files and directories up to 2 levels deep\n* The `create` command cannot be used if the specified `path` already exists as a file\n* If a `command` generates a long output, it will be truncated and marked with `<response clipped>`\n* A null placeholder for a parameter unused by the selected command is treated as omitted. Required parameters still need values; omit `str_replace.new_str` rather than setting it to null when deleting a match\n\nNotes for using the `str_replace` command:\n* The `old_str` parameter should match EXACTLY one or more consecutive lines from the original file. Be mindful of whitespaces!\n* If the `old_str` parameter is not unique in the file, the replacement will not be performed. Make sure to include enough context in `old_str` to make it unique\n* The `new_str` parameter should contain the edited lines that should replace the `old_str`",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -395,27 +395,62 @@
|
||||
"description": "Absolute path to file or directory, e.g. `/repo/file.py` or `/repo`."
|
||||
},
|
||||
"file_text": {
|
||||
"type": "string",
|
||||
"description": "Required parameter of `create` command, with the content of the file to be created."
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Required string parameter of `create` command, with the content of the file to be created. A null placeholder is treated as omitted by commands that do not use this parameter."
|
||||
},
|
||||
"insert_line": {
|
||||
"type": "integer",
|
||||
"description": "Required parameter of `insert` command. The `new_str` will be inserted AFTER the line `insert_line` of `path`."
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "integer"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Required integer parameter of `insert` command. The `new_str` will be inserted AFTER the line `insert_line` of `path`. A null placeholder is treated as omitted by commands that do not use this parameter."
|
||||
},
|
||||
"new_str": {
|
||||
"type": "string",
|
||||
"description": "Optional parameter of `str_replace` command containing the new string (if not given, no string will be added). Required parameter of `insert` command containing the string to insert."
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Optional string parameter of `str_replace` command containing the new string (if omitted, no string will be added). Required string parameter of `insert` command containing the string to insert. A null placeholder is accepted only by commands that do not use this parameter."
|
||||
},
|
||||
"old_str": {
|
||||
"type": "string",
|
||||
"description": "Required parameter of `str_replace` command containing the string in `path` to replace."
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Required string parameter of `str_replace` command containing the string in `path` to replace. A null placeholder is treated as omitted by commands that do not use this parameter."
|
||||
},
|
||||
"view_range": {
|
||||
"type": "array",
|
||||
"description": "Optional parameter of `view` command when `path` points to a file. If none is given, the full file is shown. If provided, the file will be shown in the indicated line number range, e.g. [11, 12] will show lines 11 and 12. Indexing at 1 to start. Setting `[start_line, -1]` shows all lines from `start_line` to the end of the file.",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "integer"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Optional parameter of `view` command when `path` points to a file. If omitted or null, the full file is shown. If provided, the file will be shown in the indicated line number range, e.g. [11, 12] will show lines 11 and 12. Indexing at 1 to start. Setting `[start_line, -1]` shows all lines from `start_line` to the end of the file."
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
|
||||
Reference in New Issue
Block a user