mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-08-29 04:26:38 +00:00
Tianyi Cui
28442337cf
Make the checked-in minimal SDK overlay disable both one-shot shell rows and mount exactly one persistent PTY stack: Bash on Linux/macOS and PowerShell on Windows. The SDK server, explicit dsh home, persistence, editor, timeout, and reduced tool catalog remain unchanged. Update the runnable example and tutorial to list Windows x64 as supported, describe the platform-selected shell, and remove the obsolete POSIX-only restriction. This keeps the documented first Python task executable through the packaged Windows dsh profile instead of advertising a Linux-only overlay on a Windows-capable SDK.
299 lines
12 KiB
TypeScript
299 lines
12 KiB
TypeScript
import { createServer } from 'node:http'
|
|
import { mkdtemp, readFile, readdir, rm } from 'node:fs/promises'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
import { fileURLToPath } from 'node:url'
|
|
import { promisify } from 'node:util'
|
|
import { zstdDecompress } from 'node:zlib'
|
|
import { execa } from 'execa'
|
|
import { describe, expect, it } from 'vitest'
|
|
|
|
const binScript = fileURLToPath(new URL('../../../apps/cli/src/bin.ts', import.meta.url))
|
|
const repoRoot = fileURLToPath(new URL('../../..', import.meta.url))
|
|
const decompress = promisify(zstdDecompress)
|
|
|
|
function waitForLine(
|
|
lines: string[],
|
|
predicate: (value: Record<string, unknown>) => boolean,
|
|
stderr: () => string,
|
|
): Promise<Record<string, unknown>> {
|
|
return new Promise((resolve, reject) => {
|
|
const deadline = Date.now() + 30_000
|
|
const poll = (): void => {
|
|
while (lines.length > 0) {
|
|
const line = lines.shift()!
|
|
if (!line.trim()) continue
|
|
try {
|
|
const value = JSON.parse(line) as Record<string, unknown>
|
|
if (predicate(value)) {
|
|
resolve(value)
|
|
return
|
|
}
|
|
} catch {
|
|
reject(new Error(`non-JSON stdout from JSON-RPC agent runtime: ${line}`))
|
|
return
|
|
}
|
|
}
|
|
if (Date.now() >= deadline) {
|
|
reject(new Error(`timed out waiting for JSON-RPC response; stderr=${stderr()}`))
|
|
return
|
|
}
|
|
setTimeout(poll, 10)
|
|
}
|
|
poll()
|
|
})
|
|
}
|
|
|
|
describe('Python SDK dsh profile keyless smoke', () => {
|
|
it.each([
|
|
{ label: 'reports max-token turns with the default mapping config', envValue: undefined },
|
|
{ label: 'reports max-token turns with mapping enabled through env', envValue: 'true' },
|
|
{ label: 'reports max-token turns with mapping disabled through env', envValue: 'false' },
|
|
])('$label', async ({ envValue }) => {
|
|
const root = await mkdtemp(join(tmpdir(), 'dsh-python-sdk-runtime-smoke-'))
|
|
const modelRequests: Record<string, unknown>[] = []
|
|
const modelServer = createServer((request, response) => {
|
|
let body = ''
|
|
request.setEncoding('utf8')
|
|
request.on('data', (chunk: string) => { body += chunk })
|
|
request.on('end', () => {
|
|
modelRequests.push(JSON.parse(body) as Record<string, unknown>)
|
|
response.writeHead(200, { 'content-type': 'text/event-stream' })
|
|
response.write('data: {"choices":[{"delta":{"role":"assistant","content":null}}]}\n\n')
|
|
response.write('data: {"choices":[{"delta":{"content":"done"}}]}\n\n')
|
|
response.write('data: {"choices":[{"delta":{},"finish_reason":"length"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}\n\n')
|
|
response.end('data: [DONE]\n\n')
|
|
})
|
|
})
|
|
await new Promise<void>(resolve => modelServer.listen(0, '127.0.0.1', resolve))
|
|
const address = modelServer.address()
|
|
if (address === null || typeof address === 'string') throw new Error('model server did not bind a TCP port')
|
|
// The line-predicate protocol driving below is the genuinely custom part;
|
|
// execa owns spawn, the deadline, and exit settlement around it.
|
|
const child = execa(process.execPath, [
|
|
'--import',
|
|
'tsx/esm',
|
|
binScript,
|
|
'--profile',
|
|
'sdk',
|
|
], {
|
|
cwd: repoRoot,
|
|
env: {
|
|
DSH_HOME: join(root, '.dsh'),
|
|
DSH_PERMISSION_MODE: 'danger-full-access',
|
|
DSH_TELEMETRY_DISABLED: '1',
|
|
DEEPSEEK_API_KEY: 'keyless-smoke-no-call',
|
|
DEEPSEEK_BASE_URL: `http://127.0.0.1:${address.port}`,
|
|
...(envValue === undefined ? {} : { DSH_MAX_TOKENS_AS_SUCCESS: envValue }),
|
|
},
|
|
timeout: 35_000,
|
|
killSignal: 'SIGKILL',
|
|
reject: false,
|
|
})
|
|
const lines: string[] = []
|
|
let stdoutBuffer = ''
|
|
let stderr = ''
|
|
child.stdout.on('data', (chunk: Buffer) => {
|
|
stdoutBuffer += chunk.toString('utf8')
|
|
const parts = stdoutBuffer.split('\n')
|
|
stdoutBuffer = parts.pop() ?? ''
|
|
lines.push(...parts)
|
|
})
|
|
child.stderr.on('data', (chunk: Buffer) => { stderr += chunk.toString('utf8') })
|
|
|
|
try {
|
|
child.stdin.write(`${JSON.stringify({
|
|
jsonrpc: '2.0',
|
|
id: 1,
|
|
method: 'initialize',
|
|
params: { cwd: root, provider: 'deepseek-official', model: 'deepseek-v4-pro', maxTokens: 1234 },
|
|
})}\n`)
|
|
const initialized = await waitForLine(lines, value => value.id === 1, () => stderr)
|
|
expect(initialized).toMatchObject({
|
|
jsonrpc: '2.0',
|
|
id: 1,
|
|
result: { serverInfo: { name: 'deepseek-harness-sdk-runtime' } },
|
|
})
|
|
|
|
child.stdin.write(`${JSON.stringify({
|
|
jsonrpc: '2.0',
|
|
id: 2,
|
|
method: 'session/prompt',
|
|
params: { sessionId: 'main', contentBlocks: [{ type: 'text', text: 'inspect tools' }] },
|
|
})}\n`)
|
|
const prompt = await waitForLine(lines, value => value.id === 2, () => stderr)
|
|
expect(prompt).toMatchObject({
|
|
jsonrpc: '2.0',
|
|
id: 2,
|
|
result: { messageId: expect.any(String) as unknown },
|
|
})
|
|
const turnEnd = await waitForLine(lines, (value) => {
|
|
if (value.method !== 'session.event') return false
|
|
const params = value.params as Record<string, unknown> | undefined
|
|
const event = params?.event as Record<string, unknown> | undefined
|
|
return params?.sessionId === 'main' && event?.type === 'turn/end'
|
|
}, () => stderr)
|
|
expect(turnEnd).toMatchObject({
|
|
jsonrpc: '2.0',
|
|
method: 'session.event',
|
|
params: {
|
|
sessionId: 'main',
|
|
event: {
|
|
type: 'turn/end',
|
|
data: { reason: { kind: 'max-tokens' } },
|
|
},
|
|
},
|
|
})
|
|
const tools = modelRequests[0]?.tools as { function?: { name?: string } }[]
|
|
expect(modelRequests[0]?.max_tokens).toBe(1234)
|
|
expect(tools.map(tool => tool.function?.name)).toContain('list_subagent_models')
|
|
|
|
child.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id: 3, method: 'shutdown' })}\n`)
|
|
const shutdown = await waitForLine(lines, value => value.id === 3, () => stderr)
|
|
expect(shutdown).toMatchObject({ jsonrpc: '2.0', id: 3, result: {} })
|
|
const exit = await child
|
|
expect(exit.exitCode, `signal=${String(exit.signal)}; stderr=${stderr}`).toBe(0)
|
|
const sessionsRoot = join(root, '.dsh', 'sessions')
|
|
const files = await readdir(sessionsRoot, { recursive: true })
|
|
const log = files.find(file => file.endsWith('.jsonl.zstd'))
|
|
expect(log).toBeDefined()
|
|
const compressed = await readFile(join(sessionsRoot, log!))
|
|
expect(compressed.subarray(0, 4).toString('hex')).toBe('28b52ffd')
|
|
expect(JSON.parse((await decompress(compressed)).toString())).toMatchObject({ type: 'session', id: 'main' })
|
|
} finally {
|
|
// No-op after exit; reject: false settles on every outcome, so cleanup never races teardown.
|
|
child.kill('SIGKILL')
|
|
await child
|
|
await new Promise<void>(resolve => modelServer.close(() => { resolve() }))
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
}, 40_000)
|
|
|
|
it('boots the standalone minimal profile with its exact model-facing roster', async () => {
|
|
const root = await mkdtemp(join(tmpdir(), 'dsh-python-sdk-minimal-'))
|
|
const modelRequests: Record<string, unknown>[] = []
|
|
const modelServer = createServer((request, response) => {
|
|
let body = ''
|
|
request.setEncoding('utf8')
|
|
request.on('data', (chunk: string) => { body += chunk })
|
|
request.on('end', () => {
|
|
modelRequests.push(JSON.parse(body) as Record<string, unknown>)
|
|
response.writeHead(200, { 'content-type': 'text/event-stream' })
|
|
response.write('data: {"choices":[{"delta":{"role":"assistant","content":null}}]}\n\n')
|
|
response.write('data: {"choices":[{"delta":{"content":"done"}}]}\n\n')
|
|
response.write('data: {"choices":[{"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}\n\n')
|
|
response.end('data: [DONE]\n\n')
|
|
})
|
|
})
|
|
await new Promise<void>(resolve => modelServer.listen(0, '127.0.0.1', resolve))
|
|
const address = modelServer.address()
|
|
if (address === null || typeof address === 'string') throw new Error('model server did not bind a TCP port')
|
|
const child = execa(process.execPath, [
|
|
'--import',
|
|
'tsx/esm',
|
|
binScript,
|
|
'--profile',
|
|
'sdk-minimal',
|
|
], {
|
|
cwd: repoRoot,
|
|
env: {
|
|
DSH_HOME: join(root, '.dsh'),
|
|
DSH_SYSTEM_PROMPT: 'Minimal allowlist prompt.',
|
|
DEEPSEEK_API_KEY: 'keyless-smoke-no-call',
|
|
DEEPSEEK_BASE_URL: `http://127.0.0.1:${address.port}`,
|
|
},
|
|
timeout: 35_000,
|
|
killSignal: 'SIGKILL',
|
|
reject: false,
|
|
})
|
|
const lines: string[] = []
|
|
let stdoutBuffer = ''
|
|
let stderr = ''
|
|
child.stdout.on('data', (chunk: Buffer) => {
|
|
stdoutBuffer += chunk.toString('utf8')
|
|
const parts = stdoutBuffer.split('\n')
|
|
stdoutBuffer = parts.pop() ?? ''
|
|
lines.push(...parts)
|
|
})
|
|
child.stderr.on('data', (chunk: Buffer) => { stderr += chunk.toString('utf8') })
|
|
|
|
try {
|
|
child.stdin.write(`${JSON.stringify({
|
|
jsonrpc: '2.0',
|
|
id: 1,
|
|
method: 'initialize',
|
|
params: { cwd: root, provider: 'deepseek-official', model: 'deepseek-v4-pro' },
|
|
})}\n`)
|
|
await waitForLine(lines, value => value.id === 1, () => stderr)
|
|
child.stdin.write(`${JSON.stringify({
|
|
jsonrpc: '2.0',
|
|
id: 2,
|
|
method: 'session/prompt',
|
|
params: { sessionId: 'minimal', contentBlocks: [{ type: 'text', text: 'inspect tools' }] },
|
|
})}\n`)
|
|
await waitForLine(lines, (value) => {
|
|
const params = value.params as Record<string, unknown> | undefined
|
|
const event = params?.event as Record<string, unknown> | undefined
|
|
return params?.sessionId === 'minimal' && event?.type === 'turn/end'
|
|
}, () => stderr)
|
|
|
|
const request = modelRequests[0] as {
|
|
messages?: Array<{ role?: string; content?: unknown }>
|
|
tools?: Array<{ function?: { name?: string } }>
|
|
}
|
|
expect(request.messages?.[0]).toMatchObject({ role: 'system', content: 'Minimal allowlist prompt.' })
|
|
const shellTool = process.platform === 'win32' ? 'pwsh' : 'bash'
|
|
expect(request.tools?.map(tool => tool.function?.name).sort()).toEqual([shellTool, 'str_replace_editor'].sort())
|
|
const profile = JSON.parse(
|
|
await readFile(join(root, '.dsh', 'profiles', 'sdk-minimal', 'package.json'), 'utf8'),
|
|
) as { dsh?: { profile?: { bundles?: string[]; patchReload?: string } } }
|
|
expect(profile.dsh?.profile).toEqual({
|
|
bundles: ['@deepseek-ai/dsh-sdk-minimal'],
|
|
patchReload: 'startup',
|
|
})
|
|
|
|
child.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id: 3, method: 'shutdown' })}\n`)
|
|
await waitForLine(lines, value => value.id === 3, () => stderr)
|
|
const exit = await child
|
|
expect(exit.exitCode, `signal=${String(exit.signal)}; stderr=${stderr}`).toBe(0)
|
|
} finally {
|
|
child.kill('SIGKILL')
|
|
await child
|
|
await new Promise<void>(resolve => modelServer.close(() => { resolve() }))
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
}, 40_000)
|
|
|
|
it('rejects an invalid max-token success env value', async () => {
|
|
const root = await mkdtemp(join(tmpdir(), 'dsh-python-sdk-runtime-invalid-'))
|
|
try {
|
|
const { exitCode, stdout, stderr } = await execa(process.execPath, [
|
|
'--import',
|
|
'tsx/esm',
|
|
binScript,
|
|
'--profile',
|
|
'sdk',
|
|
], {
|
|
cwd: repoRoot,
|
|
env: {
|
|
DSH_HOME: join(root, '.dsh'),
|
|
DEEPSEEK_API_KEY: 'keyless-smoke-no-call',
|
|
DSH_MAX_TOKENS_AS_SUCCESS: 'sometimes',
|
|
},
|
|
stdin: 'ignore',
|
|
timeout: 25_000,
|
|
killSignal: 'SIGKILL',
|
|
reject: false,
|
|
})
|
|
|
|
expect(exitCode, stderr).toBe(1)
|
|
expect(stdout).toBe('')
|
|
expect(stderr).toContain('plugin tree failed to load')
|
|
expect(stderr).toContain('failed to apply loader entry sdk-jsonrpc-server (@deepseek-ai/dsh-sdk-jsonrpc-server)')
|
|
expect(stderr).toContain('sometimes')
|
|
} finally {
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
}, 30_000)
|
|
})
|