mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-08-29 04:26:38 +00:00
306 lines
12 KiB
TypeScript
306 lines
12 KiB
TypeScript
import { createServer } from 'node:http'
|
|
import { mkdtemp, readFile, readdir, rm } from 'node:fs/promises'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
import { fileURLToPath } from 'node:url'
|
|
import { promisify } from 'node:util'
|
|
import { zstdDecompress } from 'node:zlib'
|
|
import { execa } from 'execa'
|
|
import { describe, expect, it } from 'vitest'
|
|
|
|
const binScript = fileURLToPath(new URL('../../../apps/cli/src/bin.ts', import.meta.url))
|
|
const repoRoot = fileURLToPath(new URL('../../..', import.meta.url))
|
|
const decompress = promisify(zstdDecompress)
|
|
|
|
function waitForLine(
|
|
lines: string[],
|
|
predicate: (value: Record<string, unknown>) => boolean,
|
|
stderr: () => string,
|
|
): Promise<Record<string, unknown>> {
|
|
return new Promise((resolve, reject) => {
|
|
const deadline = Date.now() + 30_000
|
|
const poll = (): void => {
|
|
while (lines.length > 0) {
|
|
const line = lines.shift()!
|
|
if (!line.trim()) continue
|
|
try {
|
|
const value = JSON.parse(line) as Record<string, unknown>
|
|
if (predicate(value)) {
|
|
resolve(value)
|
|
return
|
|
}
|
|
} catch {
|
|
reject(new Error(`non-JSON stdout from JSON-RPC agent runtime: ${line}`))
|
|
return
|
|
}
|
|
}
|
|
if (Date.now() >= deadline) {
|
|
reject(new Error(`timed out waiting for JSON-RPC response; stderr=${stderr()}`))
|
|
return
|
|
}
|
|
setTimeout(poll, 10)
|
|
}
|
|
poll()
|
|
})
|
|
}
|
|
|
|
describe('Python SDK dsh profile keyless smoke', () => {
|
|
it.each([
|
|
{ label: 'reports max-token turns with the default mapping config', envValue: undefined },
|
|
{ label: 'reports max-token turns with mapping enabled through env', envValue: 'true' },
|
|
{ label: 'reports max-token turns with mapping disabled through env', envValue: 'false' },
|
|
])('$label', async ({ envValue }) => {
|
|
const root = await mkdtemp(join(tmpdir(), 'dsh-python-sdk-runtime-smoke-'))
|
|
const modelRequests: Record<string, unknown>[] = []
|
|
const modelServer = createServer((request, response) => {
|
|
let body = ''
|
|
request.setEncoding('utf8')
|
|
request.on('data', (chunk: string) => { body += chunk })
|
|
request.on('end', () => {
|
|
modelRequests.push(JSON.parse(body) as Record<string, unknown>)
|
|
response.writeHead(200, { 'content-type': 'text/event-stream' })
|
|
response.write('data: {"choices":[{"delta":{"role":"assistant","content":null}}]}\n\n')
|
|
response.write('data: {"choices":[{"delta":{"content":"done"}}]}\n\n')
|
|
response.write('data: {"choices":[{"delta":{},"finish_reason":"length"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}\n\n')
|
|
response.end('data: [DONE]\n\n')
|
|
})
|
|
})
|
|
await new Promise<void>(resolve => modelServer.listen(0, '127.0.0.1', resolve))
|
|
const address = modelServer.address()
|
|
if (address === null || typeof address === 'string') throw new Error('model server did not bind a TCP port')
|
|
// The line-predicate protocol driving below is the genuinely custom part;
|
|
// execa owns spawn, the deadline, and exit settlement around it.
|
|
const child = execa(process.execPath, [
|
|
'--import',
|
|
'tsx/esm',
|
|
binScript,
|
|
'--profile',
|
|
'sdk',
|
|
], {
|
|
cwd: repoRoot,
|
|
env: {
|
|
DSH_HOME: join(root, '.dsh'),
|
|
DSH_PERMISSION_MODE: 'danger-full-access',
|
|
DSH_TELEMETRY_DISABLED: '1',
|
|
DEEPSEEK_API_KEY: 'keyless-smoke-no-call',
|
|
DEEPSEEK_BASE_URL: `http://127.0.0.1:${address.port}`,
|
|
...(envValue === undefined ? {} : { DSH_MAX_TOKENS_AS_SUCCESS: envValue }),
|
|
},
|
|
timeout: 35_000,
|
|
killSignal: 'SIGKILL',
|
|
reject: false,
|
|
})
|
|
const lines: string[] = []
|
|
let stdoutBuffer = ''
|
|
let stderr = ''
|
|
child.stdout.on('data', (chunk: Buffer) => {
|
|
stdoutBuffer += chunk.toString('utf8')
|
|
const parts = stdoutBuffer.split('\n')
|
|
stdoutBuffer = parts.pop() ?? ''
|
|
lines.push(...parts)
|
|
})
|
|
child.stderr.on('data', (chunk: Buffer) => { stderr += chunk.toString('utf8') })
|
|
|
|
try {
|
|
child.stdin.write(`${JSON.stringify({
|
|
jsonrpc: '2.0',
|
|
id: 1,
|
|
method: 'initialize',
|
|
params: {
|
|
cwd: root,
|
|
provider: 'deepseek-official',
|
|
model: 'deepseek-v4-pro',
|
|
reasoningEffort: 'max',
|
|
maxTokens: 1234,
|
|
},
|
|
})}\n`)
|
|
const initialized = await waitForLine(lines, value => value.id === 1, () => stderr)
|
|
expect(initialized).toMatchObject({
|
|
jsonrpc: '2.0',
|
|
id: 1,
|
|
result: { serverInfo: { name: 'deepseek-harness-sdk-runtime' } },
|
|
})
|
|
|
|
child.stdin.write(`${JSON.stringify({
|
|
jsonrpc: '2.0',
|
|
id: 2,
|
|
method: 'session/prompt',
|
|
params: { sessionId: 'main', contentBlocks: [{ type: 'text', text: 'inspect tools' }] },
|
|
})}\n`)
|
|
const prompt = await waitForLine(lines, value => value.id === 2, () => stderr)
|
|
expect(prompt).toMatchObject({
|
|
jsonrpc: '2.0',
|
|
id: 2,
|
|
result: { messageId: expect.any(String) as unknown },
|
|
})
|
|
const turnEnd = await waitForLine(lines, (value) => {
|
|
if (value.method !== 'session.event') return false
|
|
const params = value.params as Record<string, unknown> | undefined
|
|
const event = params?.event as Record<string, unknown> | undefined
|
|
return params?.sessionId === 'main' && event?.type === 'turn/end'
|
|
}, () => stderr)
|
|
expect(turnEnd).toMatchObject({
|
|
jsonrpc: '2.0',
|
|
method: 'session.event',
|
|
params: {
|
|
sessionId: 'main',
|
|
event: {
|
|
type: 'turn/end',
|
|
data: { reason: { kind: 'max-tokens' } },
|
|
},
|
|
},
|
|
})
|
|
const tools = modelRequests[0]?.tools as { function?: { name?: string } }[]
|
|
expect(modelRequests[0]?.reasoning_effort).toBe('max')
|
|
expect(modelRequests[0]?.max_tokens).toBe(1234)
|
|
expect(tools.map(tool => tool.function?.name)).toContain('list_subagent_models')
|
|
|
|
child.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id: 3, method: 'shutdown' })}\n`)
|
|
const shutdown = await waitForLine(lines, value => value.id === 3, () => stderr)
|
|
expect(shutdown).toMatchObject({ jsonrpc: '2.0', id: 3, result: {} })
|
|
const exit = await child
|
|
expect(exit.exitCode, `signal=${String(exit.signal)}; stderr=${stderr}`).toBe(0)
|
|
const sessionsRoot = join(root, '.dsh', 'sessions')
|
|
const files = await readdir(sessionsRoot, { recursive: true })
|
|
const log = files.find(file => file.endsWith('.jsonl.zstd'))
|
|
expect(log).toBeDefined()
|
|
const compressed = await readFile(join(sessionsRoot, log!))
|
|
expect(compressed.subarray(0, 4).toString('hex')).toBe('28b52ffd')
|
|
expect(JSON.parse((await decompress(compressed)).toString())).toMatchObject({ type: 'session', id: 'main' })
|
|
} finally {
|
|
// No-op after exit; reject: false settles on every outcome, so cleanup never races teardown.
|
|
child.kill('SIGKILL')
|
|
await child
|
|
await new Promise<void>(resolve => modelServer.close(() => { resolve() }))
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
}, 40_000)
|
|
|
|
it('boots the standalone minimal profile with its exact model-facing roster', async () => {
|
|
const root = await mkdtemp(join(tmpdir(), 'dsh-python-sdk-minimal-'))
|
|
const modelRequests: Record<string, unknown>[] = []
|
|
const modelServer = createServer((request, response) => {
|
|
let body = ''
|
|
request.setEncoding('utf8')
|
|
request.on('data', (chunk: string) => { body += chunk })
|
|
request.on('end', () => {
|
|
modelRequests.push(JSON.parse(body) as Record<string, unknown>)
|
|
response.writeHead(200, { 'content-type': 'text/event-stream' })
|
|
response.write('data: {"choices":[{"delta":{"role":"assistant","content":null}}]}\n\n')
|
|
response.write('data: {"choices":[{"delta":{"content":"done"}}]}\n\n')
|
|
response.write('data: {"choices":[{"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":1}}\n\n')
|
|
response.end('data: [DONE]\n\n')
|
|
})
|
|
})
|
|
await new Promise<void>(resolve => modelServer.listen(0, '127.0.0.1', resolve))
|
|
const address = modelServer.address()
|
|
if (address === null || typeof address === 'string') throw new Error('model server did not bind a TCP port')
|
|
const child = execa(process.execPath, [
|
|
'--import',
|
|
'tsx/esm',
|
|
binScript,
|
|
'--profile',
|
|
'sdk-minimal',
|
|
], {
|
|
cwd: repoRoot,
|
|
env: {
|
|
DSH_HOME: join(root, '.dsh'),
|
|
DSH_SYSTEM_PROMPT: 'Minimal allowlist prompt.',
|
|
DEEPSEEK_API_KEY: 'keyless-smoke-no-call',
|
|
DEEPSEEK_BASE_URL: `http://127.0.0.1:${address.port}`,
|
|
},
|
|
timeout: 35_000,
|
|
killSignal: 'SIGKILL',
|
|
reject: false,
|
|
})
|
|
const lines: string[] = []
|
|
let stdoutBuffer = ''
|
|
let stderr = ''
|
|
child.stdout.on('data', (chunk: Buffer) => {
|
|
stdoutBuffer += chunk.toString('utf8')
|
|
const parts = stdoutBuffer.split('\n')
|
|
stdoutBuffer = parts.pop() ?? ''
|
|
lines.push(...parts)
|
|
})
|
|
child.stderr.on('data', (chunk: Buffer) => { stderr += chunk.toString('utf8') })
|
|
|
|
try {
|
|
child.stdin.write(`${JSON.stringify({
|
|
jsonrpc: '2.0',
|
|
id: 1,
|
|
method: 'initialize',
|
|
params: { cwd: root, provider: 'deepseek-official', model: 'deepseek-v4-pro' },
|
|
})}\n`)
|
|
await waitForLine(lines, value => value.id === 1, () => stderr)
|
|
child.stdin.write(`${JSON.stringify({
|
|
jsonrpc: '2.0',
|
|
id: 2,
|
|
method: 'session/prompt',
|
|
params: { sessionId: 'minimal', contentBlocks: [{ type: 'text', text: 'inspect tools' }] },
|
|
})}\n`)
|
|
await waitForLine(lines, (value) => {
|
|
const params = value.params as Record<string, unknown> | undefined
|
|
const event = params?.event as Record<string, unknown> | undefined
|
|
return params?.sessionId === 'minimal' && event?.type === 'turn/end'
|
|
}, () => stderr)
|
|
|
|
const request = modelRequests[0] as {
|
|
messages?: Array<{ role?: string; content?: unknown }>
|
|
tools?: Array<{ function?: { name?: string } }>
|
|
}
|
|
expect(request.messages?.[0]).toMatchObject({ role: 'system', content: 'Minimal allowlist prompt.' })
|
|
const shellTool = process.platform === 'win32' ? 'pwsh' : 'bash'
|
|
expect(request.tools?.map(tool => tool.function?.name).sort()).toEqual([shellTool, 'str_replace_editor'].sort())
|
|
const profile = JSON.parse(
|
|
await readFile(join(root, '.dsh', 'profiles', 'sdk-minimal', 'package.json'), 'utf8'),
|
|
) as { dsh?: { profile?: { bundles?: string[]; patchReload?: string } } }
|
|
expect(profile.dsh?.profile).toEqual({
|
|
bundles: ['@deepseek-ai/dsh-sdk-minimal'],
|
|
patchReload: 'startup',
|
|
})
|
|
|
|
child.stdin.write(`${JSON.stringify({ jsonrpc: '2.0', id: 3, method: 'shutdown' })}\n`)
|
|
await waitForLine(lines, value => value.id === 3, () => stderr)
|
|
const exit = await child
|
|
expect(exit.exitCode, `signal=${String(exit.signal)}; stderr=${stderr}`).toBe(0)
|
|
} finally {
|
|
child.kill('SIGKILL')
|
|
await child
|
|
await new Promise<void>(resolve => modelServer.close(() => { resolve() }))
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
}, 40_000)
|
|
|
|
it('rejects an invalid max-token success env value', async () => {
|
|
const root = await mkdtemp(join(tmpdir(), 'dsh-python-sdk-runtime-invalid-'))
|
|
try {
|
|
const { exitCode, stdout, stderr } = await execa(process.execPath, [
|
|
'--import',
|
|
'tsx/esm',
|
|
binScript,
|
|
'--profile',
|
|
'sdk',
|
|
], {
|
|
cwd: repoRoot,
|
|
env: {
|
|
DSH_HOME: join(root, '.dsh'),
|
|
DEEPSEEK_API_KEY: 'keyless-smoke-no-call',
|
|
DSH_MAX_TOKENS_AS_SUCCESS: 'sometimes',
|
|
},
|
|
stdin: 'ignore',
|
|
timeout: 25_000,
|
|
killSignal: 'SIGKILL',
|
|
reject: false,
|
|
})
|
|
|
|
expect(exitCode, stderr).toBe(1)
|
|
expect(stdout).toBe('')
|
|
expect(stderr).toContain('plugin tree failed to load')
|
|
expect(stderr).toContain('failed to apply loader entry sdk-jsonrpc-server (@deepseek-ai/dsh-sdk-jsonrpc-server)')
|
|
expect(stderr).toContain('sometimes')
|
|
} finally {
|
|
await rm(root, { recursive: true, force: true })
|
|
}
|
|
}, 30_000)
|
|
})
|