mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-09-14 04:01:35 +00:00
The first call pays the full pwsh cold-start latency (spawn + .NET + PSReadLine + Defender) inside the tool deadline. A 60s bound on the fully loaded self-hosted Windows pool is exceeded often enough to reset the session mid-test: two master CI runs (2026-09-01, runs 33524764567 and 33534262413) each failed at ~62s with the second call observing a fresh session (cwd back at root, env empty). 300s matches the product default so cold start no longer races the budget.
176 lines
7.4 KiB
TypeScript
176 lines
7.4 KiB
TypeScript
import { spawnSync } from 'node:child_process'
|
|
import { mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
import { pathToFileURL } from 'node:url'
|
|
import { afterEach, describe, expect, it } from 'vitest'
|
|
import { Context } from '@deepseek-ai/cordis'
|
|
import Loader from '@deepseek-ai/cordis-plugin-loader'
|
|
import Include from '@deepseek-ai/cordis-plugin-include'
|
|
import { ToolCallId } from '@deepseek-ai/dsh-llm'
|
|
import { Session, SessionId } from '@deepseek-ai/dsh-session'
|
|
import SessionProjectionRegistry from '@deepseek-ai/dsh-session-projection'
|
|
import AgentRegistry, { Inbox } from '@deepseek-ai/dsh-agent'
|
|
import type { Agent } from '@deepseek-ai/dsh-agent'
|
|
import TerminalSessionService from '@deepseek-ai/dsh-terminal'
|
|
import * as TerminalBash from '@deepseek-ai/dsh-terminal-bash'
|
|
import SandboxProvider from '@deepseek-ai/dsh-sandbox'
|
|
import type { ConfinedArgv, SandboxPolicy } from '@deepseek-ai/dsh-sandbox'
|
|
import SandboxPolicyService from '@deepseek-ai/dsh-sandbox-policy'
|
|
import LocalSubprocessService from '@deepseek-ai/dsh-subprocess-local'
|
|
import { resolvePwshPath } from '@deepseek-ai/dsh-pwsh-local/src/resolve.ts'
|
|
import SystemPrompt from '@deepseek-ai/dsh-system-prompt'
|
|
import ToolRegistry from '@deepseek-ai/dsh-tools'
|
|
import * as ToolPwshPersistent from '@deepseek-ai/dsh-tool-pwsh-persistent'
|
|
|
|
const hasPwsh = spawnSync(
|
|
resolvePwshPath(), ['-NoLogo', '-NoProfile', '-NonInteractive', '-Command', '$true'],
|
|
{ encoding: 'utf8' },
|
|
).status === 0
|
|
|
|
let root: string | undefined
|
|
let context: Context | undefined
|
|
|
|
afterEach(async () => {
|
|
await context?.fiber.dispose()
|
|
context = undefined
|
|
if (root !== undefined) await rm(root, { recursive: true, force: true })
|
|
root = undefined
|
|
})
|
|
|
|
class PassthroughSandbox extends SandboxProvider {
|
|
confine(argv: readonly string[], _policy: SandboxPolicy): ConfinedArgv {
|
|
return { argv: [...argv], enforcement: 'full', denialSignatures: [], runnerFailureRules: [] }
|
|
}
|
|
}
|
|
|
|
function agent(ctx: Context, cwd: string): Agent {
|
|
const id = SessionId('persistent-pwsh-loader-agent')
|
|
const scope = ctx.plugin(() => {})
|
|
const session = Session.create(id, [], { version: 0, id, createdAt: 0, cwd, isSeeded: false })
|
|
const value: Agent = {
|
|
id,
|
|
options: {},
|
|
session,
|
|
inbox: new Inbox(session, { inserted: () => {}, discarded: () => {}, claimed: () => {} }),
|
|
status: 'idle',
|
|
ctx: scope.ctx,
|
|
send: () => {},
|
|
followup: () => {},
|
|
steer: () => ({ outcome: Promise.resolve({ status: 'rejected' as const }) }),
|
|
inject: () => {},
|
|
cancel() {},
|
|
runMaintenance: task => task(new AbortController().signal),
|
|
whenIdle: () => Promise.resolve(),
|
|
}
|
|
ctx.agents.register(value)
|
|
return value
|
|
}
|
|
|
|
function text(result: { content: { type: string; text?: string }[] }): string {
|
|
return result.content.filter(block => block.type === 'text').map(block => block.text).join('')
|
|
}
|
|
|
|
describe.skipIf(!hasPwsh)('persistent pwsh through a real cordis.yml Loader composition', () => {
|
|
it('preserves cwd and environment across calls', async () => {
|
|
root = await realpath(await mkdtemp(join(tmpdir(), 'dsh-persistent-pwsh-loader-')))
|
|
const configPath = join(root, 'cordis.yml')
|
|
await writeFile(configPath, [
|
|
"- name: '@deepseek-ai/dsh-agent'",
|
|
"- name: '@deepseek-ai/dsh-system-prompt'",
|
|
"- name: '@deepseek-ai/dsh-tools'",
|
|
"- name: '@deepseek-ai/dsh-terminal'",
|
|
"- name: '@deepseek-ai/dsh-test-sandbox'",
|
|
"- name: '@deepseek-ai/dsh-session-projection'",
|
|
"- name: '@deepseek-ai/dsh-sandbox-policy'",
|
|
' config:',
|
|
' mode: danger-full-access',
|
|
` workspaceRoot: ${JSON.stringify(root)}`,
|
|
"- name: '@deepseek-ai/dsh-subprocess-local'",
|
|
"- name: '@deepseek-ai/dsh-terminal-bash'",
|
|
' config:',
|
|
' shellDialect: pwsh',
|
|
' pollIntervalMs: 10',
|
|
' exactProbeAfterMs: 20',
|
|
' idleSilenceMs: 300',
|
|
' handoffGraceMs: 300',
|
|
' scrollbackLines: 20000',
|
|
// The first call pays the full pwsh cold-start latency (spawn + .NET +
|
|
// PSReadLine + Defender) inside the tool deadline; a 60s bound on the
|
|
// fully loaded self-hosted Windows pool is exceeded often enough to
|
|
// reset the session mid-test (2026-09-01, two runs ~62s each). 300s
|
|
// matches the product default so cold start no longer races the budget.
|
|
' timeoutMs: 300000',
|
|
' disposeGraceMs: 500',
|
|
"- name: '@deepseek-ai/dsh-tool-pwsh-persistent'",
|
|
' config:',
|
|
' timeoutMs: 300000',
|
|
'',
|
|
].join('\n'))
|
|
|
|
context = new Context()
|
|
context.baseUrl = pathToFileURL(root).href + '/'
|
|
await context.plugin(Loader)
|
|
context.loader.builtins.include = Include
|
|
const modules = new Map<string, unknown>([
|
|
['@deepseek-ai/dsh-agent', AgentRegistry],
|
|
['@deepseek-ai/dsh-system-prompt', SystemPrompt],
|
|
['@deepseek-ai/dsh-tools', ToolRegistry],
|
|
['@deepseek-ai/dsh-terminal', TerminalSessionService],
|
|
['@deepseek-ai/dsh-test-sandbox', PassthroughSandbox],
|
|
['@deepseek-ai/dsh-session-projection', SessionProjectionRegistry],
|
|
['@deepseek-ai/dsh-sandbox-policy', SandboxPolicyService],
|
|
['@deepseek-ai/dsh-subprocess-local', LocalSubprocessService],
|
|
['@deepseek-ai/dsh-terminal-bash', TerminalBash],
|
|
['@deepseek-ai/dsh-tool-pwsh-persistent', ToolPwshPersistent],
|
|
])
|
|
context.loader.internal = {
|
|
version: 'v2',
|
|
async import(specifier: string) {
|
|
if (!modules.has(specifier)) throw new Error(`unexpected Loader import: ${specifier}`)
|
|
return modules.get(specifier)
|
|
},
|
|
} as unknown as NonNullable<typeof context.loader.internal>
|
|
await context.loader.create({ name: 'cordis:include', config: { path: pathToFileURL(configPath).href } })
|
|
await context.loader.await()
|
|
|
|
const owner = agent(context, root)
|
|
const signal = new AbortController().signal
|
|
const execute = (id: string, command: string) => context!.tools.execute({
|
|
signal,
|
|
callId: ToolCallId(id),
|
|
name: 'pwsh',
|
|
arguments: { command },
|
|
agent: owner,
|
|
})
|
|
|
|
expect(context.tools.schemas().map(schema => schema.name)).toEqual(['pwsh'])
|
|
await execute('state', '$env:KEEP = "loader"; New-Item -ItemType Directory -Force -Path nested | Out-Null; Set-Location nested')
|
|
const observed = text(await execute('observe', 'Write-Output "cwd=$PWD keep=$env:KEEP"'))
|
|
expect(observed).toContain(`cwd=${join(root, 'nested')} keep=loader`)
|
|
expect(observed).not.toContain('DSH_PERSISTENT_PWSH')
|
|
|
|
const multiline = text(await execute(
|
|
'multiline',
|
|
'$value = "line one"\nWrite-Output "${value}:it\'s fine"',
|
|
))
|
|
expect(multiline).toBe("line one:it's fine")
|
|
expect(multiline).not.toContain('DSH_PERSISTENT_PWSH')
|
|
|
|
const hereString = text(await execute(
|
|
'here-string',
|
|
"$h = @'\nalpha\nbeta\n'@\nWrite-Output $h",
|
|
))
|
|
expect(hereString).toBe('alpha\nbeta')
|
|
|
|
const large = text(await execute('large-output', '1..12050 | ForEach-Object { $_ }'))
|
|
expect(large.startsWith('1\n2\n3\n')).toBe(true)
|
|
expect(large).toContain('<response clipped>')
|
|
expect(large).not.toContain('beginning of this command output was dropped')
|
|
|
|
const exited = text(await execute('exit', 'exit'))
|
|
expect(exited).toContain('next pwsh call starts from the workspace')
|
|
expect(text(await execute('after-exit', 'Write-Output "$PWD"'))).toBe(root)
|
|
}, 300_000)
|
|
})
|