mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-08-29 04:26:38 +00:00
refactor(code-runtime): generate shared subprocess runner
This commit is contained in:
@@ -112,6 +112,8 @@
|
||||
"gen-module-graph": "tsx scripts/gen-module-graph.ts",
|
||||
"gen-scoped-events": "tsx scripts/gen-scoped-events.ts",
|
||||
"verify-scoped-events": "tsx scripts/gen-scoped-events.ts --check",
|
||||
"gen-code-runtime-runner": "tsx scripts/gen-code-runtime-runner.ts",
|
||||
"verify-code-runtime-runner": "tsx scripts/gen-code-runtime-runner.ts --check",
|
||||
"verify-module-graph": "tsx scripts/gen-module-graph.ts --check",
|
||||
"constraints": "tsx scripts/check-workspace-constraints.ts",
|
||||
"doc-sync": "tsx scripts/run-gates.ts doc-sync",
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
*/
|
||||
|
||||
import { inspect } from 'node:util'
|
||||
import type { CodeJsonValue } from '@deepseek-ai/dsh-code-runtime'
|
||||
import type { DoneMessage, ReplyMessage, WorkerBootData, WorkerToHost } from './protocol.ts'
|
||||
import { jsonStringBytesUpTo, jsonValueBytesUpTo, truncateJsonStringBytes } from './output-json.ts'
|
||||
import { decodeWorkerJson, encodeWorkerJson, snapshotCodeJsonValue } from './worker-json.ts'
|
||||
@@ -310,6 +311,7 @@ export function wireReplies(port: BootstrapPort, pending: Map<number, PendingCal
|
||||
* @param pending - the id-keyed map each posted call parks its handles in.
|
||||
* @param nextId - the shared mutable id counter (worker-issued correlation ids).
|
||||
* @param errorClasses - per-namespace constructors shared with program globals.
|
||||
* @param maxFrameBytes - optional serialized transport cap checked before posting.
|
||||
* @returns one namespace object per declaration, in declaration order.
|
||||
*/
|
||||
export function makeNamespaces(
|
||||
@@ -318,6 +320,7 @@ export function makeNamespaces(
|
||||
pending: Map<number, PendingCall>,
|
||||
nextId: { value: number },
|
||||
errorClasses: Map<string, BindingErrorConstructor> = makeBindingErrorClasses(data),
|
||||
maxFrameBytes?: number,
|
||||
): Record<string, unknown>[] {
|
||||
return data.namespaces.map(({ global, names }) => {
|
||||
const errorClass = errorClasses.get(global)
|
||||
@@ -335,6 +338,11 @@ export function makeNamespaces(
|
||||
if (detached === undefined) {
|
||||
return Promise.reject(bindingFailure(errorClass, name, 'binding arguments must be lossless JSON'))
|
||||
}
|
||||
const call = { type: 'call' as const, id: nextId.value, global, name, args: encodeWorkerJson(detached) }
|
||||
if (maxFrameBytes !== undefined
|
||||
&& jsonValueBytesUpTo(call as unknown as CodeJsonValue, maxFrameBytes) === undefined) {
|
||||
return Promise.reject(bindingFailure(errorClass, name, 'binding arguments exceed maxFrameBytes'))
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const id = nextId.value++
|
||||
pending.set(id, {
|
||||
@@ -344,7 +352,7 @@ export function makeNamespaces(
|
||||
},
|
||||
})
|
||||
try {
|
||||
port.postMessage({ type: 'call', id, global, name, args: encodeWorkerJson(detached) })
|
||||
port.postMessage(call)
|
||||
} catch (error: unknown) {
|
||||
pending.delete(id)
|
||||
const message = `binding arguments must be structured-cloneable: ${error instanceof CapturedError ? error.message : String(error)}`
|
||||
@@ -364,12 +372,14 @@ export function makeNamespaces(
|
||||
* @param port - host message port or test double.
|
||||
* @param data - the boot payload the host sent.
|
||||
* @param streams - stdout/stderr objects captured as program logs.
|
||||
* @param maxFrameBytes - optional serialized transport cap checked before posting.
|
||||
* @returns after posting the done message.
|
||||
*/
|
||||
export async function runWorkerMain(
|
||||
port: BootstrapPort,
|
||||
data: WorkerBootData,
|
||||
streams: { stdout: PatchableStream; stderr: PatchableStream },
|
||||
maxFrameBytes?: number,
|
||||
): Promise<void> {
|
||||
const logs = new LogBuffer(
|
||||
data.maxOutputBytes,
|
||||
@@ -384,7 +394,7 @@ export async function runWorkerMain(
|
||||
|
||||
const nextId = { value: 1 }
|
||||
const errorClasses = makeBindingErrorClasses(data)
|
||||
const namespaces = makeNamespaces(data, port, pending, nextId, errorClasses)
|
||||
const namespaces = makeNamespaces(data, port, pending, nextId, errorClasses, maxFrameBytes)
|
||||
const errorClassParameters: string[] = []
|
||||
const errorClassValues: BindingErrorConstructor[] = []
|
||||
for (const namespace of data.namespaces) {
|
||||
@@ -420,5 +430,8 @@ export async function runWorkerMain(
|
||||
...prepareException(error, logs.remainingOutputBytes(), data.maxOutputBytes),
|
||||
}
|
||||
}
|
||||
port.postMessage(done)
|
||||
port.postMessage(maxFrameBytes !== undefined
|
||||
&& jsonValueBytesUpTo(done as unknown as CodeJsonValue, maxFrameBytes) === undefined
|
||||
? { type: 'output-limit' }
|
||||
: done)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
/** Shared host mechanics for local and subprocess-hosted TypeScript worker runtimes. */
|
||||
|
||||
import { stripTypeScriptTypes } from 'node:module'
|
||||
import type { Readable } from 'node:stream'
|
||||
import type {
|
||||
CodeBindingNamespace,
|
||||
CodeJsonValue,
|
||||
@@ -15,6 +16,28 @@ import type { WorkerJsonWire } from './worker-json.ts'
|
||||
/** Smallest cap that can represent an empty log array and failure message. */
|
||||
export const MIN_RUNTIME_OUTPUT_BYTES = 4
|
||||
|
||||
/**
|
||||
* Resolve after a worker pipe emits queued data or closes during termination.
|
||||
* @param stream - captured worker or child-process pipe.
|
||||
* @returns after no more queued bytes can arrive.
|
||||
*/
|
||||
export function waitForRuntimePipeDrain(stream: Readable): Promise<void> {
|
||||
if (stream.readableEnded || stream.destroyed) return Promise.resolve()
|
||||
return new Promise((resolve) => {
|
||||
const done = (): void => {
|
||||
stream.off('end', done)
|
||||
stream.off('close', done)
|
||||
stream.off('error', done)
|
||||
resolve()
|
||||
}
|
||||
stream.once('end', done)
|
||||
stream.once('close', done)
|
||||
stream.once('error', done)
|
||||
/* v8 ignore next -- termination can win the adjacent listener-registration race. */
|
||||
if (stream.readableEnded || stream.destroyed) done()
|
||||
})
|
||||
}
|
||||
|
||||
const IDENTIFIER = /^[A-Za-z_$][A-Za-z0-9_$]*$/
|
||||
const RESERVED_WORDS = new Set([
|
||||
'await', 'break', 'case', 'catch', 'class', 'const', 'continue', 'debugger', 'default', 'delete', 'do',
|
||||
@@ -223,5 +246,6 @@ export class RuntimeOutputLedger {
|
||||
}
|
||||
|
||||
export { decodeWorkerJson, encodeWorkerJson, snapshotCodeJsonValue } from './worker-json.ts'
|
||||
export { jsonValueBytesUpTo } from './output-json.ts'
|
||||
export { jsonStringBytesUpTo, jsonValueBytesUpTo } from './output-json.ts'
|
||||
export { runWorkerMain } from './bootstrap.ts'
|
||||
export type { WorkerJsonWire } from './worker-json.ts'
|
||||
|
||||
@@ -286,6 +286,23 @@ describe('makeNamespaces', () => {
|
||||
expect(nextId.value).toBe(1)
|
||||
})
|
||||
|
||||
it('rejects an oversized transport frame before posting or allocating a call id', async () => {
|
||||
const port = new FakePort()
|
||||
const pending = new Map<number, PendingCall>()
|
||||
const nextId = { value: 1 }
|
||||
const data = { namespaces: [toolNamespace(['x'])] }
|
||||
const [tools] = makeNamespaces(
|
||||
data, port, pending, nextId, makeBindingErrorClasses(data), 64,
|
||||
) as [Record<string, (args: unknown) => Promise<unknown>>]
|
||||
|
||||
await expect(tools.x?.({ text: 'x'.repeat(64) })).rejects.toMatchObject({
|
||||
name: 'ToolCallError', toolName: 'x', message: 'binding arguments exceed maxFrameBytes',
|
||||
})
|
||||
expect(port.sent).toEqual([])
|
||||
expect(pending.size).toBe(0)
|
||||
expect(nextId.value).toBe(1)
|
||||
})
|
||||
|
||||
it('uses ordinary Error for non-tools namespace failures', async () => {
|
||||
const deniedPort = new FakePort()
|
||||
deniedPort.respond = message => message.type === 'call'
|
||||
@@ -343,6 +360,16 @@ describe('runWorkerMain', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('reports output-limit before posting a completion that expands past the transport cap', async () => {
|
||||
const port = new FakePort()
|
||||
await runWorkerMain(port, {
|
||||
maxOutputBytes: 1_000,
|
||||
code: 'return Array.from({ length: 100 }, () => [])',
|
||||
namespaces: [],
|
||||
}, fakeStreams(), 100)
|
||||
expect(port.sent.at(-1)).toEqual({ type: 'output-limit' })
|
||||
})
|
||||
|
||||
it('reports a thrown program error on the done message', async () => {
|
||||
const port = new FakePort()
|
||||
await runWorkerMain(port, { ...BOOT, code: 'throw new Error("boom")', namespaces: [] }, fakeStreams())
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,390 @@
|
||||
/** Typed source for the dependency-free execution-world runner bundle. */
|
||||
|
||||
import { Buffer } from 'node:buffer'
|
||||
import { fork } from 'node:child_process'
|
||||
import type { ChildProcess } from 'node:child_process'
|
||||
import { createInterface } from 'node:readline'
|
||||
import type { Readable } from 'node:stream'
|
||||
import { inspect } from 'node:util'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads'
|
||||
import {
|
||||
decodeWorkerJson,
|
||||
encodeWorkerJson,
|
||||
jsonStringBytesUpTo,
|
||||
runWorkerMain,
|
||||
waitForRuntimePipeDrain,
|
||||
} from '@deepseek-ai/dsh-code-runtime-worker/runtime-host'
|
||||
import type { WorkerJsonWire } from '@deepseek-ai/dsh-code-runtime-worker/runtime-host'
|
||||
|
||||
type WorkerBootData = Parameters<typeof runWorkerMain>[1]
|
||||
type Controller = ChildProcess & { stdout: Readable; stderr: Readable }
|
||||
type FailureKind = 'exception' | 'timeout' | 'abort' | 'worker-exit' | 'invalid-output' | 'output-limit'
|
||||
|
||||
interface RuntimeBootData extends WorkerBootData {
|
||||
type: 'boot'
|
||||
maxFrameBytes: number
|
||||
maxOldGenerationSizeMb: number
|
||||
computeMs: number
|
||||
}
|
||||
|
||||
interface RuntimeFailure {
|
||||
kind: FailureKind
|
||||
message: string
|
||||
}
|
||||
|
||||
interface RuntimeCall {
|
||||
type: 'call'
|
||||
id: number
|
||||
global: string
|
||||
name: string
|
||||
args: WorkerJsonWire | null
|
||||
}
|
||||
|
||||
type RuntimeReply =
|
||||
| { type: 'reply'; id: number; ok: true; value: unknown }
|
||||
| { type: 'reply'; id: number; ok: false; message: string }
|
||||
|
||||
type RuntimeMessage = RuntimeCall
|
||||
| { type: 'log'; text: string }
|
||||
| { type: 'output-limit' }
|
||||
| { type: 'done'; value?: WorkerJsonWire | null; error?: RuntimeFailure }
|
||||
|
||||
const failureKinds = new Set<FailureKind>([
|
||||
'exception', 'timeout', 'abort', 'worker-exit', 'invalid-output', 'output-limit',
|
||||
])
|
||||
|
||||
let maxFrameBytes = 0
|
||||
|
||||
function recordOf(value: unknown): Record<string, unknown> | undefined {
|
||||
return typeof value === 'object' && value !== null ? value as Record<string, unknown> : undefined
|
||||
}
|
||||
|
||||
function encodeJsonBounded(value: unknown, maxBytes: number): string | undefined {
|
||||
try {
|
||||
const json: unknown = JSON.stringify(value)
|
||||
return typeof json === 'string' && Buffer.byteLength(json) <= maxBytes ? json : undefined
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
function emitJson(json: string): void {
|
||||
process.stdout.write(json)
|
||||
process.stdout.write('\n')
|
||||
}
|
||||
|
||||
function emitFrame(message: RuntimeMessage): boolean {
|
||||
const json = encodeJsonBounded(message, maxFrameBytes)
|
||||
if (json === undefined) return false
|
||||
emitJson(json)
|
||||
return true
|
||||
}
|
||||
|
||||
function validFailure(value: unknown): value is RuntimeFailure {
|
||||
const record = recordOf(value)
|
||||
return record !== undefined
|
||||
&& typeof record.kind === 'string'
|
||||
&& failureKinds.has(record.kind as FailureKind)
|
||||
&& typeof record.message === 'string'
|
||||
}
|
||||
|
||||
function validWorkerFailure(value: unknown): value is RuntimeFailure {
|
||||
return validFailure(value)
|
||||
&& (value.kind === 'exception' || value.kind === 'invalid-output' || value.kind === 'output-limit')
|
||||
}
|
||||
|
||||
function runtimeBoot(value: unknown): RuntimeBootData | undefined {
|
||||
const record = recordOf(value)
|
||||
if (record === undefined
|
||||
|| record.type !== 'boot'
|
||||
|| typeof record.code !== 'string'
|
||||
|| !Array.isArray(record.namespaces)
|
||||
|| !Number.isSafeInteger(record.maxOutputBytes)
|
||||
|| (record.maxOutputBytes as number) < 4
|
||||
|| !Number.isSafeInteger(record.maxFrameBytes)
|
||||
|| (record.maxFrameBytes as number) < (record.maxOutputBytes as number)
|
||||
|| typeof record.computeMs !== 'number'
|
||||
|| !Number.isFinite(record.computeMs)
|
||||
|| (record.computeMs) <= 0
|
||||
|| typeof record.maxOldGenerationSizeMb !== 'number'
|
||||
|| !Number.isFinite(record.maxOldGenerationSizeMb)
|
||||
|| (record.maxOldGenerationSizeMb) <= 0) return undefined
|
||||
return record as unknown as RuntimeBootData
|
||||
}
|
||||
|
||||
function runtimeReply(value: unknown): RuntimeReply | undefined {
|
||||
const record = recordOf(value)
|
||||
if (record === undefined || record.type !== 'reply' || typeof record.id !== 'number' || typeof record.ok !== 'boolean') return undefined
|
||||
return record.ok
|
||||
? { type: 'reply', id: record.id, ok: true, value: record.value }
|
||||
: { type: 'reply', id: record.id, ok: false, message: String(record.message) }
|
||||
}
|
||||
|
||||
function transportWireOrNull(input: unknown): WorkerJsonWire | null {
|
||||
const value = decodeWorkerJson(input)
|
||||
return value === undefined ? null : encodeWorkerJson(value)
|
||||
}
|
||||
|
||||
function waitForChildExit(child: ChildProcess): Promise<void> {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve()
|
||||
return new Promise((resolve) => { child.once('exit', () => { resolve() }) })
|
||||
}
|
||||
|
||||
function frameLimitFailure(): RuntimeMessage {
|
||||
return {
|
||||
type: 'done',
|
||||
error: { kind: 'worker-exit', message: 'code runtime bridge frame exceeded maxFrameBytes' },
|
||||
}
|
||||
}
|
||||
|
||||
function runLauncher(): void {
|
||||
const input = createInterface({ input: process.stdin, crlfDelay: Infinity })
|
||||
let controller: Controller | undefined
|
||||
let maxOutputBytes = 0
|
||||
let logBytes = 2
|
||||
let logEntries = 0
|
||||
let settling = false
|
||||
|
||||
const finish = (message: RuntimeMessage): void => {
|
||||
if (settling) return
|
||||
const encoded = encodeJsonBounded(message, maxFrameBytes)
|
||||
?? encodeJsonBounded(frameLimitFailure(), maxFrameBytes)
|
||||
settling = true
|
||||
if (encoded !== undefined) emitJson(encoded)
|
||||
const current = controller
|
||||
controller = undefined
|
||||
const drain = current === undefined
|
||||
? Promise.resolve()
|
||||
: new Promise<void>((resolve) => { setImmediate(resolve) }).then(async () => {
|
||||
const stdoutDrained = waitForRuntimePipeDrain(current.stdout)
|
||||
const stderrDrained = waitForRuntimePipeDrain(current.stderr)
|
||||
const exited = waitForChildExit(current)
|
||||
current.kill('SIGKILL')
|
||||
await Promise.all([exited, stdoutDrained, stderrDrained])
|
||||
})
|
||||
void drain.catch((error: unknown) => {
|
||||
process.stderr.write(`dsh-code-runtime-subprocess controller cleanup error: ${String(error)}\n`)
|
||||
}).then(() => {
|
||||
input.close()
|
||||
process.stdin.destroy()
|
||||
})
|
||||
}
|
||||
|
||||
const forwardLog = (text: string): void => {
|
||||
if (settling) return
|
||||
const separator = logEntries > 0 ? 1 : 0
|
||||
const cost = jsonStringBytesUpTo(text, maxOutputBytes - logBytes - separator)
|
||||
if (cost === undefined) {
|
||||
finish({ type: 'output-limit' })
|
||||
return
|
||||
}
|
||||
logBytes += cost + separator
|
||||
logEntries += 1
|
||||
if (!emitFrame({ type: 'log', text })) finish(frameLimitFailure())
|
||||
}
|
||||
|
||||
const startController = (boot: RuntimeBootData): void => {
|
||||
maxOutputBytes = boot.maxOutputBytes
|
||||
maxFrameBytes = boot.maxFrameBytes
|
||||
controller = fork(fileURLToPath(import.meta.url), [], {
|
||||
env: { DSH_CODE_RUNTIME_CONTROLLER: '1' },
|
||||
detached: false,
|
||||
execArgv: [],
|
||||
stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
|
||||
}) as Controller
|
||||
const current = controller
|
||||
current.stdout.on('data', (data: Buffer) => { forwardLog(data.toString('utf8')) })
|
||||
current.stderr.on('data', (data: Buffer) => { forwardLog(data.toString('utf8')) })
|
||||
current.on('message', (raw: unknown) => {
|
||||
const message = recordOf(raw)
|
||||
if (message === undefined) return
|
||||
if (message.type === 'log' && typeof message.text === 'string') {
|
||||
forwardLog(message.text)
|
||||
return
|
||||
}
|
||||
if (settling) return
|
||||
if (message.type === 'call'
|
||||
&& typeof message.id === 'number'
|
||||
&& typeof message.global === 'string'
|
||||
&& typeof message.name === 'string') {
|
||||
if (!emitFrame({
|
||||
type: 'call', id: message.id, global: message.global, name: message.name, args: transportWireOrNull(message.args),
|
||||
})) finish(frameLimitFailure())
|
||||
} else if (message.type === 'output-limit') {
|
||||
finish({ type: 'output-limit' })
|
||||
} else if (message.type === 'done') {
|
||||
if (message.error !== undefined) {
|
||||
if (validFailure(message.error)) finish({ type: 'done', error: message.error })
|
||||
} else {
|
||||
finish({ type: 'done', ...message.value === undefined ? {} : { value: transportWireOrNull(message.value) } })
|
||||
}
|
||||
}
|
||||
})
|
||||
current.on('error', (error: Error) => {
|
||||
finish({ type: 'done', error: { kind: 'worker-exit', message: `remote controller error: ${error.message}` } })
|
||||
})
|
||||
current.on('exit', (code: number | null) => {
|
||||
if (!settling) {
|
||||
finish({ type: 'done', error: { kind: 'worker-exit', message: `remote controller exited with code ${code} before completing` } })
|
||||
}
|
||||
})
|
||||
current.send(boot, (error: Error | null) => {
|
||||
if (error !== null) {
|
||||
finish({ type: 'done', error: { kind: 'worker-exit', message: `remote controller boot failed: ${error.message}` } })
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
input.on('line', (line: string) => {
|
||||
let raw: unknown
|
||||
try {
|
||||
raw = JSON.parse(line) as unknown
|
||||
} catch (error: unknown) {
|
||||
process.stderr.write(`dsh-code-runtime-subprocess frame error: ${String(error)}\n`)
|
||||
finish({ type: 'done', error: { kind: 'worker-exit', message: 'remote runner received a malformed frame' } })
|
||||
return
|
||||
}
|
||||
if (controller === undefined) {
|
||||
const boot = runtimeBoot(raw)
|
||||
if (boot === undefined) {
|
||||
finish({ type: 'done', error: { kind: 'worker-exit', message: 'remote runner received an invalid boot frame' } })
|
||||
return
|
||||
}
|
||||
startController(boot)
|
||||
return
|
||||
}
|
||||
const reply = runtimeReply(raw)
|
||||
if (reply !== undefined) {
|
||||
controller.send(reply, (error: Error | null) => {
|
||||
if (error !== null) {
|
||||
finish({ type: 'done', error: { kind: 'worker-exit', message: `remote controller reply failed: ${error.message}` } })
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
input.on('close', () => {
|
||||
if (controller !== undefined && !settling) {
|
||||
finish({ type: 'done', error: { kind: 'abort', message: 'remote runner input closed' } })
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function runController(): void {
|
||||
let worker: Worker | undefined
|
||||
let finished = false
|
||||
let computeTimer: NodeJS.Timeout | undefined
|
||||
let controllerMaxFrameBytes = 0
|
||||
|
||||
const send = (message: RuntimeMessage): boolean => {
|
||||
if (process.send === undefined) return false
|
||||
if (controllerMaxFrameBytes > 0 && encodeJsonBounded(message, controllerMaxFrameBytes) === undefined) return false
|
||||
process.send(message)
|
||||
return true
|
||||
}
|
||||
|
||||
const finish = (message: RuntimeMessage): void => {
|
||||
if (finished) return
|
||||
finished = true
|
||||
clearInterval(computeTimer)
|
||||
const bounded = controllerMaxFrameBytes > 0 && encodeJsonBounded(message, controllerMaxFrameBytes) === undefined
|
||||
? frameLimitFailure()
|
||||
: message
|
||||
const current = worker
|
||||
worker = undefined
|
||||
const drain = current === undefined
|
||||
? Promise.resolve()
|
||||
: new Promise<void>((resolve) => { setImmediate(resolve) }).then(async () => {
|
||||
const stdoutDrained = waitForRuntimePipeDrain(current.stdout)
|
||||
const stderrDrained = waitForRuntimePipeDrain(current.stderr)
|
||||
await Promise.all([current.terminate(), stdoutDrained, stderrDrained])
|
||||
})
|
||||
void drain.catch((error: unknown) => {
|
||||
send({ type: 'log', text: `dsh-code-runtime-subprocess worker cleanup error: ${String(error)}\n` })
|
||||
}).then(() => {
|
||||
if (process.send === undefined) {
|
||||
process.exitCode = 1
|
||||
return
|
||||
}
|
||||
process.send(bounded, () => { if (process.connected) process.disconnect() })
|
||||
})
|
||||
}
|
||||
|
||||
process.on('message', (raw: unknown) => {
|
||||
if (worker === undefined) {
|
||||
const boot = runtimeBoot(raw)
|
||||
if (boot === undefined) {
|
||||
finish({ type: 'done', error: { kind: 'worker-exit', message: 'remote controller received an invalid boot frame' } })
|
||||
return
|
||||
}
|
||||
controllerMaxFrameBytes = boot.maxFrameBytes
|
||||
worker = new Worker(new URL(import.meta.url), {
|
||||
workerData: boot,
|
||||
env: {},
|
||||
execArgv: [],
|
||||
stdout: true,
|
||||
stderr: true,
|
||||
resourceLimits: { maxOldGenerationSizeMb: boot.maxOldGenerationSizeMb },
|
||||
})
|
||||
const current = worker
|
||||
current.stdout.on('data', (data: Buffer) => {
|
||||
if (!send({ type: 'log', text: data.toString('utf8') })) finish(frameLimitFailure())
|
||||
})
|
||||
current.stderr.on('data', (data: Buffer) => {
|
||||
if (!send({ type: 'log', text: data.toString('utf8') })) finish(frameLimitFailure())
|
||||
})
|
||||
current.on('message', (messageRaw: unknown) => {
|
||||
const message = recordOf(messageRaw)
|
||||
if (message === undefined) return
|
||||
if (message.type === 'call'
|
||||
&& typeof message.id === 'number'
|
||||
&& typeof message.global === 'string'
|
||||
&& typeof message.name === 'string') {
|
||||
if (!send({
|
||||
type: 'call', id: message.id, global: message.global, name: message.name, args: transportWireOrNull(message.args),
|
||||
})) finish(frameLimitFailure())
|
||||
} else if (message.type === 'log' && typeof message.text === 'string') {
|
||||
if (!send({ type: 'log', text: message.text })) finish(frameLimitFailure())
|
||||
} else if (message.type === 'output-limit') {
|
||||
finish({ type: 'output-limit' })
|
||||
} else if (message.type === 'done') {
|
||||
if (message.error !== undefined) {
|
||||
if (validWorkerFailure(message.error)) finish({ type: 'done', error: message.error })
|
||||
} else {
|
||||
finish({ type: 'done', ...message.value === undefined ? {} : { value: transportWireOrNull(message.value) } })
|
||||
}
|
||||
}
|
||||
})
|
||||
current.on('error', (error: Error) => {
|
||||
finish({
|
||||
type: 'done',
|
||||
error: { kind: 'worker-exit', message: `worker error: ${error.stack || error.message || inspect(error)}` },
|
||||
})
|
||||
})
|
||||
current.on('exit', (code: number) => {
|
||||
if (!finished) {
|
||||
finish({ type: 'done', error: { kind: 'worker-exit', message: `worker exited with code ${code} before completing` } })
|
||||
}
|
||||
})
|
||||
computeTimer = setInterval(() => {
|
||||
if (worker !== undefined && worker.performance.eventLoopUtilization().active > boot.computeMs) {
|
||||
finish({ type: 'done', error: { kind: 'timeout', message: `compute budget exhausted (${boot.computeMs}ms busy)` } })
|
||||
}
|
||||
}, 25)
|
||||
return
|
||||
}
|
||||
const reply = runtimeReply(raw)
|
||||
if (reply !== undefined) worker.postMessage(reply)
|
||||
})
|
||||
process.on('disconnect', () => { if (worker !== undefined && !finished) void worker.terminate() })
|
||||
}
|
||||
|
||||
if (!isMainThread) {
|
||||
if (parentPort === null) throw new Error('remote worker requires parentPort')
|
||||
const boot = workerData as RuntimeBootData
|
||||
void runWorkerMain(parentPort, boot, { stdout: process.stdout, stderr: process.stderr }, boot.maxFrameBytes)
|
||||
} else if (process.env.DSH_CODE_RUNTIME_CONTROLLER === '1') {
|
||||
runController()
|
||||
} else {
|
||||
runLauncher()
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
/** Generate the subprocess Code Runtime's dependency-free runner bundle. */
|
||||
|
||||
import { existsSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import { resolve } from 'node:path'
|
||||
import { build } from 'tsdown'
|
||||
|
||||
const root = resolve(import.meta.dirname, '..')
|
||||
const ENTRY = 'packages/code-runtime/code-runtime-subprocess/src/runner.ts'
|
||||
const OUT = 'packages/code-runtime/code-runtime-subprocess/src/runner-source.generated.ts'
|
||||
|
||||
/**
|
||||
* Bundle the typed runner and shared worker implementation into one source literal.
|
||||
* @returns generated TypeScript module consumed by the subprocess backend.
|
||||
*/
|
||||
export async function renderCodeRuntimeRunner(): Promise<string> {
|
||||
const bundles = await build({
|
||||
config: false,
|
||||
entry: [resolve(root, ENTRY)],
|
||||
format: ['esm'],
|
||||
platform: 'node',
|
||||
target: 'es2024',
|
||||
write: false,
|
||||
dts: false,
|
||||
clean: false,
|
||||
minify: true,
|
||||
logLevel: 'silent',
|
||||
report: false,
|
||||
deps: { alwaysBundle: ['@deepseek-ai/dsh-code-runtime-worker'] },
|
||||
})
|
||||
try {
|
||||
const chunks = bundles.flatMap(bundle => bundle.chunks).filter(chunk => chunk.type === 'chunk')
|
||||
if (chunks.length !== 1) throw new Error(`gen-code-runtime-runner: expected one chunk, received ${chunks.length}`)
|
||||
const chunk = chunks[0]
|
||||
if (chunk === undefined) throw new Error('gen-code-runtime-runner: runner chunk is missing')
|
||||
const external = chunk.imports.filter(specifier => !specifier.startsWith('node:'))
|
||||
if (external.length > 0) {
|
||||
throw new Error(`gen-code-runtime-runner: runner retained external imports: ${external.join(', ')}`)
|
||||
}
|
||||
return [
|
||||
'/**',
|
||||
' * Generated dependency-free execution-world runner.',
|
||||
' * Do not edit by hand; run `pnpm run gen-code-runtime-runner`.',
|
||||
' */',
|
||||
'',
|
||||
`export const CODE_RUNNER_SOURCE = ${JSON.stringify(chunk.code)}`,
|
||||
'',
|
||||
].join('\n')
|
||||
} finally {
|
||||
await Promise.all(bundles.map(async (bundle) => { await bundle[Symbol.asyncDispose]() }))
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const content = await renderCodeRuntimeRunner()
|
||||
const output = resolve(root, OUT)
|
||||
if (process.argv.includes('--check')) {
|
||||
const committed = existsSync(output) ? readFileSync(output, 'utf8') : null
|
||||
if (committed === content) {
|
||||
console.log(`gen-code-runtime-runner: ${OUT} is up to date.`)
|
||||
return
|
||||
}
|
||||
console.error(`gen-code-runtime-runner: ${OUT} is stale. Run \`pnpm run gen-code-runtime-runner\` and commit it.`)
|
||||
process.exitCode = 1
|
||||
return
|
||||
}
|
||||
writeFileSync(output, content)
|
||||
console.log(`gen-code-runtime-runner: wrote ${OUT}.`)
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.filename === resolve(process.argv[1])) await main()
|
||||
@@ -585,6 +585,7 @@ function docSyncLeafGates(options: {
|
||||
pnpmScript('persistence-catalog', 'verify-persistence-catalog', { label: 'persistence catalog' }),
|
||||
pnpmScript('doc-graphs', 'verify-doc-graphs', { label: 'doc graphs' }),
|
||||
pnpmScript('scoped-events', 'verify-scoped-events', { label: 'scoped events' }),
|
||||
pnpmScript('code-runtime-runner', 'verify-code-runtime-runner', { label: 'code-runtime runner' }),
|
||||
pnpmScript('markdown-wrap', 'verify-md-wrap', { label: 'markdown wrap' }),
|
||||
pnpmScript('markdown-links', 'verify-md-links', { label: 'markdown links' }),
|
||||
pnpmScript('public-repository-links', 'verify-public-repository-links', { label: 'public repository links' }),
|
||||
|
||||
+18
-78
@@ -1,16 +1,5 @@
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import tsconfigPaths from 'vite-tsconfig-paths'
|
||||
import { resolvePwshPath } from './packages/bash/pwsh-local/src/resolve.ts'
|
||||
import { defineConfig } from 'vitest/config'
|
||||
import { standardDecoratorPlugin, vitestExecArgv } from './vitest.shared.ts'
|
||||
import { COVERAGE_EXEMPT_ENV, coverageExemptHeavySuites } from './scripts/coverage-exempt.ts'
|
||||
|
||||
// Prints exact `path:line:col` records for every uncovered statement, branch
|
||||
// path, and function when a file misses the per-file 100% gate — the built-in
|
||||
// threshold ERRORs name only the file. Absolute path because istanbul-reports
|
||||
// require()s custom reporters (which is also why the reporter is CJS).
|
||||
const uncoveredLocationsReporter = fileURLToPath(new URL('./scripts/coverage-uncovered-locations.cjs', import.meta.url))
|
||||
|
||||
// Resolution facade shared by every plugin instance below: tsconfig.base.json
|
||||
// has no include, which vite-tsconfig-paths treats as match-all, so its paths
|
||||
@@ -20,14 +9,7 @@ const pathsPlugin = (): ReturnType<typeof tsconfigPaths> => tsconfigPaths({ proj
|
||||
|
||||
const windowsUnsupportedPackages = process.platform === 'win32'
|
||||
? [
|
||||
// Bash-requiring suites (a real POSIX shell is unavailable on Windows).
|
||||
// The pwsh-requiring suites (pwsh-local, tool-pwsh) deliberately stay
|
||||
// INCLUDED: PowerShell ships with Windows, so they run natively here.
|
||||
// Replacing the old 'packages/bash/*' glob with this explicit list also
|
||||
// newly INCLUDES packages/bash/bash (the pure seam package) on Windows.
|
||||
'packages/bash/bash-local',
|
||||
'packages/bash/bash-sandbox',
|
||||
'packages/bash/tool-bash',
|
||||
'packages/bash/*',
|
||||
'packages/hooks/*',
|
||||
'packages/subprocess/*',
|
||||
'packages/pty/pty-local',
|
||||
@@ -44,20 +26,10 @@ const windowsCoverageExclusions = process.platform === 'win32'
|
||||
'packages/lsp/lsp-local/src/connection.ts',
|
||||
'packages/lsp/lsp-local/src/index.ts',
|
||||
'packages/lsp/lsp-local/src/instance.ts',
|
||||
'packages/ui/tui/src/index.ts',
|
||||
]
|
||||
: []
|
||||
|
||||
// Mirrors windowsCoverageExclusions: pwsh-local's run/start/lifecycle suites
|
||||
// self-skip without a real pwsh (executor.spec.ts hasPwsh), leaving this file
|
||||
// far below per-file 100% on pwsh-less hosts; the exemption keeps those hosts
|
||||
// green while CI runners ship pwsh and still enforce the full bar. The probe
|
||||
// runs the suites' own resolution (the dependency-free resolve.ts module),
|
||||
// so the exemption is active exactly when the suites skip — a mismatched
|
||||
// narrower probe could exempt the file on hosts whose suites actually run.
|
||||
const pwshCoverageExclusions = spawnSync(resolvePwshPath(), ['-NoLogo', '-NoProfile', '-NonInteractive', '-Command', '$true'], { encoding: 'utf8' }).status === 0
|
||||
? []
|
||||
: ['packages/bash/pwsh-local/src/index.ts']
|
||||
|
||||
const testIncludes = [
|
||||
'packages/*/*/tests/**/*.spec.{ts,tsx}',
|
||||
'apps/*/tests/**/*.spec.ts',
|
||||
@@ -65,17 +37,6 @@ const testIncludes = [
|
||||
'scripts/**/*.spec.ts',
|
||||
]
|
||||
|
||||
// The instrumented coverage gate sets this env; the exempt heavy suites then
|
||||
// run beside it uninstrumented (membership contract in scripts/coverage-exempt.ts).
|
||||
// A set-but-not-'1' value is a misconfiguration, not a silent no-op.
|
||||
const coverageExemptRaw = process.env[COVERAGE_EXEMPT_ENV]
|
||||
if (coverageExemptRaw !== undefined && coverageExemptRaw !== '' && coverageExemptRaw !== '1') {
|
||||
throw new Error(`vitest config: ${COVERAGE_EXEMPT_ENV} must be '1' or unset, got ${JSON.stringify(coverageExemptRaw)}.`)
|
||||
}
|
||||
const coverageExemptExcludes = coverageExemptRaw === '1'
|
||||
? coverageExemptHeavySuites.map(suite => suite.exclude)
|
||||
: []
|
||||
|
||||
// These suites exercise process-global state, process APIs, or timing-sensitive process I/O
|
||||
// that worker threads cannot isolate reliably under aggregate gate contention.
|
||||
// Keep the narrow exception in forks while the rest of the inventory avoids per-file processes.
|
||||
@@ -88,48 +49,39 @@ const processBoundTests = [
|
||||
]
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [pathsPlugin(), standardDecoratorPlugin()],
|
||||
plugins: [pathsPlugin()],
|
||||
test: {
|
||||
setupFiles: ['./scripts/test-invariants.ts'],
|
||||
// .tsx: client component specs (jsdom via per-file @vitest-environment pragma).
|
||||
include: testIncludes,
|
||||
exclude: windowsUnsupportedPackages.map(path => `${path}/tests/**/*.spec.ts`),
|
||||
// One coverage invocation aggregates both projects. Regular suites fork on
|
||||
// POSIX for Node stability and use threads on Windows; process-bound suites
|
||||
// always fork.
|
||||
// One coverage invocation aggregates both projects. Most suites use threads
|
||||
// for lower startup/IPC overhead; only explicit process-bound suites fork.
|
||||
projects: [
|
||||
{
|
||||
plugins: [pathsPlugin(), standardDecoratorPlugin()],
|
||||
plugins: [pathsPlugin()],
|
||||
test: {
|
||||
name: 'thread-safe',
|
||||
execArgv: vitestExecArgv,
|
||||
// Node 24 has aborted in its CJS lexer (v8::ToLocalChecked Empty
|
||||
// MaybeLocal in cjs_lexer::Parse) from worker threads on macOS
|
||||
// arm64 and later on Linux. A fork contains that external runtime
|
||||
// failure to the test process; Windows keeps the thread pool, where
|
||||
// the abort has not reproduced and process spawn is costlier.
|
||||
pool: process.platform === 'win32' ? 'threads' : 'forks',
|
||||
// Node 24 has aborted in its CJS lexer from a macOS arm64 worker
|
||||
// thread. A fork contains that external runtime failure to the test
|
||||
// process; other hosts retain the lower-overhead thread pool.
|
||||
pool: process.platform === 'darwin' ? 'forks' : 'threads',
|
||||
setupFiles: ['./scripts/test-invariants.ts'],
|
||||
include: testIncludes,
|
||||
exclude: [
|
||||
...windowsUnsupportedPackages.map(path => `${path}/tests/**/*.spec.ts`),
|
||||
...processBoundTests,
|
||||
...coverageExemptExcludes,
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
plugins: [pathsPlugin(), standardDecoratorPlugin()],
|
||||
plugins: [pathsPlugin()],
|
||||
test: {
|
||||
name: 'process-bound',
|
||||
execArgv: vitestExecArgv,
|
||||
pool: 'forks',
|
||||
setupFiles: ['./scripts/test-invariants.ts'],
|
||||
include: processBoundTests,
|
||||
exclude: [
|
||||
...windowsUnsupportedPackages.map(path => `${path}/tests/**/*.spec.ts`),
|
||||
...coverageExemptExcludes,
|
||||
],
|
||||
exclude: windowsUnsupportedPackages.map(path => `${path}/tests/**/*.spec.ts`),
|
||||
},
|
||||
},
|
||||
],
|
||||
@@ -146,8 +98,7 @@ export default defineConfig({
|
||||
'packages/*/*/src/types.ts',
|
||||
'packages/*/*/src/bin.ts',
|
||||
'packages/*/*/src/worker.ts',
|
||||
// A killed executable lint-contract test can leave a non-product source probe behind.
|
||||
'packages/*/*/src/oxlint-contract-*.ts',
|
||||
'packages/code-runtime/code-runtime-subprocess/src/runner.ts',
|
||||
// GUI step-1 skeleton (PR #500): client/web UI files whose remaining
|
||||
// branches need a browser-grade harness the jsdom lane doesn't cover
|
||||
// yet. TODO(gui): cover and remove as the client test lane matures.
|
||||
@@ -156,7 +107,6 @@ export default defineConfig({
|
||||
'packages/client/ui-primitives/src/markdown/plain-text.ts',
|
||||
'packages/client/ui-question/src/client/QuestionComposer.tsx',
|
||||
'packages/client/ui-primitives/src/Menu.tsx',
|
||||
'packages/client/ui-primitives/src/RiskConfirmation.tsx',
|
||||
'packages/client/ui-workspace/src/client/WorkspaceBrowser.tsx',
|
||||
'packages/client/ui-workspace/src/client/WorkspacePicker.tsx',
|
||||
'packages/client/web-react/src/*',
|
||||
@@ -181,10 +131,6 @@ export default defineConfig({
|
||||
'packages/client/hmr/src/invariant.ts',
|
||||
'packages/client/connection/src/index.ts',
|
||||
'packages/client/connection/src/http-bridge.ts',
|
||||
// This assembly imports generated Host-for-Client code that exists
|
||||
// only in lib; the post-build built-bin smoke executes both entries.
|
||||
'packages/api/remotes/src/index.ts',
|
||||
'packages/api/remotes/src/client/index.ts',
|
||||
// Slash/command/input round: per-file gaps deferred with the same
|
||||
// client-lane debt. TODO(gui): cover and remove with the lane above.
|
||||
'packages/client/connection/src/client/fixture.ts',
|
||||
@@ -209,13 +155,9 @@ export default defineConfig({
|
||||
'packages/client/ui-sidebar/src/client/index.ts',
|
||||
'packages/client/ui-skill/src/client/index.ts',
|
||||
'packages/client/ui-workspace/src/client/index.ts',
|
||||
'packages/client/test-runtime/src/translate.ts',
|
||||
'packages/client/ui-primitives/src/JsonTree.tsx',
|
||||
// Typert generator: correctness is pinned by its fixture suites and
|
||||
// the byte-for-byte catalog reproduction test; per-file coverage
|
||||
// would put whole-workspace compiler analysis under v8
|
||||
// instrumentation — the coverage lane's longest tail.
|
||||
'packages/typert/generator/src/*.ts',
|
||||
'packages/typert/generator/src/analyzer.ts',
|
||||
'packages/typert/generator/src/renderer.ts',
|
||||
'packages/typert/generator/src/cordis-catalog.ts',
|
||||
'packages/host/apiproxy/src/index.ts',
|
||||
'packages/host/apiproxy/src/invariant.ts',
|
||||
'packages/host/apiproxy/src/api-proxy.ts',
|
||||
@@ -225,9 +167,9 @@ export default defineConfig({
|
||||
'packages/ui/commands/src/index.ts',
|
||||
'packages/ui/commands/src/invariant.ts',
|
||||
'packages/session-projection/session-projection/src/index.ts',
|
||||
'packages/ui/tui/src/index.ts',
|
||||
...windowsUnsupportedPackages.map(path => `${path}/src/**/*.ts`),
|
||||
...windowsCoverageExclusions,
|
||||
...pwshCoverageExclusions,
|
||||
],
|
||||
// 100% or it doesn't merge (docs/testing.md: excessive tests are welcome).
|
||||
// Per-file so a well-covered big file can't subsidize a bare one.
|
||||
@@ -240,9 +182,7 @@ export default defineConfig({
|
||||
functions: 100,
|
||||
lines: 100,
|
||||
},
|
||||
reporter: process.env.CI
|
||||
? ['text', uncoveredLocationsReporter]
|
||||
: ['text', 'html', uncoveredLocationsReporter],
|
||||
reporter: process.env.CI ? ['text'] : ['text', 'html'],
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user