mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-09-11 04:00:38 +00:00
fix(code-runtime-python): reap same-group children and correct log-budget bound
Address review findings on the CPython backend: - CRITICAL: a model program could leave a descendant in the child's own process group that ignores SIGTERM but releases the inherited pipes, so the leader's `close` fired and settle() cancelled the pending SIGKILL before it escalated — run()/dispose() returned while that child lived. kill() now unrefs the grace timer and settle() no longer clears it, so the SIGKILL reaches the whole group; killGroup swallows ESRCH when the group is already gone (the normal case). Adds a real-subprocess regression test. - WARNING: the maxLogBytes/maxValueBytes load bound divided the frame ceiling by 6 for escape expansion, but both budgets are metered in already-escaped serialized bytes, so a payload occupies at most cap+envelope on the wire. Bound is now ceiling-envelope; drop the unused escape constant. - Narrow the runtime.spec.ts header to "no subprocess mocks" (it mocks node:fs.copyFileSync for staging-failure cases). - Use full-width punctuation in the README.zh.md prose per translation rules; re-record the pair.
This commit is contained in:
Chinesezjc
@@ -3,4 +3,4 @@
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write packages/code-runtime/code-runtime-python/README.md
|
||||
README.md: 31c493ee119f145c7cf11ed22e2c191226919410
|
||||
README.zh.md: 6f808ed7c768985be38c6b9f4d6cb701e702cd76
|
||||
README.zh.md: 60233e71c3830e5acdb43b80ba7cdccfceef064a
|
||||
|
||||
@@ -22,7 +22,7 @@ host 与 CPython 子进程在子进程的 fd 3 上交换一个无版本号的 JS
|
||||
|
||||
## Configuration
|
||||
|
||||
每个上限都是带默认值的、经校验的 `Config` 字段,可从 `cordis.yml` 修改(无硬编码可调项)。`cpuSeconds`(默认 60)是 `RLIMIT_CPU` 的整秒预算;子进程把软限设为 `cpuSeconds`、硬限设为 `cpuSeconds + 1`,因此内核在软限处发出的 `SIGXCPU` 被归类为 `timeout`,而 +1 秒的硬限是 `SIGKILL` 兜底。`maxWallMs`(默认 600000)是墙钟上限,为一个在等待无人 resolve 的 promise 的程序兜住 CPU 时间。`addressSpaceMb`(默认 512)是 `RLIMIT_AS` 上限,在 Darwin 上不施加(那里映射进每个进程的 dyld 共享缓存超过任何实际上限;`cpuSeconds` 与 `maxWallMs` 仍约束运行)。`maxLogBytes`(默认 65536)是共享的捕获日志字节预算;`maxValueBytes`(默认 32768)为完成值设上限;`graceMs`(默认 3000)是 `SIGTERM`→`SIGKILL` 的 grace 窗口;`pythonBin`(默认 `python3`)是解释器,在子进程以空环境启动前先对 `PATH` 解析。
|
||||
每个上限都是带默认值的、经校验的 `Config` 字段，可从 `cordis.yml` 修改（无硬编码可调项）。`cpuSeconds`（默认 60）是 `RLIMIT_CPU` 的整秒预算；子进程把软限设为 `cpuSeconds`、硬限设为 `cpuSeconds + 1`，因此内核在软限处发出的 `SIGXCPU` 被归类为 `timeout`，而 +1 秒的硬限是 `SIGKILL` 兜底。`maxWallMs`（默认 600000）是墙钟上限，为一个在等待无人 resolve 的 promise 的程序兜住 CPU 时间。`addressSpaceMb`（默认 512）是 `RLIMIT_AS` 上限，在 Darwin 上不施加（那里映射进每个进程的 dyld 共享缓存超过任何实际上限；`cpuSeconds` 与 `maxWallMs` 仍约束运行）。`maxLogBytes`（默认 65536）是共享的捕获日志字节预算；`maxValueBytes`（默认 32768）为完成值设上限；`graceMs`（默认 3000）是 `SIGTERM`→`SIGKILL` 的 grace 窗口；`pythonBin`（默认 `python3`）是解释器，在子进程以空环境启动前先对 `PATH` 解析。
|
||||
|
||||
## Model Experience
|
||||
|
||||
@@ -35,4 +35,4 @@ No direct invalidation; the named consumer owns any request-prefix changes.
|
||||
## Known Limitations and Deferred Work
|
||||
|
||||
- **跨语言 guard 覆盖执行值与帧字段集，但不覆盖字段类型** —— `tests/protocol-mirror.e2e.ts` 使用真实 `python3` 比较 `PROTOCOL_FD`、日志截断标记，以及每个 `TypedDict` 的必填和可选字段。跨 TypeScript 与 Python 比较字段类型在此没有机械等价物，因此类型级漂移由 review 加后端真子进程套件负责。
|
||||
- **`RLIMIT_AS` 在 macOS 上不施加** —— 在 exec 时映射进每个进程的 dyld 共享缓存超过任何实际的地址空间上限,内核会拒绝该 `setrlimit` 调用,故 `addressSpaceMb` 在那里被跳过。`cpuSeconds` 与 `maxWallMs` 仍约束每一次运行。
|
||||
- **`RLIMIT_AS` 在 macOS 上不施加** —— 在 exec 时映射进每个进程的 dyld 共享缓存超过任何实际的地址空间上限，内核会拒绝该 `setrlimit` 调用，故 `addressSpaceMb` 在那里被跳过。`cpuSeconds` 与 `maxWallMs` 仍约束每一次运行。
|
||||
|
||||
@@ -208,13 +208,6 @@ const MAX_PENDING_CHUNKS = 1024
|
||||
*/
|
||||
const FRAME_ENVELOPE_BYTES = 64
|
||||
|
||||
/**
|
||||
* The most bytes one payload character can occupy once JSON-escaped: a control
|
||||
* character renders as `\uXXXX`. Used with {@link FRAME_ENVELOPE_BYTES} to turn
|
||||
* the frame ceiling into an admissible output cap.
|
||||
*/
|
||||
const MAX_JSON_ESCAPE_EXPANSION = 6
|
||||
|
||||
/**
|
||||
* Extra time added to `graceMs` before the post-kill close-deadline force-settles
|
||||
* a run whose `close` never fires (a setsid-escaped orphan holds our inherited
|
||||
@@ -496,12 +489,14 @@ export class PythonCodeRuntime extends CodeRuntime {
|
||||
// carry is unsatisfiable: a completion or log entry that the cap admits
|
||||
// arrives as an over-ceiling frame and fails the run as `worker-exit`
|
||||
// instead of the `output-limit` the cap describes — a silent inversion, so
|
||||
// it fails at load. The bound subtracts the frame's own envelope, since the
|
||||
// ceiling covers the whole line: worst case is every payload byte escaping
|
||||
// to six (`\uXXXX` per control character), so the admissible cap is
|
||||
// `(ceiling - envelope) / 6`.
|
||||
// it fails at load. Both budgets are metered in SERIALIZED (JSON-escaped)
|
||||
// bytes — the host log ledger charges `Buffer.byteLength(JSON.stringify(text))`
|
||||
// and `checkDoneValue` measures the escaped form — so a payload admitted
|
||||
// under the cap occupies at most `cap + envelope` bytes on the wire; escaping
|
||||
// is already inside the charge and must not be multiplied in again. The
|
||||
// admissible cap is therefore `ceiling - envelope`.
|
||||
for (const key of ['maxLogBytes', 'maxValueBytes'] as const) {
|
||||
const limit = Math.floor((FRAME_CEILING_BYTES - FRAME_ENVELOPE_BYTES) / MAX_JSON_ESCAPE_EXPANSION)
|
||||
const limit = FRAME_CEILING_BYTES - FRAME_ENVELOPE_BYTES
|
||||
if (this.config[key] > limit) {
|
||||
throw new Error(`dsh-code-runtime-python: config.${key} must not exceed ${limit} (a payload that large cannot cross the ${FRAME_CEILING_BYTES}-byte fd-3 frame ceiling, so the run would fail as worker-exit rather than output-limit), got ${String(this.config[key])}`)
|
||||
}
|
||||
@@ -966,7 +961,6 @@ export class PythonCodeRuntime extends CodeRuntime {
|
||||
// Escalate SIGTERM → grace → SIGKILL on the entire process group. Idempotent
|
||||
// via `killing`.
|
||||
let killing = false
|
||||
let graceTimer: NodeJS.Timeout | undefined
|
||||
// A backstop for the one case `close` cannot cover: model code that starts
|
||||
// a descendant with `os.setsid()`/`start_new_session=True` moves it into a
|
||||
// fresh process group, so the SIGTERM/SIGKILL aimed at the child's group
|
||||
@@ -989,7 +983,21 @@ export class PythonCodeRuntime extends CodeRuntime {
|
||||
if (killing) return
|
||||
killing = true
|
||||
killGroup('SIGTERM')
|
||||
graceTimer = setTimeout(() => { killGroup('SIGKILL') }, this.config.graceMs)
|
||||
// The SIGKILL is left to fire on its own timer and is deliberately NOT
|
||||
// cancelled at settlement. A model program can leave a descendant in the
|
||||
// SAME process group `kill(-pid)` targets — no setsid, so it stays in the
|
||||
// group — that ignores SIGTERM but releases the inherited stdout/stderr/
|
||||
// fd-3 pipes: the leader then exits, its `close` fires (the pipes drained),
|
||||
// and settle() runs while that descendant is still alive. Cancelling the
|
||||
// timer there would strand it, breaking "no subprocess outlives the fiber".
|
||||
// Letting the timer elapse SIGKILLs the whole group, reaching the survivor;
|
||||
// `killGroup` swallows ESRCH, so firing against an already-dead group (the
|
||||
// normal case, where the leader was the only member) is harmless. `unref`
|
||||
// so a pending SIGKILL never keeps the host process alive after run()
|
||||
// resolves. (A setsid-escaped orphan in a FRESH group is the different case
|
||||
// `closeDeadline` in finish() covers, since `close` never fires there.)
|
||||
const graceTimer = setTimeout(() => { killGroup('SIGKILL') }, this.config.graceMs)
|
||||
graceTimer.unref()
|
||||
}
|
||||
|
||||
let finishResolve!: () => void
|
||||
@@ -1005,7 +1013,11 @@ export class PythonCodeRuntime extends CodeRuntime {
|
||||
const settle = (result: Omit<CodeRunResult, 'logs'>): void => {
|
||||
if (resolved) return
|
||||
resolved = true
|
||||
if (graceTimer !== undefined) clearTimeout(graceTimer)
|
||||
// The grace-window SIGKILL timer is intentionally NOT cleared here: a
|
||||
// same-group descendant that ignored SIGTERM but released the pipes lets
|
||||
// `close` fire (and settle() run) while it is still alive, so the pending
|
||||
// SIGKILL must remain armed to reap it (see kill()). The timer is
|
||||
// `unref`'d, so leaving it pending cannot keep the host process alive.
|
||||
if (closeDeadline !== undefined) clearTimeout(closeDeadline)
|
||||
// Drop from `live` only at settlement (close / pid-less spawn failure),
|
||||
// NOT at finish(): between finish() and the child's `close` the child
|
||||
|
||||
@@ -30,9 +30,10 @@ vi.mock('node:fs', async (importOriginal) => {
|
||||
})
|
||||
|
||||
/**
|
||||
* Integration suite over REAL python3 subprocesses (no mocks — subprocess is
|
||||
* cheap and local, per docs/testing.md's real-over-mock policy). Each test
|
||||
* builds a fresh runtime so budgets can be tuned per case.
|
||||
* Integration suite over REAL python3 subprocesses (no subprocess mocks — it is
|
||||
* cheap and local, per docs/testing.md's real-over-mock policy; the only mock is
|
||||
* `node:fs.copyFileSync` for the staging-failure cases). Each test builds a fresh
|
||||
* runtime so budgets can be tuned per case.
|
||||
*/
|
||||
async function setup(config: Config = {}) {
|
||||
const ctx = new Context()
|
||||
@@ -1921,6 +1922,80 @@ describe('PythonCodeRuntime — budgets, termination, disposal', () => {
|
||||
expect(elapsed).toBeGreaterThanOrEqual(1_500)
|
||||
expect(elapsed).toBeLessThan(5_000)
|
||||
}, 8000)
|
||||
|
||||
it('reaps a same-group child that ignores SIGTERM and releases the pipes before close', async () => {
|
||||
// The same-group counterpart to the setsid-orphan case above. A descendant
|
||||
// left in the child's OWN process group (no setsid, so `kill(-pid)` reaches
|
||||
// it) can ignore SIGTERM yet still release the inherited stdout/stderr/fd 3
|
||||
// it does not hold — here by giving the Popen child DEVNULL streams and
|
||||
// letting close_fds drop fd 3. The leader then writes `done` and exits, its
|
||||
// `close` fires because the pipes drained, and settle() runs while that
|
||||
// descendant is still alive. If settle() cancelled the grace-window SIGKILL
|
||||
// the descendant would outlive the fiber; leaving the unref'd timer to fire
|
||||
// SIGKILLs the whole group and reaps it.
|
||||
//
|
||||
// The descendant must have SIG_IGN installed BEFORE the host sends SIGTERM,
|
||||
// or it dies from the default SIGTERM whether the fix is present or not — so
|
||||
// it writes a readiness marker after trapping and the leader waits for that
|
||||
// marker before returning. The descendant sleeps 30 s as a safety net so a
|
||||
// broken fix cannot leak it forever; the assertion window is far shorter, so
|
||||
// it genuinely tests the SIGKILL reaping rather than the self-timeout.
|
||||
const handoff = await mkdtemp(join(tmpdir(), 'dsh-samegroup-'))
|
||||
const readyMarker = join(handoff, 'ready')
|
||||
const { runtime } = await setup({ maxWallMs: 10_000, graceMs: 300 })
|
||||
let reportedPid!: (pid: number) => void
|
||||
const childPid = new Promise<number>((resolve) => { reportedPid = resolve })
|
||||
const result = await runtime.run({
|
||||
program: [
|
||||
'import subprocess, sys, os, time',
|
||||
`marker = ${JSON.stringify(readyMarker)}`,
|
||||
// Same group (no start_new_session); ignores SIGTERM; holds none of the
|
||||
// leader's pipes (DEVNULL std streams, close_fds drops fd 3). It writes
|
||||
// the marker (its argv[1]) only AFTER the trap is installed, so the
|
||||
// leader cannot return — and the host cannot send SIGTERM — before the
|
||||
// descendant ignores it.
|
||||
'code = "import signal, sys, time; signal.signal(signal.SIGTERM, signal.SIG_IGN); open(sys.argv[1], \'w\').close(); time.sleep(30)"',
|
||||
'child = subprocess.Popen([sys.executable, "-c", code, marker],',
|
||||
' stdin=subprocess.DEVNULL,',
|
||||
' stdout=subprocess.DEVNULL,',
|
||||
' stderr=subprocess.DEVNULL)',
|
||||
'deadline = time.time() + 5',
|
||||
'while not os.path.exists(marker) and time.time() < deadline:',
|
||||
' time.sleep(0.02)',
|
||||
'await tools.report({"pid": child.pid})',
|
||||
'return "spawned"',
|
||||
].join('\n'),
|
||||
bindings: tools({
|
||||
report: async (args) => {
|
||||
reportedPid((args as { pid: number }).pid)
|
||||
return 'ok'
|
||||
},
|
||||
}),
|
||||
})
|
||||
expect(result.error).toBeUndefined()
|
||||
expect(result.value).toBe('spawned')
|
||||
const pid = await childPid
|
||||
expect(Number.isInteger(pid) && pid > 0).toBe(true)
|
||||
// The trap really installed before the leader returned, so this is the
|
||||
// SIGTERM-ignoring descendant, not one that would have died to the default.
|
||||
expect(existsSync(readyMarker)).toBe(true)
|
||||
// run() resolved inside the grace window, so the descendant is still alive
|
||||
// here; the pending SIGKILL reaps it shortly after graceMs. Poll until it is
|
||||
// gone, well within the descendant's own 30 s self-timeout.
|
||||
const deadline = Date.now() + 5_000
|
||||
const alive = (): boolean => {
|
||||
try {
|
||||
process.kill(pid, 0)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
while (alive() && Date.now() < deadline) {
|
||||
await new Promise(resolve => setTimeout(resolve, 50))
|
||||
}
|
||||
expect(() => process.kill(pid, 0)).toThrow(/ESRCH/)
|
||||
}, 15_000)
|
||||
})
|
||||
|
||||
describe('PythonCodeRuntime — hostile peer', () => {
|
||||
|
||||
Reference in New Issue
Block a user