diff --git a/.agents/notes/implemented/bug-fix/2026-07-31-code-runtime-python-settlement-fixes.i18n.yaml b/.agents/notes/implemented/bug-fix/2026-07-31-code-runtime-python-settlement-fixes.i18n.yaml index c2bcbbeee6..922d1bdfbd 100644 --- a/.agents/notes/implemented/bug-fix/2026-07-31-code-runtime-python-settlement-fixes.i18n.yaml +++ b/.agents/notes/implemented/bug-fix/2026-07-31-code-runtime-python-settlement-fixes.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/bug-fix/2026-07-31-code-runtime-python-settlement-fixes.md -2026-07-31-code-runtime-python-settlement-fixes.md: 7bb01f08f62994029680cdab3572a10c8971474b -2026-07-31-code-runtime-python-settlement-fixes.zh.md: 169cc34d3edc82ea52c89c37da4db8e9e63dd661 +2026-07-31-code-runtime-python-settlement-fixes.md: aa8e4ce513b9b9c4aaa07b5363bc497c6a4d0a9c +2026-07-31-code-runtime-python-settlement-fixes.zh.md: 36a5d5d06286ee07db3564f6f62c0f9d79288391 diff --git a/.agents/notes/implemented/bug-fix/2026-07-31-code-runtime-python-settlement-fixes.md b/.agents/notes/implemented/bug-fix/2026-07-31-code-runtime-python-settlement-fixes.md index 7bb01f08f6..aa8e4ce513 100644 --- a/.agents/notes/implemented/bug-fix/2026-07-31-code-runtime-python-settlement-fixes.md +++ b/.agents/notes/implemented/bug-fix/2026-07-31-code-runtime-python-settlement-fixes.md @@ -54,13 +54,13 @@ Also in `src/index.ts`, `spawn` is called before the settlement Promise executor ### Stray pipe output is aggregated by line, not by transport chunk -Also in `src/index.ts`, native stdout/stderr bytes (C-extension writes, `os.write` past the pipe buffer) were pushed to `logs` one entry per Node `data` chunk. `logs` entries are joined with `\n` downstream (Code Mode), so a single newline-free write larger than one pipe read — arriving as several `data` chunks — read back with model-visible newlines inserted at arbitrary transport boundaries. Capture now accumulates raw `Buffer` chunks with a running byte counter (the same shape as the fd-3 reader, and for the same reason: a string `+=` accumulator re-copies the whole residual per chunk and scanning it from index 0 each chunk is a second quadratic — both O(N²) on a large newline-free write), splits on the raw `0x0a` byte, and admits one entry per complete line. A newline never appears inside a UTF-8 multibyte sequence, so decoding each split line is safe without a streaming decoder. The residual stays bounded by the ledger: when its raw byte count would cross the budget with no newline in sight it is flushed (admitted and truncated) immediately, and once the ledger has truncated, buffering stops so a newline-free flood cannot retain host memory for output that can never be admitted. The per-entry charge itself is metered by SERIALIZED cost through `jsonStringCostUpTo`, which walks the string to the cap and stops — the previous `Buffer.byteLength(JSON.stringify(text))` allocated the whole escaped form first, so a near-budget control-char-dense line under a large `maxLogBytes` could momentarily allocate over a gigabyte just to measure it. The residual is flushed on the pipe's `end` and also explicitly in the `closeDeadline` handler before it destroys the streams: a `setsid` escapee holding the pipes open forces settlement through that path without an `end`, so a final newline-free `os.write(1, …)` the leader emitted before exiting would otherwise be dropped from `logs`. +Also in `src/index.ts`, native stdout/stderr bytes (C-extension writes, `os.write` past the pipe buffer) were pushed to `logs` one entry per Node `data` chunk. `logs` entries are joined with `\n` downstream (Code Mode), so a single newline-free write larger than one pipe read — arriving as several `data` chunks — read back with model-visible newlines inserted at arbitrary transport boundaries. Capture now accumulates raw `Buffer` chunks (the same shape as the fd-3 reader, and for the same reasons: a string `+=` accumulator re-copies the whole residual per chunk and scanning it from index 0 each chunk is a second quadratic — both O(N²) on a large newline-free write), splits on the raw `0x0a` byte, and admits one entry per complete line. A newline never appears inside a UTF-8 multibyte sequence, so decoding each split line is safe without a streaming decoder. Three separate bounds keep the residual from exhausting host memory, each mirroring the fd-3 reader: the fragment list SEALS into finished blocks past `MAX_PENDING_CHUNKS` so a program pacing single-byte `os.write`s cannot accumulate millions of live Buffer objects (whose per-object overhead no byte count sees); the residual is flushed when its running SERIALIZED cost — tracked per byte through `serializedBufferCost`, a lower bound on the admitted line's exact cost — would cross the budget, so a control-char-dense flood flushes at roughly a sixth of the raw bytes rather than accumulating a full budget's worth of raw bytes first; and once the ledger has truncated, buffering stops so nothing accumulates for output that can never be admitted. The per-entry charge itself is metered by SERIALIZED cost through `jsonStringCostUpTo`, which walks the string to the cap and stops — the previous `Buffer.byteLength(JSON.stringify(text))` allocated the whole escaped form first, so a near-budget control-char-dense line under a large `maxLogBytes` could momentarily allocate over a gigabyte just to measure it. Both cost functions charge a LONE surrogate the full six escaped bytes (`\uXXXX` under ES2019 well-formed `JSON.stringify`), not the three bytes `Buffer.byteLength` reports for its U+FFFD rendering: a forged `log` frame flooding `\ud800` escapes would otherwise be undercharged by half and admit roughly twice the budget. The residual is flushed on the pipe's `end` and also explicitly in the `closeDeadline` handler before it destroys the streams: a `setsid` escapee holding the pipes open forces settlement through that path without an `end`, so a final newline-free `os.write(1, …)` the leader emitted before exiting would otherwise be dropped from `logs`. ## Testing -- `tests/boot-write-failure.spec.ts` mocks `spawn` so the fd-3 pipe throws on the boot write — the one path a real subprocess cannot be coerced into — and asserts `run()` resolves a `worker-exit` rather than rejecting. A sibling case makes the mocked `spawn` throw SYNCHRONOUSLY and asserts `run()` still resolves a `worker-exit` and leaves no NEW `dsh-code-runtime-python-*` directory in `tmpdir` (a set difference against a pre-run snapshot, so a sibling worker's concurrent create or delete cannot flake the assertion). Both are isolated in this spec so the real-subprocess suite is untouched. +- `tests/boot-write-failure.spec.ts` mocks `spawn` so the fd-3 pipe throws on the boot write — the one path a real subprocess cannot be coerced into — and asserts `run()` resolves a `worker-exit` rather than rejecting. A sibling case makes the mocked `spawn` throw SYNCHRONOUSLY and asserts `run()` still resolves a `worker-exit` and removes its staging directory, keyed off the exact bootstrap path the mocked `spawn` received in its argv so a sibling worker's concurrent staging cannot flake it. Both are isolated in this spec so the real-subprocess suite is untouched. - `tests/residual-detach.spec.ts` unit-tests `detachResidual`: the carried copy equals the residual, owns a backing store sized to its own length (fixture kept above Node's Buffer pool threshold), and does not share the source frame's `ArrayBuffer`. -- `tests/runtime.spec.ts` — the output-cap case asserts the `ceiling - envelope` bound (268435392) and its message. A daemon-thread case drives four threads emitting unterminated writes through settlement's flush. A native-write case writes 200 KiB with no newline via `os.write` under a raised `maxLogBytes` and asserts it reads back as EXACTLY one log entry (proving stray output is aggregated by line, not split at pipe-chunk boundaries); a companion writes `b"one\ntwo\nthree"` and asserts three entries (proving real newlines still delimit). A newline-free-flood case writes 2 MiB under a 4 KiB `maxLogBytes` and asserts the capture ends at the truncation marker and stays under budget (proving the residual is bounded by the ledger, not buffered whole); a NUL-flood companion writes 4000 newline-free NULs under the same budget and asserts truncation (proving the residual is charged by SERIALIZED cost, ~6× raw, measured without allocating the escaped copy); a short-escape case writes a line mixing a tab, quote, backslash, a `\uXXXX` control, a multibyte character, and ASCII, asserting it round-trips verbatim (exercising every branch of `jsonStringCostUpTo`). A closeDeadline-flush case has the leader write a newline-free diagnostic then spawn a `setsid` orphan holding the pipes open, and asserts the diagnostic survives in `logs` (proving the residual is flushed before the deadline destroys the streams). The same-group reap case spawns a SIGTERM-ignoring same-group descendant that releases the pipes and bumps a heartbeat file; the test asserts the heartbeat STOPS after the grace-window SIGKILL — an assertion robust whether the killed descendant is reaped or lingers as a zombie, so it holds where PID 1 does not wait() orphans. A dispose-after-resolve case asserts `dispose()` of a completed run with a same-group survivor returns only after the survivor stops executing (proving the run stays in `live` until its group is reaped), with an `expect(afterDispose).toBeGreaterThan(0)` guard so the frozen-heartbeat assertion cannot pass vacuously when the file was never written. A deadline case busy-blocks the event loop past both timers and asserts the survivor's heartbeat freezes (proving the poll's deadline arm sends SIGKILL itself rather than cancelling the unfired escalation). The cross-loop case runs a binding from a worker thread's own `asyncio.run` loop while the main coroutine yields with `await asyncio.sleep`, asserting the reply round-trips instead of timing out; a companion case abandons a thread's call so its loop closes, then answers it before a later binding — asserting the pump survives the closed-loop `call_soon_threadsafe` (host-gated ordering makes it deterministic, fail-before hangs the later binding to the wall clock). The inherited-soft-limit case runs the interpreter through a `ulimit -S -t` wrapper that sets a CPU soft limit below `cpuSeconds` and asserts the applied `RLIMIT_CPU` soft is the inherited value, not the configured one (CPU rather than address space, since macOS ignores `ulimit -v`); a companion inherits a 1 s CPU soft, has the program trap SIGXCPU and busy-loop past it, and asserts the settlement recheck reports a timeout — proving the recheck uses the effective soft, not the configured `cpuSeconds`. A control-heavy-diagnostic case raises a NUL-flood exception under a small `maxValueBytes` and asserts the serialized frame fits (proving the diagnostic is metered by serialized cost). A non-integer-budget case asserts a fractional `maxLogBytes`/`maxValueBytes` rejects at load. +- `tests/runtime.spec.ts` — the output-cap case asserts the `ceiling - envelope` bound (268435392) and its message. A daemon-thread case drives four threads emitting unterminated writes through settlement's flush. A native-write case writes 200 KiB with no newline via `os.write` under a raised `maxLogBytes` and asserts it reads back as EXACTLY one log entry (proving stray output is aggregated by line, not split at pipe-chunk boundaries); a companion writes `b"one\ntwo\nthree"` and asserts three entries (proving real newlines still delimit). A newline-free-flood case writes 2 MiB under a 4 KiB `maxLogBytes` and asserts the capture ends at the truncation marker and stays under budget (proving the residual is bounded by the ledger, not buffered whole); a NUL-flood companion writes 4000 newline-free NULs under the same budget and asserts truncation (proving the residual is charged by SERIALIZED cost, ~6× raw, measured without allocating the escaped copy); a short-escape case writes a line mixing a tab, quote, backslash, a `\uXXXX` control, a multibyte character, and ASCII, asserting it round-trips verbatim (exercising every branch of `jsonStringCostUpTo`); a lone-surrogate case forges an fd-3 `log` frame flooding 1000 `\ud800` escapes under a 4 KiB budget and asserts truncation (the count sits in the window where charging 3 bytes would admit and 6 bytes truncates, proving the surrogate is charged its full escaped width); a stray-sealing case paces 60000 single-byte newline-free `os.write(1, …)` calls under a raised budget with `Buffer.concat` wrapped to measure copy volume, asserting the trickle coalesces to one entry and the cumulative copy stays linear (proving the fragment list seals into blocks past `MAX_PENDING_CHUNKS` rather than re-merging). A closeDeadline-flush case has the leader write a newline-free diagnostic then spawn a `setsid` orphan holding the pipes open, and asserts the diagnostic survives in `logs` (proving the residual is flushed before the deadline destroys the streams). The same-group reap case spawns a SIGTERM-ignoring same-group descendant that releases the pipes and bumps a heartbeat file; the test asserts the heartbeat STOPS after the grace-window SIGKILL — an assertion robust whether the killed descendant is reaped or lingers as a zombie, so it holds where PID 1 does not wait() orphans. A dispose-after-resolve case asserts `dispose()` of a completed run with a same-group survivor returns only after the survivor stops executing (proving the run stays in `live` until its group is reaped), with an `expect(afterDispose).toBeGreaterThan(0)` guard so the frozen-heartbeat assertion cannot pass vacuously when the file was never written. A deadline case busy-blocks the event loop past both timers and asserts the survivor's heartbeat freezes (proving the poll's deadline arm sends SIGKILL itself rather than cancelling the unfired escalation). The cross-loop case runs a binding from a worker thread's own `asyncio.run` loop while the main coroutine yields with `await asyncio.sleep`, asserting the reply round-trips instead of timing out; a companion case abandons a thread's call so its loop closes, then answers it before a later binding — asserting the pump survives the closed-loop `call_soon_threadsafe` (host-gated ordering makes it deterministic, fail-before hangs the later binding to the wall clock). The inherited-soft-limit case runs the interpreter through a `ulimit -S -t` wrapper that sets a CPU soft limit below `cpuSeconds` and asserts the applied `RLIMIT_CPU` soft is the inherited value, not the configured one (CPU rather than address space, since macOS ignores `ulimit -v`); a companion inherits a 1 s CPU soft, has the program trap SIGXCPU and busy-loop past it, and asserts the settlement recheck reports a timeout — proving the recheck uses the effective soft, not the configured `cpuSeconds`. A control-heavy-diagnostic case raises a NUL-flood exception under a small `maxValueBytes` and asserts the serialized frame fits (proving the diagnostic is metered by serialized cost). A non-integer-budget case asserts a fractional `maxLogBytes`/`maxValueBytes` rejects at load. ## Alternatives considered diff --git a/.agents/notes/implemented/bug-fix/2026-07-31-code-runtime-python-settlement-fixes.zh.md b/.agents/notes/implemented/bug-fix/2026-07-31-code-runtime-python-settlement-fixes.zh.md index 169cc34d3e..36a5d5d062 100644 --- a/.agents/notes/implemented/bug-fix/2026-07-31-code-runtime-python-settlement-fixes.zh.md +++ b/.agents/notes/implemented/bug-fix/2026-07-31-code-runtime-python-settlement-fixes.zh.md @@ -54,13 +54,13 @@ Status: implemented ### Stray pipe output is aggregated by line, not by transport chunk -同样在 `src/index.ts` 中，原生 stdout／stderr 字节（C 扩展写入、越过管道缓冲区的 `os.write`）过去每来一个 Node `data` 分片就被推入 `logs` 一条条目。`logs` 条目在下游（Code Mode）会用 `\n` 拼接，因此一次大于单次管道读取、且不含换行符的写入——它以若干个 `data` 分片到达——回读时会在任意传输边界处被插入模型可见的换行符。现在捕获会以一个持续推进的字节计数器累积原始 `Buffer` 分片（与 fd-3 读取器同一形态，出于同一原因：一个字符串 `+=` 累加器会为每个分片重新复制整份残余数据，而每个分片都从索引 0 扫描它则是第二重平方——在一次大的不含换行符的写入上二者都是 O(N²)），在原始的 `0x0a` 字节处切分，并为每个完整行准入一条条目。换行符绝不会出现在一个 UTF-8 多字节序列内部，因此对每个切出的行做解码无需流式解码器即可安全进行。该残余数据仍受账本约束：当它的原始字节计数在看不到换行符的情况下将要越过预算时，会被立即冲刷（准入并截断）；而一旦账本已经截断，缓冲便停止，从而一场不含换行符的洪泛无法为永远无法被准入的输出保留宿主内存。每条条目的计费本身通过 `jsonStringCostUpTo` 按序列化开销计量，它把字符串走到上限即停止——先前的 `Buffer.byteLength(JSON.stringify(text))` 会先分配出整份转义后的形式，因此在一个较大的 `maxLogBytes` 之下，一行接近预算、控制字符密集的内容，仅仅为了度量它就可能瞬时分配超过一 GB。该残余数据会在管道 `end` 时冲刷，也会在 `closeDeadline` 处理器销毁流之前被显式冲刷：一个持有管道不放的 `setsid` 逃逸者会迫使结算在没有 `end` 的情况下走那条路径，因此 leader 在退出前发出的最后一次不含换行符的 `os.write(1, …)` 否则会从 `logs` 中被丢弃。 +同样在 `src/index.ts` 中，原生 stdout／stderr 字节（C 扩展写入、越过管道缓冲区的 `os.write`）过去每来一个 Node `data` 分片就被推入 `logs` 一条条目。`logs` 条目在下游（Code Mode）会用 `\n` 拼接，因此一次大于单次管道读取、且不含换行符的写入——它以若干个 `data` 分片到达——回读时会在任意传输边界处被插入模型可见的换行符。现在捕获会累积原始 `Buffer` 分片（与 fd-3 读取器同一形态，出于同样的原因：一个字符串 `+=` 累加器会为每个分片重新复制整份残余数据，而每个分片都从索引 0 扫描它则是第二重平方——在一次大的不含换行符的写入上二者都是 O(N²)），在原始的 `0x0a` 字节处切分，并为每个完整行准入一条条目。换行符绝不会出现在一个 UTF-8 多字节序列内部，因此对每个切出的行做解码无需流式解码器即可安全进行。三条相互独立的界限使残余数据不至于耗尽宿主内存，每一条都与 fd-3 读取器相对应：分片列表在越过 `MAX_PENDING_CHUNKS` 后会封存（SEAL）为已完成的块，因此一个以单字节 `os.write` 控速的程序无法累积起数以百万计的存活 Buffer 对象（其逐对象开销是任何字节计数都看不到的）；当残余数据持续推进的序列化（SERIALIZED）开销——通过 `serializedBufferCost` 逐字节跟踪，它是被准入行确切开销的一个下界——将要越过预算时，残余数据会被冲刷，因此一场控制字符密集的洪泛会在大约六分之一的原始字节处就冲刷，而不是先累积起满满一个预算份额的原始字节；而一旦账本已经截断，缓冲便停止，从而不会为永远无法被准入的输出累积任何内容。每条条目的计费本身通过 `jsonStringCostUpTo` 按序列化开销计量，它把字符串走到上限即停止——先前的 `Buffer.byteLength(JSON.stringify(text))` 会先分配出整份转义后的形式，因此在一个较大的 `maxLogBytes` 之下，一行接近预算、控制字符密集的内容，仅仅为了度量它就可能瞬时分配超过一 GB。两个开销函数都会给一个孤立（LONE）代理项计满六个转义字节（在 ES2019 良构 `JSON.stringify` 下为 `\uXXXX`），而不是 `Buffer.byteLength` 为其 U+FFFD 渲染所报告的三个字节：否则一个伪造的、以 `\ud800` 转义洪泛的 `log` 帧会被少计一半，并放行大约两倍于预算的内容。该残余数据会在管道 `end` 时冲刷，也会在 `closeDeadline` 处理器销毁流之前被显式冲刷：一个持有管道不放的 `setsid` 逃逸者会迫使结算在没有 `end` 的情况下走那条路径，因此 leader 在退出前发出的最后一次不含换行符的 `os.write(1, …)` 否则会从 `logs` 中被丢弃。 ## Testing -- `tests/boot-write-failure.spec.ts` 对 `spawn` 做 mock，使 fd-3 管道在引导写入时抛出异常（这是真实子进程无法被迫进入的唯一路径），并断言 `run()` resolve 出一个 `worker-exit` 而非 reject。一个同级用例让被 mock 的 `spawn` 同步抛出，并断言 `run()` 仍然 resolve 出一个 `worker-exit`，且不会在 `tmpdir` 中留下任何新的 `dsh-code-runtime-python-*` 目录（相对一份运行前快照做集合差分，因此一个同级 worker 的并发创建或删除不会让该断言变得不稳定）。两者都被隔离在这个 spec 中，因此真实子进程测试套件不受影响。 +- `tests/boot-write-failure.spec.ts` 对 `spawn` 做 mock，使 fd-3 管道在引导写入时抛出异常（这是真实子进程无法被迫进入的唯一路径），并断言 `run()` resolve 出一个 `worker-exit` 而非 reject。一个同级用例让被 mock 的 `spawn` 同步抛出，并断言 `run()` 仍然 resolve 出一个 `worker-exit`，且会移除它的暂存目录——以被 mock 的 `spawn` 在其 argv 中收到的确切引导路径为准，因此一个同级 worker 的并发暂存不会让它变得不稳定。两者都被隔离在这个 spec 中，因此真实子进程测试套件不受影响。 - `tests/residual-detach.spec.ts` 对 `detachResidual` 做单元测试：向前传递的副本与残余数据相等、拥有一个大小与其自身长度一致的底层存储（fixture 保持在 Node 的 Buffer 池阈值之上），并且不与源帧的 `ArrayBuffer` 共享。 -- `tests/runtime.spec.ts`：output-cap 用例断言 `ceiling - envelope` 上界（268435392）及其消息。一个 daemon 线程用例驱动四个线程穿过结算的 flush 发出未结束的写入。一个 native-write 用例在抬高后的 `maxLogBytes` 之下，通过 `os.write` 写入 200 KiB 且不含换行符，断言它回读时恰好是一条日志条目（证明散逸输出是按行聚合的，而不是在管道分片边界处被切开）；一个配套用例写入 `b"one\ntwo\nthree"`，断言得到三条条目（证明真正的换行符仍然起分隔作用）。一个 newline-free-flood 用例在一个 4 KiB 的 `maxLogBytes` 之下写入 2 MiB，断言捕获终止于截断标记且保持在预算之内（证明残余数据受账本约束，而不是被整体缓冲）；一个 NUL-flood 配套用例在同一预算之下写入 4000 个不含换行符的 NUL，断言发生截断（证明残余数据是按序列化开销计费的，约为原始的 6 倍，且在度量时不分配转义后的副本）；一个 short-escape 用例写入一行混合了制表符、引号、反斜杠、一个 `\uXXXX` 控制字符、一个多字节字符和 ASCII 的内容，断言它原样完成往返（覆盖 `jsonStringCostUpTo` 的每一条分支）。一个 closeDeadline-flush 用例让 leader 写入一段不含换行符的诊断，随后 spawn 一个持有管道不放的 `setsid` 孤儿进程，断言该诊断在 `logs` 中存留下来（证明残余数据在截止时间销毁流之前被冲刷）。same-group 回收用例 spawn 一个忽略 SIGTERM 的同进程组后代，它释放管道并递增一个心跳文件；该测试断言在宽限窗口的 SIGKILL 之后心跳停止：无论被杀死的后代是被回收还是作为僵尸进程滞留，这个断言都成立，因此它在 PID 1 不 wait() 孤儿进程的环境下同样成立。一个 dispose-after-resolve 用例断言，对一个已完成、且存在同进程组存活者的运行调用 `dispose()`，只有在该存活者停止执行之后才返回（证明该运行会一直留在 `live` 中，直到它的进程组被回收），并带有一个 `expect(afterDispose).toBeGreaterThan(0)` 守卫，使得当心跳文件从未被写入时，冻结心跳的断言不会被空洞地通过。一个 deadline 用例忙阻塞事件循环越过两个定时器，断言该存活者的心跳冻结（证明轮询的截止时间分支自身发送 SIGKILL，而不是取消尚未触发的升级）。cross-loop 用例在主协程通过 `await asyncio.sleep` 让出时，从一个工作线程自己的 `asyncio.run` 事件循环运行一个绑定，断言该回复完成往返而不是超时；一个配套用例放弃某个线程的调用，使其事件循环关闭，随后在一个后续绑定之前回答它——断言 pump 在关闭事件循环上的 `call_soon_threadsafe` 之后仍然存活（由宿主门控的顺序使其具有确定性，未修复时会把后续绑定拖到墙钟上挂起）。inherited-soft-limit 用例通过一个 `ulimit -S -t` 包装脚本运行解释器，将 CPU 软限制设为低于 `cpuSeconds`，并断言实际应用的 `RLIMIT_CPU` 软限制是继承来的值，而不是配置的值（用 CPU 而非地址空间，因为 macOS 忽略 `ulimit -v`）。一个配套用例继承 1 秒的 CPU 软限制，让程序捕获 SIGXCPU 并忙循环越过它，断言结算复查报告 timeout——证明复查用的是实际生效的软限制，而不是配置的 `cpuSeconds`。一个 control-heavy-diagnostic 用例在一个较小的 `maxValueBytes` 之下抛出一个 NUL 洪泛异常，断言序列化后的帧能放得下（证明该诊断是按序列化开销计量的）。一个 non-integer-budget 用例断言一个小数的 `maxLogBytes`／`maxValueBytes` 在加载期被拒绝。 +- `tests/runtime.spec.ts`：output-cap 用例断言 `ceiling - envelope` 上界（268435392）及其消息。一个 daemon 线程用例驱动四个线程穿过结算的 flush 发出未结束的写入。一个 native-write 用例在抬高后的 `maxLogBytes` 之下，通过 `os.write` 写入 200 KiB 且不含换行符，断言它回读时恰好是一条日志条目（证明散逸输出是按行聚合的，而不是在管道分片边界处被切开）；一个配套用例写入 `b"one\ntwo\nthree"`，断言得到三条条目（证明真正的换行符仍然起分隔作用）。一个 newline-free-flood 用例在一个 4 KiB 的 `maxLogBytes` 之下写入 2 MiB，断言捕获终止于截断标记且保持在预算之内（证明残余数据受账本约束，而不是被整体缓冲）；一个 NUL-flood 配套用例在同一预算之下写入 4000 个不含换行符的 NUL，断言发生截断（证明残余数据是按序列化开销计费的，约为原始的 6 倍，且在度量时不分配转义后的副本）；一个 short-escape 用例写入一行混合了制表符、引号、反斜杠、一个 `\uXXXX` 控制字符、一个多字节字符和 ASCII 的内容，断言它原样完成往返（覆盖 `jsonStringCostUpTo` 的每一条分支）；一个 lone-surrogate 用例在一个 4 KiB 预算之下伪造一个以 1000 个 `\ud800` 转义洪泛的 fd-3 `log` 帧，断言发生截断（该计数正落在计 3 字节会放行、计 6 字节则截断的窗口内，证明该代理项是按其完整转义宽度计费的）；一个 stray-sealing 用例在抬高后的预算之下控速发出 60000 次单字节、不含换行符的 `os.write(1, …)` 调用，并对 `Buffer.concat` 做包装以度量复制量，断言这股细流合并为一条条目、且累积复制量保持线性（证明分片列表在越过 `MAX_PENDING_CHUNKS` 后封存为块，而不是反复重新合并）。一个 closeDeadline-flush 用例让 leader 写入一段不含换行符的诊断，随后 spawn 一个持有管道不放的 `setsid` 孤儿进程，断言该诊断在 `logs` 中存留下来（证明残余数据在截止时间销毁流之前被冲刷）。same-group 回收用例 spawn 一个忽略 SIGTERM 的同进程组后代，它释放管道并递增一个心跳文件；该测试断言在宽限窗口的 SIGKILL 之后心跳停止：无论被杀死的后代是被回收还是作为僵尸进程滞留，这个断言都成立，因此它在 PID 1 不 wait() 孤儿进程的环境下同样成立。一个 dispose-after-resolve 用例断言，对一个已完成、且存在同进程组存活者的运行调用 `dispose()`，只有在该存活者停止执行之后才返回（证明该运行会一直留在 `live` 中，直到它的进程组被回收），并带有一个 `expect(afterDispose).toBeGreaterThan(0)` 守卫，使得当心跳文件从未被写入时，冻结心跳的断言不会被空洞地通过。一个 deadline 用例忙阻塞事件循环越过两个定时器，断言该存活者的心跳冻结（证明轮询的截止时间分支自身发送 SIGKILL，而不是取消尚未触发的升级）。cross-loop 用例在主协程通过 `await asyncio.sleep` 让出时，从一个工作线程自己的 `asyncio.run` 事件循环运行一个绑定，断言该回复完成往返而不是超时；一个配套用例放弃某个线程的调用，使其事件循环关闭，随后在一个后续绑定之前回答它——断言 pump 在关闭事件循环上的 `call_soon_threadsafe` 之后仍然存活（由宿主门控的顺序使其具有确定性，未修复时会把后续绑定拖到墙钟上挂起）。inherited-soft-limit 用例通过一个 `ulimit -S -t` 包装脚本运行解释器，将 CPU 软限制设为低于 `cpuSeconds`，并断言实际应用的 `RLIMIT_CPU` 软限制是继承来的值，而不是配置的值（用 CPU 而非地址空间，因为 macOS 忽略 `ulimit -v`）。一个配套用例继承 1 秒的 CPU 软限制，让程序捕获 SIGXCPU 并忙循环越过它，断言结算复查报告 timeout——证明复查用的是实际生效的软限制，而不是配置的 `cpuSeconds`。一个 control-heavy-diagnostic 用例在一个较小的 `maxValueBytes` 之下抛出一个 NUL 洪泛异常，断言序列化后的帧能放得下（证明该诊断是按序列化开销计量的）。一个 non-integer-budget 用例断言一个小数的 `maxLogBytes`／`maxValueBytes` 在加载期被拒绝。 ## Alternatives considered diff --git a/packages/code-runtime/code-runtime-python/src/index.ts b/packages/code-runtime/code-runtime-python/src/index.ts index b6b9bd8b59..9eddcb01df 100644 --- a/packages/code-runtime/code-runtime-python/src/index.ts +++ b/packages/code-runtime/code-runtime-python/src/index.ts @@ -312,6 +312,23 @@ const TRUNCATION_MARKER = '… [truncated]' */ const TRUNCATION_MARKER_BYTES = Buffer.byteLength(TRUNCATION_MARKER, 'utf8') +/** + * Serialized JSON byte width of one character, given its code point and the + * one-character string. Control characters below 0x20 escape to `\uXXXX` (6) + * except the five with short forms `\b \t \n \f \r` (2); `"` and `\` escape to + * 2; a LONE surrogate escapes to `\uXXXX` (6) under ES2019 well-formed + * `JSON.stringify`; everything else rides at its raw UTF-8 width. + * @param code - the character's code point. + * @param character - the one-character (or one-code-point) string. + * @returns the character's serialized JSON byte width. + */ +function serializedCharCost(code: number, character: string): number { + if (code < 0x20) return code === 0x08 || code === 0x09 || code === 0x0a || code === 0x0c || code === 0x0d ? 2 : 6 + if (code === 0x22 || code === 0x5c) return 2 + if (code >= 0xd800 && code <= 0xdfff) return 6 + return Buffer.byteLength(character, 'utf8') +} + /** * Serialized JSON-string cost of `text` (the two quotes plus each character's * escaped byte width), measured WITHOUT materializing the escaped copy, and @@ -319,7 +336,9 @@ const TRUNCATION_MARKER_BYTES = Buffer.byteLength(TRUNCATION_MARKER, 'utf8') * allocate the whole escaped form first — up to sixfold a control-char-dense * string — so a near-budget line under a large `maxLogBytes` could momentarily * allocate over a gigabyte just to measure it. This walks code point by code - * point and stops at the cap, so the measurement allocates nothing. + * point (a matched surrogate pair yields its combined code point ≥ 0x10000; a + * lone surrogate yields a value in 0xD800–0xDFFF that {@link serializedCharCost} + * charges the full six escaped bytes) and stops at the cap, allocating nothing. * @param text - the candidate string. * @param maxBytes - the largest serialized size the caller can admit. * @returns the exact serialized byte cost, or `undefined` once it exceeds `maxBytes`. @@ -327,22 +346,38 @@ const TRUNCATION_MARKER_BYTES = Buffer.byteLength(TRUNCATION_MARKER, 'utf8') function jsonStringCostUpTo(text: string, maxBytes: number): number | undefined { let bytes = 2 // the enclosing quotes for (const character of text) { - const code = character.codePointAt(0) as number - // Control characters below 0x20 escape to `\uXXXX` (6) except the five with - // short forms `\b \t \n \f \r` (2); `"` and `\` escape to 2; everything else - // rides at its raw UTF-8 width. - if (code < 0x20) { - bytes += code === 0x08 || code === 0x09 || code === 0x0a || code === 0x0c || code === 0x0d ? 2 : 6 - } else if (code === 0x22 || code === 0x5c) { - bytes += 2 - } else { - bytes += Buffer.byteLength(character, 'utf8') - } + bytes += serializedCharCost(character.codePointAt(0) as number, character) if (bytes > maxBytes) return undefined } return bytes } +/** + * Serialized-cost lower bound of raw UTF-8 `buf`, charged per byte without + * decoding: a control byte below 0x20 costs 6 (`\uXXXX`) or 2 (the five + * short-form escapes), `"`/`\` cost 2, and every other byte — including each + * byte of a multibyte sequence — costs at least 1. It is exact for valid UTF-8 + * (a W-byte character serializes to W bytes) and a lower bound for invalid bytes + * (each decodes to U+FFFD at 3 bytes but is charged 1); since each byte costs at + * least its raw 1, the total is always ≥ the raw byte count, so a threshold on + * this cost flushes no later than a raw-byte threshold and strictly earlier for + * control-dense output. Used to bound the stray-capture residual by what the + * ledger can actually admit rather than by raw length, so a NUL flood under a + * large `maxLogBytes` flushes at roughly a sixth of the raw bytes instead of + * accumulating the full budget's worth before `admit` truncates it. + * @param buf - raw bytes from a stdout/stderr pipe chunk. + * @returns the summed per-byte serialized cost. + */ +function serializedBufferCost(buf: Buffer): number { + let cost = 0 + for (const byte of buf) { + if (byte < 0x20) cost += byte === 0x08 || byte === 0x09 || byte === 0x0a || byte === 0x0c || byte === 0x0d ? 2 : 6 + else if (byte === 0x22 || byte === 0x5c) cost += 2 + else cost += 1 + } + return cost +} + /** * Cap a done-frame `error.message` to `maxValueBytes` host-side: a forged done * frame can carry an arbitrarily long message, so truncate by RAW UTF-8 byte @@ -547,8 +582,9 @@ export class PythonCodeRuntime extends CodeRuntime { // arrives as an over-ceiling frame and fails the run as `worker-exit` // instead of the `output-limit` the cap describes — a silent inversion, so // it fails at load. Both budgets are metered in SERIALIZED (JSON-escaped) - // bytes — the host log ledger charges `Buffer.byteLength(JSON.stringify(text))`, - // `checkDoneValue` measures the escaped form, and the producing-side + // bytes — the host log ledger charges the serialized cost via + // `jsonStringCostUpTo`, which walks to the cap without allocating the escaped + // copy, `checkDoneValue` measures the escaped form, and the producing-side // `_cap_message` in the child also caps by serialized cost (which is why a // capped diagnostic still fits its frame) — so a payload admitted under the // cap occupies at most `cap + envelope` bytes on the wire; escaping is @@ -784,27 +820,46 @@ export class PythonCodeRuntime extends CodeRuntime { // child's own `log` frames are already line-granular; stray capture // matches them by splitting on `\n`. // - // Buffered as raw `Buffer` chunks with a running byte counter, exactly - // like the fd-3 reader below and for the same reasons: a string `+=` - // accumulator re-copies the whole residual on every pipe chunk (quadratic - // on a large newline-free write), and scanning it from index 0 each chunk - // is a second quadratic. Appending a chunk is O(1); the split happens only - // when a `\n` actually arrived. A newline never appears inside a UTF-8 - // multibyte sequence (continuation bytes are 0x80–0xBF), so splitting on - // the raw 0x0a byte and decoding each complete line is safe without a - // streaming decoder — a line's bytes are whole by construction. - interface StrayBuffer { chunks: Buffer[]; bytes: number } - const strayOut: StrayBuffer = { chunks: [], bytes: 0 } - const strayErr: StrayBuffer = { chunks: [], bytes: 0 } + // Buffered as raw `Buffer` chunks with a running SERIALIZED-cost counter, + // exactly like the fd-3 reader below and for the same reasons: a string + // `+=` accumulator re-copies the whole residual on every pipe chunk + // (quadratic on a large newline-free write), and scanning it from index 0 + // each chunk is a second quadratic. Appending a chunk is O(1); the split + // happens only when a `\n` actually arrived. A newline never appears inside + // a UTF-8 multibyte sequence (continuation bytes are 0x80–0xBF), so + // splitting on the raw 0x0a byte and decoding each complete line is safe + // without a streaming decoder — a line's bytes are whole by construction. + // + // `chunks` also seals into `blocks` past MAX_PENDING_CHUNKS, mirroring the + // fd-3 reader: without it a program pacing one-byte newline-free + // `os.write`s accumulates one Buffer object per write, and the object plus + // backing-store overhead — which no byte or cost count sees — exhausts the + // host heap far below the budget. Sealing bounds the live object count. + interface StrayBuffer { chunks: Buffer[]; blocks: Buffer[]; cost: number } + const strayOut: StrayBuffer = { chunks: [], blocks: [], cost: 0 } + const strayErr: StrayBuffer = { chunks: [], blocks: [], cost: 0 } const captureStray = (stray: StrayBuffer, chunk: Buffer): void => { // Once the ledger has truncated, stop buffering: admit() is a no-op past // that point, so continuing to accumulate would retain host memory for // output that can never be admitted. if (logsTruncated) return stray.chunks.push(chunk) - stray.bytes += chunk.length + // Track SERIALIZED cost, not raw bytes: a control-char-dense residual + // (a NUL flood) serializes several-fold, so a raw-byte threshold would + // let it grow to the full budget's worth of RAW bytes — up to ~6x what + // the ledger can admit — before flushing. The per-byte cost is a lower + // bound on the admitted line's exact cost, so flushing when it crosses + // the budget bounds the residual by what `admit` can actually keep. + stray.cost += serializedBufferCost(chunk) + // Bound the live fragment count (see the seal rationale above), before + // any concat so an over-count payload is never copied whole first. + if (stray.chunks.length >= MAX_PENDING_CHUNKS) { + stray.blocks.push(Buffer.concat(stray.chunks)) + stray.chunks = [] + } if (chunk.includes(0x0a)) { - let buffered = Buffer.concat(stray.chunks) + let buffered = Buffer.concat(stray.blocks.length > 0 ? [...stray.blocks, ...stray.chunks] : stray.chunks) + stray.blocks = [] let newline: number while ((newline = buffered.indexOf(0x0a)) >= 0) { admit(buffered.subarray(0, newline).toString('utf8')) @@ -813,17 +868,16 @@ export class PythonCodeRuntime extends CodeRuntime { // Carry the residual as a fresh right-sized copy, not the subarray view // (which would pin the whole concat allocation). See detachResidual. stray.chunks = detachResidual(buffered) - stray.bytes = buffered.length + stray.cost = serializedBufferCost(buffered) } // Newline-free residual is bounded by the ledger, not left to grow with // the stream: an `os.write(1, b"A"*N)` flood carrying no newline would // otherwise accumulate N bytes in host memory before `end`. When the - // pending residual would cross the budget, admit it now — admit() charges - // its serialized cost, truncates, and marks the ledger, and the - // truncation short-circuit above stops buffering on the next chunk. The - // raw byte count is a safe lower bound on the serialized cost, so this - // fires no later than the budget is genuinely at risk. - if (stray.bytes + 3 > logBudget) { + // pending residual's serialized cost would cross the budget, flush it now + // — admit() charges the exact serialized cost, truncates, and marks the + // ledger, and the truncation short-circuit above stops buffering on the + // next chunk. `+ 3` covers the two quotes and one separator admit adds. + if (stray.cost + 3 > logBudget) { flushStray(stray) } } @@ -831,14 +885,15 @@ export class PythonCodeRuntime extends CodeRuntime { // above, on the pipe's `end` (normal drain), and — for the setsid-escapee // path where destroy() forces settlement without an `end` — explicitly in // the closeDeadline handler. Idempotent: it clears what it admits, so a - // later flush is a no-op. The `chunks.length` guard is the only emptiness + // later flush is a no-op. The `chunks`/`blocks` guard is the only emptiness // check needed — `data` never emits a zero-length Buffer, so a non-empty - // chunk list always decodes to a non-empty tail. + // fragment list always decodes to a non-empty tail. function flushStray(stray: StrayBuffer): void { - if (stray.chunks.length === 0) return - const tail = Buffer.concat(stray.chunks).toString('utf8') + if (stray.chunks.length === 0 && stray.blocks.length === 0) return + const tail = Buffer.concat([...stray.blocks, ...stray.chunks]).toString('utf8') stray.chunks = [] - stray.bytes = 0 + stray.blocks = [] + stray.cost = 0 admit(tail) } child.stdout.on('data', (chunk: Buffer) => { captureStray(strayOut, chunk) }) diff --git a/packages/code-runtime/code-runtime-python/tests/boot-write-failure.spec.ts b/packages/code-runtime/code-runtime-python/tests/boot-write-failure.spec.ts index 37b3c9c8c4..359bcc33ec 100644 --- a/packages/code-runtime/code-runtime-python/tests/boot-write-failure.spec.ts +++ b/packages/code-runtime/code-runtime-python/tests/boot-write-failure.spec.ts @@ -1,7 +1,7 @@ import { EventEmitter } from 'node:events' -import { readdirSync } from 'node:fs' +import { existsSync } from 'node:fs' +import { dirname } from 'node:path' import { PassThrough } from 'node:stream' -import { tmpdir } from 'node:os' import { afterEach, describe, expect, it, vi } from 'vitest' import { Context } from 'cordis' @@ -74,14 +74,18 @@ describe('PythonCodeRuntime — boot-write failure', () => { // misuse) and stranded the staging directory materializePyScripts had just // written, which only settle() removes. The fix catches it, unlinks the // directory, and resolves the same `worker-exit` class as an async ENOENT. - // Snapshot as a SET, then assert no dir NEW relative to it survives. Strict - // array equality would flake: vitest's forks pool runs runtime.spec.ts in a - // sibling worker that concurrently creates and removes - // `dsh-code-runtime-python-*` dirs, so a concurrent create OR delete in the - // window would fail `toEqual`. The set difference is immune to both — it - // only asserts THIS run left nothing behind. - const before = new Set(readdirSync(tmpdir()).filter(name => name.startsWith('dsh-code-runtime-python-'))) - spawnMock.mockImplementation(() => { throw Object.assign(new Error('EMFILE: too many open files'), { code: 'EMFILE' }) }) + // + // Capture THIS run's exact staging dir from the argv the mocked spawn + // received (`['-I', /bootstrap.py]`) and assert only that path is gone. + // A tmpdir scan — even a set difference against a pre-run snapshot — would + // flake under vitest's forks pool: a sibling worker creating its own + // `dsh-code-runtime-python-*` dir in the window reads as a leak here. Keying + // off our own argv is fully isolated from concurrent staging. + let stagedBootstrap: string | undefined + spawnMock.mockImplementation((_bin: string, args: string[]) => { + stagedBootstrap = args[args.length - 1] + throw Object.assign(new Error('EMFILE: too many open files'), { code: 'EMFILE' }) + }) const ctx = new Context() const fiber = await ctx.plugin(PythonCodeRuntime) const runtime = ctx.codeRuntime as InstanceType @@ -90,8 +94,8 @@ describe('PythonCodeRuntime — boot-write failure', () => { expect(result.error?.kind).toBe('worker-exit') expect(result.error?.message).toContain('python spawn error') - const leaked = readdirSync(tmpdir()).filter(name => name.startsWith('dsh-code-runtime-python-') && !before.has(name)) - expect(leaked).toEqual([]) + expect(stagedBootstrap).toBeDefined() + expect(existsSync(dirname(stagedBootstrap as string))).toBe(false) await fiber.dispose() }) }) diff --git a/packages/code-runtime/code-runtime-python/tests/runtime.spec.ts b/packages/code-runtime/code-runtime-python/tests/runtime.spec.ts index 14326e363f..ac962ce270 100644 --- a/packages/code-runtime/code-runtime-python/tests/runtime.spec.ts +++ b/packages/code-runtime/code-runtime-python/tests/runtime.spec.ts @@ -747,6 +747,33 @@ describe('PythonCodeRuntime — programs and bindings', () => { expect(result.logs.at(-1)).toBe(logTruncationMarker(4096)) }) + it('charges a lone surrogate its full six escaped bytes, not three', async () => { + // A forged `log` frame carrying `\ud800` escapes materializes lone + // surrogates after JSON.parse. `Buffer.byteLength` of U+FFFD is 3, but + // ES2019 well-formed `JSON.stringify` emits `\ud800` at 6 bytes, so charging + // the raw width would admit ~2x the configured budget of serialized bytes + // (the same family as the NUL-flood undercount, at 2x rather than 6x). The + // cost walker charges surrogates the full 6, so a flood truncates at budget. + // Forged on fd 3 because Python stdout will not emit lone surrogates. + const { runtime } = await setup({ maxLogBytes: 4096 }) + const result = await runtime.run({ + program: [ + 'import os', + // 1000 \ud800 escapes: charged at the buggy raw width 1000 * 3 = 3000 + // bytes fits under 4096 (wrongly admitted), but the correct serialized + // width 1000 * 6 = 6000 bytes is over budget — so the ledger must + // truncate. The count sits in the 683..1365 window where the two + // chargings disagree, making the test discriminate. + String.raw`frame = b'{"type":"log","text":"' + b'\\ud800' * 1000 + b'"}\n'`, + 'os.write(3, frame)', + 'return None', + ].join('\n'), + bindings: [], + }) + expect(result.error).toBeUndefined() + expect(result.logs.at(-1)).toBe(logTruncationMarker(4096)) + }) + it('charges the exact serialized cost of short-escape and quote/backslash characters', async () => { // Exercises every branch of jsonStringCostUpTo's per-character cost: a tab // and other C0 controls with short JSON forms (\t etc., 2 bytes), a quote @@ -3074,6 +3101,51 @@ describe('PythonCodeRuntime — hostile peer', () => { expect(copied).toBeLessThan(256 * 1024) }, 40_000) + it('seals trickled stray fragments into blocks without recopying the sealed prefix', async () => { + // The stray-capture buffer has the same object-overhead exposure as the fd-3 + // reader above: each newline-free `data` chunk is its own Buffer, so a + // program pacing single-byte `os.write(1, ...)` accumulates one object per + // write, which the serialized-cost counter cannot see. Past MAX_PENDING_CHUNKS + // the fragments seal into a finished block; re-merging the whole residual at + // each threshold instead would copy the sealed prefix again and again, making + // the cumulative copy volume quadratic. `Buffer.concat` is wrapped to measure + // that volume — both shapes admit the same final log entry, so the copy total + // is the discriminator. maxLogBytes is raised so the trickle is retained, + // not truncated, which is what forces the fragments to accumulate and seal. + const realConcat = Buffer.concat.bind(Buffer) + let copied = 0 + Buffer.concat = (list: readonly Uint8Array[], total?: number): Buffer => { + for (const part of list) copied += part.length + return realConcat(list, total) + } + let result: CodeRunResult + try { + const { runtime } = await setup({ maxLogBytes: 200_000, maxWallMs: 30_000 }) + result = await runtime.run({ + program: [ + 'import os', + 'for _ in range(60000):', + ' os.write(1, b"x")', + ' os.sched_yield()', + 'os.write(1, b"\\n")', + 'return "done"', + ].join('\n'), + bindings: [], + }) + } finally { + Buffer.concat = realConcat + } + expect(result.error).toBeUndefined() + expect(result.value).toBe('done') + // The trickle coalesces into one log line (no interior newlines). Its exact + // length depends on pipe coalescing, but it is one entry and non-empty. + expect(result.logs.length).toBe(1) + expect((result.logs[0] as string).length).toBeGreaterThan(0) + // Sealing keeps each byte copied a bounded number of times; re-merging the + // whole residual per threshold would push the total far past this. + expect(copied).toBeLessThan(2 * 1024 * 1024) + }, 40_000) + it('caps a huge exception diagnostic child-side before it crosses the wire', async () => { // A program can raise with a multi-megabyte message; the child must cap // it at maxValueBytes before formatting/sending, not ship the whole