fix(code-runtime-python): charge illegal UTF-8 by its U+FFFD width on both log paths

The host stray-capture cost function charged illegal UTF-8 bytes (0x80-0xC1, 0xF5-0xFF, and orphaned multibyte leads) the raw 1, but toString('utf8') renders each as U+FFFD (3 serialized bytes). A b"\xff" flood was undercounted threefold, so the residual grew to a full budget's worth of raw bytes before flushing and, near a large maxLogBytes, expanded toward a ~1 GiB peak in the flush's concat plus toString. Replace serializedBufferCost with accrueStrayCost, a cross-chunk UTF-8 walker that charges each byte its decoded serialized width; carry its sequence state on each StrayBuffer. The child _LogStream had the same-family bug: its early-flush trigger compared _pending_chars (character count) against remaining (a serialized-byte budget), so a 30M-NUL newline-free flood stayed under a 50 MB char trigger yet encoded to ~180 MB at settlement, breaching RLIMIT_AS as worker-exit. Track _pending_cost via the _JSON_BYTE_COST table and trigger on it; keep _pending_chars for the char-based slice bounds. Correct the note's surrogate claim (only the string-walking jsonStringCostUpTo charges a lone surrogate six bytes; the byte walker never sees one). Shrink the post-truncation fixture below PIPE_BUF for a deterministic single callback. List the shared stdout/stderr budget as a third honest fail-before exception (cross-pipe arrival timing is nondeterministic). Add illegal-UTF-8, broken-multibyte, and child-log-flood regression tests; sync the zh pair.
2026-09-07 04:10:42 +00:00 · 2026-08-31 14:22:37 +08:00
parent 45814baf26
commit dbff8ffba3
6 changed files with 224 additions and 44 deletions
@@ -2,5 +2,5 @@
 # side as of the last confirmed-consistent state. Both languages carry equal authority;
 # after editing either side, bring the other along and re-record with:
 #   pnpm run verify-translation-pairing --write .agents/notes/implemented/bug-fix/2026-07-31-code-runtime-python-settlement-fixes.md
-2026-07-31-code-runtime-python-settlement-fixes.md: a62b6c67da185081bce7895af242473797722423
-2026-07-31-code-runtime-python-settlement-fixes.zh.md: c04131399c50d53749656111a478dc3c756e32cb
+2026-07-31-code-runtime-python-settlement-fixes.md: b667ec543512ede1c1fe0402122943e6a13f7488
+2026-07-31-code-runtime-python-settlement-fixes.zh.md: f21464d4e96a42552c96e27cb222ac31b6bf2e75
@@ -167,10 +167,22 @@ class _LogStream(io.TextIOBase):
        # ``print("x", end="")`` must not concatenate quadratically.
        self._pending: list[str] = []
        self._pending_chars = 0
+        # Running serialized JSON cost of the pending tail, kept beside the
+        # character count because the early-flush trigger charges against
+        # ``remaining`` (a serialized-byte budget) and a control byte serializes
+        # to up to six bytes.
+        self._pending_cost = 0

    def writable(self) -> bool:  # noqa: D401 -- inherited contract
        return True

+    @staticmethod
+    def _fragment_cost(chunk: str) -> int:
+        # Serialized JSON cost of one pending fragment WITHOUT the enclosing
+        # quotes, so the running total mirrors what LogBuffer charges at
+        # settlement. Mirrors :func:`_json_string_cost` minus its two quotes.
+        return sum(_JSON_BYTE_COST[b] for b in chunk.encode("utf-8", errors="replace"))
+
    def write(self, text: str) -> int:  # noqa: D401 -- inherited contract
        # Serialize the whole read-modify-write against the settlement flush and
        # any other thread's write: model code may spawn daemon threads that keep
@@ -222,6 +234,7 @@ class _LogStream(io.TextIOBase):
                    line = "".join(self._pending)
                    self._pending = []
                    self._pending_chars = 0
+                    self._pending_cost = 0
                    self._logs.push(line)
                pos = newline + 1
            # Scan by offset and STOP once the ledger is exhausted: a single
@@ -251,6 +264,7 @@ class _LogStream(io.TextIOBase):
                    tail = text[pos:]
                    self._pending.append(tail)
                    self._pending_chars = len(tail)
+                    self._pending_cost = self._fragment_cost(tail)
                else:
                    # The ledger ran out with text still unscanned, so that text
                    # IS being dropped and the run must say so. One push is
@@ -270,11 +284,18 @@ class _LogStream(io.TextIOBase):
        else:
            self._pending.append(text)
            self._pending_chars += len(text)
+            self._pending_cost += self._fragment_cost(text)
        # A newline-free flood must hit the budget while running, not at
        # settlement: once the buffered tail alone can no longer fit the
-        # ledger (chars lower-bound the serialized cost), push it through — LogBuffer
-        # truncates, emits the marker once, and swallows everything after.
-        if self._pending_chars > self._logs.remaining:
+        # ledger, push it through — LogBuffer truncates, emits the marker once,
+        # and swallows everything after. Trigger on the SERIALIZED cost, not the
+        # character count: a control byte serializes to up to six bytes, so the
+        # char-count version undercharged control-char floods by up to 6x and a
+        # newline-free flood of ~30M NUL characters (each 1 char but 6 serialized
+        # bytes) stayed under a char-count trigger yet encoded to ~180 MB at
+        # settlement, breaching RLIMIT_AS. Serialized cost >= char count, so this
+        # fires no later than before and strictly earlier for control-dense text.
+        if self._pending_cost > self._logs.remaining:
            self._push_bounded_prefix()
        return len(text)

@@ -307,6 +328,7 @@ class _LogStream(io.TextIOBase):
                break
        self._pending = []
        self._pending_chars = 0
+        self._pending_cost = 0
        self._logs.push("".join(parts))

    def flush(self) -> None:  # noqa: D401 -- inherited contract
@@ -333,6 +355,7 @@ class _LogStream(io.TextIOBase):
                self._logs.push("".join(self._pending))
                self._pending = []
                self._pending_chars = 0
+                self._pending_cost = 0


 # ---------------------------------------------------------------------------
@@ -353,27 +353,80 @@ function jsonStringCostUpTo(text: string, maxBytes: number): number | undefined
 }

 /**
- * Serialized-cost lower bound of raw UTF-8 `buf`, charged per byte without
- * decoding: a control byte below 0x20 costs 6 (`\uXXXX`) or 2 (the five
- * short-form escapes), `"`/`\` cost 2, and every other byte — including each
- * byte of a multibyte sequence — costs at least 1. It is exact for valid UTF-8
- * (a W-byte character serializes to W bytes) and a lower bound for invalid bytes
- * (each decodes to U+FFFD at 3 bytes but is charged 1); since each byte costs at
- * least its raw 1, the total is always ≥ the raw byte count, so a threshold on
- * this cost flushes no later than a raw-byte threshold and strictly earlier for
- * control-dense output. Used to bound the stray-capture residual by what the
- * ledger can actually admit rather than by raw length, so a NUL flood under a
- * large `maxLogBytes` flushes at roughly a sixth of the raw bytes instead of
- * accumulating the full budget's worth before `admit` truncates it.
- * @param buf - raw bytes from a stdout/stderr pipe chunk.
- * @returns the summed per-byte serialized cost.
+ * Cross-chunk UTF-8 state for {@link accrueStrayCost}: `expected` continuation
+ * bytes still needed to finish the in-progress sequence, and its total `width`.
+ * Both zero between sequences. Carried on each {@link StrayBuffer} so a multibyte
+ * character split across pipe `data` chunks is costed as one character, not as
+ * two broken fragments.
 */
-function serializedBufferCost(buf: Buffer): number {
+interface Utf8CostState { expected: number; width: number }
+
+/**
+ * Accrue the serialized JSON cost of raw pipe bytes `buf`, decoding UTF-8
+ * structurally so a byte that `toString('utf8')` would render as U+FFFD is
+ * charged the three bytes that replacement character serializes to — not the one
+ * byte a naive per-byte tally gives it. Without this a `b"\xff" * N` flood (every
+ * byte illegal, so U+FFFD each) counted `cost = raw`, letting the residual grow
+ * to a full budget's worth of RAW bytes before flushing; near a large
+ * `maxLogBytes` that retained ~256 MiB, then `flushStray`'s `Buffer.concat` +
+ * `toString` expanded it to a ~1 GiB peak before `admit`'s exact check could
+ * truncate. A control byte below 0x20 still costs 6 (`\uXXXX`) or 2 (the five
+ * short escapes); `"`/`\` cost 2; ASCII costs 1; a structurally valid multibyte
+ * sequence costs its byte width (2/3/4); any byte outside a valid structure
+ * costs 3. Exotic structurally-valid-but-invalid encodings (overlong forms,
+ * CESU-8 surrogates) are charged their structural width rather than the larger
+ * per-byte U+FFFD cost — a bounded under-count on inputs a flood cannot cheaply
+ * produce, and `admit`'s exact `jsonStringCostUpTo` on the decoded string remains
+ * the truncation backstop. `state` carries the in-progress sequence across
+ * chunks; a sequence left unfinished at the stream's end is decoded by the final
+ * `flushStray` and costed exactly there.
+ * @param buf - raw bytes from a stdout/stderr pipe chunk.
+ * @param state - the pipe's carried UTF-8 sequence state, mutated in place.
+ * @returns the serialized cost accrued by the bytes that resolved in this call.
+ */
+function accrueStrayCost(buf: Buffer, state: Utf8CostState): number {
  let cost = 0
-  for (const byte of buf) {
-    if (byte < 0x20) cost += byte === 0x08 || byte === 0x09 || byte === 0x0a || byte === 0x0c || byte === 0x0d ? 2 : 6
-    else if (byte === 0x22 || byte === 0x5c) cost += 2
-    else cost += 1
+  let index = 0
+  while (index < buf.length) {
+    const byte = buf[index] as number
+    if (state.expected > 0) {
+      if (byte >= 0x80 && byte <= 0xbf) {
+        state.expected -= 1
+        if (state.expected === 0) {
+          cost += state.width
+          state.width = 0
+        }
+        index += 1
+        continue
+      }
+      // The sequence broke before completing: every byte consumed so far
+      // (`width - expected`) is an invalid byte that decodes to U+FFFD (3). Then
+      // reprocess this byte as a fresh start (no index advance).
+      cost += (state.width - state.expected) * 3
+      state.expected = 0
+      state.width = 0
+      continue
+    }
+    if (byte < 0x20) {
+      cost += byte === 0x08 || byte === 0x09 || byte === 0x0a || byte === 0x0c || byte === 0x0d ? 2 : 6
+    } else if (byte === 0x22 || byte === 0x5c) {
+      cost += 2
+    } else if (byte < 0x80) {
+      cost += 1
+    } else if (byte >= 0xc2 && byte <= 0xdf) {
+      state.expected = 1
+      state.width = 2
+    } else if (byte >= 0xe0 && byte <= 0xef) {
+      state.expected = 2
+      state.width = 3
+    } else if (byte >= 0xf0 && byte <= 0xf4) {
+      state.expected = 3
+      state.width = 4
+    } else {
+      // 0x80–0xc1 and 0xf5–0xff never begin a valid sequence: U+FFFD (3).
+      cost += 3
+    }
+    index += 1
  }
  return cost
 }
@@ -838,9 +891,9 @@ export class PythonCodeRuntime extends CodeRuntime {
      // `os.write`s accumulates one Buffer object per write, and the object plus
      // backing-store overhead — which no byte or cost count sees — exhausts the
      // host heap far below the budget. Sealing bounds the live object count.
-      interface StrayBuffer { chunks: Buffer[]; blocks: Buffer[]; cost: number }
-      const strayOut: StrayBuffer = { chunks: [], blocks: [], cost: 0 }
-      const strayErr: StrayBuffer = { chunks: [], blocks: [], cost: 0 }
+      interface StrayBuffer { chunks: Buffer[]; blocks: Buffer[]; cost: number; utf8: Utf8CostState }
+      const strayOut: StrayBuffer = { chunks: [], blocks: [], cost: 0, utf8: { expected: 0, width: 0 } }
+      const strayErr: StrayBuffer = { chunks: [], blocks: [], cost: 0, utf8: { expected: 0, width: 0 } }
      const captureStray = (stray: StrayBuffer, chunk: Buffer): void => {
        // Once the ledger has truncated, stop buffering: admit() is a no-op past
        // that point, so continuing to accumulate would retain host memory for
@@ -848,12 +901,12 @@ export class PythonCodeRuntime extends CodeRuntime {
        if (logsTruncated) return
        stray.chunks.push(chunk)
        // Track SERIALIZED cost, not raw bytes: a control-char-dense residual
-        // (a NUL flood) serializes several-fold, so a raw-byte threshold would
-        // let it grow to the full budget's worth of RAW bytes — up to ~6x what
-        // the ledger can admit — before flushing. The per-byte cost is a lower
-        // bound on the admitted line's exact cost, so flushing when it crosses
-        // the budget bounds the residual by what `admit` can actually keep.
-        stray.cost += serializedBufferCost(chunk)
+        // (a NUL or illegal-UTF-8 flood) serializes several-fold, so a raw-byte
+        // threshold would let it grow to the full budget's worth of RAW bytes
+        // before flushing. `accrueStrayCost` decodes UTF-8 structurally across
+        // chunks (via `stray.utf8`) so a byte that renders as U+FFFD is charged
+        // its three serialized bytes, not one.
+        stray.cost += accrueStrayCost(chunk, stray.utf8)
        // Bound the live fragment count (see the seal rationale above), before
        // any concat so an over-count payload is never copied whole first.
        if (stray.chunks.length >= MAX_PENDING_CHUNKS) {
@@ -870,8 +923,12 @@ export class PythonCodeRuntime extends CodeRuntime {
          }
          // Carry the residual as a fresh right-sized copy, not the subarray view
          // (which would pin the whole concat allocation). See detachResidual.
+          // The residual begins at a character boundary (a newline is never
+          // inside a multibyte sequence), so its cost and UTF-8 state recompute
+          // cleanly from a fresh walk.
          stray.chunks = detachResidual(buffered)
-          stray.cost = serializedBufferCost(buffered)
+          stray.utf8 = { expected: 0, width: 0 }
+          stray.cost = accrueStrayCost(buffered, stray.utf8)
        }
        // Newline-free residual is bounded by the ledger, not left to grow with
        // the stream: an `os.write(1, b"A"*N)` flood carrying no newline would
@@ -904,6 +961,7 @@ export class PythonCodeRuntime extends CodeRuntime {
        stray.chunks = []
        stray.blocks = []
        stray.cost = 0
+        stray.utf8 = { expected: 0, width: 0 }
        admit(tail)
      }
      child.stdout.on('data', (chunk: Buffer) => { captureStray(strayOut, chunk) })
@@ -747,6 +747,70 @@ describe('PythonCodeRuntime — programs and bindings', () => {
    expect(result.logs.at(-1)).toBe(logTruncationMarker(4096))
  })

+  it('bounds a newline-free NUL flood through sys.stdout by serialized cost, not char count', async () => {
+    // `_LogStream` (the child's sys.stdout wrapper) buffers newline-free writes
+    // and early-flushes once the pending tail can no longer fit the ledger.
+    // Charging that trigger by CHARACTER count undercharged a control-char flood
+    // by up to 6x: 30M NUL chars stay under a 50 MB char-count trigger yet
+    // serialize to ~180 MB, which the settlement flush then allocated at once —
+    // breaching a 64 MB RLIMIT_AS and surfacing as worker-exit instead of the
+    // truncation marker. Driving the flood through sys.stdout.write (not
+    // os.write, which bypasses the wrapper into host stray capture) exercises the
+    // in-child stream. On Linux CI the pre-fix trigger dies on RLIMIT_AS; the
+    // serialized-cost trigger flushes while running, so the run completes and
+    // ends at the marker. (RLIMIT_AS is skipped on Darwin — bootstrap.py — so the
+    // worker-exit repro is Linux-only; locally this asserts the happy path.)
+    const { runtime } = await setup({ maxLogBytes: 50_000_000, addressSpaceMb: 64, maxWallMs: 20_000 })
+    const result = await runtime.run({
+      program: ['import sys', 'sys.stdout.write("\\x00" * 30_000_000)', 'return None'].join('\n'),
+      bindings: [],
+    })
+    expect(result.error).toBeUndefined()
+    expect(result.logs.at(-1)).toBe(logTruncationMarker(50_000_000))
+  })
+
+  it('bounds an illegal-UTF-8 native residual by its U+FFFD-decoded cost', async () => {
+    // Every 0xFF byte is illegal in any UTF-8 sequence, so `toString('utf8')`
+    // renders each as U+FFFD (3 serialized bytes). `accrueStrayCost` must charge
+    // that 3, not the raw 1: otherwise the newline-free residual grows to a full
+    // budget's worth of RAW bytes before flushing — a ~3x undercount that near a
+    // large maxLogBytes retains hundreds of MiB then expands toward a ~1 GiB peak
+    // in flushStray's concat + toString. Paced single-byte writes (each its own
+    // `data` chunk, like the sealing case) expose the sub-chunk accrual: charged
+    // at 3 the residual crosses a 3072-byte budget after ~1024 bytes and flushes;
+    // charged at 1 it would need ~3072 bytes, so the peak residual triples. The
+    // largest merged buffer is the discriminator.
+    const realConcat = Buffer.concat.bind(Buffer)
+    let maxConcat = 0
+    Buffer.concat = (list: readonly Uint8Array[], total?: number): Buffer<ArrayBuffer> => {
+      const merged = realConcat(list, total)
+      if (merged.length > maxConcat) maxConcat = merged.length
+      return merged
+    }
+    let result: CodeRunResult
+    try {
+      const { runtime } = await setup({ maxLogBytes: 3072, maxWallMs: 30_000 })
+      result = await runtime.run({
+        program: [
+          'import os',
+          'for _ in range(6000):',
+          '    os.write(1, b"\\xff")',
+          '    os.sched_yield()',
+          'return None',
+        ].join('\n'),
+        bindings: [],
+      })
+    } finally {
+      Buffer.concat = realConcat
+    }
+    expect(result.error).toBeUndefined()
+    expect(result.logs.at(-1)).toBe(logTruncationMarker(3072))
+    // Charged at 3, the residual flushes around 1024 raw bytes; the largest
+    // merged buffer stays well under 2048. A raw-byte undercount would let it
+    // reach ~3072 before flushing, so 2048 discriminates.
+    expect(maxConcat).toBeLessThan(2048)
+  })
+
  it('charges a lone surrogate its full six escaped bytes, not three', async () => {
    // A forged `log` frame carrying `\ud800` escapes materializes lone
    // surrogates after JSON.parse. `Buffer.byteLength` of U+FFFD is 3, but
@@ -779,10 +843,14 @@ describe('PythonCodeRuntime — programs and bindings', () => {
    // exhausts maxLogBytes: the first line's admit truncates and marks the
    // ledger, and the second line's admit — reached in the same `data` callback
    // — must be the post-truncation no-op. Proves that branch is exercised, so
-    // it carries no v8-ignore.
+    // it carries no v8-ignore. Kept to 109 bytes (< the smallest PIPE_BUF, 512 on
+    // macOS) so the whole payload lands in ONE atomic write and one `data`
+    // callback — the two newlines cannot split across callbacks and leave the
+    // branch un-exercised, which would be a hard-to-attribute per-file coverage
+    // flake. 103 payload bytes still exceed the 64-byte budget, so it truncates.
    const { runtime } = await setup({ maxLogBytes: 64 })
    const result = await runtime.run({
-      program: ['import os', 'os.write(1, b"A" * 5000 + b"\\nSECOND\\n")', 'return None'].join('\n'),
+      program: ['import os', 'os.write(1, b"A" * 100 + b"\\nSECOND\\n")', 'return None'].join('\n'),
      bindings: [],
    })
    expect(result.error).toBeUndefined()
@@ -790,6 +858,29 @@ describe('PythonCodeRuntime — programs and bindings', () => {
    expect(result.logs.join('\n')).not.toContain('SECOND')
  })

+  it('charges a broken multibyte sequence its U+FFFD bytes, split across pipe chunks', async () => {
+    // A 3-byte lead (0xE4) whose continuation never arrives — the next byte is a
+    // fresh ASCII 'A' — must be costed as U+FFFD (3) for the orphaned lead, not
+    // folded into a phantom character. Driven byte-by-byte so the lead and the
+    // breaking byte land in separate `data` chunks, exercising accrueStrayCost's
+    // cross-chunk broken-sequence branch. The run completes and the bytes are
+    // captured (rendered U+FFFD by toString), proving the walk resynchronizes.
+    const { runtime } = await setup({ maxLogBytes: 1024 })
+    const result = await runtime.run({
+      program: [
+        'import os',
+        'os.write(1, b"\\xe4")',
+        'os.sched_yield()',
+        'os.write(1, b"A\\n")',
+        'return None',
+      ].join('\n'),
+      bindings: [],
+    })
+    expect(result.error).toBeUndefined()
+    expect(result.logs.join('')).toContain('A')
+    expect(result.logs.join('')).toContain('�')
+  })
+
  it('charges the exact serialized cost of short-escape and quote/backslash characters', async () => {
    // Exercises every branch of jsonStringCostUpTo's per-character cost: a tab
    // and other C0 controls with short JSON forms (\t etc., 2 bytes), a quote