mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-09-11 04:00:38 +00:00
fix(code-runtime-python): charge structurally-valid-but-illegal UTF-8 and newline-path logs by decoded cost
accrueStrayCost accepted any 0x80-0xBF continuation, so a CESU-8 surrogate
(ED A0 80) or overlong (E0 80 80) — structurally well-formed but illegal, and
as cheap to flood as 0xFF — was charged its structural width 3 while
toString('utf8') renders each byte as its own U+FFFD (cost 9). Validate each
lead's first-continuation range (WHATWG E0/ED/F0/F4 bounds) and charge 3 per
byte of any sequence outside it, folding a broken prefix to one U+FFFD.
The child _LogStream newline path had the same char-vs-serialized gap the
newline-free trigger had: its per-line fit checks (first reconstructed line and
each subsequent line) compared character count against the serialized-byte
budget, so a control-char line passed and _logs.push encoded it whole, breaching
RLIMIT_AS. Route every check through _fragment_cost_upto, which sums per-char
costs from _json_char_cost over a start/end sub-range without slicing or
encoding and stops at the budget.
Decline arrival-order stray flushing: the two pipes' data events interleave
nondeterministically and logs carries no cross-pipe ordering guarantee, so a
fixed drain order is as valid as any and an arrival-tick branch could not be
covered without a flaky test.
Add CESU-8/overlong, newline-path-flood, and all-lead-class reassembly
regression tests; fix the note's now-inaccurate CESU/illegal-byte claims and a
fixture byte-count comment; sync the zh pair.
This commit is contained in:
Chinesezjc
@@ -178,26 +178,31 @@ class _LogStream(io.TextIOBase):
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _fragment_cost_upto(chunk: str, limit: int) -> int:
|
||||
# Serialized JSON cost of one fragment's characters (no enclosing quotes),
|
||||
# scanning the str directly and STOPPING once the running total passes
|
||||
# ``limit`` so the walk is bounded by a budget's worth of characters
|
||||
# however large the write is. A control character serializes to up to six
|
||||
# bytes, so a plain character count undercharges a control-char flood by
|
||||
# up to 6x: 30M NUL characters stay under a 50 MB character budget yet
|
||||
# serialize to ~180 MB, which the settlement flush would then allocate at
|
||||
# once and breach RLIMIT_AS. Measuring the true serialized cost fixes that,
|
||||
# but ``.encode`` to measure it would itself be the copy the
|
||||
def _fragment_cost_upto(chunk: str, limit: int, start: int = 0, end: "int | None" = None) -> int:
|
||||
# Serialized JSON cost of ``chunk[start:end]``'s characters (no enclosing
|
||||
# quotes), indexing the str directly and STOPPING once the running total
|
||||
# passes ``limit`` so the walk is bounded by a budget's worth of
|
||||
# characters however large the write is. A control character serializes to
|
||||
# up to six bytes, so a plain character count undercharges a control-char
|
||||
# flood by up to 6x: 30M NUL characters stay under a 50 MB character budget
|
||||
# yet serialize to ~180 MB, which the settlement flush would then allocate
|
||||
# at once and breach RLIMIT_AS. Measuring the true serialized cost fixes
|
||||
# that, but ``.encode`` to measure it would itself be the copy the
|
||||
# ``_push_bounded_prefix`` path exists to avoid (a single 340 MiB write
|
||||
# under a tight addressSpaceMb dies on that encode), and re-scanning the
|
||||
# whole pending list per write would be quadratic under a daemon-thread
|
||||
# flood — so the caller accumulates this per-fragment result once and the
|
||||
# ``limit`` cap keeps each scan bounded.
|
||||
# ``limit`` cap keeps each scan bounded. ``start``/``end`` weigh a
|
||||
# sub-range without slicing it (the slice on a 340 MiB write would be that
|
||||
# same copy); CPython ``str`` indexing is O(1) per character.
|
||||
cost = 0
|
||||
for char in chunk:
|
||||
cost += _json_char_cost(ord(char))
|
||||
stop = len(chunk) if end is None else end
|
||||
index = start
|
||||
while index < stop:
|
||||
cost += _json_char_cost(ord(chunk[index]))
|
||||
if cost > limit:
|
||||
return cost
|
||||
index += 1
|
||||
return cost
|
||||
|
||||
def write(self, text: str) -> int: # noqa: D401 -- inherited contract
|
||||
@@ -236,7 +241,16 @@ class _LogStream(io.TextIOBase):
|
||||
pos = 0
|
||||
if self._pending:
|
||||
newline = text.index("\n")
|
||||
if self._pending_chars + newline + 3 > self._logs.remaining:
|
||||
# Weigh the reconstructed first line by SERIALIZED cost, not
|
||||
# character count: `_pending_cost` already holds the buffered
|
||||
# chunks' cost, and the first line's cost is scanned up to the
|
||||
# newline without slicing `text` (the slice on a 340 MiB write
|
||||
# would be the copy this path avoids). A character-count check
|
||||
# undercharged a control-char line — 30M NUL characters plus a
|
||||
# newline pass `chars + 3 > remaining` under a 50 MB budget, then
|
||||
# `_logs.push` would encode the 30M-char join and breach RLIMIT_AS.
|
||||
first_line_cost = self._fragment_cost_upto(text, self._logs.remaining, end=newline)
|
||||
if self._pending_cost + first_line_cost + 2 > self._logs.remaining:
|
||||
# The reconstructed first line cannot fit the ledger, so
|
||||
# LogBuffer would reject it whole: copy only the prefix that
|
||||
# fails its cheap bound and drop the chunks. The slice is
|
||||
@@ -264,14 +278,17 @@ class _LogStream(io.TextIOBase):
|
||||
newline = text.find("\n", pos)
|
||||
if newline < 0:
|
||||
break
|
||||
# Bound the SLICE the same way LogBuffer bounds the encode: a
|
||||
# first line far above the ledger would be copied whole before
|
||||
# push could reject it, and that copy is the allocation an
|
||||
# over-budget write cannot afford. Copy only a budget-sized
|
||||
# prefix, which push still rejects on its own cheap bound (the
|
||||
# prefix is longer than `remaining`), so the marker is emitted
|
||||
# and the oversized line is never materialized.
|
||||
if newline - pos + 3 > self._logs.remaining:
|
||||
# Bound the SLICE by SERIALIZED cost, not character count: a line
|
||||
# whose escaped form exceeds the ledger would be copied whole
|
||||
# before push could reject it, and a control-char-dense line
|
||||
# (30M NUL characters plus a newline) passes a `chars + 3 >
|
||||
# remaining` check under a large budget yet encodes to ~6x that,
|
||||
# so `_logs.push` would allocate the encode and breach RLIMIT_AS.
|
||||
# `_fragment_cost_upto` scans up to the newline without slicing and
|
||||
# stops at `remaining`, so an over-budget line takes the bounded
|
||||
# prefix path; push still rejects that prefix on its own cheap
|
||||
# bound, emits the marker, and never materializes the full line.
|
||||
if self._fragment_cost_upto(text, self._logs.remaining, start=pos, end=newline) + 2 > self._logs.remaining:
|
||||
self._logs.push(text[pos:pos + self._logs.remaining + 4])
|
||||
break
|
||||
self._logs.push(text[pos:newline])
|
||||
|
||||
@@ -354,32 +354,33 @@ function jsonStringCostUpTo(text: string, maxBytes: number): number | undefined
|
||||
|
||||
/**
|
||||
* Cross-chunk UTF-8 state for {@link accrueStrayCost}: `expected` continuation
|
||||
* bytes still needed to finish the in-progress sequence, and its total `width`.
|
||||
* Both zero between sequences. Carried on each {@link StrayBuffer} so a multibyte
|
||||
* character split across pipe `data` chunks is costed as one character, not as
|
||||
* two broken fragments.
|
||||
* bytes still needed to finish the in-progress sequence, its total `width`, and
|
||||
* `lowerFirst`/`upperFirst`, the valid range for the NEXT continuation byte
|
||||
* (only the first continuation of a 3- or 4-byte lead is range-restricted; once
|
||||
* consumed, later continuations accept the full 0x80–0xBF). All zero between
|
||||
* sequences. Carried on each {@link StrayBuffer} so a multibyte character split
|
||||
* across pipe `data` chunks is costed as one character.
|
||||
*/
|
||||
interface Utf8CostState { expected: number; width: number }
|
||||
interface Utf8CostState { expected: number; width: number; lowerFirst: number; upperFirst: number }
|
||||
|
||||
/**
|
||||
* Accrue the serialized JSON cost of raw pipe bytes `buf`, decoding UTF-8
|
||||
* structurally so a byte that `toString('utf8')` would render as U+FFFD is
|
||||
* charged the three bytes that replacement character serializes to — not the one
|
||||
* byte a naive per-byte tally gives it. Without this a `b"\xff" * N` flood (every
|
||||
* byte illegal, so U+FFFD each) counted `cost = raw`, letting the residual grow
|
||||
* to a full budget's worth of RAW bytes before flushing; near a large
|
||||
* `maxLogBytes` that retained ~256 MiB, then `flushStray`'s `Buffer.concat` +
|
||||
* `toString` expanded it to a ~1 GiB peak before `admit`'s exact check could
|
||||
* truncate. A control byte below 0x20 still costs 6 (`\uXXXX`) or 2 (the five
|
||||
* short escapes); `"`/`\` cost 2; ASCII costs 1; a structurally valid multibyte
|
||||
* sequence costs its byte width (2/3/4); any byte outside a valid structure
|
||||
* costs 3. Exotic structurally-valid-but-invalid encodings (overlong forms,
|
||||
* CESU-8 surrogates) are charged their structural width rather than the larger
|
||||
* per-byte U+FFFD cost — a bounded under-count on inputs a flood cannot cheaply
|
||||
* produce, and `admit`'s exact `jsonStringCostUpTo` on the decoded string remains
|
||||
* the truncation backstop. `state` carries the in-progress sequence across
|
||||
* chunks; a sequence left unfinished at the stream's end is decoded by the final
|
||||
* `flushStray` and costed exactly there.
|
||||
* Accrue the serialized JSON cost of raw pipe bytes `buf`, decoding UTF-8 the way
|
||||
* `toString('utf8')` (WHATWG) would so a byte that renders as U+FFFD is charged
|
||||
* the three bytes that replacement character serializes to. A naive tally that
|
||||
* charged every byte 1 let a `b"\xff"` flood (every byte illegal → U+FFFD each)
|
||||
* grow the residual to a full budget's worth of raw bytes before flushing; near
|
||||
* a large `maxLogBytes` that retained ~256 MiB, then `flushStray`'s
|
||||
* `Buffer.concat` + `toString` expanded it to a ~1 GiB peak. Charging only the
|
||||
* structural width would leave the same gap for structurally-well-formed but
|
||||
* ILLEGAL sequences a flood produces just as cheaply — a CESU-8 surrogate
|
||||
* (`ED A0 80`) or an overlong (`E0 80 80`) decodes to THREE U+FFFD (cost 9), not
|
||||
* one width-3 character, so this validates each lead's first continuation range
|
||||
* (WHATWG: `E0`→A0-BF, `ED`→80-9F, `F0`→90-BF, `F4`→80-8F, others 80-BF) and
|
||||
* charges 3 per byte of any sequence that breaks. A control byte below 0x20
|
||||
* costs 6 (`\uXXXX`) or 2 (five short escapes); `"`/`\` cost 2; ASCII costs 1; a
|
||||
* fully valid multibyte sequence costs its byte width (2/3/4). `state` carries
|
||||
* the in-progress sequence across chunks; an unfinished tail at stream end is
|
||||
* decoded by the final `flushStray` and costed exactly there.
|
||||
* @param buf - raw bytes from a stdout/stderr pipe chunk.
|
||||
* @param state - the pipe's carried UTF-8 sequence state, mutated in place.
|
||||
* @returns the serialized cost accrued by the bytes that resolved in this call.
|
||||
@@ -390,7 +391,12 @@ function accrueStrayCost(buf: Buffer, state: Utf8CostState): number {
|
||||
while (index < buf.length) {
|
||||
const byte = buf[index] as number
|
||||
if (state.expected > 0) {
|
||||
if (byte >= 0x80 && byte <= 0xbf) {
|
||||
// The valid range for THIS continuation: the lead-specific range applies
|
||||
// to the first continuation only, then reverts to the full 0x80–0xBF.
|
||||
const consumed = state.width - state.expected
|
||||
const lower = consumed === 1 ? state.lowerFirst : 0x80
|
||||
const upper = consumed === 1 ? state.upperFirst : 0xbf
|
||||
if (byte >= lower && byte <= upper) {
|
||||
state.expected -= 1
|
||||
if (state.expected === 0) {
|
||||
cost += state.width
|
||||
@@ -399,10 +405,11 @@ function accrueStrayCost(buf: Buffer, state: Utf8CostState): number {
|
||||
index += 1
|
||||
continue
|
||||
}
|
||||
// The sequence broke before completing: every byte consumed so far
|
||||
// (`width - expected`) is an invalid byte that decodes to U+FFFD (3). Then
|
||||
// reprocess this byte as a fresh start (no index advance).
|
||||
cost += (state.width - state.expected) * 3
|
||||
// The sequence broke. WHATWG's maximal-subpart rule folds the bytes
|
||||
// consumed so far into ONE U+FFFD (cost 3), then reprocesses this byte as
|
||||
// a fresh start (no index advance). Charging per consumed byte would
|
||||
// over-count, which is memory-safe but wrong; folding to one is exact.
|
||||
cost += 3
|
||||
state.expected = 0
|
||||
state.width = 0
|
||||
continue
|
||||
@@ -416,12 +423,20 @@ function accrueStrayCost(buf: Buffer, state: Utf8CostState): number {
|
||||
} else if (byte >= 0xc2 && byte <= 0xdf) {
|
||||
state.expected = 1
|
||||
state.width = 2
|
||||
state.lowerFirst = 0x80
|
||||
state.upperFirst = 0xbf
|
||||
} else if (byte >= 0xe0 && byte <= 0xef) {
|
||||
state.expected = 2
|
||||
state.width = 3
|
||||
// Exclude the overlong (E0 80-9F) and CESU-8 surrogate (ED A0-BF) ranges.
|
||||
state.lowerFirst = byte === 0xe0 ? 0xa0 : 0x80
|
||||
state.upperFirst = byte === 0xed ? 0x9f : 0xbf
|
||||
} else if (byte >= 0xf0 && byte <= 0xf4) {
|
||||
state.expected = 3
|
||||
state.width = 4
|
||||
// Exclude the overlong (F0 80-8F) and out-of-range (F4 90-BF) leads.
|
||||
state.lowerFirst = byte === 0xf0 ? 0x90 : 0x80
|
||||
state.upperFirst = byte === 0xf4 ? 0x8f : 0xbf
|
||||
} else {
|
||||
// 0x80–0xc1 and 0xf5–0xff never begin a valid sequence: U+FFFD (3).
|
||||
cost += 3
|
||||
@@ -892,8 +907,8 @@ export class PythonCodeRuntime extends CodeRuntime {
|
||||
// backing-store overhead — which no byte or cost count sees — exhausts the
|
||||
// host heap far below the budget. Sealing bounds the live object count.
|
||||
interface StrayBuffer { chunks: Buffer[]; blocks: Buffer[]; cost: number; utf8: Utf8CostState }
|
||||
const strayOut: StrayBuffer = { chunks: [], blocks: [], cost: 0, utf8: { expected: 0, width: 0 } }
|
||||
const strayErr: StrayBuffer = { chunks: [], blocks: [], cost: 0, utf8: { expected: 0, width: 0 } }
|
||||
const strayOut: StrayBuffer = { chunks: [], blocks: [], cost: 0, utf8: { expected: 0, width: 0, lowerFirst: 0, upperFirst: 0 } }
|
||||
const strayErr: StrayBuffer = { chunks: [], blocks: [], cost: 0, utf8: { expected: 0, width: 0, lowerFirst: 0, upperFirst: 0 } }
|
||||
const captureStray = (stray: StrayBuffer, chunk: Buffer): void => {
|
||||
// Once the ledger has truncated, stop buffering: admit() is a no-op past
|
||||
// that point, so continuing to accumulate would retain host memory for
|
||||
@@ -927,7 +942,7 @@ export class PythonCodeRuntime extends CodeRuntime {
|
||||
// inside a multibyte sequence), so its cost and UTF-8 state recompute
|
||||
// cleanly from a fresh walk.
|
||||
stray.chunks = detachResidual(buffered)
|
||||
stray.utf8 = { expected: 0, width: 0 }
|
||||
stray.utf8 = { expected: 0, width: 0, lowerFirst: 0, upperFirst: 0 }
|
||||
stray.cost = accrueStrayCost(buffered, stray.utf8)
|
||||
}
|
||||
// Newline-free residual is bounded by the ledger, not left to grow with
|
||||
@@ -940,7 +955,11 @@ export class PythonCodeRuntime extends CodeRuntime {
|
||||
// When the sum would cross the budget, flush both now. admit() charges
|
||||
// the exact serialized cost, truncates, and marks the ledger, and the
|
||||
// truncation short-circuit above stops buffering on the next chunk.
|
||||
// `+ 3` covers the two quotes and one separator admit adds.
|
||||
// `+ 3` covers the two quotes and one separator admit adds. The two
|
||||
// pipes are independent OS streams whose `data` events already interleave
|
||||
// nondeterministically with each other and with the child's own fd-3
|
||||
// `log` frames, so `logs` carries no cross-pipe ordering guarantee to
|
||||
// preserve here; a fixed drain order is as valid as any.
|
||||
if (strayOut.cost + strayErr.cost + 3 > logBudget) {
|
||||
flushStray(strayOut)
|
||||
flushStray(strayErr)
|
||||
@@ -961,7 +980,7 @@ export class PythonCodeRuntime extends CodeRuntime {
|
||||
stray.chunks = []
|
||||
stray.blocks = []
|
||||
stray.cost = 0
|
||||
stray.utf8 = { expected: 0, width: 0 }
|
||||
stray.utf8 = { expected: 0, width: 0, lowerFirst: 0, upperFirst: 0 }
|
||||
admit(tail)
|
||||
}
|
||||
child.stdout.on('data', (chunk: Buffer) => { captureStray(strayOut, chunk) })
|
||||
|
||||
@@ -778,6 +778,32 @@ describe('PythonCodeRuntime — programs and bindings', () => {
|
||||
expect(result.logs.at(-1)).toBe(logTruncationMarker(20_000_000))
|
||||
})
|
||||
|
||||
it('bounds a NEWLINE-terminated NUL flood through sys.stdout by serialized cost', async () => {
|
||||
// The newline path of `_LogStream.write` scans and pushes each completed
|
||||
// LINE. Its per-line fit check charged CHARACTER count, so a control-char
|
||||
// line (a chunk of NULs ending in a newline) passed `chars + 3 > remaining`
|
||||
// under a large budget yet `_logs.push` then encoded the whole line at
|
||||
// settlement — the same RLIMIT_AS breach as the newline-free path, on a
|
||||
// different branch. The check now weighs the line by serialized cost via
|
||||
// `_fragment_cost_upto` (scanning to the newline without slicing), so an
|
||||
// over-budget line takes the bounded-prefix path and the run truncates. Each
|
||||
// 1 MiB NUL chunk is newline-terminated so it exercises the line branch;
|
||||
// driven through sys.stdout.write, Linux-only RLIMIT_AS repro, macOS happy path.
|
||||
const { runtime } = await setup({ maxLogBytes: 20_000_000, addressSpaceMb: 512, maxWallMs: 30_000 })
|
||||
const result = await runtime.run({
|
||||
program: [
|
||||
'import sys',
|
||||
'chunk = "\\x00" * (1024 * 1024) + "\\n"',
|
||||
'for _ in range(200):',
|
||||
' sys.stdout.write(chunk)',
|
||||
'return None',
|
||||
].join('\n'),
|
||||
bindings: [],
|
||||
})
|
||||
expect(result.error).toBeUndefined()
|
||||
expect(result.logs.at(-1)).toBe(logTruncationMarker(20_000_000))
|
||||
})
|
||||
|
||||
it('bounds an illegal-UTF-8 native residual by its U+FFFD-decoded cost', async () => {
|
||||
// Every 0xFF byte is illegal in any UTF-8 sequence, so `toString('utf8')`
|
||||
// renders each as U+FFFD (3 serialized bytes). `accrueStrayCost` must charge
|
||||
@@ -820,6 +846,51 @@ describe('PythonCodeRuntime — programs and bindings', () => {
|
||||
expect(maxConcat).toBeLessThan(2048)
|
||||
})
|
||||
|
||||
it('charges a structurally-valid but illegal UTF-8 sequence its U+FFFD-decoded cost', async () => {
|
||||
// A CESU-8 lone surrogate `ED A0 80` is structurally well-formed (a 3-byte
|
||||
// lead plus two 0x80–0xBF continuations) but ILLEGAL: `toString('utf8')`
|
||||
// renders each of the three bytes as its own U+FFFD (serialized cost 9), not
|
||||
// one width-3 character. The newline-free flush trigger weighs the residual
|
||||
// through `accrueStrayCost`, which must validate each lead's
|
||||
// first-continuation range (ED excludes A0–BF) and charge the true 9 — else a
|
||||
// CESU flood undercounts 3x and the residual grows toward a full budget's raw
|
||||
// bytes before flushing, the same peak-memory vector as the 0xFF case. The
|
||||
// bytes are written one at a time (each its own `data` chunk, no pipe
|
||||
// coalescing) and `Buffer.concat` is wrapped to measure the peak residual.
|
||||
const realConcat = Buffer.concat.bind(Buffer)
|
||||
let maxConcat = 0
|
||||
Buffer.concat = (list: readonly Uint8Array[], total?: number): Buffer<ArrayBuffer> => {
|
||||
const merged = realConcat(list, total)
|
||||
if (merged.length > maxConcat) maxConcat = merged.length
|
||||
return merged
|
||||
}
|
||||
let result: CodeRunResult
|
||||
try {
|
||||
const { runtime } = await setup({ maxLogBytes: 3072, maxWallMs: 30_000 })
|
||||
result = await runtime.run({
|
||||
program: [
|
||||
'import os',
|
||||
'seq = (0xed, 0xa0, 0x80)',
|
||||
'for _ in range(2000):',
|
||||
' for b in seq:',
|
||||
' os.write(1, bytes((b,)))',
|
||||
' os.sched_yield()',
|
||||
'return None',
|
||||
].join('\n'),
|
||||
bindings: [],
|
||||
})
|
||||
} finally {
|
||||
Buffer.concat = realConcat
|
||||
}
|
||||
expect(result.error).toBeUndefined()
|
||||
expect(result.logs.at(-1)).toBe(logTruncationMarker(3072))
|
||||
// Each 3-byte sequence costs 9 (three U+FFFD), so single-byte-paced the
|
||||
// residual crosses the 3072 budget after ~342 raw bytes and flushes; the
|
||||
// largest merged buffer stays well under 2048. Charging the structural width
|
||||
// 3 would need ~1024 raw bytes, tripling the peak past 2048.
|
||||
expect(maxConcat).toBeLessThan(2048)
|
||||
})
|
||||
|
||||
it('charges a lone surrogate its full six escaped bytes, not three', async () => {
|
||||
// A forged `log` frame carrying `\ud800` escapes materializes lone
|
||||
// surrogates after JSON.parse. `Buffer.byteLength` of U+FFFD is 3, but
|
||||
@@ -852,11 +923,11 @@ describe('PythonCodeRuntime — programs and bindings', () => {
|
||||
// exhausts maxLogBytes: the first line's admit truncates and marks the
|
||||
// ledger, and the second line's admit — reached in the same `data` callback
|
||||
// — must be the post-truncation no-op. Proves that branch is exercised, so
|
||||
// it carries no v8-ignore. Kept to 109 bytes (< the smallest PIPE_BUF, 512 on
|
||||
// it carries no v8-ignore. Kept to 108 bytes (< the smallest PIPE_BUF, 512 on
|
||||
// macOS) so the whole payload lands in ONE atomic write and one `data`
|
||||
// callback — the two newlines cannot split across callbacks and leave the
|
||||
// branch un-exercised, which would be a hard-to-attribute per-file coverage
|
||||
// flake. 103 payload bytes still exceed the 64-byte budget, so it truncates.
|
||||
// flake. The first line's 100 bytes already exceed the 64-byte budget, so it truncates.
|
||||
const { runtime } = await setup({ maxLogBytes: 64 })
|
||||
const result = await runtime.run({
|
||||
program: ['import os', 'os.write(1, b"A" * 100 + b"\\nSECOND\\n")', 'return None'].join('\n'),
|
||||
@@ -3561,19 +3632,23 @@ describe('PythonCodeRuntime — hostile peer', () => {
|
||||
|
||||
it('reassembles multibyte UTF-8 split across stray-output pipe chunks', async () => {
|
||||
// A single os.write far past the 64 KiB pipe buffer forces multiple
|
||||
// 'data' chunks; when the boundary lands inside the emoji's 4-byte
|
||||
// sequence, per-chunk decoding would corrupt it into replacement
|
||||
// characters. Raw bytes are buffered and only decoded once a complete line
|
||||
// (or the whole tail at flush) is assembled, so the split sequence is whole
|
||||
// by the time it is decoded.
|
||||
// 'data' chunks; when the boundary lands inside a multibyte sequence,
|
||||
// per-chunk decoding would corrupt it into replacement characters. Raw bytes
|
||||
// are buffered and only decoded once a complete line (or the whole tail at
|
||||
// flush) is assembled, so the split sequence is whole by the time it is
|
||||
// decoded. The payload spans every valid multibyte lead class so
|
||||
// accrueStrayCost's per-lead continuation ranges are all exercised: U+0900
|
||||
// (E0 A4 80, the range-restricted E0 lead), U+4F60 and U+597D (E4/E5, plain
|
||||
// 3-byte), U+1F600 (F0, the range-restricted F0 lead), and U+10FFFF (F4 8F
|
||||
// BF BF, the range-restricted F4 lead).
|
||||
const { runtime } = await setup({ maxLogBytes: 1024 * 1024 })
|
||||
const result = await runtime.run({
|
||||
program: [
|
||||
'import os',
|
||||
// os.write is one syscall and returns a partial count on a full
|
||||
// pipe, so loop until the whole payload (odd prefix -> a chunk
|
||||
// boundary lands inside the emoji's 4-byte sequence) is out.
|
||||
String.raw`payload = b"a" * 65535 + "\u4f60\u597d\U0001f600".encode("utf-8")`,
|
||||
// boundary lands inside a multibyte sequence) is out.
|
||||
String.raw`payload = b"a" * 65535 + "\u0900\u4f60\u597d\U0001f600\U0010ffff".encode("utf-8")`,
|
||||
'view = memoryview(payload)',
|
||||
'while view:',
|
||||
' view = view[os.write(1, view):]',
|
||||
@@ -3583,7 +3658,7 @@ describe('PythonCodeRuntime — hostile peer', () => {
|
||||
})
|
||||
expect(result.error).toBeUndefined()
|
||||
const text = result.logs.join('')
|
||||
expect(text).toContain('\u4f60\u597d\u{1f600}')
|
||||
expect(text).toContain('\u0900\u4f60\u597d\u{1f600}\u{10ffff}')
|
||||
expect(text).not.toContain('\ufffd')
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user