From 8a60b5f0056ec3636742c7c270c52fb3385cb749 Mon Sep 17 00:00:00 2001 From: Chinesezjc Date: Sun, 2 Aug 2026 17:45:11 +0800 Subject: [PATCH] feat(code-runtime-python): make the TypedDict wire mirror an executable gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address the two standing review suggestions in this layer rather than deferring them to PR #4: - Extend tests/protocol-mirror.e2e.ts to read each py/protocol.py TypedDict's required/optional key set and assert it against the wire field names src/protocol.ts declares (global included, via functional TypedDict). The round-12 class of drift — a renamed/dropped field, or one side making a field optional the other requires — now fails a test instead of relying on review. Field types remain review-guarded (no mechanical TS/Python equivalent). - Drop the forward references to PR #4's internal mechanisms from this layer's prose: the "256 MiB frame ceiling" figure and the "(index.ts)" fd-3 pinning citation become an abstract "host-side inbound frame-size cap" so the JSDoc, spec, README, and Agent Note describe only what this layer owns. Update both README sides and the Agent Note (both languages) to state the mirror is now executable, and re-record their i18n pairings. --- ...code-runtime-python-fd3-protocol.i18n.yaml | 4 +- ...-07-31-code-runtime-python-fd3-protocol.md | 4 +- ...-31-code-runtime-python-fd3-protocol.zh.md | 4 +- .../code-runtime-python/README.i18n.yaml | 4 +- .../code-runtime-python/README.md | 2 +- .../code-runtime-python/README.zh.md | 2 +- .../code-runtime-python/src/protocol.ts | 21 ++++--- .../tests/protocol-mirror.e2e.ts | 60 ++++++++++++++++--- .../tests/protocol.spec.ts | 4 +- 9 files changed, 76 insertions(+), 29 deletions(-) diff --git a/.agents/notes/implemented/architecture/2026-07-31-code-runtime-python-fd3-protocol.i18n.yaml b/.agents/notes/implemented/architecture/2026-07-31-code-runtime-python-fd3-protocol.i18n.yaml index 33df2bf488..f716091e5b 100644 --- a/.agents/notes/implemented/architecture/2026-07-31-code-runtime-python-fd3-protocol.i18n.yaml +++ b/.agents/notes/implemented/architecture/2026-07-31-code-runtime-python-fd3-protocol.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-07-31-code-runtime-python-fd3-protocol.md -2026-07-31-code-runtime-python-fd3-protocol.md: 142f6aaf8093ec3e76249fecb40a6fbb13d80500 -2026-07-31-code-runtime-python-fd3-protocol.zh.md: 5a371240bb816379d50cc6331f0c6971cf37209a +2026-07-31-code-runtime-python-fd3-protocol.md: fff3ed7a6e42cfc5372c7c8a3124a33ebcacab32 +2026-07-31-code-runtime-python-fd3-protocol.zh.md: 88a6d493b35b976abf3676dd00182da1270c4fec diff --git a/.agents/notes/implemented/architecture/2026-07-31-code-runtime-python-fd3-protocol.md b/.agents/notes/implemented/architecture/2026-07-31-code-runtime-python-fd3-protocol.md index 142f6aaf80..fff3ed7a6e 100644 --- a/.agents/notes/implemented/architecture/2026-07-31-code-runtime-python-fd3-protocol.md +++ b/.agents/notes/implemented/architecture/2026-07-31-code-runtime-python-fd3-protocol.md @@ -28,7 +28,7 @@ Frames are JSON-lines on fd 3, one object per line, leaving stdout/stderr free f ## Mirror alignment -Round-12 review of #436 found `py/protocol.py` stale against `src/protocol.ts` in three declarations — `LogMessage` lacked `truncated`, `DoneMessage.error` lacked `kind`, and `Namespace` lacked the optional `errorClass`. This PR aligns all three when lifting the file, so the stale mirror is not carried forward. Because the declarations are `TypedDict`s (no runtime enforcement on the trusted Python side), an automated guard covers only what both sides execute: `tests/protocol-mirror.e2e.ts` spawns a real `python3`, reads `PROTOCOL_FD` and `log_truncation_marker` from `py/protocol.py`, and asserts they equal the TypeScript constants across several byte budgets. +Round-12 review of #436 found `py/protocol.py` stale against `src/protocol.ts` in three declarations — `LogMessage` lacked `truncated`, `DoneMessage.error` lacked `kind`, and `Namespace` lacked the optional `errorClass`. This PR aligns all three when lifting the file, so the stale mirror is not carried forward. To keep it aligned, `tests/protocol-mirror.e2e.ts` spawns a real `python3` and asserts, against `src/protocol.ts`: `PROTOCOL_FD` and `log_truncation_marker` (the two surfaces both sides execute), and each `TypedDict`'s required/optional wire field set — so a renamed or dropped field, or one side making a field optional the other requires (exactly the round-12 drift), fails the test. Field *types* are not compared across the language boundary; that residue stays with review. ## Alternatives considered @@ -40,4 +40,4 @@ Round-12 review of #436 found `py/protocol.py` stale against `src/protocol.ts` i Bought: the fd-3 protocol and its hostile-input codec land as a self-contained, fully unit-covered layer, and the py/ts mirror drift the round-12 review found is fixed with an executing guard against its recurrence. The backend-core PR builds on a reviewed wire contract. -Cost: `src/index.ts` and `package.json` are introduced minimally here and edited (not created) by the backend-core PR. The `TypedDict` shapes in `py/protocol.py` beyond the two executed surfaces remain guarded by review plus the backend's real-subprocess suite, not by the mirror e2e test — an inherent limit of comparing type declarations across languages. +Cost: `src/index.ts` and `package.json` are introduced minimally here and edited (not created) by the backend-core PR. The mirror e2e compares field NAMES and required/optional-ness across the two sides but not field TYPES — comparing type declarations across TypeScript and Python has no mechanical equivalent, so that residue stays with review plus the backend's real-subprocess suite. diff --git a/.agents/notes/implemented/architecture/2026-07-31-code-runtime-python-fd3-protocol.zh.md b/.agents/notes/implemented/architecture/2026-07-31-code-runtime-python-fd3-protocol.zh.md index 5a371240bb..88a6d493b3 100644 --- a/.agents/notes/implemented/architecture/2026-07-31-code-runtime-python-fd3-protocol.zh.md +++ b/.agents/notes/implemented/architecture/2026-07-31-code-runtime-python-fd3-protocol.zh.md @@ -28,7 +28,7 @@ CPython code-runtime 后端（`@deepseek-ai/dsh-code-runtime-python`，分多个 ## Mirror alignment -#436 的 round-12 review 发现 `py/protocol.py` 相对 `src/protocol.ts` 有三处声明陈旧——`LogMessage` 缺 `truncated`、`DoneMessage.error` 缺 `kind`、`Namespace` 缺可选的 `errorClass`。本 PR 在搬运该文件时对齐了这三处，不把陈旧镜像带过来。由于这些声明是 `TypedDict`（在受信任的 Python 侧无运行时强制），自动化 guard 只覆盖两侧都会执行的部分：`tests/protocol-mirror.e2e.ts` 启动一个真实 `python3`，从 `py/protocol.py` 读取 `PROTOCOL_FD` 与 `log_truncation_marker`，并在若干字节预算下断言它们等于 TypeScript 常量。 +#436 的 round-12 review 发现 `py/protocol.py` 相对 `src/protocol.ts` 有三处声明陈旧——`LogMessage` 缺 `truncated`、`DoneMessage.error` 缺 `kind`、`Namespace` 缺可选的 `errorClass`。本 PR 在搬运该文件时对齐了这三处，不把陈旧镜像带过来。为持续保持对齐，`tests/protocol-mirror.e2e.ts` 启动一个真实 `python3`，对照 `src/protocol.ts` 断言：`PROTOCOL_FD` 与 `log_truncation_marker`（两侧都会执行的面），以及每个 `TypedDict` 的必填/可选 wire 字段集——于是字段被重命名或删除、或一侧把另一侧要求的字段改成可选（正是 round-12 那类漂移），测试即失败。字段的*类型*不跨语言边界比较，那部分残留留给 review。 ## Alternatives considered @@ -40,4 +40,4 @@ CPython code-runtime 后端（`@deepseek-ai/dsh-code-runtime-python`，分多个 收获：fd-3 协议及其敌意输入 codec 作为自包含、unit 全覆盖的一层落地，round-12 review 发现的 py/ts 镜像漂移被修复，并有一个执行中的 guard 防其复发。backend-core PR 建立在已 review 的 wire contract 之上。 -代价：`src/index.ts` 与 `package.json` 在此以最小形态引入，并由 backend-core PR 编辑（而非创建）。`py/protocol.py` 中两个可执行面之外的 `TypedDict` 形状仍由 review 加后端真子进程套件守护，而非 mirror e2e 测试——这是跨语言比较类型声明的固有局限。 +代价：`src/index.ts` 与 `package.json` 在此以最小形态引入，并由 backend-core PR 编辑（而非创建）。mirror e2e 比较两侧的字段名与必填/可选性，但不比较字段类型——跨 TypeScript 与 Python 比较类型声明无机械等价物，那部分残留留给 review 加后端真子进程套件。 diff --git a/packages/code-runtime/code-runtime-python/README.i18n.yaml b/packages/code-runtime/code-runtime-python/README.i18n.yaml index f096202ad7..4d7725dafc 100644 --- a/packages/code-runtime/code-runtime-python/README.i18n.yaml +++ b/packages/code-runtime/code-runtime-python/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/code-runtime/code-runtime-python/README.md -README.md: 62e899941ac8eadb2046b1bffae0a3c9c6e14308 -README.zh.md: 5a7aa880830bccc4ebd647d7b009ec7e4f0d6307 +README.md: d0491c478d04a8436199bc23fd79917e8c019b1c +README.zh.md: 63d7d38ee05b561e60a3adf387c5c1292c37a7a2 diff --git a/packages/code-runtime/code-runtime-python/README.md b/packages/code-runtime/code-runtime-python/README.md index 62e899941a..d0491c478d 100644 --- a/packages/code-runtime/code-runtime-python/README.md +++ b/packages/code-runtime/code-runtime-python/README.md @@ -25,5 +25,5 @@ No direct invalidation; the named consumer owns any request-prefix changes. ## Known Limitations and Deferred Work -- **The cross-language guard covers only the two runtime-executed surfaces** — `PROTOCOL_FD` and the log truncation marker. The `TypedDict` frame shapes in `py/protocol.py` mirror `src/protocol.ts` by review, not by an automated check: comparing type declarations across TypeScript and Python has no mechanical equivalent here, so a future shape drift is caught by review plus the backend's real-subprocess suite rather than this package's tests. +- **The cross-language guard covers the runtime-executed surfaces and the frame field shapes** — `tests/protocol-mirror.e2e.ts` spawns a real `python3` and asserts, against `src/protocol.ts`, both `PROTOCOL_FD` / the log truncation marker text AND each `TypedDict`'s required/optional wire field set in `py/protocol.py`. What it does not compare is the field *types* (e.g. that `cpuSeconds` is an `int` on both sides): comparing type declarations across TypeScript and Python has no mechanical equivalent here, so a type-level drift is still caught by review plus the backend's real-subprocess suite rather than this package's tests. - **The `PythonCodeRuntime` implementation and its Python-side JSON codec are not in this layer** — they ship in the backend-core PR on top of this branch; `src/index.ts` re-exports only the protocol vocabulary until then. diff --git a/packages/code-runtime/code-runtime-python/README.zh.md b/packages/code-runtime/code-runtime-python/README.zh.md index 5a7aa88083..63d7d38ee0 100644 --- a/packages/code-runtime/code-runtime-python/README.zh.md +++ b/packages/code-runtime/code-runtime-python/README.zh.md @@ -25,5 +25,5 @@ host 与 CPython 子进程在子进程的 fd 3 上交换一个无版本号的 JS ## Known Limitations and Deferred Work -- **跨语言 guard 只覆盖两个运行时执行的面** —— `PROTOCOL_FD` 与日志截断标记。`py/protocol.py` 中的 `TypedDict` 帧形状靠 review 而非自动化检查来镜像 `src/protocol.ts`：跨 TypeScript 与 Python 比较类型声明在此无机械等价物，故未来的形状漂移由 review 加后端真子进程套件捕获，而非本包的测试。 +- **跨语言 guard 覆盖运行时执行的面与帧字段形状** —— `tests/protocol-mirror.e2e.ts` 启动一个真实 `python3`，对照 `src/protocol.ts` 断言 `PROTOCOL_FD` / 日志截断标记文本，以及 `py/protocol.py` 中每个 `TypedDict` 的必填/可选 wire 字段集。它不比较字段的*类型*（例如 `cpuSeconds` 两侧都是 `int`）：跨 TypeScript 与 Python 比较类型声明在此无机械等价物，故类型级漂移仍由 review 加后端真子进程套件捕获，而非本包的测试。 - **`PythonCodeRuntime` 实现与 Python 侧 JSON codec 不在本层** —— 它们在基于本分支的 backend-core PR 中交付；在那之前 `src/index.ts` 只 re-export 协议词汇。 diff --git a/packages/code-runtime/code-runtime-python/src/protocol.ts b/packages/code-runtime/code-runtime-python/src/protocol.ts index 94489b8904..80eda4ab8d 100644 --- a/packages/code-runtime/code-runtime-python/src/protocol.ts +++ b/packages/code-runtime/code-runtime-python/src/protocol.ts @@ -7,8 +7,9 @@ */ // The protocol channel is fd 3 from the child's perspective — the host pins it -// positionally via `stdio: ['pipe','pipe','pipe','pipe']` (index.ts), and the -// Python bootstrap reads the same constant from its own protocol.py. +// positionally via `stdio: ['pipe','pipe','pipe','pipe']` when it spawns the +// child, and the Python bootstrap reads the same constant from its own +// protocol.py. /** * What the host sends immediately after spawn, as the first line on fd 3. The @@ -194,12 +195,13 @@ function scalarJson(current: unknown): string { * crossed. This bounds the INCREMENTAL allocation the check itself would add on * top of the already-parsed value — the escaped-string copy, the enqueued * children, the per-key `JSON.stringify` — not the parse that produced `value`. - * That upstream width is bounded separately: the host reads fd 3 into a fixed - * 256 MiB receive buffer (a later stack layer), so `value` cannot already be - * larger than that when it reaches here, while `maxValueBytes` defaults to - * 32 KiB. The traversal rejects over-budget BEFORE materializing a string's - * escaped form or enqueuing an array's/object's children, so a below-ceiling - * forgery cannot force those secondary allocations. Object key COUNTING is + * That upstream width is bounded separately, by the host-side cap on inbound + * fd-3 frame size before `JSON.parse` runs (owned by the runtime that reads the + * channel), so `value` cannot be arbitrarily large when it reaches here, while + * `maxValueBytes` defaults to 32 KiB. The traversal rejects over-budget BEFORE + * materializing a string's escaped form or enqueuing an array's/object's + * children, so a forgery within that frame cap cannot force those secondary + * allocations. Object key COUNTING is * unavoidably O(keys) — JS has no lazy own-key iterator, and the parse already * built the key set — but the check still refuses the per-entry work before the * enqueue loop. A non-lossless number (non-finite, negative zero) is caught only @@ -353,7 +355,8 @@ function* ownValues(record: object): Generator { * cap, so there is no budget to reject a wide payload against the way * {@link checkDoneValue} does. The traversal therefore holds ONE cursor per * NESTING LEVEL (an array or {@link ownValues} iterator) instead of one entry - * per member: a forged flat `args` just below the 256 MiB frame ceiling would + * per member: a forged flat `args` at the top of the host's inbound frame-size + * cap would * otherwise push tens of millions of stack entries — and `Object.values` would * copy each object's full breadth — allocating hundreds of megabytes beyond * what `JSON.parse` already holds. Iterative either way, so a deep frame diff --git a/packages/code-runtime/code-runtime-python/tests/protocol-mirror.e2e.ts b/packages/code-runtime/code-runtime-python/tests/protocol-mirror.e2e.ts index d79a659c09..3a191ddbb2 100644 --- a/packages/code-runtime/code-runtime-python/tests/protocol-mirror.e2e.ts +++ b/packages/code-runtime/code-runtime-python/tests/protocol-mirror.e2e.ts @@ -6,13 +6,14 @@ import { describe, expect, it } from 'vitest' import { logTruncationMarker } from '../src/protocol.ts' /** - * Cross-language mirror check for the two protocol surfaces the host and the - * CPython subprocess share at runtime, spawning a real `python3` to read them - * from `py/protocol.py`. `src/protocol.ts` and `py/protocol.py` declare the same - * frame vocabulary on two sides of the wire; the only values both sides EXECUTE - * against are `PROTOCOL_FD` (the fd the channel is pinned to) and the log - * truncation marker text (emitted verbatim by whichever ledger exhausts first), - * so a drift there silently corrupts a live run. Self-skips when no `python3` is + * Cross-language mirror check between `src/protocol.ts` and `py/protocol.py`, + * spawning a real `python3` to read the Python side. Two things are asserted: + * the runtime surfaces both sides EXECUTE against — `PROTOCOL_FD` and the log + * truncation marker text, where a drift silently corrupts a live run — and the + * per-frame wire field sets (required/optional keys of each `TypedDict`), which + * turns the otherwise review-only shape mirror into an executable check that + * catches the round-12 kind of drift (a renamed/dropped field, or one side + * making a field optional the other requires). Self-skips when no `python3` is * on PATH — CI provides one; the pure-TS `protocol.spec.ts` covers the host * codec unconditionally. */ @@ -46,10 +47,53 @@ describe.skipIf(!python3Available)('protocol.py mirrors protocol.ts at runtime', ].join('\n') const { stdout } = await execFileAsync('python3', ['-I', '-c', probe]) const seen = JSON.parse(stdout) as { fd: number; markers: string[] } - // fd 3 is the wire contract, not a tunable: index.ts pins it positionally. + // fd 3 is the wire contract, not a tunable: the host pins it positionally + // when it spawns the child. expect(seen.fd).toBe(3) expect(seen.markers).toEqual(budgets.map(budget => logTruncationMarker(budget))) }) + + it('agrees on every frame type\'s wire field set between the TS and Python declarations', async () => { + // Turn the TypedDict mirror from a review-only obligation into an executable + // check: read each Python TypedDict's required/optional key sets and assert + // them against the wire field names the TS side declares. `global` is the + // reserved-keyword key the Python side carries via functional TypedDict — + // catching exactly the round-12 kind of drift (a renamed/dropped field, an + // optional field the other side made required). + const probe = [ + 'import json, sys', + `sys.path.insert(0, ${JSON.stringify(pyDir)})`, + 'import protocol as p', + 'def keys(td): return {"required": sorted(td.__required_keys__), "optional": sorted(td.__optional_keys__)}', + 'print(json.dumps({', + ' "BootMessage": keys(p.BootMessage),', + ' "Namespace": keys(p.Namespace),', + ' "RunMessage": keys(p.RunMessage),', + ' "BootAckMessage": keys(p.BootAckMessage),', + ' "CallMessage": keys(p.CallMessage),', + ' "LogMessage": keys(p.LogMessage),', + ' "DoneErrorField": keys(p.DoneErrorField),', + ' "DoneMessage": keys(p.DoneMessage),', + ' "ErrorClass": keys(p.ErrorClass),', + '}))', + ].join('\n') + const { stdout } = await execFileAsync('python3', ['-I', '-c', probe]) + const seen = JSON.parse(stdout) as Record + // The wire field sets each frame carries, mirroring src/protocol.ts. `global` + // is the JSON key `CallMessage`/`Namespace` send (a Python keyword, declared + // functionally on the Python side). + expect(seen).toEqual({ + BootMessage: { required: ['addressSpaceBytes', 'cpuSeconds', 'maxLogBytes', 'maxValueBytes', 'namespaces', 'type'], optional: [] }, + Namespace: { required: ['global', 'names'], optional: ['errorClass'] }, + RunMessage: { required: ['program', 'type'], optional: [] }, + BootAckMessage: { required: ['type'], optional: [] }, + CallMessage: { required: ['args', 'global', 'id', 'name', 'type'], optional: [] }, + LogMessage: { required: ['text', 'type'], optional: ['truncated'] }, + DoneErrorField: { required: ['kind', 'message'], optional: [] }, + DoneMessage: { required: ['type'], optional: ['error', 'value'] }, + ErrorClass: { required: ['memberNameProperty', 'name'], optional: [] }, + }) + }) }) it('names the py/ directory that ships with the package', () => { diff --git a/packages/code-runtime/code-runtime-python/tests/protocol.spec.ts b/packages/code-runtime/code-runtime-python/tests/protocol.spec.ts index a33b9e905b..465aa06a21 100644 --- a/packages/code-runtime/code-runtime-python/tests/protocol.spec.ts +++ b/packages/code-runtime/code-runtime-python/tests/protocol.spec.ts @@ -135,8 +135,8 @@ describe('lossless-number scan', () => { it('walks wide arrays and objects one member at a time', () => { // `call.args` carries no seam byte cap, so a wide forged payload has no // budget to be rejected against — the walk must hold one cursor per - // NESTING LEVEL, not one entry per member, or a flat payload just below - // the 256 MiB frame ceiling would allocate tens of millions of stack + // NESTING LEVEL, not one entry per member, or a flat payload at the top of + // the host's inbound frame-size cap would allocate tens of millions of stack // entries (and `Object.values` a second full-breadth copy). Observable // through the boundary: a wide payload whose per-member cost the old shape // would have paid still scans, and a violation ANYWHERE in it is found