mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-08-30 04:40:37 +00:00
Chinesezjc
be839a8e53
Two gaps from the previous round's fixes: - checkDoneValue flagged a non-lossless number but skipped counting its encoded bytes, so a value over budget ONLY through that number classified as non-lossless instead of over-budget (e.g. [Infinity] at cap 3, whose encoding is 10 bytes). Count the scalar's bytes even when flagging, so the budget check wins as the JSDoc promises. Add cap-3 regression cases. - The mirror e2e compared the Python TypedDict keys against a hand-written constant, so a field change on the TS side alone would not fail it, and the reply frames were not probed at all. Introduce WIRE_FRAME_FIELDS in protocol.ts, bound to each frame interface's key set via `satisfies` (a renamed/removed field breaks typecheck — verified), and drive the mirror test from it, now covering ReplyOk/ReplyErr too. The test therefore fails on one-sided drift from either language.
95 lines
4.3 KiB
TypeScript
95 lines
4.3 KiB
TypeScript
import { execFile } from 'node:child_process'
|
|
import { existsSync } from 'node:fs'
|
|
import { fileURLToPath } from 'node:url'
|
|
import { promisify } from 'node:util'
|
|
import { describe, expect, it } from 'vitest'
|
|
import { logTruncationMarker, WIRE_FRAME_FIELDS } from '../src/protocol.ts'
|
|
|
|
/**
|
|
* Cross-language mirror check between `src/protocol.ts` and `py/protocol.py`,
|
|
* spawning a real `python3` to read the Python side. Two things are asserted:
|
|
* the runtime surfaces both sides EXECUTE against — `PROTOCOL_FD` and the log
|
|
* truncation marker text, where a drift silently corrupts a live run — and the
|
|
* per-frame wire field sets (required/optional keys of each `TypedDict`), which
|
|
* turns the otherwise review-only shape mirror into an executable check that
|
|
* catches the round-12 kind of drift (a renamed/dropped field, or one side
|
|
* making a field optional the other requires). Self-skips when no `python3` is
|
|
* on PATH — CI provides one; the pure-TS `protocol.spec.ts` covers the host
|
|
* codec unconditionally.
|
|
*/
|
|
|
|
const execFileAsync = promisify(execFile)
|
|
const pyDir = fileURLToPath(new URL('../py', import.meta.url))
|
|
|
|
async function hasPython3(): Promise<boolean> {
|
|
try {
|
|
await execFileAsync('python3', ['--version'])
|
|
return true
|
|
} catch {
|
|
return false
|
|
}
|
|
}
|
|
|
|
const python3Available = await hasPython3()
|
|
|
|
describe.skipIf(!python3Available)('protocol.py mirrors protocol.ts at runtime', () => {
|
|
it('agrees on PROTOCOL_FD and the log truncation marker across byte budgets', async () => {
|
|
const budgets = [1, 65536, 1048576]
|
|
const probe = [
|
|
'import json, sys',
|
|
`sys.path.insert(0, ${JSON.stringify(pyDir)})`,
|
|
'from protocol import PROTOCOL_FD, log_truncation_marker',
|
|
`budgets = ${JSON.stringify(budgets)}`,
|
|
'print(json.dumps({',
|
|
' "fd": PROTOCOL_FD,',
|
|
' "markers": [log_truncation_marker(b) for b in budgets],',
|
|
'}))',
|
|
].join('\n')
|
|
const { stdout } = await execFileAsync('python3', ['-I', '-c', probe])
|
|
const seen = JSON.parse(stdout) as { fd: number; markers: string[] }
|
|
// fd 3 is the wire contract, not a tunable: the host pins it positionally
|
|
// when it spawns the child.
|
|
expect(seen.fd).toBe(3)
|
|
expect(seen.markers).toEqual(budgets.map(budget => logTruncationMarker(budget)))
|
|
})
|
|
|
|
it('agrees on every frame type\'s wire field set between the TS and Python declarations', async () => {
|
|
// Turn the TypedDict mirror from a review-only obligation into an executable
|
|
// check: read each Python TypedDict's required/optional key sets and assert
|
|
// them against WIRE_FRAME_FIELDS — the TS-side source of truth bound to the
|
|
// frame interfaces by `satisfies` in protocol.ts, so a rename or a removed
|
|
// field on the TS side breaks typecheck and an added field breaks this
|
|
// comparison (the Python side would carry it). Covers the reply frames too.
|
|
// `global` is the reserved-keyword wire key the Python side carries via a
|
|
// functional TypedDict. This catches the round-12 kind of drift on EITHER
|
|
// side of the wire.
|
|
const pyNames = Object.keys(WIRE_FRAME_FIELDS)
|
|
const probe = [
|
|
'import json, sys',
|
|
`sys.path.insert(0, ${JSON.stringify(pyDir)})`,
|
|
'import protocol as p',
|
|
'def keys(td): return {"required": sorted(td.__required_keys__), "optional": sorted(td.__optional_keys__)}',
|
|
`names = ${JSON.stringify(pyNames)}`,
|
|
'print(json.dumps({n: keys(getattr(p, n)) for n in names}))',
|
|
].join('\n')
|
|
const { stdout } = await execFileAsync('python3', ['-I', '-c', probe])
|
|
const seen = JSON.parse(stdout) as Record<string, { required: string[]; optional: string[] }>
|
|
// Normalize the TS source of truth to the same sorted shape Python reports.
|
|
const expected = Object.fromEntries(
|
|
Object.entries(WIRE_FRAME_FIELDS).map(([name, sets]) => [
|
|
name,
|
|
{ required: [...sets.required].sort(), optional: [...sets.optional].sort() },
|
|
]),
|
|
)
|
|
expect(seen).toEqual(expected)
|
|
})
|
|
})
|
|
|
|
it('names the py/ directory that ships with the package', () => {
|
|
// Resolves py/ relative to this test file; the same directory ships in the
|
|
// package.json `files` whitelist (`py/**/*.py`). The tests/ directory itself
|
|
// is not published — this asserts the source-tree layout the mirror test
|
|
// depends on, so it holds even when python3 is absent from the runner.
|
|
expect(existsSync(pyDir)).toBe(true)
|
|
})
|