mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-08-29 04:26:38 +00:00
fix(ci): preserve Windows coverage failures
This commit is contained in:
@@ -2,5 +2,5 @@
|
||||
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
||||
# after editing either side, bring the other along and re-record with:
|
||||
# pnpm run verify-translation-pairing --write .agents/notes/implemented/process/2026-08-18-in-job-partitioned-coverage.md
|
||||
2026-08-18-in-job-partitioned-coverage.md: d6b8f98095ebb77c6caf88ce67999e683c71f74c
|
||||
2026-08-18-in-job-partitioned-coverage.zh.md: 36a49b91b2544c611bf77350bda66b06a2708ba6
|
||||
2026-08-18-in-job-partitioned-coverage.md: b7335c490c4a4921e5c786809e0db492613ef5c8
|
||||
2026-08-18-in-job-partitioned-coverage.zh.md: 5e2a6a60d59751a22c7d664d4bc50d93390c6995
|
||||
|
||||
@@ -22,7 +22,7 @@ The coordinator waits for every child, validates that the blob directory contain
|
||||
|
||||
## Failure and output semantics
|
||||
|
||||
Partition children inherit the coordinator's stdout and stderr. The coverage gate opts into `run-gates` streaming, so test progress and failures reach CI logs as they occur without buffering the complete log in the scheduler or printing it a second time at completion. When a child settles unsuccessfully, the coordinator immediately prints its spawn error, exit code, or signal before validating the complete blob set.
|
||||
Partition children stream stdout and stderr through the coordinator. The coverage gate opts into `run-gates` streaming, so test progress and failures reach CI logs as they occur without buffering the complete log in the scheduler. The coordinator also retains a bounded 64 KiB combined tail per child; when a child settles unsuccessfully, it prints the spawn error, exit code, or signal and repeats that tail before validating the complete blob set, keeping the specific Vitest failure beside the final partition diagnostic.
|
||||
|
||||
A normal failed test still emits a blob through `--coverage.reportOnFailure`, allowing the merge to report the complete coverage state before the coordinator returns failure. Spawn failure, signal termination, non-zero exit, a missing or extra blob, or a failed merge all make the gate fail. The coordinator removes only its owned coverage tree and unlinks a link-shaped path instead of recursively following it.
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ Status: implemented
|
||||
|
||||
## 失败与输出语义
|
||||
|
||||
分区子进程继承协调器的 stdout 与 stderr。覆盖率门禁选择 `run-gates` 流式输出,因此测试进度与失败会在发生时进入 CI 日志;调度器不会缓冲完整日志,也不会在结束时重复打印。子进程以失败状态结算时,协调器会立即打印其 spawn 错误、退出码或信号,再校验完整的 blob 集合。
|
||||
分区子进程通过协调器流式传递 stdout 与 stderr。覆盖率门禁选择 `run-gates` 流式输出,因此测试进度与失败会在发生时进入 CI 日志,调度器不会缓冲完整日志。协调器还会为每个子进程保留一份有界的 64 KiB 混合输出尾部;子进程以失败状态结算时,它会打印 spawn 错误、退出码或信号,并在校验完整 blob 集合前重印这份尾部,使具体 Vitest 失败与最终分区诊断相邻。
|
||||
|
||||
普通测试失败仍通过 `--coverage.reportOnFailure` 产出 blob,使合并步骤可以先报告完整覆盖率状态,再由协调器返回失败。spawn 失败、信号终止、非零退出、blob 缺失或多余,以及合并失败都会让门禁失败。协调器只删除自己拥有的覆盖率目录树;若该路径是链接,则只 unlink,不递归跟随。
|
||||
|
||||
|
||||
@@ -65,17 +65,19 @@ interface RealInstanceFixture {
|
||||
readonly workspace: string
|
||||
}
|
||||
|
||||
type ResponsesScript = readonly ResponsesBehavior[] | ((workspace: string) => readonly ResponsesBehavior[])
|
||||
|
||||
async function realInstanceFixture(
|
||||
script: readonly ResponsesBehavior[],
|
||||
script: ResponsesScript,
|
||||
): Promise<RealInstanceFixture> {
|
||||
const root = mkdtempSync(join(tmpdir(), 'dsh-codex-real-'))
|
||||
roots.push(root)
|
||||
const workspace = join(root, 'workspace')
|
||||
const codexHome = join(root, 'codex-home')
|
||||
const fixture = await startResponsesFixture(script)
|
||||
fixtures.push(fixture)
|
||||
mkdirSync(workspace)
|
||||
mkdirSync(codexHome)
|
||||
const fixture = await startResponsesFixture(typeof script === 'function' ? script(workspace) : script)
|
||||
fixtures.push(fixture)
|
||||
writeFileSync(join(codexHome, 'config.toml'), [
|
||||
'model = "fixture-model"',
|
||||
'model_provider = "fixture"',
|
||||
@@ -133,7 +135,7 @@ async function realRuntime(): Promise<RealRuntime> {
|
||||
}
|
||||
|
||||
async function realHarness(
|
||||
script: readonly ResponsesBehavior[],
|
||||
script: ResponsesScript,
|
||||
permissionMode?: CodexPermissionMode,
|
||||
): Promise<{
|
||||
readonly harness: RealHarness
|
||||
@@ -385,25 +387,30 @@ describe('real @openai/codex 0.147.0 product', () => {
|
||||
|
||||
it('executes an explicitly selected dangerous bypass write in the isolated workspace', async () => {
|
||||
const sideEffect = 'bypass-side-effect'
|
||||
const command = `echo bypass>${sideEffect}`
|
||||
const commandCalls = [
|
||||
{
|
||||
name: 'exec_command',
|
||||
arguments: {
|
||||
cmd: command,
|
||||
const { harness, fixture } = await realHarness((workspace): readonly ResponsesBehavior[] => {
|
||||
const target = join(workspace, sideEffect)
|
||||
const command = process.platform === 'win32'
|
||||
? `powershell.exe -NoLogo -NoProfile -NonInteractive -Command "Set-Content -LiteralPath '${target.replaceAll("'", "''")}' -Value 'bypass' -NoNewline"`
|
||||
: `printf bypass > ${JSON.stringify(target)}`
|
||||
const commandCalls = [
|
||||
{
|
||||
name: 'exec_command',
|
||||
arguments: {
|
||||
cmd: command,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'shell_command',
|
||||
arguments: {
|
||||
command,
|
||||
{
|
||||
name: 'shell_command',
|
||||
arguments: {
|
||||
command,
|
||||
},
|
||||
},
|
||||
},
|
||||
] as const
|
||||
const { harness } = await realHarness([
|
||||
{ kind: 'advertisedFunctionCall', choices: commandCalls },
|
||||
{ kind: 'complete', text: 'bypass complete' },
|
||||
], 'dangerously-bypass-approvals-and-sandbox')
|
||||
] as const
|
||||
return [
|
||||
{ kind: 'advertisedFunctionCall', choices: commandCalls },
|
||||
{ kind: 'complete', text: 'bypass complete' },
|
||||
]
|
||||
}, 'dangerously-bypass-approvals-and-sandbox')
|
||||
const target = join(harness.workspace, sideEffect)
|
||||
const run = await harness.ctx.subagents.start('codex', {
|
||||
prompt: [{ type: 'text', text: 'Create the fixture side effect.' }],
|
||||
@@ -414,6 +421,7 @@ describe('real @openai/codex 0.147.0 product', () => {
|
||||
output: [{ type: 'text', text: 'bypass complete' }],
|
||||
stopReason: 'completed',
|
||||
})
|
||||
expect(existsSync(target), JSON.stringify(fixture.requests.at(-1)?.body.input)).toBe(true)
|
||||
expect(readFileSync(target, 'utf8').trim()).toBe('bypass')
|
||||
await run.dispose()
|
||||
await expectQuiescent(harness.handles)
|
||||
|
||||
@@ -129,7 +129,7 @@ describe('coverage partition coordinator', () => {
|
||||
const runCommand = vi.fn(async (command: CoverageCommand) => {
|
||||
await writeBlob(command)
|
||||
return command.label === 'partition 2/2'
|
||||
? { exitCode: 1, signalCode: null }
|
||||
? { exitCode: 1, signalCode: null, outputTail: 'specific Vitest failure' }
|
||||
: passed
|
||||
})
|
||||
const coordinator = new CoveragePartitionCoordinator({
|
||||
@@ -141,6 +141,9 @@ describe('coverage partition coordinator', () => {
|
||||
|
||||
await expect(coordinator.run()).resolves.toBe(1)
|
||||
expect(reported).toHaveBeenCalledWith('coverage-partitions: FAIL partition 2/2 (exit 1)')
|
||||
expect(reported).toHaveBeenCalledWith(
|
||||
'coverage-partitions: output tail for partition 2/2:\nspecific Vitest failure',
|
||||
)
|
||||
expect(runCommand).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
|
||||
|
||||
@@ -34,6 +34,8 @@ export interface CoverageCommandResult {
|
||||
signalCode: NodeJS.Signals | null
|
||||
/** Spawn failure recorded independently from process completion. */
|
||||
error?: string
|
||||
/** Bounded combined stdout/stderr tail repeated when the command fails. */
|
||||
outputTail?: string
|
||||
}
|
||||
|
||||
/** Execute one coordinator command with inherited output. */
|
||||
@@ -120,6 +122,9 @@ export class CoveragePartitionCoordinator {
|
||||
const result = await this.runCommand(command)
|
||||
if (commandFailed(result)) {
|
||||
console.error(`coverage-partitions: FAIL ${command.label} (${commandFailureReason(result)})`)
|
||||
if (result.outputTail !== undefined && result.outputTail !== '') {
|
||||
console.error(`coverage-partitions: output tail for ${command.label}:\n${result.outputTail}`)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}))
|
||||
@@ -202,6 +207,7 @@ export class CoveragePartitionCoordinator {
|
||||
/** Spawn one pnpm-backed command without a platform shell. */
|
||||
function runCoverageCommand(command: CoverageCommand): Promise<CoverageCommandResult> {
|
||||
return new Promise((resolveCommand) => {
|
||||
let outputTail = ''
|
||||
const env = { ...process.env }
|
||||
for (const [name, value] of Object.entries(command.env)) {
|
||||
if (value === undefined) Reflect.deleteProperty(env, name)
|
||||
@@ -210,17 +216,32 @@ function runCoverageCommand(command: CoverageCommand): Promise<CoverageCommandRe
|
||||
const child = spawn(process.execPath, command.args, {
|
||||
cwd: command.cwd,
|
||||
env,
|
||||
stdio: 'inherit',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
})
|
||||
child.stdout.setEncoding('utf8')
|
||||
child.stderr.setEncoding('utf8')
|
||||
child.stdout.on('data', (chunk: string) => {
|
||||
process.stdout.write(chunk)
|
||||
outputTail = appendOutputTail(outputTail, chunk)
|
||||
})
|
||||
child.stderr.on('data', (chunk: string) => {
|
||||
process.stderr.write(chunk)
|
||||
outputTail = appendOutputTail(outputTail, chunk)
|
||||
})
|
||||
child.once('error', (error: Error) => {
|
||||
resolveCommand({ exitCode: null, signalCode: null, error: error.message })
|
||||
resolveCommand({ exitCode: null, signalCode: null, error: error.message, outputTail })
|
||||
})
|
||||
child.once('exit', (exitCode, signalCode) => {
|
||||
resolveCommand({ exitCode, signalCode })
|
||||
child.once('close', (exitCode, signalCode) => {
|
||||
resolveCommand({ exitCode, signalCode, outputTail })
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function appendOutputTail(previous: string, chunk: string): string {
|
||||
const combined = previous + chunk
|
||||
return combined.length <= 65_536 ? combined : combined.slice(-65_536)
|
||||
}
|
||||
|
||||
function commandFailed(result: CoverageCommandResult): boolean {
|
||||
return result.exitCode !== 0 || result.signalCode !== null || result.error !== undefined
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user