mirror of
https://github.com/deepseek-ai/deepseek-harness.git
synced 2026-09-09 04:02:35 +00:00
test(session-snapshot): preserve historical retired fixture roles
This commit is contained in:
@@ -22,6 +22,7 @@ import { readFile, readdir, writeFile } from 'node:fs/promises'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { isSurfaceEligibleType } from '@deepseek-ai/dsh-session/surface'
|
||||
import { SESSION_FORMAT_VERSION } from '@deepseek-ai/dsh-session'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { type AgentUnderTest, type HarvestedLog, type InputScript, runScenario } from './harness.ts'
|
||||
import {
|
||||
@@ -1146,19 +1147,24 @@ export function stabilizeRefreshLog(
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the selected generation of every parent and child fixture against storage policy.
|
||||
* Check every selected role for tool/path defects and current generations for canonical prompt/identity storage.
|
||||
* Historical roles retain their released bytes, including roles retired by the current writer.
|
||||
* @param dir Scenario directory containing canonical Session fixtures.
|
||||
* @param scenarioName Scenario name used in failure diagnostics.
|
||||
* @returns Resolves when all selected fixtures satisfy the storage checks.
|
||||
*/
|
||||
export async function assertSessionFixtureStorage(dir: string, scenarioName: string): Promise<void> {
|
||||
const files = await sessionFixtures(dir)
|
||||
const currentFixtures: string[] = []
|
||||
for (const file of files) {
|
||||
const fixture = await readFile(join(dir, file), 'utf8')
|
||||
const version = assertSessionFixtureVersion(file, fixture)
|
||||
expect(unknownToolCallIds(fixture), `${scenarioName}/${file} contains UNKNOWN_TOOL`)
|
||||
.toEqual([])
|
||||
expect(fixture, `${scenarioName}/${file} carries a non-canonical macOS cwd token`)
|
||||
.not.toContain('/private{{cwd}}')
|
||||
if (version !== SESSION_FORMAT_VERSION) continue
|
||||
currentFixtures.push(fixture)
|
||||
expect(scrubSystemPrompts(fixture), `${scenarioName}/${file} carries an unscrubbed system prompt`)
|
||||
.toEqual(fixture)
|
||||
expect(scrubToolSchemas(fixture), `${scenarioName}/${file} carries unscrubbed tool schemas`)
|
||||
@@ -1166,8 +1172,7 @@ export async function assertSessionFixtureStorage(dir: string, scenarioName: str
|
||||
expect(systemPromptPrecedesRequests(fixture), `${scenarioName}/${file} has a request/header with no preceding system/message`)
|
||||
.toBe(true)
|
||||
}
|
||||
const fixtures = await Promise.all(files.map(file => readFile(join(dir, file), 'utf8')))
|
||||
expect(redactSessionSnapshotIds(fixtures), `${scenarioName}: identity redaction fixed point`).toEqual(fixtures)
|
||||
expect(redactSessionSnapshotIds(currentFixtures), `${scenarioName}: identity redaction fixed point`).toEqual(currentFixtures)
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Vendored
+6
@@ -0,0 +1,6 @@
|
||||
{"type":"session","version":3,"id":"{{session:2}}","createdAt":800,"cwd":"{{cwd}}","parentSession":"{{session:1}}","isSeeded":false,"delegationDepth":1}
|
||||
{"type":"turn/start","data":{"turn":1}}
|
||||
{"type":"step/start","data":{"turn":1,"step":1}}
|
||||
{"type":"system/message","data":{"turn":1,"step":1,"message":{"id":"{{message:1}}","role":"system","content":[{"type":"text","text":"{{system}}"}],"source":{"kind":"plugin","plugin":"@deepseek-ai/dsh-system-prompt"}}},"surfaceOp":"append"}
|
||||
{"type":"request/header","data":{"header":{"config":{"provider":"fake","model":"fake"},"tools":"{{tools}}"},"reason":"initial"}}
|
||||
{"type":"user/message","data":{"role":"user","content":[{"type":"text","text":"same inherited message"}],"source":{"kind":"user"},"id":"{{message:2}}"},"surfaceOp":"append"}
|
||||
Vendored
+6
@@ -0,0 +1,6 @@
|
||||
{"type":"session","version":3,"id":"{{session:1}}","createdAt":700,"cwd":"{{cwd}}","isSeeded":false,"delegationDepth":0}
|
||||
{"type":"turn/start","data":{"turn":1}}
|
||||
{"type":"step/start","data":{"turn":1,"step":1}}
|
||||
{"type":"system/message","data":{"turn":1,"step":1,"message":{"id":"{{message:1}}","role":"system","content":[{"type":"text","text":"{{system}}"}],"source":{"kind":"plugin","plugin":"@deepseek-ai/dsh-system-prompt"}}},"surfaceOp":"append"}
|
||||
{"type":"request/header","data":{"header":{"config":{"provider":"fake","model":"fake"},"tools":"{{tools}}"},"reason":"initial"}}
|
||||
{"type":"user/message","data":{"role":"user","content":[{"type":"text","text":"same inherited message"}],"source":{"kind":"user"},"id":"{{message:2}}"},"surfaceOp":"append"}
|
||||
@@ -1,7 +1,9 @@
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, expect, test } from 'vitest'
|
||||
import { SESSION_FORMAT_VERSION } from '@deepseek-ai/dsh-session'
|
||||
import { sessionFixtureName } from '../src/session-files.ts'
|
||||
import { assertSessionFixtureStorage } from '../src/suite.ts'
|
||||
|
||||
const roots: string[] = []
|
||||
@@ -13,28 +15,57 @@ const system = { type: 'system/message', data: { message: {
|
||||
role: 'system', content: [{ type: 'text', text: '{{system}}' }],
|
||||
} } }
|
||||
const request = { type: 'request/header', data: { header: { tools: '{{tools}}' } } }
|
||||
const historicalRequest = { type: 'request/header', data: { header: { system: '{{system}}', tools: '{{tools}}' } } }
|
||||
|
||||
function fixture(id: number, version: number, events: unknown[]): string {
|
||||
return [{ type: 'session', version, id: `{{session:${id}}}`, createdAt: 0, delegationDepth: id - 1 }, ...events]
|
||||
.map(record => JSON.stringify(record)).join('\n') + '\n'
|
||||
}
|
||||
|
||||
function registerGuard(childEvents: unknown[]): () => Promise<void> {
|
||||
function registerGuard(childEvents: unknown[], historicalEvents?: unknown[]): () => Promise<void> {
|
||||
const root = mkdtempSync(join(tmpdir(), 'snapshot-storage-policy-'))
|
||||
roots.push(root)
|
||||
const dir = join(root, 'pin')
|
||||
mkdirSync(dir)
|
||||
writeFileSync(join(dir, 'session.v2.jsonl'), fixture(1, 2, [system, request]))
|
||||
// Each role selects its highest generation; the older child deliberately violates the policy.
|
||||
writeFileSync(join(dir, sessionFixtureName(0, SESSION_FORMAT_VERSION)), fixture(1, SESSION_FORMAT_VERSION, [system, request]))
|
||||
// Each role selects its highest generation; the older child predates system/message.
|
||||
writeFileSync(join(dir, 'session.1.v1.jsonl'), fixture(2, 1, [request]))
|
||||
writeFileSync(join(dir, 'session.1.v2.jsonl'), fixture(2, 2, childEvents))
|
||||
return () => assertSessionFixtureStorage(dir, 'pin')
|
||||
writeFileSync(join(dir, sessionFixtureName(1, SESSION_FORMAT_VERSION)), fixture(2, SESSION_FORMAT_VERSION, childEvents))
|
||||
if (historicalEvents !== undefined) {
|
||||
writeFileSync(join(dir, 'session.2.jsonl'), [
|
||||
JSON.stringify({ type: 'session', id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', createdAt: 0 }),
|
||||
...historicalEvents.map(event => JSON.stringify(event)),
|
||||
'',
|
||||
].join('\n'))
|
||||
}
|
||||
const contents = (): string[] => readdirSync(dir).sort().map(file => readFileSync(join(dir, file), 'utf8'))
|
||||
const before = contents()
|
||||
return async () => {
|
||||
await assertSessionFixtureStorage(dir, 'pin')
|
||||
expect(contents()).toEqual(before)
|
||||
}
|
||||
}
|
||||
|
||||
test('checks only the highest generation of every parent and child role', async () => {
|
||||
await expect(registerGuard([system, request])()).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
test('preserves a selected historical child without imposing current prompt or identity storage', async () => {
|
||||
await expect(registerGuard([system, request], [historicalRequest])()).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
test.each([
|
||||
['unknown tool', { type: 'tool/result', data: { error: { code: 'UNKNOWN_TOOL' } } }, 'contains UNKNOWN_TOOL'],
|
||||
['noncanonical cwd', { type: 'user/message', data: { text: '/private{{cwd}}/file' } }, 'carries a non-canonical macOS cwd token'],
|
||||
] as const)('rejects a selected historical child with %s', async (_name, event, message) => {
|
||||
await expect(registerGuard([system, request], [historicalRequest, event])()).rejects.toThrow(`pin/session.2.jsonl ${message}`)
|
||||
})
|
||||
|
||||
test('rejects an unredacted identity in a selected current child beside historical residue', async () => {
|
||||
const unredacted = { ...system, data: { message: { ...system.data.message, id: 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb' } } }
|
||||
await expect(registerGuard([unredacted, request], [historicalRequest])()).rejects.toThrow('identity redaction fixed point')
|
||||
})
|
||||
|
||||
test.each([
|
||||
['missing prompt', [request], 'has a request/header with no preceding system/message'],
|
||||
['unscrubbed prompt', [{ ...system, data: { message: {
|
||||
@@ -42,5 +73,5 @@ test.each([
|
||||
} } }, request], 'carries an unscrubbed system prompt'],
|
||||
['unscrubbed tools', [system, { type: 'request/header', data: { header: { tools: [] } } }], 'carries unscrubbed tool schemas'],
|
||||
] as const)('rejects a selected child with %s', async (_name, events, message) => {
|
||||
await expect(registerGuard([...events])()).rejects.toThrow(`pin/session.1.v2.jsonl ${message}`)
|
||||
await expect(registerGuard([...events])()).rejects.toThrow(`pin/${sessionFixtureName(1, SESSION_FORMAT_VERSION)} ${message}`)
|
||||
})
|
||||
|
||||
@@ -128,13 +128,14 @@ const RECORD_SCENARIOS: Scenario[] = [
|
||||
// committed record fixtures and expected outputs in place.
|
||||
const BOOTSTRAP = process.env.ACP_SNAPSHOT_SPEC_BOOTSTRAP === '1'
|
||||
const recordDir = BOOTSTRAP ? RECORD_SRC : mkdtempSync(join(tmpdir(), 'acp-snap-record-suite-'))
|
||||
const retiredChildFixture = readFileSync(join(RECORD_SRC, 'rec-child', 'session.1.jsonl'), 'utf8')
|
||||
const retiredChildFixture = readFileSync(join(RECORD_SRC, 'rec-child', 'session.1.v3.jsonl'), 'utf8')
|
||||
if (!BOOTSTRAP) {
|
||||
cpSync(RECORD_SRC, recordDir, { recursive: true })
|
||||
// Record mode owns its output inventory: a new scenario has no primary yet,
|
||||
// while a changed child count can leave old numbered fixtures behind.
|
||||
rmSync(join(recordDir, 'rec-pin', 'session.jsonl'))
|
||||
writeFileSync(join(recordDir, 'rec-child', 'session.2.jsonl'), retiredChildFixture)
|
||||
rmSync(join(recordDir, 'rec-pin', 'session.v3.jsonl'))
|
||||
writeFileSync(join(recordDir, 'rec-child', 'session.2.v3.jsonl'), retiredChildFixture)
|
||||
}
|
||||
const refreshDir = mkdtempSync(join(tmpdir(), 'acp-snap-refresh-suite-'))
|
||||
cpSync(REPLAY_DIR, refreshDir, { recursive: true })
|
||||
@@ -244,7 +245,7 @@ describe('defineAcpSnapshotSuite: record inventory write-back', () => {
|
||||
expect(fixture).toContain('"type":"session"')
|
||||
expect(fixture).toContain('"cwd":"{{cwd}}"')
|
||||
if (!BOOTSTRAP) {
|
||||
expect(readFileSync(join(recordDir, 'rec-child', 'session.2.jsonl'), 'utf8')).toBe(retiredChildFixture)
|
||||
expect(readFileSync(join(recordDir, 'rec-child', 'session.2.v3.jsonl'), 'utf8')).toBe(retiredChildFixture)
|
||||
}
|
||||
expect(readFileSync(join(recordDir, 'rec-child', 'tool-schemas.1.expected.json'), 'utf8'))
|
||||
.toContain('"name": "t1"')
|
||||
|
||||
Reference in New Issue
Block a user