chore(session): retire the v1-to-v2 benchmark script

The repository-only benchmark, its unit-suite spec, the root script, and the
README section are removed; the accepted acceptance run's figures stay in the
embedded-assistant-streams note as recorded facts. The note also records why
the migration edge reuses the `dsh-llm` stream helpers instead of frozen
copies and why its target validation re-checks message/stream agreement
itself. A validation test that recomputed `RELEASED_V2_EVENT_TYPES` from the
same expression as its definition is dropped.
This commit is contained in:
Tianyi Cui
2026-09-03 17:50:09 +08:00
parent cf97e3e455
commit 51adfbf52c
10 changed files with 8 additions and 877 deletions
@@ -1,67 +0,0 @@
import { describe, expect, it } from 'vitest'
import {
ACCEPTANCE_DEFAULTS,
SMOKE_DEFAULTS,
parseOptions,
percentile,
} from './benchmark-session-format-v1-to-v2.ts'
describe('v2 performance acceptance options', () => {
it('pins the full acceptance and non-gating smoke specifications', () => {
expect(parseOptions([])).toEqual({
...ACCEPTANCE_DEFAULTS,
smoke: false,
help: false,
})
expect(parseOptions(['--smoke'])).toEqual({
...SMOKE_DEFAULTS,
smoke: true,
help: false,
})
})
it('accepts split and equals-form overrides independently of option order', () => {
expect(parseOptions([
'--samples=9',
'--smoke',
'--runs', '2',
'--warmups=0',
'--threshold-percent', '4',
])).toEqual({
runs: 2,
warmups: 0,
samples: 9,
thresholdPercent: 4,
smoke: true,
help: false,
})
})
it.each([
[['--unknown'], /unknown benchmark option/],
[['--runs'], /requires a numeric value/],
[['--runs', '0'], /runs must be positive/],
[['--samples', '-1'], /samples must be a non-negative safe integer/],
[['--threshold-percent', '1.5'], /threshold-percent must be a non-negative safe integer/],
] as const)('rejects invalid options %j', (arguments_, expected) => {
expect(() => parseOptions(arguments_)).toThrow(expected)
})
})
describe('v2 performance acceptance statistics', () => {
it('uses the PR3 discrete percentile estimator without mutating input', () => {
const values = [4, 1, 3, 2]
expect(percentile(values, 0)).toBe(1)
expect(percentile(values, 0.5)).toBe(3)
expect(percentile(values, 0.95)).toBe(4)
expect(percentile(values, 1)).toBe(4)
expect(values).toEqual([4, 1, 3, 2])
})
it('rejects empty samples and invalid fractions', () => {
expect(() => percentile([], 0.5)).toThrow(/at least one sample/)
expect(() => percentile([1], -0.1)).toThrow(/between zero and one/)
expect(() => percentile([1], 1.1)).toThrow(/between zero and one/)
expect(() => percentile([1], Number.NaN)).toThrow(/between zero and one/)
})
})
@@ -1,774 +0,0 @@
/**
* Manual performance acceptance for the released v2 Session format.
*
* Run the full gate from the repository root with:
*
* pnpm run benchmark:session-format-v1-to-v2
*
* Use `--smoke` for a short correctness and reporting pass. The smoke mode
* reports timing deltas but does not enforce the acceptance ceiling.
*/
import { deepStrictEqual, ok } from 'node:assert'
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { performance } from 'node:perf_hooks'
import { pathToFileURL } from 'node:url'
import { Context } from '@deepseek-ai/cordis'
import {
KNOWN_SESSION_EVENT_TYPES,
Session,
SessionId,
SessionLogOffset,
} from '@deepseek-ai/dsh-session'
import type {
SessionEvent,
SessionHeader,
SessionId as SessionIdType,
} from '@deepseek-ai/dsh-session'
import type { SessionPersistence } from '@deepseek-ai/dsh-session-persistence'
import {
snapshotSessionFormatArtifact,
} from '@deepseek-ai/dsh-session-format'
import type {
EncodedSessionFormatArtifact,
SessionFormatArtifact,
SessionFormatEvent,
SessionFormatJsonObject,
} from '@deepseek-ai/dsh-session-format'
import {
assertReleasedV1Artifact,
releasedV1SessionFormatCodec,
} from '@deepseek-ai/dsh-session-format-v0-to-v1'
import {
assertReleasedV2Artifact,
releasedV2SessionFormatCodec,
restoreReleasedV2Artifact,
sessionFormatV1ToV2,
} from '../packages/session/session-format-v1-to-v2/src/index.ts'
import {
sessionFormatCatalog,
} from '../packages/session/session-format-catalog/src/generated.ts'
import {
validateInstalledCurrentSessionArtifact,
} from '../packages/session/session-format-catalog/src/current.ts'
import {
compressZstdFrame,
createZstdFrameDecoder,
scanZstdFrames,
} from '../packages/session/session-persistence-jsonl/src/zstd.ts'
import JsonlSessionPersistence from '@deepseek-ai/dsh-session-persistence-jsonl'
import {
generationLogPath,
type JsonlCompression,
} from '../packages/session/session-persistence-jsonl/src/format.ts'
import SessionProjectionRegistry from '@deepseek-ai/dsh-session-projection'
import TokenMeter from '@deepseek-ai/dsh-token-meter'
/** Reproducible full-acceptance sampling specification. */
export const ACCEPTANCE_DEFAULTS = Object.freeze({
runs: 3,
warmups: 100,
samples: 600,
thresholdPercent: 5,
})
/** Short diagnostic sampling specification; it is not an acceptance result. */
export const SMOKE_DEFAULTS = Object.freeze({
runs: 1,
warmups: 3,
samples: 20,
thresholdPercent: 5,
})
interface BenchmarkOptions {
readonly runs: number
readonly warmups: number
readonly samples: number
readonly thresholdPercent: number
readonly smoke: boolean
readonly help: boolean
}
interface PhysicalInput {
readonly header: unknown
readonly rows: readonly unknown[]
}
interface PhysicalFixture {
readonly raw: Buffer
readonly zstd: Buffer
}
interface Fixture {
readonly name: 'small' | '100-turn'
readonly turns: number
readonly v1: SessionFormatArtifact
readonly v2: SessionFormatArtifact
readonly v1Physical: PhysicalFixture
readonly v2Physical: PhysicalFixture
}
interface CurrentReadCase {
readonly label: string
readonly root: string
readonly id: SessionIdType
readonly compression: JsonlCompression
readonly physical: PhysicalInput
readonly expected: SessionFormatArtifact
}
interface Distribution {
readonly medianUs: number
readonly p95Us: number
}
interface PairedResult {
readonly direct: Distribution
readonly catalog: Distribution
readonly medianRegressionPercent: number
readonly p95RegressionPercent: number
readonly passed: boolean
}
type StoredRead = SessionFormatArtifact
interface EventExtras {
readonly surfaceOp?: SessionFormatEvent['surfaceOp']
readonly sourceEventSeqs?: SessionFormatEvent['sourceEventSeqs']
}
let resultSink = 0
/** Parse benchmark-only CLI options without mutating process-global state. */
export function parseOptions(argv: readonly string[]): BenchmarkOptions {
const smoke = argv.includes('--smoke')
const defaults = smoke ? SMOKE_DEFAULTS : ACCEPTANCE_DEFAULTS
const values: {
runs: number
warmups: number
samples: number
thresholdPercent: number
} = { ...defaults }
let help = false
for (let index = 0; index < argv.length; index += 1) {
const argument = argv[index] as string
if (argument === '--smoke') continue
if (argument === '--help' || argument === '-h') {
help = true
continue
}
const match = /^--(runs|warmups|samples|threshold-percent)(?:=(.+))?$/.exec(argument)
if (match === null) throw new Error(`unknown benchmark option ${JSON.stringify(argument)}`)
const name = match[1]
if (name === undefined) throw new Error(`unknown benchmark option ${JSON.stringify(argument)}`)
const raw = match[2] ?? argv[index + 1]
if (raw === undefined || (match[2] === undefined && raw.startsWith('--'))) {
throw new Error(`${name} requires a numeric value`)
}
if (match[2] === undefined) index += 1
const value = Number(raw)
if (!Number.isSafeInteger(value) || value < 0) {
throw new Error(`${name} must be a non-negative safe integer`)
}
if (name !== 'warmups' && value === 0) throw new Error(`${name} must be positive`)
switch (name) {
case 'runs': values.runs = value; break
case 'warmups': values.warmups = value; break
case 'samples': values.samples = value; break
case 'threshold-percent': values.thresholdPercent = value; break
}
}
return { ...values, smoke, help }
}
/** Calculate the discrete percentile used by the acceptance measurement. */
export function percentile(values: readonly number[], fraction: number): number {
if (values.length === 0) throw new Error('percentile requires at least one sample')
if (!Number.isFinite(fraction) || fraction < 0 || fraction > 1) {
throw new Error('percentile fraction must be between zero and one')
}
const sorted = [...values].sort((left, right) => left - right)
return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * fraction))] as number
}
async function main(): Promise<void> {
const options = parseOptions(process.argv.slice(2))
if (options.help) {
printHelp()
return
}
selfCheckStatistics()
const root = await mkdtemp(join(tmpdir(), 'dsh-v2-format-acceptance-'))
try {
const fixtures = await Promise.all([
createFixture('small', 1),
createFixture('100-turn', 100),
])
const cases = await materializeCurrentReadCases(root, fixtures)
await validateFixtureReads(cases)
console.log('Session format v2 catalog-dispatch overhead acceptance')
console.log(
'Direct-current/no-dispatch baseline: the released-v2 codec restores the same parsed physical rows. '
+ 'The candidate routes those rows through the static format catalog; public handles validate each file once.',
)
console.log(
`Sampling: ${options.runs} run(s), ${options.warmups} warmup pair(s), `
+ `${options.samples} alternating measured pair(s), ${options.thresholdPercent}% median/p95 ceiling.`,
)
if (options.smoke) console.log('Mode: diagnostic smoke; timing ceiling is reported but not enforced.')
console.log('')
reportFixtureSizes(fixtures)
console.log('')
let accepted = true
console.log('Released-v2 restoration dispatch overhead (pooled hot samples)')
for (const benchmarkCase of cases) {
const result = runPairedCurrentRead(benchmarkCase, options)
accepted &&= result.passed
printPairedResult(benchmarkCase.label, result)
}
console.log('')
console.log('Absolute costs (informational; no speedup claim)')
for (const fixture of fixtures) {
const migration = runDistribution(
() => { consumeArtifact(sessionFormatV1ToV2.migrate(fixture.v1)) },
options,
)
printDistribution(`${fixture.name} v1->v2 migration`, migration)
const session = restoredSession(fixture.v2)
const tokenMeter = runDistribution(
() => { consumeMeasurement(measureWithFreshTokenMeter(session)) },
options,
)
printDistribution(`${fixture.name} TokenMeter cold replay+measure`, tokenMeter)
}
console.log('')
await reportMemory(fixtures, cases)
const memory = process.memoryUsage()
console.log(
`process memory after run: heapUsed=${formatBytes(memory.heapUsed)}, `
+ `heapTotal=${formatBytes(memory.heapTotal)}, rss=${formatBytes(memory.rss)}`,
)
if (options.smoke) {
console.log('')
console.log('SMOKE COMPLETE (non-acceptance timing sample)')
} else if (accepted) {
console.log('')
console.log('PASS: every pooled catalog-dispatch median and p95 regression is within the 5% ceiling.')
} else {
console.error('')
console.error('FAIL: at least one pooled catalog-dispatch median or p95 regression exceeds the 5% ceiling.')
process.exitCode = 1
}
} finally {
await rm(root, { recursive: true, force: true })
}
}
function printHelp(): void {
console.log(`Usage: node --expose-gc --import tsx/esm ${process.argv[1] ?? '<benchmark>'} [options]
Options:
--smoke Use 1 run, 3 warmups, and 20 samples; do not enforce timing.
--runs N Independent runs (acceptance default: 3).
--warmups N Alternating warmup pairs per run (acceptance default: 100).
--samples N Alternating measured pairs per run (acceptance default: 600).
--threshold-percent N Median and p95 regression ceiling (acceptance default: 5).
--help Show this help.`)
}
function selfCheckStatistics(): void {
deepStrictEqual(parseOptions([]), { ...ACCEPTANCE_DEFAULTS, smoke: false, help: false })
deepStrictEqual(parseOptions(['--smoke']), { ...SMOKE_DEFAULTS, smoke: true, help: false })
deepStrictEqual(percentile([4, 1, 3, 2], 0.5), 3)
deepStrictEqual(percentile([1, 2, 3, 4, 5], 0.95), 5)
}
async function createFixture(name: Fixture['name'], turns: number): Promise<Fixture> {
const v1 = buildV1Artifact(name, turns)
assertReleasedV1Artifact(v1)
const v2 = sessionFormatV1ToV2.migrate(v1)
assertReleasedV2Artifact(v2)
validateInstalledCurrentSessionArtifact(v2)
ok(v2.events.some(event => event.type === 'assistant/message'
&& Array.isArray((event.data as SessionFormatJsonObject)['stream'])
&& ((event.data as SessionFormatJsonObject)['stream'] as readonly unknown[]).length > 0))
const v1Physical = await encodePhysical(releasedV1SessionFormatCodec.encodeArtifact(v1, { packChunks: true }))
const v2Physical = await encodePhysical(releasedV2SessionFormatCodec.encodeArtifact(v2))
const v1Decoded = releasedV1SessionFormatCodec.decodeArtifact(...physicalArguments(parseRaw(v1Physical.raw)))
const v2Decoded = releasedV2SessionFormatCodec.decodeArtifact(...physicalArguments(parseRaw(v2Physical.raw)))
deepStrictEqual(v1Decoded, v1)
deepStrictEqual(v2Decoded, v2)
deepStrictEqual(
releasedV1SessionFormatCodec.decodeArtifact(...physicalArguments(parseZstd(v1Physical.zstd))),
v1,
)
deepStrictEqual(
releasedV2SessionFormatCodec.decodeArtifact(...physicalArguments(parseZstd(v2Physical.zstd))),
v2,
)
return { name, turns, v1, v2, v1Physical, v2Physical }
}
function buildV1Artifact(name: string, turns: number): SessionFormatArtifact {
const events: SessionFormatEvent[] = []
const append = (
type: string,
time: number,
data: SessionFormatEvent['data'],
extras: EventExtras = {},
): number => {
const seq = events.length
events.push({ type, seq, time, data, ...extras })
return seq
}
for (let turn = 1; turn <= turns; turn += 1) {
const baseTime = turn * 1_000
append('turn/start', baseTime, { turn })
append('user/message', baseTime + 1, {
id: `user-${name}-${turn}`,
role: 'user',
content: [{
type: 'text',
text: `Turn ${turn}: inspect the deterministic workspace report and explain the relevant changes clearly.`,
}],
source: { kind: 'user' },
}, { surfaceOp: 'append' })
append('step/start', baseTime + 2, { turn, step: 1 })
append('request/header', baseTime + 3, {
header: {
config: { provider: 'mock', model: 'benchmark-model', maxTokens: 2_048 },
system: 'Answer with a concise explanation grounded in the supplied report.',
tools: [],
},
reason: 'initial',
})
const reasoningParts = [
`Turn ${turn} establishes the requested scope. `,
'The durable facts are checked against the current session state. ',
'The response keeps the relevant behavior and omits unrelated details. ',
'The final wording records the observable result directly.',
]
const textParts = [
`For turn ${turn}, the report confirms the requested behavior. `,
'The current state is internally consistent, ',
'the persisted events retain their required ordering, ',
'and the replayed message matches the provider stream. ',
'No unrelated setting changes are included. ',
'The result remains deterministic across repeated reads. ',
'The validation path checks the complete artifact. ',
'This completes the requested analysis.',
]
const usage = {
inputTokens: 800 + turn * 8,
outputTokens: 160,
totalTokens: 1_080 + turn * 8,
cacheReadTokens: 120,
cacheWriteTokens: 0,
reasoningTokens: 56,
}
const chunkSeqs: number[] = []
chunkSeqs.push(append('assistant/chunk', baseTime + 10, {
turn, step: 1, chunk: { type: 'block-start', index: 0, blockType: 'reasoning' },
}))
for (const [index, text] of reasoningParts.entries()) {
chunkSeqs.push(append('assistant/chunk', baseTime + 11 + index, {
turn, step: 1, chunk: { type: 'reasoning-delta', index: 0, text },
}))
}
chunkSeqs.push(append('assistant/chunk', baseTime + 15, {
turn,
step: 1,
chunk: {
type: 'block-end',
index: 0,
block: { type: 'reasoning', text: reasoningParts.join('') },
},
}))
chunkSeqs.push(append('assistant/chunk', baseTime + 16, {
turn, step: 1, chunk: { type: 'block-start', index: 1, blockType: 'text' },
}))
for (const [index, text] of textParts.entries()) {
chunkSeqs.push(append('assistant/chunk', baseTime + 17 + index, {
turn, step: 1, chunk: { type: 'text-delta', index: 1, text },
}))
}
chunkSeqs.push(append('assistant/chunk', baseTime + 25, {
turn,
step: 1,
chunk: { type: 'block-end', index: 1, block: { type: 'text', text: textParts.join('') } },
}))
chunkSeqs.push(append('assistant/chunk', baseTime + 26, {
turn, step: 1, chunk: { type: 'usage', usage },
}))
chunkSeqs.push(append('assistant/chunk', baseTime + 27, {
turn, step: 1, chunk: { type: 'finish', reason: { kind: 'stop' } },
}))
append('assistant/message', baseTime + 28, {
turn,
step: 1,
message: {
id: `assistant-${name}-${turn}`,
role: 'assistant',
content: [
{ type: 'reasoning', text: reasoningParts.join('') },
{ type: 'text', text: textParts.join('') },
],
source: { kind: 'model', provider: 'mock', model: 'benchmark-model' },
},
usage,
}, { surfaceOp: 'append', sourceEventSeqs: chunkSeqs })
append('step/end', baseTime + 29, { turn, step: 1 })
append('turn/end', baseTime + 30, { turn, reason: { kind: 'completed' } })
}
return snapshotSessionFormatArtifact({
header: {
version: 1,
id: `v2-performance-${name}`,
createdAt: 1,
cwd: '/benchmark',
isSeeded: false,
delegationDepth: 0,
agentPreset: 'benchmark',
},
inheritedEventCount: 0,
events,
}, `${name} v1 benchmark artifact`)
}
async function encodePhysical(encoded: EncodedSessionFormatArtifact): Promise<PhysicalFixture> {
const headerLine = `${JSON.stringify(encoded.header)}\n`
const body = `${encoded.rows.map(row => JSON.stringify(row)).join('\n')}\n`
const raw = Buffer.from(headerLine + body)
const zstd = Buffer.concat([
await compressZstdFrame(headerLine),
await compressZstdFrame(body),
])
return { raw, zstd }
}
async function materializeCurrentReadCases(
root: string,
fixtures: readonly Fixture[],
): Promise<readonly CurrentReadCase[]> {
const cases: CurrentReadCase[] = []
for (const fixture of fixtures) {
for (const compression of ['none', 'zstd'] as const) {
const caseRoot = join(root, `${fixture.name}-${compression}`)
const id = SessionId(fixture.v2.header.id)
const path = generationLogPath(caseRoot, fixture.v2.header.cwd, id, 2, compression)
await mkdir(dirname(path), { recursive: true })
await writeFile(path, compression === 'none' ? fixture.v2Physical.raw : fixture.v2Physical.zstd)
cases.push({
label: `${compression === 'none' ? 'raw' : 'Zstandard'} ${fixture.name}`,
root: caseRoot,
id,
compression,
physical: compression === 'none'
? parseRaw(fixture.v2Physical.raw)
: parseZstd(fixture.v2Physical.zstd),
expected: fixture.v2,
})
}
}
return cases
}
async function validateFixtureReads(cases: readonly CurrentReadCase[]): Promise<void> {
for (const benchmarkCase of cases) {
const mounted = await mountBackend(benchmarkCase)
try {
const handle = await mounted.persistence.open(benchmarkCase.id, 'read')
try {
deepStrictEqual(handle.header, benchmarkCase.expected.header)
deepStrictEqual(handle.inheritedEventCount, benchmarkCase.expected.inheritedEventCount)
deepStrictEqual(await handle.read(), benchmarkCase.expected.events)
} finally {
await handle.close()
}
const direct = directCurrentRead(benchmarkCase)
const catalog = catalogCurrentRead(benchmarkCase)
deepStrictEqual(direct, benchmarkCase.expected)
deepStrictEqual(catalog, direct)
} finally {
await mounted.dispose()
}
}
}
async function mountBackend(benchmarkCase: CurrentReadCase): Promise<{
readonly persistence: SessionPersistence
readonly dispose: () => Promise<void>
}> {
const context = new Context()
await context.plugin(JsonlSessionPersistence, {
root: benchmarkCase.root,
compression: benchmarkCase.compression,
})
return {
persistence: context.sessionPersistence,
dispose: async () => context.fiber.dispose(),
}
}
function directCurrentRead(benchmarkCase: CurrentReadCase): StoredRead {
const decoded = snapshotSessionFormatArtifact(
releasedV2SessionFormatCodec.decodeArtifact(
benchmarkCase.physical.header,
benchmarkCase.physical.rows,
),
'direct released-v2 decoded artifact',
)
const source = snapshotSessionFormatArtifact(decoded, 'direct released-v2 source')
const restored = restoreReleasedV2Artifact(source, KNOWN_SESSION_EVENT_TYPES)
validateInstalledCurrentSessionArtifact(restored)
return snapshotSessionFormatArtifact(restored, 'direct current Session restoration')
}
function catalogCurrentRead(benchmarkCase: CurrentReadCase): StoredRead {
return sessionFormatCatalog.migrate(sessionFormatCatalog.decodeArtifact(
benchmarkCase.physical.header,
benchmarkCase.physical.rows,
))
}
function parseRaw(bytes: Buffer): PhysicalInput {
return parseJsonl(bytes.toString('utf8'))
}
function parseZstd(bytes: Buffer): PhysicalInput {
const scan = scanZstdFrames(bytes)
if (scan.tornStart !== undefined || scan.frames.length !== 2) {
throw new Error('benchmark Zstandard artifact must contain two complete frames')
}
const decoder = createZstdFrameDecoder()
let text = ''
for (const plaintext of decoder.decode(bytes, scan.frames)) text += plaintext.toString('utf8')
return parseJsonl(text)
}
function parseJsonl(text: string): PhysicalInput {
if (!text.endsWith('\n')) throw new Error('benchmark artifact lacks its final newline')
const lines = text.slice(0, -1).split('\n')
const header = JSON.parse(lines[0] as string) as unknown
const rows = lines.slice(1).map(line => JSON.parse(line) as unknown)
return { header, rows }
}
function physicalArguments(input: PhysicalInput): [unknown, readonly unknown[]] {
return [input.header, input.rows]
}
function runPairedCurrentRead(
benchmarkCase: CurrentReadCase,
options: BenchmarkOptions,
): PairedResult {
const pooledDirect: number[] = []
const pooledCatalog: number[] = []
for (let run = 0; run < options.runs; run += 1) {
forceGc()
for (let warmup = 0; warmup < options.warmups; warmup += 1) {
if ((warmup + run) % 2 === 0) {
consumeStored(directCurrentRead(benchmarkCase))
consumeStored(catalogCurrentRead(benchmarkCase))
} else {
consumeStored(catalogCurrentRead(benchmarkCase))
consumeStored(directCurrentRead(benchmarkCase))
}
}
const direct: number[] = []
const catalog: number[] = []
for (let sample = 0; sample < options.samples; sample += 1) {
if ((sample + run) % 2 === 0) {
direct.push(timedUs(() => { consumeStored(directCurrentRead(benchmarkCase)) }))
catalog.push(timedUs(() => { consumeStored(catalogCurrentRead(benchmarkCase)) }))
} else {
catalog.push(timedUs(() => { consumeStored(catalogCurrentRead(benchmarkCase)) }))
direct.push(timedUs(() => { consumeStored(directCurrentRead(benchmarkCase)) }))
}
}
pooledDirect.push(...direct)
pooledCatalog.push(...catalog)
const directRun = distribution(direct)
const catalogRun = distribution(catalog)
console.log(
` ${benchmarkCase.label} run ${run + 1}: direct ${formatDistribution(directRun)}; `
+ `catalog ${formatDistribution(catalogRun)}`,
)
}
const direct = distribution(pooledDirect)
const catalog = distribution(pooledCatalog)
const medianRegressionPercent = percentChange(catalog.medianUs, direct.medianUs)
const p95RegressionPercent = percentChange(catalog.p95Us, direct.p95Us)
return {
direct,
catalog,
medianRegressionPercent,
p95RegressionPercent,
passed: medianRegressionPercent <= options.thresholdPercent
&& p95RegressionPercent <= options.thresholdPercent,
}
}
function runDistribution(operation: () => void, options: BenchmarkOptions): Distribution {
const samples: number[] = []
for (let run = 0; run < options.runs; run += 1) {
forceGc()
for (let warmup = 0; warmup < options.warmups; warmup += 1) operation()
for (let sample = 0; sample < options.samples; sample += 1) samples.push(timedUs(operation))
}
return distribution(samples)
}
function distribution(samples: readonly number[]): Distribution {
return { medianUs: percentile(samples, 0.5), p95Us: percentile(samples, 0.95) }
}
function timedUs(operation: () => void): number {
const started = performance.now()
operation()
return (performance.now() - started) * 1_000
}
function consumeArtifact(artifact: SessionFormatArtifact): void {
resultSink = (resultSink + artifact.events.length + artifact.header.version) | 0
}
function consumeStored(stored: StoredRead): void {
resultSink = (resultSink + stored.events.length + stored.header.version) | 0
}
function consumeMeasurement(measurement: ReturnType<TokenMeter['measure']>): void {
resultSink = (resultSink + measurement.totalTokens + measurement.nodes.length) | 0
}
function restoredSession(artifact: SessionFormatArtifact): Session {
return Session.fromRestore(
SessionId(artifact.header.id),
artifact.events as SessionEvent[],
artifact.header as unknown as SessionHeader,
SessionLogOffset(artifact.inheritedEventCount),
)
}
function measureWithFreshTokenMeter(session: Session): ReturnType<TokenMeter['measure']> {
const context = new Context()
new SessionProjectionRegistry(context)
const meter = new TokenMeter(context)
return meter.measure(session)
}
function reportFixtureSizes(fixtures: readonly Fixture[]): void {
console.log('Logical and physical representation')
for (const fixture of fixtures) {
const eventReduction = reductionPercent(fixture.v2.events.length, fixture.v1.events.length)
const rawReduction = reductionPercent(fixture.v2Physical.raw.length, fixture.v1Physical.raw.length)
const zstdReduction = reductionPercent(fixture.v2Physical.zstd.length, fixture.v1Physical.zstd.length)
console.log(
` ${fixture.name} (${fixture.turns} turn(s)): logical events ${fixture.v1.events.length} -> ${fixture.v2.events.length} `
+ `(${eventReduction.toFixed(3)}% fewer); raw ${formatBytes(fixture.v1Physical.raw.length)} -> `
+ `${formatBytes(fixture.v2Physical.raw.length)} (change ${formatSigned(-rawReduction)}%); `
+ `Zstandard ${formatBytes(fixture.v1Physical.zstd.length)} -> ${formatBytes(fixture.v2Physical.zstd.length)} `
+ `(change ${formatSigned(-zstdReduction)}%).`,
)
}
}
function printPairedResult(label: string, result: PairedResult): void {
console.log(
` ${label} pooled: direct ${formatDistribution(result.direct)}; catalog ${formatDistribution(result.catalog)}; `
+ `regression median=${formatSigned(result.medianRegressionPercent)}%, `
+ `p95=${formatSigned(result.p95RegressionPercent)}% [${result.passed ? 'within ceiling' : 'exceeds ceiling'}]`,
)
}
function printDistribution(label: string, result: Distribution): void {
console.log(` ${label}: ${formatDistribution(result)}`)
}
async function reportMemory(
fixtures: readonly Fixture[],
cases: readonly CurrentReadCase[],
): Promise<void> {
console.log('Retained heap (informational; one forced-GC observation)')
if (globalThis.gc === undefined) {
console.log(' unavailable: rerun with --expose-gc for retained-heap observations')
return
}
for (const fixture of fixtures) {
const rawCase = cases.find(candidate => candidate.label === `raw ${fixture.name}`) as CurrentReadCase
const direct = await retainedHeap(() => directCurrentRead(rawCase))
const catalog = await retainedHeap(() => catalogCurrentRead(rawCase))
const migration = await retainedHeap(() => sessionFormatV1ToV2.migrate(fixture.v1))
const session = restoredSession(fixture.v2)
const tokenMeter = await retainedHeap(() => measureWithFreshTokenMeter(session))
console.log(
` ${fixture.name}: direct current=${formatSignedBytes(direct)}, `
+ `catalog current=${formatSignedBytes(catalog)}, migration=${formatSignedBytes(migration)}, `
+ `TokenMeter=${formatSignedBytes(tokenMeter)}`,
)
}
}
async function retainedHeap(operation: () => object | Promise<object>): Promise<number> {
forceGc()
const before = process.memoryUsage().heapUsed
let retained: object | undefined = await operation()
forceGc()
const after = process.memoryUsage().heapUsed
resultSink = (resultSink + Object.keys(retained).length) | 0
retained = undefined
forceGc()
return after - before
}
function forceGc(): void {
globalThis.gc?.()
}
function percentChange(value: number, baseline: number): number {
return (value / baseline - 1) * 100
}
function reductionPercent(value: number, baseline: number): number {
return (1 - value / baseline) * 100
}
function formatDistribution(value: Distribution): string {
return `median=${value.medianUs.toFixed(3)} us p95=${value.p95Us.toFixed(3)} us`
}
function formatSigned(value: number): string {
return `${value >= 0 ? '+' : ''}${value.toFixed(3)}`
}
function formatBytes(value: number): string {
if (Math.abs(value) < 1_024) return `${value} B`
if (Math.abs(value) < 1_024 * 1_024) return `${(value / 1_024).toFixed(2)} KiB`
return `${(value / (1_024 * 1_024)).toFixed(2)} MiB`
}
function formatSignedBytes(value: number): string {
return `${value >= 0 ? '+' : '-'}${formatBytes(Math.abs(value))}`
}
if (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error: unknown) => {
console.error(error instanceof Error ? error.stack ?? error.message : String(error))
process.exitCode = 1
})
}