Files
deepseek-harness/scripts/benchmark-npm-resolution.ts
T

552 lines
21 KiB
TypeScript

/** Benchmark npm's dependency-tree resolution against an all-local registry. */
import { execFileSync, spawn, spawnSync, type ChildProcess } from 'node:child_process'
import { globSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { createServer, type Server } from 'node:http'
import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { performance } from 'node:perf_hooks'
import { parseArgs } from 'node:util'
const TARGET_PACKAGE = '@deepseek-ai/dsh'
const DEFAULT_TIMEOUT_MS = 300_000
const TERMINATION_GRACE_MS = 1_000
const FORCED_EXIT_TIMEOUT_MS = 5_000
const WORKSPACE_MANIFEST_GLOBS = [
'apps/*/package.json',
'packages/*/*/package.json',
'vendor/*/package.json',
'native/landlock-run/package.json',
'native/landlock-run/packages/*/package.json',
]
const INSTALLED_MANIFEST_GLOBS = [
'node_modules/.pnpm/*/node_modules/*/package.json',
'node_modules/.pnpm/*/node_modules/@*/*/package.json',
]
const PUBLISHED_FIELDS = [
'dependencies',
'optionalDependencies',
'peerDependencies',
'peerDependenciesMeta',
'engines',
'os',
'cpu',
'bin',
] as const
interface PackageManifest {
readonly name?: unknown
readonly version?: unknown
readonly dependencies?: Record<string, string>
readonly optionalDependencies?: Record<string, string>
readonly peerDependencies?: Record<string, string>
readonly peerDependenciesMeta?: Record<string, unknown>
readonly engines?: unknown
readonly os?: unknown
readonly cpu?: unknown
readonly bin?: unknown
}
interface RegistryVersion extends PackageManifest {
readonly name: string
readonly version: string
}
/** Package versions served by the local benchmark registry. */
export type RegistryIndex = ReadonlyMap<string, ReadonlyMap<string, RegistryVersion>>
/** Parsed command-line options for one benchmark invocation. */
export interface BenchmarkOptions {
readonly ref?: string
readonly runs: number
readonly timeoutMs: number
readonly maxMs?: number
}
/** One measured npm resolution. */
export interface BenchmarkRun {
readonly durationMs: number
readonly registryRequests: number
readonly archiveRequests: number
readonly unknownPackages: readonly string[]
}
/** Published-package fields retained in npm's package-lock layout. */
export interface NpmLockPackage {
readonly name?: string
readonly version?: string
readonly dependencies?: Readonly<Record<string, string>>
readonly optionalDependencies?: Readonly<Record<string, string>>
readonly peerDependencies?: Readonly<Record<string, string>>
readonly peerDependenciesMeta?: Readonly<Record<string, { readonly optional?: boolean }>>
}
/** The installed paths selected by npm without materializing package archives. */
export interface NpmPackageLock {
readonly lockfileVersion: number
readonly packages: Readonly<Record<string, NpmLockPackage>>
}
/** npm resolution observations together with its computed install layout. */
export interface NpmPackageLockResolution extends BenchmarkRun {
readonly packageLock: NpmPackageLock
}
/** Parse one positive-integer command-line option or use its default. */
export function parsePositiveIntegerOption(raw: string | undefined, fallback: number, name: string): number {
if (raw === undefined) return fallback
const value = Number.parseInt(raw, 10)
if (!Number.isSafeInteger(value) || value < 1 || String(value) !== raw) {
throw new Error(`${name} must be a positive integer, got ${JSON.stringify(raw)}`)
}
return value
}
/**
* Parse supported benchmark arguments.
* @param args - Command-line arguments after the script path.
* @returns Validated benchmark options.
*/
export function parseBenchmarkOptions(args: readonly string[]): BenchmarkOptions {
const normalized = args[0] === '--' ? args.slice(1) : args
const { values } = parseArgs({
args: [...normalized],
options: {
ref: { type: 'string' },
runs: { type: 'string' },
'timeout-ms': { type: 'string' },
'max-ms': { type: 'string' },
},
allowPositionals: false,
})
const maxMs = values['max-ms'] === undefined
? undefined
: parsePositiveIntegerOption(values['max-ms'], 0, '--max-ms')
return {
runs: parsePositiveIntegerOption(values.runs, 1, '--runs'),
timeoutMs: parsePositiveIntegerOption(values['timeout-ms'], DEFAULT_TIMEOUT_MS, '--timeout-ms'),
...(values.ref === undefined ? {} : { ref: values.ref }),
...(maxMs === undefined ? {} : { maxMs }),
}
}
function workspaceManifestPath(path: string): boolean {
return /^(?:apps\/[^/]+|packages\/[^/]+\/[^/]+|vendor\/[^/]+|native\/landlock-run(?:\/packages\/[^/]+)?)\/package\.json$/.test(path)
}
function workspaceManifestPaths(root: string, ref: string | undefined): string[] {
if (ref === undefined) return globSync(WORKSPACE_MANIFEST_GLOBS, { cwd: root }).sort()
return execFileSync('git', ['ls-tree', '-r', '--name-only', ref, '--', 'apps', 'packages', 'vendor', 'native'], {
cwd: root,
encoding: 'utf8',
}).split('\n').filter(workspaceManifestPath).sort()
}
function readGitFiles(root: string, ref: string, paths: readonly string[]): ReadonlyMap<string, string> {
const output = execFileSync('git', ['cat-file', '--batch'], {
cwd: root,
input: paths.map(path => `${ref}:${path}\n`).join(''),
maxBuffer: 64 * 1024 * 1024,
})
const contents = new Map<string, string>()
let offset = 0
for (const path of paths) {
const headerEnd = output.indexOf(0x0a, offset)
if (headerEnd < 0) throw new Error(`git cat-file returned no header for ${ref}:${path}`)
const header = output.subarray(offset, headerEnd).toString('utf8')
if (header.endsWith(' missing')) throw new Error(`git ref ${ref} has no ${path}`)
const size = Number.parseInt(header.split(' ')[2] ?? '', 10)
if (!Number.isSafeInteger(size) || size < 0) {
throw new Error(`git cat-file returned an invalid size for ${ref}:${path}`)
}
const contentStart = headerEnd + 1
const contentEnd = contentStart + size
if (output[contentEnd] !== 0x0a) throw new Error(`git cat-file truncated ${ref}:${path}`)
contents.set(path, output.subarray(contentStart, contentEnd).toString('utf8'))
offset = contentEnd + 1
}
return contents
}
/**
* Convert a workspace protocol range to the range published by pnpm pack.
* @param range - Dependency range from a workspace manifest.
* @param targetVersion - Current version of the referenced workspace package.
* @returns The registry-facing semver range.
*/
export function publishWorkspaceRange(range: string, targetVersion: string): string {
if (range === 'workspace:*') return targetVersion
if (range === 'workspace:^') return `^${targetVersion}`
if (range === 'workspace:~') return `~${targetVersion}`
if (range.startsWith('workspace:')) return range.slice('workspace:'.length)
return range
}
function copyPublishedManifest(
source: PackageManifest,
workspaceVersions: ReadonlyMap<string, string>,
): RegistryVersion | undefined {
if (typeof source.name !== 'string' || typeof source.version !== 'string') return undefined
const output: Record<string, unknown> = { name: source.name, version: source.version }
for (const field of PUBLISHED_FIELDS) {
const value = source[field]
if (value === undefined) continue
if (field === 'dependencies' || field === 'optionalDependencies' || field === 'peerDependencies') {
output[field] = Object.fromEntries(Object.entries(value as Record<string, string>).map(([name, range]) => {
const targetVersion = workspaceVersions.get(name)
return [name, targetVersion === undefined ? range : publishWorkspaceRange(range, targetVersion)]
}))
} else {
output[field] = structuredClone(value)
}
}
return output as unknown as RegistryVersion
}
function addManifest(index: Map<string, Map<string, RegistryVersion>>, manifest: RegistryVersion): void {
const versions = index.get(manifest.name) ?? new Map<string, RegistryVersion>()
versions.set(manifest.version, manifest)
index.set(manifest.name, versions)
}
/**
* Build registry metadata from installed external packages and workspace manifests.
* @param root - Repository root containing the pnpm virtual store.
* @param ref - Optional Git ref used instead of working-tree workspace manifests.
* @returns Package metadata served by the benchmark registry.
*/
export function buildRegistryIndex(root: string, ref?: string): RegistryIndex {
const index = new Map<string, Map<string, RegistryVersion>>()
for (const path of globSync(INSTALLED_MANIFEST_GLOBS, { cwd: root }).sort()) {
const manifest = JSON.parse(readFileSync(resolve(root, path), 'utf8')) as PackageManifest
const copied = copyPublishedManifest(manifest, new Map())
if (copied !== undefined) addManifest(index, copied)
}
const paths = workspaceManifestPaths(root, ref)
const refContents = ref === undefined ? undefined : readGitFiles(root, ref, paths)
const workspace = paths.map(path =>
JSON.parse(refContents?.get(path) ?? readFileSync(resolve(root, path), 'utf8')) as PackageManifest)
const workspaceVersions = new Map(workspace.flatMap(manifest =>
typeof manifest.name === 'string' && typeof manifest.version === 'string'
? [[manifest.name, manifest.version] as const]
: []))
for (const manifest of workspace) {
const copied = copyPublishedManifest(manifest, workspaceVersions)
if (copied !== undefined) addManifest(index, copied)
}
return index
}
function latestVersion(versions: ReadonlyMap<string, RegistryVersion>): string {
const sorted = [...versions.keys()].sort((left, right) => left.localeCompare(right, 'en', { numeric: true }))
const latest = sorted.at(-1)
if (latest === undefined) throw new Error('local registry package has no versions')
return latest
}
function listen(server: Server): Promise<number> {
return new Promise((resolveListen, reject) => {
server.once('error', reject)
server.listen(0, '127.0.0.1', () => {
server.off('error', reject)
const address = server.address()
if (address === null || typeof address === 'string') {
reject(new Error('local registry did not expose a TCP port'))
return
}
resolveListen(address.port)
})
})
}
function close(server: Server): Promise<void> {
return new Promise((resolveClose, reject) => {
server.close((error) => {
if (error === undefined) resolveClose()
else reject(error)
})
})
}
function npmExecutable(): string {
return process.platform === 'win32' ? 'npm.cmd' : 'npm'
}
function delay(ms: number): Promise<void> {
return new Promise(resolveDelay => setTimeout(resolveDelay, ms))
}
function signalProcessTree(child: ChildProcess, signal: 'SIGTERM' | 'SIGKILL'): void {
if (child.pid === undefined) {
child.kill(signal)
return
}
if (process.platform === 'win32') {
const force = signal === 'SIGKILL' ? ['/F'] : []
const result = spawnSync('taskkill', ['/PID', String(child.pid), '/T', ...force], {
stdio: 'ignore',
windowsHide: true,
})
if (result.error !== undefined) throw result.error
if (result.status !== 0 && child.exitCode === null && child.signalCode === null) child.kill(signal)
return
}
try {
process.kill(-child.pid, signal)
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error
}
}
/**
* Run one command with bounded process-tree termination after its deadline.
* @param command - Executable path or name.
* @param args - Arguments passed without shell interpolation on POSIX.
* @param options - Working directory, environment, timeout, and termination grace.
* @returns Exit facts, captured output, duration, and whether timeout handling began.
*/
export async function runCommandWithTimeout(
command: string,
args: readonly string[],
options: {
readonly cwd: string
readonly env: NodeJS.ProcessEnv
readonly timeoutMs: number
readonly terminationGraceMs?: number
},
): Promise<{ status: number | null; signal: NodeJS.Signals | null; durationMs: number; output: string; timedOut: boolean }> {
const started = performance.now()
const child = spawn(command, [...args], {
cwd: options.cwd,
detached: process.platform !== 'win32',
env: options.env,
shell: process.platform === 'win32',
stdio: ['ignore', 'pipe', 'pipe'],
})
let output = ''
child.stdout.setEncoding('utf8')
child.stderr.setEncoding('utf8')
child.stdout.on('data', (chunk) => { output += String(chunk) })
child.stderr.on('data', (chunk) => { output += String(chunk) })
const exited = new Promise<{ status: number | null; signal: NodeJS.Signals | null }>((resolveExit, reject) => {
child.once('error', reject)
child.once('close', (status, signal) => { resolveExit({ status, signal }) })
})
let timeout: NodeJS.Timeout | undefined
try {
const first = await Promise.race([
exited.then(outcome => ({ type: 'exit' as const, outcome })),
new Promise<{ type: 'timeout' }>((resolveTimeout) => {
timeout = setTimeout(() => { resolveTimeout({ type: 'timeout' }) }, options.timeoutMs)
}),
])
if (first.type === 'exit') {
return { ...first.outcome, durationMs: performance.now() - started, output, timedOut: false }
}
signalProcessTree(child, 'SIGTERM')
await delay(options.terminationGraceMs ?? TERMINATION_GRACE_MS)
signalProcessTree(child, 'SIGKILL')
const forced = await Promise.race([
exited,
delay(FORCED_EXIT_TIMEOUT_MS).then(() => undefined),
])
if (forced === undefined) throw new Error('timed-out process tree did not exit after SIGKILL')
return { ...forced, durationMs: performance.now() - started, output, timedOut: true }
} finally {
if (timeout !== undefined) clearTimeout(timeout)
}
}
function readNpmPackageLock(path: string): NpmPackageLock {
const parsed: unknown = JSON.parse(readFileSync(path, 'utf8'))
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
throw new Error('npm produced an invalid package-lock.json')
}
const { lockfileVersion, packages } = parsed as Record<string, unknown>
if (!Number.isSafeInteger(lockfileVersion) || packages === null
|| typeof packages !== 'object' || Array.isArray(packages)) {
throw new Error('npm produced an invalid package-lock.json')
}
return parsed as NpmPackageLock
}
async function runNpm(
cwd: string,
registry: string,
timeoutMs: number,
): Promise<{ durationMs: number; output: string; timedOut: boolean }> {
const npmrc = join(cwd, '.npmrc')
const globalNpmrc = join(cwd, '.npmrc-global')
writeFileSync(npmrc, `registry=${registry}\n@deepseek-ai:registry=${registry}\n`)
writeFileSync(globalNpmrc, '')
const inheritedEnvironment = Object.fromEntries(Object.entries(process.env)
.filter(([name]) => !name.toLowerCase().startsWith('npm_config_')))
const result = await runCommandWithTimeout(npmExecutable(), [
'install',
'--package-lock-only',
'--ignore-scripts',
'--no-audit',
'--no-fund',
'--loglevel=error',
'--include=peer',
'--install-strategy=hoisted',
'--legacy-peer-deps=false',
`--registry=${registry}`,
], {
cwd,
env: {
...inheritedEnvironment,
npm_config_cache: join(cwd, '.npm-cache'),
npm_config_globalconfig: globalNpmrc,
npm_config_userconfig: npmrc,
npm_config_update_notifier: 'false',
},
timeoutMs,
})
if (result.timedOut) return result
if (result.status !== 0) {
throw new Error(`npm install exited ${String(result.status)} after ${result.durationMs.toFixed(0)} ms\n${result.output.trim()}`)
}
return result
}
/**
* Ask npm to compute an install layout without downloading package archives.
* @param index - Package metadata exposed through the local registry.
* @param dependencies - Root dependencies whose install layout npm computes.
* @param timeoutMs - Hard wall-clock limit for the npm child process.
* @returns The package lock plus timing and registry-request observations.
*/
export async function resolveNpmPackageLock(
index: RegistryIndex,
dependencies: Readonly<Record<string, string>>,
timeoutMs: number,
): Promise<NpmPackageLockResolution> {
let registryRequests = 0
let archiveRequests = 0
const unknownPackages = new Set<string>()
let registry = ''
const server = createServer((request, response) => {
registryRequests++
const pathname = new URL(request.url ?? '/', registry).pathname
if (pathname.startsWith('/tarballs/')) {
archiveRequests++
response.writeHead(500, { 'content-type': 'application/json' })
response.end(JSON.stringify({ error: 'package-lock-only benchmark requested an archive' }))
return
}
const name = decodeURIComponent(pathname.slice(1))
const versions = index.get(name)
if (versions === undefined) {
unknownPackages.add(name)
response.writeHead(404, { 'content-type': 'application/json' })
response.end(JSON.stringify({ error: 'not_found' }))
return
}
const materialized = Object.fromEntries([...versions].map(([version, manifest]) => [version, {
...manifest,
dist: { tarball: `${registry}tarballs/${encodeURIComponent(name)}-${version}.tgz` },
}]))
const body = JSON.stringify({
name,
'dist-tags': { latest: latestVersion(versions) },
versions: materialized,
})
response.writeHead(200, {
'content-type': 'application/json',
'content-length': Buffer.byteLength(body),
})
response.end(body)
})
const port = await listen(server)
registry = `http://127.0.0.1:${String(port)}/`
const consumer = mkdtempSync(join(tmpdir(), 'dsh-npm-resolution-'))
try {
writeFileSync(join(consumer, 'package.json'), `${JSON.stringify({
name: 'dsh-npm-resolution-benchmark',
version: '0.0.0',
private: true,
dependencies,
}, null, 2)}\n`)
const result = await runNpm(consumer, registry, timeoutMs)
if (result.timedOut) throw new Error(`npm resolution exceeded ${String(timeoutMs)} ms`)
return {
durationMs: result.durationMs,
registryRequests,
archiveRequests,
unknownPackages: [...unknownPackages].sort(),
packageLock: readNpmPackageLock(join(consumer, 'package-lock.json')),
}
} finally {
server.closeAllConnections()
await close(server)
rmSync(consumer, { recursive: true, force: true })
}
}
/**
* Resolve the CLI install graph once without downloading package archives.
* @param index - Package metadata exposed through the local registry.
* @param targetVersion - Version of `@deepseek-ai/dsh` to install.
* @param timeoutMs - Hard wall-clock limit for the npm child process.
* @returns Timing and registry-request observations.
*/
export async function benchmarkNpmResolution(
index: RegistryIndex,
targetVersion: string,
timeoutMs: number,
): Promise<BenchmarkRun> {
const result = await resolveNpmPackageLock(index, { [TARGET_PACKAGE]: targetVersion }, timeoutMs)
return {
durationMs: result.durationMs,
registryRequests: result.registryRequests,
archiveRequests: result.archiveRequests,
unknownPackages: result.unknownPackages,
}
}
async function main(): Promise<void> {
const options = parseBenchmarkOptions(process.argv.slice(2))
const root = resolve(import.meta.dirname, '..')
const started = performance.now()
const index = buildRegistryIndex(root, options.ref)
const targetVersions = index.get(TARGET_PACKAGE)
if (targetVersions === undefined) throw new Error(`local registry contains no ${TARGET_PACKAGE}`)
const targetVersion = latestVersion(targetVersions)
const npmVersion = execFileSync(npmExecutable(), ['--version'], { encoding: 'utf8' }).trim()
console.log(
`benchmark-npm-resolution: npm ${npmVersion}, ${options.ref === undefined ? 'working tree' : options.ref}, `
+ `${String(index.size)} package name(s), setup ${(performance.now() - started).toFixed(0)} ms.`,
)
const durations: number[] = []
for (let run = 1; run <= options.runs; run++) {
const result = await benchmarkNpmResolution(index, targetVersion, options.timeoutMs)
durations.push(result.durationMs)
console.log(
`benchmark-npm-resolution: run ${String(run)}/${String(options.runs)} resolved ${TARGET_PACKAGE}@${targetVersion}`
+ ` in ${(result.durationMs / 1000).toFixed(2)} s with ${String(result.registryRequests)} metadata request(s)`
+ ` and ${String(result.unknownPackages.length)} local 404 package name(s).`,
)
if (result.archiveRequests > 0) throw new Error('npm requested package archives during the metadata-only benchmark')
}
const minimum = Math.min(...durations)
const maximum = Math.max(...durations)
console.log(
`benchmark-npm-resolution: ${String(options.runs)} run(s), min ${(minimum / 1000).toFixed(2)} s, max ${(maximum / 1000).toFixed(2)} s.`,
)
if (options.maxMs !== undefined && maximum > options.maxMs) {
throw new Error(`npm resolution exceeded --max-ms=${String(options.maxMs)} (max ${maximum.toFixed(0)} ms)`)
}
}
if (import.meta.main) {
try {
await main()
} catch (error) {
console.error(`benchmark-npm-resolution: ${error instanceof Error ? error.message : String(error)}`)
process.exitCode = 1
}
}