From 08e546eff13780b8cd879db7923f62bd44f76786 Mon Sep 17 00:00:00 2001 From: _Kerman Date: Thu, 20 Aug 2026 16:25:01 +0800 Subject: [PATCH] feat(storage-json): one-time migration of a legacy whole-unit file to per-record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opening a per-record unit splits a legacy `/.json` (the pre-per-record single-file layout) into per-record documents; an already-present new record wins, and the legacy file is deleted once every record migrated. The migration also runs when the new tree is absent — the fresh-upgrade shape — and foreign, shapeless, or malformed legacy files are left alone. This preserves previously cached session titles, list metadata, stats, and subagent identity across the medium change. --- .../storage-json/src/per-record-unit.ts | 76 ++++++++++++++++--- .../storage-json/tests/json-backend.spec.ts | 66 ++++++++++++++++ 2 files changed, 132 insertions(+), 10 deletions(-) diff --git a/packages/storage/storage-json/src/per-record-unit.ts b/packages/storage/storage-json/src/per-record-unit.ts index 20fd0a39c9..956dd742c2 100644 --- a/packages/storage/storage-json/src/per-record-unit.ts +++ b/packages/storage/storage-json/src/per-record-unit.ts @@ -14,6 +14,10 @@ * bricks the whole unit, and a version bump discards stale records instead * of migrating them. Record keys become path segments, so they must be * path-safe (`[a-zA-Z0-9_-]+`); an unsafe key rejects at write. + * + * One-time migration: a legacy whole-unit file `/.json` (the + * pre-per-record layout) is split into per-record documents on first open, + * new records winning, and deleted once every record migrated. * @module @deepseek-ai/dsh-storage-json/src/per-record-unit */ @@ -63,27 +67,79 @@ async function loadPerRecordState(descriptor: KvUnitDescriptor, dir: string): Pr global: null, tables: new Map(descriptor.tables.map(table => [table, new Map()])), } - let entries: Dirent[] + let entries: Dirent[] | undefined try { entries = await readdir(dir, { withFileTypes: true }) } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error - return state + // Missing directory = empty unit; the legacy migration below still runs + // (the fresh-upgrade shape is exactly an absent new tree). } - for (const entry of entries) { - if (entry.isDirectory()) { - const records = state.tables.get(entry.name) - if (records !== undefined) { - await loadTableRecords(records, descriptor.version, join(dir, entry.name)) + if (entries !== undefined) { + for (const entry of entries) { + if (entry.isDirectory()) { + const records = state.tables.get(entry.name) + if (records !== undefined) { + await loadTableRecords(records, descriptor.version, join(dir, entry.name)) + } + } else if (entry.isFile() && entry.name === 'global.json' && descriptor.hasGlobal) { + const global = await readRecord(join(dir, entry.name), descriptor.version) + if (global !== undefined) state.global = global } - } else if (entry.isFile() && entry.name === 'global.json' && descriptor.hasGlobal) { - const global = await readRecord(join(dir, entry.name), descriptor.version) - if (global !== undefined) state.global = global } } + await migrateLegacyUnit(descriptor, dir, state) return state } +/** + * One-time migration of a legacy whole-unit file (`/.json`, the + * pre-per-record layout). Every record it holds that the new tree lacks is + * written as a per-record document — an already-present new record wins — + * and the legacy file is deleted only after all records migrated. A missing + * legacy file, or one that is unreadable, foreign (another unit's name), or + * not a unit document, is left alone: the migration is idempotent, and the + * legacy file's absence is the "migrated" marker. + * @param descriptor - Static identity and shape of the unit. + * @param dir - The per-record unit directory (`/`). + * @param state - The tree state loaded so far; migrated records are added. + */ +async function migrateLegacyUnit(descriptor: KvUnitDescriptor, dir: string, state: UnitState): Promise { + const legacyPath = join(dirname(dir), `${descriptor.name}.json`) + let text: string | undefined + try { + text = await readFile(legacyPath, 'utf8') + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error + return + } + // The legacy document is runtime data: only `unit.name` and the tables map + // shape are checked here — the record values are migrated as-is and the + // domain layer's schemas judge them. + let document: { unit?: { name?: unknown }; tables?: unknown } + try { + document = JSON.parse(text) as { unit?: { name?: unknown }; tables?: unknown } + } catch { + return // Malformed legacy file: not ours to interpret or delete. + } + if (document.unit?.name !== descriptor.name) return + const tables = document.tables + if (typeof tables !== 'object' || tables === null) return + const recordsByTable = tables as Record> + for (const [table, records] of Object.entries(recordsByTable)) { + const target = state.tables.get(table) + if (target === undefined) continue + for (const [key, value] of Object.entries(records)) { + if (target.has(key)) continue // An existing new record wins. + const path = join(dir, table, `${key}.json`) + await mkdir(dirname(path), { recursive: true, mode: 0o700 }) + await writeAtomic(path, serializeRecord(descriptor.version, value)) + target.set(key, value) + } + } + await rm(legacyPath, { force: true }) // Every record migrated: drop the legacy file. +} + /** Read one declared table's record documents into `records`. */ async function loadTableRecords(records: Map, version: number, dir: string): Promise { for (const file of await readdir(dir, { withFileTypes: true })) { diff --git a/packages/storage/storage-json/tests/json-backend.spec.ts b/packages/storage/storage-json/tests/json-backend.spec.ts index 4365a5e3e0..e3ae010a44 100644 --- a/packages/storage/storage-json/tests/json-backend.spec.ts +++ b/packages/storage/storage-json/tests/json-backend.spec.ts @@ -339,4 +339,70 @@ describe('per-record layout', () => { await backend.close() await chmod(path, 0o600) }) + + it('migrates a legacy whole-unit file once, new records win, then deletes it', async () => { + const root = await freshRoot() + // A legacy single-layout file for the same unit (any older version); + // the extra table is not declared and must be skipped. + await writeFile(join(root, 'recs.json'), JSON.stringify({ + unit: { name: 'recs', version: 3 }, + global: null, + tables: { t: { old1: { v: 1 }, old2: { v: 2 } }, undeclared: { k: { v: 0 } } }, + }), 'utf8') + // A new per-record row that must win over its legacy namesake. + await mkdir(join(root, 'recs', 't'), { recursive: true }) + await writeFile(join(root, 'recs', 't', 'old1.json'), JSON.stringify({ version: 2, record: { v: 9 } }), 'utf8') + const backend = new JsonStorageBackend(root) + const unit = await backend.kv.open(descriptor) + expect(await unit.loadAll()).toEqual({ tables: { t: { old1: { v: 9 }, old2: { v: 2 } } }, global: null }) + // The legacy file was deleted; a reopen reads the migrated tree. + await expect(readFile(join(root, 'recs.json'), 'utf8')).rejects.toMatchObject({ code: 'ENOENT' }) + await unit.close() + const unit2 = await backend.kv.open(descriptor) + expect(await unit2.loadAll()).toEqual({ tables: { t: { old1: { v: 9 }, old2: { v: 2 } } }, global: null }) + await backend.close() + }) + + it('leaves a foreign, shapeless, or malformed legacy file alone', async () => { + const root = await freshRoot() + await writeFile(join(root, 'recs.json'), JSON.stringify({ unit: { name: 'other', version: 3 }, tables: {} }), 'utf8') + const backend = new JsonStorageBackend(root) + const unit = await backend.kv.open(descriptor) + expect(await unit.loadAll()).toEqual({ tables: { t: {} }, global: null }) + await expect(readFile(join(root, 'recs.json'), 'utf8')).resolves.toContain('other') + await unit.close() + await backend.close() + + const root2 = await freshRoot() + await writeFile(join(root2, 'recs.json'), JSON.stringify({ tables: { t: { k: { v: 1 } } } }), 'utf8') + const backend2 = new JsonStorageBackend(root2) + const unit2 = await backend2.kv.open(descriptor) + expect(await unit2.loadAll()).toEqual({ tables: { t: {} }, global: null }) + await expect(readFile(join(root2, 'recs.json'), 'utf8')).resolves.toContain('tables') + await backend2.close() + + const root3 = await freshRoot() + // A directory where the legacy file should be: the migration read fails loudly. + await mkdir(join(root3, 'recs.json')) + const backend3 = new JsonStorageBackend(root3) + const unit3 = await backend3.kv.open(descriptor) + await expect(unit3.loadAll()).rejects.toMatchObject({ code: 'EISDIR' }) + await backend3.close() + + const root4 = await freshRoot() + await writeFile(join(root4, 'recs.json'), 'not json at all', 'utf8') + const backend4 = new JsonStorageBackend(root4) + const unit4 = await backend4.kv.open(descriptor) + expect(await unit4.loadAll()).toEqual({ tables: { t: {} }, global: null }) + await expect(readFile(join(root4, 'recs.json'), 'utf8')).resolves.toBe('not json at all') + await backend4.close() + + const root5 = await freshRoot() + await writeFile(join(root5, 'recs.json'), JSON.stringify({ unit: { name: 'recs' }, tables: 'not an object' }), 'utf8') + const backend5 = new JsonStorageBackend(root5) + const unit5 = await backend5.kv.open(descriptor) + expect(await unit5.loadAll()).toEqual({ tables: { t: {} }, global: null }) + await expect(readFile(join(root5, 'recs.json'), 'utf8')).resolves.toContain('not an object') + await backend5.close() + }) })