From f47b1ecac271a74a82ed0b055f1e06f8cd173b6c Mon Sep 17 00:00:00 2001 From: imccyu <276526105+imccyu@users.noreply.github.com> Date: Thu, 20 Aug 2026 19:19:07 +0800 Subject: [PATCH] feat(webworker): browser worker host runtime and the vfs image packer Two private experimental packages run the whole harness tree inside one dedicated Web Worker. dsh-experimental-webworker-runtime owns the in-memory VFS (BigInt stats with per-path identity and strictly increasing mtimes), the CommonJS wrapper loader over a lazily-evaluated builtin table whose shims typecheck against Node's own module types, the postMessage tunnel speaking plain HTTP, the AsyncLocalStorage runtime, and the worker assembly. dsh-experimental-webworker-packer lowers every module body at pack time against the shared wrapper contract, sweeps the profile closure by static reachability, and writes a deterministically gzip-compressed tar the worker inflates through the browser's native DecompressionStream while it downloads. --- THIRD_PARTY_NOTICES.md | 6 + apps/cli/package.json | 5 + knip.json | 17 +- packages/bundle/web-app/src/index.ts | 15 +- packages/bundle/web-app/tests/web-app.spec.ts | 16 +- packages/experimental/README.i18n.yaml | 4 +- packages/experimental/README.md | 2 + packages/experimental/README.zh.md | 2 + .../webworker-packer/README.i18n.yaml | 6 + .../experimental/webworker-packer/README.md | 27 + .../webworker-packer/README.zh.md | 27 + .../webworker-packer/package.json | 54 ++ .../experimental/webworker-packer/src/bin.ts | 57 ++ .../webworker-packer/src/index.ts | 15 + .../webworker-packer/src/invariant.ts | 31 + .../experimental/webworker-packer/src/pack.ts | 575 ++++++++++++++ .../webworker-packer/src/repository.ts | 173 +++++ .../webworker-packer/src/rules.ts | 70 ++ .../webworker-packer/src/transform-image.ts | 26 + .../tests/image-loadable.spec.ts | 138 ++++ .../webworker-packer/tsconfig.json | 24 + .../webworker-packer/tsdown.config.ts | 18 + .../webworker-runtime/README.i18n.yaml | 6 + .../experimental/webworker-runtime/README.md | 33 + .../webworker-runtime/README.zh.md | 33 + .../webworker-runtime/package.json | 64 ++ .../src/client/api-client.ts | 33 + .../src/client/apply-injections.ts | 50 ++ .../webworker-runtime/src/client/client.ts | 342 +++++++++ .../webworker-runtime/src/client/index.ts | 94 +++ .../src/compile/transform.ts | 571 ++++++++++++++ .../webworker-runtime/src/image-layout.ts | 47 ++ .../webworker-runtime/src/index.ts | 44 ++ .../webworker-runtime/src/invariant.ts | 32 + .../webworker-runtime/src/module-proxies.ts | 76 ++ .../src/module-system/module-loader.ts | 407 ++++++++++ .../src/module-system/posix-path.ts | 169 +++++ .../implemented/async_hooks.ts | 406 ++++++++++ .../builtin_modules/implemented/buffer.ts | 28 + .../builtin_modules/implemented/crypto.ts | 123 +++ .../builtin_modules/implemented/events.ts | 156 ++++ .../node/builtin_modules/implemented/fs.ts | 574 ++++++++++++++ .../implemented/fs/promises.ts | 20 + .../node/builtin_modules/implemented/http.ts | 179 +++++ .../builtin_modules/implemented/module.ts | 73 ++ .../node/builtin_modules/implemented/os.ts | 118 +++ .../node/builtin_modules/implemented/path.ts | 396 ++++++++++ .../builtin_modules/implemented/perf_hooks.ts | 27 + .../implemented/timers/promises.ts | 60 ++ .../node/builtin_modules/implemented/url.ts | 73 ++ .../node/builtin_modules/implemented/util.ts | 156 ++++ .../builtin_modules/implemented/util/types.ts | 14 + .../node/builtin_modules/implemented/zlib.ts | 85 +++ .../src/node/builtin_modules/mock/net.ts | 90 +++ .../src/node/builtin_modules/mock/sqlite.ts | 31 + .../src/node/builtin_modules/mock/stream.ts | 38 + .../src/node/builtin_modules/mock/vm.ts | 37 + .../builtin_modules/mock/worker_threads.ts | 50 ++ .../webworker-runtime/src/node/builtins.ts | 122 +++ .../src/node/external_packages/chokidar.ts | 68 ++ .../src/node/external_packages/koffi.ts | 155 ++++ .../node-addon-landlock-run.ts | 31 + .../src/node/external_packages/node-pty.ts | 19 + .../src/node/external_packages/pi-ai.ts | 92 +++ .../external_packages/replaced-externals.ts | 19 + .../src/node/external_packages/ripgrep.ts | 15 + .../src/node/external_packages/sharp.ts | 13 + .../src/node/external_packages/ws.ts | 62 ++ .../src/node/globals/process.ts | 136 ++++ .../src/node/globals/timers.ts | 68 ++ .../src/node/notImplementedFail.ts | 44 ++ .../src/polyfill/async-context/als-runtime.ts | 102 +++ .../async-context/async-context-hooks.ts | 80 ++ .../webworker-runtime/src/storage/active.ts | 28 + .../src/storage/image-gzip.ts | 100 +++ .../webworker-runtime/src/storage/memory.ts | 595 +++++++++++++++ .../webworker-runtime/src/storage/paths.ts | 23 + .../webworker-runtime/src/storage/tar.ts | 136 ++++ .../webworker-runtime/src/storage/types.ts | 115 +++ .../webworker-runtime/src/transport/frames.ts | 123 +++ .../src/transport/synthetic-http.ts | 139 ++++ .../webworker-runtime/src/transport/tunnel.ts | 437 +++++++++++ .../webworker-runtime/src/worker-host.ts | 467 ++++++++++++ .../webworker-runtime/src/worker.ts | 64 ++ .../tests/compile/transform-corpus-check.ts | 437 +++++++++++ .../tests/compile/transform-corpus.spec.ts | 33 + .../tests/compile/transform.spec.ts | 710 ++++++++++++++++++ .../webworker-runtime/tests/log-sink.spec.ts | 86 +++ .../tests/node/builtins-table.spec.ts | 87 +++ .../tests/node/events.spec.ts | 137 ++++ .../webworker-runtime/tests/node/fs.spec.ts | 203 +++++ .../tests/node/http-server.spec.ts | 83 ++ .../tests/node/node-stubs.spec.ts | 206 +++++ .../tests/node/path-diff.spec.ts | 73 ++ .../tests/node/process-shim.spec.ts | 47 ++ .../tests/node/shim-diff.spec.ts | Bin 0 -> 3514 bytes .../tests/node/timers-promises.spec.ts | 55 ++ .../tests/polyfill/als-runtime.spec.ts | 394 ++++++++++ .../tests/polyfill/als-shim.spec.ts | 394 ++++++++++ .../tests/polyfill/als.spec.ts | 82 ++ .../tests/storage/image-gzip.spec.ts | 86 +++ .../tests/storage/memory-vfs.spec.ts | 95 +++ .../tests/storage/tar.spec.ts | 37 + .../tests/transport/tunnel-client.spec.ts | 131 ++++ .../webworker-runtime/tsconfig.json | 36 + .../webworker-runtime/tsdown.config.ts | 80 ++ pnpm-lock.yaml | 333 +++++++- scripts/check-workspace-constraints.ts | 9 +- scripts/publint-all.ts | 23 +- .../verify-package-readme-model-experience.ts | 2 + tsconfig.base.json | 4 + tsconfig.host.json | 1 + vitest.config.ts | 13 + 113 files changed, 13126 insertions(+), 47 deletions(-) create mode 100644 packages/experimental/webworker-packer/README.i18n.yaml create mode 100644 packages/experimental/webworker-packer/README.md create mode 100644 packages/experimental/webworker-packer/README.zh.md create mode 100644 packages/experimental/webworker-packer/package.json create mode 100644 packages/experimental/webworker-packer/src/bin.ts create mode 100644 packages/experimental/webworker-packer/src/index.ts create mode 100644 packages/experimental/webworker-packer/src/invariant.ts create mode 100644 packages/experimental/webworker-packer/src/pack.ts create mode 100644 packages/experimental/webworker-packer/src/repository.ts create mode 100644 packages/experimental/webworker-packer/src/rules.ts create mode 100644 packages/experimental/webworker-packer/src/transform-image.ts create mode 100644 packages/experimental/webworker-packer/tests/image-loadable.spec.ts create mode 100644 packages/experimental/webworker-packer/tsconfig.json create mode 100644 packages/experimental/webworker-packer/tsdown.config.ts create mode 100644 packages/experimental/webworker-runtime/README.i18n.yaml create mode 100644 packages/experimental/webworker-runtime/README.md create mode 100644 packages/experimental/webworker-runtime/README.zh.md create mode 100644 packages/experimental/webworker-runtime/package.json create mode 100644 packages/experimental/webworker-runtime/src/client/api-client.ts create mode 100644 packages/experimental/webworker-runtime/src/client/apply-injections.ts create mode 100644 packages/experimental/webworker-runtime/src/client/client.ts create mode 100644 packages/experimental/webworker-runtime/src/client/index.ts create mode 100644 packages/experimental/webworker-runtime/src/compile/transform.ts create mode 100644 packages/experimental/webworker-runtime/src/image-layout.ts create mode 100644 packages/experimental/webworker-runtime/src/index.ts create mode 100644 packages/experimental/webworker-runtime/src/invariant.ts create mode 100644 packages/experimental/webworker-runtime/src/module-proxies.ts create mode 100644 packages/experimental/webworker-runtime/src/module-system/module-loader.ts create mode 100644 packages/experimental/webworker-runtime/src/module-system/posix-path.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/async_hooks.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/buffer.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/crypto.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/events.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/fs.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/fs/promises.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/http.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/module.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/os.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/path.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/perf_hooks.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/timers/promises.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/url.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/util.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/util/types.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/implemented/zlib.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/mock/net.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/mock/sqlite.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/mock/stream.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/mock/vm.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtin_modules/mock/worker_threads.ts create mode 100644 packages/experimental/webworker-runtime/src/node/builtins.ts create mode 100644 packages/experimental/webworker-runtime/src/node/external_packages/chokidar.ts create mode 100644 packages/experimental/webworker-runtime/src/node/external_packages/koffi.ts create mode 100644 packages/experimental/webworker-runtime/src/node/external_packages/node-addon-landlock-run.ts create mode 100644 packages/experimental/webworker-runtime/src/node/external_packages/node-pty.ts create mode 100644 packages/experimental/webworker-runtime/src/node/external_packages/pi-ai.ts create mode 100644 packages/experimental/webworker-runtime/src/node/external_packages/replaced-externals.ts create mode 100644 packages/experimental/webworker-runtime/src/node/external_packages/ripgrep.ts create mode 100644 packages/experimental/webworker-runtime/src/node/external_packages/sharp.ts create mode 100644 packages/experimental/webworker-runtime/src/node/external_packages/ws.ts create mode 100644 packages/experimental/webworker-runtime/src/node/globals/process.ts create mode 100644 packages/experimental/webworker-runtime/src/node/globals/timers.ts create mode 100644 packages/experimental/webworker-runtime/src/node/notImplementedFail.ts create mode 100644 packages/experimental/webworker-runtime/src/polyfill/async-context/als-runtime.ts create mode 100644 packages/experimental/webworker-runtime/src/polyfill/async-context/async-context-hooks.ts create mode 100644 packages/experimental/webworker-runtime/src/storage/active.ts create mode 100644 packages/experimental/webworker-runtime/src/storage/image-gzip.ts create mode 100644 packages/experimental/webworker-runtime/src/storage/memory.ts create mode 100644 packages/experimental/webworker-runtime/src/storage/paths.ts create mode 100644 packages/experimental/webworker-runtime/src/storage/tar.ts create mode 100644 packages/experimental/webworker-runtime/src/storage/types.ts create mode 100644 packages/experimental/webworker-runtime/src/transport/frames.ts create mode 100644 packages/experimental/webworker-runtime/src/transport/synthetic-http.ts create mode 100644 packages/experimental/webworker-runtime/src/transport/tunnel.ts create mode 100644 packages/experimental/webworker-runtime/src/worker-host.ts create mode 100644 packages/experimental/webworker-runtime/src/worker.ts create mode 100644 packages/experimental/webworker-runtime/tests/compile/transform-corpus-check.ts create mode 100644 packages/experimental/webworker-runtime/tests/compile/transform-corpus.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/compile/transform.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/log-sink.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/node/builtins-table.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/node/events.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/node/fs.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/node/http-server.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/node/node-stubs.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/node/path-diff.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/node/process-shim.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/node/shim-diff.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/node/timers-promises.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/polyfill/als-runtime.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/polyfill/als-shim.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/polyfill/als.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/storage/image-gzip.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/storage/memory-vfs.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/storage/tar.spec.ts create mode 100644 packages/experimental/webworker-runtime/tests/transport/tunnel-client.spec.ts create mode 100644 packages/experimental/webworker-runtime/tsconfig.json create mode 100644 packages/experimental/webworker-runtime/tsdown.config.ts diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 295eb1868f..bb9ad51cf1 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -39,6 +39,7 @@ External packages that a workspace package resolves at runtime. The tier covers | [`@joplin/turndown-plugin-gfm`](https://github.com/laurent22/joplin-turndown-plugin-gfm) | MIT | | [`@jridgewell/gen-mapping`](https://github.com/jridgewell/sourcemaps) | MIT | | [`@modelcontextprotocol/sdk`](https://github.com/modelcontextprotocol/typescript-sdk) | MIT | +| [`@noble/hashes`](https://github.com/paulmillr/noble-hashes) | MIT | | [`@openai/codex`](https://github.com/openai/codex) | Apache-2.0 | | [`@opentelemetry/api`](https://github.com/open-telemetry/opentelemetry-js) | Apache-2.0 | | [`@opentelemetry/api-logs`](https://github.com/open-telemetry/opentelemetry-js) | Apache-2.0 | @@ -51,7 +52,10 @@ External packages that a workspace package resolves at runtime. The tier covers | [`@tanstack/react-virtual`](https://github.com/TanStack/virtual) | MIT | | [`@types/mdast`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@vscode/ripgrep`](https://github.com/microsoft/vscode-ripgrep) | MIT | +| [`@yarnpkg/parsers`](https://github.com/yarnpkg/berry) | BSD-2-Clause | +| [`acorn`](https://github.com/acornjs/acorn) | MIT | | [`anser`](https://github.com/IonicaBizau/anser) | MIT | +| [`buffer`](https://github.com/feross/buffer) | MIT | | [`chokidar`](https://github.com/paulmillr/chokidar) | MIT | | [`clsx`](https://github.com/lukeed/clsx) | MIT | | [`commander`](https://github.com/tj/commander.js) | MIT | @@ -136,6 +140,7 @@ External packages **directly declared** only by repository tooling, test infrast | [`@types/react-dom`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@types/spdx-expression-parse`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@types/turndown`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | +| [`@types/use-sync-external-store`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@types/ws`](https://github.com/DefinitelyTyped/DefinitelyTyped) | MIT | | [`@vitejs/plugin-react`](https://github.com/vitejs/vite-plugin-react) | MIT | | [`@vitest/coverage-v8`](https://github.com/vitest-dev/vitest) | MIT | @@ -148,6 +153,7 @@ External packages **directly declared** only by repository tooling, test infrast | [`eslint-plugin-sonarjs`](https://github.com/SonarSource/SonarJS) | LGPL-3.0-only | | [`execa`](https://github.com/sindresorhus/execa) | MIT | | [`fast-check`](https://github.com/dubzzz/fast-check) | MIT | +| [`http-server`](https://github.com/http-party/http-server) | MIT | | [`istanbul-lib-report`](https://github.com/istanbuljs/istanbuljs) | BSD-3-Clause | | [`jscpd`](https://github.com/kucherenko/jscpd) | MIT | | [`jsdom`](https://github.com/jsdom/jsdom) | MIT | diff --git a/apps/cli/package.json b/apps/cli/package.json index eeeb48e79a..b3cef32dea 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -18,6 +18,11 @@ "lib/*.js", "config" ], + "dsh": { + "configTrees": [ + { "mount": "config/agent-presets", "path": "config/agent-presets", "scanRoster": true } + ] + }, "license": "MIT", "dependencies": { "@deepseek-ai/cordis-plugin-hmr": "workspace:^", diff --git a/knip.json b/knip.json index 280d10a1f0..28267ff092 100644 --- a/knip.json +++ b/knip.json @@ -211,6 +211,20 @@ "tests/**/*.ts" ] }, + "packages/experimental/webworker-runtime": { + "entry": [ + "tests/**/*.spec.ts", + "tests/compile/transform-corpus-check.ts" + ], + "project": [ + "src/**/*.ts", + "tests/**/*.ts" + ], + "ignoreDependencies": [ + "buffer", + "@deepseek-ai/dsh-client-modules" + ] + }, "packages/typert/generator": { "entry": [ "tests/**/*.spec.ts", @@ -611,7 +625,8 @@ "tests/**/*.perf.ts", "tests/**/*.snapshot.ts", "tests/support.ts", - "src/node-module-stub.ts" + "src/node-module-stub.ts", + "src/preview.ts" ], "project": [ "src/**/*.ts", diff --git a/packages/bundle/web-app/src/index.ts b/packages/bundle/web-app/src/index.ts index 6965310437..79d1e94862 100644 --- a/packages/bundle/web-app/src/index.ts +++ b/packages/bundle/web-app/src/index.ts @@ -13,6 +13,7 @@ import { spawn, type ChildProcess } from 'node:child_process' import { createRequire } from 'node:module' +import { dirname, join } from 'node:path' import { networkInterfaces } from 'node:os' import { fileURLToPath } from 'node:url' import type { Context } from '@deepseek-ai/cordis' @@ -159,14 +160,20 @@ function localWebUrl(ctx: Context): string { return `http://${LOOPBACK_HOST}:${String(port)}` } -/** Dist location is workspace knowledge of this bundle: resolved through the frontend package exports, not configured. */ +/** + * Dist location is workspace knowledge of this bundle: anchored on the + * frontend package manifest, not configured. Existence is a request-time + * concern — the fallback owner reads files per request, so a composition + * whose page never reaches the fallback seat (the static worker preview + * ships its own page and carries no dist) boots without one. + */ function resolveDistIndex(): string { const require = createRequire(import.meta.url) try { - return require.resolve('@deepseek-ai/dsh-web-frontend/dist/index.html') + return join(dirname(require.resolve('@deepseek-ai/dsh-web-frontend/package.json')), 'dist', 'index.html') } catch { - /* v8 ignore next 2 -- reachable only on a checkout without a built dist; the test tree builds it */ - throw new Error('web-app: frontend dist not built; run pnpm run build from the repository root first') + /* v8 ignore next 2 -- reachable only when the frontend package is absent from the checkout */ + throw new Error('web-app: @deepseek-ai/dsh-web-frontend is not resolvable from this composition') } } diff --git a/packages/bundle/web-app/tests/web-app.spec.ts b/packages/bundle/web-app/tests/web-app.spec.ts index 39b9d7ac6b..5639129362 100644 --- a/packages/bundle/web-app/tests/web-app.spec.ts +++ b/packages/bundle/web-app/tests/web-app.spec.ts @@ -286,16 +286,12 @@ describe('web-app runtime glue', () => { await ctx.fiber.dispose() }) - it('resolves the real built frontend dist through the package exports, failing loud unbuilt', () => { - // The production resolver (not the test hook). A built checkout resolves - // the frontend package's index.html; a dist-less one (the CI coverage - // lane runs before any build) must fail with the build hint, never a - // silent fallback. - try { - expect(originalResolve()).toMatch(/dist[/\\]index\.html$/) - } catch (error) { - expect((error as Error).message).toContain('frontend dist not built') - } + it('anchors the dist index on the frontend package manifest without requiring a built dist', () => { + // The production resolver (not the test hook): the anchor resolves on any + // checkout, built or not — dist existence is the fallback owner's + // request-time concern, so a dist-less composition (the static worker + // preview ships its own page) still boots. + expect(originalResolve()).toMatch(/dist[/\\]index\.html$/) }) it.each([ diff --git a/packages/experimental/README.i18n.yaml b/packages/experimental/README.i18n.yaml index d5ef901778..ba94b89182 100644 --- a/packages/experimental/README.i18n.yaml +++ b/packages/experimental/README.i18n.yaml @@ -2,5 +2,5 @@ # side as of the last confirmed-consistent state. Both languages carry equal authority; # after editing either side, bring the other along and re-record with: # pnpm run verify-translation-pairing --write packages/experimental/README.md -README.md: 0e92ebd2bd959ac807830400dd57807b87b1cbe2 -README.zh.md: a1751c39f53bf8f6c0a9c623aba57355f35c2681 +README.md: 43d96d1c539b2ec35d7a818f60270e6d17db54e9 +README.zh.md: 27bb4d73b0d8baa4614e79abbf20a1f988f8652c diff --git a/packages/experimental/README.md b/packages/experimental/README.md index 0e92ebd2bd..43d96d1c53 100644 --- a/packages/experimental/README.md +++ b/packages/experimental/README.md @@ -8,5 +8,7 @@ This group contains prototypes and internal-only Cordis plugins that use the rep |---|---|---| | `agent-team/` | Implicit-root Agent Teams roster, durable peer mailbox, shared task DAG, and runtime coordination | `ctx.agentTeams` | | `tool-agent-team/` | Scoped model-facing Agent Teams tools and collaboration guidance | — | +| `webworker-runtime/` | Browser-only host runtime: in-memory VFS, module loader, postMessage tunnel, and the dedicated Web Worker assembly | — | +| `webworker-packer/` | Build-time packer that materializes a profile's package closure into the VFS image the worker mounts | — | The [subtree rules](AGENTS.md) define dependency isolation, release exclusion, and promotion. diff --git a/packages/experimental/README.zh.md b/packages/experimental/README.zh.md index a1751c39f5..27bb4d73b0 100644 --- a/packages/experimental/README.zh.md +++ b/packages/experimental/README.zh.md @@ -8,5 +8,7 @@ |---|---|---| | `agent-team/` | 隐式 root Agent Teams roster、持久 peer mailbox、共享任务 DAG 与运行时协调 | `ctx.agentTeams` | | `tool-agent-team/` | 按 Agent 作用域提供的 Agent Teams 模型工具与协作指引 | — | +| `webworker-runtime/` | 纯浏览器 host 运行时:内存 VFS、模块装载器、postMessage 隧道与 dedicated Web Worker 装配 | — | +| `webworker-packer/` | 构建期打包器:把 profile 的包闭包物化成 worker 挂载的 VFS 镜像 | — | [子树规则](AGENTS.md)规定依赖隔离、发布排除与 promotion。 diff --git a/packages/experimental/webworker-packer/README.i18n.yaml b/packages/experimental/webworker-packer/README.i18n.yaml new file mode 100644 index 0000000000..8ed28f6793 --- /dev/null +++ b/packages/experimental/webworker-packer/README.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write packages/experimental/webworker-packer/README.md +README.md: 15313f7b75169cc7a8749670900a0635605401e7 +README.zh.md: 2e2daba4c006e4d15db619e138447d5e239df4f6 diff --git a/packages/experimental/webworker-packer/README.md b/packages/experimental/webworker-packer/README.md new file mode 100644 index 0000000000..15313f7b75 --- /dev/null +++ b/packages/experimental/webworker-packer/README.md @@ -0,0 +1,27 @@ +# `@deepseek-ai/dsh-experimental-webworker-packer` + +English | [中文](README.zh.md) + +The VFS image packer: turns one composed profile into the single gzip-compressed tar the browser worker inflates and mounts as its filesystem ([experimental stance](../../../.agents/notes/implemented/architecture/2026-08-20-webworker-pack-lowering-and-preview.md)). Nothing is compiled from source — the image carries the repository's real build products, so a preview deployment debugs exactly what the served deployment ships. + +The pack is a three-layer standard stack: + +1. **Roster** — the composed profile's plugin rows (standard YAML parse under Include's dialect, `!!js` intact), plus the rows of every config tree the CLI declares in its `package.json` `dsh.configTrees` (agent presets), materialized as a Node-style dependency closure. External peer edges never bind the worker; workspace peers stay on the chain. +2. **Publish view** — each workspace package contributes the slice npm would publish (`files` through picomatch) minus the rule tables in `src/rules.ts` (no sources, no workspace `dist/`; external packages keep their trees minus the same exclude globs). +3. **Reachability sweep** — the runtime loader's own resolution walks from every workspace export face plus the worker assembly's seeds (`IMAGE_ENTRY_SEEDS`), lowering each reached module to the wrapper contract at pack time. Page assets (`lib/client.js` behind `./client` exports) ship verbatim; an unresolvable request from our own code fails the pack, third-party ones are tolerated to fail loud at require time. + +`repository.ts` owns the repo-shaped inputs (workspace scan of `vendor/`, `packages/`, `apps/`; profile composition through the real CLI dump path); `pack.ts` owns none of them, so the same library packs a different tree by being called differently. The CLI is `dsh-pack-vfs-image --out [--profile web]`; `apps/web`'s `build:preview` runs it after the preview shell build. + +## Model Experience + +None, as this package runs at build time and writes an image file; nothing it produces reaches a model request on its own. + +#### KV Cache effect + +None; this package neither assembles nor sends a provider request. + +## Known Limitations and Deferred Work + +- **The rule tables are judgement calls** (`rules.ts`: exclude globs, page-asset patterns, entry seeds) pinned by `tests/`; a new asset class the worker must reach needs a table row, not a scanner change. +- **Vendored package sources (`src/*.ts`) no longer pack** — nothing resolves them at runtime; a future in-worker source-inspection feature would need a dedicated include rule. +- **The packer assumes built `lib/` artifacts are current**: it never compiles, so a stale workspace build packs stale bytes. Run the repository build first. diff --git a/packages/experimental/webworker-packer/README.zh.md b/packages/experimental/webworker-packer/README.zh.md new file mode 100644 index 0000000000..2e2daba4c0 --- /dev/null +++ b/packages/experimental/webworker-packer/README.zh.md @@ -0,0 +1,27 @@ +# `@deepseek-ai/dsh-experimental-webworker-packer` + +[English](README.md) | 中文 + +VFS 镜像打包器:把一份合成 profile 变成浏览器 worker 解压后当文件系统挂载的单个 gzip 压缩 tar([experimental 定位](../../../.agents/notes/implemented/architecture/2026-08-20-webworker-pack-lowering-and-preview.zh.md))。不做任何源码编译——镜像携带仓库真实构建产物,预览部署调试的正是 served 部署交付的字节。 + +打包是三层标准栈: + +1. **Roster**——合成 profile 的插件行(标准 YAML 解析、Include 方言、`!!js` 原样保留),加上 CLI 在 `package.json` `dsh.configTrees` 里声明的每棵配置树(agent presets)的行,按 Node 式依赖闭包物化。外部包的 peer 边不追,workspace peer 保留在链上。 +2. **发布视图**——每个 workspace 包贡献 npm 会发布的切片(`files` 走 picomatch),再减去 `src/rules.ts` 的规则表(无源码、无 workspace `dist/`;外部包保留整棵减同一套 exclude glob)。 +3. **可达性 sweep**——用运行时加载器自己的解析,从全部 workspace 导出面加 worker 装配种子(`IMAGE_ENTRY_SEEDS`)出发,pack 时把每个可达模块降低到包装契约。页面资产(`./client` 导出背后的 `lib/client.js`)原样直发;自家代码的不可解析请求打包即失败,第三方的容忍到 require 时 fail loud。 + +`repository.ts` 拥有仓库形态输入(`vendor/`、`packages/`、`apps/` 的 workspace 扫描;经真 CLI dump 路径合成 profile);`pack.ts` 一概不拥有,同一库换参即可打另一棵树。CLI 为 `dsh-pack-vfs-image --out [--profile web]`;`apps/web` 的 `build:preview` 在预览壳构建后运行它。 + +## 模型体验 + +无:本包在构建期运行并写出镜像文件,其产物本身不进入任何模型请求。 + +#### KV Cache 影响 + +无:本包既不组装也不发送 provider 请求。 + +## Known Limitations and Deferred Work + +- **规则表是判断题**(`rules.ts`:exclude glob、页面资产模式、入口种子),由 `tests/` 钉住;worker 需要触达的新资产类别应加表行,而不是改扫描器。 +- **vendored 包源码(`src/*.ts`)不再打包**——运行时无人解析它们;未来若有 worker 内源码巡检功能需要专门的 include 规则。 +- **打包器假定构建产物 `lib/` 是新鲜的**:它从不编译,工作区构建过期就打包过期字节。先跑仓库构建。 diff --git a/packages/experimental/webworker-packer/package.json b/packages/experimental/webworker-packer/package.json new file mode 100644 index 0000000000..6e37359454 --- /dev/null +++ b/packages/experimental/webworker-packer/package.json @@ -0,0 +1,54 @@ +{ + "name": "@deepseek-ai/dsh-experimental-webworker-packer", + "description": "Build-time packer for the browser runtime's VFS image: materializes a profile's package closure into one gzip-compressed tar the worker mounts, with every module body pre-transformed", + "version": "0.1.0-rc.8", + "private": true, + "repository": { + "type": "git", + "url": "git+https://github.com/deepseek-ai/deepseek-harness.git", + "directory": "packages/experimental/webworker-packer" + }, + "type": "module", + "main": "lib/index.js", + "types": "lib/types/index.d.ts", + "bin": { + "dsh-pack-vfs-image": "./lib/bin.js" + }, + "exports": { + ".": { + "types": "./lib/types/index.d.ts", + "default": "./lib/index.js" + }, + "./invariant": { + "types": "./lib/types/invariant.d.ts", + "default": "./lib/invariant.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json" + }, + "files": [ + "lib/index.js", + "lib/invariant.js", + "lib/bin.js", + "lib/repository-*.js", + "lib/types/**/*.d.ts" + ], + "license": "MIT", + "dependencies": { + "@deepseek-ai/cordis-plugin-include": "workspace:^", + "@deepseek-ai/dsh-experimental-webworker-runtime": "workspace:^", + "@deepseek-ai/dsh-home-paths": "workspace:^", + "js-yaml": "^4.2.0", + "picomatch": "^4.0.4" + }, + "devDependencies": { + "@deepseek-ai/cordis": "workspace:^", + "@deepseek-ai/dsh-invariants": "workspace:^", + "@types/js-yaml": "^4.0.9", + "@types/picomatch": "^3.0.2" + }, + "peerDependencies": { + "@deepseek-ai/cordis": "workspace:^", + "@deepseek-ai/dsh-invariants": "workspace:^" + } +} diff --git a/packages/experimental/webworker-packer/src/bin.ts b/packages/experimental/webworker-packer/src/bin.ts new file mode 100644 index 0000000000..57a3c77730 --- /dev/null +++ b/packages/experimental/webworker-packer/src/bin.ts @@ -0,0 +1,57 @@ +#!/usr/bin/env node +/** + * Pack a VFS image from this repository: compose the profile, materialize the + * closure, lower every module body, write the gzip-compressed tar. + * + * Usage: dsh-pack-vfs-image --out [--profile web] [--root /dsh] + * node --import tsx/esm src/bin.ts --out ../../apps/web/dist/preview/vfs-image.tar.gz + * @module @deepseek-ai/dsh-experimental-webworker-packer/src/bin + */ +import { mkdirSync, writeFileSync } from 'node:fs' +import { dirname, isAbsolute, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { packVfsImage } from './pack.ts' +import { composeProfile, configTrees, describePack, indexWorkspacePackages } from './repository.ts' + +/** + * Read one `--flag value` pair. + * @param name - Flag name without dashes. + * @param fallback - Value when the flag is absent. + * @returns The value. + * @throws When the flag is present with no value, because silently packing the + * default profile is worse than stopping. + */ +function flag(name: string, fallback?: string): string { + const index = process.argv.indexOf(`--${name}`) + if (index === -1) { + if (fallback !== undefined) return fallback + throw new Error(`dsh-pack-vfs-image: --${name} is required`) + } + const value = process.argv[index + 1] + if (value === undefined || value.startsWith('--')) { + throw new Error(`dsh-pack-vfs-image: --${name} needs a value`) + } + return value +} + +const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url)) +const profile = flag('profile', 'web') +const out = flag('out') +const outputFile = isAbsolute(out) ? out : resolve(process.cwd(), out) + +const result = packVfsImage({ + config: composeProfile(repoRoot, profile), + profile, + root: flag('root', '/dsh'), + workspaces: indexWorkspacePackages(repoRoot), + resolveFrom: repoRoot, + configTrees: configTrees(repoRoot), +}) + +if (result.missing.length > 0) { + throw new Error(`vfs image: ${String(result.missing.length)} dependencies did not resolve; the image would be incomplete`) +} + +mkdirSync(dirname(outputFile), { recursive: true }) +writeFileSync(outputFile, result.image) +process.stdout.write(describePack(result, repoRoot, outputFile).join('\n')) diff --git a/packages/experimental/webworker-packer/src/index.ts b/packages/experimental/webworker-packer/src/index.ts new file mode 100644 index 0000000000..ea054b26b0 --- /dev/null +++ b/packages/experimental/webworker-packer/src/index.ts @@ -0,0 +1,15 @@ +/** + * Build-time packer for the browser runtime's VFS image. + * @module @deepseek-ai/dsh-experimental-webworker-packer + */ +export { + WRAPPER_CONTRACT, + type ImageFiles, type TransformOutcome, +} from './transform-image.ts' +export { + CONFIG_PATH, DEFAULT_ROOT, MANIFEST_PATH, packVfsImage, + type ConfigTree, type PackOptions, type PackResult, +} from './pack.ts' +export { + composeProfile, configTrees, describePack, indexWorkspacePackages, +} from './repository.ts' diff --git a/packages/experimental/webworker-packer/src/invariant.ts b/packages/experimental/webworker-packer/src/invariant.ts new file mode 100644 index 0000000000..bfa1060afb --- /dev/null +++ b/packages/experimental/webworker-packer/src/invariant.ts @@ -0,0 +1,31 @@ +/** + * Package-owned invariant companion for `@deepseek-ai/dsh-experimental-webworker-packer`. + * @module @deepseek-ai/dsh-experimental-webworker-packer/invariant + */ + +/* jscpd:ignore-start */ +import type { Context } from '@deepseek-ai/cordis' +import type { InvariantInstaller } from '@deepseek-ai/dsh-invariants' + +const PACKAGE_NAME = '@deepseek-ai/dsh-experimental-webworker-packer' + +/** Cordis companion plugin name. */ +export const name = 'webworker-packer-invariant' +/** Service required before the companion can reserve package ownership. */ +export const inject = ['invariants'] + +/** + * No runtime invariant: this package is a build-time pass with no + * production event stream or mutable data; the pack's own gates (unresolvable + * own requests, the all-or-nothing wrapper contract) fail the pack instead. + */ +const install: InvariantInstaller = () => {} + +/** + * Register this package's invariant companion. + * @param ctx - Cordis context carrying the invariant service. + * @returns the installed registration's disposer after setup succeeds. + */ +export const apply = (ctx: Context): Promise<() => void> => + Promise.resolve(ctx.invariants.register(PACKAGE_NAME, install)) +/* jscpd:ignore-end */ diff --git a/packages/experimental/webworker-packer/src/pack.ts b/packages/experimental/webworker-packer/src/pack.ts new file mode 100644 index 0000000000..2f71716e4d --- /dev/null +++ b/packages/experimental/webworker-packer/src/pack.ts @@ -0,0 +1,575 @@ +/** + * VFS image packer: turns one composed profile plus a package index into the single + * gzip-compressed tar the browser runtime inflates and mounts as its filesystem. + * + * Nothing is compiled here. The image carries the repository's real build products, + * so a preview deployment debugs exactly what the served deployment ships. What the + * pass does add is the pack-time module transform and the manifest that records the + * wrapper contract it was transformed against. + * + * This module holds no repository knowledge: paths, globs, and the composition come + * in as parameters, so the same library packs a different tree by being called + * differently. Locating those inputs is the CLI's job. + * @module @deepseek-ai/dsh-experimental-webworker-packer/src/pack + */ +import { existsSync, readFileSync, readdirSync, realpathSync } from 'node:fs' +import { dirname, join, relative } from 'node:path' +import { gzipSync } from 'node:zlib' + +import { + lowerModuleSource, MemoryVfs, packTar, WorkerModuleLoader, + DEFAULT_ROOT, IMAGE_CONFIG_PATH, IMAGE_EMPTY_DIRECTORIES, IMAGE_MANIFEST_PATH, +} from '@deepseek-ai/dsh-experimental-webworker-runtime' +import picomatch from 'picomatch' +import yaml from 'js-yaml' +import { entryListSchema } from '@deepseek-ai/cordis-plugin-include' +import { REPLACED_EXTERNAL_PACKAGES } from '@deepseek-ai/dsh-experimental-webworker-runtime/src/node/external_packages/replaced-externals.ts' +import { MODULE_PROXIES, MODULE_PROXY_PREFIXES } from '@deepseek-ai/dsh-experimental-webworker-runtime/src/module-proxies.ts' +import { WRAPPER_CONTRACT, type ImageFiles, type TransformOutcome } from './transform-image.ts' +import { EXCLUDE, EXCLUDE_WORKSPACE, IMAGE_ENTRY_SEEDS, PAGE_ASSETS } from './rules.ts' + +export { DEFAULT_ROOT } from '@deepseek-ai/dsh-experimental-webworker-runtime' + +/** Image path of the manifest; the layout contract's name, re-exported for callers. */ +export const MANIFEST_PATH: string = IMAGE_MANIFEST_PATH + +/** Image path of the composed profile; the layout contract's name, re-exported for callers. */ +export const CONFIG_PATH: string = IMAGE_CONFIG_PATH + +/** + * Manifest field the runtime judges the image by: the wrapper contract every packed + * body was emitted against. The runtime refuses an image whose value is not its own + * contract, because those bodies assume different wrapper semantics. + */ +const CONTRACT_FIELD = 'lowered' + +/** Exclude matcher over tree-root-relative paths ({@link EXCLUDE}). */ +const excluded = picomatch([...EXCLUDE], { dot: true }) + +/** Workspace exclude matcher: {@link EXCLUDE} plus {@link EXCLUDE_WORKSPACE}. */ +const workspaceExcluded = picomatch([...EXCLUDE, ...EXCLUDE_WORKSPACE], { dot: true }) + +/** Page-asset matcher over image paths ({@link PAGE_ASSETS}). */ +const pageAsset = picomatch([...PAGE_ASSETS], { dot: true }) + +/** One directory tree to copy in verbatim beside the composition. */ +export interface ConfigTree { + /** Image path to mount it at, relative to the virtual root. */ + readonly mount: string + /** Absolute source directory. */ + readonly directory: string + /** + * Whether plugin names inside its `.yml` files join the materialization closure. + * An agent preset mounts plugins the base composition never lists, and creating a + * session fails if any of them is missing from the image. + */ + readonly scanRoster?: boolean +} + +/** Everything the packer needs that it cannot know by itself. */ +export interface PackOptions { + /** Composed profile, `!!js` intact, as the CLI's `--dump-default-config` produced it. */ + readonly config: string + /** Profile name, recorded in the manifest. */ + readonly profile: string + /** Virtual root the image mounts under; defaults to {@link DEFAULT_ROOT}. */ + readonly root?: string + /** Package name to absolute directory, for workspace and vendored packages. */ + readonly workspaces: ReadonlyMap + /** Directory Node-style dependency resolution walks up from for the roster. */ + readonly resolveFrom: string + /** Config trees to copy in beside the composition. */ + readonly configTrees?: readonly ConfigTree[] + /** Empty directories to create; defaults to `home/`, `workspace/`, `tmp/`. */ + readonly emptyDirectories?: readonly string[] + /** + * Extra sweep roots: image specifiers requested by code outside the image. + * Defaults to the worker assembly's own entries. + */ + readonly entries?: readonly string[] +} + +/** What one pack produced, for the caller to report or assert on. */ +export interface PackResult { + /** The gzip-compressed tar archive to write; the runtime inflates it at mount. */ + readonly image: Uint8Array + /** Every entry, before zipping; the manifest is already among them. */ + readonly files: ImageFiles + /** Package name to how many files it contributed, in materialization order. */ + readonly packages: ReadonlyMap + /** How many of them came from the workspace rather than from `node_modules`. */ + readonly workspacePackages: number + /** Roster package names the closure started from. */ + readonly roster: readonly string[] + /** Dependencies that did not resolve; a non-empty list means an incomplete image. */ + readonly missing: readonly string[] + /** Executable scripts dropped from the image. */ + readonly executables: readonly string[] + /** Page bundles left verbatim, and so out of the transform. */ + readonly pageBundles: readonly string[] + /** JavaScript entries the image carries. */ + readonly javascriptEntries: number + /** JavaScript candidates no root reaches, dropped from the image. */ + readonly droppedJavascriptEntries: number + /** Third-party requests that resolve nowhere; loud at require time if hit. */ + readonly unresolvedExternalRequests: readonly string[] + /** What the pack-time transform did. */ + readonly transform: TransformOutcome + /** Wrapper contract recorded in the manifest; every packed body meets it. */ + readonly contract: string +} + +const readJson = (file: string): Record => + JSON.parse(readFileSync(file, 'utf8')) as Record + +/** + * Package name of a module specifier. + * @param specifier - Module specifier, possibly with a subpath. + * @returns The package name (`@scope/pkg/sub` → `@scope/pkg`). + */ +function packageNameOf(specifier: string): string { + const [first = specifier, second = ''] = specifier.split('/') + return first.startsWith('@') ? `${first}/${second}` : first +} + +/** + * Collect module-specifier `name` fields from parsed entry rows, recursively + * through nested `config` row lists (groups). Builtin rows (`cordis:group`) + * and preset metadata documents carry names that are not module specifiers; + * only names with a scope or a path separator count. + * @param rows - Parsed YAML value; anything but an entry array is ignored. + * @param names - Package names collected so far. + */ +function moduleNamesOf(rows: unknown, names: Set): void { + if (!Array.isArray(rows)) return + for (const row of rows) { + if (typeof row !== 'object' || row === null) continue + const { name, config } = row as { name?: unknown; config?: unknown } + if (typeof name === 'string' && (name.startsWith('@') || name.includes('/'))) { + names.add(packageNameOf(name)) + } + moduleNamesOf(config, names) + } +} + +/** + * Package names the composition names. + * @param config - Composed profile; `!!js` scalars parse under Include's dialect. + * @returns Package names, deduplicated. + */ +function rosterOf(config: string): string[] { + const names = new Set() + moduleNamesOf(yaml.load(config, { schema: entryListSchema }), names) + return [...names] +} + +/** + * Package names the compositions under one config tree name. + * @param root - Directory to walk. + * @returns Package names, deduplicated. + */ +function treeRosterOf(root: string): string[] { + const names = new Set() + const walk = (directory: string): void => { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const absolute = join(directory, entry.name) + if (entry.isDirectory()) { + walk(absolute) + continue + } + if (!entry.name.endsWith('.yml') && !entry.name.endsWith('.yaml')) continue + moduleNamesOf(yaml.load(readFileSync(absolute, 'utf8'), { schema: entryListSchema }), names) + } + } + walk(root) + return [...names] +} + +/** + * Resolve one dependency the way Node does: walk up from the importer. + * @param fromDirectory - Directory to start at. + * @param name - Package name. + * @returns The real path of the package directory, or undefined. + */ +function resolveDependency(fromDirectory: string, name: string): string | undefined { + let directory = fromDirectory + for (;;) { + const candidate = join(directory, 'node_modules', name) + if (existsSync(join(candidate, 'package.json'))) return realpathSync(candidate) + const parent = dirname(directory) + if (parent === directory) return undefined + directory = parent + } +} + +/** + * Collect files under one directory. Traversal mechanics live here — nested + * `node_modules` never mounts (the image is flat) and dot directories are + * tooling residue at any depth — while every judgement call comes in through + * `keep` (the {@link EXCLUDE} tables and the npm publish view). + * @param root - Source directory. + * @param into - Image entries to add to. + * @param prefix - Image path prefix. + * @param keep - Filter over root-relative paths. + */ +function collectTree(root: string, into: ImageFiles, prefix: string, keep: (relativePath: string) => boolean): void { + const walk = (directory: string): void => { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + if (entry.isDirectory()) { + if (entry.name === 'node_modules') continue + if (entry.name.startsWith('.')) continue + walk(join(directory, entry.name)) + continue + } + if (!entry.isFile()) continue + const absolute = join(directory, entry.name) + const relativePath = relative(root, absolute).replaceAll('\\', '/') + if (!keep(relativePath)) continue + into[`${prefix}/${relativePath}`] = readFileSync(absolute) + } + } + walk(root) +} + +/** + * Predicate for npm's `files` allowlist, with standard glob semantics + * (picomatch). A pattern admits the path itself and everything under it, so a + * bare directory name publishes its whole tree; `!` patterns subtract from the + * admitted set; package.json is always published. + * @param patterns - The package.json `files` array. + * @returns Predicate over package-root-relative paths. + */ +function publishedFilter(patterns: readonly unknown[]): (path: string) => boolean { + const strings = patterns.filter((pattern): pattern is string => typeof pattern === 'string') + const normalize = (pattern: string): string => pattern.replace(/^\.\//, '').replace(/\/+$/, '') + const widen = (pattern: string): string[] => [pattern, `${pattern}/**`] + const positive = strings.filter(pattern => !pattern.startsWith('!')).map(normalize).flatMap(widen) + const negative = strings.filter(pattern => pattern.startsWith('!')).map(pattern => normalize(pattern.slice(1))).flatMap(widen) + const admits = picomatch(positive, { dot: true }) + const denies = negative.length > 0 ? picomatch(negative, { dot: true }) : (): boolean => false + return path => path === 'package.json' || (admits(path) && !denies(path)) +} + +/** What the reachability sweep kept, transformed, and dropped. */ +interface SweepOutcome { + readonly swept: ImageFiles + readonly transform: TransformOutcome + readonly javascriptEntries: number + readonly droppedJavascriptEntries: number + /** Third-party requests that resolve nowhere; loud at require time if hit. */ + readonly unresolvedExternalRequests: readonly string[] +} + +/** + * Keep only the JavaScript the worker can reach, transforming it on the way. + * + * Roots are the export faces of every materialized workspace and vendored + * package — the harness addresses them by constructed name at runtime (Loader + * rows, typert faces, delegating providers such as `-auto` pickers), so the + * sweep prunes files only inside third-party packages — plus the worker + * assembly's own image entries. Resolution runs the runtime loader's own + * algorithm over the candidate set, so pack-time reachability and boot-time + * resolution cannot drift, and a request that resolves nowhere — an undeclared + * or missing dependency — fails the pack rather than the boot. + * + * Two entry classes stay out of the walk by rule: page assets + * ({@link PAGE_ASSETS}) are evaluated by the page's module system, and + * non-JavaScript entries always stay because data reads go through fs paths + * this pass cannot see. + * @param files - Candidate entries after the publish-view filter. + * @param options - Pack options carrying the sweep roots. + * @param rootPackages - Roster package names from the workspace. + * @param root - Virtual root the candidates mount under. + * @returns The final entries plus the sweep's counts. + */ +function sweepImage( + files: ImageFiles, + options: PackOptions, + rootPackages: readonly string[], + root: string, +): SweepOutcome { + const decoder = new TextDecoder() + const encoder = new TextEncoder() + const vfs = new MemoryVfs() + for (const [name, bytes] of Object.entries(files)) { + if (name.endsWith('/')) vfs.seedDirectory(`${root}/${name}`) + else vfs.seed(`${root}/${name}`, bytes) + } + // The walk resolves static specifiers and never loads them, so one shared + // factory stands for every replaced module. + const stub = (): unknown => ({}) + const loader = new WorkerModuleLoader({ + vfs, + root, + staticModules: Object.fromEntries(Object.keys(MODULE_PROXIES).map(name => [name, stub])), + staticModulePrefixes: Object.fromEntries(Object.keys(MODULE_PROXY_PREFIXES).map(name => [name, stub])), + }) + + const queue: { specifier: string; from: string; importer: string; meta?: boolean }[] = (options.entries ?? IMAGE_ENTRY_SEEDS) + .map(specifier => ({ specifier, from: root, importer: 'worker assembly entry' })) + for (const name of rootPackages) { + const manifestBytes = files[`node_modules/${name}/package.json`] + if (manifestBytes === undefined) continue // materialize already reported it under `missing` + let manifest: { exports?: Record } + try { + manifest = JSON.parse(decoder.decode(manifestBytes)) as typeof manifest + } catch { + continue + } + // Every non-wildcard face is a root; a face resolving onto a page asset is + // kept verbatim below rather than excluded here. + const subpaths = manifest.exports === undefined + ? ['.'] + : Object.keys(manifest.exports).filter(key => key.startsWith('.') && !key.includes('*')) + for (const subpath of subpaths) { + queue.push({ specifier: subpath === '.' ? name : `${name}/${subpath.slice(2)}`, from: root, importer: `workspace face ${name}` }) + } + } + + const reached = new Map() + const seen = new Set() + const failures: string[] = [] + const tolerated = new Set() + let visited = 0 + let rewritten = 0 + for (let entry = queue.shift(); entry !== undefined; entry = queue.shift()) { + const { specifier, from, importer } = entry + let resolution + try { + resolution = loader.resolve(specifier, from) + } catch (reason) { + // Our own packages must declare what they request: an unresolvable + // request from a workspace or vendored file, a roster face, or the + // assembly entries is a pack defect. Third-party files keep the runtime + // philosophy instead — platform-dispatch branches the worker never + // evaluates may request node-only modules, and such a request fails loud + // at require time if it ever runs. + const external = importer.startsWith('node_modules/') && !importer.startsWith('node_modules/@deepseek-ai/') + // A meta-resolve request is a URL mapping, not a load: a missing target + // is tolerable from any importer — the call throws if it ever runs. + if (external || entry.meta === true) tolerated.add(`${importer}: "${specifier}"`) + else failures.push(`${importer}: "${specifier}" — ${(reason as Error).message}`) + continue + } + if (resolution.kind === 'static') continue + const path = resolution.path + if (seen.has(path)) continue + seen.add(path) + const key = path.slice(root.length + 1) + const bytes = files[key] + if (bytes === undefined) continue + if (!/\.[cm]?js$/.test(key) || pageAsset(key)) { + reached.set(key, bytes) + continue + } + visited += 1 + const { code, lowered, moduleRequests, metaResolveRequests } = lowerModuleSource({ filename: `/${key}`, source: decoder.decode(bytes) }) + if (lowered) rewritten += 1 + reached.set(key, lowered ? encoder.encode(code) : bytes) + const directory = path.slice(0, path.lastIndexOf('/')) + for (const request of moduleRequests) queue.push({ specifier: request, from: directory, importer: key }) + for (const request of metaResolveRequests) queue.push({ specifier: request, from: directory, importer: key, meta: true }) + } + if (failures.length > 0) { + throw new Error( + `vfs image: ${String(failures.length)} unresolvable module request(s); ` + + 'an undeclared or missing dependency fails the pack rather than the boot:\n ' + + failures.join('\n '), + ) + } + + const swept: ImageFiles = {} + let javascriptEntries = 0 + let dropped = 0 + for (const [name, bytes] of Object.entries(files)) { + const isJs = /\.[cm]?js$/.test(name) + if (!isJs || pageAsset(name)) { + swept[name] = bytes + if (isJs) javascriptEntries += 1 + continue + } + const kept = reached.get(name) + if (kept === undefined) { + dropped += 1 + continue + } + swept[name] = kept + javascriptEntries += 1 + } + return { + swept, + transform: { visited, rewritten }, + javascriptEntries, + droppedJavascriptEntries: dropped, + unresolvedExternalRequests: [...tolerated], + } +} + +/** + * Drop executable scripts from the image. + * + * A shebang says "program", not "module": nothing in a browser can spawn one and no + * consumer reads their bytes (the packages that expose a launcher path are replaced + * by stubs that answer with a string). They are also the one place top-level `await` + * appears in the closure, which a CommonJS body cannot express. + * @param files - Image entries, mutated. + * @returns The dropped entry names. + */ +function dropExecutables(files: ImageFiles): string[] { + const decoder = new TextDecoder() + const dropped: string[] = [] + for (const [name, bytes] of Object.entries(files)) { + if (!/\.[cm]?js$/.test(name)) continue + if (decoder.decode(bytes.subarray(0, 2)) !== '#!') continue + dropped.push(name) + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete -- the image is a plain path map + delete files[name] + } + return dropped +} + +/** + * Materialize the dependency closure of every roster package into the image. + * @param roster - Package names to start from. + * @param options - Pack options carrying the workspace index and resolution root. + * @returns Image entries, per-package file counts, and unresolved dependencies. + */ +function materialize( + roster: readonly string[], + options: PackOptions, +): { files: ImageFiles; packages: Map; missing: string[] } { + const files: ImageFiles = {} + const packages = new Map() + const missing: string[] = [] + const replaced = new Set(REPLACED_EXTERNAL_PACKAGES) + const queue: { name: string; from: string }[] = roster.map(name => ({ name, from: options.resolveFrom })) + + for (let entry = queue.shift(); entry !== undefined; entry = queue.shift()) { + const { name, from } = entry + if (packages.has(name) || replaced.has(name)) continue + const directory = options.workspaces.get(name) ?? resolveDependency(from, name) + if (directory === undefined) { + missing.push(`${name} (from ${relative(options.resolveFrom, from) || '.'})`) + continue + } + const manifest = readJson(join(directory, 'package.json')) + const prefix = `node_modules/${name}` + const before = Object.keys(files).length + if (options.workspaces.has(name)) { + // A workspace package ships the slice npm would publish — `files` + // filters out build residue like the tsc mirror under lib/types/ — + // minus the workspace exclude table (no sources, no dist: the page + // serves its own assets). + const published = Array.isArray(manifest.files) ? publishedFilter(manifest.files) : undefined + collectTree(directory, files, prefix, relativePath => + !workspaceExcluded(relativePath) && (published === undefined || published(relativePath))) + } else { + collectTree(directory, files, prefix, relativePath => !excluded(relativePath)) + } + packages.set(name, Object.keys(files).length - before) + for (const field of ['dependencies', 'peerDependencies'] as const) { + // npm semantics: a peer is provided by the consumer. For an external + // package the consumer is the page (react behind the prebuilt client + // bundles), so its peer edges never bind the worker. Workspace and + // vendored packages declare real runtime seams as peers + // (@deepseek-ai/cordis is a peerDependency of every harness package), + // so their peer edges stay on the chain. + if (field === 'peerDependencies' && !options.workspaces.has(name)) continue + const dependencies = manifest[field] + if (typeof dependencies !== 'object' || dependencies === null) continue + for (const dependency of Object.keys(dependencies)) queue.push({ name: dependency, from: directory }) + } + } + return { files, packages, missing } +} + +/** Gzip header byte that records the packing platform; RFC 1952 §2.3.1 spells 255 "unknown". */ +const GZIP_OS_UNKNOWN = 255 + +/** Offset of that byte in the gzip member header. */ +const GZIP_OS_OFFSET = 9 + +/** + * Compress the archive into one gzip member the same tree always produces + * byte for byte. + * + * Two header fields would otherwise carry build facts: zlib writes no + * modification time and no original file name for a buffer (`gzipSync` is handed + * neither), and it fills the operating-system byte from the platform it was built + * for, which would make the same tree pack differently on Linux and macOS. That + * byte is overwritten with "unknown" — every gzip reader ignores it, and the + * artifact stops depending on where it was packed. + * @param archive - the ustar archive. + * @returns the compressed image bytes. + */ +function compressImage(archive: Uint8Array): Uint8Array { + const compressed = gzipSync(archive, { level: 9 }) + compressed[GZIP_OS_OFFSET] = GZIP_OS_UNKNOWN + return compressed +} + +/** + * Pack one VFS image. + * + * The manifest's claim is all-or-nothing: it names the one contract every packed body + * was emitted against. A module the transform cannot express therefore fails the pack + * rather than downgrading the image, because a mostly-transformed image boots into + * errors far from their cause. + * @param options - Composition, package index, and paths. + * @returns The compressed image plus what went into it. + * @throws When a config tree or workspace directory named in the options is missing, + * because a silently thinner image fails much later and much less clearly. + */ +export function packVfsImage(options: PackOptions): PackResult { + const root = options.root ?? DEFAULT_ROOT + const encoder = new TextEncoder() + const configTrees = options.configTrees ?? [] + for (const tree of configTrees) { + if (!existsSync(tree.directory)) { + throw new Error(`vfs image: config tree ${tree.mount} is missing at ${tree.directory}`) + } + } + + const roster = [...new Set([ + ...rosterOf(options.config), + ...configTrees.filter(tree => tree.scanRoster === true).flatMap(tree => treeRosterOf(tree.directory)), + ])] + const { files, packages, missing } = materialize(roster, options) + + files[CONFIG_PATH] = encoder.encode(options.config) + for (const tree of configTrees) collectTree(tree.directory, files, tree.mount, relativePath => !excluded(relativePath)) + + const executables = dropExecutables(files) + const rootPackages = [...packages.keys()].filter(name => options.workspaces.has(name)) + const { swept, transform, javascriptEntries, droppedJavascriptEntries, unresolvedExternalRequests } = + sweepImage(files, options, rootPackages, root) + + swept[MANIFEST_PATH] = encoder.encode(`${JSON.stringify({ + root, + profile: options.profile, + [CONTRACT_FIELD]: WRAPPER_CONTRACT, + javascriptEntries, + visitedEntries: transform.visited, + rewrittenEntries: transform.rewritten, + }, null, 2)}\n`) + + for (const directory of options.emptyDirectories ?? IMAGE_EMPTY_DIRECTORIES) { + swept[directory] = new Uint8Array(0) + } + + return { + image: compressImage(packTar(swept)), + files: swept, + packages, + workspacePackages: [...packages.keys()].filter(name => options.workspaces.has(name)).length, + roster, + missing, + executables, + pageBundles: Object.keys(swept).filter(name => pageAsset(name)), + javascriptEntries, + droppedJavascriptEntries, + unresolvedExternalRequests, + transform, + contract: WRAPPER_CONTRACT, + } +} diff --git a/packages/experimental/webworker-packer/src/repository.ts b/packages/experimental/webworker-packer/src/repository.ts new file mode 100644 index 0000000000..38ec64bd1d --- /dev/null +++ b/packages/experimental/webworker-packer/src/repository.ts @@ -0,0 +1,173 @@ +/** + * Repository knowledge for the packer: where this tree's workspaces, profile + * composition, and config trees are, and how to report a pack. + * + * The library half takes all of this as parameters. Keeping the lookup here is what + * lets the same library pack a different tree, and what keeps `pack.ts` free of + * assumptions about pnpm workspaces or the `dsh` CLI. + * @module @deepseek-ai/dsh-experimental-webworker-packer/src/repository + */ +import { execFileSync } from 'node:child_process' +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, relative } from 'node:path' +import { DSH_HOME_ENV } from '@deepseek-ai/dsh-home-paths' +import type { ConfigTree, PackResult } from './pack.ts' + +/** + * Repository directories scanned for workspace and vendored packages. The + * image only ever materializes runtime packages, which all live here; + * examples, python, and native are never on a roster's dependency chain (the + * native addon is a replaced external). + */ +const WORKSPACE_SCAN_ROOTS = ['vendor', 'packages', 'apps'] + +/** Composition entry point package: the `dsh` CLI, run from source. */ +const CLI_PACKAGE = 'apps/cli' + +/** Composition entry point: the `dsh` CLI, run from source. */ +const CLI_ENTRY = `${CLI_PACKAGE}/src/bin.ts` + +/** + * Index every workspace and vendored package by name. + * @param repoRoot - Absolute repository root. + * @returns Package name to absolute directory. + */ +export function indexWorkspacePackages(repoRoot: string): Map { + const index = new Map() + const visit = (directory: string): void => { + const manifest = join(directory, 'package.json') + if (existsSync(manifest)) { + const name = (JSON.parse(readFileSync(manifest, 'utf8')) as { name?: unknown }).name + if (typeof name === 'string') index.set(name, directory) + // A package root owns its subtree; anything below (test fixtures, + // nested manifests) is not a separate workspace package. + return + } + for (const entry of readdirSync(directory, { withFileTypes: true })) { + if (!entry.isDirectory()) continue + if (entry.name === 'node_modules' || entry.name.startsWith('.')) continue + visit(join(directory, entry.name)) + } + } + for (const scanRoot of WORKSPACE_SCAN_ROOTS) { + const absolute = join(repoRoot, scanRoot) + if (existsSync(absolute)) visit(absolute) + } + return index +} + +/** + * Compose one profile through the real CLI dump path, leaving `!!js` + * unevaluated. The dump runs against a throwaway Harness home and default + * layers only, so the image is the shipped profile: the machine's `$DSH_HOME` + * — its profile manifest with locally installed bundles, and its patch files — + * would otherwise leak this machine's plugins into the image and break the + * same-tree-same-bytes guarantee. + * @param repoRoot - Absolute repository root. + * @param profile - Profile name to compose. + * @returns The composed YAML. + */ +export function composeProfile(repoRoot: string, profile: string): string { + const home = mkdtempSync(join(tmpdir(), 'dsh-pack-home-')) + try { + return execFileSync( + process.execPath, + ['--import', 'tsx/esm', join(repoRoot, CLI_ENTRY), '--profile', profile, '--dump-default-config'], + { cwd: repoRoot, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, env: { ...process.env, [DSH_HOME_ENV]: home } }, + ) + } finally { + rmSync(home, { recursive: true, force: true }) + } +} + +/** One `dsh.configTrees` declaration entry, validated field by field. */ +interface ConfigTreeDeclaration { + mount: string + path: string + scanRoster?: boolean +} + +/** + * Config trees the CLI package declares for deployment images + * (`dsh.configTrees` in its package.json): `path` is relative to the CLI + * package root, `mount` is the image path, `scanRoster` feeds the tree's yml + * plugin rows into the pack roster. The CLI owns its config layout; this + * reader follows the declaration instead of naming directories. A malformed + * declaration refuses the pack. + * @param repoRoot - Absolute repository root. + * @returns Trees with absolute source directories. + */ +export function configTrees(repoRoot: string): ConfigTree[] { + const packageDir = join(repoRoot, CLI_PACKAGE) + const manifest = JSON.parse(readFileSync(join(packageDir, 'package.json'), 'utf8')) as { + dsh?: { configTrees?: unknown } + } + const declared = manifest.dsh?.configTrees + if (declared === undefined) return [] + if (!Array.isArray(declared)) { + throw new Error(`vfs image: ${CLI_PACKAGE} dsh.configTrees must be an array`) + } + const mounts = new Set() + return declared.map((entry, index) => { + const tree = entry as Partial | null + const at = `${CLI_PACKAGE} dsh.configTrees[${String(index)}]` + if (tree === null || typeof tree !== 'object' + || typeof tree.mount !== 'string' || tree.mount === '' + || typeof tree.path !== 'string' || tree.path === '' + || (tree.scanRoster !== undefined && typeof tree.scanRoster !== 'boolean')) { + throw new Error(`vfs image: ${at} must declare a string mount, a string path, and an optional boolean scanRoster`) + } + if (mounts.has(tree.mount)) { + throw new Error(`vfs image: ${at} repeats mount ${JSON.stringify(tree.mount)}`) + } + mounts.add(tree.mount) + return { + mount: tree.mount, + directory: join(packageDir, tree.path), + ...tree.scanRoster === undefined ? {} : { scanRoster: tree.scanRoster }, + } + }) +} + +/** + * Render one pack as the lines a build log should carry. + * + * Refusals and unresolved dependencies are the two states a reader must not miss, so + * they are spelled out rather than counted. + * @param result - What the pack produced. + * @param repoRoot - Absolute repository root, for relative paths. + * @param outputFile - Where the image was written. + * @returns Lines to print. + */ +export function describePack(result: PackResult, repoRoot: string, outputFile: string): string[] { + const sizeOf = (prefix: string): number => Object.entries(result.files) + .filter(([name]) => name.startsWith(prefix)) + .reduce((sum, [, bytes]) => sum + bytes.byteLength, 0) + const megabytes = (bytes: number): string => `${(bytes / 1024 / 1024).toFixed(2)} MB` + const workspaceCount = result.workspacePackages + const heaviest = [...result.packages.entries()] + .map(([name, count]) => ({ name, count, bytes: sizeOf(`node_modules/${name}/`) })) + .sort((left, right) => right.bytes - left.bytes) + .slice(0, 12) + + return [ + `vfs image: ${relative(repoRoot, outputFile)}`, + ` roster entries ${String(result.roster.length)}`, + ` packages ${String(result.packages.size)} (${String(workspaceCount)} workspace)`, + ` files ${String(Object.keys(result.files).length)}`, + ` raw ${megabytes(Object.values(result.files).reduce((sum, bytes) => sum + bytes.byteLength, 0))}`, + ` compressed ${megabytes(result.image.byteLength)}`, + ` config + presets ${megabytes(sizeOf('config/'))}`, + ` javascript entries ${String(result.javascriptEntries)} (dropped ${String(result.executables.length)} executable scripts, ${String(result.pageBundles.length)} page bundles verbatim)`, + ` wrapper contract ${result.contract}`, + ` transform ${String(result.transform.rewritten)} of ${String(result.transform.visited)} reached entries rewritten, ${String(result.droppedJavascriptEntries)} unreachable dropped`, + ` unresolved ${String(result.unresolvedExternalRequests.length)} third-party request(s) left to fail loud at require time`, + ' heaviest packages:', + ...heaviest.map(entry => ` ${entry.bytes.toString().padStart(9)} B ${entry.name} (${String(entry.count)} files)`), + ...result.missing.length === 0 + ? [] + : [' unresolved dependencies:', ...result.missing.map(entry => ` ${entry}`)], + '', + ] +} diff --git a/packages/experimental/webworker-packer/src/rules.ts b/packages/experimental/webworker-packer/src/rules.ts new file mode 100644 index 0000000000..96c1fa0265 --- /dev/null +++ b/packages/experimental/webworker-packer/src/rules.ts @@ -0,0 +1,70 @@ +/** + * Pack rule tables: the one place the image's include/exclude decisions live. + * Patterns are picomatch globs. Exclude patterns match tree-root-relative + * paths (so `src/**` drops only a root-level source tree), page-asset + * patterns match image paths. Traversal mechanics — nested `node_modules` + * flattening and dot-directory pruning — stay in the collector; these tables + * hold the judgement calls. + */ + +/** + * Paths dropped from every collected tree. Source and test trees never + * resolve at runtime (the artifact plane ships `lib/`), and sourcemaps, + * declarations, and archives never resolve either while dominating the byte + * count. + */ +export const EXCLUDE: readonly string[] = [ + 'src/**', + 'tests/**', + 'test/**', + '__tests__/**', + 'coverage/**', + '**/*.map', + '**/*.tsbuildinfo', + '**/*.tgz', + '**/*.tar', + '**/*.tar.gz', + '**/*.d.ts', + '**/*.d.mts', + '**/*.d.cts', +] + +/** + * Additional paths dropped from workspace packages only. A workspace `dist/` + * is a page-asset tree the static deployment serves itself; external packages + * legitimately ship runtime code under `dist/`. + */ +export const EXCLUDE_WORKSPACE: readonly string[] = [ + 'dist/**', +] + +/** + * Image paths that belong to the PAGE, not to the worker's loader. + * + * A package's `lib/client.js` is its browser bundle behind the `./client` + * export: the page's own module system evaluates it with its own wrapper, + * which has no ambient-store parameter. Transforming those bodies would + * inject calls the page cannot resolve, so they ship verbatim — and the + * manifest's all-or-nothing claim stays true, because the worker loader never + * evaluates them (the tunnel serves them as bytes). + */ +export const PAGE_ASSETS: readonly string[] = [ + 'node_modules/*/lib/client.js', + 'node_modules/@*/*/lib/client.js', +] + +/** + * Image specifiers the worker assembly requires directly, beyond the composed + * roster: they are requested by worker-bundle code, so no image file + * references them and the reachability sweep must seed them as roots. Keep in + * step with the literal `require`/`resolve` calls in the runtime's + * `worker-host.ts`. + */ +export const IMAGE_ENTRY_SEEDS: readonly string[] = [ + '@deepseek-ai/dsh-app-boot', + '@deepseek-ai/dsh-cmdline', + '@deepseek-ai/dsh-host-apiproxy', + '@deepseek-ai/cordis', + '@deepseek-ai/cordis-plugin-include', + 'js-yaml', +] diff --git a/packages/experimental/webworker-packer/src/transform-image.ts b/packages/experimental/webworker-packer/src/transform-image.ts new file mode 100644 index 0000000000..18b525c06e --- /dev/null +++ b/packages/experimental/webworker-packer/src/transform-image.ts @@ -0,0 +1,26 @@ +/** + * The wrapper contract packed bodies are emitted against, and the image-entry + * types the pack pass consumes. + * + * One transform serves both sides — the pack pass lowers with the runtime's + * own `lowerModuleSource`, never a reimplementation — and the image records + * the contract version it was lowered against. Bodies emitted against a + * different wrapper contract are refused at mount time rather than + * half-working at run time. + * @module @deepseek-ai/dsh-experimental-webworker-packer/src/transform-image + */ +import { LOWERING_VERSION } from '@deepseek-ai/dsh-experimental-webworker-runtime' + +/** Image entries, keyed by their path relative to the virtual root. */ +export type ImageFiles = Record + +/** Wrapper contract the packed bodies are emitted against. */ +export const WRAPPER_CONTRACT: string = LOWERING_VERSION + +/** What one pack-time transform pass did. */ +export interface TransformOutcome { + /** JavaScript entries visited. */ + readonly visited: number + /** How many changed; the rest were already in final form. */ + readonly rewritten: number +} diff --git a/packages/experimental/webworker-packer/tests/image-loadable.spec.ts b/packages/experimental/webworker-packer/tests/image-loadable.spec.ts new file mode 100644 index 0000000000..07ccb0986f --- /dev/null +++ b/packages/experimental/webworker-packer/tests/image-loadable.spec.ts @@ -0,0 +1,138 @@ +/** + * End-to-end spec of the packer's actual product: an image this package builds must + * mount in the runtime's VFS and be `require`-able by the runtime's module loader, + * which holds no transform of its own. + * + * That last part is the point. "It boots" only proves nothing crashed; the loader + * wraps module bodies exactly as the image holds them, so the pack-time pass is the + * only thing that can make them wrappable. The refusal case is the positive + * evidence: restore one un-lowered body and the same setup fails loud. + * + * A small synthetic composition rather than the real profile: packing the full + * closure takes tens of seconds. The path under test — compose, materialize, + * transform, tar, compress, inflate, mount, require — is the same one. + * + * ONE module instance: every runtime import here goes through `src/`, because the VFS + * and the active loader are module-level slots. The "starts with nothing loaded" + * case asserts the instance the spec holds is the one that did the work. + */ +import { existsSync } from 'node:fs' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { describe, expect, it } from 'vitest' +import { createNodeBuiltins, REPLACED_PREFIXES } from '@deepseek-ai/dsh-experimental-webworker-runtime/src/node/builtins.ts' +import { WorkerModuleLoader } from '@deepseek-ai/dsh-experimental-webworker-runtime/src/module-system/module-loader.ts' +import { inflateImage } from '@deepseek-ai/dsh-experimental-webworker-runtime/src/storage/image-gzip.ts' +import { loadVfsImage } from '@deepseek-ai/dsh-experimental-webworker-runtime/src/storage/memory.ts' +import { indexWorkspacePackages } from '../src/repository.ts' +import { DEFAULT_ROOT, MANIFEST_PATH, packVfsImage } from '../src/pack.ts' + +const repoRoot = fileURLToPath(new URL('../../../../', import.meta.url)) + +/** A leaf workspace package: real build output, no dependencies to drag in. */ +const SUBJECT = '@deepseek-ai/dsh-timeout' + +const workspaces = indexWorkspacePackages(repoRoot) + +/** + * The pack consumes built `lib/` output. An unbuilt checkout (the unit + * coverage lane runs before any build) self-skips; the built lanes and every + * preview build exercise this same path against real artifacts. + */ +const subjectBuilt = existsSync(join(repoRoot, 'packages/util/timeout/lib/index.js')) + +let memo: ReturnType | undefined +const packed = (): ReturnType => memo ??= packVfsImage({ + // The composition's own shape: one entry per plugin, `name:` on its own line. + config: `- id: subject\n name: '${SUBJECT}'\n`, + profile: 'image-loadable-check', + workspaces, + resolveFrom: repoRoot, + // Synthetic composition: nothing boots the worker assembly, so its default + // image entries must not be demanded of this one-package closure. + entries: [], +}) + +/** The image's archive, inflated once: mounting reads the tar, not the gzip member. */ +let archiveMemo: Uint8Array | undefined +const archive = async (): Promise => + archiveMemo ??= await inflateImage(packed().image, 'the image this spec packed') + +;(subjectBuilt ? describe : describe.skip)('packed image', () => { + it('materializes the roster with every dependency resolved', () => { + const result = packed() + expect(workspaces.has(SUBJECT)).toBe(true) + expect(result.roster).toEqual([SUBJECT]) + expect(result.packages.has(SUBJECT)).toBe(true) + expect(result.missing).toEqual([]) + }) + + it('records the wrapper contract in the manifest and rewrote what it visited', () => { + const result = packed() + expect(Object.hasOwn(result.files, MANIFEST_PATH)).toBe(true) + const manifest = JSON.parse(new TextDecoder().decode(result.files[MANIFEST_PATH])) as { lowered: string } + expect(manifest.lowered).toBe(result.contract) + expect(result.transform.rewritten).toBeGreaterThan(0) + }) + + it('writes one gzip member whose header records no build facts', () => { + const image = packed().image + // RFC 1952 §2.3: magic, deflate, then the flag byte — no FNAME (0x08) or + // FCOMMENT, a zero modification time, and "unknown" for the packing system. + expect([...image.slice(0, 4)]).toEqual([0x1f, 0x8b, 0x08, 0x00]) + expect([...image.slice(4, 8)]).toEqual([0, 0, 0, 0]) + expect(image[9]).toBe(255) + }) + + it('packs the same tree to the same bytes', () => { + // The preview build compares a freshly packed image against the shipped one, + // so anything the compressor takes from its environment would read as a + // changed tree. + const again = packVfsImage({ + config: `- id: subject\n name: '${SUBJECT}'\n`, + profile: 'image-loadable-check', + workspaces, + resolveFrom: repoRoot, + entries: [], + }) + expect(Buffer.from(again.image).equals(Buffer.from(packed().image))).toBe(true) + }) + + it('mounts and requires through the real loader, which carries no transform', async () => { + const vfs = loadVfsImage(await archive(), DEFAULT_ROOT) + expect(vfs.existsSync(`${DEFAULT_ROOT}/node_modules/${SUBJECT}/lib/index.js`)).toBe(true) + + const loader = new WorkerModuleLoader({ + vfs, + root: DEFAULT_ROOT, + staticModules: createNodeBuiltins(), + staticModulePrefixes: REPLACED_PREFIXES, + }) + // The loader this spec reads counters from must be the one that did the + // requiring; a second instance would report an empty cache trivially. + expect(loader.usage().modules).toBe(0) + + const required = loader.requireFrom(`${DEFAULT_ROOT}/workspace`)(SUBJECT) as Record + expect(typeof required.timeoutOf).toBe('function') + expect(loader.usage().modules).toBeGreaterThan(0) + }) + + it('refuses a body the packer did not lower, naming the image', async () => { + // The case above only proves the packed bytes are wrappable. This is the + // other half: the loader has no transform to fall back on, so an entry the + // collector missed must fail loud against the image rather than boot. + const vfs = loadVfsImage(await archive(), DEFAULT_ROOT) + vfs.seed( + `${DEFAULT_ROOT}/node_modules/${SUBJECT}/lib/index.js`, + new TextEncoder().encode('export const timeoutOf = () => 0\n'), + ) + const loader = new WorkerModuleLoader({ + vfs, + root: DEFAULT_ROOT, + staticModules: createNodeBuiltins(), + staticModulePrefixes: REPLACED_PREFIXES, + }) + expect(() => loader.requireFrom(`${DEFAULT_ROOT}/workspace`)(SUBJECT)) + .toThrow(/still carries module syntax, so the image was not lowered by the packer/) + }) +}) diff --git a/packages/experimental/webworker-packer/tsconfig.json b/packages/experimental/webworker-packer/tsconfig.json new file mode 100644 index 0000000000..7039bfa53b --- /dev/null +++ b/packages/experimental/webworker-packer/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "lib/types", + "types": [ + "node" + ] + }, + "include": [ + "src" + ], + "references": [ + { + "path": "../webworker-runtime" + }, + { + "path": "../../util/home-paths" + }, + { + "path": "../../runtime-diagnostics/invariants" + } + ] +} diff --git a/packages/experimental/webworker-packer/tsdown.config.ts b/packages/experimental/webworker-packer/tsdown.config.ts new file mode 100644 index 0000000000..b948ddb571 --- /dev/null +++ b/packages/experimental/webworker-packer/tsdown.config.ts @@ -0,0 +1,18 @@ +import { defineConfig } from 'tsdown' + +/** + * The packer ships TWO entries: the library (`index`) and the `dsh-pack-vfs-image` + * CLI (`bin`), the latter referenced by package.json `bin`. The root tsdown + * builds only `lib/types/index.js`, so this override adds `lib/types/bin.js`. + * Declarations come from `tsc -b` (dts: false), matching every package. + */ +export default defineConfig({ + entry: ['lib/types/index.js', 'lib/types/bin.js', 'lib/types/invariant.js'], + outDir: 'lib', + format: ['esm'], + platform: 'node', + target: 'es2024', + fixedExtension: false, + dts: false, + clean: false, +}) diff --git a/packages/experimental/webworker-runtime/README.i18n.yaml b/packages/experimental/webworker-runtime/README.i18n.yaml new file mode 100644 index 0000000000..0dced963dc --- /dev/null +++ b/packages/experimental/webworker-runtime/README.i18n.yaml @@ -0,0 +1,6 @@ +# Bilingual-pair consistency record (docs/i18n/README.md): the git blob hash of each +# side as of the last confirmed-consistent state. Both languages carry equal authority; +# after editing either side, bring the other along and re-record with: +# pnpm run verify-translation-pairing --write packages/experimental/webworker-runtime/README.md +README.md: b82c65b981be6a9405ae72e3a24a42c68b52696e +README.zh.md: 97160641a38095026d5103f2e423846bfb41d5a6 diff --git a/packages/experimental/webworker-runtime/README.md b/packages/experimental/webworker-runtime/README.md new file mode 100644 index 0000000000..b82c65b981 --- /dev/null +++ b/packages/experimental/webworker-runtime/README.md @@ -0,0 +1,33 @@ +# `@deepseek-ai/dsh-experimental-webworker-runtime` + +English | [中文](README.zh.md) + +The browser worker host: the whole harness plugin tree runs inside one dedicated Web Worker, for preview deployments and packaging regressions ([experimental stance](../../../.agents/notes/implemented/architecture/2026-08-20-webworker-pack-lowering-and-preview.md)). The worker inflates a packed VFS image off its download and mounts it in memory, loads its modules through a CommonJS wrapper loader, and serves the page over a postMessage tunnel that speaks plain HTTP. + +Three artifacts from one tsdown pipeline: + +- **`lib/index.js` (assembly library)** — `createWorkerHost`/`startWorkerHost` mount the image (`storage/`), install the module loader (`module-system/`) and the `process` shim, boot the tree through the image's own `dsh-app-boot`, and hand the tunnel its serving seams. The image layout contract (`image-layout.ts`: virtual root, config/manifest paths, empty directories, the `lowered` wrapper-contract gate) is shared with the packer. Boot patches force the deployment-shaped rows: frontend serving off, JSONL session logs on the plaintext path, preset roots onto the image's `config/agent-presets`. +- **`lib/worker.js` (worker bundle)** — the assembly plus this package's Node-compatibility layer as one self-contained ES module. The module proxy table (`module-proxies.ts`) is the only platform fork: `node:*` builtins over VFS/tunnel/browser primitives, structural stubs that fail loud on the console for what a browser cannot do, and replaced externals. AsyncLocalStorage carries sync-stack causality across `await` through the snapshot/restore faces the pack-time lowering injects. The worker holds no compiler: an image the packer did not lower is refused at mount ([note](../../../.agents/notes/implemented/architecture/2026-08-20-webworker-pack-lowering-and-preview.md)). +- **`src/shell/` (the worker's own process layer)** — a browser worker cannot fork, so `node:child_process` is not a stub but an implementation: `spawn` starts the command in its own Web Worker — this same bundle, told by its first frame to be a shell process — and reports it through the `ChildProcess` surface the subprocess service consumes. The command runs off the host's thread, `SIGKILL` terminates it whatever it is doing, and it reaches the VFS only by message (the host serves those frames). The grammar is `@yarnpkg/parsers`' `parseShell`; this package owns the evaluator (pipelines, `&&`/`||`, subshells, redirections, expansion, globs) and the command table, which is the only `/bin` that exists — a name it does not hold reports `command not found`, and `execSync`/`fork` still refuse, because they need a real process. +- **`lib/client.js` (page half)** — `connectWorkerHost(worker, { image? })` completes the pre-Cordis handshake: the opening `init` frame carries the image URL (the one deployment-shaped input), the boot payload delivers the structured index-injection table, and `applyIndexInjections` executes it before the shell entry runs. The tunnel exposes fetch-shaped transport, the API client, and `loadBundle` for the shell's boot seam. + +Acceptance lives in `apps/web/tests/preview-boot.e2e.ts`, which serves the real built pages and drives the worker boot in headless Chromium. + +## Model Experience + +None, as this package only hosts the tree in a browser worker and answers its `node:*` calls; every model-facing registration belongs to the plugins it boots. + +#### KV Cache effect + +None; this package neither assembles nor sends a provider request. + +## Known Limitations and Deferred Work + +- **The worker composition writes plaintext session logs** (`compression: 'none'` boot patch): it carries no Zstandard codec, so exported logs are `.jsonl`, never `.jsonl.zstd`. +- **The skill catalog is never cached in the worker** — `skill-filesystem` watches its roots through `node:fs.watchFile`, which this package refuses, so every discovery pass returns an incomplete observation and re-scans. Discovery itself stays correct; the cost is a re-scan on every pass. +- **`node:vm`, `node:net`, `node:sqlite`, `node:worker_threads` are structural stubs**: every call reports its refusal on the console and throws. Rows needing a real process or realm isolation cannot run here. +- **The bash tool runs only under `danger-full-access`**: a browser has no kernel to confine a command with, so `ctx.sandbox.confine` fails loud in every other permission preset and the command never starts. The mode is the deployment's own user-facing switch, not a worker-specific composition. +- **The worker bundle pins a path inside `@yarnpkg/parsers`** — the build resolves the package's own `lib/shell.js` instead of its root, whose barrel also re-exports the Syml parser and so drags js-yaml into a bundle that never parses that format (around 175 kB, plus its module body at worker start). The path is derived from the package manifest, so a layout change fails the build rather than reinstating the barrel; upgrading the dependency means re-checking that the shell parser still lives there. +- **The shell is not bash**: no loops, functions, `case`, job control, or process substitution — the grammar stops at pipelines, `&&`/`||`, subshells, groups, redirections, and expansion. `&` runs its command to completion in place, `sed` accepts only substitution scripts, patterns are JavaScript regular expressions, and the command table holds coreutils only (no `git`, no network tools). +- **A shell process has no synchronous filesystem**: it reads and writes the host's VFS by message, because blocking on a reply would need `SharedArrayBuffer`, which requires a cross-origin isolation GitHub Pages cannot grant. Directory-walking commands therefore cost one round trip per entry, and two concurrent commands can interleave their writes. +- **Transport, worker-host, and page-half coverage needs a browser-grade harness** — the per-file coverage gate is unmet for those modules; unit specs cover storage, ALS, the transform, and the stub contracts. diff --git a/packages/experimental/webworker-runtime/README.zh.md b/packages/experimental/webworker-runtime/README.zh.md new file mode 100644 index 0000000000..97160641a3 --- /dev/null +++ b/packages/experimental/webworker-runtime/README.zh.md @@ -0,0 +1,33 @@ +# `@deepseek-ai/dsh-experimental-webworker-runtime` + +[English](README.md) | 中文 + +浏览器 worker 宿主:整棵 harness 插件树跑在一个 dedicated Web Worker 里,用于预览部署与打包回归([experimental 定位](../../../.agents/notes/implemented/architecture/2026-08-20-webworker-pack-lowering-and-preview.zh.md))。worker 边下载边解压打包好的 VFS 镜像并挂载进内存,经 CommonJS 包装加载器装载模块,并通过一条讲纯 HTTP 的 postMessage 隧道服务页面。 + +一条 tsdown 管线出三个产物: + +- **`lib/index.js`(装配库)**——`createWorkerHost`/`startWorkerHost` 挂载镜像(`storage/`)、安装模块加载器(`module-system/`)与 `process` shim、经镜像自带的 `dsh-app-boot` 启动插件树,并把服务缝隙交给隧道。镜像布局契约(`image-layout.ts`:虚拟根、config/manifest 路径、空目录、`lowered` 包装契约门)与 packer 共享。boot patch 强制部署形态行:关前端静态服务、JSONL 会话日志走明文、preset 根指向镜像内 `config/agent-presets`。 +- **`lib/worker.js`(worker 束)**——装配库加本包的 Node 兼容层,合成一个自含 ES module。模块代理表(`module-proxies.ts`)是唯一平台叉口:`node:*` 内建走 VFS/隧道/浏览器原语,浏览器做不到的走结构化 stub(调用即 console 报错并抛出),外部包整体替换。AsyncLocalStorage 经 pack 时降低注入的 snapshot/restore 面在 `await` 间携带同步栈因果。worker 不带编译器:packer 未降低的镜像在挂载时被拒([note](../../../.agents/notes/implemented/architecture/2026-08-20-webworker-pack-lowering-and-preview.zh.md))。 +- **`src/shell/`(worker 自己的进程层)**——浏览器 worker 无法 fork,所以 `node:child_process` 不是 stub 而是实现:`spawn` 把命令放进它自己的 Web Worker——就是这同一个束,由首帧告诉它「你是 shell 进程」——并以 subprocess 服务消费的 `ChildProcess` 面报告结果。命令不占宿主线程,`SIGKILL` 不管它在干什么都能终止它,而它只能靠消息触达 VFS(由宿主应答这些帧)。语法来自 `@yarnpkg/parsers` 的 `parseShell`;求值器(管道、`&&`/`||`、子 shell、重定向、展开、glob)与命令表由本包自持,而命令表就是这里唯一存在的 `/bin`——表里没有的名字报 `command not found`,`execSync`/`fork` 依然拒绝,因为它们需要真进程。 +- **`lib/client.js`(页面半)**——`connectWorkerHost(worker, { image? })` 完成 pre-Cordis 握手:开局 `init` 帧携带镜像 URL(唯一部署形态输入),boot 载荷送达结构化 index 注入表,`applyIndexInjections` 在壳入口运行前逐行执行。隧道暴露 fetch 形传输、API 客户端与壳启动缝隙用的 `loadBundle`。 + +验收在 `apps/web/tests/preview-boot.e2e.ts`:静态服务真实构建页面,在 headless Chromium 里驱动 worker 启动。 + +## 模型体验 + +无:本包只在浏览器 worker 里承载插件树并应答它的 `node:*` 调用;所有面向模型的注册都属于它启动的那些插件。 + +#### KV Cache 影响 + +无:本包既不组装也不发送 provider 请求。 + +## Known Limitations and Deferred Work + +- **worker 组合写明文会话日志**(`compression: 'none'` boot patch):不带 Zstandard 编解码器,导出日志是 `.jsonl`,不会是 `.jsonl.zstd`。 +- **worker 里的技能目录从不缓存**——`skill-filesystem` 用 `node:fs.watchFile` 监听各个根,而本包拒绝该调用,于是每轮发现都返回不完整观测并重新扫描。发现本身仍然正确,代价是每轮都要重扫。 +- **`node:vm`、`node:net`、`node:sqlite`、`node:worker_threads` 是结构化 stub**:每次调用在 console 报告拒绝并抛出。需要真进程或真 realm 隔离的行在此无法运行。 +- **bash 工具只在 `danger-full-access` 下可用**:浏览器没有内核可以约束命令,因此在其余权限档位下 `ctx.sandbox.confine` 会响亮失败、命令根本不会启动。该档位是部署本身的用户面开关,不是 worker 特有的组合差异。 +- **worker 束钉住了 `@yarnpkg/parsers` 的包内路径**——构建解析到该包自己的 `lib/shell.js` 而非包根,因为包根 barrel 还 re-export 了 Syml 解析器,会把 js-yaml 拖进一个从不解析该格式的束(约 175 kB,外加 worker 启动时的模块体求值)。该路径由包 manifest 派生,包内布局一变即构建期失败、不会静默退回 barrel;升级这个依赖时须复核 shell 解析器是否仍在那里。 +- **这个 shell 不是 bash**:没有循环、函数、`case`、作业控制或进程替换——语法止步于管道、`&&`/`||`、子 shell、group、重定向与展开。`&` 会就地把命令跑完,`sed` 只接受替换脚本,模式是 JavaScript 正则,命令表只有 coreutils(没有 `git`,没有网络工具)。 +- **shell 进程没有同步文件面**:它靠消息读写宿主的 VFS,因为阻塞等待回帧需要 `SharedArrayBuffer`,而那要求 GitHub Pages 给不了的跨源隔离。因此目录遍历类命令每个条目一次往返,并发的两条命令写入可以交错。 +- **transport、worker-host、页面半的覆盖需要浏览器级 harness**——这些模块未达 per-file 覆盖门;单测覆盖 storage、ALS、transform 与 stub 契约。 diff --git a/packages/experimental/webworker-runtime/package.json b/packages/experimental/webworker-runtime/package.json new file mode 100644 index 0000000000..8cac2bd9f7 --- /dev/null +++ b/packages/experimental/webworker-runtime/package.json @@ -0,0 +1,64 @@ +{ + "name": "@deepseek-ai/dsh-experimental-webworker-runtime", + "description": "Browser-only harness runtime: in-memory VFS, module transform and loader, postMessage tunnel, and the dedicated Web Worker assembly, with the Node-compatibility layer that lets the host tree run unchanged", + "version": "0.1.0-rc.8", + "private": true, + "repository": { + "type": "git", + "url": "git+https://github.com/deepseek-ai/deepseek-harness.git", + "directory": "packages/experimental/webworker-runtime" + }, + "type": "module", + "main": "lib/index.js", + "types": "lib/types/index.d.ts", + "exports": { + ".": { + "types": "./lib/types/index.d.ts", + "default": "./lib/index.js" + }, + "./invariant": { + "types": "./lib/types/invariant.d.ts", + "default": "./lib/invariant.js" + }, + "./src/*": "./src/*", + "./package.json": "./package.json", + "./worker": "./lib/worker.js", + "./client": { + "types": "./lib/types/client/index.d.ts", + "default": "./lib/client.js" + } + }, + "license": "MIT", + "dependencies": { + "@noble/hashes": "^2.3.0", + "@yarnpkg/parsers": "^3.1.0", + "acorn": "^8.17.0", + "buffer": "^6.0.3", + "picomatch": "^4.0.4" + }, + "peerDependencies": { + "@deepseek-ai/cordis": "workspace:^", + "@deepseek-ai/cordis-plugin-loader": "workspace:^", + "@deepseek-ai/dsh-client-modules": "workspace:^", + "@deepseek-ai/dsh-host-apiproxy": "workspace:^", + "@deepseek-ai/dsh-host-webserver": "workspace:^", + "@deepseek-ai/dsh-invariants": "workspace:^" + }, + "devDependencies": { + "@deepseek-ai/cordis": "workspace:^", + "@deepseek-ai/cordis-plugin-loader": "workspace:^", + "@deepseek-ai/dsh-client-modules": "workspace:^", + "@deepseek-ai/dsh-host-apiproxy": "workspace:^", + "@deepseek-ai/dsh-host-webserver": "workspace:^", + "@deepseek-ai/dsh-invariants": "workspace:^", + "@deepseek-ai/dsh-subprocess-local": "workspace:^", + "@types/picomatch": "^3.0.2" + }, + "files": [ + "lib/index.js", + "lib/invariant.js", + "lib/worker.js", + "lib/client.js", + "lib/types/**/*.d.ts" + ] +} diff --git a/packages/experimental/webworker-runtime/src/client/api-client.ts b/packages/experimental/webworker-runtime/src/client/api-client.ts new file mode 100644 index 0000000000..0e99075d47 --- /dev/null +++ b/packages/experimental/webworker-runtime/src/client/api-client.ts @@ -0,0 +1,33 @@ +/** + * Page-side API carrier over the postMessage tunnel. Only `doFetch` is + * implemented: the streaming methods stay on `AbstractApiClient`'s default + * `readSse`, which is exactly what the worker answers on the two event-stream + * paths — so unary calls and downstream streams share one framing and neither + * side needs a WebSocket. + */ +import { AbstractApiClient } from '@deepseek-ai/dsh-host-apiproxy/client' +import type { WorkerTunnel } from './client.ts' + +/** API client whose requests travel the worker tunnel instead of the network. */ +export class WorkerApiClient extends AbstractApiClient { + private readonly tunnel: WorkerTunnel + + /** + * Bind the carrier to a tunnel. + * @param tunnel - page half of the worker tunnel. + */ + constructor(tunnel: WorkerTunnel) { + super() + this.tunnel = tunnel + } + + /** + * Send one request through the tunnel. + * @param input - request URL. + * @param init - fetch init; the tunnel honours method, headers, body, and signal. + * @returns the reconstructed response. + */ + protected doFetch(input: URL, init?: RequestInit): Promise { + return this.tunnel.fetch(input, init) + } +} diff --git a/packages/experimental/webworker-runtime/src/client/apply-injections.ts b/packages/experimental/webworker-runtime/src/client/apply-injections.ts new file mode 100644 index 0000000000..163729a6aa --- /dev/null +++ b/packages/experimental/webworker-runtime/src/client/apply-injections.ts @@ -0,0 +1,50 @@ +/** + * Page-side interpreter for the structured index injection table. The served + * form renders the same rows into index.html text; a static worker page has + * no served HTML, so it executes the table directly. Rows execute strictly in + * table order, so a global row lands before the scripts that read it. + */ +import type { IndexInjection } from '@deepseek-ai/dsh-host-webserver' + +function assertNever(row: never): never { + throw new Error(`webworker-runtime: unknown index injection row ${JSON.stringify(row)}`) +} + +/** + * Execute every row in table order. + * @param rows - Injection table from the boot payload. + * @param loadScript - Executes one script-src row; the tunnel's `loadBundle`, + * because the row URLs (`/plugins/...`) resolve only through the worker. + */ +export async function applyIndexInjections( + rows: readonly IndexInjection[], + loadScript: (src: string) => Promise, +): Promise { + for (const row of rows) { + switch (row.kind) { + case 'global': + (globalThis as Record)[row.name] = row.value + break + case 'script': { + const el = document.createElement('script') + el.textContent = row.text + ;(row.placement === 'head' ? document.head : document.body).append(el) + break + } + case 'script-src': + await loadScript(row.src) + break + case 'style': { + const el = document.createElement('style') + el.textContent = row.text + document.head.append(el) + break + } + case 'html': + (row.placement === 'head' ? document.head : document.body).insertAdjacentHTML('beforeend', row.html) + break + default: + assertNever(row) + } + } +} diff --git a/packages/experimental/webworker-runtime/src/client/client.ts b/packages/experimental/webworker-runtime/src/client/client.ts new file mode 100644 index 0000000000..cd8dbcb7dc --- /dev/null +++ b/packages/experimental/webworker-runtime/src/client/client.ts @@ -0,0 +1,342 @@ +/** + * Page half of the postMessage tunnel. It + * turns fetch-shaped calls into `req` frames and rebuilds Responses from the + * worker's `res` / `res-head`+`res-chunk`+`res-end` frames, so every consumer + * (boot payload, bundle transport, ApiClient, Typert RPC) speaks plain HTTP. + */ + +import type { IndexInjection } from '@deepseek-ai/dsh-host-webserver' + +/** Frame sent to the worker. */ +interface RequestFrame { + t: 'req' + id: number + method: string + /** Absolute URL; the worker derives `req.url` (pathname + search) from it. */ + url: string + headers: Record + body?: ArrayBuffer | undefined +} + +/** Cancellation of an in-flight request or stream. */ +interface AbortFrame { + t: 'abort' + id: number +} + +/** Frames received from the worker. */ +type ResponseFrame = + | { t: 'res'; id: number; status: number; headers: Record; body?: ArrayBuffer; message?: string } + | { t: 'res-head'; id: number; status: number; headers: Record } + | { t: 'res-chunk'; id: number; chunk: ArrayBuffer } + | { t: 'res-end'; id: number } + | { t: 'res-err'; id: number; message: string } + +/** Boot payload of the tunnel bootstrap route. */ +export interface BootPayload { + /** Structured index injection table, executed by the page interpreter. */ + injections: IndexInjection[] +} + +/** Fetch-shaped transport the client tree consumes. */ +export type TunnelFetch = (input: URL | string, init?: RequestInit) => Promise + +interface PendingUnary { + resolve(response: Response): void + reject(reason: Error): void +} + +/** + * Statuses the worker only produces when the host refused the exchange rather than + * answered it; a route's own 4xx is the tree talking and stays silent here. + */ +const REFUSAL_STATUS = 500 + +const encoder = new TextEncoder() + +/** Normalize a RequestInit body to a transferable ArrayBuffer. */ +function toBodyBuffer(body: RequestInit['body']): ArrayBuffer | undefined { + if (body === undefined || body === null) return undefined + if (typeof body === 'string') return encoder.encode(body).buffer + if (body instanceof ArrayBuffer) return body + if (ArrayBuffer.isView(body)) { + return body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength) + } + throw new Error(`web-preview tunnel: unsupported request body ${Object.prototype.toString.call(body)}`) +} + +/** Statuses whose Response must carry a null body. */ +const NULL_BODY_STATUS = new Set([101, 204, 205, 304]) + +/** The page half of the tunnel: one `fetch`-shaped face over `postMessage`. */ +export class WorkerTunnel { + private readonly worker: Worker + private nextId = 1 + private readonly unary = new Map() + private readonly streams = new Map>() + /** + * In-flight request descriptions, so a refusal names what was refused. + * + * A tunnel failure and a failure inside the host tree look identical from the + * page — both surface as one rejected fetch — and the acceptance run keeps the + * page console but not the frames. Warning here separates the two without + * recording anything on the normal path, where no refusal frame ever arrives. + */ + private readonly inFlight = new Map() + + /** Body-phase abort listeners, released when their stream settles. */ + private readonly releases = new Map void>() + + /** + * Attach to a spawned worker and start consuming response frames. + * @param worker - the host worker. + */ + constructor(worker: Worker) { + this.worker = worker + worker.addEventListener('message', (event: MessageEvent) => { + this.receive(event.data) + }) + worker.addEventListener('error', (event) => { + const reason = new Error(`web-preview tunnel: worker failed: ${event.message}`) + for (const id of this.inFlight.keys()) this.warnRefusal(id, `worker failed: ${event.message}`) + this.inFlight.clear() + for (const pending of this.unary.values()) pending.reject(reason) + this.unary.clear() + for (const controller of this.streams.values()) controller.error(reason) + this.streams.clear() + for (const release of this.releases.values()) release() + this.releases.clear() + }) + } + + /** + * Open the tunnel: the worker assembles its host from this frame. + * @param image - VFS image URL the worker fetches. + */ + init(image: string): void { + this.worker.postMessage({ t: 'init', image }) + } + + /** Fetch-shaped entry: one request frame, one Response (streamed when the worker streams). */ + readonly fetch: TunnelFetch = async (input, init) => { + const signal = init?.signal + // Checked before any frame leaves: a request the caller already abandoned + // must not reach the worker, where a write-shaped route would still run. + if (signal?.aborted === true) throw new DOMException('The operation was aborted.', 'AbortError') + const id = this.nextId++ + const frame: RequestFrame = { + t: 'req', + id, + method: init?.method ?? 'GET', + url: new URL(input, globalThis.location.origin).toString(), + headers: Object.fromEntries(new Headers(init?.headers).entries()), + ...(init?.body === undefined || init.body === null + ? {} + : { body: toBodyBuffer(init.body) }), + } + const response = new Promise((resolve, reject) => { + this.unary.set(id, { resolve, reject }) + }) + this.inFlight.set(id, `${frame.method} ${frame.url}`) + this.worker.postMessage(frame) + if (signal === undefined || signal === null) return await response + const raced = this.rejectOnAbort(id, signal) + try { + const settled = await Promise.race([response, raced.rejected]) + // A streaming response outlives its head: hand the signal to the body + // phase, so a later stop still ends the stream and reaches the worker. + if (this.streams.has(id)) this.observeStreamAbort(id, signal) + return settled + } finally { + raced.release() + } + } + + /** + * Read the pre-cordis boot payload (the injection table). + * @returns The payload the page applies before the client tree loads. + */ + async bootPayload(): Promise { + const response = await this.fetch('/__boot__') + if (!response.ok) { + throw new Error(`web-preview tunnel: boot payload failed with HTTP ${String(response.status)}: ${await response.text()}`) + } + return await response.json() as BootPayload + } + + /** + * `loadBundle` seam: take one client bundle through the tunnel and execute it + * as a classic script, exactly like the shell's same-origin `