fix(webworker): align filesystem semantics with Node

This commit is contained in:
imccyu
2026-08-24 10:37:05 +08:00
parent 4f80422595
commit 5ad9b128f9
16 changed files with 614 additions and 160 deletions
@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write .agents/notes/implemented/architecture/2026-08-23-webworker-vfs-watch-and-landlock.md
2026-08-23-webworker-vfs-watch-and-landlock.md: 61254092e0f32e8e4291fd7c21489684110a1d15
2026-08-23-webworker-vfs-watch-and-landlock.zh.md: 32e1b36e0ef17e4574252693e246f9d7cd4d4712
2026-08-23-webworker-vfs-watch-and-landlock.md: 2705c63aa6bf0f2e2de33d00029a4f41e1b1d4af
2026-08-23-webworker-vfs-watch-and-landlock.zh.md: 4470720e1ff968aa06578b231b054c9053f27283
@@ -20,13 +20,13 @@ The filesystem compatibility boundary follows the [Worker Node face decision](20
The mutation record is shared with WebFS persistence rather than defining a second notification path. Writes carry their complete post-commit bytes and virtual permission bits, plus an append offset when only a tail changed. `MemoryVfs` accepts an optional asynchronous `VfsMutationSink`, sends the same records to that sink and live watcher subscribers, and exposes `flush()` through file-handle `sync()` and `datasync()`. Hydration supplies `{ mode, mtimeMs }` explicitly, so image permissions and durable timestamps cannot occupy the same positional argument. This change mounts no durable sink; it keeps the synchronous in-memory tree authoritative so an OPFS or user-directory mirror can hydrate before publication and write behind without changing `node:fs`.
The `node:fs` implementation provides callback `stat` and `lstat`, `watch`, `watchFile`, `unwatchFile`, `FSWatcher`, and `StatWatcher`; `node:fs/promises.watch` provides the abortable async iterator. One path shares one `StatWatcher` across listeners, listener-specific unwatching leaves peers active, and missing paths report zero-valued Stats before later creation, deletion, and recreation transitions. Callback dispatch captures the registration-time async context and checks closure before every queued delivery.
The `node:fs` implementation provides callback `stat` and `lstat`, `watch`, `watchFile`, `unwatchFile`, `FSWatcher`, and `StatWatcher`; `node:fs/promises.watch` provides the abortable async iterator. One path shares one `StatWatcher` across listeners, listener-specific unwatching leaves peers active, and missing paths report zero-valued Stats before later creation, deletion, and recreation transitions. Callback dispatch captures the registration-time async context and checks closure before every queued delivery. A pre-aborted callback watch returns its watcher before asynchronously closing it, while a pre-aborted promise watch rejects its first iterator read with `AbortError`.
`fs.watch` maps entry creation, removal, and rename destinations to `rename`, and maps content or mode changes to `change`. Non-recursive directory watches report immediate child names; recursive watches report paths relative to the watched directory. The VFS has no symlinks, so this implementation does not invent symlink events.
### Streams and unchanged npm packages
`node:stream` uses the maintained `readable-stream` browser implementation for `Readable`, `Writable`, `Duplex`, `Transform`, `PassThrough`, pipeline helpers, async iteration, backpressure, aborts, and teardown ordering. The compatibility module sets the byte high-water default to the 64 KiB value used by the repository's Node 22+ engines. VFS-backed `ReadStream` and `WriteStream` supply file descriptors, inclusive ranges, encoding, append or replace behavior, byte accounting, AbortSignal handling, and `open`/`ready`/`finish`/`end`/`close` ordering.
`node:stream` uses the maintained `readable-stream` browser implementation for `Readable`, `Writable`, `Duplex`, `Transform`, `PassThrough`, pipeline helpers, async iteration, backpressure, aborts, and teardown ordering. The compatibility module sets the byte high-water default to the 64 KiB value used by the repository's Node 22+ engines. VFS-backed `ReadStream` and `WriteStream` supply file descriptors, inclusive ranges, encoding, append or replace behavior, byte accounting, AbortSignal handling, and `open`/`ready`/`finish`/`end`/`close` ordering. Descriptors retain their opened file identity and access mode across rename, replacement, and unlink; hard links share that identity and subsequent content or mode changes, while truncation zero-fills growth.
Chokidar and readdirp are ordinary image dependencies, not module replacements. Their package code runs unchanged and imports the Worker implementations of `node:fs`, `node:fs/promises`, `node:stream`, `node:events`, `node:path`, and `node:os`. Chokidar therefore retains its own initial scan, `ready`, polling, atomic-write normalization, write-settle delay, shared watcher, and close behavior.
@@ -36,7 +36,7 @@ Chokidar and readdirp are ordinary image dependencies, not module replacements.
The process layer has a table of Worker platform executables identified by logical executable name rather than one package-manager path. Its `landlock-run` provider accepts a bare command or an absolute launcher path, parses the native package's unchanged CLI, validates every grant root, and delegates the inner argv to the existing shell process runner. `node:child_process` performs only generic executable lookup, output delivery, and settlement. The unchanged package's synchronous `probe()` therefore observes the provider through `spawnSync` and reports `full`. A usage error, missing grant root, or unknown inner executable prints one `landlock-run: ...` line, exits `125`, and never runs the inner command. The bwrap probe remains unavailable, so the unmodified `sandbox-local` Linux chain selects this Landlock backend.
Each launched process receives its own `ShellFileSystem` guard. `stat`, `list`, and `readText` require a read-only or read-write grant; `writeText`, `mkdir`, and `remove` require a read-write grant; `rename` requires both source and destination to be writable. Denials carry `EACCES` and `permission denied`, preserving `bash-sandbox` denial classification. `/tmp` maps to the VFS `/dsh/tmp`, while `/dev/null` is a virtual empty-read and discarded-write file that stores no bytes.
Each launched process receives its own `ShellFileSystem` guard. `stat`, `list`, and `readText` require a read-only or read-write grant; `writeText`, `mkdir`, and `remove` require a read-write grant; `rename` requires both source and destination to be writable. Grant roots normalize trailing separators before containment checks. Denials carry `EACCES` and `permission denied`, preserving `bash-sandbox` denial classification. `/tmp` maps to the VFS `/dsh/tmp`, while `/dev/null` is a virtual empty-read and discarded-write file that stores no bytes.
The Worker's `full` verdict covers every file operation expressible through its shell command table and Host-served VFS protocol. It does not claim Linux kernel Landlock, arbitrary native executable support, or protection against a future shell program that bypasses `ShellFileSystem`.
@@ -20,13 +20,13 @@ Web Worker preview 启动与 Node host 相同的 Web profile 和 Agent preset。
Mutation record 与 WebFS 持久化共用,而不建立第二条通知路径。Write 记录携带提交后的完整字节与虚拟权限位,并在只有尾部变化时携带 append offset。`MemoryVfs` 接受可选的异步 `VfsMutationSink`,把同一批记录交给 sink 与实时 watcher 订阅方,并通过文件句柄的 `sync()``datasync()` 暴露 `flush()`。水合通过显式的 `{ mode, mtimeMs }` 传入元数据,因此镜像权限与持久化时间戳不会占用同一个位置参数。本次变更不挂载 durable sink;同步内存树继续作为权威,因此 OPFS 或用户目录 mirror 可以先水合、再异步写回,而无需改变 `node:fs`
`node:fs` 实现 callback `stat``lstat``watch``watchFile``unwatchFile``FSWatcher``StatWatcher``node:fs/promises.watch` 提供可由 abort 取消的异步迭代器。同一路径的 listener 共享一个 `StatWatcher`,按 listener 取消监听不会影响其他 listener;缺失路径先报告零值 Stats,随后再报告创建、删除和重建状态。Callback 分发捕获注册时的异步上下文,并在每次排队交付前检查 watcher 是否已经关闭。
`node:fs` 实现 callback `stat``lstat``watch``watchFile``unwatchFile``FSWatcher``StatWatcher``node:fs/promises.watch` 提供可由 abort 取消的异步迭代器。同一路径的 listener 共享一个 `StatWatcher`,按 listener 取消监听不会影响其他 listener;缺失路径先报告零值 Stats,随后再报告创建、删除和重建状态。Callback 分发捕获注册时的异步上下文,并在每次排队交付前检查 watcher 是否已经关闭。预先 abort 的 callback watcher 先返回对象、再异步关闭;预先 abort 的 promise watcher 在第一次读取 iterator 时以 `AbortError` 拒绝。
`fs.watch` 把条目创建、删除和 rename 目标映射为 `rename`,把内容或 mode 变化映射为 `change`。非递归目录 watcher 报告直接子项名,递归 watcher 报告相对被监听目录的路径。VFS 没有符号链接,因此该实现不会制造符号链接事件。
### Stream 与未修改的 NPM 包
`node:stream` 使用维护中的 `readable-stream` 浏览器实现来提供 `Readable``Writable``Duplex``Transform``PassThrough`、pipeline helper、异步迭代、backpressure、abort 和 teardown 顺序。兼容模块把字节流 high-water mark 默认值设为仓库 Node 22+ 引擎使用的 64 KiB。VFS 支持的 `ReadStream``WriteStream` 提供文件描述符、闭区间范围、encoding、追加或替换行为、字节计数、AbortSignal 处理,以及 `open``ready``finish``end``close` 顺序。
`node:stream` 使用维护中的 `readable-stream` 浏览器实现来提供 `Readable``Writable``Duplex``Transform``PassThrough`、pipeline helper、异步迭代、backpressure、abort 和 teardown 顺序。兼容模块把字节流 high-water mark 默认值设为仓库 Node 22+ 引擎使用的 64 KiB。VFS 支持的 `ReadStream``WriteStream` 提供文件描述符、闭区间范围、encoding、追加或替换行为、字节计数、AbortSignal 处理,以及 `open``ready``finish``end``close` 顺序。Descriptor 在 rename、replacement 和 unlink 后仍保留打开时的文件身份与访问模式;hard link 共享该身份及后续内容和 mode 变化,truncate 增长则用零字节填充。
Chokidar 和 readdirp 作为普通镜像依赖运行,不属于模块 replacement。它们的包代码保持原样,并导入 Worker 实现的 `node:fs``node:fs/promises``node:stream``node:events``node:path``node:os`。因此,初次扫描、`ready`、polling、原子写归一化、写入稳定等待、共享 watcher 与关闭行为仍由 Chokidar 自己负责。
@@ -36,7 +36,7 @@ Chokidar 和 readdirp 作为普通镜像依赖运行,不属于模块 replaceme
进程层持有按逻辑可执行文件名识别的 Worker 平台可执行文件表,而不依赖某一个包管理器路径。其 `landlock-run` provider 接受裸命令或绝对 launcher 路径,解析 native 包未经修改的 CLI、校验每个授权根,并把内部 argv 交给既有 shell 进程 runner。`node:child_process` 只负责通用的可执行文件查找、输出投递与结束处理。因此,原包的同步 `probe()` 会通过 `spawnSync` 观察到该 provider 并报告 `full`。用法错误、缺失的授权根或未知内部可执行文件只输出一行 `landlock-run: ...`,以 `125` 退出,并且绝不运行内部命令。bwrap 仍探测为不可用,因此未修改的 `sandbox-local` Linux 选择链会选中该 Landlock 后端。
每个已启动进程分别获得一个 `ShellFileSystem` guard。`stat``list``readText` 需要只读或读写授权;`writeText``mkdir``remove` 需要读写授权;`rename` 要求源和目标都可写。拒绝错误包含 `EACCES``permission denied`,从而保持 `bash-sandbox` 的拒绝分类。`/tmp` 映射到 VFS 的 `/dsh/tmp``/dev/null` 则是空读、丢弃写入且不保存任何字节的虚拟文件。
每个已启动进程分别获得一个 `ShellFileSystem` guard。`stat``list``readText` 需要只读或读写授权;`writeText``mkdir``remove` 需要读写授权;`rename` 要求源和目标都可写。Grant root 在 containment 检查前去除尾部分隔符。拒绝错误包含 `EACCES``permission denied`,从而保持 `bash-sandbox` 的拒绝分类。`/tmp` 映射到 VFS 的 `/dsh/tmp``/dev/null` 则是空读、丢弃写入且不保存任何字节的虚拟文件。
Worker 的 `full` 结论覆盖 shell 命令表和 Host 服务 VFS 协议能够表达的全部文件操作。它不表示 Linux 内核 Landlock、不支持任意 native 可执行文件,也无法约束未来绕过 `ShellFileSystem` 的 shell 程序。
@@ -2,5 +2,5 @@
# side as of the last confirmed-consistent state. Both languages carry equal authority;
# after editing either side, bring the other along and re-record with:
# pnpm run verify-translation-pairing --write packages/experimental/webworker-runtime/README.md
README.md: 5b63856b826d4f8bc8b6ff56626c6e7a1ed663b1
README.zh.md: 59b31bd308077b8eeaa60edf5bd23a75924eaa07
README.md: df6dacad35273f8c636a86c7c260b4f5d1958a6a
README.zh.md: 721770149c04169d813d2b5d3d4faafdccf1dec5
@@ -7,7 +7,7 @@ The browser worker host: the whole harness plugin tree runs inside one dedicated
Three artifacts from one tsdown pipeline:
- **`lib/index.js` (assembly library)** — `createWorkerHost`/`startWorkerHost` mount the base image and any ordered data overlays (`storage/`), install the module loader (`module-system/`) and the `process` shim, boot the tree through the image's own `dsh-app-boot`, and hand the tunnel its serving seams. Overlays may replace files only under `home/` and `workspace/`; they cannot replace the base manifest, configuration, or modules. The image layout contract (`image-layout.ts`: virtual root, config/manifest paths, empty directories, the `lowered` wrapper-contract gate) is shared with the packer. Boot patches force the deployment-shaped rows: frontend serving off, JSONL session logs on the plaintext path, preset roots onto the image's `config/agent-presets`.
- **`lib/worker.js` (worker bundle)** — the assembly plus this package's Node-compatibility layer as one self-contained ES module. The module proxy table (`module-proxies.ts`) is the only platform fork: `node:*` builtins over VFS/tunnel/browser primitives, structural stubs that fail loud on the console for what a browser cannot do, and native/binary package replacements. VFS mutations drive `node:fs` callback, polling, and promise watchers; `readable-stream` supplies the stream state machine used by file streams and unchanged image packages such as Chokidar and readdirp. AsyncLocalStorage carries sync-stack causality across `await` through the snapshot/restore faces the pack-time lowering injects. The worker holds no compiler: an image the packer did not lower is refused at mount ([note](../../../.agents/notes/implemented/architecture/2026-08-20-webworker-pack-lowering-and-preview.md)).
- **`lib/worker.js` (worker bundle)** — the assembly plus this package's Node-compatibility layer as one self-contained ES module. The module proxy table (`module-proxies.ts`) is the only platform fork: `node:*` builtins over VFS/tunnel/browser primitives, structural stubs that fail loud on the console for what a browser cannot do, and native/binary package replacements. VFS mutations drive `node:fs` callback, polling, and promise watchers; open descriptors retain file identity and access mode across rename, replacement, and unlink; `readable-stream` supplies the stream state machine used by file streams and unchanged image packages such as Chokidar and readdirp. AsyncLocalStorage carries sync-stack causality across `await` through the snapshot/restore faces the pack-time lowering injects. The worker holds no compiler: an image the packer did not lower is refused at mount ([note](../../../.agents/notes/implemented/architecture/2026-08-20-webworker-pack-lowering-and-preview.md)).
- **`src/shell/` (the worker's own process layer)** — a browser worker cannot fork, so `node:child_process` is not a stub but an implementation: `spawn` starts the command in its own Web Worker — this same bundle, told by its first frame to be a shell process — and reports it through the `ChildProcess` surface the subprocess service consumes. The command runs off the host's thread, `SIGKILL` terminates it whatever it is doing, and it reaches the VFS only by message (the host serves those frames). Worker platform executables preserve native-package protocols such as Landlock without replacing their JavaScript packages or coupling their implementations to `node:child_process`; ordinary commands use the package's evaluator and coreutils command table. The grammar is `@yarnpkg/parsers`' `parseShell`, while `execSync`/`fork` still refuse because they need a real process.
- **`lib/client.js` (page half)** — startup has two independent stages. `chooseWorkerHostSource({ image?, fixtureManifest? })` optionally owns the boot barrier and fixture manifest: without `preview-fixture` it waits at the source chooser, while a valid query selects directly; either path returns ordered overlays. `connectWorkerHost(worker, { image?, overlays? })` remains the public base-runtime connector; callers that skip the chooser get an empty overlay list. `apps/web` invokes both and supplies its statically bundled Worker. The opening `init` frame carries the base and ordered overlay URLs, the boot payload delivers the structured index-injection table, and `applyIndexInjections` executes it before the shell entry runs. The tunnel exposes fetch-shaped transport, the API client, and `loadBundle` for the shell's boot seam.
@@ -7,7 +7,7 @@
一条 tsdown 管线出三个产物:
- **`lib/index.js`(装配库)**——`createWorkerHost`/`startWorkerHost` 挂载基础镜像和按序排列的数据 overlays(`storage/`)、安装模块加载器(`module-system/`)与 `process` shim、经镜像自带的 `dsh-app-boot` 启动插件树,并把服务缝隙交给隧道。Overlay 只能替换 `home/``workspace/` 下的文件,不能替换基础 manifest、配置或模块。镜像布局契约(`image-layout.ts`:虚拟根、config/manifest 路径、空目录、`lowered` 包装契约门)与 packer 共享。boot patch 强制部署形态行:关前端静态服务、JSONL 会话日志走明文、preset 根指向镜像内 `config/agent-presets`
- **`lib/worker.js`(worker 束)**——装配库加本包的 Node 兼容层,合成一个自含 ES module。模块代理表(`module-proxies.ts`)是唯一平台叉口:`node:*` 内建走 VFS、隧道和浏览器原语,浏览器做不到的走结构化 stub(调用即在 console 报错并抛出),native/binary 包则替换执行后端。VFS mutation 驱动 `node:fs` 的 callback、polling 和 promise watcher`readable-stream` 提供文件流以及 Chokidar、readdirp 等未修改镜像包所用的流状态机。AsyncLocalStorage 经 pack 时降低注入的 snapshot/restore 面在 `await` 间携带同步栈因果。worker 不带编译器:packer 未降低的镜像在挂载时被拒([note](../../../.agents/notes/implemented/architecture/2026-08-20-webworker-pack-lowering-and-preview.zh.md))。
- **`lib/worker.js`(worker 束)**——装配库加本包的 Node 兼容层,合成一个自含 ES module。模块代理表(`module-proxies.ts`)是唯一平台叉口:`node:*` 内建走 VFS、隧道和浏览器原语,浏览器做不到的走结构化 stub(调用即在 console 报错并抛出),native/binary 包则替换执行后端。VFS mutation 驱动 `node:fs` 的 callback、polling 和 promise watcher打开的 descriptor 在 rename、replacement 和 unlink 后仍保留文件身份与访问模式;`readable-stream` 提供文件流以及 Chokidar、readdirp 等未修改镜像包所用的流状态机。AsyncLocalStorage 经 pack 时降低注入的 snapshot/restore 面在 `await` 间携带同步栈因果。worker 不带编译器:packer 未降低的镜像在挂载时被拒([note](../../../.agents/notes/implemented/architecture/2026-08-20-webworker-pack-lowering-and-preview.zh.md))。
- **`src/shell/`(worker 自己的进程层)**——浏览器 worker 无法 fork,所以 `node:child_process` 不是 stub 而是实现:`spawn` 把命令放进它自己的 Web Worker——就是这同一个束,由首帧告诉它「你是 shell 进程」——并以 subprocess 服务消费的 `ChildProcess` 面报告结果。命令不占宿主线程,`SIGKILL` 不管它在干什么都能终止它,而它只能靠消息触达 VFS(由宿主应答这些帧)。Worker 平台 executable 在不替换 JavaScript 包、也不把具体实现耦合进 `node:child_process` 的情况下保持 Landlock 等 native 包协议;普通命令使用本包的求值器与 coreutils 命令表。语法来自 `@yarnpkg/parsers``parseShell`,而 `execSync`/`fork` 依然拒绝,因为它们需要真进程。
- **`lib/client.js`(页面半)**——启动分为相互独立的两段。`chooseWorkerHostSource({ image?, fixtureManifest? })` 可选地拥有 boot barrier 与 fixture manifest:没有 `preview-fixture` 时停在来源选择面板,合法 query 则直接选择;两条路径都返回按序排列的 overlays。`connectWorkerHost(worker, { image?, overlays? })` 仍是公开的基础运行态连接器;调用方跳过选择器时 overlay 列表为空。`apps/web` 调用这两段并提供静态打包的 Worker。开局 `init` 帧携带基础镜像与按序排列的 overlay URLboot 载荷送达结构化 index 注入表,`applyIndexInjections` 在壳入口运行前逐行执行。隧道暴露 fetch 形传输、API 客户端与壳启动缝隙用的 `loadBundle`
@@ -17,6 +17,7 @@ export interface PreviewFixtureManifestEntry {
/** Complete built-in fixture catalog consumed before Worker startup. */
export interface PreviewFixtureManifest {
readonly version: number
/** Required default fixture id, or null when the chooser should default to an empty overlay. */
readonly defaultFixture: string | null
readonly fixtures: readonly PreviewFixtureManifestEntry[]
}
@@ -0,0 +1,14 @@
/** Build the Node-style cancellation error shared by abortable builtin APIs. */
/**
* Create an `AbortError` carrying Node's stable error code.
* @param reason - Optional AbortSignal reason exposed as the error cause.
* @returns A Node-compatible abort error.
*/
export function abortError(reason?: unknown): Error & { code: string; cause?: unknown } {
const error = new Error('The operation was aborted') as Error & { code: string; cause?: unknown }
error.name = 'AbortError'
error.code = 'ABORT_ERR'
if (reason !== undefined) error.cause = reason
return error
}
@@ -5,6 +5,7 @@ import { captureAsyncContext, runWithAsyncContext } from './async_hooks.ts'
import { basename, relative, resolve, sep } from './path.ts'
import { requireActiveVfs } from '../../../storage/active.ts'
import type { VfsBigIntStats, VfsMutation, VfsStats } from '../../../storage/types.ts'
import { abortError } from './abort-error.ts'
type PathArg = string | URL | Uint8Array
type WatchListener = (eventType: 'rename' | 'change', filename: string | Buffer | null) => void
@@ -82,14 +83,6 @@ const contains = (parent: string, child: string): boolean =>
const overlaps = (left: string, right: string): boolean => contains(left, right) || contains(right, left)
const abortError = (reason?: unknown): Error & { code: string; cause?: unknown } => {
const error = new Error('The operation was aborted') as Error & { code: string }
error.name = 'AbortError'
error.code = 'ABORT_ERR'
if (reason !== undefined) error.cause = reason
return error
}
/** `fs.FSWatcher` over VFS mutations. */
export class FSWatcher extends EventEmitter {
private readonly disposeMutation: () => void
@@ -122,9 +115,8 @@ export class FSWatcher extends EventEmitter {
this.signal = options.signal
this.onAbort = options.signal === undefined ? undefined : () => { this.close() }
if (options.signal?.aborted === true) {
this.disposeMutation()
this.closed = true
throw abortError(options.signal.reason)
this.close()
return
}
options.signal?.addEventListener('abort', this.onAbort as () => void, { once: true })
}
@@ -372,6 +364,10 @@ export function watchAsync(
const onAbort = (): void => { settleFailure(abortError(options.signal?.reason)) }
const start = (): void => {
if (watcher !== undefined || closed || failure !== undefined) return
if (options.signal?.aborted === true) {
settleFailure(abortError(options.signal.reason))
return
}
try {
watcher = watch(path, options, (eventType, filename) => {
const event = { eventType, filename }
@@ -5,10 +5,13 @@
* file descriptors, `mkdtemp`, access checks, watchers, streams, and the promise face.
*/
import { requireActiveVfs } from '../../../storage/active.ts'
import type { Vfs, VfsBigIntStats, VfsStatOptions, VfsStats, VfsWriteOptions } from '../../../storage/types.ts'
import type {
Vfs, VfsBigIntStats, VfsOpenFile, VfsStatOptions, VfsStats, VfsWriteOptions,
} from '../../../storage/types.ts'
import { Buffer } from 'buffer'
import { Readable, Writable } from './stream.ts'
import { dirname } from './path.ts'
import { abortError } from './abort-error.ts'
import {
FSWatcher, StatWatcher, unwatchFile, watch, watchAsync, watchFile,
} from './fs-watch.ts'
@@ -166,11 +169,14 @@ export function stat(
const callback = typeof optionsOrCallback === 'function' ? optionsOrCallback : maybeCallback
if (callback === undefined) throw new TypeError('The "callback" argument must be of type function')
queueMicrotask(() => {
let result: VfsStats | VfsBigIntStats
try {
callback(null, statSync(path, options))
result = statSync(path, options)
} catch (error) {
callback(error as NodeJS.ErrnoException)
return
}
callback(null, result)
})
}
@@ -290,9 +296,8 @@ export function accessSync(path: PathArg): void {
}
interface OpenFile {
path: string
file: VfsOpenFile
position: number
append: boolean
}
const openFiles = new Map<number, OpenFile>()
@@ -308,25 +313,20 @@ let nextFd = 3
*/
export function openSync(path: PathArg, flags = 'r', mode?: number): number {
const target = asPath(path)
const exists = vfs().existsSync(target)
if (flags.includes('x') && exists) {
const error = new Error(`EEXIST: file already exists, open '${target}'`) as Error & { code: string; path: string }
error.code = 'EEXIST'
error.path = target
throw error
}
if (flags.startsWith('r')) vfs().realpathSync(target)
else if (flags.startsWith('w') || !exists) {
vfs().writeFileSync(target, new Uint8Array(0), mode === undefined ? undefined : { mode })
}
const file = vfs().openFileSync(target, flags, mode)
const fd = nextFd++
openFiles.set(fd, { path: target, position: 0, append: flags.startsWith('a') })
openFiles.set(fd, { file, position: 0 })
return fd
}
const fileOf = (fd: number, syscall: string): OpenFile => {
const file = openFiles.get(fd)
if (file === undefined) throw new Error(`EBADF: bad file descriptor, ${syscall}`)
if (file === undefined) {
const error = new Error(`EBADF: bad file descriptor, ${syscall}`) as Error & { code: string; syscall: string }
error.code = 'EBADF'
error.syscall = syscall
throw error
}
return file
}
@@ -347,9 +347,8 @@ export function readSync(
position: number | null = null,
): number {
const file = fileOf(fd, 'read')
const bytes = bytesOf(file.path)
const from = position ?? file.position
const slice = bytes.subarray(from, from + length)
const slice = file.file.read(from, length)
buffer.set(slice, offset)
if (position === null) file.position = from + slice.byteLength
return slice.byteLength
@@ -364,17 +363,10 @@ export function readSync(
export function writeSync(fd: number, data: string | Uint8Array): number {
const file = fileOf(fd, 'write')
const bytes = typeof data === 'string' ? new TextEncoder().encode(data) : data
if (file.append) {
vfs().appendFileSync(file.path, bytes)
return bytes.byteLength
}
const existing = vfs().existsSync(file.path) ? bytesOf(file.path) : new Uint8Array(0)
const merged = new Uint8Array(Math.max(existing.byteLength, file.position + bytes.byteLength))
merged.set(existing, 0)
merged.set(bytes, file.position)
vfs().writeFileSync(file.path, merged)
file.position += bytes.byteLength
return bytes.byteLength
const position = file.file.append ? file.file.stat().size : file.position
const bytesWritten = file.file.write(position, bytes)
file.position = position + bytesWritten
return bytesWritten
}
/**
@@ -382,17 +374,16 @@ export function writeSync(fd: number, data: string | Uint8Array): number {
* @param fd - descriptor.
*/
export function closeSync(fd: number): void {
openFiles.delete(fd)
if (!openFiles.delete(fd)) fileOf(fd, 'close')
}
/**
* Create a second name for one file's contents. Hard links do not exist in the
* VFS, so the bytes are copied.
* Create a second name for one file identity.
* @param from - existing path.
* @param to - new path.
*/
export function linkSync(from: PathArg, to: PathArg): void {
writeFileSync(to, bytesOf(asPath(from)))
vfs().linkSync(asPath(from), asPath(to))
}
/**
@@ -424,30 +415,40 @@ export interface FileHandle {
export function openHandleSync(path: PathArg, flags = 'r', mode?: number): FileHandle {
const target = asPath(path)
const directory = vfs().existsSync(target) && vfs().statSync(target).isDirectory()
const append = flags.startsWith('a')
const fd = directory ? -1 : openSync(target, flags, mode)
let closed = false
const descriptor = (syscall: string): OpenFile => fileOf(fd, syscall)
return {
fd,
readFile: async (options?: EncodingOption) => readFileSync(target, options),
// Node appends when the handle was opened with 'a'. The JSONL session log
// depends on it — `open(path, 'a')` then `writeFile(batch)` — and replacing
// the file there destroys the header frame its reader requires.
readFile: async (options?: EncodingOption) => {
if (directory) return readFileSync(target, options)
const open = descriptor('read')
const bytes = open.file.read(open.position, Math.max(0, open.file.stat().size - open.position))
open.position += bytes.length
const encoding = encodingOf(options)
return encoding === undefined || encoding === 'utf8' || encoding === 'utf-8'
? (encoding === undefined ? asBuffer(bytes) : new TextDecoder().decode(bytes))
: asBuffer(bytes).toString(encoding)
},
writeFile: async (data: string | Uint8Array) => {
if (append) appendFileSync(target, data)
else writeFileSync(target, data)
if (directory) writeFileSync(target, data)
else writeSync(fd, data)
},
write: async (data: string | Uint8Array) => ({ bytesWritten: writeSync(fd, data) }),
read: async (buffer: Uint8Array, offset = 0, length = buffer.byteLength, position: number | null = null) => ({
bytesRead: readSync(fd, buffer, offset, length, position),
buffer,
}),
stat: async () => statSync(target) as VfsStats,
stat: async () => directory ? statSync(target) as VfsStats : descriptor('fstat').file.stat(),
truncate: async (length = 0) => {
writeFileSync(target, bytesOf(target).subarray(0, length))
if (directory) writeFileSync(target, new Uint8Array(length))
else descriptor('ftruncate').file.truncate(length)
},
sync: async () => { await vfs().flush() },
datasync: async () => { await vfs().flush() },
close: async () => {
if (closed) return
closed = true
if (fd !== -1) closeSync(fd)
},
}
@@ -477,12 +478,8 @@ export interface WriteStreamOptions {
signal?: AbortSignal
}
const aborted = (reason?: unknown): Error => {
const error = new Error('The operation was aborted', { cause: reason }) as Error & { code: string }
error.name = 'AbortError'
error.code = 'ABORT_ERR'
return error
}
/** Node implements file-stream `autoClose` through the stream's `autoDestroy` state. */
const streamAutoDestroy = (autoClose: boolean | undefined): boolean => autoClose ?? true
/** Read stream over one VFS file. */
export class ReadStream extends Readable {
@@ -503,7 +500,7 @@ export class ReadStream extends Readable {
constructor(path: PathArg, options: ReadStreamOptions = {}) {
super({
autoDestroy: options.autoClose ?? true,
autoDestroy: streamAutoDestroy(options.autoClose),
emitClose: options.emitClose ?? true,
highWaterMark: options.highWaterMark ?? 64 * 1024,
})
@@ -513,7 +510,7 @@ export class ReadStream extends Readable {
this.flags = options.flags ?? 'r'
this.position = this.start
this.signal = options.signal
this.onAbort = options.signal === undefined ? undefined : () => { this.destroy(aborted(options.signal?.reason)) }
this.onAbort = options.signal === undefined ? undefined : () => { this.destroy(abortError(options.signal?.reason)) }
if (options.encoding !== undefined && options.encoding !== null) this.setEncoding(options.encoding)
options.signal?.addEventListener('abort', this.onAbort as () => void, { once: true })
}
@@ -524,7 +521,7 @@ export class ReadStream extends Readable {
return
}
if (this.signal?.aborted === true) {
callback(aborted(this.signal.reason))
callback(abortError(this.signal.reason))
return
}
try {
@@ -598,7 +595,7 @@ export class WriteStream extends Writable {
constructor(path: PathArg, options: WriteStreamOptions = {}) {
super({
autoDestroy: options.autoClose ?? true,
autoDestroy: streamAutoDestroy(options.autoClose),
decodeStrings: true,
defaultEncoding: options.encoding ?? 'utf8',
emitClose: options.emitClose ?? true,
@@ -609,7 +606,7 @@ export class WriteStream extends Writable {
this.mode = options.mode
this.start = options.start
this.signal = options.signal
this.onAbort = options.signal === undefined ? undefined : () => { this.destroy(aborted(options.signal?.reason)) }
this.onAbort = options.signal === undefined ? undefined : () => { this.destroy(abortError(options.signal?.reason)) }
options.signal?.addEventListener('abort', this.onAbort as () => void, { once: true })
}
@@ -619,7 +616,7 @@ export class WriteStream extends Writable {
return
}
if (this.signal?.aborted === true) {
callback(aborted(this.signal.reason))
callback(abortError(this.signal.reason))
return
}
try {
@@ -772,13 +769,12 @@ export const promises = {
mkdirSync(dirname(target), { recursive: true })
writeFileSync(target, bytesOf(source))
},
// The VFS has no inodes, so a hard link is a byte copy: the caller's contract
// is only that both names read the same content until one is removed.
// The VFS keeps both names attached to one file identity until either name is removed.
link: async (from: PathArg, to: PathArg): Promise<void> => { linkSync(from, to) },
open: async (path: PathArg, flags?: string, mode?: number): Promise<FileHandle> => openHandleSync(path, flags, mode),
opendir: async (path: PathArg): Promise<Dir> => opendirSync(path),
truncate: async (path: PathArg, length = 0): Promise<void> => {
writeFileSync(path, bytesOf(asPath(path)).subarray(0, length))
vfs().truncateSync(asPath(path), length)
},
watch: watchAsync,
constants,
@@ -51,7 +51,8 @@ export function parseLandlockArguments(args: readonly string[]): LandlockInvocat
/** Map the host launcher's temp path into the Worker VFS. */
function vfsPath(path: string, cwd: string): string {
const absolute = resolve(cwd, path)
const resolved = resolve(cwd, path)
const absolute = resolved.length > 1 ? resolved.replace(/\/+$/u, '') : resolved
if (absolute === '/tmp') return DSH_TMP
if (absolute.startsWith('/tmp/')) return `${DSH_TMP}${absolute.slice('/tmp'.length)}`
return absolute
@@ -8,7 +8,7 @@ import { dirname, join, normalize, resolve, SEP } from '../module-system/posix-p
import { IMAGE_OVERLAY_DIRECTORIES } from '../image-layout.ts'
import { parseTar } from './tar.ts'
import type {
Vfs, VfsBigIntStats, VfsDir, VfsDirent, VfsEncoding, VfsError, VfsFileHandle, VfsMutation,
Vfs, VfsBigIntStats, VfsDir, VfsDirent, VfsEncoding, VfsError, VfsFileHandle, VfsMutation, VfsOpenFile,
VfsMutationListener, VfsMutationSink, VfsReadOptions, VfsSeedOptions, VfsStatOptions, VfsStats, VfsWriteOptions,
} from './types.ts'
@@ -20,6 +20,8 @@ interface FileNode {
mtimeMs: number
/** Permission bits (`0o777` mask), set at creation and changed only by `chmod`. */
mode: number
/** Stable identity shared by hard links and retained by open descriptors. */
identity?: bigint
}
/** Creation default for files, Node's `0o666` under the classic `022` umask. */
@@ -80,7 +82,14 @@ function statsOf(size: number, mtimeMs: number, directory: boolean, ino: bigint,
* @param mode - Stored permission bits of the entry.
* @returns Stats in the shape Node returns under `{ bigint: true }`.
*/
function bigIntStatsOf(size: number, mtimeMs: number, directory: boolean, ino: bigint, mode: number): VfsBigIntStats {
function bigIntStatsOf(
size: number,
mtimeMs: number,
directory: boolean,
ino: bigint,
mode: number,
nlink = 1,
): VfsBigIntStats {
const milliseconds = BigInt(Math.trunc(mtimeMs))
const nanoseconds = milliseconds * 1_000_000n
const time = new Date(mtimeMs)
@@ -89,7 +98,7 @@ function bigIntStatsOf(size: number, mtimeMs: number, directory: boolean, ino: b
mode: BigInt((directory ? 0o040000 : 0o100000) | (mode & 0o777)),
dev: 1n,
ino,
nlink: 1n,
nlink: BigInt(nlink),
mtimeMs: milliseconds,
mtimeNs: nanoseconds,
ctimeMs: milliseconds,
@@ -112,6 +121,49 @@ function bigIntStatsOf(size: number, mtimeMs: number, directory: boolean, ino: b
}
}
interface OpenMode {
readonly readable: boolean
readonly writable: boolean
readonly append: boolean
readonly create: boolean
readonly truncate: boolean
readonly exclusive: boolean
}
/** Parse the Node string flags supported by the compatibility filesystem. */
function openMode(flags: string): OpenMode {
const base = flags[0]
const suffix = flags.slice(1).split('')
const validSuffix = suffix.every(flag => flag === '+' || flag === 'x' || flag === 's')
const uniqueSuffix = new Set(suffix).size === suffix.length
if ((base !== 'r' && base !== 'w' && base !== 'a') || !validSuffix || !uniqueSuffix
|| base === 'r' && flags.includes('x')) {
const error = new TypeError(`The argument 'flags' is invalid. Received '${flags}'`) as TypeError & { code: string }
error.code = 'ERR_INVALID_ARG_VALUE'
throw error
}
return {
readable: base === 'r' || flags.includes('+'),
writable: base !== 'r' || flags.includes('+'),
append: base === 'a',
create: base === 'w' || base === 'a',
truncate: base === 'w',
exclusive: flags.includes('x'),
}
}
/** Resize bytes exactly, preserving the prefix and zero-filling growth. */
function resize(bytes: Uint8Array, length: number): Uint8Array {
if (!Number.isSafeInteger(length) || length < 0) {
const error = new RangeError(`The value of "len" is out of range. It must be >= 0. Received ${String(length)}`) as RangeError & { code: string }
error.code = 'ERR_OUT_OF_RANGE'
throw error
}
const resized = new Uint8Array(length)
resized.set(bytes.subarray(0, length))
return resized
}
/** Construction inputs for {@link MemoryVfs}. */
export interface MemoryVfsOptions {
/** Durable write-behind observer; absent leaves the filesystem ephemeral. */
@@ -133,9 +185,8 @@ export class MemoryVfs implements Vfs {
private readonly mutationListeners = new Set<VfsMutationListener>()
private readonly sink: VfsMutationSink | undefined
private temporaries = 0
// Identity per path, assigned on first stat and dropped when the path goes:
// the filesystem service builds its version token from `ino` plus the
// timestamp, so a recreated path must not look like the entry it replaced.
// Directories retain path identities. File identities live on FileNode so
// descriptors, renames, and hard links continue to address the same file.
private readonly identities = new Map<string, bigint>()
private lastIdentity = 0n
@@ -256,9 +307,9 @@ export class MemoryVfs implements Vfs {
: this.directories.has(target)
? [0, this.directoryMtimes.get(target) ?? 0, true, this.directoryModes.get(target) ?? DEFAULT_DIRECTORY_MODE] as const
: fail('ENOENT', 'stat', target)
const identity = this.identityOf(target)
const identity = node === undefined ? this.identityOf(target) : this.identityOfFile(node)
return options?.bigint === true
? bigIntStatsOf(size, mtimeMs, directory, identity, mode)
? bigIntStatsOf(size, mtimeMs, directory, identity, mode, node === undefined ? 1 : this.pathsOf(node).length)
: statsOf(size, mtimeMs, directory, identity, mode)
}
@@ -276,7 +327,62 @@ export class MemoryVfs implements Vfs {
return this.lastIdentity
}
/** Forget a removed path's identity, so a recreated path reports a new one. */
/** @returns The inode-like identity retained by a file node across names. */
private identityOfFile(node: FileNode): bigint {
if (node.identity !== undefined) return node.identity
this.lastIdentity += 1n
node.identity = this.lastIdentity
return node.identity
}
/** @returns Every currently linked path for one file node. */
private pathsOf(node: FileNode): string[] {
const paths: string[] = []
for (const [path, candidate] of this.files) {
if (candidate === node) paths.push(path)
}
return paths
}
/** Publish a content or metadata write for every hard link to one node. */
private publishFile(node: FileNode, appendedFrom?: number): void {
for (const path of this.pathsOf(node)) {
this.publish({
kind: 'write', path, bytes: node.bytes, mode: node.mode, entryChanged: false,
...appendedFrom === undefined ? {} : { appendedFrom },
})
}
}
/** Replace bytes on one file identity and notify all linked paths. */
private replaceFile(node: FileNode, bytes: Uint8Array, appendedFrom?: number): void {
node.bytes = bytes
node.mtimeMs = this.touchNode(node)
this.publishFile(node, appendedFrom)
}
/** Write at one offset, zero-filling any gap. */
private writeFileNode(node: FileNode, position: number, data: Uint8Array): number {
const offset = Math.max(0, position)
const previousLength = node.bytes.length
const bytes = new Uint8Array(Math.max(previousLength, offset + data.length))
bytes.set(node.bytes)
bytes.set(data, offset)
this.replaceFile(node, bytes, offset === previousLength ? previousLength : undefined)
return data.length
}
/** Resize one file identity and notify all linked paths. */
private truncateFile(node: FileNode, length: number): void {
this.replaceFile(node, resize(node.bytes, length))
}
/** @returns Plain stats for an open file, including after its last name is removed. */
private fileStats(node: FileNode): VfsStats {
return statsOf(node.bytes.length, node.mtimeMs, false, this.identityOfFile(node), node.mode)
}
/** Forget removed directory identities, so recreated paths report new ones. */
private forgetIdentity(target: string): void {
this.identities.delete(target)
const prefix = `${target}${SEP}`
@@ -296,7 +402,12 @@ export class MemoryVfs implements Vfs {
* @returns Now, or one millisecond past the entry's current time.
*/
private touch(target: string): number {
const previous = this.files.get(target)?.mtimeMs
return this.touchNode(this.files.get(target))
}
/** @returns A modification time strictly newer than one file node's current value. */
private touchNode(node?: FileNode): number {
const previous = node?.mtimeMs
const now = Date.now()
return previous === undefined ? now : Math.max(now, previous + 1)
}
@@ -402,11 +513,14 @@ export class MemoryVfs implements Vfs {
const previous = this.files.get(target)
const mode = previous?.mode ?? (options?.mode !== undefined ? options.mode & 0o777 : DEFAULT_FILE_MODE)
const bytes = typeof data === 'string' ? encoder.encode(data) : data
this.files.set(target, { bytes, mtimeMs: this.touch(target), mode })
if (previous === undefined) this.touchDirectory(dirname(target))
this.publish({
kind: 'write', path: target, bytes, mode, entryChanged: previous === undefined,
})
if (previous !== undefined) {
this.replaceFile(previous, bytes)
return
}
const node: FileNode = { bytes, mtimeMs: this.touch(target), mode }
this.files.set(target, node)
this.touchDirectory(dirname(target))
this.publish({ kind: 'write', path: target, bytes, mode, entryChanged: true })
}
/**
@@ -453,46 +567,89 @@ export class MemoryVfs implements Vfs {
...this.handleTail(target),
}
}
const exists = this.files.has(target)
if (flags.startsWith('r') && !exists) fail('ENOENT', 'open', target)
if (flags.startsWith('wx') && exists) fail('EEXIST', 'open', target)
if (!flags.startsWith('r') && !this.directories.has(dirname(target))) fail('ENOENT', 'open', target)
const creation = mode === undefined ? {} : { mode }
if (flags.startsWith('w') && !flags.startsWith('wx')) this.writeFileSync(target, new Uint8Array(), creation)
if (flags.startsWith('wx')) this.writeFileSync(target, new Uint8Array(), { flag: 'wx', ...creation })
if (flags.startsWith('a') && !exists) this.writeFileSync(target, new Uint8Array(), creation)
const appending = flags.startsWith('a')
const file = this.openFileSync(target, flags, mode)
let position = 0
let closed = false
const current = (syscall: string): VfsOpenFile => {
if (closed) fail('EBADF', syscall, target)
return file
}
return {
write: async (data: string | Uint8Array): Promise<{ bytesWritten: number }> => {
const bytes = typeof data === 'string' ? encoder.encode(data) : data
this.appendFileSync(target, bytes)
return { bytesWritten: bytes.length }
const descriptor = current('write')
const offset = descriptor.append ? descriptor.stat().size : position
const bytesWritten = descriptor.write(offset, bytes)
position = offset + bytesWritten
return { bytesWritten }
},
// A handle opened for append must append here too: session persistence
// opens the log with `a` and writes each batch through this method, so a
// truncating write would replace the whole log with the newest batch.
writeFile: async (data: string | Uint8Array): Promise<void> => {
if (appending) this.appendFileSync(target, data)
else this.writeFileSync(target, data)
const bytes = typeof data === 'string' ? encoder.encode(data) : data
const descriptor = current('write')
const offset = descriptor.append ? descriptor.stat().size : position
position = offset + descriptor.write(offset, bytes)
},
readFile: async (options?: VfsReadOptions): Promise<string | Uint8Array> => {
const descriptor = current('read')
const bytes = descriptor.read(position, Math.max(0, descriptor.stat().size - position))
position += bytes.length
return encodingOf(options) === undefined ? bytes : decoder.decode(bytes)
},
readFile: async (options?: VfsReadOptions): Promise<string | Uint8Array> => this.readFileSync(target, options),
truncate: async (length = 0): Promise<void> => {
const node = this.files.get(target)
if (node === undefined) fail('ENOENT', 'ftruncate', target)
const bytes = node.bytes.slice(0, length)
this.files.set(target, { bytes, mtimeMs: this.touch(target), mode: node.mode })
this.publish({ kind: 'write', path: target, bytes, mode: node.mode, entryChanged: false })
current('ftruncate').truncate(length)
},
...this.handleTail(target),
stat: async (): Promise<VfsStats> => current('fstat').stat(),
sync: async (): Promise<void> => { current('fsync'); await this.flush() },
datasync: async (): Promise<void> => { current('fdatasync'); await this.flush() },
close: async (): Promise<void> => { closed = true },
}
}
/**
* The handle members that do not depend on how the file was opened.
*
* Open one synchronous descriptor over a stable file identity.
* @param path - File path.
* @param flags - Node open flags.
* @param mode - Permission bits applied only when a file is created.
* @returns An open file that survives path rename, replacement, and unlink.
*/
openFileSync(path: string, flags = 'r', mode?: number): VfsOpenFile {
const target = this.key(path)
const access = openMode(flags)
const existing = this.files.get(target)
if (this.directories.has(target)) fail('EISDIR', 'open', target)
if (access.exclusive && existing !== undefined) fail('EEXIST', 'open', target)
if (!access.create && existing === undefined) fail('ENOENT', 'open', target)
if (access.create && existing === undefined) {
this.writeFileSync(target, new Uint8Array(), mode === undefined ? undefined : { mode })
} else if (access.truncate && existing !== undefined) {
this.truncateFile(existing, 0)
}
const node = this.files.get(target)
if (node === undefined) fail('ENOENT', 'open', target)
return {
readable: access.readable,
writable: access.writable,
append: access.append,
read: (position, length) => {
if (!access.readable) fail('EBADF', 'read', target)
return node.bytes.subarray(position, position + length)
},
write: (position, data) => {
if (!access.writable) fail('EBADF', 'write', target)
return this.writeFileNode(node, access.append ? node.bytes.length : position, data)
},
truncate: (length) => {
if (!access.writable) fail('EINVAL', 'ftruncate', target)
this.truncateFile(node, length)
},
stat: () => this.fileStats(node),
}
}
/**
* Directory-handle members for metadata, durability, and release.
* `sync`/`datasync` settle an attached durable sink; an ephemeral filesystem
* resolves immediately. `close` releases nothing, so both directory and file
* handles share this tail.
* resolves immediately and `close` releases nothing.
* @param target - Normalized path the handle was opened on.
* @returns Metadata plus the no-op durability and release calls.
*/
@@ -515,14 +672,7 @@ export class MemoryVfs implements Vfs {
const existing = this.files.get(target)
const addition = typeof data === 'string' ? encoder.encode(data) : data
if (existing === undefined) { this.writeFileSync(target, addition); return }
const merged = new Uint8Array(existing.bytes.length + addition.length)
merged.set(existing.bytes)
merged.set(addition, existing.bytes.length)
this.files.set(target, { bytes: merged, mtimeMs: this.touch(target), mode: existing.mode })
this.publish({
kind: 'write', path: target, bytes: merged, mode: existing.mode,
entryChanged: false, appendedFrom: existing.bytes.length,
})
this.writeFileNode(existing, existing.bytes.length, addition)
}
/**
@@ -533,8 +683,10 @@ export class MemoryVfs implements Vfs {
renameSync(from: string, to: string): void {
const source = this.key(from)
const destination = this.key(to)
if (source === destination) return
const node = this.files.get(source)
if (node !== undefined) {
if (this.directories.has(destination)) fail('EISDIR', 'rename', destination)
if (!this.directories.has(dirname(destination))) fail('ENOENT', 'rename', destination)
this.files.delete(source)
this.files.set(destination, node)
@@ -588,9 +740,8 @@ export class MemoryVfs implements Vfs {
/**
* Give existing bytes a second name.
*
* There are no inodes here, so the two names share the bytes present at link
* time and diverge on the next write through either name; session persistence
* links a finished file to a stable name, which this satisfies.
* Both names retain one file identity, so writes and metadata changes through
* either name remain visible through the other until that name is removed.
* @param existing - Source file path.
* @param next - Additional path; its parent must exist and it must be free.
*/
@@ -615,9 +766,7 @@ export class MemoryVfs implements Vfs {
const target = this.key(path)
const node = this.files.get(target)
if (node === undefined) fail('ENOENT', 'truncate', target)
const bytes = node.bytes.slice(0, length)
this.files.set(target, { bytes, mtimeMs: this.touch(target), mode: node.mode })
this.publish({ kind: 'write', path: target, bytes, mode: node.mode, entryChanged: false })
this.truncateFile(node, length)
}
/**
@@ -630,7 +779,7 @@ export class MemoryVfs implements Vfs {
const node = this.files.get(target)
if (node !== undefined) {
node.mode = mode & 0o777
this.publish({ kind: 'chmod', path: target, mode: node.mode })
for (const path of this.pathsOf(node)) this.publish({ kind: 'chmod', path, mode: node.mode })
return
}
if (this.directories.has(target)) {
@@ -1,8 +1,8 @@
/**
* Filesystem interfaces shared by every VFS backend. The shipped implementation
* is in memory; a browser-persistent backend would implement the same faces. Errors carry
* Node's `code` values because roster plugins branch on them (`ENOENT` for
* optional files, `EACCES` for read-only trees).
* is in memory; browser persistence hydrates it and consumes its committed
* mutation stream. Errors carry Node's `code` values because roster plugins
* branch on them (`ENOENT` for optional files, `EACCES` for read-only trees).
* @module @deepseek-ai/dsh-experimental-webworker-runtime/src/storage/types
*/
@@ -22,7 +22,7 @@ export interface VfsError extends Error {
/** Subset of `fs.Stats` the roster reads. */
export interface VfsStats {
readonly size: number
/** Stable identity while an entry exists; recreation receives another value. */
/** Stable file identity across rename and hard links; recreation receives another value. */
readonly ino: number
readonly mtimeMs: number
readonly ctimeMs: number
@@ -54,7 +54,7 @@ export interface VfsBigIntStats {
readonly mode: bigint
/** One virtual device holds the whole image. */
readonly dev: bigint
/** Identity of the entry at this path; a removed and recreated path gets a new one. */
/** File identity retained across rename and hard links; recreation gets a new one. */
readonly ino: bigint
readonly nlink: bigint
readonly mtimeMs: bigint
@@ -125,6 +125,40 @@ export interface VfsFileHandle {
close(): Promise<void>
}
/** Open-file identity used by synchronous Node-style descriptors. */
export interface VfsOpenFile {
/** Whether reads are allowed by the flags used at open time. */
readonly readable: boolean
/** Whether writes and truncation are allowed by the flags used at open time. */
readonly writable: boolean
/** Whether each write targets the current end of the opened file. */
readonly append: boolean
/**
* Read bytes from the opened file identity.
* @param position - Absolute byte offset.
* @param length - Maximum byte count.
* @returns A view of the available bytes.
*/
read(position: number, length: number): Uint8Array
/**
* Write bytes to the opened file identity.
* @param position - Absolute byte offset, ignored for append descriptors.
* @param data - Bytes to write.
* @returns Number of bytes written.
*/
write(position: number, data: Uint8Array): number
/**
* Resize the opened file, zero-filling growth.
* @param length - Target byte length.
*/
truncate(length: number): void
/**
* Read metadata from the opened file identity.
* @returns Current file metadata, including after rename or unlink.
*/
stat(): VfsStats
}
/**
* One completed change to the authoritative in-memory filesystem.
*
@@ -203,6 +237,8 @@ export interface Vfs {
unlinkSync(path: string): void
rmSync(path: string, options?: { recursive?: boolean; force?: boolean }): void
mkdtempSync(prefix: string): string
/** Open and retain one file identity until its Node descriptor closes. */
openFileSync(path: string, flags?: string, mode?: number): VfsOpenFile
seed(path: string, data: string | Uint8Array, options?: VfsSeedOptions): void
seedDirectory(path: string, options?: VfsSeedOptions): void
usage(): { files: number; directories: number; bytes: number }
@@ -238,6 +238,14 @@ it('normalizes relative grants and denies sibling-prefix escapes and unreadable
expect(result.stdout).not.toContain('private')
})
it('treats trailing-slash grants as the same subtree', async () => {
const invocation = parseLandlockArguments(['--rw', '/tmp/', '--', 'true'])
if (invocation.kind !== 'run') throw new Error('expected a confined run invocation')
const guarded = await landlockFileSystem(hostFileSystem(), invocation, WORKSPACE)
await guarded.writeText('/tmp/nested.txt', 'allowed')
expect(vfs.readFileSync(`${TMP}/nested.txt`, 'utf8')).toBe('allowed')
})
it('presents the virtual device directory without storing it in the VFS', async () => {
const child = spawn(launcherPath(), [
...grantArgs({ readOnly: ['/'], readWrite: ['/dev/null'] }),
@@ -1,12 +1,17 @@
/** Node differential checks for the Worker filesystem watcher and stream faces. */
import {
closeSync as closeNodeSync,
createReadStream as createNodeReadStream,
createWriteStream as createNodeWriteStream,
mkdtempSync,
openSync as openNodeSync,
readSync as readNodeSync,
readFileSync,
renameSync as renameNodeSync,
rmSync,
unwatchFile as unwatchNodeFile,
watchFile as watchNodeFile,
writeSync as writeNodeSync,
writeFileSync,
} from 'node:fs'
import { tmpdir } from 'node:os'
@@ -106,6 +111,139 @@ async function writeScenario(create: () => WritableFileStream): Promise<{
}
describe('file streams', () => {
it('keeps an opened file identity across rename, replacement, and unlink', () => {
const nativeRoot = mkdtempSync(join(tmpdir(), 'dsh-stream-diff-'))
nativeRoots.push(nativeRoot)
const nativePath = join(nativeRoot, 'identity.txt')
const workerPath = `${VFS_ROOT}/identity.txt`
const nativeScenario = (): string[] => {
writeFileSync(nativePath, 'original')
const fd = openNodeSync(nativePath, 'r')
renameNodeSync(nativePath, `${nativePath}.moved`)
writeFileSync(nativePath, 'replacement')
const beforeUnlink = Buffer.alloc(16)
const firstCount = readNodeSync(fd, beforeUnlink, 0, beforeUnlink.length, 0)
rmSync(`${nativePath}.moved`)
const afterUnlink = Buffer.alloc(16)
const secondCount = readNodeSync(fd, afterUnlink, 0, afterUnlink.length, 0)
closeNodeSync(fd)
return [beforeUnlink.subarray(0, firstCount).toString(), afterUnlink.subarray(0, secondCount).toString()]
}
const workerScenario = (): string[] => {
vfs.writeFileSync(workerPath, 'original')
const fd = workerFs.openSync(workerPath, 'r')
vfs.renameSync(workerPath, `${workerPath}.moved`)
vfs.writeFileSync(workerPath, 'replacement')
const beforeUnlink = Buffer.alloc(16)
const firstCount = workerFs.readSync(fd, beforeUnlink, 0, beforeUnlink.length, 0)
vfs.rmSync(`${workerPath}.moved`)
const afterUnlink = Buffer.alloc(16)
const secondCount = workerFs.readSync(fd, afterUnlink, 0, afterUnlink.length, 0)
workerFs.closeSync(fd)
return [beforeUnlink.subarray(0, firstCount).toString(), afterUnlink.subarray(0, secondCount).toString()]
}
expect(workerScenario()).toEqual(nativeScenario())
})
it('keeps a read stream on the file opened before an atomic replacement', async () => {
const nativeRoot = mkdtempSync(join(tmpdir(), 'dsh-stream-diff-'))
nativeRoots.push(nativeRoot)
const nativePath = join(nativeRoot, 'stream-identity.txt')
const workerPath = `${VFS_ROOT}/stream-identity.txt`
writeFileSync(nativePath, 'original')
vfs.writeFileSync(workerPath, 'original')
const readAfterReplacement = async (
stream: AsyncIterable<Uint8Array> & { once(event: string, listener: () => void): unknown },
replace: () => void,
): Promise<string> => {
stream.once('open', replace)
const chunks: Uint8Array[] = []
for await (const chunk of stream) chunks.push(chunk)
return Buffer.concat(chunks).toString()
}
const native = await readAfterReplacement(createNodeReadStream(nativePath, { highWaterMark: 2 }), () => {
renameNodeSync(nativePath, `${nativePath}.moved`)
writeFileSync(nativePath, 'replacement')
})
const worker = await readAfterReplacement(workerFs.createReadStream(workerPath, { highWaterMark: 2 }), () => {
vfs.renameSync(workerPath, `${workerPath}.moved`)
vfs.writeFileSync(workerPath, 'replacement')
})
expect(worker).toBe(native)
})
it('rejects descriptor operations that conflict with the open mode', () => {
const nativeRoot = mkdtempSync(join(tmpdir(), 'dsh-stream-diff-'))
nativeRoots.push(nativeRoot)
const nativePath = join(nativeRoot, 'mode.txt')
const workerPath = `${VFS_ROOT}/mode.txt`
writeFileSync(nativePath, 'content')
vfs.writeFileSync(workerPath, 'content')
const codeOf = (run: () => unknown): string | undefined => {
try {
run()
return undefined
} catch (error) {
return (error as NodeJS.ErrnoException).code
}
}
const nativeReadOnly = openNodeSync(nativePath, 'r')
const workerReadOnly = workerFs.openSync(workerPath, 'r')
expect(codeOf(() => workerFs.writeSync(workerReadOnly, 'x')))
.toBe(codeOf(() => writeNodeSync(nativeReadOnly, 'x')))
closeNodeSync(nativeReadOnly)
workerFs.closeSync(workerReadOnly)
const nativeWriteOnly = openNodeSync(nativePath, 'w')
const workerWriteOnly = workerFs.openSync(workerPath, 'w')
expect(codeOf(() => workerFs.readSync(workerWriteOnly, Buffer.alloc(1), 0, 1, 0)))
.toBe(codeOf(() => readNodeSync(nativeWriteOnly, Buffer.alloc(1), 0, 1, 0)))
closeNodeSync(nativeWriteOnly)
workerFs.closeSync(workerWriteOnly)
})
it('keeps hard-link identity and content shared through the Node face', () => {
const source = `${VFS_ROOT}/linked-source.txt`
const alias = `${VFS_ROOT}/linked-alias.txt`
workerFs.writeFileSync(source, 'one')
workerFs.linkSync(source, alias)
expect(workerFs.statSync(alias, { bigint: true }).ino)
.toBe(workerFs.statSync(source, { bigint: true }).ino)
workerFs.appendFileSync(alias, '-two')
expect(workerFs.readFileSync(source, 'utf8')).toBe('one-two')
})
it('reports incompatible read and write stream flags as EBADF', async () => {
const path = `${VFS_ROOT}/stream-mode.txt`
vfs.writeFileSync(path, 'content')
const writeError = nextValue<NodeJS.ErrnoException>((resolve) => {
const stream = workerFs.createWriteStream(path, { flags: 'r' })
stream.once('error', resolve)
stream.end('x')
})
await expect(writeError).resolves.toMatchObject({ code: 'EBADF' })
const read = workerFs.createReadStream(path, { flags: 'w' })
const readError = nextValue<NodeJS.ErrnoException>((resolve) => { read.once('error', resolve) })
read.resume()
await expect(readError).resolves.toMatchObject({ code: 'EBADF' })
})
it('zero-extends through promise and file-handle truncate', async () => {
const path = `${VFS_ROOT}/truncate.txt`
vfs.writeFileSync(path, new Uint8Array([1, 2]))
await workerFsp.truncate(path, 4)
expect([...workerFs.readFileSync(path) as Uint8Array]).toEqual([1, 2, 0, 0])
const handle = await workerFsp.open(path, 'r+')
await handle.truncate(6)
await handle.close()
expect([...workerFs.readFileSync(path) as Uint8Array]).toEqual([1, 2, 0, 0, 0, 0])
})
it('matches Node chunking, inclusive ranges, and read lifecycle ordering', async () => {
const nativeRoot = mkdtempSync(join(tmpdir(), 'dsh-stream-diff-'))
nativeRoots.push(nativeRoot)
@@ -175,6 +313,51 @@ describe('file streams', () => {
expect(error).toMatchObject({ name: 'AbortError', code: 'ABORT_ERR' })
})
it('keeps autoClose false descriptors open until explicit stream close', async () => {
const readPath = `${VFS_ROOT}/manual-read-close.txt`
vfs.writeFileSync(readPath, 'content')
const read = workerFs.createReadStream(readPath, { autoClose: false })
read.resume()
await nextValue<undefined>((resolve) => { read.once('end', () => { resolve(undefined) }) })
const readFd = read.fd
expect(readFd).not.toBeNull()
expect(read.destroyed).toBe(false)
expect(() => workerFs.readSync(readFd as number, Buffer.alloc(1), 0, 1, 0)).not.toThrow()
const readClosed = nextValue<undefined>((resolve) => { read.once('close', () => { resolve(undefined) }) })
read.close()
await readClosed
expect(() => workerFs.readSync(readFd as number, Buffer.alloc(1), 0, 1, 0)).toThrow(/EBADF/)
const write = workerFs.createWriteStream(`${VFS_ROOT}/manual-write-close.txt`, { autoClose: false })
write.end('a')
await nextValue<undefined>((resolve) => { write.once('finish', () => { resolve(undefined) }) })
const writeFd = write.fd
expect(writeFd).not.toBeNull()
expect(write.destroyed).toBe(false)
expect(workerFs.writeSync(writeFd as number, 'b')).toBe(1)
const writeClosed = nextValue<undefined>((resolve) => { write.once('close', () => { resolve(undefined) }) })
write.close()
await writeClosed
expect(workerFs.readFileSync(`${VFS_ROOT}/manual-write-close.txt`, 'utf8')).toBe('ab')
vfs.writeFileSync(`${VFS_ROOT}/manual-error-close.txt`, 'content')
const errored = workerFs.createWriteStream(`${VFS_ROOT}/manual-error-close.txt`, {
flags: 'r',
autoClose: false,
})
const error = nextValue<NodeJS.ErrnoException>((resolve) => { errored.once('error', resolve) })
errored.end('rejected')
await expect(error).resolves.toMatchObject({ code: 'EBADF' })
const errorFd = errored.fd
expect(errorFd).not.toBeNull()
expect(errored.destroyed).toBe(false)
expect(() => workerFs.readSync(errorFd as number, Buffer.alloc(1), 0, 1, 0)).not.toThrow()
const errorClosed = nextValue<undefined>((resolve) => { errored.once('close', () => { resolve(undefined) }) })
errored.destroy()
await errorClosed
expect(() => workerFs.readSync(errorFd as number, Buffer.alloc(1), 0, 1, 0)).toThrow(/EBADF/)
})
it('matches Node positional overwrite and missing-file failure', async () => {
const nativeRoot = mkdtempSync(join(tmpdir(), 'dsh-stream-diff-'))
nativeRoots.push(nativeRoot)
@@ -258,6 +441,22 @@ async function watchFileScenario(
}
describe('watchers', () => {
it('does not catch exceptions thrown by a successful stat callback', () => {
const path = `${VFS_ROOT}/callback.txt`
vfs.writeFileSync(path, 'value')
const failure = new Error('callback failed')
let calls = 0
const dispatch = vi.spyOn(globalThis, 'queueMicrotask').mockImplementation((callback) => { callback() })
expect(() => {
workerFs.stat(path, () => {
calls += 1
throw failure
})
}).toThrow(failure)
expect(calls).toBe(1)
dispatch.mockRestore()
})
it('matches Node watchFile state transitions for a missing and recreated file', async () => {
const nativeRoot = mkdtempSync(join(tmpdir(), 'dsh-watch-diff-'))
nativeRoots.push(nativeRoot)
@@ -398,16 +597,20 @@ describe('watchers', () => {
await expect(event).resolves.toEqual(['rename', 'file.txt'])
})
it('rejects an already-aborted callback watcher without retaining a subscription', () => {
it('returns an asynchronously closing watcher for a pre-aborted signal', async () => {
const controller = new AbortController()
const reason = new Error('already stopped')
controller.abort(reason)
try {
workerFs.watch(VFS_ROOT, { signal: controller.signal })
throw new Error('watch unexpectedly opened')
} catch (error) {
expect(error).toMatchObject({ name: 'AbortError', code: 'ABORT_ERR', cause: reason })
}
controller.abort(new Error('already stopped'))
const order: string[] = []
const watcher = workerFs.watch(VFS_ROOT, { signal: controller.signal })
const closed = nextValue<undefined>((resolve) => {
watcher.once('close', () => {
order.push('close')
resolve(undefined)
})
})
order.push('return')
await closed
expect(order).toEqual(['return', 'close'])
expect(() => { vfs.writeFileSync(`${VFS_ROOT}/after-abort.txt`, 'x') }).not.toThrow()
})
@@ -474,6 +677,14 @@ describe('watchers', () => {
await expect(iterator.next()).rejects.toMatchObject({ name: 'AbortError', code: 'ABORT_ERR' })
})
it('rejects the first promise-watch read for a pre-aborted signal', async () => {
const controller = new AbortController()
const reason = new Error('already stopped')
controller.abort(reason)
const iterator = workerFsp.watch(VFS_ROOT, { signal: controller.signal })[Symbol.asyncIterator]()
await expect(iterator.next()).rejects.toMatchObject({ name: 'AbortError', code: 'ABORT_ERR', cause: reason })
})
it('lets promise-watch return interrupt a pending next call', async () => {
const iterator = workerFsp.watch(VFS_ROOT)[Symbol.asyncIterator]()
const pending = iterator.next()
@@ -40,10 +40,7 @@ describe('entry identity', () => {
expect(identity(vfs, '/dsh/skills/git/SKILL.md')).not.toBe(before)
})
it('assigns the destination of a rename an identity of its own', () => {
// Identity belongs to the path, not to the bytes: a renamed-over path must
// stop looking like the entry it replaced, which is the property the guard
// reads. The source identity deliberately does not follow the move.
it('moves the source identity when a file replaces another path', () => {
const vfs = new MemoryVfs()
vfs.seed('/dsh/from.txt', 'moved')
vfs.seed('/dsh/to.txt', 'replaced')
@@ -51,7 +48,7 @@ describe('entry identity', () => {
vfs.renameSync('/dsh/from.txt', '/dsh/to.txt')
const renamed = identity(vfs, '/dsh/to.txt')
expect(vfs.readFileSync('/dsh/to.txt', 'utf8')).toBe('moved')
expect([renamed === source, renamed === destination]).toEqual([false, false])
expect([renamed === source, renamed === destination]).toEqual([true, false])
})
})
@@ -92,6 +89,16 @@ describe('modification time', () => {
expect(modified(vfs, '/dsh/log.jsonl')).toBe(1_700_000_005_000)
})
it('extends truncation with zero bytes', async () => {
const vfs = new MemoryVfs()
vfs.seed('/dsh/file', new Uint8Array([1, 2]))
vfs.truncateSync('/dsh/file', 5)
expect([...vfs.readFileSync('/dsh/file') as Uint8Array]).toEqual([1, 2, 0, 0, 0])
const handle = vfs.open('/dsh/file', 'r+')
await handle.truncate(7)
expect([...vfs.readFileSync('/dsh/file') as Uint8Array]).toEqual([1, 2, 0, 0, 0, 0, 0])
})
it('advances a directory only when its immediate entry set changes', () => {
vi.spyOn(Date, 'now').mockReturnValue(1_700_000_000_000)
const vfs = new MemoryVfs()
@@ -175,6 +182,24 @@ describe('mutation publication', () => {
expect(flushes).toBe(1)
})
it('publishes descriptor writes at the file identity current path', () => {
const mutations: VfsMutation[] = []
const vfs = new MemoryVfs()
vfs.seed('/dsh/source', 'old')
const descriptor = vfs.openFileSync('/dsh/source', 'r+')
vfs.subscribe((mutation) => { mutations.push(mutation) })
vfs.renameSync('/dsh/source', '/dsh/destination')
mutations.length = 0
descriptor.write(0, new TextEncoder().encode('new'))
expect(mutations.map(mutation => mutation.path)).toEqual(['/dsh/destination'])
expect(vfs.readFileSync('/dsh/destination', 'utf8')).toBe('new')
vfs.unlinkSync('/dsh/destination')
mutations.length = 0
descriptor.write(0, new TextEncoder().encode('detached'))
expect(mutations).toEqual([])
expect(new TextDecoder().decode(descriptor.read(0, descriptor.stat().size))).toBe('detached')
})
it('decomposes a directory rename into replayable destination state', () => {
const recorded: VfsMutation[] = []
const vfs = new MemoryVfs({
@@ -196,13 +221,30 @@ describe('mutation publication', () => {
})
describe('hard links', () => {
it('shares the bytes present at link time and diverges on the next write', () => {
it('shares identity, bytes, and mode until one name is removed', () => {
const vfs = new MemoryVfs()
vfs.seed('/dsh/session.jsonl', 'committed\n')
vfs.linkSync('/dsh/session.jsonl', '/dsh/session-latest.jsonl')
expect(identity(vfs, '/dsh/session-latest.jsonl')).toBe(identity(vfs, '/dsh/session.jsonl'))
expect(vfs.readFileSync('/dsh/session-latest.jsonl', 'utf8')).toBe('committed\n')
const changedPaths: string[] = []
vfs.subscribe((mutation) => { changedPaths.push(mutation.path) })
vfs.appendFileSync('/dsh/session.jsonl', 'appended\n')
expect(changedPaths).toEqual(['/dsh/session.jsonl', '/dsh/session-latest.jsonl'])
expect(vfs.readFileSync('/dsh/session.jsonl', 'utf8')).toBe('committed\nappended\n')
expect(vfs.readFileSync('/dsh/session-latest.jsonl', 'utf8')).toBe('committed\n')
expect(vfs.readFileSync('/dsh/session-latest.jsonl', 'utf8')).toBe('committed\nappended\n')
vfs.chmodSync('/dsh/session-latest.jsonl', 0o600)
expect((vfs.statSync('/dsh/session.jsonl') as VfsStats).mode & 0o777).toBe(0o600)
vfs.unlinkSync('/dsh/session-latest.jsonl')
expect(vfs.readFileSync('/dsh/session.jsonl', 'utf8')).toBe('committed\nappended\n')
})
it('rejects renaming a file over an existing directory', () => {
const vfs = new MemoryVfs()
vfs.seed('/dsh/file', 'value')
vfs.seedDirectory('/dsh/directory')
expect(() => { vfs.renameSync('/dsh/file', '/dsh/directory') }).toThrow(expect.objectContaining({ code: 'EISDIR' }))
expect(vfs.readFileSync('/dsh/file', 'utf8')).toBe('value')
expect(vfs.statSync('/dsh/directory').isDirectory()).toBe(true)
})
})