diff --git a/.editorconfig b/.editorconfig index 4cd71ed..395bbdd 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,6 +10,7 @@ trim_trailing_whitespace = true [*.{md,mdx}] indent_size = 2 +max_line_length = 120 [*.{yml,yaml}] @@ -17,3 +18,6 @@ indent_size = 2 [*.toml] indent_size = 2 + +[Makefile] +indent_style = tab diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..a899103 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,14 @@ +version: 2 +updates: + - package-ecosystem: "cargo" + directory: "/" + schedule: + interval: "daily" + open-pull-requests-limit: 10 + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "rust-toolchain" + schedule: + interval: "daily" diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index ad51dcb..51170d3 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -28,10 +28,8 @@ jobs: matrix: command: - cargo fmt --all -- --check - - cargo check --all-targets - - cargo check --target wasm32-unknown-unknown - - cargo clippy --all-targets - - cargo clippy --target wasm32-unknown-unknown + - cargo check --target wasm32-unknown-unknown --all-targets + - cargo clippy --target wasm32-unknown-unknown --all-targets steps: - uses: actions/checkout@v5 - uses: wireapp/core-crypto/.github/actions/setup-and-cache-rust@main @@ -47,7 +45,7 @@ jobs: fail-fast: false matrix: command: - - cargo nextest run + - cargo nextest run --no-tests warn - cargo test --doc steps: diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..bbcd6a7 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,11 @@ +{ + "rust-analyzer.cargo.target": "wasm32-unknown-unknown", + "[markdown]": { + "editor.rulers": [ + 80, + 120 + ], + "rewrap.autoWrap.enabled": true, + "rewrap.wrappingColumn": 120 + } +} diff --git a/Cargo.toml b/Cargo.toml index 35b8a91..3b1d2f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,5 +4,35 @@ version = "0.1.0" edition = "2024" description = "Implements a `StorageBackend` which delegates to OPFS" license = "GPL-3.0-only" +repository = "https://github.com/wireapp/redb-opfs" + +[lib] +crate-type = ["lib", "cdylib"] [dependencies] +derive_more = { version = "2.0.1", features = ["display", "error", "from"] } +parking_lot = "0.12.4" +# Temporary! Should be next released version containing https://github.com/cberner/redb/pull/1084 +redb = { git = "https://github.com/cberner/redb", branch = "master", version = "3.0" } +wasm-bindgen = "0.2.101" +wasm-bindgen-futures = "0.4.51" + +[target.'cfg(target_family = "wasm")'.dependencies] +js-sys = "0.3.80" +web-sys = { version = "0.3.80", features = [ + "DedicatedWorkerGlobalScope", + "DomException", + "FileSystemDirectoryHandle", + "FileSystemFileHandle", + "FileSystemGetDirectoryOptions", + "FileSystemGetFileOptions", + "FileSystemReadWriteOptions", + "FileSystemSyncAccessHandle", + "StorageManager", + "WorkerGlobalScope", + "WorkerNavigator", +] } + + +[profile.release] +lto = true diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..eb8041c --- /dev/null +++ b/Makefile @@ -0,0 +1,71 @@ +RUST_RS_FILES := $(shell find . -type f -name '*.rs' 2>/dev/null | LC_ALL=C sort) +RUST_SOURCES := Cargo.toml Cargo.lock $(RUST_RS_FILES) +WASM_OUT := ts/gen/redb-opfs_bg.wasm +DTS_OUT := ts/gen/redb-opfs.d.ts +JS_OUT := ts/gen/redb-opfs.js + +# If RELEASE is nonempty, build in release mode +# Otherwise build in dev mode, which is much faster +WASM_BUILD_ARGS := $(if $(RELEASE),,--dev) + +.PHONY: clean clean-bun clean-pack clean-wwex +clean: + cargo clean + $(MAKE) clean-bun + $(MAKE) clean-pack + $(MAKE) clean-wwex + +clean-bun: + rm -rf $(WWEX)/node_modules + +clean-pack: + rm -rf ts/gen + +clean-wwex: + rm -rf $(WWTARGET) + +Cargo.lock: Cargo.toml + cargo check + @touch $@ + +$(JS_OUT) $(DTS_OUT) $(WASM_OUT) &: $(RUST_SOURCES) + wasm-pack build \ + --locked \ + --no-pack \ + --out-dir ts/gen \ + --out-name redb-opfs \ + --mode normal \ + --target web \ + $(WASM_BUILD_ARGS) + +# human name for building wasm +.PHONY: wasm-build +wasm-build: $(WASM_OUT) + +WWEX := examples/web-worker +WWEX_HTML := $(WWEX)/src/index.html +WWEX_TS := $(shell find $(WWEX)/src -type f -name '*.ts' 2>/dev/null | LC_ALL=C sort) +WWEX_SOURCES := $(WWEX)/bun.lock $(WWEX)/package.json $(WWEX_TS) +WWTARGET := target/web-worker-example +BUNDLE_INDEX := $(WWTARGET)/index.js +BUNDLE_WORKER := $(WWTARGET)/worker.js +BUNDLE_WASM := $(WWTARGET)/redb-opfs_bg.wasm + +$(BUNDLE_INDEX) $(BUNDLE_WORKER) $(BUNDLE_WASM) &: $(WWEX_HTML) $(WASM_OUT) $(JS_OUT) $(WWEX_SOURCES) + cd examples/web-worker/src && \ + bun build \ + --target browser \ + --format esm \ + index.ts \ + worker.ts \ + --outdir ../../../$(WWTARGET) + cp $(WWEX_HTML) $(WASM_OUT) $(WWTARGET) + + +.PHONY: web-worker-example +web-worker-example: $(BUNDLE_INDEX) $(BUNDLE_WORKER) $(BUNDLE_WASM) +# cargo install --locked miniserve + miniserve \ + --index index.html \ + --port 8000 \ + $(WWTARGET) diff --git a/README.md b/README.md index 4df0462..33a530b 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,94 @@ # `redb-opfs`: Implements a `StorageBackend` which delegates to OPFS -This allows compilation and deployment on `wasm32-unknown-unknown`. +This allows deployment on `wasm32-unknown-unknown`. -> [!WARNING] -> The contents of this README are a statement of intent, not an accurate reflection of the current state of the project. +> [!WARNING] The contents of this README are a statement of intent, not an accurate reflection of the current state of +> the project. > > Carefully inspect the code and/or generated documentation before relying on this library. ## Usage -- Add this dependency to your project: +It is important to understand that Redb's [`StorageBackend` +interface](https://docs.rs/redb/latest/redb/trait.StorageBackend.html) is fundamentally synchronous, and OPFS is +fundamentally asynchronous. There's a simple way to tie these two things +together--[`block_on`](https://docs.rs/futures-lite/latest/futures_lite/future/fn.block_on.html)--but that method is +illegal on the main thread, in order not to block the UI. - ```sh - cargo add redb-opfs - ``` +> [!IMPORTANT] The `OpfsBackend` instance **must** run on a web worker. -- Explicitly choose this backend when initializing your `Database`: +This gives rise to two use cases. - ```rust - use redb_opfs::OpfsStorageBackend; +### Your Rust code is already running in a web worker - let database = redb::Builder::new() - .create_with_backend(OpfsStorageBackend::new("my-db"))?; - ``` +This case is nice and simple; everything stays within Rust. Just explicitly choose this backend when initializing your +`Database`: -- Go nuts! +```rust +use redb_opfs::OpfsBackend; + +let database = redb::Builder::new() + .create_with_backend(OpfsBackend::new("my-db")?)?; +``` + +### Your Rust code is running in the main thread + +> [!NOTE] Running in this configuration introduces unavoidable performance penalties; when possible, you should prefer +> to run all your Rust code within a web worker to avoid these. + +In this case we need to instantiate the `OpfsBackend` on a web worker and then instantiate the handle on the main +thread. + +You'll want to use the `worker-shim.js` worker file to initialize the worker, and then hand that worker to the +`OpfsBackendHandle` + +```js +import { WorkerHandle } from "./redb-opfs"; + +const redbOpfsWorker = new Worker("worker-shim.js"); +const workerHandle = WorkerHandle(redbOpfsWorker); + +// now pass that handle to your rust code, using a mechanism of your choice. +``` + +As you're writing your own Rust anyway, you have your own means of getting the handle into your code from there. To keep +life simple, there exists `impl TryFrom for WorkerHandle`. + +Once you have that, usage is fairly simple: + +```rust +use redb_opfs::WorkerHandle; + +let worker_handle = WorkerHandle::try_from(my_js_value)?; +let database = redb::Builder::new() + .create_with_backend(worker_hanndle)?; +``` + +## Building + +### Prerequisites for WASM + +- [`wasm-bindgen` cli](https://github.com/wasm-bindgen/wasm-bindgen?tab=readme-ov-file#install-wasm-bindgen-cli) +- [wasm-pack](https://github.com/drager/wasm-pack) +- [GNU Make](https://www.gnu.org/software/make/) + +## Examples + +### Web Worker + +This example demonstrates the simplest possible case: `redb-opfs` is compiled to WASM and deployed, where it is used +only from Typescript code without interacting at all with other Rust code. In this use case, the only thing it gives us +is a fully-synchronous interface to OPFS. + +#### Prerequisites + +- [bun](https://bun.com/) +- [miniserve](https://github.com/svenstaro/miniserve) + +#### Usage + +- `make web-worker-example` +- point your browser at any of the listed URLs the server has bound itself to ## License @@ -32,4 +96,5 @@ Licensed only under [GPL-3.0](./LICENSE). ### Contribution -Any contribution intentionally submitted for inclusion in this work shall be licensed as above, without any additional terms or conditions. +Any contribution intentionally submitted for inclusion in this work shall be licensed as above, without any additional +terms or conditions. diff --git a/examples/web-worker/.gitignore b/examples/web-worker/.gitignore new file mode 100644 index 0000000..9b1ee42 --- /dev/null +++ b/examples/web-worker/.gitignore @@ -0,0 +1,175 @@ +# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore + +# Logs + +logs +_.log +npm-debug.log_ +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Caches + +.cache + +# Diagnostic reports (https://nodejs.org/api/report.html) + +report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json + +# Runtime data + +pids +_.pid +_.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover + +lib-cov + +# Coverage directory used by tools like istanbul + +coverage +*.lcov + +# nyc test coverage + +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) + +.grunt + +# Bower dependency directory (https://bower.io/) + +bower_components + +# node-waf configuration + +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) + +build/Release + +# Dependency directories + +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) + +web_modules/ + +# TypeScript cache + +*.tsbuildinfo + +# Optional npm cache directory + +.npm + +# Optional eslint cache + +.eslintcache + +# Optional stylelint cache + +.stylelintcache + +# Microbundle cache + +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history + +.node_repl_history + +# Output of 'npm pack' + +*.tgz + +# Yarn Integrity file + +.yarn-integrity + +# dotenv environment variable files + +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) + +.parcel-cache + +# Next.js build output + +.next +out + +# Nuxt.js build / generate output + +.nuxt +dist + +# Gatsby files + +# Comment in the public line in if your project uses Gatsby and not Next.js + +# https://nextjs.org/blog/next-9-1#public-directory-support + +# public + +# vuepress build output + +.vuepress/dist + +# vuepress v2.x temp and cache directory + +.temp + +# Docusaurus cache and generated files + +.docusaurus + +# Serverless directories + +.serverless/ + +# FuseBox cache + +.fusebox/ + +# DynamoDB Local files + +.dynamodb/ + +# TernJS port file + +.tern-port + +# Stores VSCode versions used for testing VSCode extensions + +.vscode-test + +# yarn v2 + +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# IntelliJ based IDEs +.idea + +# Finder (MacOS) folder config +.DS_Store diff --git a/examples/web-worker/bun.lock b/examples/web-worker/bun.lock new file mode 100644 index 0000000..be4ddd9 --- /dev/null +++ b/examples/web-worker/bun.lock @@ -0,0 +1,29 @@ +{ + "lockfileVersion": 1, + "workspaces": { + "": { + "name": "run-as-web-worker", + "devDependencies": { + "@types/bun": "latest", + }, + "peerDependencies": { + "typescript": "^5.0.0", + }, + }, + }, + "packages": { + "@types/bun": ["@types/bun@1.2.21", "", { "dependencies": { "bun-types": "1.2.21" } }, "sha512-NiDnvEqmbfQ6dmZ3EeUO577s4P5bf4HCTXtI6trMc6f6RzirY5IrF3aIookuSpyslFzrnvv2lmEWv5HyC1X79A=="], + + "@types/node": ["@types/node@24.3.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-3vXmQDXy+woz+gnrTvuvNrPzekOi+Ds0ReMxw0LzBiK3a+1k0kQn9f2NWk+lgD4rJehFUmYy2gMhJ2ZI+7YP9g=="], + + "@types/react": ["@types/react@19.1.12", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-cMoR+FoAf/Jyq6+Df2/Z41jISvGZZ2eTlnsaJRptmZ76Caldwy1odD4xTr/gNV9VLj0AWgg/nmkevIyUfIIq5w=="], + + "bun-types": ["bun-types@1.2.21", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-sa2Tj77Ijc/NTLS0/Odjq/qngmEPZfbfnOERi0KRUYhT9R8M4VBioWVmMWE5GrYbKMc+5lVybXygLdibHaqVqw=="], + + "csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="], + + "typescript": ["typescript@5.9.2", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A=="], + + "undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="], + } +} diff --git a/examples/web-worker/package.json b/examples/web-worker/package.json new file mode 100644 index 0000000..87a7fd7 --- /dev/null +++ b/examples/web-worker/package.json @@ -0,0 +1,11 @@ +{ + "name": "web-worker", + "module": "src/index.ts", + "type": "module", + "devDependencies": { + "@types/bun": "latest" + }, + "peerDependencies": { + "typescript": "^5.0.0" + } +} diff --git a/examples/web-worker/src/dlog.ts b/examples/web-worker/src/dlog.ts new file mode 100644 index 0000000..ca3210b --- /dev/null +++ b/examples/web-worker/src/dlog.ts @@ -0,0 +1,6 @@ +export const DEBUG_LOGS = true; +export function dlog(...s: any[]) { + if (DEBUG_LOGS) { + console.log(...s) + } +} diff --git a/examples/web-worker/src/index.html b/examples/web-worker/src/index.html new file mode 100644 index 0000000..6102648 --- /dev/null +++ b/examples/web-worker/src/index.html @@ -0,0 +1,150 @@ + + + + + + Web Worker / OpfsUser Demo + + + + +

Web Worker / OpfsUser Demo

+
+ +
+ Store Data + + + +
+ +
+ Load Data + + + +
+ + + + + diff --git a/examples/web-worker/src/index.ts b/examples/web-worker/src/index.ts new file mode 100644 index 0000000..ed6aebb --- /dev/null +++ b/examples/web-worker/src/index.ts @@ -0,0 +1,79 @@ +import type { Message, Response } from "./types"; +import { dlog } from "./dlog"; + +type PendingRequest = { + resolve: (value: Response) => void; + reject: (reason?: any) => void; +} + +export class OpfsUser { + private worker: Worker; + private ready: Promise; + private pending = new Map(); + private nextId = 0; + + constructor(workerPath: string = "./worker.js") { + this.worker = new Worker(workerPath, { type: "module" }); + this.worker.addEventListener("error", (e) => this._onError(e)); + + // this promise will resolve exactly once when the worker sends the ready message, + // then remain resolved forever, minimizing latency + this.ready = new Promise(resolve => { + const onReady = (event: MessageEvent) => { + if (event.data?.type === "ready") { + this.worker.removeEventListener("message", onReady); + this.worker.addEventListener("message", (e) => this._onMessage(e)); + resolve(); + } + } + this.worker.addEventListener("message", onReady); + }); + } + + _onError(event: ErrorEvent) { + console.error("worker error event:", event); + throw new Error(event.message); + } + + _onMessage(event: MessageEvent) { + const { id, data, error } = event.data; + dlog(`opfsu: received message from worker (${id})`); + const pending = this.pending.get(id); + if (!pending) return; + + this.pending.delete(id); + if (error) pending.reject(error); + else pending.resolve(data); + } + + destroy() { + this.worker.removeEventListener('error', this._onError); + this.worker.removeEventListener('message', this._onMessage); + this.worker.terminate() + } + + // the dispatch function picks its own id + private async dispatch(message: Omit): Promise { + // wait for worker to report that it is ready + await this.ready; + + const id = this.nextId++; + return new Promise((resolve, reject) => { + dlog(`opfsu: dispatching (${id})`); + this.pending.set(id, { resolve, reject }); + this.worker.postMessage({ id, ...message }); + }) + } + + /** Store some data in Opfs */ + async store(offset: bigint, data: Uint8Array): Promise { + dlog(`opfsu: storing ${data.length} bytes at offset ${offset}`); + await this.dispatch({ op: "store", offset, data }) + } + + /** Retrieve `size` bytes from Opfs */ + async load(offset: bigint, size: number): Promise { + dlog(`opfsu: loading ${size} bytes at offset ${offset}`); + return await this.dispatch({ op: "load", offset, size }) + } +} diff --git a/examples/web-worker/src/types.ts b/examples/web-worker/src/types.ts new file mode 100644 index 0000000..26ed61c --- /dev/null +++ b/examples/web-worker/src/types.ts @@ -0,0 +1,19 @@ +export type Message = { + id: number; + op: "store" | "load"; + offset: bigint; + data?: Uint8Array; + size?: number; +} + +export function isMessage(value: any): value is Message { + return ( + typeof value === "object" && + value !== null && + (value.op === "store" || value.op === "load") && + typeof value.offset === "bigint" && + typeof value.id === "number" + ) +} + +export type Response = Uint8Array; diff --git a/examples/web-worker/src/worker.ts b/examples/web-worker/src/worker.ts new file mode 100644 index 0000000..a6944ff --- /dev/null +++ b/examples/web-worker/src/worker.ts @@ -0,0 +1,50 @@ +declare var self: Worker; + +import init, { OpfsBackend } from "../../../ts/gen/redb-opfs" +import { dlog } from "./dlog"; +import { isMessage, type Response } from "./types"; + +dlog("worker: initializing wasm"); +await init(); +dlog("worker: initializing OpfsBackend") +const backend = await OpfsBackend.open("my-db"); + + +dlog("worker: registering message handler"); +self.addEventListener("message", + // note this is _not_ async! The magic here is using OPFS synchronously. + (event) => { + dlog("worker: processing message"); + + var ret: Response = new Uint8Array(); + try { + if (!isMessage(event.data)) { + console.error("could not decipher event.data as message"); + return; + } + + dlog(`worker: processing ${event.data.op} (${event.data.id})`); + + switch (event.data.op) { + case "store": + const writeData = event.data.data ?? new Uint8Array(); + backend.write(event.data.offset, writeData); + backend.sync_data() + break; + case "load": + ret = new Uint8Array(event.data.size ?? 0); + backend.sync_data() + backend.read(event.data.offset, ret); + break; + } + + dlog(`worker: posting response to ${event.data.id} (${ret.length} bytes)`) + self.postMessage({ id: event.data.id, data: ret }); + } catch (e) { + dlog(`worker: responding with error to ${event.data.id} (${e})`) + self.postMessage({ id: event.data.id, error: String(e) }) + } + }); + +// send the handshake ready message so the `OpfsUser` knows we can now receive messages +self.postMessage({ type: "ready" }); diff --git a/examples/web-worker/tsconfig.json b/examples/web-worker/tsconfig.json new file mode 100644 index 0000000..238655f --- /dev/null +++ b/examples/web-worker/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + // Enable latest features + "lib": ["ESNext", "DOM"], + "target": "ESNext", + "module": "ESNext", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..23e0580 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.90" +components = ["rust-analyzer"] +targets = ["wasm32-unknown-unknown"] diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..90a859f --- /dev/null +++ b/src/error.rs @@ -0,0 +1,64 @@ +use std::io::{self, ErrorKind}; + +use js_sys::{self, JsString, Object}; +use wasm_bindgen::{JsCast, JsValue}; +use web_sys::DomException; + +#[derive(Debug, derive_more::Display, derive_more::Error, derive_more::From)] +pub struct Error(pub(crate) io::Error); + +impl From for JsValue { + fn from(value: Error) -> Self { + fn construct_error_stack(err: &dyn std::error::Error) -> js_sys::Error { + let out = js_sys::Error::new(&err.to_string()); + if let Some(source) = err.source() { + let cause = construct_error_stack(source); + out.set_cause(&cause); + } + out + } + + let stacked_error = construct_error_stack(&value); + stacked_error.set_name(&format!("{}Error", value.0.kind())); + stacked_error.into() + } +} + +impl From for Error { + fn from(value: JsValue) -> Self { + match value.dyn_ref::() { + Some(dom) => match dom.code() { + DomException::NOT_FOUND_ERR => io::Error::from(ErrorKind::NotFound), + DomException::NO_DATA_ALLOWED_ERR | DomException::NO_MODIFICATION_ALLOWED_ERR => { + io::Error::from(ErrorKind::PermissionDenied) + } + DomException::TYPE_MISMATCH_ERR => io::Error::other("type mismatch"), + _ => { + let name = dom.name(); + let message = dom.message(); + io::Error::other(format!("{name}: {message}")) + } + }, + None => { + let js_serialization = Object::from(value).to_string(); + let str = ::to_string(&js_serialization); + io::Error::other(str) + } + } + .into() + } +} + +impl Error { + pub(crate) fn ad_hoc(err: impl Into>) -> Self { + io::Error::other(err).into() + } + + pub(crate) fn into_inner(self) -> io::Error { + self.0 + } + + pub(crate) fn to_io(value: JsValue) -> io::Error { + Self::from(value).into_inner() + } +} diff --git a/src/file.rs b/src/file.rs new file mode 100644 index 0000000..16574c1 --- /dev/null +++ b/src/file.rs @@ -0,0 +1,255 @@ +//! Implementation of a simple blocking File abstraction backed by OPFS. +//! +//! Because it is based on OPFS, this will only work in a web worker. +//! +//! This implementation makes several assumptions and simplifications: +//! +//! - no handle caching is performed +//! - the "current directory" is always the root and cannot be changed +//! - fs prefixes (`c:\`, `//share`, etc) are unsupported in paths +//! - parent directory annotations (`..`) are unsupported in paths +//! - files are always opened with (effectively) read+write+create mode +//! - files are never automatically truncated on creation +//! - cursor position is always initialized at 0 +//! - necessary parent directories are always silently implicitly created + +use std::{ + io::{self, ErrorKind, Read, Seek, Write}, + path::{Component, Path, PathBuf}, +}; + +use js_sys::{Function, Promise, Reflect}; +use wasm_bindgen::{JsCast, JsValue}; +use wasm_bindgen_futures::JsFuture; +use web_sys::{ + DedicatedWorkerGlobalScope, FileSystemDirectoryHandle, FileSystemFileHandle, + FileSystemGetDirectoryOptions, FileSystemGetFileOptions, FileSystemReadWriteOptions, + FileSystemSyncAccessHandle, +}; + +use super::{Error, Result}; + +/// A blocking File abstraction that operates on OPFS via a [`FileSystemSyncAccessHandle`]. +/// +/// Because this is blocking, it can only run in the context of a web worker, i.e. a [`DedicatedWorkerGlobalScope`]. +#[derive(Debug)] +pub(crate) struct File { + pub(crate) handle: FileSystemSyncAccessHandle, + pos: u64, +} + +impl File { + pub async fn open(path: impl AsRef) -> Result { + let path = virtualize_path(path)?; + let name = path + .file_name() + .ok_or(io::Error::new( + ErrorKind::InvalidFilename, + "no filename detected", + ))? + .to_str() + .ok_or(io::Error::new( + ErrorKind::InvalidFilename, + "non utf-8 chars in filename", + ))?; + + // in a perfect world, it would be + // let parent_handle = path.parent().map(open_dir).unwrap_or_else(root).await?; + // but we can't do that as each `impl Future` is a different type, even if the + // outputs resolve to the same type. + let parent_handle = match path.parent() { + Some(parent) if parent != Path::new("") => open_dir(parent).await?, + // Some case below must be empty + Some(_) | None => root().await?, + }; + + let file_handle = get_file_handle(name, &parent_handle).await?; + + Ok(File { + handle: file_handle, + pos: 0, + }) + } + + pub fn size(&self) -> io::Result { + self.handle + .get_size() + .map(|size| size as _) + .map_err(Error::to_io) + } + + /// Truncates or extends the underlying file, updating the size of this file to become `size`. + /// + /// If `size` is less than the current file's size, then the file will be shrunk. If it is greater + /// than the currrent file's size, then the file will be extended to `size` and have all intermediate + /// data filled with 0s. + /// + /// The file's cursor is not changed. In particular, if the cursor was at the end of the file and + /// the file was shrunk using this operation, the cursor will now be past the end. + /// + /// If the requested length is greater than 9007199254740991 (max safe integer in a floating-point context), + /// this will produce an error. + pub fn set_len(&mut self, size: u64) -> io::Result<()> { + const MAX_SAFE_INT: u64 = js_sys::Number::MAX_SAFE_INTEGER as _; + if size > MAX_SAFE_INT { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("requested size {size} too large, max allowed is {MAX_SAFE_INT}"), + )); + } + self.handle + .truncate_with_f64(size as _) + .map_err(Error::to_io) + } + + /// Flush any pending changes to the file system. + pub fn flush(&self) -> io::Result<()> { + self.handle.flush().map_err(Error::to_io) + } + + fn options(&self) -> FileSystemReadWriteOptions { + let options = FileSystemReadWriteOptions::new(); + options.set_at(self.pos as _); + options + } +} + +impl Seek for File { + fn seek(&mut self, seek_from: io::SeekFrom) -> io::Result { + // `SeekFrom` semantics: https://doc.rust-lang.org/nightly/std/io/enum.SeekFrom.html + self.pos = match seek_from { + io::SeekFrom::Start(offset) => offset, + io::SeekFrom::End(offset) => { + self.size()?.checked_add_signed(offset).ok_or_else(|| { + io::Error::new( + ErrorKind::InvalidInput, + "over/underflow seeking from file end", + ) + })? + } + io::SeekFrom::Current(offset) => { + self.pos.checked_add_signed(offset).ok_or_else(|| { + io::Error::new( + ErrorKind::InvalidInput, + "over/underflow seeking from current position", + ) + })? + } + }; + Ok(self.pos) + } +} + +impl Read for File { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let bytes_read = self + .handle + .read_with_u8_array_and_options(buf, &self.options()) + .map_err(Error::to_io)? as u64; + self.pos += bytes_read; + Ok(bytes_read as _) + } +} + +impl Write for File { + fn write(&mut self, buf: &[u8]) -> io::Result { + let bytes_written = self + .handle + .write_with_u8_array_and_options(buf, &self.options()) + .map_err(Error::to_io)? as u64; + self.pos += bytes_written; + Ok(bytes_written as _) + } + + fn flush(&mut self) -> io::Result<()> { + self.handle.flush().map_err(Error::to_io) + } +} + +/// Construct a normalized version of the input path +fn virtualize_path(path: impl AsRef) -> Result { + let mut out = PathBuf::new(); + + for component in path.as_ref().components() { + match component { + std::path::Component::RootDir => out.clear(), + std::path::Component::CurDir => {} + std::path::Component::Normal(normal) => out.push(normal), + std::path::Component::Prefix(_) | std::path::Component::ParentDir => { + return Err(io::Error::new( + ErrorKind::InvalidInput, + "only normal path components are supported", + ) + .into()); + } + } + } + + Ok(out) +} + +async fn root() -> Result { + let storage = DedicatedWorkerGlobalScope::from(JsValue::from(js_sys::global())) + .navigator() + .storage(); + + let root_handle = JsFuture::from(storage.get_directory()) + .await? + .dyn_into::()?; + + Ok(root_handle) +} + +async fn open_dir(path: impl AsRef) -> Result { + async fn get_dir_handle( + parent: &FileSystemDirectoryHandle, + path: &str, + ) -> Result { + let options = FileSystemGetDirectoryOptions::new(); + options.set_create(true); + + JsFuture::from(parent.get_directory_handle_with_options(path, &options)) + .await? + .dyn_into::() + .map_err(Into::into) + } + + let mut handle = root().await?; + for component in path.as_ref().components() { + let Component::Normal(component) = component else { + // shouldn't happen though because we always virtualize ahead of time + return Err(Error::ad_hoc(format!( + "non-normal component in path: {component:?}" + ))); + }; + let component = component.to_str().ok_or(io::Error::new( + ErrorKind::InvalidFilename, + "non utf-8 chars in dir name", + ))?; + handle = get_dir_handle(&handle, component).await?; + } + + Ok(handle) +} + +async fn get_file_handle( + name: &str, + dir: &FileSystemDirectoryHandle, +) -> Result { + let options = FileSystemGetFileOptions::new(); + options.set_create(true); + let file_handle = JsFuture::from(dir.get_file_handle_with_options(name, &options)) + .await? + .dyn_into::()?; + + let file_handle = JsValue::from(file_handle); + let create_sync_access_handle_promise = + Reflect::get(&file_handle, &"createSyncAccessHandle".into())? + .dyn_into::()? + .call0(&file_handle)? + .dyn_into::()?; + let sync_access_handle = JsFuture::from(create_sync_access_handle_promise) + .await? + .dyn_into::()?; + Ok(sync_access_handle) +} diff --git a/src/file_abstraction.rs b/src/file_abstraction.rs new file mode 100644 index 0000000..b586c7d --- /dev/null +++ b/src/file_abstraction.rs @@ -0,0 +1,44 @@ +use std::io::Result; + +pub(crate) trait FileAbstraction: Sized { + /// Open the specified path. + /// + /// Must have the following conditions/flags set at initialization: + /// + /// - readable + /// - writeable + /// - created if does not exist + /// - _not_ truncated + /// - initial cursor position at 0 + async fn open(path: &str) -> Result; + + /// Get the length of this file in bytes. + fn len(&self) -> Result; +} + +#[cfg(not(target_family = "wasm"))] +impl FileAbstraction for std::fs::File { + async fn open(path: &str) -> Result { + std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(path) + } + + fn len(&self) -> Result { + self.metadata().map(|metadata| metadata.len()) + } +} + +#[cfg(target_family = "wasm")] +impl FileAbstraction for crate::file::File { + async fn open(path: &str) -> Result { + ::open(path).await.map_err(crate::Error::into_inner) + } + + fn len(&self) -> Result { + self.size() + } +} diff --git a/src/lib.rs b/src/lib.rs index b93cf3f..744b8c7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,14 +1,141 @@ -pub fn add(left: u64, right: u64) -> u64 { - left + right +//! [`OpfsBackend`] mplements a [`StorageBackend`] which delegates to [OPFS] when built for wasm. +//! +//! [OPFS]: https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system + +#[cfg(target_family = "wasm")] +mod error; +#[cfg(not(target_family = "wasm"))] +mod file { + pub use std::fs::File; +} +#[cfg(target_family = "wasm")] +mod file; +mod file_abstraction; + +use std::io::{Read as _, Seek as _, SeekFrom, Write as _}; + +use file::File; +use file_abstraction::FileAbstraction; +use parking_lot::Mutex; +use redb::StorageBackend; + +#[cfg(target_family = "wasm")] +use wasm_bindgen::prelude::*; + +#[cfg(target_family = "wasm")] +pub use error::Error; + +#[cfg(not(target_family = "wasm"))] +type Error = std::io::Error; + +pub(crate) type Result = std::result::Result; +type IoResult = std::io::Result; + +/// Implementataion of a [`StorageBackend`] which delegates to [OPFS] when built for wasm. +/// +/// **IMPORTANT**: This can only ever be used within a web worker. +/// This _may_ instantiate within the main thread, but as it blocks internally, +/// it will fail at runtime on the main thread if you attempt to actually use it. +/// +/// In native contexts, this targets the local file system. +/// +/// [OPFS]: https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system +#[cfg_attr(target_family = "wasm", wasm_bindgen)] +#[derive(Debug)] +pub struct OpfsBackend { + file: Mutex, +} + +// Safety: when targeting wasm, we're really working in a single-threaded context anyway, so +// literally everything is trivially `Send`, because there are no other threads to send it to. +// +// Note that we only need to manually implement this for wasm; in native contexts, `async_fs::File` +// already implements `Send`. +#[cfg(target_family = "wasm")] +unsafe impl Send for OpfsBackend {} + +// Safety: when targeting wasm, we don't have multiple threads to send things between, but we +// very often need to coordinate between various async contexts. For this reason we put a mutex +// around the file handle, so contention is explicitly resolved. +#[cfg(target_family = "wasm")] +unsafe impl Sync for OpfsBackend {} + +#[cfg_attr(target_family = "wasm", wasm_bindgen)] +impl OpfsBackend { + /// Open the file at the specified path. + #[cfg_attr(target_family = "wasm", wasm_bindgen(js_name = open))] + pub async fn new(path: &str) -> Result { + let file = ::open(path).await?; + let file = Mutex::new(file); + Ok(Self { file }) + } +} + +impl StorageBackend for OpfsBackend { + fn len(&self) -> IoResult { + self.file.lock().len() + } + + fn set_len(&self, len: u64) -> IoResult<()> { + self.file.lock().set_len(len) + } + + fn sync_data(&self) -> IoResult<()> { + self.file.lock().flush() + } + + fn read(&self, offset: u64, out: &mut [u8]) -> IoResult<()> { + let mut guard = self.file.lock(); + guard.seek(SeekFrom::Start(offset))?; + guard.read_exact(out)?; + Ok(()) + } + + fn write(&self, offset: u64, data: &[u8]) -> IoResult<()> { + let mut guard = self.file.lock(); + guard.seek(SeekFrom::Start(offset))?; + guard.write_all(data)?; + Ok(()) + } } -#[cfg(test)] -mod tests { - use super::*; +#[cfg(target_family = "wasm")] +#[wasm_bindgen] +#[expect(clippy::len_without_is_empty)] +impl OpfsBackend { + /// Returns the size of the file, in bytes + // + // Files have length but no trivial `is_empty` impl, so we skip that + pub fn len(&self) -> Result { + ::len(self).map_err(Into::into) + } + + /// Reads some bytes from the file at the given offset. + pub fn read(&self, offset: u64, out: &mut [u8]) -> Result<()> { + ::read(self, offset, out).map_err(Into::into) + } + + /// Truncates or extends the underlying file, updating the size of this file to become `size`. + /// + /// If `size` is less than the current file's size, then the file will be shrunk. + /// If it is greater than the current file's size, then the file will be extended to `size` + /// and have all intermediate data filled with 0s. + /// + /// The file's cursor is not changed. In particular, if the cursor was at the end of the file + /// and the file is shrunk with this operaiton, the cursor will now be past the end. + #[wasm_bindgen(js_name = "setLen")] + pub fn set_len(&self, len: u64) -> Result<()> { + ::set_len(self, len).map_err(Into::into) + } + + /// Attempts to sync all OS-internal file content to disk. This might not synchronize file metadata. + #[wasm_bindgen(js_name = "syncData")] + pub fn sync_data(&self) -> Result<()> { + ::sync_data(self).map_err(Into::into) + } - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); + /// Writes some bytes to the file at the given offset. + pub fn write(&self, offset: u64, data: &[u8]) -> Result<()> { + ::write(self, offset, data).map_err(Into::into) } }