Skip to content

Commit 5a8e140

Browse files
committed
wip: add an example of using the raw OpfsBackend within a worker
We're getting closer! We have now demonstrated that initializing the wasm module can succeed. But the app still hangs on `OpfsBackend.open`. So we need to sort that.
1 parent d803e51 commit 5a8e140

File tree

13 files changed

+541
-7
lines changed

13 files changed

+541
-7
lines changed

.gitignore

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,2 @@
11
/target
22
/Cargo.lock
3-
/ts/

Cargo.toml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,7 @@ derive_more = { version = "2.0.1", features = ["display", "error"] }
1515
futures-lite = "2.6"
1616
# Temporary! Should be next released version after https://github.com/cberner/redb/pull/1065 is merged
1717
redb = { git = "https://github.com/coriolinus/redb", branch = "prgn/feat/wasm32-unknown-unknown-support", version = "3.0" }
18-
# Temporary! Should be next released version after https://github.com/utooland/tokio-fs-ext/pull/3 is merged
19-
tokio-fs-ext = { git = "https://github.com/coriolinus/tokio-fs-ext.git", branch = "prgn/feat/file-set-len", version = "0.6.1" }
18+
tokio-fs-ext = "0.6.3"
2019
wasm-bindgen = "0.2.101"
2120
wasm-bindgen-futures = "0.4.51"
2221

Makefile

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,21 @@
11
RUST_RS_FILES := $(shell find . -type f -name '*.rs' 2>/dev/null | LC_ALL=C sort)
22
RUST_SOURCES := Cargo.toml Cargo.lock $(RUST_RS_FILES)
3-
WASM_OUT := ts/redb-opfs_bg.wasm
4-
DTS_OUT := ts/redb-opfs.d.ts
5-
JS_OUT := ts/redb-opfs.js
3+
WASM_OUT := ts/gen/redb-opfs_bg.wasm
4+
DTS_OUT := ts/gen/redb-opfs.d.ts
5+
JS_OUT := ts/gen/redb-opfs.js
66

77
# If RELEASE is nonempty, build in release mode
88
# Otherwise build in dev mode, which is much faster
99
WASM_BUILD_ARGS := $(if $(RELEASE),,--dev)
1010

11+
Cargo.lock: Cargo.toml
12+
cargo check
13+
1114
$(JS_OUT) $(DTS_OUT) $(WASM_OUT) &: $(RUST_SOURCES)
1215
wasm-pack build \
1316
--locked \
1417
--no-pack \
15-
--out-dir ts \
18+
--out-dir ts/gen \
1619
--out-name redb-opfs \
1720
--mode normal \
1821
--target web \
@@ -21,3 +24,8 @@ $(JS_OUT) $(DTS_OUT) $(WASM_OUT) &: $(RUST_SOURCES)
2124
# human name for building wasm
2225
.PHONY: wasm-build
2326
wasm-build: $(WASM_OUT)
27+
28+
.PHONY: web-worker-example
29+
web-worker-example: $(WASM_OUT)
30+
cd examples/web-worker && \
31+
timeout 1s bun run src/index.ts

examples/web-worker/.gitignore

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
2+
3+
# Logs
4+
5+
logs
6+
_.log
7+
npm-debug.log_
8+
yarn-debug.log*
9+
yarn-error.log*
10+
lerna-debug.log*
11+
.pnpm-debug.log*
12+
13+
# Caches
14+
15+
.cache
16+
17+
# Diagnostic reports (https://nodejs.org/api/report.html)
18+
19+
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
20+
21+
# Runtime data
22+
23+
pids
24+
_.pid
25+
_.seed
26+
*.pid.lock
27+
28+
# Directory for instrumented libs generated by jscoverage/JSCover
29+
30+
lib-cov
31+
32+
# Coverage directory used by tools like istanbul
33+
34+
coverage
35+
*.lcov
36+
37+
# nyc test coverage
38+
39+
.nyc_output
40+
41+
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
42+
43+
.grunt
44+
45+
# Bower dependency directory (https://bower.io/)
46+
47+
bower_components
48+
49+
# node-waf configuration
50+
51+
.lock-wscript
52+
53+
# Compiled binary addons (https://nodejs.org/api/addons.html)
54+
55+
build/Release
56+
57+
# Dependency directories
58+
59+
node_modules/
60+
jspm_packages/
61+
62+
# Snowpack dependency directory (https://snowpack.dev/)
63+
64+
web_modules/
65+
66+
# TypeScript cache
67+
68+
*.tsbuildinfo
69+
70+
# Optional npm cache directory
71+
72+
.npm
73+
74+
# Optional eslint cache
75+
76+
.eslintcache
77+
78+
# Optional stylelint cache
79+
80+
.stylelintcache
81+
82+
# Microbundle cache
83+
84+
.rpt2_cache/
85+
.rts2_cache_cjs/
86+
.rts2_cache_es/
87+
.rts2_cache_umd/
88+
89+
# Optional REPL history
90+
91+
.node_repl_history
92+
93+
# Output of 'npm pack'
94+
95+
*.tgz
96+
97+
# Yarn Integrity file
98+
99+
.yarn-integrity
100+
101+
# dotenv environment variable files
102+
103+
.env
104+
.env.development.local
105+
.env.test.local
106+
.env.production.local
107+
.env.local
108+
109+
# parcel-bundler cache (https://parceljs.org/)
110+
111+
.parcel-cache
112+
113+
# Next.js build output
114+
115+
.next
116+
out
117+
118+
# Nuxt.js build / generate output
119+
120+
.nuxt
121+
dist
122+
123+
# Gatsby files
124+
125+
# Comment in the public line in if your project uses Gatsby and not Next.js
126+
127+
# https://nextjs.org/blog/next-9-1#public-directory-support
128+
129+
# public
130+
131+
# vuepress build output
132+
133+
.vuepress/dist
134+
135+
# vuepress v2.x temp and cache directory
136+
137+
.temp
138+
139+
# Docusaurus cache and generated files
140+
141+
.docusaurus
142+
143+
# Serverless directories
144+
145+
.serverless/
146+
147+
# FuseBox cache
148+
149+
.fusebox/
150+
151+
# DynamoDB Local files
152+
153+
.dynamodb/
154+
155+
# TernJS port file
156+
157+
.tern-port
158+
159+
# Stores VSCode versions used for testing VSCode extensions
160+
161+
.vscode-test
162+
163+
# yarn v2
164+
165+
.yarn/cache
166+
.yarn/unplugged
167+
.yarn/build-state.yml
168+
.yarn/install-state.gz
169+
.pnp.*
170+
171+
# IntelliJ based IDEs
172+
.idea
173+
174+
# Finder (MacOS) folder config
175+
.DS_Store

examples/web-worker/README.md

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# run-as-web-worker
2+
3+
Demonstrate simple usage of the OPFS functionality within a web worker.
4+
5+
## Setup
6+
7+
To install dependencies:
8+
9+
```bash
10+
bun install
11+
```
12+
13+
To run:
14+
15+
```bash
16+
bun run index.ts
17+
```
18+
19+
This project was created using `bun init` in bun v1.2.1. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.

examples/web-worker/bun.lock

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"lockfileVersion": 1,
3+
"workspaces": {
4+
"": {
5+
"name": "run-as-web-worker",
6+
"devDependencies": {
7+
"@types/bun": "latest",
8+
},
9+
"peerDependencies": {
10+
"typescript": "^5.0.0",
11+
},
12+
},
13+
},
14+
"packages": {
15+
"@types/bun": ["@types/[email protected]", "", { "dependencies": { "bun-types": "1.2.21" } }, "sha512-NiDnvEqmbfQ6dmZ3EeUO577s4P5bf4HCTXtI6trMc6f6RzirY5IrF3aIookuSpyslFzrnvv2lmEWv5HyC1X79A=="],
16+
17+
"@types/node": ["@types/[email protected]", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-3vXmQDXy+woz+gnrTvuvNrPzekOi+Ds0ReMxw0LzBiK3a+1k0kQn9f2NWk+lgD4rJehFUmYy2gMhJ2ZI+7YP9g=="],
18+
19+
"@types/react": ["@types/[email protected]", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-cMoR+FoAf/Jyq6+Df2/Z41jISvGZZ2eTlnsaJRptmZ76Caldwy1odD4xTr/gNV9VLj0AWgg/nmkevIyUfIIq5w=="],
20+
21+
"bun-types": ["[email protected]", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-sa2Tj77Ijc/NTLS0/Odjq/qngmEPZfbfnOERi0KRUYhT9R8M4VBioWVmMWE5GrYbKMc+5lVybXygLdibHaqVqw=="],
22+
23+
"csstype": ["[email protected]", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="],
24+
25+
"typescript": ["[email protected]", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A=="],
26+
27+
"undici-types": ["[email protected]", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="],
28+
}
29+
}

examples/web-worker/package.json

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"name": "web-worker",
3+
"module": "src/index.ts",
4+
"type": "module",
5+
"devDependencies": {
6+
"@types/bun": "latest"
7+
},
8+
"peerDependencies": {
9+
"typescript": "^5.0.0"
10+
}
11+
}

examples/web-worker/src/dlog.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
export const DEBUG_LOGS = true;
2+
export function dlog(...s: any[]) {
3+
if (DEBUG_LOGS) {
4+
console.log(...s)
5+
}
6+
}

examples/web-worker/src/index.ts

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import { Semaphore, type Lock } from "./semaphore";
2+
import type { Message, Response } from "./types";
3+
import { dlog } from "./dlog";
4+
5+
class OpfsUser {
6+
private dispatchSemaphore: Semaphore;
7+
private responseSemaphore: Semaphore;
8+
private responseLock: Lock | null;
9+
private response: Response | null;
10+
private worker: Worker;
11+
12+
constructor(label?: string, workerPath: string = "./src/worker.ts") {
13+
this.dispatchSemaphore = new Semaphore(`${label ?? workerPath} dispatch`, 1);
14+
this.responseSemaphore = new Semaphore(`${label ?? workerPath} response`, 1);
15+
this.response = null;
16+
this.responseLock = null;
17+
18+
this.worker = new Worker(workerPath, {
19+
preload: "../../ts/gen/redb-opfs"
20+
});
21+
this.worker.addEventListener("error", (event) => {
22+
throw new Error(event.message);
23+
});
24+
this.worker.addEventListener("message",
25+
(event) => {
26+
dlog("opfsu: received message from worker");
27+
this.response = event.data;
28+
this.releaseResponseLock();
29+
});
30+
}
31+
32+
private releaseResponseLock() {
33+
if (this.responseLock !== null) {
34+
this.responseLock.release();
35+
}
36+
}
37+
38+
private async dispatch(message: Message): Promise<Response> {
39+
const dispatchLock = await this.dispatchSemaphore.acquire();
40+
41+
try {
42+
// get the response lock. This will be released when we receive a response.
43+
// This means we can efficiently block on a reacquisition attempt.
44+
this.responseLock = await this.responseSemaphore.acquire();
45+
dlog(`opfsu: dispatching message to worker`);
46+
47+
this.worker.postMessage(message);
48+
// now wait until the response is received (and immediately release it)
49+
dlog(`opfsu: waiting for response from worker`);
50+
(await this.responseSemaphore.acquire()).release();
51+
// now we have a response we can return
52+
const response = this.response ?? new Uint8Array();
53+
this.response = null;
54+
return response;
55+
} finally {
56+
dispatchLock.release()
57+
this.releaseResponseLock()
58+
}
59+
}
60+
61+
/** Store some data in Opfs */
62+
async store(offset: bigint, data: Uint8Array): Promise<void> {
63+
dlog(`opfsu: storing ${data.length} bytes at offset ${offset}`);
64+
await this.dispatch({ op: "store", offset, data })
65+
}
66+
67+
/** Retrieve `size` bytes from Opfs */
68+
async load(offset: bigint, size: number): Promise<Uint8Array> {
69+
dlog(`opfsu: loading ${size} bytes at offset ${offset}`);
70+
return await this.dispatch({ op: "load", offset, size })
71+
}
72+
}
73+
74+
// lets cache some squares in opfs so we never have to recompute them
75+
const opfsUser = new OpfsUser("opfs");
76+
77+
const squares = new Uint8Array(16);
78+
for (var i = 0; i < 16; i++) {
79+
squares[i] = i * i;
80+
}
81+
82+
// store the squares
83+
console.log("main: storing data")
84+
await opfsUser.store(BigInt(0), squares);
85+
86+
// what's the square of 4?
87+
console.log("main: retrieving data")
88+
const foursquared = (await opfsUser.load(BigInt(4), 1))[0];
89+
console.log("main: square of 4 is: ", foursquared);

0 commit comments

Comments
 (0)