[WasmFS] Add JSImplBackend (#16209)

This refactors the existing JSFileBackend to use a new model for creating JS
backends. A JSImplBackend is defined, which adds a layer of indirection in
JS, allowing a backend to be implemented almost 100% in JS. That is,
JSImplBackend is a new backend that forwards operations to an implemenation
in JS.

See full docs in js_impl_backend.h. Note that the diffs here may not be as
useful as reading the files in their final state. In particular, look at the how the
JSFileBackend is now implemented in a minimal way in C++ in
js_file_backend.cpp.

This is a step towards it being practical to make backends like Node etc.

None of the actual JSFileBackend logic is changed, just moved around.

Author: kripken

src/library_wasmfs.js

@@ -1,13 +1,9 @@
-var WasmfsLibrary = {
-  $wasmFS$JSMemoryFiles : [],
-  $wasmFS$JSMemoryFreeList: [],
+var WasmFSLibrary = {
   $wasmFS$preloadedFiles: [],
   $wasmFS$preloadedDirs: [],
   $FS__deps: [
     '$wasmFS$preloadedFiles',
     '$wasmFS$preloadedDirs',
-    '$wasmFS$JSMemoryFiles',
-    '$wasmFS$JSMemoryFreeList',
     '$asyncLoad',
 #if !MINIMAL_RUNTIME
     // TODO: when preload-plugins are not used, we do not need this.
@@ -145,64 +141,54 @@ var WasmfsLibrary = {
     var len = lengthBytesUTF8(s) + 1;
     stringToUTF8(s, fileNameBuffer, len);
   },
-  _wasmfs_write_js_file: function(index, buffer, length, offset) {
-    try {
-      if (!wasmFS$JSMemoryFiles[index]) {
-        // Initialize typed array on first write operation.
-        wasmFS$JSMemoryFiles[index] = new Uint8Array(offset + length);
-      }
-
-      if (offset + length > wasmFS$JSMemoryFiles[index].length) {
-        // Resize the typed array if the length of the write buffer exceeds its capacity.
-        var oldContents = wasmFS$JSMemoryFiles[index];
-        var newContents = new Uint8Array(offset + length);
-        newContents.set(oldContents);
-        wasmFS$JSMemoryFiles[index] = newContents;
-      }
-
-      wasmFS$JSMemoryFiles[index].set(HEAPU8.subarray(buffer, buffer + length), offset);
-      return 0;
-    } catch (err) {
-      return {{{ cDefine('EIO') }}};
-    }
+  _wasmfs_get_preloaded_file_size: function(index) {
+    return wasmFS$preloadedFiles[index].fileData.length;
   },
-  _wasmfs_read_js_file: function(index, buffer, length, offset) {
-    try {
-      HEAPU8.set(wasmFS$JSMemoryFiles[index].subarray(offset, offset + length), buffer);
-      return 0;
-    } catch (err) {
-      return {{{ cDefine('EIO') }}};
-    }
+  _wasmfs_copy_preloaded_file_data: function(index, buffer) {
+    HEAPU8.set(wasmFS$preloadedFiles[index].fileData, buffer);
   },
-  _wasmfs_get_js_file_size: function(index) {
-    return wasmFS$JSMemoryFiles[index] ? wasmFS$JSMemoryFiles[index].length : 0;
+
+  // Backend support. wasmFS$backends will contain a mapping of backend IDs to
+  // the JS code that implements them. This is the JS side of the JSImpl class
+  // in C++, together with the js_impl calls defined right after it.
+  $wasmFS$backends: {},
+
+  _wasmfs_jsimpl_alloc_file: function(backend, file) {
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    return wasmFS$backends[backend].alloc_file(file);
   },
-  _wasmfs_create_js_file: function() {
-    // Find a free entry in the $wasmFS$JSMemoryFreeList or append a new entry to
-    // wasmFS$JSMemoryFiles.
-    if (wasmFS$JSMemoryFreeList.length) {
-      // Pop off the top of the free list.
-      var index = wasmFS$JSMemoryFreeList.pop();
-      return index;
-    }
-    wasmFS$JSMemoryFiles.push(null);
-    return wasmFS$JSMemoryFiles.length - 1;
+
+  _wasmfs_jsimpl_free_file: function(backend, file) {
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    return wasmFS$backends[backend].free_file(file);
   },
-  _wasmfs_remove_js_file: function(index) {
-    wasmFS$JSMemoryFiles[index] = null;
-    // Add the index to the free list.
-    wasmFS$JSMemoryFreeList.push(index);
+
+  _wasmfs_jsimpl_write: function(backend, file, buffer, length, offset) {
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    return wasmFS$backends[backend].write(file, buffer, length, offset);
   },
-  _wasmfs_get_preloaded_file_size: function(index) {
-    return wasmFS$preloadedFiles[index].fileData.length;
+
+  _wasmfs_jsimpl_read: function(backend, file, buffer, length, offset) {
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    return wasmFS$backends[backend].read(file, buffer, length, offset);
   },
-  _wasmfs_copy_preloaded_file_data: function(index, buffer) {
-    HEAPU8.set(wasmFS$preloadedFiles[index].fileData, buffer);
+
+  _wasmfs_jsimpl_get_size: function(backend, file) {
+#if ASSERTIONS
+    assert(wasmFS$backends[backend]);
+#endif
+    return wasmFS$backends[backend].getSize(file);
   },
 }
 
-mergeInto(LibraryManager.library, WasmfsLibrary);
+mergeInto(LibraryManager.library, WasmFSLibrary);
 
-if (WASMFS) {
-  DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.push('$FS');
-}
+DEFAULT_LIBRARY_FUNCS_TO_INCLUDE.push('$FS');

src/library_wasmfs_js_file.js

@@ -0,0 +1,53 @@
+mergeInto(LibraryManager.library, {
+  // JSFile backend: Store a file's data in JS. We map File objects in C++ to
+  // entries here that contain typed arrays.
+  $wasmFS$JSMemoryFiles: {},
+
+  _wasmfs_create_js_file_backend_js__deps: [
+    '$wasmFS$backends',
+    '$wasmFS$JSMemoryFiles',
+  ],
+  _wasmfs_create_js_file_backend_js: function(backend) {
+    wasmFS$backends[backend] = {
+      alloc_file: function(file) {
+        // Do nothing: we allocate the typed array lazily, see write()
+      },
+      free_file: function(file) {
+        // Release the memory, as it now has no references to it any more.
+        wasmFS$JSMemoryFiles[file] = undefined;
+      },
+      write: function(file, buffer, length, offset) {
+        try {
+          if (!wasmFS$JSMemoryFiles[file]) {
+            // Initialize typed array on first write operation.
+            wasmFS$JSMemoryFiles[file] = new Uint8Array(offset + length);
+          }
+
+          if (offset + length > wasmFS$JSMemoryFiles[file].length) {
+            // Resize the typed array if the length of the write buffer exceeds its capacity.
+            var oldContents = wasmFS$JSMemoryFiles[file];
+            var newContents = new Uint8Array(offset + length);
+            newContents.set(oldContents);
+            wasmFS$JSMemoryFiles[file] = newContents;
+          }
+
+          wasmFS$JSMemoryFiles[file].set(HEAPU8.subarray(buffer, buffer + length), offset);
+          return 0;
+        } catch (err) {
+          return {{{ cDefine('EIO') }}};
+        }
+      },
+      read: function(file, buffer, length, offset) {
+        try {
+          HEAPU8.set(wasmFS$JSMemoryFiles[file].subarray(offset, offset + length), buffer);
+          return 0;
+        } catch (err) {
+          return {{{ cDefine('EIO') }}};
+        }
+      },
+      getSize: function(file) {
+        return wasmFS$JSMemoryFiles[file] ? wasmFS$JSMemoryFiles[file].length : 0;
+      },
+    };
+  },
+});

src/modules.js

@@ -95,6 +95,7 @@ global.LibraryManager = {
       }
     } else if (WASMFS) {
       libraries.push('library_wasmfs.js');
+      libraries.push('library_wasmfs_js_file.js');
     }
 
     // Additional JS libraries (without AUTO_JS_LIBRARIES, link to these explicitly via -lxxx.js)

system/lib/wasmfs/js_file_backend.cpp

@@ -4,74 +4,24 @@
 // found in the LICENSE file.
 
 // This file defines the JS file backend and JS file of the new file system.
-// Current Status: Work in Progress.
 // See https://github.com/emscripten-core/emscripten/issues/15041.
 
 #include "backend.h"
+#include "js_impl_backend.h"
 #include "wasmfs.h"
 
-using js_index_t = uint32_t;
+// See library_wasmfs_js_file.js
 
 extern "C" {
-int _wasmfs_write_js_file(js_index_t index,
-                          const uint8_t* buffer,
-                          size_t length,
-                          off_t offset);
-int _wasmfs_read_js_file(js_index_t index,
-                         const uint8_t* buffer,
-                         size_t length,
-                         off_t offset);
-int _wasmfs_get_js_file_size(js_index_t index);
-int _wasmfs_create_js_file();
-void _wasmfs_remove_js_file(js_index_t index);
+void _wasmfs_create_js_file_backend_js(wasmfs::backend_t);
 }
 
 namespace wasmfs {
 
-// This class describes a file that lives in JS Memory
-class JSFile : public DataFile {
-  // This index indicates the location of the JS File in the backing JS array.
-  js_index_t index;
-
-  // JSFiles will write from a Wasm Memory buffer into the backing JS array.
-  __wasi_errno_t write(const uint8_t* buf, size_t len, off_t offset) override {
-    return _wasmfs_write_js_file(index, buf, len, offset);
-  }
-
-  // JSFiles will read from the backing JS array into a Wasm Memory buffer.
-  __wasi_errno_t read(uint8_t* buf, size_t len, off_t offset) override {
-    // The caller should have already checked that the offset + len does
-    // not exceed the file's size.
-    assert(offset + len <= getSize());
-    return _wasmfs_read_js_file(index, buf, len, offset);
-  }
-
-  void flush() override {}
-
-  // The size of the JSFile is defined as the length of the backing JS array.
-  size_t getSize() override { return _wasmfs_get_js_file_size(index); }
-
-public:
-  JSFile(mode_t mode, backend_t backend) : DataFile(mode, backend) {
-    // Create a new file in the backing JS array and store its index.
-    index = _wasmfs_create_js_file();
-  }
-
-  // Remove the typed array file contents in the backing JS array.
-  ~JSFile() { _wasmfs_remove_js_file(index); }
-};
-
-class JSFileBackend : public Backend {
-
-public:
-  std::shared_ptr<DataFile> createFile(mode_t mode) override {
-    return std::make_shared<JSFile>(mode, this);
-  }
-};
-
-// This function is exposed to users to instantiate a new JSBackend.
 extern "C" backend_t wasmfs_create_js_file_backend() {
-  return wasmFS.addBackend(std::make_unique<JSFileBackend>());
+  backend_t backend = wasmFS.addBackend(std::make_unique<JSImplBackend>());
+  _wasmfs_create_js_file_backend_js(backend);
+  return backend;
 }
 
 } // namespace wasmfs

system/lib/wasmfs/js_impl_backend.h

@@ -0,0 +1,111 @@
+// Copyright 2022 The Emscripten Authors.  All rights reserved.
+// Emscripten is available under two separate licenses, the MIT license and the
+// University of Illinois/NCSA Open Source License.  Both these licenses can be
+// found in the LICENSE file.
+
+// This file defines the JS file backend and JS file of the new file system.
+// Current Status: Work in Progress.
+// See https://github.com/emscripten-core/emscripten/issues/15041.
+
+#pragma once
+
+#include "backend.h"
+#include "wasmfs.h"
+
+//
+// A JS Impl backend has files that are implemented by JS code. Each backend
+// object in C++ can call JS to define the proper JS code on that side, and we
+// keep a mapping of C++ pointer of backend => JS code in JS. Then, when we do
+// something like a read from a JSImplFile we pass it the backend, find the
+// proper code to run, and run it. This adds a layer of indirection at the JS
+// level that makes it easy to write new backends in 99% JS.
+//
+// Each file operation in the _wasmfs_jsimpl_* APIs that we call from here take
+// the backend and a pointer to this file itself. Those allow the JS to identify
+// both the backend and the particular file. TODO: We could use dense indexes
+// instead of pointers, and use an array instead of a map.
+//
+// To write a new backend in JS, you basically do the following:
+//
+//  1. Add a declaration of the C function to create the backend in the
+//     "backend creation" section of emscripten/wasmfs.h.
+//  2. Add a cpp file for the new backend, and implement the C function from 1,
+//     which should create it on both the C++ (using JSImplBackend) and JS
+//     sides. (By convention, the C function should just call into C++ and JS
+//     which do the interesting work; the C is just a thin wrapper.)
+// 3. Write a new JS library, and add the implementation of the JS method just
+//    mentioned, which should set up the mapping from the C++ backend object's
+//    address to the JS code containing the hooks to read and write etc.
+//
+// For a simple example, see js_file_backend.cpp and library_wasmfs_js_file.js
+//
+
+using js_index_t = uint32_t;
+
+extern "C" {
+// JSImpl API (see below for overview).
+void _wasmfs_jsimpl_alloc_file(js_index_t backend, js_index_t index);
+void _wasmfs_jsimpl_free_file(js_index_t backend, js_index_t index);
+int _wasmfs_jsimpl_write(js_index_t backend,
+                         js_index_t index,
+                         const uint8_t* buffer,
+                         size_t length,
+                         off_t offset);
+int _wasmfs_jsimpl_read(js_index_t backend,
+                        js_index_t index,
+                        const uint8_t* buffer,
+                        size_t length,
+                        off_t offset);
+int _wasmfs_jsimpl_get_size(js_index_t backend, js_index_t index);
+}
+
+namespace wasmfs {
+
+class JSImplFile : public DataFile {
+  js_index_t getBackendIndex() {
+    static_assert(sizeof(backend_t) == sizeof(js_index_t), "TODO: wasm64");
+    return js_index_t(getBackend());
+  }
+
+  js_index_t getFileIndex() {
+    static_assert(sizeof(this) == sizeof(js_index_t), "TODO: wasm64");
+    return js_index_t(this);
+  }
+
+  __wasi_errno_t write(const uint8_t* buf, size_t len, off_t offset) override {
+    return _wasmfs_jsimpl_write(
+      getBackendIndex(), getFileIndex(), buf, len, offset);
+  }
+
+  __wasi_errno_t read(uint8_t* buf, size_t len, off_t offset) override {
+    // The caller should have already checked that the offset + len does
+    // not exceed the file's size.
+    assert(offset + len <= getSize());
+    return _wasmfs_jsimpl_read(
+      getBackendIndex(), getFileIndex(), buf, len, offset);
+  }
+
+  void flush() override {}
+
+  size_t getSize() override {
+    return _wasmfs_jsimpl_get_size(getBackendIndex(), getFileIndex());
+  }
+
+public:
+  JSImplFile(mode_t mode, backend_t backend) : DataFile(mode, backend) {
+    _wasmfs_jsimpl_alloc_file(getBackendIndex(), getFileIndex());
+  }
+
+  ~JSImplFile() {
+    _wasmfs_jsimpl_free_file(getBackendIndex(), getFileIndex());
+  }
+};
+
+class JSImplBackend : public Backend {
+public:
+  std::shared_ptr<DataFile> createFile(mode_t mode) override {
+    return std::make_shared<JSImplFile>(mode, this);
+  }
+};
+
+} // namespace wasmfs