Skip to content

Commit

Permalink
[build] Improve tuta-wasm-loader API, allow not generating the fallback
Browse files Browse the repository at this point in the history
  • Loading branch information
@charlag
charlag committed Jul 2, 2024
1 parent f9d7b95 commit acbc430
Show file tree
Hide file tree
Showing 5 changed files with 151 additions and 93 deletions.
18 changes: 6 additions & 12 deletions buildSrc/DevBuild.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,23 +121,17 @@ importScripts("./worker.js")
{
name: "liboqs.wasm",
command: "make -f Makefile_liboqs build",
options: {
workingDir: `${process.cwd()}/libs/webassembly/`,
env: {
WASM: `${process.cwd()}/build/wasm/liboqs.wasm`,
},
optimizationLevel: "O3",
workingDir: `${process.cwd()}/libs/webassembly/`,
env: {
WASM: `${process.cwd()}/build/wasm/liboqs.wasm`,
},
},
{
name: "argon2.wasm",
command: "make -f Makefile_argon2 build",
options: {
workingDir: `${process.cwd()}/libs/webassembly/`,
env: {
WASM: `${process.cwd()}/build/wasm/argon2.wasm`,
},
optimizationLevel: "O3",
workingDir: `${process.cwd()}/libs/webassembly/`,
env: {
WASM: `${process.cwd()}/build/wasm/argon2.wasm`,
},
},
],
Expand Down
21 changes: 9 additions & 12 deletions buildSrc/buildWebapp.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,28 +87,25 @@ export async function buildWebapp({ version, stage, host, measure, minify, proje
}),
rollupWasmLoader({
output: "build/wasm",
fallback: true,
webassemblyLibraries: [
{
name: "liboqs.wasm",
command: "make -f Makefile_liboqs build",
options: {
workingDir: "libs/webassembly/",
env: {
WASM: "../../build/wasm/liboqs.wasm",
},
optimizationLevel: "O3",
workingDir: "libs/webassembly/",
env: {
WASM: "../../build/wasm/liboqs.wasm",
},
optimizationLevel: "O3",
},
{
name: "argon2.wasm",
command: "make -f Makefile_argon2 build",
options: {
workingDir: "libs/webassembly/",
env: {
WASM: "../../build/wasm/argon2.wasm",
},
optimizationLevel: "O3",
workingDir: "libs/webassembly/",
env: {
WASM: "../../build/wasm/argon2.wasm",
},
optimizationLevel: "O3",
},
],
}),
Expand Down
25 changes: 16 additions & 9 deletions packages/tuta-wasm-loader/lib/WasmHandler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,27 +7,32 @@ export interface WasmGeneratorOptions {
env?: Record<string, any>
}

export interface FallbackGeneratorOptions extends WasmGeneratorOptions {
/** Optimization level for the JavaScript fallback */
optimizationLevel: string
export interface FallbackOptions {
/** Tool needed to transpile the wasm file to a JavaScript file */
wasm2jsPath?: string
}

async function generateWasm(command: string, options: WasmGeneratorOptions) {
const runner = util.promisify(exec)
await runner(`${command}`, {
const promise = runner(`${command}`, {
env: {
...process.env,
...options.env,
},
cwd: options.workingDir ?? process.cwd(),
})
promise.child.stdout?.on("data", (data) => {
console.log("wasm build:", data)
})
promise.child.stderr?.on("data", (data) => {
console.log("wasm build:", data)
})
await promise
}

async function generateWasmFallback(wasmFilePath: string, options: FallbackGeneratorOptions) {
async function generateWasmFallback(wasmFilePath: string, options: WasmGeneratorOptions, fallbackOptions: FallbackOptions, optLevel: string | undefined) {
const transpiler = util.promisify(exec)
const result = await transpiler(`${options.wasm2jsPath ?? "wasm2js"} ${wasmFilePath} -${options.optimizationLevel}`, {
const result = await transpiler(`${fallbackOptions.wasm2jsPath ?? "wasm2js"} ${wasmFilePath} -${optLevel ?? "O3"}`, {
env: {
...process.env,
...options.env,
Expand All @@ -36,13 +41,15 @@ async function generateWasmFallback(wasmFilePath: string, options: FallbackGener
return result.stdout
}

async function generateImportCode(wasmFilePath: string) {
const fallback = `wasm-fallback:${path.basename(wasmFilePath)}`
async function generateImportCode(wasmFilePath: string, enableFallback: boolean) {
const fallback = enableFallback
? `await import("wasm-fallback:${path.basename(wasmFilePath)}")`
: `(() => {throw new TypeError("WASM is not supported")})()`
return `
async function loadWasm(options) {
const shouldForceFallback = options && options.forceFallback
if (typeof WebAssembly !== "object" || typeof WebAssembly.instantiate !== "function" || shouldForceFallback) {
return await import("${fallback}").catch((e) => console.log(e))
return ${fallback}
} else if (typeof process !== "undefined") {
const {readFile} = await import("node:fs/promises")
const {dirname, join} = await import("node:path")
Expand Down
161 changes: 113 additions & 48 deletions packages/tuta-wasm-loader/lib/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,90 @@
import { OnLoadResult, OnResolveResult, PluginBuild } from "esbuild"
import path from "node:path"
import { FallbackGeneratorOptions, generateImportCode, generateWasm, generateWasmFallback } from "./WasmHandler.js"
import { FallbackOptions, generateImportCode, generateWasm, generateWasmFallback, WasmGeneratorOptions } from "./WasmHandler.js"
import * as fs from "node:fs"

export interface Library {
export interface Library extends WasmGeneratorOptions {
/**
* Name of the module, how it is imported in JS.
*/
name: string
/**
* Command to run to generate WASM.
*/
command: string
options: FallbackGeneratorOptions
/** Optimization level for the JavaScript fallback */
optimizationLevel?: string
}

export interface LoadOptions {
/** Tool needed to transpile the wasm file to a JavaScript file */
wasm2jsPath?: string
/**
* Top-level plugin options
*/
export interface PluginOptions {
/** Output path for the webassembly files */
output: string
/** List of webassembly files to be compiled and get fallback generated */
webassemblyLibraries: Library[]
/** Whether to generate JS fallback for WASM. If {@code true} or an object it will generate fallback code. */
fallback?: FallbackOptions | boolean
}

function findLib(libraries: Library[], requestedLib: string): Library {
const lib = libraries.find((lib) => lib.name === requestedLib)
/** Private options */
interface NormalizedOptions {
output: string
libraries: Map<string, Library>
fallback: FallbackOptions | null
}

function parseLibraries(libraries: unknown): Map<string, Library> {
if (!Array.isArray(libraries)) {
throw new Error(`Invalid webassemblyLibraries, expected array, got: ${libraries}`)
}
const librariesMap = new Map()
for (const library of libraries) {
if (
typeof library.name !== "string" ||
typeof library.command !== "string" ||
("optimizationLevel" in library && typeof library.optimizationLevel !== "string") ||
(library.workingDir != null && typeof library.workingDir !== "string") ||
(library.env != null && typeof library.env !== "object")
) {
throw new Error(`Invalid library: ${JSON.stringify(library)}`)
}
librariesMap.set(library.name, library)
}
return librariesMap
}

function parseOptions(options: PluginOptions): NormalizedOptions {
let output: string
if (typeof options.output === "string") {
output = options.output
} else {
throw new Error("Invalid output")
}

let fallback: FallbackOptions | null
if (options.fallback === true) {
fallback = { wasm2jsPath: undefined }
} else if (options.fallback === false || options.fallback == null) {
fallback = null
} else if (typeof options.fallback === "object") {
if (options.fallback.wasm2jsPath != null && typeof options.fallback.wasm2jsPath !== "string") {
throw new Error("Invalid wasm2jsPath")
}
fallback = options.fallback
} else {
throw new Error("Invalid fallback")
}
return {
output,
libraries: parseLibraries(options.webassemblyLibraries),
fallback,
}
}

function findLib({ libraries }: NormalizedOptions, requestedLib: string): Library {
const lib = libraries.get(requestedLib)
if (!lib) throw new Error(`${requestedLib} isn't included in build options`)

return lib
Expand All @@ -31,60 +96,63 @@ function createOutputFolderStructure(output: string) {
}
}

function esbuildWasmLoader(options: LoadOptions) {
export function esbuildWasmLoader(options: PluginOptions) {
const normalizedOptions = parseOptions(options)
const fallbackOptions = normalizedOptions.fallback
return {
name: "wasm",
setup(build: PluginBuild) {
createOutputFolderStructure(options.output)
createOutputFolderStructure(normalizedOptions.output)

build.onResolve({ filter: /\.(?:wasm)$/ }, async (args): Promise<OnResolveResult | undefined> => {
build.onResolve({ filter: /\.wasm$/ }, async (args): Promise<OnResolveResult | undefined> => {
if (args.resolveDir === "" && !args.path.includes("wasm-fallback")) return

if (args.path.includes("wasm-fallback")) {
if (fallbackOptions && args.path.includes("wasm-fallback")) {
return {
path: args.path.replaceAll("wasm-fallback:", ""),
namespace: "wasm-fallback",
}
}

const lib = findLib(options.webassemblyLibraries, args.path)
const lib = findLib(normalizedOptions, args.path)

await generateWasm(lib.command, lib.options)
await generateWasm(lib.command, lib)

return {
path: path.join("wasm", args.path),
namespace: "wasm-loader",
}
})

build.onResolve({ filter: /\.(?:wasm-fallback)$/ }, async (args): Promise<OnResolveResult | undefined> => {
if (args.resolveDir === "") return
return {
path: args.path,
namespace: "wasm-fallback",
}
})
if (fallbackOptions) {
build.onResolve({ filter: /\.wasm-fallback$/ }, async (args): Promise<OnResolveResult | undefined> => {
if (args.resolveDir === "") return
return {
path: args.path,
namespace: "wasm-fallback",
}
})
}

build.onLoad({ filter: /.*/, namespace: "wasm-loader" }, async (args): Promise<OnLoadResult> => {
return {
contents: await generateImportCode(args.path),
contents: await generateImportCode(args.path, fallbackOptions != null),
loader: "js",
}
})

build.onLoad({ filter: /.*/, namespace: "wasm-fallback" }, async (args): Promise<OnLoadResult> => {
const buildPath = options.output
const lib = findLib(options.webassemblyLibraries, args.path)
return {
contents: await generateWasmFallback(path.join(buildPath, args.path), {
optimizationLevel: lib.options.optimizationLevel,
env: lib.options.env,
wasm2jsPath: lib.options.wasm2jsPath,
}),
loader: "js",
}
})
build.onResolve({ filter: /node:*/, namespace: "wasm-loader" }, async (args) => {
if (fallbackOptions) {
build.onLoad({ filter: /.*/, namespace: "wasm-fallback" }, async (args): Promise<OnLoadResult> => {
const buildPath = normalizedOptions.output
const lib = findLib(normalizedOptions, args.path)
const contents = await generateWasmFallback(path.join(buildPath, args.path), lib, fallbackOptions, lib.optimizationLevel)
return {
contents: contents,
loader: "js",
}
})
}
build.onResolve({ filter: /node:*/, namespace: "wasm-loader" }, async (_) => {
return {
external: true,
}
Expand All @@ -93,8 +161,9 @@ function esbuildWasmLoader(options: LoadOptions) {
}
}

function rollupWasmLoader(options: LoadOptions & { output: string }) {
export function rollupWasmLoader(options: PluginOptions & { output: string }) {
createOutputFolderStructure(options.output)
const normalizedOptions = parseOptions(options)

return {
name: "wasm",
Expand All @@ -118,23 +187,19 @@ function rollupWasmLoader(options: LoadOptions & { output: string }) {
if (id.startsWith("\0wasm-loader")) {
const wasmLib = id.replaceAll("\0wasm-loader:", "")

const lib = findLib(options.webassemblyLibraries, wasmLib)
await generateWasm(lib.command, lib.options)
const lib = findLib(normalizedOptions, wasmLib)
await generateWasm(lib.command, lib)

return await generateImportCode(path.join("wasm", wasmLib))
} else if (id.startsWith("\0wasm-fallback")) {
return await generateImportCode(path.join("wasm", wasmLib), true)
} else if (id.startsWith("\0wasm-fallback") && normalizedOptions.fallback) {
const wasmLib = id.replaceAll("\0wasm-fallback:", "")
const lib = findLib(options.webassemblyLibraries, wasmLib)
const wasmPath = path.join(options.output, wasmLib)
const lib = findLib(normalizedOptions, wasmLib)
const wasmPath = path.join(normalizedOptions.output, wasmLib)

return await generateWasmFallback(wasmPath, {
optimizationLevel: lib.options.optimizationLevel,
env: lib.options.env,
wasm2jsPath: lib.options.wasm2jsPath,
})
return await generateWasmFallback(wasmPath, lib, normalizedOptions.fallback, lib.optimizationLevel)
}
},
}
}

export { esbuildWasmLoader, rollupWasmLoader, esbuildWasmLoader as default }
export { esbuildWasmLoader as default }
19 changes: 7 additions & 12 deletions test/TestBuilder.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,27 +121,22 @@ export async function runTestBuild({ clean, fast = false }) {
}),
esbuildWasmLoader({
output: `${process.cwd()}/build/wasm`,
fallback: true,
webassemblyLibraries: [
{
name: "liboqs.wasm",
command: "make -f Makefile_liboqs build",
options: {
workingDir: `${process.cwd()}/../libs/webassembly/`,
env: {
WASM: `${process.cwd()}/build/wasm/liboqs.wasm`,
},
optimizationLevel: "O3",
workingDir: `${process.cwd()}/../libs/webassembly/`,
env: {
WASM: `${process.cwd()}/build/wasm/liboqs.wasm`,
},
},
{
name: "argon2.wasm",
command: "make -f Makefile_argon2 build",
options: {
workingDir: `${process.cwd()}/../libs/webassembly/`,
env: {
WASM: `${process.cwd()}/build/wasm/argon2.wasm`,
},
optimizationLevel: "O3",
workingDir: `${process.cwd()}/../libs/webassembly/`,
env: {
WASM: `${process.cwd()}/build/wasm/argon2.wasm`,
},
},
],
Expand Down

0 comments on commit acbc430

Please sign in to comment.