Skip to content

fix: avoid crash on OOM by safe-guarding uninitialized VM pointer #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jtakakura wants to merge 1 commit into from
Closed

fix: avoid crash on OOM by safe-guarding uninitialized VM pointer #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ Format based on [Keep a Changelog](https://keepachangelog.com/).

## [Unreleased]

### Changed
- `invoke`/`invokeInterpreterOnly` now return `error.ModuleNotFullyLoaded` if the underlying VM is uninitialized (e.g., after OOM in `loadLinked`). This is a new error variant in the public API. Embedders matching on specific errors should handle this case. See API docs for details.

## [1.8.0] - 2026-04-21

### Added
Expand Down
2 changes: 1 addition & 1 deletion docs/api-boundary.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ Types and functions listed here are covered by SemVer guarantees.
| `runtime.Module` | Internal decoded module |
| `runtime.Instance` | Internal instance |
| `runtime.VmImpl` | Internal VM implementation |
| `WasmModule.loadLinked` | Two-phase instantiation with shared store |
| `WasmModule.loadLinked` | Two-phase instantiation with shared store. **Note:** If OOM occurs after Phase 1, the returned module will have `vm = null` and `apply_error = error.OutOfMemory`. Such a module is discoverable (e.g., via exports), but unusable: any attempt to invoke will return `error.ModuleNotFullyLoaded`. This is intentional for shared store consistency. |
| `WasmModule.registerExportsTo` | Register to external store |
| `WasmModule.setWitInfo` | Attach WIT metadata |
| `WasmModule.getWitFunc` | Lookup WIT function |
Expand Down
8 changes: 4 additions & 4 deletions src/c_api.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1218,8 +1218,8 @@ test "c_api: config set vm limits" {
defer zwasm_module_delete(module.?);

const mod = &module.?.module.*;
try testing.expectEqual(@as(?u64, 9999), mod.vm.fuel);
try testing.expectEqual(@as(?u64, 65536), mod.vm.max_memory_bytes);
try testing.expectEqual(true, mod.vm.force_interpreter);
try testing.expect(mod.vm.deadline_ns != null);
try testing.expectEqual(@as(?u64, 9999), mod.vm.?.fuel);
try testing.expectEqual(@as(?u64, 65536), mod.vm.?.max_memory_bytes);
try testing.expectEqual(true, mod.vm.?.force_interpreter);
try testing.expect(mod.vm.?.deadline_ns != null);
}
14 changes: 7 additions & 7 deletions src/cli.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,7 @@ fn cmdRun(allocator: Allocator, args: []const []const u8, stdout: *std.Io.Writer
// Enable profiling if requested (note: disables JIT for accurate opcode counting)
var profile = vm_mod.Profile.init();
if (profile_mode) {
module.vm.profile = &profile;
module.vm.?.profile = &profile;
try stderr.print("[note] --profile disables JIT for accurate opcode counting\n", .{});
try stderr.flush();
}
Expand All @@ -475,7 +475,7 @@ fn cmdRun(allocator: Allocator, args: []const []const u8, stdout: *std.Io.Writer
.dump_jit_func = dump_jit_func,
};
if (trace_categories != 0 or dump_regir_func != null or dump_jit_func != null) {
module.vm.trace = &trace_config;
module.vm.?.trace = &trace_config;
}

// Lookup export info for type-aware parsing and validation
Expand Down Expand Up @@ -614,7 +614,7 @@ fn cmdRun(allocator: Allocator, args: []const []const u8, stdout: *std.Io.Writer

// Enable profiling if requested
var wasi_profile = vm_mod.Profile.init();
if (profile_mode) module.vm.profile = &wasi_profile;
if (profile_mode) module.vm.?.profile = &wasi_profile;

// Enable tracing if requested
var wasi_trace_config = trace_mod.TraceConfig{
Expand All @@ -623,7 +623,7 @@ fn cmdRun(allocator: Allocator, args: []const []const u8, stdout: *std.Io.Writer
.dump_jit_func = dump_jit_func,
};
if (trace_categories != 0 or dump_regir_func != null or dump_jit_func != null) {
module.vm.trace = &wasi_trace_config;
module.vm.?.trace = &wasi_trace_config;
}

var no_args = [_]u64{};
Expand Down Expand Up @@ -1267,7 +1267,7 @@ fn cmdBatch(allocator: Allocator, wasm_bytes: []const u8, imports: []const types
.dump_jit_func = dump_jit_func,
};
if (trace_categories != 0 or dump_regir_func != null or dump_jit_func != null) {
module.vm.trace = &batch_trace_config;
module.vm.?.trace = &batch_trace_config;
}

const stdin = std.fs.File.stdin();
Expand Down Expand Up @@ -1399,8 +1399,8 @@ fn cmdBatch(allocator: Allocator, wasm_bytes: []const u8, imports: []const types
}
// Execute start function if present (v2 spec: partial init persists on trap)
if (lm.module.start) |start_idx| {
lm.vm.reset();
lm.vm.invokeByIndex(&lm.instance, start_idx, &.{}, &.{}) catch {
lm.vm.?.reset();
lm.vm.?.invokeByIndex(&lm.instance, start_idx, &.{}, &.{}) catch {
try stdout.print("error start trapped\n", .{});
try stdout.flush();
continue;
Expand Down
192 changes: 132 additions & 60 deletions src/types.zig
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ pub const WasmModule = struct {
/// WIT function signatures (set via setWitInfo).
wit_funcs: []const wit_parser.WitFunc = &[_]wit_parser.WitFunc{},
/// Cached VM instance — reused across invoke() calls to avoid stack reallocation.
vm: *rt.vm_mod.Vm = undefined,
vm: ?*rt.vm_mod.Vm = null,
/// Owned wasm bytes (from WAT conversion). Freed on deinit.
owned_wasm_bytes: ?[]const u8 = null,
/// Persistent fuel budget from Config. Decremented across all invocations.
Expand Down Expand Up @@ -409,9 +409,10 @@ pub const WasmModule = struct {

self.vm = allocator.create(rt.vm_mod.Vm) catch {
// OOM after phase 1 — module stays alive (leak) to keep store valid
self.vm = null;
return .{ .module = self, .apply_error = error.OutOfMemory };
};
self.vm.* = rt.vm_mod.Vm.init(allocator);
self.vm.?.* = rt.vm_mod.Vm.init(allocator);

// Phase 2: apply active element/data segments (may partially fail).
var apply_error: ?anyerror = null;
Expand Down Expand Up @@ -464,28 +465,30 @@ pub const WasmModule = struct {
self.wit_funcs = &[_]wit_parser.WitFunc{};

self.vm = try allocator.create(rt.vm_mod.Vm);
errdefer allocator.destroy(self.vm);
self.vm.* = rt.vm_mod.Vm.init(allocator);
errdefer if (self.vm) |vm| allocator.destroy(vm);
self.vm.?.* = rt.vm_mod.Vm.init(allocator);
self.max_memory_bytes = config.max_memory_bytes;
self.force_interpreter = config.force_interpreter;
self.timeout_ms = config.timeout_ms;
self.fuel = config.fuel;
if (self.fuel) |f| self.vm.fuel = f;
if (self.max_memory_bytes) |mb| self.vm.max_memory_bytes = mb;
if (self.force_interpreter) |fi| self.vm.force_interpreter = fi;
if (self.timeout_ms) |ms| self.vm.setDeadlineTimeoutMs(ms);
const vm = self.vm.?;
if (self.fuel) |f| vm.fuel = f;
if (self.max_memory_bytes) |mb| vm.max_memory_bytes = mb;
if (self.force_interpreter) |fi| vm.force_interpreter = fi;
if (self.timeout_ms) |ms| vm.setDeadlineTimeoutMs(ms);

// Execute start function if present.
// Only apply persistent settings to the VM when explicitly set — a null
// persistent field means "inherit whatever the caller set on self.vm.*".
if (self.module.start) |start_idx| {
self.vm.reset();
if (self.fuel) |f| self.vm.fuel = f;
if (self.max_memory_bytes) |mb| self.vm.max_memory_bytes = mb;
if (self.force_interpreter) |fi| self.vm.force_interpreter = fi;
if (self.timeout_ms) |ms| self.vm.setDeadlineTimeoutMs(ms);
try self.vm.invokeByIndex(&self.instance, start_idx, &.{}, &.{});
self.fuel = self.vm.fuel;
// Use the already-declared vm constant
vm.reset();
if (self.fuel) |f| vm.fuel = f;
if (self.max_memory_bytes) |mb| vm.max_memory_bytes = mb;
if (self.force_interpreter) |fi| vm.force_interpreter = fi;
if (self.timeout_ms) |ms| vm.setDeadlineTimeoutMs(ms);
try vm.invokeByIndex(&self.instance, start_idx, &.{}, &.{});
self.fuel = vm.fuel;
}

return self;
Expand All @@ -501,7 +504,7 @@ pub const WasmModule = struct {
allocator.free(ei.result_types);
}
if (self.export_fns.len > 0) allocator.free(self.export_fns);
allocator.destroy(self.vm);
if (self.vm) |vm| allocator.destroy(vm);
self.instance.deinit();
if (self.wasi_ctx) |*wc| wc.deinit();
self.module.deinit();
Expand All @@ -514,17 +517,27 @@ pub const WasmModule = struct {
/// Args and results are passed as u64 arrays.
///
/// Persistent module settings (`self.fuel` / `self.timeout_ms` /
/// `self.force_interpreter`) override `self.vm.*` only when set (non-null).
/// A null persistent field preserves whatever the caller set directly on
/// `self.vm`, since `self.vm.reset()` does not clear these fields.
/// `self.force_interpreter`) override the underlying VM's corresponding
/// fields only when set (non-null). A null persistent field preserves
/// whatever the caller set directly on the VM instance, since `vm.reset()` does
/// not clear these fields.
///
/// Returns `error.ModuleNotFullyLoaded` if the underlying VM is null (e.g.,
/// after OOM in `loadLinked`).
pub fn invoke(self: *WasmModule, name: []const u8, args: []const u64, results: []u64) !void {
self.vm.reset();
if (self.fuel) |f| self.vm.fuel = f;
if (self.max_memory_bytes) |mb| self.vm.max_memory_bytes = mb;
if (self.force_interpreter) |fi| self.vm.force_interpreter = fi;
if (self.timeout_ms) |ms| self.vm.setDeadlineTimeoutMs(ms);
defer if (self.fuel != null) { self.fuel = self.vm.fuel; };
try self.vm.invoke(&self.instance, name, args, results);
if (self.vm) |vm| {
vm.reset();
if (self.fuel) |f| vm.fuel = f;
if (self.max_memory_bytes) |mb| vm.max_memory_bytes = mb;
if (self.force_interpreter) |fi| vm.force_interpreter = fi;
if (self.timeout_ms) |ms| vm.setDeadlineTimeoutMs(ms);
defer if (self.fuel != null) {
self.fuel = vm.fuel;
};
try vm.invoke(&self.instance, name, args, results);
} else {
return error.ModuleNotFullyLoaded;
}
}

/// Invoke using only the stack-based interpreter, bypassing RegIR and JIT.
Expand All @@ -533,15 +546,21 @@ pub const WasmModule = struct {
/// mode selection — whether set via `module.force_interpreter` or directly
/// on `module.vm.force_interpreter` — survives a diagnostic interpreter call.
pub fn invokeInterpreterOnly(self: *WasmModule, name: []const u8, args: []const u64, results: []u64) !void {
self.vm.reset();
if (self.fuel) |f| self.vm.fuel = f;
if (self.max_memory_bytes) |mb| self.vm.max_memory_bytes = mb;
if (self.timeout_ms) |ms| self.vm.setDeadlineTimeoutMs(ms);
const saved_fi = self.vm.force_interpreter;
self.vm.force_interpreter = true;
defer self.vm.force_interpreter = saved_fi;
defer if (self.fuel != null) { self.fuel = self.vm.fuel; };
try self.vm.invoke(&self.instance, name, args, results);
if (self.vm) |vm| {
vm.reset();
if (self.fuel) |f| vm.fuel = f;
if (self.max_memory_bytes) |mb| vm.max_memory_bytes = mb;
if (self.timeout_ms) |ms| vm.setDeadlineTimeoutMs(ms);
const saved_fi = vm.force_interpreter;
vm.force_interpreter = true;
defer vm.force_interpreter = saved_fi;
defer if (self.fuel != null) {
self.fuel = vm.fuel;
};
try vm.invoke(&self.instance, name, args, results);
} else {
return error.ModuleNotFullyLoaded;
}
}

/// Read bytes from linear memory at the given offset.
Expand Down Expand Up @@ -1186,7 +1205,7 @@ test "nqueens(8) = 92 — regir only (JIT disabled)" {

// Enable profiling to disable JIT (JIT is skipped when profile != null)
var profile = rt.vm_mod.Profile.init();
wasm_mod.vm.profile = &profile;
wasm_mod.vm.?.profile = &profile;

var args = [_]u64{8};
var results = [_]u64{0};
Expand Down Expand Up @@ -1458,42 +1477,42 @@ test "force_interpreter — persistence across invoke and invokeInterpreterOnly"
// Pattern A — legacy direct-vm: caller sets vm.force_interpreter; persistent
// field left null; invoke() must not clobber the caller's choice.
wasm_mod.force_interpreter = null;
wasm_mod.vm.force_interpreter = true;
wasm_mod.vm.?.force_interpreter = true;
try wasm_mod.invoke("f", &.{}, &results);
try testing.expect(wasm_mod.vm.force_interpreter == true);
try testing.expect(wasm_mod.vm.?.force_interpreter == true);
try testing.expectEqual(@as(u64, 42), results[0]);

// invokeInterpreterOnly under Pattern A must restore vm.force_interpreter
// to the caller's value (true), not to the persistent-field default.
try wasm_mod.invokeInterpreterOnly("f", &.{}, &results);
try testing.expect(wasm_mod.vm.force_interpreter == true);
try testing.expect(wasm_mod.vm.?.force_interpreter == true);

// Pattern B — new persistent-field override. vm.force_interpreter gets
// overridden from `module.force_interpreter` on every invoke.
wasm_mod.vm.force_interpreter = false;
wasm_mod.vm.?.force_interpreter = false;
wasm_mod.force_interpreter = true;
try wasm_mod.invoke("f", &.{}, &results);
try testing.expect(wasm_mod.vm.force_interpreter == true);
try testing.expect(wasm_mod.vm.?.force_interpreter == true);

// invokeInterpreterOnly under Pattern B restores to true (the value live on
// vm at entry), so a subsequent regular invoke still sees interpreter mode.
try wasm_mod.invokeInterpreterOnly("f", &.{}, &results);
try testing.expect(wasm_mod.vm.force_interpreter == true);
try testing.expect(wasm_mod.vm.?.force_interpreter == true);
try wasm_mod.invoke("f", &.{}, &results);
try testing.expect(wasm_mod.vm.force_interpreter == true);
try testing.expect(wasm_mod.vm.?.force_interpreter == true);

// Pattern C — persistent field explicitly cleared to false wins over a
// prior vm.force_interpreter = true caller mutation.
wasm_mod.force_interpreter = false;
wasm_mod.vm.force_interpreter = true;
wasm_mod.vm.?.force_interpreter = true;
try wasm_mod.invoke("f", &.{}, &results);
try testing.expect(wasm_mod.vm.force_interpreter == false);
try testing.expect(wasm_mod.vm.?.force_interpreter == false);

// Pattern D — null persistent + false vm stays false.
wasm_mod.force_interpreter = null;
wasm_mod.vm.force_interpreter = false;
wasm_mod.vm.?.force_interpreter = false;
try wasm_mod.invoke("f", &.{}, &results);
try testing.expect(wasm_mod.vm.force_interpreter == false);
try testing.expect(wasm_mod.vm.?.force_interpreter == false);
}

test "fuel and timeout — persistence and caller-set preservation" {
Expand All @@ -1510,24 +1529,24 @@ test "fuel and timeout — persistence and caller-set preservation" {

// Pattern A — caller sets vm.fuel directly; persistent null must not wipe it.
wasm_mod.fuel = null;
wasm_mod.vm.fuel = 1_000;
wasm_mod.vm.?.fuel = 1_000;
try wasm_mod.invoke("f", &.{}, &results);
try testing.expect(wasm_mod.vm.fuel != null);
try testing.expect(wasm_mod.vm.?.fuel != null);

// Pattern B — persistent module.fuel overrides per-invoke.
wasm_mod.fuel = 500;
wasm_mod.vm.fuel = null;
wasm_mod.vm.?.fuel = null;
try wasm_mod.invoke("f", &.{}, &results);
try testing.expect(wasm_mod.vm.fuel != null);
try testing.expect(wasm_mod.vm.fuel.? <= 500);
try testing.expect(wasm_mod.vm.?.fuel != null);
try testing.expect(wasm_mod.vm.?.fuel.? <= 500);

// timeout — caller-set deadline must not be wiped by null persistent.
wasm_mod.timeout_ms = null;
wasm_mod.vm.setDeadlineTimeoutMs(5_000);
const deadline_before = wasm_mod.vm.deadline_ns;
wasm_mod.vm.?.setDeadlineTimeoutMs(5_000);
const deadline_before = wasm_mod.vm.?.deadline_ns;
try wasm_mod.invoke("f", &.{}, &results);
try testing.expect(wasm_mod.vm.deadline_ns != null);
try testing.expectEqual(deadline_before, wasm_mod.vm.deadline_ns);
try testing.expect(wasm_mod.vm.?.deadline_ns != null);
try testing.expectEqual(deadline_before, wasm_mod.vm.?.deadline_ns);
}

test "WasmModule.Config applies VM limits" {
Expand All @@ -1540,8 +1559,61 @@ test "WasmModule.Config applies VM limits" {
});
defer wasm_mod.deinit();

try testing.expectEqual(@as(?u64, 12345), wasm_mod.vm.fuel);
try testing.expectEqual(@as(?u64, 1048576), wasm_mod.vm.max_memory_bytes);
try testing.expectEqual(true, wasm_mod.vm.force_interpreter);
try testing.expect(wasm_mod.vm.deadline_ns != null);
try testing.expectEqual(@as(?u64, 12345), wasm_mod.vm.?.fuel);
try testing.expectEqual(@as(?u64, 1048576), wasm_mod.vm.?.max_memory_bytes);
try testing.expectEqual(true, wasm_mod.vm.?.force_interpreter);
try testing.expect(wasm_mod.vm.?.deadline_ns != null);
}

test "loadLinked OOM after phase 1: invoke returns ModuleNotFullyLoaded" {
const FailingAllocator = std.testing.FailingAllocator;
const wasm_bytes = @embedFile("testdata/01_add.wasm");

var found_store: ?rt.store_mod.Store = null;
defer if (found_store) |*s| s.deinit();

var found_module: ?*WasmModule = null;
var found_apply_error: ?anyerror = null;
defer if (found_module) |m| m.deinit();

// Find a fail index that reaches phase 1 and fails when creating vm.
// Keep this ceiling high so allocator-count changes do not make the test brittle.
const max_fail_index: usize = 65_536;
var fail_index: usize = 0;

while (fail_index < max_fail_index) : (fail_index += 1) {
var store = rt.store_mod.Store.init(testing.allocator);

var failing = FailingAllocator.init(testing.allocator, .{ .fail_index = fail_index });
const linked = WasmModule.loadLinked(failing.allocator(), wasm_bytes, &store) catch |err| {
try testing.expectEqual(error.OutOfMemory, err);
store.deinit();
continue;
};

if (linked.apply_error) |apply_err| {
if (apply_err == error.OutOfMemory and linked.module.vm == null) {
found_module = linked.module;
found_apply_error = apply_err;
found_store = store;
// Patch: reassign allocator to avoid dangling reference (UB)
found_module.?.allocator = testing.allocator;
break;
}
}

linked.module.deinit();
store.deinit();
}

try testing.expect(found_module != null);
try testing.expectEqual(@as(?anyerror, error.OutOfMemory), found_apply_error);
try testing.expect(found_module.?.vm == null);

var args = [_]u64{ 1, 2 };
var results = [_]u64{0};
try testing.expectError(
error.ModuleNotFullyLoaded,
found_module.?.invoke("add", &args, &results),
);
}