LibWasm: Try really hard to avoid touching the value stack

This commit adds a register allocator, with 8 available "register"
slots.
In testing with various random blobs, this moves anywhere from 30% to
74% of value accesses into predefined slots, and is about a ~20% perf
increase end-to-end.

To actually make this usable, a few structural changes were also made:
- we no longer do one instruction per interpret call
- trapping is an (unlikely) exit condition
- the label and frame stacks are replaced with linked lists with a huge
  node cache size, as we only need to touch the last element and
  push/pop is very frequent.
This commit is contained in:
Ali Mohammad Pur 2025-08-02 20:30:44 +02:00 committed by Ali Mohammad Pur
commit 0e5ecef848
Notes: github-actions[bot] 2025-08-08 10:55:57 +00:00
9 changed files with 3217 additions and 1824 deletions

File diff suppressed because it is too large Load diff

View file

@ -52,54 +52,62 @@ struct BytecodeInterpreter final : public Interpreter {
IndirectCall,
};
template<bool HasCompiledList, bool HasDynamicInsnLimit>
void interpret_impl(Configuration&, Expression const&);
protected:
void interpret_instruction(Configuration&, InstructionPointer&, Instruction const&);
void branch_to_label(Configuration&, LabelIndex);
template<typename ReadT, typename PushT>
void load_and_push(Configuration&, Instruction const&);
bool load_and_push(Configuration&, Instruction const&);
template<typename PopT, typename StoreT>
void pop_and_store(Configuration&, Instruction const&);
bool pop_and_store(Configuration&, Instruction const&);
template<typename StoreT>
bool store_value(Configuration&, Instruction const&, StoreT, size_t address_source);
template<size_t N>
void pop_and_store_lane_n(Configuration&, Instruction const&);
bool pop_and_store_lane_n(Configuration&, Instruction const&);
template<size_t M, size_t N, template<typename> typename SetSign>
void load_and_push_mxn(Configuration&, Instruction const&);
bool load_and_push_mxn(Configuration&, Instruction const&);
template<size_t N>
void load_and_push_lane_n(Configuration&, Instruction const&);
bool load_and_push_lane_n(Configuration&, Instruction const&);
template<size_t N>
void load_and_push_zero_n(Configuration&, Instruction const&);
bool load_and_push_zero_n(Configuration&, Instruction const&);
template<size_t M>
void load_and_push_m_splat(Configuration&, Instruction const&);
bool load_and_push_m_splat(Configuration&, Instruction const&);
template<size_t M, template<size_t> typename NativeType>
void set_top_m_splat(Configuration&, NativeType<M>);
template<size_t M, template<size_t> typename NativeType>
void pop_and_push_m_splat(Configuration&, Instruction const&);
template<typename M, template<typename> typename SetSign, typename VectorType = Native128ByteVectorOf<M, SetSign>>
VectorType pop_vector(Configuration&);
void store_to_memory(Configuration&, Instruction::MemoryArgument const&, ReadonlyBytes data, u32 base);
void call_address(Configuration&, FunctionAddress, CallAddressSource = CallAddressSource::DirectCall);
VectorType pop_vector(Configuration&, size_t source);
bool store_to_memory(Configuration&, Instruction::MemoryArgument const&, ReadonlyBytes data, u32 base);
bool call_address(Configuration&, FunctionAddress, CallAddressSource = CallAddressSource::DirectCall);
template<typename PopTypeLHS, typename PushType, typename Operator, typename PopTypeRHS = PopTypeLHS, typename... Args>
void binary_numeric_operation(Configuration&, Args&&...);
bool binary_numeric_operation(Configuration&, Args&&...);
template<typename PopType, typename PushType, typename Operator, typename... Args>
void unary_operation(Configuration&, Args&&...);
bool unary_operation(Configuration&, Args&&...);
template<typename T>
T read_value(ReadonlyBytes data);
ALWAYS_INLINE bool trap_if_not(bool value, StringView reason)
{
if (!value)
if (!value) [[unlikely]] {
m_trap = Trap { ByteString(reason) };
return !m_trap.has<Empty>();
return true;
}
return false;
}
template<typename... Rest>
ALWAYS_INLINE bool trap_if_not(bool value, StringView reason, CheckedFormatString<StringView, Rest...> format, Rest const&... args)
{
if (!value)
if (!value) [[unlikely]] {
m_trap = Trap { ByteString::formatted(move(format), reason, args...) };
return !m_trap.has<Empty>();
return true;
}
return false;
}
Variant<Trap, Empty> m_trap;

View file

@ -15,7 +15,8 @@ void Configuration::unwind(Badge<CallFrameHandle>, CallFrameHandle const& frame_
{
m_frame_stack.take_last();
m_depth--;
m_ip = frame_handle.ip;
m_ip = frame_handle.ip.value();
m_locals_base = m_frame_stack.is_empty() ? nullptr : m_frame_stack.unchecked_last().locals().data();
}
Result Configuration::call(Interpreter& interpreter, FunctionAddress address, Vector<Value> arguments)

View file

@ -8,6 +8,7 @@
#include <AK/DoublyLinkedList.h>
#include <LibWasm/AbstractMachine/AbstractMachine.h>
#include <LibWasm/Types.h>
namespace Wasm {
@ -23,12 +24,13 @@ public:
Label label(frame.arity(), frame.expression().instructions().size(), m_value_stack.size());
frame.label_index() = m_label_stack.size();
if (auto hint = frame.expression().stack_usage_hint(); hint.has_value())
m_value_stack.ensure_capacity(*hint);
m_value_stack.ensure_capacity(*hint + m_value_stack.size());
m_frame_stack.append(move(frame));
m_label_stack.append(label);
m_locals_base = m_frame_stack.unchecked_last().locals().data();
}
ALWAYS_INLINE auto& frame() const { return m_frame_stack.last(); }
ALWAYS_INLINE auto& frame() { return m_frame_stack.last(); }
ALWAYS_INLINE auto& frame() const { return m_frame_stack.unchecked_last(); }
ALWAYS_INLINE auto& frame() { return m_frame_stack.unchecked_last(); }
ALWAYS_INLINE auto& ip() const { return m_ip; }
ALWAYS_INLINE auto& ip() { return m_ip; }
ALWAYS_INLINE auto& depth() const { return m_depth; }
@ -40,6 +42,10 @@ public:
ALWAYS_INLINE auto& store() const { return m_store; }
ALWAYS_INLINE auto& store() { return m_store; }
ALWAYS_INLINE Value const* raw_locals() const { return m_locals_base; }
ALWAYS_INLINE Value const& local(LocalIndex index) const { return m_locals_base[index.value()]; }
ALWAYS_INLINE Value& local(LocalIndex index) { return m_locals_base[index.value()]; }
struct CallFrameHandle {
explicit CallFrameHandle(Configuration& configuration)
: ip(configuration.ip())
@ -66,14 +72,60 @@ public:
void dump_stack();
ALWAYS_INLINE FLATTEN void push_to_destination(Value value)
{
if (destination == Dispatch::RegisterOrStack::Stack) {
value_stack().unchecked_append(value);
return;
}
regs[to_underlying(destination)] = value;
}
ALWAYS_INLINE FLATTEN Value& source_value(u8 index)
{
// Note: The last source in a dispatch *must* be equal to the destination for this to be valid.
auto const source = sources[index];
if (source == Dispatch::RegisterOrStack::Stack)
return value_stack().unsafe_last();
return regs[to_underlying(source)];
}
ALWAYS_INLINE FLATTEN Value take_source(u8 index)
{
auto const source = sources[index];
if (source == Dispatch::RegisterOrStack::Stack)
return value_stack().unsafe_take_last();
return regs[to_underlying(source)];
}
union {
struct {
Dispatch::RegisterOrStack sources[3];
Dispatch::RegisterOrStack destination;
};
u32 sources_and_destination;
};
Value regs[Dispatch::RegisterOrStack::CountRegisters] = {
Value(0),
Value(0),
Value(0),
Value(0),
Value(0),
Value(0),
Value(0),
Value(0),
};
private:
Store& m_store;
Vector<Value> m_value_stack;
DoublyLinkedList<Label, 32> m_label_stack;
DoublyLinkedList<Frame, 32> m_frame_stack;
Vector<Value, 64, FastLastAccess::Yes> m_value_stack;
DoublyLinkedList<Label, 128> m_label_stack;
DoublyLinkedList<Frame, 128> m_frame_stack;
size_t m_depth { 0 };
InstructionPointer m_ip;
u64 m_ip { 0 };
bool m_should_limit_instruction_count { false };
Value* m_locals_base { nullptr };
};
}

View file

@ -1337,7 +1337,7 @@ VALIDATE_INSTRUCTION(select_typed)
// https://webassembly.github.io/spec/core/bikeshed/#variable-instructions%E2%91%A2
VALIDATE_INSTRUCTION(local_get)
{
auto index = instruction.arguments().get<LocalIndex>();
auto index = instruction.local_index();
TRY(validate(index));
stack.append(m_context.locals[index.value()]);
@ -1346,7 +1346,7 @@ VALIDATE_INSTRUCTION(local_get)
VALIDATE_INSTRUCTION(local_set)
{
auto index = instruction.arguments().get<LocalIndex>();
auto index = instruction.local_index();
TRY(validate(index));
auto& value_type = m_context.locals[index.value()];
@ -1357,7 +1357,7 @@ VALIDATE_INSTRUCTION(local_set)
VALIDATE_INSTRUCTION(local_tee)
{
auto index = instruction.arguments().get<LocalIndex>();
auto index = instruction.local_index();
TRY(validate(index));
auto& value_type = m_context.locals[index.value()];
@ -3699,7 +3699,7 @@ VALIDATE_INSTRUCTION(f64x2_convert_low_i32x4_u)
ErrorOr<void, ValidationError> Validator::validate(Instruction const& instruction, Stack& stack, bool& is_constant)
{
switch (instruction.opcode().value()) {
#define M(name, integer_value) \
#define M(name, integer_value, ...) \
case Instructions::name.value(): \
dbgln_if(WASM_VALIDATOR_DEBUG, "checking {}, stack = {}", #name, stack); \
return validate_instruction<integer_value>(instruction, stack, is_constant);
@ -3738,6 +3738,9 @@ ErrorOr<Validator::ExpressionTypeResult, ValidationError> Validator::validate(Ex
expression.set_stack_usage_hint(stack.max_known_size());
// Now that we're in happy land, try to compile the expression down to a list of labels to help dispatch.
expression.compiled_instructions = try_compile_instructions(expression, m_context.functions.span());
return ExpressionTypeResult { stack.release_vector(), is_constant_expression };
}

View file

@ -14,453 +14,454 @@ AK_TYPEDEF_DISTINCT_ORDERED_ID(u64, OpCode);
namespace Instructions {
// name, opcode, pops, pushes
#define ENUMERATE_SINGLE_BYTE_WASM_OPCODES(M) \
M(unreachable, 0x00) \
M(nop, 0x01) \
M(block, 0x02) \
M(loop, 0x03) \
M(if_, 0x04) \
M(structured_else, 0x05) \
M(structured_end, 0x0b) \
M(br, 0x0c) \
M(br_if, 0x0d) \
M(br_table, 0x0e) \
M(return_, 0x0f) \
M(call, 0x10) \
M(call_indirect, 0x11) \
M(drop, 0x1a) \
M(select, 0x1b) \
M(select_typed, 0x1c) \
M(local_get, 0x20) \
M(local_set, 0x21) \
M(local_tee, 0x22) \
M(global_get, 0x23) \
M(global_set, 0x24) \
M(table_get, 0x25) \
M(table_set, 0x26) \
M(i32_load, 0x28) \
M(i64_load, 0x29) \
M(f32_load, 0x2a) \
M(f64_load, 0x2b) \
M(i32_load8_s, 0x2c) \
M(i32_load8_u, 0x2d) \
M(i32_load16_s, 0x2e) \
M(i32_load16_u, 0x2f) \
M(i64_load8_s, 0x30) \
M(i64_load8_u, 0x31) \
M(i64_load16_s, 0x32) \
M(i64_load16_u, 0x33) \
M(i64_load32_s, 0x34) \
M(i64_load32_u, 0x35) \
M(i32_store, 0x36) \
M(i64_store, 0x37) \
M(f32_store, 0x38) \
M(f64_store, 0x39) \
M(i32_store8, 0x3a) \
M(i32_store16, 0x3b) \
M(i64_store8, 0x3c) \
M(i64_store16, 0x3d) \
M(i64_store32, 0x3e) \
M(memory_size, 0x3f) \
M(memory_grow, 0x40) \
M(i32_const, 0x41) \
M(i64_const, 0x42) \
M(f32_const, 0x43) \
M(f64_const, 0x44) \
M(i32_eqz, 0x45) \
M(i32_eq, 0x46) \
M(i32_ne, 0x47) \
M(i32_lts, 0x48) \
M(i32_ltu, 0x49) \
M(i32_gts, 0x4a) \
M(i32_gtu, 0x4b) \
M(i32_les, 0x4c) \
M(i32_leu, 0x4d) \
M(i32_ges, 0x4e) \
M(i32_geu, 0x4f) \
M(i64_eqz, 0x50) \
M(i64_eq, 0x51) \
M(i64_ne, 0x52) \
M(i64_lts, 0x53) \
M(i64_ltu, 0x54) \
M(i64_gts, 0x55) \
M(i64_gtu, 0x56) \
M(i64_les, 0x57) \
M(i64_leu, 0x58) \
M(i64_ges, 0x59) \
M(i64_geu, 0x5a) \
M(f32_eq, 0x5b) \
M(f32_ne, 0x5c) \
M(f32_lt, 0x5d) \
M(f32_gt, 0x5e) \
M(f32_le, 0x5f) \
M(f32_ge, 0x60) \
M(f64_eq, 0x61) \
M(f64_ne, 0x62) \
M(f64_lt, 0x63) \
M(f64_gt, 0x64) \
M(f64_le, 0x65) \
M(f64_ge, 0x66) \
M(i32_clz, 0x67) \
M(i32_ctz, 0x68) \
M(i32_popcnt, 0x69) \
M(i32_add, 0x6a) \
M(i32_sub, 0x6b) \
M(i32_mul, 0x6c) \
M(i32_divs, 0x6d) \
M(i32_divu, 0x6e) \
M(i32_rems, 0x6f) \
M(i32_remu, 0x70) \
M(i32_and, 0x71) \
M(i32_or, 0x72) \
M(i32_xor, 0x73) \
M(i32_shl, 0x74) \
M(i32_shrs, 0x75) \
M(i32_shru, 0x76) \
M(i32_rotl, 0x77) \
M(i32_rotr, 0x78) \
M(i64_clz, 0x79) \
M(i64_ctz, 0x7a) \
M(i64_popcnt, 0x7b) \
M(i64_add, 0x7c) \
M(i64_sub, 0x7d) \
M(i64_mul, 0x7e) \
M(i64_divs, 0x7f) \
M(i64_divu, 0x80) \
M(i64_rems, 0x81) \
M(i64_remu, 0x82) \
M(i64_and, 0x83) \
M(i64_or, 0x84) \
M(i64_xor, 0x85) \
M(i64_shl, 0x86) \
M(i64_shrs, 0x87) \
M(i64_shru, 0x88) \
M(i64_rotl, 0x89) \
M(i64_rotr, 0x8a) \
M(f32_abs, 0x8b) \
M(f32_neg, 0x8c) \
M(f32_ceil, 0x8d) \
M(f32_floor, 0x8e) \
M(f32_trunc, 0x8f) \
M(f32_nearest, 0x90) \
M(f32_sqrt, 0x91) \
M(f32_add, 0x92) \
M(f32_sub, 0x93) \
M(f32_mul, 0x94) \
M(f32_div, 0x95) \
M(f32_min, 0x96) \
M(f32_max, 0x97) \
M(f32_copysign, 0x98) \
M(f64_abs, 0x99) \
M(f64_neg, 0x9a) \
M(f64_ceil, 0x9b) \
M(f64_floor, 0x9c) \
M(f64_trunc, 0x9d) \
M(f64_nearest, 0x9e) \
M(f64_sqrt, 0x9f) \
M(f64_add, 0xa0) \
M(f64_sub, 0xa1) \
M(f64_mul, 0xa2) \
M(f64_div, 0xa3) \
M(f64_min, 0xa4) \
M(f64_max, 0xa5) \
M(f64_copysign, 0xa6) \
M(i32_wrap_i64, 0xa7) \
M(i32_trunc_sf32, 0xa8) \
M(i32_trunc_uf32, 0xa9) \
M(i32_trunc_sf64, 0xaa) \
M(i32_trunc_uf64, 0xab) \
M(i64_extend_si32, 0xac) \
M(i64_extend_ui32, 0xad) \
M(i64_trunc_sf32, 0xae) \
M(i64_trunc_uf32, 0xaf) \
M(i64_trunc_sf64, 0xb0) \
M(i64_trunc_uf64, 0xb1) \
M(f32_convert_si32, 0xb2) \
M(f32_convert_ui32, 0xb3) \
M(f32_convert_si64, 0xb4) \
M(f32_convert_ui64, 0xb5) \
M(f32_demote_f64, 0xb6) \
M(f64_convert_si32, 0xb7) \
M(f64_convert_ui32, 0xb8) \
M(f64_convert_si64, 0xb9) \
M(f64_convert_ui64, 0xba) \
M(f64_promote_f32, 0xbb) \
M(i32_reinterpret_f32, 0xbc) \
M(i64_reinterpret_f64, 0xbd) \
M(f32_reinterpret_i32, 0xbe) \
M(f64_reinterpret_i64, 0xbf) \
M(i32_extend8_s, 0xc0) \
M(i32_extend16_s, 0xc1) \
M(i64_extend8_s, 0xc2) \
M(i64_extend16_s, 0xc3) \
M(i64_extend32_s, 0xc4) \
M(ref_null, 0xd0) \
M(ref_is_null, 0xd1) \
M(ref_func, 0xd2)
M(unreachable, 0x00, 0, 0) \
M(nop, 0x01, 0, 0) \
M(block, 0x02, 0, -1) \
M(loop, 0x03, 0, -1) \
M(if_, 0x04, 1, -1) \
M(structured_else, 0x05, -1, -1) \
M(structured_end, 0x0b, -1, -1) \
M(br, 0x0c, 0, -1) \
M(br_if, 0x0d, 1, -1) \
M(br_table, 0x0e, 1, -1) \
M(return_, 0x0f, -1, -1) \
M(call, 0x10, -1, -1) \
M(call_indirect, 0x11, -1, -1) \
M(drop, 0x1a, 1, 0) \
M(select, 0x1b, 3, 1) \
M(select_typed, 0x1c, 3, 1) \
M(local_get, 0x20, 0, 1) \
M(local_set, 0x21, 1, 0) \
M(local_tee, 0x22, 1, 1) \
M(global_get, 0x23, 0, 1) \
M(global_set, 0x24, 1, 0) \
M(table_get, 0x25, 1, 1) \
M(table_set, 0x26, 2, 0) \
M(i32_load, 0x28, 1, 1) \
M(i64_load, 0x29, 1, 1) \
M(f32_load, 0x2a, 1, 1) \
M(f64_load, 0x2b, 1, 1) \
M(i32_load8_s, 0x2c, 1, 1) \
M(i32_load8_u, 0x2d, 1, 1) \
M(i32_load16_s, 0x2e, 1, 1) \
M(i32_load16_u, 0x2f, 1, 1) \
M(i64_load8_s, 0x30, 1, 1) \
M(i64_load8_u, 0x31, 1, 1) \
M(i64_load16_s, 0x32, 1, 1) \
M(i64_load16_u, 0x33, 1, 1) \
M(i64_load32_s, 0x34, 1, 1) \
M(i64_load32_u, 0x35, 1, 1) \
M(i32_store, 0x36, 2, 0) \
M(i64_store, 0x37, 2, 0) \
M(f32_store, 0x38, 2, 0) \
M(f64_store, 0x39, 2, 0) \
M(i32_store8, 0x3a, 2, 0) \
M(i32_store16, 0x3b, 2, 0) \
M(i64_store8, 0x3c, 2, 0) \
M(i64_store16, 0x3d, 2, 0) \
M(i64_store32, 0x3e, 2, 0) \
M(memory_size, 0x3f, 0, 1) \
M(memory_grow, 0x40, 1, 1) \
M(i32_const, 0x41, 0, 1) \
M(i64_const, 0x42, 0, 1) \
M(f32_const, 0x43, 0, 1) \
M(f64_const, 0x44, 0, 1) \
M(i32_eqz, 0x45, 1, 1) \
M(i32_eq, 0x46, 2, 1) \
M(i32_ne, 0x47, 2, 1) \
M(i32_lts, 0x48, 2, 1) \
M(i32_ltu, 0x49, 2, 1) \
M(i32_gts, 0x4a, 2, 1) \
M(i32_gtu, 0x4b, 2, 1) \
M(i32_les, 0x4c, 2, 1) \
M(i32_leu, 0x4d, 2, 1) \
M(i32_ges, 0x4e, 2, 1) \
M(i32_geu, 0x4f, 2, 1) \
M(i64_eqz, 0x50, 1, 1) \
M(i64_eq, 0x51, 2, 1) \
M(i64_ne, 0x52, 2, 1) \
M(i64_lts, 0x53, 2, 1) \
M(i64_ltu, 0x54, 2, 1) \
M(i64_gts, 0x55, 2, 1) \
M(i64_gtu, 0x56, 2, 1) \
M(i64_les, 0x57, 2, 1) \
M(i64_leu, 0x58, 2, 1) \
M(i64_ges, 0x59, 2, 1) \
M(i64_geu, 0x5a, 2, 1) \
M(f32_eq, 0x5b, 2, 1) \
M(f32_ne, 0x5c, 2, 1) \
M(f32_lt, 0x5d, 2, 1) \
M(f32_gt, 0x5e, 2, 1) \
M(f32_le, 0x5f, 2, 1) \
M(f32_ge, 0x60, 2, 1) \
M(f64_eq, 0x61, 2, 1) \
M(f64_ne, 0x62, 2, 1) \
M(f64_lt, 0x63, 2, 1) \
M(f64_gt, 0x64, 2, 1) \
M(f64_le, 0x65, 2, 1) \
M(f64_ge, 0x66, 2, 1) \
M(i32_clz, 0x67, 1, 1) \
M(i32_ctz, 0x68, 1, 1) \
M(i32_popcnt, 0x69, 1, 1) \
M(i32_add, 0x6a, 2, 1) \
M(i32_sub, 0x6b, 2, 1) \
M(i32_mul, 0x6c, 2, 1) \
M(i32_divs, 0x6d, 2, 1) \
M(i32_divu, 0x6e, 2, 1) \
M(i32_rems, 0x6f, 2, 1) \
M(i32_remu, 0x70, 2, 1) \
M(i32_and, 0x71, 2, 1) \
M(i32_or, 0x72, 2, 1) \
M(i32_xor, 0x73, 2, 1) \
M(i32_shl, 0x74, 2, 1) \
M(i32_shrs, 0x75, 2, 1) \
M(i32_shru, 0x76, 2, 1) \
M(i32_rotl, 0x77, 2, 1) \
M(i32_rotr, 0x78, 2, 1) \
M(i64_clz, 0x79, 1, 1) \
M(i64_ctz, 0x7a, 1, 1) \
M(i64_popcnt, 0x7b, 1, 1) \
M(i64_add, 0x7c, 2, 1) \
M(i64_sub, 0x7d, 2, 1) \
M(i64_mul, 0x7e, 2, 1) \
M(i64_divs, 0x7f, 2, 1) \
M(i64_divu, 0x80, 2, 1) \
M(i64_rems, 0x81, 2, 1) \
M(i64_remu, 0x82, 2, 1) \
M(i64_and, 0x83, 2, 1) \
M(i64_or, 0x84, 2, 1) \
M(i64_xor, 0x85, 2, 1) \
M(i64_shl, 0x86, 2, 1) \
M(i64_shrs, 0x87, 2, 1) \
M(i64_shru, 0x88, 2, 1) \
M(i64_rotl, 0x89, 2, 1) \
M(i64_rotr, 0x8a, 2, 1) \
M(f32_abs, 0x8b, 1, 1) \
M(f32_neg, 0x8c, 1, 1) \
M(f32_ceil, 0x8d, 1, 1) \
M(f32_floor, 0x8e, 1, 1) \
M(f32_trunc, 0x8f, 1, 1) \
M(f32_nearest, 0x90, 1, 1) \
M(f32_sqrt, 0x91, 1, 1) \
M(f32_add, 0x92, 2, 1) \
M(f32_sub, 0x93, 2, 1) \
M(f32_mul, 0x94, 2, 1) \
M(f32_div, 0x95, 2, 1) \
M(f32_min, 0x96, 2, 1) \
M(f32_max, 0x97, 2, 1) \
M(f32_copysign, 0x98, 2, 1) \
M(f64_abs, 0x99, 1, 1) \
M(f64_neg, 0x9a, 1, 1) \
M(f64_ceil, 0x9b, 1, 1) \
M(f64_floor, 0x9c, 1, 1) \
M(f64_trunc, 0x9d, 1, 1) \
M(f64_nearest, 0x9e, 1, 1) \
M(f64_sqrt, 0x9f, 1, 1) \
M(f64_add, 0xa0, 2, 1) \
M(f64_sub, 0xa1, 2, 1) \
M(f64_mul, 0xa2, 2, 1) \
M(f64_div, 0xa3, 2, 1) \
M(f64_min, 0xa4, 2, 1) \
M(f64_max, 0xa5, 2, 1) \
M(f64_copysign, 0xa6, 2, 1) \
M(i32_wrap_i64, 0xa7, 1, 1) \
M(i32_trunc_sf32, 0xa8, 1, 1) \
M(i32_trunc_uf32, 0xa9, 1, 1) \
M(i32_trunc_sf64, 0xaa, 1, 1) \
M(i32_trunc_uf64, 0xab, 1, 1) \
M(i64_extend_si32, 0xac, 1, 1) \
M(i64_extend_ui32, 0xad, 1, 1) \
M(i64_trunc_sf32, 0xae, 1, 1) \
M(i64_trunc_uf32, 0xaf, 1, 1) \
M(i64_trunc_sf64, 0xb0, 1, 1) \
M(i64_trunc_uf64, 0xb1, 1, 1) \
M(f32_convert_si32, 0xb2, 1, 1) \
M(f32_convert_ui32, 0xb3, 1, 1) \
M(f32_convert_si64, 0xb4, 1, 1) \
M(f32_convert_ui64, 0xb5, 1, 1) \
M(f32_demote_f64, 0xb6, 1, 1) \
M(f64_convert_si32, 0xb7, 1, 1) \
M(f64_convert_ui32, 0xb8, 1, 1) \
M(f64_convert_si64, 0xb9, 1, 1) \
M(f64_convert_ui64, 0xba, 1, 1) \
M(f64_promote_f32, 0xbb, 1, 1) \
M(i32_reinterpret_f32, 0xbc, 1, 1) \
M(i64_reinterpret_f64, 0xbd, 1, 1) \
M(f32_reinterpret_i32, 0xbe, 1, 1) \
M(f64_reinterpret_i64, 0xbf, 1, 1) \
M(i32_extend8_s, 0xc0, 1, 1) \
M(i32_extend16_s, 0xc1, 1, 1) \
M(i64_extend8_s, 0xc2, 1, 1) \
M(i64_extend16_s, 0xc3, 1, 1) \
M(i64_extend32_s, 0xc4, 1, 1) \
M(ref_null, 0xd0, 0, 1) \
M(ref_is_null, 0xd1, 1, 1) \
M(ref_func, 0xd2, 0, 1)
// These are synthetic opcodes, they are _not_ seen in wasm with these values.
#define ENUMERATE_MULTI_BYTE_WASM_OPCODES(M) \
M(i32_trunc_sat_f32_s, 0xfc00000000000000ull) \
M(i32_trunc_sat_f32_u, 0xfc00000000000001ull) \
M(i32_trunc_sat_f64_s, 0xfc00000000000002ull) \
M(i32_trunc_sat_f64_u, 0xfc00000000000003ull) \
M(i64_trunc_sat_f32_s, 0xfc00000000000004ull) \
M(i64_trunc_sat_f32_u, 0xfc00000000000005ull) \
M(i64_trunc_sat_f64_s, 0xfc00000000000006ull) \
M(i64_trunc_sat_f64_u, 0xfc00000000000007ull) \
M(memory_init, 0xfc00000000000008ull) \
M(data_drop, 0xfc00000000000009ull) \
M(memory_copy, 0xfc0000000000000aull) \
M(memory_fill, 0xfc0000000000000bull) \
M(table_init, 0xfc0000000000000cull) \
M(elem_drop, 0xfc0000000000000dull) \
M(table_copy, 0xfc0000000000000eull) \
M(table_grow, 0xfc0000000000000full) \
M(table_size, 0xfc00000000000010ull) \
M(table_fill, 0xfc00000000000011ull) \
M(v128_load, 0xfd00000000000000ull) \
M(v128_load8x8_s, 0xfd00000000000001ull) \
M(v128_load8x8_u, 0xfd00000000000002ull) \
M(v128_load16x4_s, 0xfd00000000000003ull) \
M(v128_load16x4_u, 0xfd00000000000004ull) \
M(v128_load32x2_s, 0xfd00000000000005ull) \
M(v128_load32x2_u, 0xfd00000000000006ull) \
M(v128_load8_splat, 0xfd00000000000007ull) \
M(v128_load16_splat, 0xfd00000000000008ull) \
M(v128_load32_splat, 0xfd00000000000009ull) \
M(v128_load64_splat, 0xfd0000000000000aull) \
M(v128_store, 0xfd0000000000000bull) \
M(v128_const, 0xfd0000000000000cull) \
M(i8x16_shuffle, 0xfd0000000000000dull) \
M(i8x16_swizzle, 0xfd0000000000000eull) \
M(i8x16_splat, 0xfd0000000000000full) \
M(i16x8_splat, 0xfd00000000000010ull) \
M(i32x4_splat, 0xfd00000000000011ull) \
M(i64x2_splat, 0xfd00000000000012ull) \
M(f32x4_splat, 0xfd00000000000013ull) \
M(f64x2_splat, 0xfd00000000000014ull) \
M(i8x16_extract_lane_s, 0xfd00000000000015ull) \
M(i8x16_extract_lane_u, 0xfd00000000000016ull) \
M(i8x16_replace_lane, 0xfd00000000000017ull) \
M(i16x8_extract_lane_s, 0xfd00000000000018ull) \
M(i16x8_extract_lane_u, 0xfd00000000000019ull) \
M(i16x8_replace_lane, 0xfd0000000000001aull) \
M(i32x4_extract_lane, 0xfd0000000000001bull) \
M(i32x4_replace_lane, 0xfd0000000000001cull) \
M(i64x2_extract_lane, 0xfd0000000000001dull) \
M(i64x2_replace_lane, 0xfd0000000000001eull) \
M(f32x4_extract_lane, 0xfd0000000000001full) \
M(f32x4_replace_lane, 0xfd00000000000020ull) \
M(f64x2_extract_lane, 0xfd00000000000021ull) \
M(f64x2_replace_lane, 0xfd00000000000022ull) \
M(i8x16_eq, 0xfd00000000000023ull) \
M(i8x16_ne, 0xfd00000000000024ull) \
M(i8x16_lt_s, 0xfd00000000000025ull) \
M(i8x16_lt_u, 0xfd00000000000026ull) \
M(i8x16_gt_s, 0xfd00000000000027ull) \
M(i8x16_gt_u, 0xfd00000000000028ull) \
M(i8x16_le_s, 0xfd00000000000029ull) \
M(i8x16_le_u, 0xfd0000000000002aull) \
M(i8x16_ge_s, 0xfd0000000000002bull) \
M(i8x16_ge_u, 0xfd0000000000002cull) \
M(i16x8_eq, 0xfd0000000000002dull) \
M(i16x8_ne, 0xfd0000000000002eull) \
M(i16x8_lt_s, 0xfd0000000000002full) \
M(i16x8_lt_u, 0xfd00000000000030ull) \
M(i16x8_gt_s, 0xfd00000000000031ull) \
M(i16x8_gt_u, 0xfd00000000000032ull) \
M(i16x8_le_s, 0xfd00000000000033ull) \
M(i16x8_le_u, 0xfd00000000000034ull) \
M(i16x8_ge_s, 0xfd00000000000035ull) \
M(i16x8_ge_u, 0xfd00000000000036ull) \
M(i32x4_eq, 0xfd00000000000037ull) \
M(i32x4_ne, 0xfd00000000000038ull) \
M(i32x4_lt_s, 0xfd00000000000039ull) \
M(i32x4_lt_u, 0xfd0000000000003aull) \
M(i32x4_gt_s, 0xfd0000000000003bull) \
M(i32x4_gt_u, 0xfd0000000000003cull) \
M(i32x4_le_s, 0xfd0000000000003dull) \
M(i32x4_le_u, 0xfd0000000000003eull) \
M(i32x4_ge_s, 0xfd0000000000003full) \
M(i32x4_ge_u, 0xfd00000000000040ull) \
M(f32x4_eq, 0xfd00000000000041ull) \
M(f32x4_ne, 0xfd00000000000042ull) \
M(f32x4_lt, 0xfd00000000000043ull) \
M(f32x4_gt, 0xfd00000000000044ull) \
M(f32x4_le, 0xfd00000000000045ull) \
M(f32x4_ge, 0xfd00000000000046ull) \
M(f64x2_eq, 0xfd00000000000047ull) \
M(f64x2_ne, 0xfd00000000000048ull) \
M(f64x2_lt, 0xfd00000000000049ull) \
M(f64x2_gt, 0xfd0000000000004aull) \
M(f64x2_le, 0xfd0000000000004bull) \
M(f64x2_ge, 0xfd0000000000004cull) \
M(v128_not, 0xfd0000000000004dull) \
M(v128_and, 0xfd0000000000004eull) \
M(v128_andnot, 0xfd0000000000004full) \
M(v128_or, 0xfd00000000000050ull) \
M(v128_xor, 0xfd00000000000051ull) \
M(v128_bitselect, 0xfd00000000000052ull) \
M(v128_any_true, 0xfd00000000000053ull) \
M(v128_load8_lane, 0xfd00000000000054ull) \
M(v128_load16_lane, 0xfd00000000000055ull) \
M(v128_load32_lane, 0xfd00000000000056ull) \
M(v128_load64_lane, 0xfd00000000000057ull) \
M(v128_store8_lane, 0xfd00000000000058ull) \
M(v128_store16_lane, 0xfd00000000000059ull) \
M(v128_store32_lane, 0xfd0000000000005aull) \
M(v128_store64_lane, 0xfd0000000000005bull) \
M(v128_load32_zero, 0xfd0000000000005cull) \
M(v128_load64_zero, 0xfd0000000000005dull) \
M(f32x4_demote_f64x2_zero, 0xfd0000000000005eull) \
M(f64x2_promote_low_f32x4, 0xfd0000000000005full) \
M(i8x16_abs, 0xfd00000000000060ull) \
M(i8x16_neg, 0xfd00000000000061ull) \
M(i8x16_popcnt, 0xfd00000000000062ull) \
M(i8x16_all_true, 0xfd00000000000063ull) \
M(i8x16_bitmask, 0xfd00000000000064ull) \
M(i8x16_narrow_i16x8_s, 0xfd00000000000065ull) \
M(i8x16_narrow_i16x8_u, 0xfd00000000000066ull) \
M(f32x4_ceil, 0xfd00000000000067ull) \
M(f32x4_floor, 0xfd00000000000068ull) \
M(f32x4_trunc, 0xfd00000000000069ull) \
M(f32x4_nearest, 0xfd0000000000006aull) \
M(i8x16_shl, 0xfd0000000000006bull) \
M(i8x16_shr_s, 0xfd0000000000006cull) \
M(i8x16_shr_u, 0xfd0000000000006dull) \
M(i8x16_add, 0xfd0000000000006eull) \
M(i8x16_add_sat_s, 0xfd0000000000006full) \
M(i8x16_add_sat_u, 0xfd00000000000070ull) \
M(i8x16_sub, 0xfd00000000000071ull) \
M(i8x16_sub_sat_s, 0xfd00000000000072ull) \
M(i8x16_sub_sat_u, 0xfd00000000000073ull) \
M(f64x2_ceil, 0xfd00000000000074ull) \
M(f64x2_floor, 0xfd00000000000075ull) \
M(i8x16_min_s, 0xfd00000000000076ull) \
M(i8x16_min_u, 0xfd00000000000077ull) \
M(i8x16_max_s, 0xfd00000000000078ull) \
M(i8x16_max_u, 0xfd00000000000079ull) \
M(f64x2_trunc, 0xfd0000000000007aull) \
M(i8x16_avgr_u, 0xfd0000000000007bull) \
M(i16x8_extadd_pairwise_i8x16_s, 0xfd0000000000007cull) \
M(i16x8_extadd_pairwise_i8x16_u, 0xfd0000000000007dull) \
M(i32x4_extadd_pairwise_i16x8_s, 0xfd0000000000007eull) \
M(i32x4_extadd_pairwise_i16x8_u, 0xfd0000000000007full) \
M(i16x8_abs, 0xfd00000000000080ull) \
M(i16x8_neg, 0xfd00000000000081ull) \
M(i16x8_q15mulr_sat_s, 0xfd00000000000082ull) \
M(i16x8_all_true, 0xfd00000000000083ull) \
M(i16x8_bitmask, 0xfd00000000000084ull) \
M(i16x8_narrow_i32x4_s, 0xfd00000000000085ull) \
M(i16x8_narrow_i32x4_u, 0xfd00000000000086ull) \
M(i16x8_extend_low_i8x16_s, 0xfd00000000000087ull) \
M(i16x8_extend_high_i8x16_s, 0xfd00000000000088ull) \
M(i16x8_extend_low_i8x16_u, 0xfd00000000000089ull) \
M(i16x8_extend_high_i8x16_u, 0xfd0000000000008aull) \
M(i16x8_shl, 0xfd0000000000008bull) \
M(i16x8_shr_s, 0xfd0000000000008cull) \
M(i16x8_shr_u, 0xfd0000000000008dull) \
M(i16x8_add, 0xfd0000000000008eull) \
M(i16x8_add_sat_s, 0xfd0000000000008full) \
M(i16x8_add_sat_u, 0xfd00000000000090ull) \
M(i16x8_sub, 0xfd00000000000091ull) \
M(i16x8_sub_sat_s, 0xfd00000000000092ull) \
M(i16x8_sub_sat_u, 0xfd00000000000093ull) \
M(f64x2_nearest, 0xfd00000000000094ull) \
M(i16x8_mul, 0xfd00000000000095ull) \
M(i16x8_min_s, 0xfd00000000000096ull) \
M(i16x8_min_u, 0xfd00000000000097ull) \
M(i16x8_max_s, 0xfd00000000000098ull) \
M(i16x8_max_u, 0xfd00000000000099ull) \
M(i16x8_avgr_u, 0xfd0000000000009bull) \
M(i16x8_extmul_low_i8x16_s, 0xfd0000000000009cull) \
M(i16x8_extmul_high_i8x16_s, 0xfd0000000000009dull) \
M(i16x8_extmul_low_i8x16_u, 0xfd0000000000009eull) \
M(i16x8_extmul_high_i8x16_u, 0xfd0000000000009full) \
M(i32x4_abs, 0xfd000000000000a0ull) \
M(i32x4_neg, 0xfd000000000000a1ull) \
M(i32x4_all_true, 0xfd000000000000a3ull) \
M(i32x4_bitmask, 0xfd000000000000a4ull) \
M(i32x4_extend_low_i16x8_s, 0xfd000000000000a7ull) \
M(i32x4_extend_high_i16x8_s, 0xfd000000000000a8ull) \
M(i32x4_extend_low_i16x8_u, 0xfd000000000000a9ull) \
M(i32x4_extend_high_i16x8_u, 0xfd000000000000aaull) \
M(i32x4_shl, 0xfd000000000000abull) \
M(i32x4_shr_s, 0xfd000000000000acull) \
M(i32x4_shr_u, 0xfd000000000000adull) \
M(i32x4_add, 0xfd000000000000aeull) \
M(i32x4_sub, 0xfd000000000000b1ull) \
M(i32x4_mul, 0xfd000000000000b5ull) \
M(i32x4_min_s, 0xfd000000000000b6ull) \
M(i32x4_min_u, 0xfd000000000000b7ull) \
M(i32x4_max_s, 0xfd000000000000b8ull) \
M(i32x4_max_u, 0xfd000000000000b9ull) \
M(i32x4_dot_i16x8_s, 0xfd000000000000baull) \
M(i32x4_extmul_low_i16x8_s, 0xfd000000000000bcull) \
M(i32x4_extmul_high_i16x8_s, 0xfd000000000000bdull) \
M(i32x4_extmul_low_i16x8_u, 0xfd000000000000beull) \
M(i32x4_extmul_high_i16x8_u, 0xfd000000000000bfull) \
M(i64x2_abs, 0xfd000000000000c0ull) \
M(i64x2_neg, 0xfd000000000000c1ull) \
M(i64x2_all_true, 0xfd000000000000c3ull) \
M(i64x2_bitmask, 0xfd000000000000c4ull) \
M(i64x2_extend_low_i32x4_s, 0xfd000000000000c7ull) \
M(i64x2_extend_high_i32x4_s, 0xfd000000000000c8ull) \
M(i64x2_extend_low_i32x4_u, 0xfd000000000000c9ull) \
M(i64x2_extend_high_i32x4_u, 0xfd000000000000caull) \
M(i64x2_shl, 0xfd000000000000cbull) \
M(i64x2_shr_s, 0xfd000000000000ccull) \
M(i64x2_shr_u, 0xfd000000000000cdull) \
M(i64x2_add, 0xfd000000000000ceull) \
M(i64x2_sub, 0xfd000000000000d1ull) \
M(i64x2_mul, 0xfd000000000000d5ull) \
M(i64x2_eq, 0xfd000000000000d6ull) \
M(i64x2_ne, 0xfd000000000000d7ull) \
M(i64x2_lt_s, 0xfd000000000000d8ull) \
M(i64x2_gt_s, 0xfd000000000000d9ull) \
M(i64x2_le_s, 0xfd000000000000daull) \
M(i64x2_ge_s, 0xfd000000000000dbull) \
M(i64x2_extmul_low_i32x4_s, 0xfd000000000000dcull) \
M(i64x2_extmul_high_i32x4_s, 0xfd000000000000ddull) \
M(i64x2_extmul_low_i32x4_u, 0xfd000000000000deull) \
M(i64x2_extmul_high_i32x4_u, 0xfd000000000000dfull) \
M(f32x4_abs, 0xfd000000000000e0ull) \
M(f32x4_neg, 0xfd000000000000e1ull) \
M(f32x4_sqrt, 0xfd000000000000e3ull) \
M(f32x4_add, 0xfd000000000000e4ull) \
M(f32x4_sub, 0xfd000000000000e5ull) \
M(f32x4_mul, 0xfd000000000000e6ull) \
M(f32x4_div, 0xfd000000000000e7ull) \
M(f32x4_min, 0xfd000000000000e8ull) \
M(f32x4_max, 0xfd000000000000e9ull) \
M(f32x4_pmin, 0xfd000000000000eaull) \
M(f32x4_pmax, 0xfd000000000000ebull) \
M(f64x2_abs, 0xfd000000000000ecull) \
M(f64x2_neg, 0xfd000000000000edull) \
M(f64x2_sqrt, 0xfd000000000000efull) \
M(f64x2_add, 0xfd000000000000f0ull) \
M(f64x2_sub, 0xfd000000000000f1ull) \
M(f64x2_mul, 0xfd000000000000f2ull) \
M(f64x2_div, 0xfd000000000000f3ull) \
M(f64x2_min, 0xfd000000000000f4ull) \
M(f64x2_max, 0xfd000000000000f5ull) \
M(f64x2_pmin, 0xfd000000000000f6ull) \
M(f64x2_pmax, 0xfd000000000000f7ull) \
M(i32x4_trunc_sat_f32x4_s, 0xfd000000000000f8ull) \
M(i32x4_trunc_sat_f32x4_u, 0xfd000000000000f9ull) \
M(f32x4_convert_i32x4_s, 0xfd000000000000faull) \
M(f32x4_convert_i32x4_u, 0xfd000000000000fbull) \
M(i32x4_trunc_sat_f64x2_s_zero, 0xfd000000000000fcull) \
M(i32x4_trunc_sat_f64x2_u_zero, 0xfd000000000000fdull) \
M(f64x2_convert_low_i32x4_s, 0xfd000000000000feull) \
M(f64x2_convert_low_i32x4_u, 0xfd000000000000ffull)
#define ENUMERATE_MULTI_BYTE_WASM_OPCODES(M) \
M(i32_trunc_sat_f32_s, 0xfc00000000000000ull, 1, 1) \
M(i32_trunc_sat_f32_u, 0xfc00000000000001ull, 1, 1) \
M(i32_trunc_sat_f64_s, 0xfc00000000000002ull, 1, 1) \
M(i32_trunc_sat_f64_u, 0xfc00000000000003ull, 1, 1) \
M(i64_trunc_sat_f32_s, 0xfc00000000000004ull, 1, 1) \
M(i64_trunc_sat_f32_u, 0xfc00000000000005ull, 1, 1) \
M(i64_trunc_sat_f64_s, 0xfc00000000000006ull, 1, 1) \
M(i64_trunc_sat_f64_u, 0xfc00000000000007ull, 1, 1) \
M(memory_init, 0xfc00000000000008ull, 3, 0) \
M(data_drop, 0xfc00000000000009ull, 0, 0) \
M(memory_copy, 0xfc0000000000000aull, 3, 0) \
M(memory_fill, 0xfc0000000000000bull, 3, 0) \
M(table_init, 0xfc0000000000000cull, 3, 0) \
M(elem_drop, 0xfc0000000000000dull, 0, 0) \
M(table_copy, 0xfc0000000000000eull, 3, 0) \
M(table_grow, 0xfc0000000000000full, 2, 1) \
M(table_size, 0xfc00000000000010ull, 0, 1) \
M(table_fill, 0xfc00000000000011ull, 3, 0) \
M(v128_load, 0xfd00000000000000ull, 1, 1) \
M(v128_load8x8_s, 0xfd00000000000001ull, 1, 1) \
M(v128_load8x8_u, 0xfd00000000000002ull, 1, 1) \
M(v128_load16x4_s, 0xfd00000000000003ull, 1, 1) \
M(v128_load16x4_u, 0xfd00000000000004ull, 1, 1) \
M(v128_load32x2_s, 0xfd00000000000005ull, 1, 1) \
M(v128_load32x2_u, 0xfd00000000000006ull, 1, 1) \
M(v128_load8_splat, 0xfd00000000000007ull, 1, 1) \
M(v128_load16_splat, 0xfd00000000000008ull, 1, 1) \
M(v128_load32_splat, 0xfd00000000000009ull, 1, 1) \
M(v128_load64_splat, 0xfd0000000000000aull, 1, 1) \
M(v128_store, 0xfd0000000000000bull, 2, 0) \
M(v128_const, 0xfd0000000000000cull, 0, 1) \
M(i8x16_shuffle, 0xfd0000000000000dull, 2, 1) \
M(i8x16_swizzle, 0xfd0000000000000eull, 2, 1) \
M(i8x16_splat, 0xfd0000000000000full, 1, 1) \
M(i16x8_splat, 0xfd00000000000010ull, 1, 1) \
M(i32x4_splat, 0xfd00000000000011ull, 1, 1) \
M(i64x2_splat, 0xfd00000000000012ull, 1, 1) \
M(f32x4_splat, 0xfd00000000000013ull, 1, 1) \
M(f64x2_splat, 0xfd00000000000014ull, 1, 1) \
M(i8x16_extract_lane_s, 0xfd00000000000015ull, 1, 1) \
M(i8x16_extract_lane_u, 0xfd00000000000016ull, 1, 1) \
M(i8x16_replace_lane, 0xfd00000000000017ull, 2, 1) \
M(i16x8_extract_lane_s, 0xfd00000000000018ull, 1, 1) \
M(i16x8_extract_lane_u, 0xfd00000000000019ull, 1, 1) \
M(i16x8_replace_lane, 0xfd0000000000001aull, 2, -1) \
M(i32x4_extract_lane, 0xfd0000000000001bull, 1, 1) \
M(i32x4_replace_lane, 0xfd0000000000001cull, 2, 1) \
M(i64x2_extract_lane, 0xfd0000000000001dull, 1, 1) \
M(i64x2_replace_lane, 0xfd0000000000001eull, 2, 1) \
M(f32x4_extract_lane, 0xfd0000000000001full, 1, 1) \
M(f32x4_replace_lane, 0xfd00000000000020ull, 2, 1) \
M(f64x2_extract_lane, 0xfd00000000000021ull, 1, 1) \
M(f64x2_replace_lane, 0xfd00000000000022ull, 2, 1) \
M(i8x16_eq, 0xfd00000000000023ull, 2, 1) \
M(i8x16_ne, 0xfd00000000000024ull, 2, 1) \
M(i8x16_lt_s, 0xfd00000000000025ull, 2, 1) \
M(i8x16_lt_u, 0xfd00000000000026ull, 2, 1) \
M(i8x16_gt_s, 0xfd00000000000027ull, 2, 1) \
M(i8x16_gt_u, 0xfd00000000000028ull, 2, 1) \
M(i8x16_le_s, 0xfd00000000000029ull, 2, 1) \
M(i8x16_le_u, 0xfd0000000000002aull, 2, 1) \
M(i8x16_ge_s, 0xfd0000000000002bull, 2, 1) \
M(i8x16_ge_u, 0xfd0000000000002cull, 2, 1) \
M(i16x8_eq, 0xfd0000000000002dull, 2, 1) \
M(i16x8_ne, 0xfd0000000000002eull, 2, 1) \
M(i16x8_lt_s, 0xfd0000000000002full, 2, 1) \
M(i16x8_lt_u, 0xfd00000000000030ull, 2, 1) \
M(i16x8_gt_s, 0xfd00000000000031ull, 2, 1) \
M(i16x8_gt_u, 0xfd00000000000032ull, 2, 1) \
M(i16x8_le_s, 0xfd00000000000033ull, 2, 1) \
M(i16x8_le_u, 0xfd00000000000034ull, 2, 1) \
M(i16x8_ge_s, 0xfd00000000000035ull, 2, 1) \
M(i16x8_ge_u, 0xfd00000000000036ull, 2, 1) \
M(i32x4_eq, 0xfd00000000000037ull, 2, 1) \
M(i32x4_ne, 0xfd00000000000038ull, 2, 1) \
M(i32x4_lt_s, 0xfd00000000000039ull, 2, 1) \
M(i32x4_lt_u, 0xfd0000000000003aull, 2, 1) \
M(i32x4_gt_s, 0xfd0000000000003bull, 2, 1) \
M(i32x4_gt_u, 0xfd0000000000003cull, 2, 1) \
M(i32x4_le_s, 0xfd0000000000003dull, 2, 1) \
M(i32x4_le_u, 0xfd0000000000003eull, 2, 1) \
M(i32x4_ge_s, 0xfd0000000000003full, 2, 1) \
M(i32x4_ge_u, 0xfd00000000000040ull, 2, 1) \
M(f32x4_eq, 0xfd00000000000041ull, 2, 1) \
M(f32x4_ne, 0xfd00000000000042ull, 2, 1) \
M(f32x4_lt, 0xfd00000000000043ull, 2, 1) \
M(f32x4_gt, 0xfd00000000000044ull, 2, 1) \
M(f32x4_le, 0xfd00000000000045ull, 2, 1) \
M(f32x4_ge, 0xfd00000000000046ull, 2, 1) \
M(f64x2_eq, 0xfd00000000000047ull, 2, 1) \
M(f64x2_ne, 0xfd00000000000048ull, 2, 1) \
M(f64x2_lt, 0xfd00000000000049ull, 2, 1) \
M(f64x2_gt, 0xfd0000000000004aull, 2, 1) \
M(f64x2_le, 0xfd0000000000004bull, 2, 1) \
M(f64x2_ge, 0xfd0000000000004cull, 2, 1) \
M(v128_not, 0xfd0000000000004dull, 1, 1) \
M(v128_and, 0xfd0000000000004eull, 2, 1) \
M(v128_andnot, 0xfd0000000000004full, 2, 1) \
M(v128_or, 0xfd00000000000050ull, 2, 1) \
M(v128_xor, 0xfd00000000000051ull, 2, 1) \
M(v128_bitselect, 0xfd00000000000052ull, 3, 1) \
M(v128_any_true, 0xfd00000000000053ull, 1, 1) \
M(v128_load8_lane, 0xfd00000000000054ull, 2, 1) \
M(v128_load16_lane, 0xfd00000000000055ull, 2, 1) \
M(v128_load32_lane, 0xfd00000000000056ull, 2, 1) \
M(v128_load64_lane, 0xfd00000000000057ull, 2, 1) \
M(v128_store8_lane, 0xfd00000000000058ull, 2, 0) \
M(v128_store16_lane, 0xfd00000000000059ull, 2, 0) \
M(v128_store32_lane, 0xfd0000000000005aull, 2, 0) \
M(v128_store64_lane, 0xfd0000000000005bull, 2, 0) \
M(v128_load32_zero, 0xfd0000000000005cull, 1, 1) \
M(v128_load64_zero, 0xfd0000000000005dull, 1, 1) \
M(f32x4_demote_f64x2_zero, 0xfd0000000000005eull, 1, 1) \
M(f64x2_promote_low_f32x4, 0xfd0000000000005full, 1, 1) \
M(i8x16_abs, 0xfd00000000000060ull, 1, 1) \
M(i8x16_neg, 0xfd00000000000061ull, 1, 1) \
M(i8x16_popcnt, 0xfd00000000000062ull, 1, 1) \
M(i8x16_all_true, 0xfd00000000000063ull, 1, 1) \
M(i8x16_bitmask, 0xfd00000000000064ull, 1, 1) \
M(i8x16_narrow_i16x8_s, 0xfd00000000000065ull, 2, 1) \
M(i8x16_narrow_i16x8_u, 0xfd00000000000066ull, 2, 1) \
M(f32x4_ceil, 0xfd00000000000067ull, 1, 1) \
M(f32x4_floor, 0xfd00000000000068ull, 1, 1) \
M(f32x4_trunc, 0xfd00000000000069ull, 1, 1) \
M(f32x4_nearest, 0xfd0000000000006aull, 1, 1) \
M(i8x16_shl, 0xfd0000000000006bull, 2, 1) \
M(i8x16_shr_s, 0xfd0000000000006cull, 2, 1) \
M(i8x16_shr_u, 0xfd0000000000006dull, 2, 1) \
M(i8x16_add, 0xfd0000000000006eull, 2, 1) \
M(i8x16_add_sat_s, 0xfd0000000000006full, 2, 1) \
M(i8x16_add_sat_u, 0xfd00000000000070ull, 2, 1) \
M(i8x16_sub, 0xfd00000000000071ull, 2, 1) \
M(i8x16_sub_sat_s, 0xfd00000000000072ull, 2, 1) \
M(i8x16_sub_sat_u, 0xfd00000000000073ull, 2, 1) \
M(f64x2_ceil, 0xfd00000000000074ull, 1, 1) \
M(f64x2_floor, 0xfd00000000000075ull, 1, 1) \
M(i8x16_min_s, 0xfd00000000000076ull, 2, 1) \
M(i8x16_min_u, 0xfd00000000000077ull, 2, 1) \
M(i8x16_max_s, 0xfd00000000000078ull, 2, 1) \
M(i8x16_max_u, 0xfd00000000000079ull, 2, 1) \
M(f64x2_trunc, 0xfd0000000000007aull, 1, 1) \
M(i8x16_avgr_u, 0xfd0000000000007bull, 2, 1) \
M(i16x8_extadd_pairwise_i8x16_s, 0xfd0000000000007cull, 1, 1) \
M(i16x8_extadd_pairwise_i8x16_u, 0xfd0000000000007dull, 1, 1) \
M(i32x4_extadd_pairwise_i16x8_s, 0xfd0000000000007eull, 1, 1) \
M(i32x4_extadd_pairwise_i16x8_u, 0xfd0000000000007full, 1, 1) \
M(i16x8_abs, 0xfd00000000000080ull, 1, 1) \
M(i16x8_neg, 0xfd00000000000081ull, 1, 1) \
M(i16x8_q15mulr_sat_s, 0xfd00000000000082ull, 2, 1) \
M(i16x8_all_true, 0xfd00000000000083ull, 1, 1) \
M(i16x8_bitmask, 0xfd00000000000084ull, 1, 1) \
M(i16x8_narrow_i32x4_s, 0xfd00000000000085ull, 2, 1) \
M(i16x8_narrow_i32x4_u, 0xfd00000000000086ull, 2, 1) \
M(i16x8_extend_low_i8x16_s, 0xfd00000000000087ull, 1, 1) \
M(i16x8_extend_high_i8x16_s, 0xfd00000000000088ull, 1, 1) \
M(i16x8_extend_low_i8x16_u, 0xfd00000000000089ull, 1, 1) \
M(i16x8_extend_high_i8x16_u, 0xfd0000000000008aull, 1, 1) \
M(i16x8_shl, 0xfd0000000000008bull, 2, 1) \
M(i16x8_shr_s, 0xfd0000000000008cull, 2, 1) \
M(i16x8_shr_u, 0xfd0000000000008dull, 2, 1) \
M(i16x8_add, 0xfd0000000000008eull, 2, 1) \
M(i16x8_add_sat_s, 0xfd0000000000008full, 2, 1) \
M(i16x8_add_sat_u, 0xfd00000000000090ull, 2, 1) \
M(i16x8_sub, 0xfd00000000000091ull, 2, 1) \
M(i16x8_sub_sat_s, 0xfd00000000000092ull, 2, 1) \
M(i16x8_sub_sat_u, 0xfd00000000000093ull, 2, 1) \
M(f64x2_nearest, 0xfd00000000000094ull, 1, 1) \
M(i16x8_mul, 0xfd00000000000095ull, 2, 1) \
M(i16x8_min_s, 0xfd00000000000096ull, 2, 1) \
M(i16x8_min_u, 0xfd00000000000097ull, 2, 1) \
M(i16x8_max_s, 0xfd00000000000098ull, 2, 1) \
M(i16x8_max_u, 0xfd00000000000099ull, 2, 1) \
M(i16x8_avgr_u, 0xfd0000000000009bull, 2, 1) \
M(i16x8_extmul_low_i8x16_s, 0xfd0000000000009cull, 2, 1) \
M(i16x8_extmul_high_i8x16_s, 0xfd0000000000009dull, 2, 1) \
M(i16x8_extmul_low_i8x16_u, 0xfd0000000000009eull, 2, 1) \
M(i16x8_extmul_high_i8x16_u, 0xfd0000000000009full, 2, 1) \
M(i32x4_abs, 0xfd000000000000a0ull, 1, 1) \
M(i32x4_neg, 0xfd000000000000a1ull, 1, 1) \
M(i32x4_all_true, 0xfd000000000000a3ull, 1, 1) \
M(i32x4_bitmask, 0xfd000000000000a4ull, 1, 1) \
M(i32x4_extend_low_i16x8_s, 0xfd000000000000a7ull, 1, 1) \
M(i32x4_extend_high_i16x8_s, 0xfd000000000000a8ull, 1, 1) \
M(i32x4_extend_low_i16x8_u, 0xfd000000000000a9ull, 1, 1) \
M(i32x4_extend_high_i16x8_u, 0xfd000000000000aaull, 1, 1) \
M(i32x4_shl, 0xfd000000000000abull, 2, 1) \
M(i32x4_shr_s, 0xfd000000000000acull, 2, 1) \
M(i32x4_shr_u, 0xfd000000000000adull, 2, 1) \
M(i32x4_add, 0xfd000000000000aeull, 2, 1) \
M(i32x4_sub, 0xfd000000000000b1ull, 2, 1) \
M(i32x4_mul, 0xfd000000000000b5ull, 2, 1) \
M(i32x4_min_s, 0xfd000000000000b6ull, 2, 1) \
M(i32x4_min_u, 0xfd000000000000b7ull, 2, 1) \
M(i32x4_max_s, 0xfd000000000000b8ull, 2, 1) \
M(i32x4_max_u, 0xfd000000000000b9ull, 2, 1) \
M(i32x4_dot_i16x8_s, 0xfd000000000000baull, 2, 1) \
M(i32x4_extmul_low_i16x8_s, 0xfd000000000000bcull, 2, 1) \
M(i32x4_extmul_high_i16x8_s, 0xfd000000000000bdull, 2, 1) \
M(i32x4_extmul_low_i16x8_u, 0xfd000000000000beull, 2, 1) \
M(i32x4_extmul_high_i16x8_u, 0xfd000000000000bfull, 2, 1) \
M(i64x2_abs, 0xfd000000000000c0ull, 1, 1) \
M(i64x2_neg, 0xfd000000000000c1ull, 1, 1) \
M(i64x2_all_true, 0xfd000000000000c3ull, 1, 1) \
M(i64x2_bitmask, 0xfd000000000000c4ull, 1, 1) \
M(i64x2_extend_low_i32x4_s, 0xfd000000000000c7ull, 1, 1) \
M(i64x2_extend_high_i32x4_s, 0xfd000000000000c8ull, 1, 1) \
M(i64x2_extend_low_i32x4_u, 0xfd000000000000c9ull, 1, 1) \
M(i64x2_extend_high_i32x4_u, 0xfd000000000000caull, 1, 1) \
M(i64x2_shl, 0xfd000000000000cbull, 2, 1) \
M(i64x2_shr_s, 0xfd000000000000ccull, 2, 1) \
M(i64x2_shr_u, 0xfd000000000000cdull, 2, 1) \
M(i64x2_add, 0xfd000000000000ceull, 2, 1) \
M(i64x2_sub, 0xfd000000000000d1ull, 2, 1) \
M(i64x2_mul, 0xfd000000000000d5ull, 2, 1) \
M(i64x2_eq, 0xfd000000000000d6ull, 2, 1) \
M(i64x2_ne, 0xfd000000000000d7ull, 2, 1) \
M(i64x2_lt_s, 0xfd000000000000d8ull, 2, 1) \
M(i64x2_gt_s, 0xfd000000000000d9ull, 2, 1) \
M(i64x2_le_s, 0xfd000000000000daull, 2, 1) \
M(i64x2_ge_s, 0xfd000000000000dbull, 2, 1) \
M(i64x2_extmul_low_i32x4_s, 0xfd000000000000dcull, 2, 1) \
M(i64x2_extmul_high_i32x4_s, 0xfd000000000000ddull, 2, 1) \
M(i64x2_extmul_low_i32x4_u, 0xfd000000000000deull, 2, 1) \
M(i64x2_extmul_high_i32x4_u, 0xfd000000000000dfull, 2, 1) \
M(f32x4_abs, 0xfd000000000000e0ull, 1, 1) \
M(f32x4_neg, 0xfd000000000000e1ull, 1, 1) \
M(f32x4_sqrt, 0xfd000000000000e3ull, 1, 1) \
M(f32x4_add, 0xfd000000000000e4ull, 2, 1) \
M(f32x4_sub, 0xfd000000000000e5ull, 2, 1) \
M(f32x4_mul, 0xfd000000000000e6ull, 2, 1) \
M(f32x4_div, 0xfd000000000000e7ull, 2, 1) \
M(f32x4_min, 0xfd000000000000e8ull, 2, 1) \
M(f32x4_max, 0xfd000000000000e9ull, 2, 1) \
M(f32x4_pmin, 0xfd000000000000eaull, 2, 1) \
M(f32x4_pmax, 0xfd000000000000ebull, 2, 1) \
M(f64x2_abs, 0xfd000000000000ecull, 1, 1) \
M(f64x2_neg, 0xfd000000000000edull, 1, 1) \
M(f64x2_sqrt, 0xfd000000000000efull, 1, 1) \
M(f64x2_add, 0xfd000000000000f0ull, 2, 1) \
M(f64x2_sub, 0xfd000000000000f1ull, 2, 1) \
M(f64x2_mul, 0xfd000000000000f2ull, 2, 1) \
M(f64x2_div, 0xfd000000000000f3ull, 2, 1) \
M(f64x2_min, 0xfd000000000000f4ull, 2, 1) \
M(f64x2_max, 0xfd000000000000f5ull, 2, 1) \
M(f64x2_pmin, 0xfd000000000000f6ull, 2, 1) \
M(f64x2_pmax, 0xfd000000000000f7ull, 2, 1) \
M(i32x4_trunc_sat_f32x4_s, 0xfd000000000000f8ull, 1, 1) \
M(i32x4_trunc_sat_f32x4_u, 0xfd000000000000f9ull, 1, 1) \
M(f32x4_convert_i32x4_s, 0xfd000000000000faull, 1, 1) \
M(f32x4_convert_i32x4_u, 0xfd000000000000fbull, 1, 1) \
M(i32x4_trunc_sat_f64x2_s_zero, 0xfd000000000000fcull, 1, 1) \
M(i32x4_trunc_sat_f64x2_u_zero, 0xfd000000000000fdull, 1, 1) \
M(f64x2_convert_low_i32x4_s, 0xfd000000000000feull, 1, 1) \
M(f64x2_convert_low_i32x4_u, 0xfd000000000000ffull, 1, 1)
#define ENUMERATE_WASM_OPCODES(M) \
ENUMERATE_SINGLE_BYTE_WASM_OPCODES(M) \
ENUMERATE_MULTI_BYTE_WASM_OPCODES(M)
#define M(name, value) static constexpr OpCode name = value;
#define M(name, value, ...) static constexpr OpCode name = value;
ENUMERATE_WASM_OPCODES(M)
#undef M

View file

@ -435,6 +435,8 @@ void Printer::print(Wasm::Instruction const& instruction)
print_indent();
print("({}", instruction_name(instruction.opcode()));
if (instruction.arguments().has<u8>()) {
if (instruction.opcode() == Instructions::local_get || instruction.opcode() == Instructions::local_set || instruction.opcode() == Instructions::local_tee)
print(" (local index {})", instruction.local_index());
print(")\n");
} else {
print(" ");
@ -478,7 +480,10 @@ void Printer::print(Wasm::Instruction const& instruction)
[&](Instruction::TableTableArgs const& args) { print("(table_table (table index {}) (table index {}))", args.lhs.value(), args.rhs.value()); },
[&](ValueType const& type) { print(type); },
[&](Vector<ValueType> const&) { print("(types...)"); },
[&](auto const& value) { print("{}", value); });
[&](auto const& value) { print("(const {})", value); });
if (instruction.local_index().value())
print(" (local index {})", instruction.local_index().value());
print(")\n");
}

View file

@ -461,14 +461,33 @@ public:
{
}
explicit Instruction(OpCode opcode, LocalIndex argument)
: m_opcode(opcode)
, m_local_index(argument)
, m_arguments(static_cast<u8>(0))
{
}
template<typename Arg1>
explicit Instruction(OpCode opcode, LocalIndex argument0, Arg1&& argument1)
: m_opcode(opcode)
, m_local_index(argument0)
, m_arguments(forward<Arg1>(argument1))
{
}
static ParseResult<Instruction> parse(ConstrainedStream& stream);
auto& opcode() const { return m_opcode; }
auto& arguments() const { return m_arguments; }
auto& arguments() { return m_arguments; }
LocalIndex local_index() const { return m_local_index; }
private:
OpCode m_opcode { 0 };
LocalIndex m_local_index;
Variant<
BlockType,
DataIndex,
@ -478,7 +497,7 @@ private:
IndirectCallArgs,
LabelIndex,
LaneIndex,
LocalIndex,
LocalIndex, // Only used by instructions that take more than one local index (currently only fused ops).
MemoryArgument,
MemoryAndLaneArgument,
MemoryCopyArgs,
@ -501,6 +520,34 @@ private:
m_arguments;
};
struct Dispatch {
enum RegisterOrStack : u8 {
R0,
R1,
R2,
R3,
R4,
R5,
R6,
R7,
CountRegisters,
Stack = CountRegisters,
};
Instruction const* instruction { nullptr };
union {
struct {
RegisterOrStack sources[3];
RegisterOrStack destination;
};
u32 sources_and_destination;
};
};
struct CompiledInstructions {
Vector<Dispatch> dispatches;
Vector<Instruction, 0, FastLastAccess::Yes> extra_instruction_storage;
};
struct SectionId {
public:
enum class SectionIdKind : u8 {
@ -710,6 +757,7 @@ public:
void set_stack_usage_hint(size_t value) const { m_stack_usage_hint = value; }
auto stack_usage_hint() const { return m_stack_usage_hint; }
mutable CompiledInstructions compiled_instructions;
private:
Vector<Instruction> m_instructions;
@ -1056,4 +1104,6 @@ private:
Optional<ByteString> m_validation_error;
};
CompiledInstructions try_compile_instructions(Expression const&, Span<FunctionType const> functions);
}

View file

@ -274,6 +274,7 @@ ErrorOr<int> ladybird_main(Main::Arguments arguments)
{
StringView filename;
bool print = false;
bool print_compiled = false;
bool attempt_instantiate = false;
bool export_all_imports = false;
bool wasi = false;
@ -286,6 +287,7 @@ ErrorOr<int> ladybird_main(Main::Arguments arguments)
Core::ArgsParser parser;
parser.add_positional_argument(filename, "File name to parse", "file");
parser.add_option(print, "Print the parsed module", "print", 'p');
parser.add_option(print_compiled, "Print the compiled module", "print-compiled");
parser.add_option(attempt_instantiate, "Attempt to instantiate the module", "instantiate", 'i');
parser.add_option(exported_function_to_execute, "Attempt to execute the named exported function from the module (implies -i)", "execute", 'e', "name");
parser.add_option(export_all_imports, "Export noop functions corresponding to imports", "export-noop");
@ -352,7 +354,7 @@ ErrorOr<int> ladybird_main(Main::Arguments arguments)
printer.print(*parse_result);
}
if (attempt_instantiate) {
if (attempt_instantiate || print_compiled) {
Wasm::AbstractMachine machine;
Optional<Wasm::Wasi::Implementation> wasi_impl;
@ -476,6 +478,7 @@ ErrorOr<int> ladybird_main(Main::Arguments arguments)
print_link_error(link_result.error());
return 1;
}
auto result = machine.instantiate(*parse_result, link_result.release_value());
if (result.is_error()) {
warnln("Module instantiation failed: {}", result.error().error);
@ -483,6 +486,78 @@ ErrorOr<int> ladybird_main(Main::Arguments arguments)
}
auto module_instance = result.release_value();
if (print_compiled) {
for (auto address : module_instance->functions()) {
auto function = machine.store().get(address)->get_pointer<Wasm::WasmFunction>();
if (!function)
continue;
auto& expression = function->code().func().body();
if (expression.compiled_instructions.dispatches.is_empty())
continue;
ByteString export_name;
for (auto& entry : function->module().exports()) {
if (entry.value() == address) {
export_name = ByteString::formatted(" '{}'", entry.name());
break;
}
}
TRY(g_stdout->write_until_depleted(ByteString::formatted("Function #{}{} (stack usage = {}):\n", address.value(), export_name, expression.stack_usage_hint())));
Wasm::Printer printer { *g_stdout, 1 };
for (size_t ip = 0; ip < expression.compiled_instructions.dispatches.size(); ++ip) {
auto& dispatch = expression.compiled_instructions.dispatches[ip];
ByteString regs;
auto first = true;
ssize_t in_count = 0;
bool has_out = false;
#define M(name, _, ins, outs) \
case Wasm::Instructions::name.value(): \
in_count = ins; \
has_out = outs != 0; \
break;
switch (dispatch.instruction->opcode().value()) {
ENUMERATE_WASM_OPCODES(M)
}
#undef M
constexpr auto reg_name = [](Wasm::Dispatch::RegisterOrStack reg) -> ByteString {
if (reg == Wasm::Dispatch::RegisterOrStack::Stack)
return "stack"sv;
return ByteString::formatted("reg{}", to_underlying(reg));
};
if (in_count > -1) {
for (ssize_t index = 0; index < in_count; ++index) {
if (first)
regs = ByteString::formatted("{} ({}", regs, reg_name(dispatch.sources[index]));
else
regs = ByteString::formatted("{}, {}", regs, reg_name(dispatch.sources[index]));
first = false;
}
if (has_out) {
if (first)
regs = ByteString::formatted(" () -> {}", reg_name(dispatch.destination));
else
regs = ByteString::formatted("{}) -> {}", regs, reg_name(dispatch.destination));
} else {
if (first)
regs = ByteString::formatted(" () -x");
else
regs = ByteString::formatted("{}) -x", regs);
}
}
if (!regs.is_empty())
regs = ByteString::formatted(" {{{:<33} }}", regs);
TRY(g_stdout->write_until_depleted(ByteString::formatted(" [{:>03}]", ip)));
TRY(g_stdout->write_until_depleted(regs.bytes()));
printer.print(*dispatch.instruction);
}
TRY(g_stdout->write_until_depleted("\n"sv.bytes()));
}
}
auto print_func = [&](auto const& address) {
Wasm::FunctionInstance* fn = machine.store().get(address);
g_stdout->write_until_depleted(ByteString::formatted("- Function with address {}, ptr = {}\n", address.value(), fn)).release_value_but_fixme_should_propagate_errors();