LibWasm: Move the interpreter IP out of the configuration object

This, along with moving the sources and destination out of the config
object, makes it so we don't have to double-deref to get to them on each
instruction, leading to a ~15% perf improvement on dispatch.

Libraries/LibWasm/AbstractMachine/AbstractMachine.cpp

@@ -270,7 +270,7 @@ InstantiationResult AbstractMachine::instantiate(Module const& module, Vector<Ex
                 auxiliary_instance,
                 Vector<Value> {},
                 entry,
-                entry.instructions().size(),
+                entry.instructions().size() - 1,
             });
             auto result = config.execute(interpreter);
             if (result.is_trap())

Libraries/LibWasm/AbstractMachine/AbstractMachine.h

@@ -572,6 +572,8 @@ public:
     DataInstance* get(DataAddress);
     ElementInstance* get(ElementAddress);
 
+    MemoryInstance* unsafe_get(MemoryAddress address) { return &m_memories.data()[address.value()]; }
+
 private:
     Vector<FunctionInstance> m_functions;
     Vector<TableInstance> m_tables;

Libraries/LibWasm/AbstractMachine/BytecodeInterpreter.h

@@ -13,6 +13,14 @@
 
 namespace Wasm {
 
+union SourcesAndDestination {
+    struct {
+        Dispatch::RegisterOrStack sources[3];
+        Dispatch::RegisterOrStack destination;
+    };
+    u32 sources_and_destination;
+};
+
 struct WASM_API BytecodeInterpreter final : public Interpreter {
     explicit BytecodeInterpreter(StackInfo const& stack_info)
         : m_stack_info(stack_info)
@@ -57,37 +65,37 @@ struct WASM_API BytecodeInterpreter final : public Interpreter {
     void interpret_impl(Configuration&, Expression const&);
 
 protected:
-    void branch_to_label(Configuration&, LabelIndex);
+    InstructionPointer branch_to_label(Configuration&, LabelIndex);
     template<typename ReadT, typename PushT>
-    bool load_and_push(Configuration&, Instruction const&);
+    bool load_and_push(Configuration&, Instruction const&, SourcesAndDestination const&);
     template<typename PopT, typename StoreT>
-    bool pop_and_store(Configuration&, Instruction const&);
+    bool pop_and_store(Configuration&, Instruction const&, SourcesAndDestination const&);
     template<typename StoreT>
-    bool store_value(Configuration&, Instruction const&, StoreT, size_t address_source);
+    bool store_value(Configuration&, Instruction const&, StoreT, size_t address_source, SourcesAndDestination const&);
     template<size_t N>
-    bool pop_and_store_lane_n(Configuration&, Instruction const&);
+    bool pop_and_store_lane_n(Configuration&, Instruction const&, SourcesAndDestination const&);
     template<size_t M, size_t N, template<typename> typename SetSign>
-    bool load_and_push_mxn(Configuration&, Instruction const&);
+    bool load_and_push_mxn(Configuration&, Instruction const&, SourcesAndDestination const&);
     template<size_t N>
-    bool load_and_push_lane_n(Configuration&, Instruction const&);
+    bool load_and_push_lane_n(Configuration&, Instruction const&, SourcesAndDestination const&);
     template<size_t N>
-    bool load_and_push_zero_n(Configuration&, Instruction const&);
+    bool load_and_push_zero_n(Configuration&, Instruction const&, SourcesAndDestination const&);
     template<size_t M>
-    bool load_and_push_m_splat(Configuration&, Instruction const&);
+    bool load_and_push_m_splat(Configuration&, Instruction const&, SourcesAndDestination const&);
     template<size_t M, template<size_t> typename NativeType>
-    void set_top_m_splat(Configuration&, NativeType<M>);
+    void set_top_m_splat(Configuration&, NativeType<M>, SourcesAndDestination const&);
     template<size_t M, template<size_t> typename NativeType>
-    void pop_and_push_m_splat(Configuration&, Instruction const&);
+    void pop_and_push_m_splat(Configuration&, Instruction const&, SourcesAndDestination const&);
     template<typename M, template<typename> typename SetSign, typename VectorType = Native128ByteVectorOf<M, SetSign>>
-    VectorType pop_vector(Configuration&, size_t source);
+    VectorType pop_vector(Configuration&, size_t source, SourcesAndDestination const&);
     bool store_to_memory(Configuration&, Instruction::MemoryArgument const&, ReadonlyBytes data, u32 base);
     bool call_address(Configuration&, FunctionAddress, CallAddressSource = CallAddressSource::DirectCall);
 
     template<typename PopTypeLHS, typename PushType, typename Operator, typename PopTypeRHS = PopTypeLHS, typename... Args>
-    bool binary_numeric_operation(Configuration&, Args&&...);
+    bool binary_numeric_operation(Configuration&, SourcesAndDestination const&, Args&&...);
 
     template<typename PopType, typename PushType, typename Operator, typename... Args>
-    bool unary_operation(Configuration&, Args&&...);
+    bool unary_operation(Configuration&, SourcesAndDestination const&, Args&&...);
 
     template<typename T>
     T read_value(ReadonlyBytes data);

Libraries/LibWasm/AbstractMachine/Configuration.cpp

@@ -11,11 +11,10 @@
 
 namespace Wasm {
 
-void Configuration::unwind(Badge<CallFrameHandle>, CallFrameHandle const& frame_handle)
+void Configuration::unwind(Badge<CallFrameHandle>, CallFrameHandle const&)
 {
     m_frame_stack.take_last();
     m_depth--;
-    m_ip = frame_handle.ip.value();
     m_locals_base = m_frame_stack.is_empty() ? nullptr : m_frame_stack.unchecked_last().locals().data();
 }
 

Libraries/LibWasm/AbstractMachine/Configuration.h

@@ -21,7 +21,7 @@ public:
 
     void set_frame(Frame frame)
     {
-        Label label(frame.arity(), frame.expression().instructions().size(), m_value_stack.size());
+        Label label(frame.arity(), frame.expression().instructions().size() - 1, m_value_stack.size());
         frame.label_index() = m_label_stack.size();
         if (auto hint = frame.expression().stack_usage_hint(); hint.has_value())
             m_value_stack.ensure_capacity(*hint + m_value_stack.size());
@@ -50,8 +50,7 @@ public:
 
     struct CallFrameHandle {
         explicit CallFrameHandle(Configuration& configuration)
-            : ip(configuration.ip())
+            : configuration(configuration)
-            , configuration(configuration)
         {
             configuration.depth()++;
         }
@@ -61,7 +60,6 @@ public:
             configuration.unwind({}, *this);
         }
 
-        InstructionPointer ip { 0 };
         Configuration& configuration;
     };
 
@@ -74,7 +72,7 @@ public:
 
     void dump_stack();
 
-    ALWAYS_INLINE FLATTEN void push_to_destination(Value value)
+    ALWAYS_INLINE FLATTEN void push_to_destination(Value value, Dispatch::RegisterOrStack destination)
     {
         if (destination == Dispatch::RegisterOrStack::Stack) {
             value_stack().unchecked_append(value);
@@ -83,7 +81,7 @@ public:
         regs.data()[to_underlying(destination)] = value;
     }
 
-    ALWAYS_INLINE FLATTEN Value& source_value(u8 index)
+    ALWAYS_INLINE FLATTEN Value& source_value(u8 index, Dispatch::RegisterOrStack const* sources)
     {
         // Note: The last source in a dispatch *must* be equal to the destination for this to be valid.
         auto const source = sources[index];
@@ -92,7 +90,7 @@ public:
         return regs.data()[to_underlying(source)];
     }
 
-    ALWAYS_INLINE FLATTEN Value take_source(u8 index)
+    ALWAYS_INLINE FLATTEN Value take_source(u8 index, Dispatch::RegisterOrStack const* sources)
     {
         auto const source = sources[index];
         if (source == Dispatch::RegisterOrStack::Stack)
@@ -100,14 +98,6 @@ public:
         return regs.data()[to_underlying(source)];
     }
 
-    union {
-        struct {
-            Dispatch::RegisterOrStack sources[3];
-            Dispatch::RegisterOrStack destination;
-        };
-        u32 sources_and_destination;
-    };
-
     Array<Value, Dispatch::RegisterOrStack::CountRegisters> regs = {
         Value(0),
         Value(0),

Libraries/LibWasm/AbstractMachine/Validator.cpp

@@ -3700,6 +3700,12 @@ VALIDATE_INSTRUCTION(f64x2_convert_low_i32x4_u)
     return stack.take_and_put<ValueType::V128>(ValueType::V128);
 }
 
+VALIDATE_INSTRUCTION(synthetic_end_expression)
+{
+    is_constant = true;
+    return {}; // Always valid.
+}
+
 ErrorOr<void, ValidationError> Validator::validate(Instruction const& instruction, Stack& stack, bool& is_constant)
 {
     switch (instruction.opcode().value()) {

Libraries/LibWasm/Opcode.h

@@ -473,7 +473,8 @@ namespace Instructions {
     M(synthetic_call_20, 0xfe0000000000000aull, 2, 0)            \
     M(synthetic_call_21, 0xfe0000000000000bull, 2, 1)            \
     M(synthetic_call_30, 0xfe0000000000000cull, 3, 0)            \
-    M(synthetic_call_31, 0xfe0000000000000dull, 3, 1)
+    M(synthetic_call_31, 0xfe0000000000000dull, 3, 1)            \
+    M(synthetic_end_expression, 0xfe0000000000000eull, 0, 0)
 
 #define ENUMERATE_WASM_OPCODES(M)         \
     ENUMERATE_SINGLE_BYTE_WASM_OPCODES(M) \
@@ -484,7 +485,7 @@ ENUMERATE_WASM_OPCODES(M)
 #undef M
 
 static constexpr inline OpCode SyntheticInstructionBase = 0xfe00000000000000ull;
-static constexpr inline size_t SyntheticInstructionCount = 14;
+static constexpr inline size_t SyntheticInstructionCount = 15;
 
 }
 

Libraries/LibWasm/Parser/Parser.cpp

@@ -978,8 +978,10 @@ ParseResult<Expression> Expression::parse(ConstrainedStream& stream, Optional<si
             stack.append(ip);
             break;
         case Instructions::structured_end.value(): {
-            if (stack.is_empty())
+            if (stack.is_empty()) {
+                instructions.empend(Instructions::synthetic_end_expression); // Synthetic noop to mark the end of the expression.
                 return Expression { move(instructions) };
+            }
             auto entry = stack.take_last();
             auto& args = instructions[entry.value()].arguments().get<Instruction::StructuredInstructionArgs>();
             // Patch the end_ip of the last structured instruction
@@ -999,6 +1001,8 @@ ParseResult<Expression> Expression::parse(ConstrainedStream& stream, Optional<si
         ++ip;
     }
 
+    instructions.empend(Instructions::synthetic_end_expression); // Synthetic noop to mark the end of the expression.
+
     return Expression { move(instructions) };
 }
 
@@ -1106,7 +1110,10 @@ ParseResult<ElementSection::Element> ElementSection::Element::parse(ConstrainedS
     if (!has_exprs) {
         auto indices = TRY(parse_vector<GenericIndexParser<FunctionIndex>>(stream));
         for (auto& index : indices) {
-            Vector<Instruction> instructions { Instruction(Instructions::ref_func, index) };
+            Vector<Instruction> instructions {
+                Instruction(Instructions::ref_func, index),
+                Instruction(Instructions::synthetic_end_expression),
+            };
             items.empend(move(instructions));
         }
     } else {

Libraries/LibWasm/Printer/Printer.cpp

@@ -1160,5 +1160,6 @@ HashMap<Wasm::OpCode, ByteString> Wasm::Names::instruction_names {
     { Instructions::synthetic_call_21, "synthetic:call.21" },
     { Instructions::synthetic_call_30, "synthetic:call.30" },
     { Instructions::synthetic_call_31, "synthetic:call.31" },
+    { Instructions::synthetic_end_expression, "synthetic:expression.end" },
 };
 HashMap<ByteString, Wasm::OpCode> Wasm::Names::instructions_by_name;

Libraries/LibWasm/Types.h

@@ -535,6 +535,7 @@ struct Dispatch {
         Stack = CountRegisters,
     };
 
+    OpCode instruction_opcode;
     Instruction const* instruction { nullptr };
     union {
         struct {