diff --git a/ARMeilleure/CodeGen/CompiledFunction.cs b/ARMeilleure/CodeGen/CompiledFunction.cs new file mode 100644 index 0000000000..61e89c2401 --- /dev/null +++ b/ARMeilleure/CodeGen/CompiledFunction.cs @@ -0,0 +1,17 @@ +using ARMeilleure.CodeGen.Unwinding; + +namespace ARMeilleure.CodeGen +{ + struct CompiledFunction + { + public byte[] Code { get; } + + public UnwindInfo UnwindInfo { get; } + + public CompiledFunction(byte[] code, UnwindInfo unwindInfo) + { + Code = code; + UnwindInfo = unwindInfo; + } + } +} \ No newline at end of file diff --git a/ARMeilleure/CodeGen/RegisterAllocators/FastLinearScan.cs b/ARMeilleure/CodeGen/RegisterAllocators/FastLinearScan.cs new file mode 100644 index 0000000000..d72c60c1da --- /dev/null +++ b/ARMeilleure/CodeGen/RegisterAllocators/FastLinearScan.cs @@ -0,0 +1,568 @@ +using ARMeilleure.Common; +using ARMeilleure.IntermediateRepresentation; +using ARMeilleure.Translation; +using System; +using System.Collections.Generic; +using System.Diagnostics; + +using static ARMeilleure.IntermediateRepresentation.OperandHelper; + +namespace ARMeilleure.CodeGen.RegisterAllocators +{ + class FastLinearScan + { + private const int InstructionGap = 2; + + private const int RegistersCount = 16; + + private const int MaxIROperands = 4; + + private class OperationInfo + { + public LinkedListNode Node { get; } + + public int IntSpillUsedRegisters { get; set; } + public int VecSpillUsedRegisters { get; set; } + + public OperationInfo(LinkedListNode node) + { + Node = node; + } + } + + private List _operationNodes; + + private int _intSpillTemps; + private int _vecSpillTemps; + + private List _intervals; + + private class CompareIntervalsEnd : IComparer + { + public int Compare(LiveInterval lhs, LiveInterval rhs) + { + if (lhs.GetEnd() == rhs.GetEnd()) + { + return 0; + } + else if (lhs.GetEnd() < rhs.GetEnd()) + { + return 1; + } + else + { + return -1; + } + } + } + + public AllocationResult RunPass(ControlFlowGraph cfg, StackAllocator stackAlloc, RegisterMasks regMasks) + { + PhiFunctions.Remove(cfg); + + BuildIntervals(cfg, regMasks); + + List[] fixedIntervals = new List[2]; + + fixedIntervals[0] = new List(); + fixedIntervals[1] = new List(); + + for (int index = 0; index < RegistersCount * 2; index++) + { + LiveInterval interval = _intervals[index]; + + if (!interval.IsEmpty) + { + InsertSorted(fixedIntervals[index & 1], interval); + } + } + + List activeIntervals = new List(); + + CompareIntervalsEnd comparer = new CompareIntervalsEnd(); + + int intFreeRegisters = regMasks.IntAvailableRegisters; + int vecFreeRegisters = regMasks.VecAvailableRegisters; + + int intUsedRegisters = 0; + int vecUsedRegisters = 0; + + intFreeRegisters = ReserveSpillTemps(ref _intSpillTemps, intFreeRegisters); + vecFreeRegisters = ReserveSpillTemps(ref _vecSpillTemps, vecFreeRegisters); + + for (int index = RegistersCount * 2; index < _intervals.Count; index++) + { + LiveInterval current = _intervals[index]; + + while (activeIntervals.Count != 0 && + activeIntervals[activeIntervals.Count - 1].GetEnd() < current.GetStart()) + { + int iIndex = activeIntervals.Count - 1; + + LiveInterval interval = activeIntervals[iIndex]; + + if (interval.Register.Type == RegisterType.Integer) + { + intFreeRegisters |= 1 << interval.Register.Index; + } + else /* if (interval.Register.Type == RegisterType.Vector) */ + { + vecFreeRegisters |= 1 << interval.Register.Index; + } + + activeIntervals.RemoveAt(iIndex); + } + + Operand local = current.Local; + + bool localIsInteger = local.Type.IsInteger(); + + int freeMask = localIsInteger ? intFreeRegisters : vecFreeRegisters; + + if (freeMask != 0) + { + List fixedIntervalsForType = fixedIntervals[localIsInteger ? 0 : 1]; + + for (int iIndex = 0; iIndex < fixedIntervalsForType.Count; iIndex++) + { + LiveInterval interval = fixedIntervalsForType[iIndex]; + + if (interval.GetStart() >= current.GetEnd()) + { + break; + } + + if (interval.Overlaps(current)) + { + freeMask &= ~(1 << interval.Register.Index); + } + } + } + + if (freeMask != 0) + { + int selectedReg = BitUtils.LowestBitSet(freeMask); + + current.Register = new Register(selectedReg, local.Type.ToRegisterType()); + + int regMask = 1 << selectedReg; + + if (localIsInteger) + { + intUsedRegisters |= regMask; + intFreeRegisters &= ~regMask; + } + else + { + vecUsedRegisters |= regMask; + vecFreeRegisters &= ~regMask; + } + } + else + { + //Spill the interval that will free the register for the longest + //amount of time, as long there's no interference of the current + //interval with a fixed interval using the same register. + bool hasRegisterSelected = false; + + RegisterType regType = current.Local.Type.ToRegisterType(); + + for (int iIndex = 0; iIndex < activeIntervals.Count; iIndex++) + { + LiveInterval spillCandidate = activeIntervals[iIndex]; + + if (spillCandidate.Register.Type != regType) + { + continue; + } + + LiveInterval fixedInterval = _intervals[GetRegisterId(spillCandidate.Register)]; + + if (fixedInterval.IsEmpty || !fixedInterval.Overlaps(current)) + { + current.Register = spillCandidate.Register; + + spillCandidate.Spill(stackAlloc.Allocate(spillCandidate.Local.Type)); + + activeIntervals.RemoveAt(iIndex); + + hasRegisterSelected = true; + + break; + } + } + + Debug.Assert(hasRegisterSelected, "Failure allocating register with spill."); + } + + InsertSorted(activeIntervals, current, comparer); + } + + for (int index = RegistersCount * 2; index < _intervals.Count; index++) + { + LiveInterval interval = _intervals[index]; + + if (interval.IsSpilled) + { + ReplaceLocalWithSpill(interval, ref intUsedRegisters, ref vecUsedRegisters); + } + else + { + ReplaceLocalWithRegister(interval); + } + } + + return new AllocationResult(intUsedRegisters, vecUsedRegisters, stackAlloc.TotalSize); + } + + private int ReserveSpillTemps(ref int tempsMask, int availableRegs) + { + for (int index = 0; index < MaxIROperands; index++) + { + int selectedReg = BitUtils.HighestBitSet(availableRegs); + + tempsMask |= 1 << selectedReg; + + availableRegs &= ~(1 << selectedReg); + } + + return availableRegs; + } + + private void ReplaceLocalWithRegister(LiveInterval interval) + { + Operand register = GetRegister(interval); + + foreach (int usePosition in interval.UsePositions()) + { + Node operation = GetOperationInfo(usePosition).Node.Value; + + for (int srcIndex = 0; srcIndex < operation.SourcesCount; srcIndex++) + { + Operand source = operation.GetSource(srcIndex); + + if (source == interval.Local) + { + operation.SetSource(srcIndex, register); + } + } + + if (operation.Dest == interval.Local) + { + operation.Dest = register; + } + } + } + + private static Operand GetRegister(LiveInterval interval) + { + Debug.Assert(!interval.IsSpilled, "Spilled intervals are not allowed."); + + return new Operand( + interval.Register.Index, + interval.Register.Type, + interval.Local.Type); + } + + private void ReplaceLocalWithSpill( + LiveInterval interval, + ref int intUsedRegisters, + ref int vecUsedRegisters) + { + Operand local = interval.Local; + + int spillOffset = interval.SpillOffset; + + foreach (int usePosition in interval.UsePositions()) + { + OperationInfo info = GetOperationInfo(usePosition); + + int tempReg = GetSpillTemp(info, local.Type); + + if (local.Type.IsInteger()) + { + intUsedRegisters |= 1 << tempReg; + } + else + { + vecUsedRegisters |= 1 << tempReg; + } + + Operand temp = new Operand(tempReg, local.Type.ToRegisterType(), local.Type); + + LinkedListNode node = info.Node; + + Node operation = node.Value; + + for (int srcIndex = 0; srcIndex < operation.SourcesCount; srcIndex++) + { + Operand source = operation.GetSource(srcIndex); + + if (source == local) + { + Operation fillOp = new Operation(Instruction.Fill, temp, Const(spillOffset)); + + node.List.AddBefore(node, fillOp); + + operation.SetSource(srcIndex, temp); + } + } + + if (operation.Dest == local) + { + Operation spillOp = new Operation(Instruction.Spill, null, Const(spillOffset), temp); + + node.List.AddAfter(node, spillOp); + + operation.Dest = temp; + } + } + } + + private OperationInfo GetOperationInfo(int position) + { + return _operationNodes[position / InstructionGap]; + } + + private int GetSpillTemp(OperationInfo info, OperandType type) + { + int selectedReg; + + if (type.IsInteger()) + { + selectedReg = BitUtils.LowestBitSet(_intSpillTemps & ~info.IntSpillUsedRegisters); + + info.IntSpillUsedRegisters |= 1 << selectedReg; + } + else + { + selectedReg = BitUtils.LowestBitSet(_vecSpillTemps & ~info.VecSpillUsedRegisters); + + info.VecSpillUsedRegisters |= 1 << selectedReg; + } + + Debug.Assert(selectedReg != -1, "Out of spill temporary registers. " + (info.Node.Value as Operation).Inst); + + return selectedReg; + } + + private static void InsertSorted( + List list, + LiveInterval interval, + IComparer comparer = null) + { + int insertIndex = list.BinarySearch(interval, comparer); + + if (insertIndex < 0) + { + insertIndex = ~insertIndex; + } + + list.Insert(insertIndex, interval); + } + + private void BuildIntervals(ControlFlowGraph cfg, RegisterMasks masks) + { + _operationNodes = new List(); + + _intervals = new List(); + + for (int index = 0; index < RegistersCount; index++) + { + _intervals.Add(new LiveInterval(new Register(index, RegisterType.Integer))); + _intervals.Add(new LiveInterval(new Register(index, RegisterType.Vector))); + } + + HashSet visited = new HashSet(); + + LiveInterval GetOrAddInterval(Operand operand) + { + LiveInterval interval; + + if (visited.Add(operand)) + { + operand.NumberLocal(_intervals.Count); + + interval = new LiveInterval(operand); + + _intervals.Add(interval); + } + else + { + interval = _intervals[GetOperandId(operand)]; + } + + return interval; + } + + int[] blockStarts = new int[cfg.Blocks.Count]; + + int operationPos = 0; + + List backwardsBranches = new List(); + + for (int index = cfg.PostOrderBlocks.Length - 1; index >= 0; index--) + { + BasicBlock block = cfg.PostOrderBlocks[index]; + + blockStarts[block.Index] = operationPos; + + for (LinkedListNode node = block.Operations.First; node != null; node = node.Next) + { + _operationNodes.Add(new OperationInfo(node)); + + Operation operation = node.Value as Operation; + + Operand dest = operation.Dest; + + if (dest != null) + { + if (dest.Kind == OperandKind.LocalVariable) + { + LiveInterval interval = GetOrAddInterval(dest); + + if (interval.IsEmpty) + { + interval.SetStart(operationPos + 1); + } + + interval.AddUsePosition(operationPos); + } + else if (dest.Kind == OperandKind.Register) + { + int iIndex = GetRegisterId(dest.GetRegister()); + + _intervals[iIndex].AddRange(operationPos + 1, operationPos + InstructionGap); + } + } + + for (int srcIndex = 0; srcIndex < operation.SourcesCount; srcIndex++) + { + Operand source = operation.GetSource(srcIndex); + + if (source.Kind == OperandKind.LocalVariable) + { + LiveInterval interval = GetOrAddInterval(source); + + Debug.Assert(!interval.IsEmpty, "Interval is empty."); + + interval.SetEnd(operationPos + 1); + interval.AddUsePosition(operationPos); + } + else if (source.Kind == OperandKind.Register) + { + int iIndex = GetRegisterId(source.GetRegister()); + + LiveInterval interval = _intervals[iIndex]; + + if (interval.IsEmpty) + { + interval.SetStart(operationPos + 1); + } + else if (interval.GetEnd() < operationPos + 1) + { + interval.SetEnd(operationPos + 1); + } + } + } + + if (operation.Inst == Instruction.Call) + { + AddIntervalCallerSavedReg(masks.IntCallerSavedRegisters, operationPos, RegisterType.Integer); + AddIntervalCallerSavedReg(masks.VecCallerSavedRegisters, operationPos, RegisterType.Vector); + } + + operationPos += InstructionGap; + } + + foreach (BasicBlock successor in Successors(block)) + { + int branchIndex = cfg.PostOrderMap[block.Index]; + int targetIndex = cfg.PostOrderMap[successor.Index]; + + //Is the branch jumping backwards? + if (targetIndex >= branchIndex) + { + int targetPos = blockStarts[successor.Index]; + + backwardsBranches.Add(new LiveRange(targetPos, operationPos)); + } + } + } + + foreach (LiveRange backwardBranch in backwardsBranches) + { + for (int iIndex = RegistersCount * 2; iIndex < _intervals.Count; iIndex++) + { + LiveInterval interval = _intervals[iIndex]; + + int start = interval.GetStart(); + int end = interval.GetEnd(); + + if (backwardBranch.Start >= start && backwardBranch.Start < end) + { + if (interval.GetEnd() < backwardBranch.End) + { + interval.SetEnd(backwardBranch.End); + } + } + + if (start > backwardBranch.Start) + { + break; + } + } + } + } + + private void AddIntervalCallerSavedReg(int mask, int operationPos, RegisterType regType) + { + while (mask != 0) + { + int regIndex = BitUtils.LowestBitSet(mask); + + Register callerSavedReg = new Register(regIndex, regType); + + int rIndex = GetRegisterId(callerSavedReg); + + _intervals[rIndex].AddRange(operationPos + 1, operationPos + InstructionGap); + + mask &= ~(1 << regIndex); + } + } + + private static int GetOperandId(Operand operand) + { + if (operand.Kind == OperandKind.LocalVariable) + { + return operand.AsInt32(); + } + else if (operand.Kind == OperandKind.Register) + { + return GetRegisterId(operand.GetRegister()); + } + else + { + throw new ArgumentException($"Invalid operand kind \"{operand.Kind}\"."); + } + } + + private static int GetRegisterId(Register register) + { + return (register.Index << 1) | (register.Type == RegisterType.Vector ? 1 : 0); + } + + private static IEnumerable Successors(BasicBlock block) + { + if (block.Next != null) + { + yield return block.Next; + } + + if (block.Branch != null) + { + yield return block.Branch; + } + } + } +} \ No newline at end of file diff --git a/ARMeilleure/CodeGen/RegisterAllocators/LinearScan.cs b/ARMeilleure/CodeGen/RegisterAllocators/LinearScan.cs index 6fd79643b9..a49a8c0a8f 100644 --- a/ARMeilleure/CodeGen/RegisterAllocators/LinearScan.cs +++ b/ARMeilleure/CodeGen/RegisterAllocators/LinearScan.cs @@ -41,11 +41,10 @@ namespace ARMeilleure.CodeGen.RegisterAllocators public int IntUsedRegisters { get; set; } public int VecUsedRegisters { get; set; } - public AllocationContext(RegisterMasks masks, int intervalsCount) + public AllocationContext(StackAllocator stackAlloc, RegisterMasks masks, int intervalsCount) { - Masks = masks; - - StackAlloc = new StackAllocator(); + StackAlloc = stackAlloc; + Masks = masks; Active = new BitMap(intervalsCount); Inactive = new BitMap(intervalsCount); @@ -69,13 +68,13 @@ namespace ARMeilleure.CodeGen.RegisterAllocators } } - public AllocationResult RunPass(ControlFlowGraph cfg, RegisterMasks regMasks) + public AllocationResult RunPass(ControlFlowGraph cfg, StackAllocator stackAlloc, RegisterMasks regMasks) { PhiFunctions.Remove(cfg); NumberLocals(cfg); - AllocationContext context = new AllocationContext(regMasks, _intervals.Count); + AllocationContext context = new AllocationContext(stackAlloc, regMasks, _intervals.Count); BuildIntervals(cfg, context); @@ -95,10 +94,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators continue; } - if (!current.HasRegister) - { - AllocateInterval(context, current, index); - } + AllocateInterval(context, current, index); } for (int index = RegistersCount * 2; index < _intervals.Count; index++) @@ -202,7 +198,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators int selectedNextUse = freePositions[selectedReg]; //Intervals starts and ends at odd positions, unless they span an entire - //block, in this case they will have ranges at even position. + //block, in this case they will have ranges at a even position. //When a interval is loaded from the stack to a register, we can only //do the split at a odd position, because otherwise the split interval //that is inserted on the list to be processed may clobber a register @@ -907,23 +903,10 @@ namespace ARMeilleure.CodeGen.RegisterAllocators interval.AddUsePosition(operationPos); } - if (node is Operation operation) + if (node is Operation operation && operation.Inst == Instruction.Call) { - if (operation.Inst == Instruction.Call) - { - AddIntervalCallerSavedReg(context.Masks.IntCallerSavedRegisters, operationPos, RegisterType.Integer); - AddIntervalCallerSavedReg(context.Masks.VecCallerSavedRegisters, operationPos, RegisterType.Vector); - } - else if (operation.Inst == Instruction.StackAlloc) - { - Operand offset = operation.GetSource(0); - - Debug.Assert(offset.Kind == OperandKind.Constant, "StackAlloc has non-constant size."); - - int spillOffset = context.StackAlloc.Allocate(offset.AsInt32()); - - operation.SetSource(0, new Operand(spillOffset)); - } + AddIntervalCallerSavedReg(context.Masks.IntCallerSavedRegisters, operationPos, RegisterType.Integer); + AddIntervalCallerSavedReg(context.Masks.VecCallerSavedRegisters, operationPos, RegisterType.Vector); } } } @@ -935,8 +918,6 @@ namespace ARMeilleure.CodeGen.RegisterAllocators { int regIndex = BitUtils.LowestBitSet(mask); - Debug.Assert(regIndex < RegistersCount, "Invalid register index."); - Register callerSavedReg = new Register(regIndex, regType); LiveInterval interval = _intervals[GetRegisterId(callerSavedReg)]; diff --git a/ARMeilleure/CodeGen/RegisterAllocators/LiveInterval.cs b/ARMeilleure/CodeGen/RegisterAllocators/LiveInterval.cs index 9448b0ac61..438562a04b 100644 --- a/ARMeilleure/CodeGen/RegisterAllocators/LiveInterval.cs +++ b/ARMeilleure/CodeGen/RegisterAllocators/LiveInterval.cs @@ -24,23 +24,7 @@ namespace ARMeilleure.CodeGen.RegisterAllocators public Operand Local { get; } - private Register _register; - - public bool HasRegister { get; private set; } - - public Register Register - { - get - { - return _register; - } - set - { - _register = value; - - HasRegister = true; - } - } + public Register Register { get; set; } public int SpillOffset { get; private set; } @@ -93,6 +77,22 @@ namespace ARMeilleure.CodeGen.RegisterAllocators return _ranges[0].Start; } + public void SetEnd(int position) + { + if (_ranges.Count != 0) + { + int lastIdx = _ranges.Count - 1; + + Debug.Assert(position != _ranges[lastIdx].Start); + + _ranges[lastIdx] = new LiveRange(_ranges[lastIdx].Start, position); + } + else + { + _ranges.Add(new LiveRange(position, position + 1)); + } + } + public int GetEnd() { if (_ranges.Count == 0) @@ -103,6 +103,35 @@ namespace ARMeilleure.CodeGen.RegisterAllocators return _ranges[_ranges.Count - 1].End; } + public void Expand(int position) + { + Expand(position, position + 1); + + _usePositions.Add(position); + } + + public void Expand(int start, int end) + { + if (_ranges.Count == 0) + { + _ranges.Add(new LiveRange(start, end)); + + return; + } + + int lastIdx = _ranges.Count - 1; + + if (_ranges[0].Start > start) + { + _ranges[0] = new LiveRange(start, _ranges[0].End); + } + + if (_ranges[lastIdx].End < end) + { + _ranges[lastIdx] = new LiveRange(_ranges[lastIdx].Start, end); + } + } + public void AddRange(int start, int end) { if (start >= end) diff --git a/ARMeilleure/CodeGen/Unwinding/UnwindInfo.cs b/ARMeilleure/CodeGen/Unwinding/UnwindInfo.cs new file mode 100644 index 0000000000..4955f1b4a5 --- /dev/null +++ b/ARMeilleure/CodeGen/Unwinding/UnwindInfo.cs @@ -0,0 +1,18 @@ +namespace ARMeilleure.CodeGen.Unwinding +{ + struct UnwindInfo + { + public UnwindPushEntry[] PushEntries { get; } + + public int PrologueSize { get; } + + public int FixedAllocSize { get; } + + public UnwindInfo(UnwindPushEntry[] pushEntries, int prologueSize, int fixedAllocSize) + { + PushEntries = pushEntries; + PrologueSize = prologueSize; + FixedAllocSize = fixedAllocSize; + } + } +} \ No newline at end of file diff --git a/ARMeilleure/CodeGen/Unwinding/UnwindPushEntry.cs b/ARMeilleure/CodeGen/Unwinding/UnwindPushEntry.cs new file mode 100644 index 0000000000..6597e2b4b9 --- /dev/null +++ b/ARMeilleure/CodeGen/Unwinding/UnwindPushEntry.cs @@ -0,0 +1,20 @@ +using ARMeilleure.IntermediateRepresentation; + +namespace ARMeilleure.CodeGen.Unwinding +{ + struct UnwindPushEntry + { + public int Index { get; } + + public RegisterType Type { get; } + + public int StreamEndOffset { get; } + + public UnwindPushEntry(int index, RegisterType type, int streamEndOffset) + { + Index = index; + Type = type; + StreamEndOffset = streamEndOffset; + } + } +} \ No newline at end of file diff --git a/ARMeilleure/CodeGen/X86/Assembler.cs b/ARMeilleure/CodeGen/X86/Assembler.cs index 5656fbb6ea..1511068336 100644 --- a/ARMeilleure/CodeGen/X86/Assembler.cs +++ b/ARMeilleure/CodeGen/X86/Assembler.cs @@ -11,6 +11,8 @@ namespace ARMeilleure.CodeGen.X86 private const int OpModRMBits = 24; + private const byte LockPrefix = 0xf0; + [Flags] private enum InstFlags { @@ -82,6 +84,8 @@ namespace ARMeilleure.CodeGen.X86 Add(X86Instruction.Cmpps, new InstInfo(BadOp, BadOp, BadOp, BadOp, 0x00000fc2, InstFlags.Vex | InstFlags.NoRexW)); Add(X86Instruction.Cmpsd, new InstInfo(BadOp, BadOp, BadOp, BadOp, 0x00000fc2, InstFlags.Vex | InstFlags.NoRexW | InstFlags.PrefixF2)); Add(X86Instruction.Cmpss, new InstInfo(BadOp, BadOp, BadOp, BadOp, 0x00000fc2, InstFlags.Vex | InstFlags.NoRexW | InstFlags.PrefixF3)); + Add(X86Instruction.Cmpxchg8b, new InstInfo(0x01000fc7, BadOp, BadOp, BadOp, BadOp, InstFlags.NoRexW)); + Add(X86Instruction.Cmpxchg16b, new InstInfo(0x01000fc7, BadOp, BadOp, BadOp, BadOp, InstFlags.RexW)); Add(X86Instruction.Comisd, new InstInfo(BadOp, BadOp, BadOp, BadOp, 0x00000f2f, InstFlags.Vex | InstFlags.NoRexW | InstFlags.Prefix66)); Add(X86Instruction.Comiss, new InstInfo(BadOp, BadOp, BadOp, BadOp, 0x00000f2f, InstFlags.Vex | InstFlags.NoRexW)); Add(X86Instruction.Cvtdq2pd, new InstInfo(BadOp, BadOp, BadOp, BadOp, 0x00000fe6, InstFlags.Vex | InstFlags.NoRexW | InstFlags.PrefixF3)); @@ -360,6 +364,20 @@ namespace ARMeilleure.CodeGen.X86 WriteByte(imm); } + public void Cmpxchg16b(X86MemoryOperand memOp) + { + WriteByte(LockPrefix); + + WriteInstruction(memOp, null, X86Instruction.Cmpxchg16b); + } + + public void Cmpxchg8b(X86MemoryOperand memOp) + { + WriteByte(LockPrefix); + + WriteInstruction(memOp, null, X86Instruction.Cmpxchg8b); + } + public void Comisd(Operand source1, Operand source2) { WriteInstruction(source1, source2, X86Instruction.Comisd); diff --git a/ARMeilleure/CodeGen/X86/CallingConvention.cs b/ARMeilleure/CodeGen/X86/CallingConvention.cs index 750f4dc561..65cdf0a38a 100644 --- a/ARMeilleure/CodeGen/X86/CallingConvention.cs +++ b/ARMeilleure/CodeGen/X86/CallingConvention.cs @@ -8,12 +8,7 @@ namespace ARMeilleure.CodeGen.X86 public static int GetIntAvailableRegisters() { - int mask = RegistersMask; - - mask &= ~(1 << (int)X86Register.Rbp); - mask &= ~(1 << (int)X86Register.Rsp); - - return mask; + return RegistersMask & ~(1 << (int)X86Register.Rsp); } public static int GetVecAvailableRegisters() diff --git a/ARMeilleure/CodeGen/X86/CodeGenContext.cs b/ARMeilleure/CodeGen/X86/CodeGenContext.cs index 198e8a8386..88f80c69d0 100644 --- a/ARMeilleure/CodeGen/X86/CodeGenContext.cs +++ b/ARMeilleure/CodeGen/X86/CodeGenContext.cs @@ -13,6 +13,8 @@ namespace ARMeilleure.CodeGen.X86 private Stream _stream; + public int StreamOffset => (int)_stream.Length; + public AllocationResult AllocResult { get; } public Assembler Assembler { get; } @@ -23,6 +25,8 @@ namespace ARMeilleure.CodeGen.X86 public int VecCalleeSaveSize { get; } + private long[] _blockOffsets; + private struct Jump { public bool IsConditional { get; } @@ -62,8 +66,6 @@ namespace ARMeilleure.CodeGen.X86 } } - private long[] _blockOffsets; - private List _jumps; private X86Condition _jNearCondition; @@ -97,16 +99,20 @@ namespace ARMeilleure.CodeGen.X86 vecCalleeSaveSize = BitUtils.CountBits(vecMask) * 16; - intMask |= 1 << (int)X86Register.Rbp; - int calleeSaveRegionSize = BitUtils.CountBits(intMask) * 8 + vecCalleeSaveSize + 8; int argsCount = maxCallArgs; - //The ABI mandates that the space for at least 4 arguments - //is reserved on the stack (this is called shadow space). - if (argsCount < 4 && maxCallArgs != -1) + if (argsCount < 0) { + //When the function has no calls, argsCount is -1. + //In this case, we don't need to allocate the shadow space. + argsCount = 0; + } + else if (argsCount < 4) + { + //The ABI mandates that the space for at least 4 arguments + //is reserved on the stack (this is called shadow space). argsCount = 4; } diff --git a/ARMeilleure/CodeGen/X86/CodeGenerator.cs b/ARMeilleure/CodeGen/X86/CodeGenerator.cs index fac111e9d5..252c8054c9 100644 --- a/ARMeilleure/CodeGen/X86/CodeGenerator.cs +++ b/ARMeilleure/CodeGen/X86/CodeGenerator.cs @@ -1,11 +1,12 @@ using ARMeilleure.CodeGen.Optimizations; using ARMeilleure.CodeGen.RegisterAllocators; +using ARMeilleure.CodeGen.Unwinding; using ARMeilleure.Common; using ARMeilleure.Diagnostics; using ARMeilleure.IntermediateRepresentation; -using ARMeilleure.Memory; using ARMeilleure.Translation; using System; +using System.Collections.Generic; using System.Diagnostics; using System.IO; @@ -29,6 +30,7 @@ namespace ARMeilleure.CodeGen.X86 Add(Instruction.BranchIfTrue, GenerateBranchIfTrue); Add(Instruction.ByteSwap, GenerateByteSwap); Add(Instruction.Call, GenerateCall); + Add(Instruction.CompareAndSwap128, GenerateCompareAndSwap128); Add(Instruction.CompareEqual, GenerateCompareEqual); Add(Instruction.CompareGreater, GenerateCompareGreater); Add(Instruction.CompareGreaterOrEqual, GenerateCompareGreaterOrEqual); @@ -48,12 +50,8 @@ namespace ARMeilleure.CodeGen.X86 Add(Instruction.DivideUI, GenerateDivideUI); Add(Instruction.Fill, GenerateFill); Add(Instruction.Load, GenerateLoad); - Add(Instruction.LoadFromContext, GenerateLoadFromContext); - Add(Instruction.LoadSx16, GenerateLoadSx16); - Add(Instruction.LoadSx32, GenerateLoadSx32); - Add(Instruction.LoadSx8, GenerateLoadSx8); - Add(Instruction.LoadZx16, GenerateLoadZx16); - Add(Instruction.LoadZx8, GenerateLoadZx8); + Add(Instruction.Load16, GenerateLoad16); + Add(Instruction.Load8, GenerateLoad8); Add(Instruction.Multiply, GenerateMultiply); Add(Instruction.Multiply64HighSI, GenerateMultiply64HighSI); Add(Instruction.Multiply64HighUI, GenerateMultiply64HighUI); @@ -72,7 +70,6 @@ namespace ARMeilleure.CodeGen.X86 Add(Instruction.Store, GenerateStore); Add(Instruction.Store16, GenerateStore16); Add(Instruction.Store8, GenerateStore8); - Add(Instruction.StoreToContext, GenerateStoreToContext); Add(Instruction.Subtract, GenerateSubtract); Add(Instruction.VectorExtract, GenerateVectorExtract); Add(Instruction.VectorExtract16, GenerateVectorExtract16); @@ -226,8 +223,10 @@ namespace ARMeilleure.CodeGen.X86 _instTable[(int)inst] = func; } - public static byte[] Generate(ControlFlowGraph cfg, MemoryManager memory) + public static CompiledFunction Generate(CompilerContext cctx) { + ControlFlowGraph cfg = cctx.Cfg; + Logger.StartPass(PassName.Optimization); Optimizer.RunPass(cfg); @@ -236,13 +235,15 @@ namespace ARMeilleure.CodeGen.X86 Logger.StartPass(PassName.PreAllocation); - PreAllocator.RunPass(cfg, memory, out int maxCallArgs); + StackAllocator stackAlloc = new StackAllocator(); + + PreAllocator.RunPass(cctx, stackAlloc, out int maxCallArgs); Logger.EndPass(PassName.PreAllocation, cfg); Logger.StartPass(PassName.RegisterAllocation); - LinearScan regAlloc = new LinearScan(); + FastLinearScan regAlloc = new FastLinearScan(); RegisterMasks regMasks = new RegisterMasks( CallingConvention.GetIntAvailableRegisters(), @@ -252,17 +253,17 @@ namespace ARMeilleure.CodeGen.X86 CallingConvention.GetIntCalleeSavedRegisters(), CallingConvention.GetVecCalleeSavedRegisters()); - AllocationResult allocResult = regAlloc.RunPass(cfg, regMasks); + AllocationResult allocResult = regAlloc.RunPass(cfg, stackAlloc, regMasks); Logger.EndPass(PassName.RegisterAllocation, cfg); + Logger.StartPass(PassName.CodeGeneration); + using (MemoryStream stream = new MemoryStream()) { CodeGenContext context = new CodeGenContext(stream, allocResult, maxCallArgs, cfg.Blocks.Count); - WritePrologue(context); - - context.Assembler.Mov(Register(X86Register.Rbp), Register(X86Register.Rcx)); + UnwindInfo unwindInfo = WritePrologue(context); foreach (BasicBlock block in cfg.Blocks) { @@ -277,7 +278,9 @@ namespace ARMeilleure.CodeGen.X86 } } - return context.GetCode(); + Logger.EndPass(PassName.CodeGeneration); + + return new CompiledFunction(context.GetCode(), unwindInfo); } } @@ -379,6 +382,16 @@ namespace ARMeilleure.CodeGen.X86 context.Assembler.Call(operation.GetSource(0)); } + private static void GenerateCompareAndSwap128(CodeGenContext context, Operation operation) + { + Operand dest = operation.Dest; + Operand src1 = operation.GetSource(0); + + X86MemoryOperand memOp = new X86MemoryOperand(OperandType.I64, src1, null, Scale.x1, 0); + + context.Assembler.Cmpxchg16b(memOp); + } + private static void GenerateCompareEqual(CodeGenContext context, Operation operation) { GenerateCompare(context, operation, X86Condition.Equal); @@ -645,55 +658,7 @@ namespace ARMeilleure.CodeGen.X86 } } - private static void GenerateLoadFromContext(CodeGenContext context, Operation operation) - { - Operand dest = operation.Dest; - Operand offset = operation.GetSource(0); - - if (offset.Kind != OperandKind.Constant) - { - throw new InvalidOperationException("LoadFromContext has non-constant context offset."); - } - - Operand rbp = Register(X86Register.Rbp); - - X86MemoryOperand memOp = new X86MemoryOperand(dest.Type, rbp, null, Scale.x1, offset.AsInt32()); - - if (dest.GetRegister().Type == RegisterType.Vector) - { - context.Assembler.Movdqu(dest, memOp); - } - else - { - context.Assembler.Mov(dest, memOp); - } - } - - private static void GenerateLoadSx16(CodeGenContext context, Operation operation) - { - Operand value = operation.Dest; - Operand address = GetMemoryOperand(operation.GetSource(0), value.Type); - - context.Assembler.Movsx16(operation.Dest, address); - } - - private static void GenerateLoadSx32(CodeGenContext context, Operation operation) - { - Operand value = operation.Dest; - Operand address = GetMemoryOperand(operation.GetSource(0), value.Type); - - context.Assembler.Movsx32(operation.Dest, address); - } - - private static void GenerateLoadSx8(CodeGenContext context, Operation operation) - { - Operand value = operation.Dest; - Operand address = GetMemoryOperand(operation.GetSource(0), value.Type); - - context.Assembler.Movsx8(operation.Dest, address); - } - - private static void GenerateLoadZx16(CodeGenContext context, Operation operation) + private static void GenerateLoad16(CodeGenContext context, Operation operation) { Operand value = operation.Dest; Operand address = GetMemoryOperand(operation.GetSource(0), value.Type); @@ -701,7 +666,7 @@ namespace ARMeilleure.CodeGen.X86 context.Assembler.Movzx16(operation.Dest, address); } - private static void GenerateLoadZx8(CodeGenContext context, Operation operation) + private static void GenerateLoad8(CodeGenContext context, Operation operation) { Operand value = operation.Dest; Operand address = GetMemoryOperand(operation.GetSource(0), value.Type); @@ -753,18 +718,6 @@ namespace ARMeilleure.CodeGen.X86 private static void GenerateReturn(CodeGenContext context, Operation operation) { - if (operation.SourcesCount != 0) - { - Operand returnReg = Register(CallingConvention.GetIntReturnRegister()); - - Operand sourceReg = operation.GetSource(0); - - if (returnReg.GetRegister() != sourceReg.GetRegister()) - { - context.Assembler.Mov(returnReg, sourceReg); - } - } - WriteEpilogue(context); context.Assembler.Return(); @@ -872,30 +825,6 @@ namespace ARMeilleure.CodeGen.X86 context.Assembler.Mov8(address, value); } - private static void GenerateStoreToContext(CodeGenContext context, Operation operation) - { - Operand offset = operation.GetSource(0); - Operand source = operation.GetSource(1); - - if (offset.Kind != OperandKind.Constant) - { - throw new InvalidOperationException("StoreToContext has non-constant context offset."); - } - - Operand rbp = Register(X86Register.Rbp); - - X86MemoryOperand memOp = new X86MemoryOperand(source.Type, rbp, null, Scale.x1, offset.AsInt32()); - - if (source.GetRegister().Type == RegisterType.Vector) - { - context.Assembler.Movdqu(memOp, source); - } - else - { - context.Assembler.Mov(memOp, source); - } - } - private static void GenerateSubtract(CodeGenContext context, Operation operation) { ValidateDestSrc1(operation); @@ -1157,36 +1086,42 @@ namespace ARMeilleure.CodeGen.X86 { context.Assembler.Comisd(operation.GetSource(0), operation.GetSource(1)); context.Assembler.Setcc(operation.Dest, X86Condition.Equal); + context.Assembler.Movzx8(operation.Dest, operation.Dest); } private static void GenerateX86Comisdge(CodeGenContext context, Operation operation) { context.Assembler.Comisd(operation.GetSource(0), operation.GetSource(1)); context.Assembler.Setcc(operation.Dest, X86Condition.AboveOrEqual); + context.Assembler.Movzx8(operation.Dest, operation.Dest); } private static void GenerateX86Comisdlt(CodeGenContext context, Operation operation) { context.Assembler.Comisd(operation.GetSource(0), operation.GetSource(1)); context.Assembler.Setcc(operation.Dest, X86Condition.Below); + context.Assembler.Movzx8(operation.Dest, operation.Dest); } private static void GenerateX86Comisseq(CodeGenContext context, Operation operation) { context.Assembler.Comiss(operation.GetSource(0), operation.GetSource(1)); context.Assembler.Setcc(operation.Dest, X86Condition.Equal); + context.Assembler.Movzx8(operation.Dest, operation.Dest); } private static void GenerateX86Comissge(CodeGenContext context, Operation operation) { context.Assembler.Comiss(operation.GetSource(0), operation.GetSource(1)); context.Assembler.Setcc(operation.Dest, X86Condition.AboveOrEqual); + context.Assembler.Movzx8(operation.Dest, operation.Dest); } private static void GenerateX86Comisslt(CodeGenContext context, Operation operation) { context.Assembler.Comiss(operation.GetSource(0), operation.GetSource(1)); context.Assembler.Setcc(operation.Dest, X86Condition.Below); + context.Assembler.Movzx8(operation.Dest, operation.Dest); } private static void GenerateX86Cvtdq2pd(CodeGenContext context, Operation operation) @@ -1785,6 +1720,7 @@ namespace ARMeilleure.CodeGen.X86 { context.Assembler.Cmp(operation.GetSource(0), operation.GetSource(1)); context.Assembler.Setcc(operation.Dest, condition); + context.Assembler.Movzx8(operation.Dest, operation.Dest); } private static void GenerateSpill(CodeGenContext context, Operation operation, int baseOffset) @@ -1840,11 +1776,11 @@ namespace ARMeilleure.CodeGen.X86 } } - private static void WritePrologue(CodeGenContext context) + private static UnwindInfo WritePrologue(CodeGenContext context) { - int mask = CallingConvention.GetIntCalleeSavedRegisters() & context.AllocResult.IntUsedRegisters; + List pushEntries = new List(); - mask |= 1 << (int)X86Register.Rbp; + int mask = CallingConvention.GetIntCalleeSavedRegisters() & context.AllocResult.IntUsedRegisters; while (mask != 0) { @@ -1852,6 +1788,8 @@ namespace ARMeilleure.CodeGen.X86 context.Assembler.Push(Register((X86Register)bit)); + pushEntries.Add(new UnwindPushEntry(bit, RegisterType.Integer, context.StreamOffset)); + mask &= ~(1 << bit); } @@ -1869,6 +1807,8 @@ namespace ARMeilleure.CodeGen.X86 context.Assembler.Movdqu(memOp, Xmm((X86Register)bit)); + pushEntries.Add(new UnwindPushEntry(bit, RegisterType.Vector, context.StreamOffset)); + mask &= ~(1 << bit); } @@ -1880,6 +1820,8 @@ namespace ARMeilleure.CodeGen.X86 { context.Assembler.Sub(Register(X86Register.Rsp), new Operand(reservedStackSize)); } + + return new UnwindInfo(pushEntries.ToArray(), context.StreamOffset, reservedStackSize); } private static void WriteEpilogue(CodeGenContext context) @@ -1912,8 +1854,6 @@ namespace ARMeilleure.CodeGen.X86 mask = CallingConvention.GetIntCalleeSavedRegisters() & context.AllocResult.IntUsedRegisters; - mask |= 1 << (int)X86Register.Rbp; - while (mask != 0) { int bit = BitUtils.HighestBitSet(mask); diff --git a/ARMeilleure/CodeGen/X86/PreAllocator.cs b/ARMeilleure/CodeGen/X86/PreAllocator.cs index e526f57ad2..35faee4d74 100644 --- a/ARMeilleure/CodeGen/X86/PreAllocator.cs +++ b/ARMeilleure/CodeGen/X86/PreAllocator.cs @@ -1,5 +1,5 @@ +using ARMeilleure.CodeGen.RegisterAllocators; using ARMeilleure.IntermediateRepresentation; -using ARMeilleure.Memory; using ARMeilleure.Translation; using System.Collections.Generic; using System.Diagnostics; @@ -10,11 +10,13 @@ namespace ARMeilleure.CodeGen.X86 { static class PreAllocator { - public static void RunPass(ControlFlowGraph cfg, MemoryManager memory, out int maxCallArgs) + public static void RunPass(CompilerContext cctx, StackAllocator stackAlloc, out int maxCallArgs) { maxCallArgs = -1; - foreach (BasicBlock block in cfg.Blocks) + Operand[] preservedArgs = new Operand[CallingConvention.GetArgumentsOnRegsCount()]; + + foreach (BasicBlock block in cctx.Cfg.Blocks) { LinkedListNode nextNode; @@ -31,15 +33,9 @@ namespace ARMeilleure.CodeGen.X86 HandleConstantCopy(node, operation); - //Comparison instructions uses CMOVcc, which does not zero the - //upper bits of the register (since it's R8), we need to ensure it - //is zero by zeroing it beforehand. - if (inst.IsComparison() || IsComparisonIntrinsic(inst)) - { - Operation copyOp = new Operation(Instruction.Copy, operation.Dest, Const(0)); + HandleFixedRegisterCopy(node, operation); - block.Operations.AddBefore(node, copyOp); - } + HandleSameDestSrc1Copy(node, operation); //Unsigned integer to FP conversions are not supported on X86. //We need to turn them into signed integer to FP conversions, and @@ -73,11 +69,19 @@ namespace ARMeilleure.CodeGen.X86 { maxCallArgs = argsCount; } + + //Copy values to registers expected by the function being called, + //as mandated by the ABI. + HandleCallWindowsAbi(stackAlloc, node, operation); + } + else if (inst == Instruction.Return) + { + HandleReturnWindowsAbi(cctx, node, preservedArgs, operation); + } + else if (inst == Instruction.LoadArgument) + { + HandleLoadArgumentWindowsAbi(cctx, node, preservedArgs, operation); } - - HandleFixedRegisterCopy(node, operation); - - HandleSameDestSrc1Copy(node, operation); } } } @@ -184,7 +188,6 @@ namespace ARMeilleure.CodeGen.X86 Operand zex = Local(OperandType.I64); temp = nodes.AddAfter(temp, new Operation(Instruction.Copy, zex, source)); - temp = nodes.AddAfter(temp, new Operation(Instruction.ConvertToFP, dest, zex)); } else /* if (source.Type == OperandType.I64) */ @@ -315,7 +318,44 @@ namespace ARMeilleure.CodeGen.X86 operation.Dest = rdx; } - //The only allowed shift register is CL. + //Handle the many restrictions of the compare and exchange (16 bytes) instruction: + //- The expected value should be in RDX:RAX. + //- The new value to be written should be in RCX:RBX. + //- The value at the memory location is loaded to RDX:RAX. + if (inst == Instruction.CompareAndSwap128) + { + void SplitOperand(Operand source, X86Register lowReg, X86Register highReg) + { + Operand lr = Gpr(lowReg, OperandType.I64); + Operand hr = Gpr(highReg, OperandType.I64); + + Operation extrL = new Operation(Instruction.VectorExtract, lr, source, Const(0)); + Operation extrH = new Operation(Instruction.VectorExtract, hr, source, Const(1)); + + node.List.AddBefore(node, extrL); + node.List.AddBefore(node, extrH); + } + + Operand src2 = operation.GetSource(1); + Operand src3 = operation.GetSource(2); + + SplitOperand(src2, X86Register.Rax, X86Register.Rdx); + SplitOperand(src3, X86Register.Rbx, X86Register.Rcx); + + Operand rax = Gpr(X86Register.Rax, OperandType.I64); + Operand rdx = Gpr(X86Register.Rdx, OperandType.I64); + + Operation insL = new Operation(Instruction.Copy, dest, rax); + Operation insH = new Operation(Instruction.VectorInsert, dest, dest, rdx, Const(1)); + + node.List.AddAfter(node, insH); + node.List.AddAfter(node, insL); + + operation.SetSource(1, Undef()); + operation.SetSource(2, Undef()); + } + + //The shift register is always implied to be CL (low 8-bits of RCX or ECX). if (inst.IsShift() && operation.GetSource(1).Kind == OperandKind.LocalVariable) { Operand rcx = Gpr(X86Register.Rcx, OperandType.I32); @@ -326,16 +366,12 @@ namespace ARMeilleure.CodeGen.X86 operation.SetSource(1, rcx); } - - //Copy values to registers expected by the function being called, - //as mandated by the ABI. - if (inst == Instruction.Call) - { - HandleCallWindowsAbi(node, operation); - } } - private static void HandleCallWindowsAbi(LinkedListNode node, Operation operation) + private static void HandleCallWindowsAbi( + StackAllocator stackAlloc, + LinkedListNode node, + Operation operation) { Operand dest = operation.Dest; @@ -344,11 +380,33 @@ namespace ARMeilleure.CodeGen.X86 Operand retValueAddr = null; + int stackAllocOffset = 0; + + int AllocateOnStack(int size) + { + //We assume that the stack allocator is initially empty (TotalSize = 0). + //Taking that into account, we can reuse the space allocated for other + //calls by keeping track of our own allocated size (stackAllocOffset). + //If the space allocated is not big enough, then we just expand it. + int offset = stackAllocOffset; + + if (stackAllocOffset + size > stackAlloc.TotalSize) + { + stackAlloc.Allocate((stackAllocOffset + size) - stackAlloc.TotalSize); + } + + stackAllocOffset += size; + + return offset; + } + if (dest != null && dest.Type == OperandType.V128) { retValueAddr = Local(OperandType.I64); - Operation allocOp = new Operation(Instruction.StackAlloc, retValueAddr, Const(dest.Type.GetSizeInBytes())); + int stackOffset = AllocateOnStack(dest.Type.GetSizeInBytes()); + + Operation allocOp = new Operation(Instruction.StackAlloc, retValueAddr, Const(stackOffset)); node.List.AddBefore(node, allocOp); @@ -369,7 +427,9 @@ namespace ARMeilleure.CodeGen.X86 { Operand stackAddr = Local(OperandType.I64); - Operation allocOp = new Operation(Instruction.StackAlloc, stackAddr, Const(source.Type.GetSizeInBytes())); + int stackOffset = AllocateOnStack(source.Type.GetSizeInBytes()); + + Operation allocOp = new Operation(Instruction.StackAlloc, stackAddr, Const(stackOffset)); node.List.AddBefore(node, allocOp); @@ -466,6 +526,121 @@ namespace ARMeilleure.CodeGen.X86 } } + private static void HandleReturnWindowsAbi( + CompilerContext cctx, + LinkedListNode node, + Operand[] preservedArgs, + Operation operation) + { + if (operation.SourcesCount == 0) + { + return; + } + + Operand source = operation.GetSource(0); + + Operand retReg; + + if (source.Type.IsInteger()) + { + retReg = Gpr(CallingConvention.GetIntReturnRegister(), source.Type); + } + else if (source.Type == OperandType.V128) + { + if (preservedArgs[0] == null) + { + Operand preservedArg = Local(OperandType.I64); + + Operand arg0 = Gpr(CallingConvention.GetIntArgumentRegister(0), OperandType.I64); + + Operation copyOp = new Operation(Instruction.Copy, preservedArg, arg0); + + cctx.Cfg.Entry.Operations.AddFirst(copyOp); + + preservedArgs[0] = preservedArg; + } + + retReg = preservedArgs[0]; + } + else /* if (regType == RegisterType.Vector) */ + { + retReg = Xmm(CallingConvention.GetVecReturnRegister(), source.Type); + } + + if (source.Type == OperandType.V128) + { + Operation retStoreOp = new Operation(Instruction.Store, null, retReg, source); + + node.List.AddBefore(node, retStoreOp); + } + else + { + Operation retCopyOp = new Operation(Instruction.Copy, retReg, source); + + node.List.AddBefore(node, retCopyOp); + } + } + + private static void HandleLoadArgumentWindowsAbi( + CompilerContext cctx, + LinkedListNode node, + Operand[] preservedArgs, + Operation operation) + { + Operand source = operation.GetSource(0); + + Debug.Assert(source.Kind == OperandKind.Constant, "Non-constant LoadArgument source kind."); + + int retArgs = cctx.FuncReturnType == OperandType.V128 ? 1 : 0; + + int index = source.AsInt32() + retArgs; + + if (index < CallingConvention.GetArgumentsOnRegsCount()) + { + Operand dest = operation.Dest; + + if (preservedArgs[index] == null) + { + Operand preservedArg; + + Operand argReg; + + if (dest.Type.IsInteger()) + { + argReg = Gpr(CallingConvention.GetIntArgumentRegister(index), dest.Type); + + preservedArg = Local(dest.Type); + } + else if (dest.Type == OperandType.V128) + { + argReg = Gpr(CallingConvention.GetIntArgumentRegister(index), OperandType.I64); + + preservedArg = Local(OperandType.I64); + } + else /* if (regType == RegisterType.Vector) */ + { + argReg = Xmm(CallingConvention.GetVecArgumentRegister(index), dest.Type); + + preservedArg = Local(dest.Type); + } + + Operation copyOp = new Operation(Instruction.Copy, preservedArg, argReg); + + cctx.Cfg.Entry.Operations.AddFirst(copyOp); + + preservedArgs[index] = preservedArg; + } + + Operation loadArgOp = new Operation(dest.Type == OperandType.V128 + ? Instruction.Load + : Instruction.Copy, dest, preservedArgs[index]); + + node.List.AddBefore(node, loadArgOp); + + Delete(node, operation); + } + } + private static void HandleSameDestSrc1Copy(LinkedListNode node, Operation operation) { if (operation.Dest == null || operation.SourcesCount == 0) @@ -618,6 +793,7 @@ namespace ARMeilleure.CodeGen.X86 switch (inst) { case Instruction.Copy: + case Instruction.LoadArgument: case Instruction.LoadFromContext: case Instruction.Spill: case Instruction.SpillArg: @@ -683,21 +859,5 @@ namespace ARMeilleure.CodeGen.X86 return inst > Instruction.X86Intrinsic_Start && inst < Instruction.X86Intrinsic_End; } - - private static bool IsComparisonIntrinsic(Instruction inst) - { - switch (inst) - { - case Instruction.X86Comisdeq: - case Instruction.X86Comisdge: - case Instruction.X86Comisdlt: - case Instruction.X86Comisseq: - case Instruction.X86Comissge: - case Instruction.X86Comisslt: - return true; - } - - return false; - } } } \ No newline at end of file diff --git a/ARMeilleure/CodeGen/X86/X86Instruction.cs b/ARMeilleure/CodeGen/X86/X86Instruction.cs index 8fbae89b0f..ee9805e9b4 100644 --- a/ARMeilleure/CodeGen/X86/X86Instruction.cs +++ b/ARMeilleure/CodeGen/X86/X86Instruction.cs @@ -19,6 +19,8 @@ namespace ARMeilleure.CodeGen.X86 Cmpps, Cmpsd, Cmpss, + Cmpxchg16b, + Cmpxchg8b, Comisd, Comiss, Cvtdq2pd, diff --git a/ARMeilleure/Diagnostics/Logger.cs b/ARMeilleure/Diagnostics/Logger.cs index 9833564253..72b3d60dd8 100644 --- a/ARMeilleure/Diagnostics/Logger.cs +++ b/ARMeilleure/Diagnostics/Logger.cs @@ -1,20 +1,34 @@ using ARMeilleure.Translation; using System; +using System.Diagnostics; namespace ARMeilleure.Diagnostics { static class Logger { + private static long _startTime; + + private static long[] _accumulatedTime; + + static Logger() + { + _accumulatedTime = new long[(int)PassName.Count]; + } + + public static bool DoLog; + public static void StartPass(PassName name) { -#if DEBUG +#if M_DEBUG WriteOutput(name + " pass started..."); + + _startTime = Stopwatch.GetTimestamp(); #endif } public static void EndPass(PassName name, ControlFlowGraph cfg) { -#if DEBUG +#if M_DEBUG EndPass(name); WriteOutput("IR after " + name + " pass:"); @@ -25,11 +39,20 @@ namespace ARMeilleure.Diagnostics public static void EndPass(PassName name) { -#if DEBUG - WriteOutput(name + " pass ended..."); +#if M_DEBUG + long elapsedTime = Stopwatch.GetTimestamp() - _startTime; + + _accumulatedTime[(int)name] += elapsedTime; + + WriteOutput($"{name} pass ended after {GetMilliseconds(_accumulatedTime[(int)name])} ms..."); #endif } + private static long GetMilliseconds(long ticks) + { + return (long)(((double)ticks / Stopwatch.Frequency) * 1000); + } + private static void WriteOutput(string text) { Console.WriteLine(text); diff --git a/ARMeilleure/Diagnostics/PassName.cs b/ARMeilleure/Diagnostics/PassName.cs index ca5e23cf9e..e37439855e 100644 --- a/ARMeilleure/Diagnostics/PassName.cs +++ b/ARMeilleure/Diagnostics/PassName.cs @@ -2,10 +2,16 @@ namespace ARMeilleure.Diagnostics { enum PassName { + Decoding, Translation, + RegisterUsage, + Dominance, SsaConstruction, Optimization, PreAllocation, - RegisterAllocation + RegisterAllocation, + CodeGeneration, + + Count } } \ No newline at end of file diff --git a/ARMeilleure/Instructions/InstEmitMemoryEx.cs b/ARMeilleure/Instructions/InstEmitMemoryEx.cs index 38565a2433..cfc7243b6c 100644 --- a/ARMeilleure/Instructions/InstEmitMemoryEx.cs +++ b/ARMeilleure/Instructions/InstEmitMemoryEx.cs @@ -84,8 +84,8 @@ namespace ARMeilleure.Instructions { Operand value = EmitLoad(context, address, exclusive, 4); - Operand valueLow = context.VectorExtract(value, Local(OperandType.I64), 0); - Operand valueHigh = context.VectorExtract(value, Local(OperandType.I64), 1); + Operand valueLow = context.VectorExtract(OperandType.I64, value, 0); + Operand valueHigh = context.VectorExtract(OperandType.I64, value, 1); SetIntOrZR(context, op.Rt, valueLow); SetIntOrZR(context, op.Rt2, valueHigh); diff --git a/ARMeilleure/Instructions/InstEmitMemoryHelper.cs b/ARMeilleure/Instructions/InstEmitMemoryHelper.cs index 188dee2a6a..0220efd5cd 100644 --- a/ARMeilleure/Instructions/InstEmitMemoryHelper.cs +++ b/ARMeilleure/Instructions/InstEmitMemoryHelper.cs @@ -144,10 +144,21 @@ namespace ARMeilleure.Instructions switch (size) { - case 0: value = context.LoadZx8 (Local(OperandType.I32), physAddr); break; - case 1: value = context.LoadZx16(Local(OperandType.I32), physAddr); break; - case 2: value = context.Load (Local(OperandType.I32), physAddr); break; - case 3: value = context.Load (Local(OperandType.I64), physAddr); break; + case 0: + value = context.Load8(physAddr); + break; + + case 1: + value = context.Load16(physAddr); + break; + + case 2: + value = context.Load(OperandType.I32, physAddr); + break; + + case 3: + value = context.Load(OperandType.I64, physAddr); + break; } SetIntOrZR(context, rt, value); @@ -186,23 +197,23 @@ namespace ARMeilleure.Instructions switch (size) { case 0: - value = context.VectorInsert8(vector, context.LoadZx8(Local(OperandType.I32), physAddr), elem); + value = context.VectorInsert8(vector, context.Load8(physAddr), elem); break; case 1: - value = context.VectorInsert16(vector, context.LoadZx16(Local(OperandType.I32), physAddr), elem); + value = context.VectorInsert16(vector, context.Load16(physAddr), elem); break; case 2: - value = context.VectorInsert(vector, context.Load(Local(OperandType.I32), physAddr), elem); + value = context.VectorInsert(vector, context.Load(OperandType.I32, physAddr), elem); break; case 3: - value = context.VectorInsert(vector, context.Load(Local(OperandType.I64), physAddr), elem); + value = context.VectorInsert(vector, context.Load(OperandType.I64, physAddr), elem); break; case 4: - value = context.Load(Local(OperandType.V128), physAddr); + value = context.Load(OperandType.V128, physAddr); break; } @@ -284,19 +295,19 @@ namespace ARMeilleure.Instructions switch (size) { case 0: - context.Store8(physAddr, context.VectorExtract8(value, Local(OperandType.I32), elem)); + context.Store8(physAddr, context.VectorExtract8(value, elem)); break; case 1: - context.Store16(physAddr, context.VectorExtract16(value, Local(OperandType.I32), elem)); + context.Store16(physAddr, context.VectorExtract16(value, elem)); break; case 2: - context.Store(physAddr, context.VectorExtract(value, Local(OperandType.FP32), elem)); + context.Store(physAddr, context.VectorExtract(OperandType.FP32, value, elem)); break; case 3: - context.Store(physAddr, context.VectorExtract(value, Local(OperandType.FP64), elem)); + context.Store(physAddr, context.VectorExtract(OperandType.FP64, value, elem)); break; case 4: @@ -337,7 +348,7 @@ namespace ARMeilleure.Instructions Operand pteAddress = context.Add(pte, pteOffset); - pte = context.Load(Local(OperandType.I64), pteAddress); + pte = context.Load(OperandType.I64, pteAddress); } while (bit < context.Memory.AddressSpaceBits); @@ -455,18 +466,27 @@ namespace ARMeilleure.Instructions MethodInfo info = typeof(NativeInterface).GetMethod(fallbackMethodName); - Operand value; + Operand value = null; if (size < 4) { - value = Local(size == 3 ? OperandType.I64 : OperandType.I32); - switch (size) { - case 0: context.VectorExtract8 (GetVec(rt), value, elem); break; - case 1: context.VectorExtract16(GetVec(rt), value, elem); break; - case 2: context.VectorExtract (GetVec(rt), value, elem); break; - case 3: context.VectorExtract (GetVec(rt), value, elem); break; + case 0: + value = context.VectorExtract8(GetVec(rt), elem); + break; + + case 1: + value = context.VectorExtract16(GetVec(rt), elem); + break; + + case 2: + value = context.VectorExtract(OperandType.I32, GetVec(rt), elem); + break; + + case 3: + value = context.VectorExtract(OperandType.I64, GetVec(rt), elem); + break; } } else diff --git a/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs b/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs index 0a2869ef92..ded192315a 100644 --- a/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs +++ b/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs @@ -387,8 +387,8 @@ namespace ARMeilleure.Instructions OperandType type = sizeF != 0 ? OperandType.FP64 : OperandType.FP32; - Operand ne0 = context.VectorExtract(GetVec(op.Rn), Local(type), 0); - Operand ne1 = context.VectorExtract(GetVec(op.Rn), Local(type), 1); + Operand ne0 = context.VectorExtract(type, GetVec(op.Rn), 0); + Operand ne1 = context.VectorExtract(type, GetVec(op.Rn), 1); Operand res = EmitSoftFloatCall(context, nameof(SoftFloat32.FPAdd), ne0, ne1); @@ -1022,9 +1022,9 @@ namespace ARMeilleure.Instructions OperandType type = sizeF != 0 ? OperandType.FP64 : OperandType.FP32; - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(type), 0); - Operand me = context.VectorExtract(GetVec(op.Rm), Local(type), 0); - Operand ae = context.VectorExtract(GetVec(op.Ra), Local(type), 0); + Operand ne = context.VectorExtract(type, GetVec(op.Rn), 0); + Operand me = context.VectorExtract(type, GetVec(op.Rm), 0); + Operand ae = context.VectorExtract(type, GetVec(op.Ra), 0); Operand res = context.Subtract(context.Multiply(context.Negate(ne), me), ae); @@ -1040,9 +1040,9 @@ namespace ARMeilleure.Instructions OperandType type = sizeF != 0 ? OperandType.FP64 : OperandType.FP32; - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(type), 0); - Operand me = context.VectorExtract(GetVec(op.Rm), Local(type), 0); - Operand ae = context.VectorExtract(GetVec(op.Ra), Local(type), 0); + Operand ne = context.VectorExtract(type, GetVec(op.Rn), 0); + Operand me = context.VectorExtract(type, GetVec(op.Rm), 0); + Operand ae = context.VectorExtract(type, GetVec(op.Ra), 0); Operand res = context.Subtract(context.Multiply(ne, me), ae); diff --git a/ARMeilleure/Instructions/InstEmitSimdCmp.cs b/ARMeilleure/Instructions/InstEmitSimdCmp.cs index e57160b125..4297c2c239 100644 --- a/ARMeilleure/Instructions/InstEmitSimdCmp.cs +++ b/ARMeilleure/Instructions/InstEmitSimdCmp.cs @@ -466,7 +466,7 @@ namespace ARMeilleure.Instructions { Operand ordMask = context.AddIntrinsic(Instruction.X86Cmpss, n, m, Const(cmpOrdered)); - Operand isOrdered = context.VectorExtract16(ordMask, Local(OperandType.I32), 0); + Operand isOrdered = context.VectorExtract16(ordMask, 0); context.BranchIfFalse(lblNaN, isOrdered); @@ -483,7 +483,7 @@ namespace ARMeilleure.Instructions { Operand ordMask = context.AddIntrinsic(Instruction.X86Cmpsd, n, m, Const(cmpOrdered)); - Operand isOrdered = context.VectorExtract16(ordMask, Local(OperandType.I32), 0); + Operand isOrdered = context.VectorExtract16(ordMask, 0); context.BranchIfFalse(lblNaN, isOrdered); @@ -512,7 +512,7 @@ namespace ARMeilleure.Instructions { OperandType type = op.Size != 0 ? OperandType.FP64 : OperandType.FP32; - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(type), 0); + Operand ne = context.VectorExtract(type, GetVec(op.Rn), 0); Operand me; if (cmpWithZero) @@ -521,7 +521,7 @@ namespace ARMeilleure.Instructions } else { - me = context.VectorExtract(GetVec(op.Rm), Local(type), 0); + me = context.VectorExtract(type, GetVec(op.Rm), 0); } Operand nzcv = EmitSoftFloatCall(context, nameof(SoftFloat32.FPCompare), ne, me, Const(signalNaNs)); @@ -625,12 +625,12 @@ namespace ARMeilleure.Instructions for (int index = 0; index < elems; index++) { - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(type), index); + Operand ne = context.VectorExtract(type, GetVec(op.Rn), index); Operand me; if (op is OpCodeSimdReg binOp) { - me = context.VectorExtract(GetVec(binOp.Rm), Local(type), index); + me = context.VectorExtract(type, GetVec(binOp.Rm), index); } else { diff --git a/ARMeilleure/Instructions/InstEmitSimdCvt.cs b/ARMeilleure/Instructions/InstEmitSimdCvt.cs index 132cfdbf17..d759a77871 100644 --- a/ARMeilleure/Instructions/InstEmitSimdCvt.cs +++ b/ARMeilleure/Instructions/InstEmitSimdCvt.cs @@ -32,7 +32,7 @@ namespace ARMeilleure.Instructions } else { - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(OperandType.FP32), 0); + Operand ne = context.VectorExtract(OperandType.FP32, GetVec(op.Rn), 0); Operand res = context.ConvertToFP(OperandType.FP64, ne); @@ -51,7 +51,7 @@ namespace ARMeilleure.Instructions } else { - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(OperandType.FP64), 0); + Operand ne = context.VectorExtract(OperandType.FP64, GetVec(op.Rn), 0); Operand res = context.ConvertToFP(OperandType.FP32, ne); @@ -60,7 +60,7 @@ namespace ARMeilleure.Instructions } else if (op.Size == 0 && op.Opc == 3) // Single -> Half. { - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(OperandType.FP32), 0); + Operand ne = context.VectorExtract(OperandType.FP32, GetVec(op.Rn), 0); MethodInfo info = typeof(SoftFloat32_16).GetMethod(nameof(SoftFloat32_16.FPConvert)); @@ -150,7 +150,7 @@ namespace ARMeilleure.Instructions } else /* if (sizeF == 1) */ { - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(OperandType.FP32), part + index); + Operand ne = context.VectorExtract(OperandType.FP32, GetVec(op.Rn), part + index); Operand e = context.ConvertToFP(OperandType.FP64, ne); @@ -209,7 +209,7 @@ namespace ARMeilleure.Instructions for (int index = 0; index < elems; index++) { - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(type), 0); + Operand ne = context.VectorExtract(type, GetVec(op.Rn), 0); if (sizeF == 0) { @@ -389,6 +389,11 @@ namespace ARMeilleure.Instructions Operand res = GetIntOrZR(context, op.Rn); + if (op.RegisterSize == RegisterSize.Int32) + { + res = context.SignExtend32(OperandType.I64, res); + } + res = EmitFPConvert(context, res, op.Size, signed: true); context.Copy(GetVec(op.Rd), context.VectorInsert(context.VectorZero(), res, 0)); @@ -400,6 +405,11 @@ namespace ARMeilleure.Instructions Operand res = GetIntOrZR(context, op.Rn); + if (op.RegisterSize == RegisterSize.Int32) + { + res = context.SignExtend32(OperandType.I64, res); + } + res = EmitFPConvert(context, res, op.Size, signed: true); res = EmitI2fFBitsMul(context, res, op.FBits); @@ -554,7 +564,7 @@ namespace ARMeilleure.Instructions for (int index = 0; index < elems; index++) { - Operand ne = context.VectorExtract(n, Local(type), index); + Operand ne = context.VectorExtract(type, n, index); Operand e = EmitRoundMathCall(context, MidpointRounding.ToEven, ne); @@ -606,7 +616,7 @@ namespace ARMeilleure.Instructions for (int index = 0; index < elems; index++) { - Operand ne = context.VectorExtract(n, Local(type), index); + Operand ne = context.VectorExtract(type, n, index); Operand e = EmitF2iFBitsMul(context, ne, fBits); @@ -655,7 +665,7 @@ namespace ARMeilleure.Instructions OperandType type = op.Size == 0 ? OperandType.FP32 : OperandType.FP64; - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(type), 0); + Operand ne = context.VectorExtract(type, GetVec(op.Rn), 0); Operand res = signed ? EmitScalarFcvts(context, emit(ne), 0) @@ -680,7 +690,7 @@ namespace ARMeilleure.Instructions OperandType type = op.Size == 0 ? OperandType.FP32 : OperandType.FP64; - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(type), 0); + Operand ne = context.VectorExtract(type, GetVec(op.Rn), 0); Operand res = signed ? EmitScalarFcvts(context, ne, op.FBits) @@ -1156,9 +1166,9 @@ namespace ARMeilleure.Instructions private static Operand EmitVectorLongExtract(EmitterContext context, int reg, int index, int size) { - Operand res = Local(size == 3 ? OperandType.I64 : OperandType.I32); + OperandType type = size == 3 ? OperandType.I64 : OperandType.I32; - return context.VectorExtract(GetVec(reg), res, index); + return context.VectorExtract(type, GetVec(reg), index); } private static Operand EmitVectorLongCreate(EmitterContext context, Operand low, Operand high) diff --git a/ARMeilleure/Instructions/InstEmitSimdHash.cs b/ARMeilleure/Instructions/InstEmitSimdHash.cs index 91f90a0866..734d00fca6 100644 --- a/ARMeilleure/Instructions/InstEmitSimdHash.cs +++ b/ARMeilleure/Instructions/InstEmitSimdHash.cs @@ -17,7 +17,7 @@ namespace ARMeilleure.Instructions Operand d = GetVec(op.Rd); - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(OperandType.I32), 0); + Operand ne = context.VectorExtract(OperandType.I32, GetVec(op.Rn), 0); Operand m = GetVec(op.Rm); @@ -32,7 +32,7 @@ namespace ARMeilleure.Instructions { OpCodeSimd op = (OpCodeSimd)context.CurrOp; - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(OperandType.I32), 0); + Operand ne = context.VectorExtract(OperandType.I32, GetVec(op.Rn), 0); MethodInfo info = typeof(SoftFallback).GetMethod(nameof(SoftFallback.FixedRotate)); @@ -47,7 +47,7 @@ namespace ARMeilleure.Instructions Operand d = GetVec(op.Rd); - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(OperandType.I32), 0); + Operand ne = context.VectorExtract(OperandType.I32, GetVec(op.Rn), 0); Operand m = GetVec(op.Rm); @@ -64,7 +64,7 @@ namespace ARMeilleure.Instructions Operand d = GetVec(op.Rd); - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(OperandType.I32), 0); + Operand ne = context.VectorExtract(OperandType.I32, GetVec(op.Rn), 0); Operand m = GetVec(op.Rm); diff --git a/ARMeilleure/Instructions/InstEmitSimdHelper.cs b/ARMeilleure/Instructions/InstEmitSimdHelper.cs index 17fecbcee6..7dae668ae1 100644 --- a/ARMeilleure/Instructions/InstEmitSimdHelper.cs +++ b/ARMeilleure/Instructions/InstEmitSimdHelper.cs @@ -375,8 +375,8 @@ namespace ARMeilleure.Instructions OperandType type = (op.Size & 1) != 0 ? OperandType.FP64 : OperandType.FP32; - Operand n = context.VectorExtract(GetVec(op.Rn), Local(type), 0); - Operand m = context.VectorExtract(GetVec(op.Rm), Local(type), op.Index); + Operand n = context.VectorExtract(type, GetVec(op.Rn), 0); + Operand m = context.VectorExtract(type, GetVec(op.Rm), op.Index); context.Copy(GetVec(op.Rd), context.VectorInsert(context.VectorZero(), emit(n, m), 0)); } @@ -387,9 +387,9 @@ namespace ARMeilleure.Instructions OperandType type = (op.Size & 1) != 0 ? OperandType.FP64 : OperandType.FP32; - Operand d = context.VectorExtract(GetVec(op.Rd), Local(type), 0); - Operand n = context.VectorExtract(GetVec(op.Rn), Local(type), 0); - Operand m = context.VectorExtract(GetVec(op.Rm), Local(type), op.Index); + Operand d = context.VectorExtract(type, GetVec(op.Rd), 0); + Operand n = context.VectorExtract(type, GetVec(op.Rn), 0); + Operand m = context.VectorExtract(type, GetVec(op.Rm), op.Index); context.Copy(GetVec(op.Rd), context.VectorInsert(context.VectorZero(), emit(d, n, m), 0)); } @@ -459,7 +459,7 @@ namespace ARMeilleure.Instructions OperandType type = (op.Size & 1) != 0 ? OperandType.FP64 : OperandType.FP32; - Operand n = context.VectorExtract(GetVec(op.Rn), Local(type), 0); + Operand n = context.VectorExtract(type, GetVec(op.Rn), 0); context.Copy(GetVec(op.Rd), context.VectorInsert(context.VectorZero(), emit(n), 0)); } @@ -470,8 +470,8 @@ namespace ARMeilleure.Instructions OperandType type = (op.Size & 1) != 0 ? OperandType.FP64 : OperandType.FP32; - Operand n = context.VectorExtract(GetVec(op.Rn), Local(type), 0); - Operand m = context.VectorExtract(GetVec(op.Rm), Local(type), 0); + Operand n = context.VectorExtract(type, GetVec(op.Rn), 0); + Operand m = context.VectorExtract(type, GetVec(op.Rm), 0); context.Copy(GetVec(op.Rd), context.VectorInsert(context.VectorZero(), emit(n, m), 0)); } @@ -482,9 +482,9 @@ namespace ARMeilleure.Instructions OperandType type = (op.Size & 1) != 0 ? OperandType.FP64 : OperandType.FP32; - Operand a = context.VectorExtract(GetVec(op.Ra), Local(type), 0); - Operand n = context.VectorExtract(GetVec(op.Rn), Local(type), 0); - Operand m = context.VectorExtract(GetVec(op.Rm), Local(type), 0); + Operand a = context.VectorExtract(type, GetVec(op.Ra), 0); + Operand n = context.VectorExtract(type, GetVec(op.Rn), 0); + Operand m = context.VectorExtract(type, GetVec(op.Rm), 0); context.Copy(GetVec(op.Rd), context.VectorInsert(context.VectorZero(), emit(a, n, m), 0)); } @@ -503,7 +503,7 @@ namespace ARMeilleure.Instructions for (int index = 0; index < elems; index++) { - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(type), index); + Operand ne = context.VectorExtract(type, GetVec(op.Rn), index); res = context.VectorInsert(res, emit(ne), index); } @@ -525,8 +525,8 @@ namespace ARMeilleure.Instructions for (int index = 0; index < elems; index++) { - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(type), index); - Operand me = context.VectorExtract(GetVec(op.Rm), Local(type), index); + Operand ne = context.VectorExtract(type, GetVec(op.Rn), index); + Operand me = context.VectorExtract(type, GetVec(op.Rm), index); res = context.VectorInsert(res, emit(ne, me), index); } @@ -548,9 +548,9 @@ namespace ARMeilleure.Instructions for (int index = 0; index < elems; index++) { - Operand de = context.VectorExtract(GetVec(op.Rd), Local(type), index); - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(type), index); - Operand me = context.VectorExtract(GetVec(op.Rm), Local(type), index); + Operand de = context.VectorExtract(type, GetVec(op.Rd), index); + Operand ne = context.VectorExtract(type, GetVec(op.Rn), index); + Operand me = context.VectorExtract(type, GetVec(op.Rm), index); res = context.VectorInsert(res, emit(de, ne, me), index); } @@ -572,8 +572,8 @@ namespace ARMeilleure.Instructions for (int index = 0; index < elems; index++) { - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(type), index); - Operand me = context.VectorExtract(GetVec(op.Rm), Local(type), op.Index); + Operand ne = context.VectorExtract(type, GetVec(op.Rn), index); + Operand me = context.VectorExtract(type, GetVec(op.Rm), op.Index); res = context.VectorInsert(res, emit(ne, me), index); } @@ -595,9 +595,9 @@ namespace ARMeilleure.Instructions for (int index = 0; index < elems; index++) { - Operand de = context.VectorExtract(GetVec(op.Rd), Local(type), index); - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(type), index); - Operand me = context.VectorExtract(GetVec(op.Rm), Local(type), op.Index); + Operand de = context.VectorExtract(type, GetVec(op.Rd), index); + Operand ne = context.VectorExtract(type, GetVec(op.Rn), index); + Operand me = context.VectorExtract(type, GetVec(op.Rm), op.Index); res = context.VectorInsert(res, emit(de, ne, me), index); } @@ -1074,11 +1074,11 @@ namespace ARMeilleure.Instructions { int pairIndex = index << 1; - Operand n0 = context.VectorExtract(GetVec(op.Rn), Local(type), pairIndex); - Operand n1 = context.VectorExtract(GetVec(op.Rn), Local(type), pairIndex + 1); + Operand n0 = context.VectorExtract(type, GetVec(op.Rn), pairIndex); + Operand n1 = context.VectorExtract(type, GetVec(op.Rn), pairIndex + 1); - Operand m0 = context.VectorExtract(GetVec(op.Rm), Local(type), pairIndex); - Operand m1 = context.VectorExtract(GetVec(op.Rm), Local(type), pairIndex + 1); + Operand m0 = context.VectorExtract(type, GetVec(op.Rm), pairIndex); + Operand m1 = context.VectorExtract(type, GetVec(op.Rm), pairIndex + 1); res = context.VectorInsert(res, emit(n0, n1), index); res = context.VectorInsert(res, emit(m0, m1), pairs + index); @@ -1416,14 +1416,25 @@ namespace ARMeilleure.Instructions { ThrowIfInvalid(index, size); - Operand res = Local(size == 3 ? OperandType.I64 : OperandType.I32); + Operand res = null; switch (size) { - case 0: context.VectorExtract8 (GetVec(reg), res, index); break; - case 1: context.VectorExtract16(GetVec(reg), res, index); break; - case 2: context.VectorExtract (GetVec(reg), res, index); break; - case 3: context.VectorExtract (GetVec(reg), res, index); break; + case 0: + res = context.VectorExtract8(GetVec(reg), index); + break; + + case 1: + res = context.VectorExtract16(GetVec(reg), index); + break; + + case 2: + res = context.VectorExtract(OperandType.I32, GetVec(reg), index); + break; + + case 3: + res = context.VectorExtract(OperandType.I64, GetVec(reg), index); + break; } if (signed) diff --git a/ARMeilleure/Instructions/InstEmitSimdMove.cs b/ARMeilleure/Instructions/InstEmitSimdMove.cs index e6feaad98d..d22a74f9de 100644 --- a/ARMeilleure/Instructions/InstEmitSimdMove.cs +++ b/ARMeilleure/Instructions/InstEmitSimdMove.cs @@ -229,7 +229,7 @@ namespace ARMeilleure.Instructions OperandType type = op.Size == 0 ? OperandType.FP32 : OperandType.FP64; - Operand me = context.VectorExtract(GetVec(op.Rm), Local(type), 0); + Operand me = context.VectorExtract(type, GetVec(op.Rm), 0); context.Copy(GetVec(op.Rd), context.VectorInsert(context.VectorZero(), me, 0)); @@ -237,7 +237,7 @@ namespace ARMeilleure.Instructions context.MarkLabel(lblTrue); - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(type), 0); + Operand ne = context.VectorExtract(type, GetVec(op.Rn), 0); context.Copy(GetVec(op.Rd), context.VectorInsert(context.VectorZero(), ne, 0)); @@ -286,7 +286,7 @@ namespace ARMeilleure.Instructions OperandType type = op.Size == 0 ? OperandType.FP32 : OperandType.FP64; - Operand ne = context.VectorExtract(GetVec(op.Rn), Local(type), 0); + Operand ne = context.VectorExtract(type, GetVec(op.Rn), 0); context.Copy(GetVec(op.Rd), context.VectorInsert(context.VectorZero(), ne, 0)); } diff --git a/ARMeilleure/Instructions/NativeInterface.cs b/ARMeilleure/Instructions/NativeInterface.cs index 0e802e4b8f..69e594eedc 100644 --- a/ARMeilleure/Instructions/NativeInterface.cs +++ b/ARMeilleure/Instructions/NativeInterface.cs @@ -201,7 +201,7 @@ namespace ARMeilleure.Instructions { ThreadContext context = GetCurrentContext(); - V128 value = context.Memory.ReadVector128((long)address); + V128 value = context.Memory.AtomicLoadInt128((long)address); context.ExclusiveAddress = GetMaskedExclusiveAddress(address); context.ExclusiveValueLow = value.GetUInt64(0); @@ -335,12 +335,9 @@ namespace ARMeilleure.Instructions if (success) { - success = context.Memory.AtomicCompareExchangeInt128( - (long)address, - context.ExclusiveValueLow, - context.ExclusiveValueHigh, - value.GetUInt64(0), - value.GetUInt64(1)); + V128 expected = new V128(context.ExclusiveValueLow, context.ExclusiveValueHigh); + + success = context.Memory.AtomicCompareExchangeInt128((long)address, expected, value); if (success) { diff --git a/ARMeilleure/IntermediateRepresentation/Instruction.cs b/ARMeilleure/IntermediateRepresentation/Instruction.cs index 8e86cc74c5..130dc0e3e8 100644 --- a/ARMeilleure/IntermediateRepresentation/Instruction.cs +++ b/ARMeilleure/IntermediateRepresentation/Instruction.cs @@ -12,6 +12,7 @@ namespace ARMeilleure.IntermediateRepresentation BranchIfTrue, ByteSwap, Call, + CompareAndSwap128, CompareEqual, CompareGreater, CompareGreaterOrEqual, @@ -32,12 +33,10 @@ namespace ARMeilleure.IntermediateRepresentation DivideUI, Fill, Load, + Load16, + Load8, + LoadArgument, LoadFromContext, - LoadSx16, - LoadSx32, - LoadSx8, - LoadZx16, - LoadZx8, Multiply, Multiply64HighSI, Multiply64HighUI, @@ -213,26 +212,6 @@ namespace ARMeilleure.IntermediateRepresentation static class InstructionExtensions { - public static bool IsComparison(this Instruction inst) - { - switch (inst) - { - case Instruction.CompareEqual: - case Instruction.CompareGreater: - case Instruction.CompareGreaterOrEqual: - case Instruction.CompareGreaterOrEqualUI: - case Instruction.CompareGreaterUI: - case Instruction.CompareLess: - case Instruction.CompareLessOrEqual: - case Instruction.CompareLessOrEqualUI: - case Instruction.CompareLessUI: - case Instruction.CompareNotEqual: - return true; - } - - return false; - } - public static bool IsShift(this Instruction inst) { switch (inst) diff --git a/ARMeilleure/Memory/MemoryManagementUnix.cs b/ARMeilleure/Memory/MemoryManagementUnix.cs index 2bb98f7808..3331fb428f 100644 --- a/ARMeilleure/Memory/MemoryManagementUnix.cs +++ b/ARMeilleure/Memory/MemoryManagementUnix.cs @@ -41,11 +41,12 @@ namespace ARMeilleure.Memory { switch (protection) { - case Memory.MemoryProtection.None: return MmapProts.PROT_NONE; - case Memory.MemoryProtection.Read: return MmapProts.PROT_READ; - case Memory.MemoryProtection.ReadAndWrite: return MmapProts.PROT_READ | MmapProts.PROT_WRITE; - case Memory.MemoryProtection.ReadAndExecute: return MmapProts.PROT_READ | MmapProts.PROT_EXEC; - case Memory.MemoryProtection.Execute: return MmapProts.PROT_EXEC; + case Memory.MemoryProtection.None: return MmapProts.PROT_NONE; + case Memory.MemoryProtection.Read: return MmapProts.PROT_READ; + case Memory.MemoryProtection.ReadAndWrite: return MmapProts.PROT_READ | MmapProts.PROT_WRITE; + case Memory.MemoryProtection.ReadAndExecute: return MmapProts.PROT_READ | MmapProts.PROT_EXEC; + case Memory.MemoryProtection.ReadWriteExecute: return MmapProts.PROT_READ | MmapProts.PROT_WRITE | MmapProts.PROT_EXEC; + case Memory.MemoryProtection.Execute: return MmapProts.PROT_EXEC; default: throw new ArgumentException($"Invalid permission \"{protection}\"."); } diff --git a/ARMeilleure/Memory/MemoryManagementWindows.cs b/ARMeilleure/Memory/MemoryManagementWindows.cs index dfcaca8a2e..c1a84c95b9 100644 --- a/ARMeilleure/Memory/MemoryManagementWindows.cs +++ b/ARMeilleure/Memory/MemoryManagementWindows.cs @@ -115,11 +115,12 @@ namespace ARMeilleure.Memory { switch (protection) { - case Memory.MemoryProtection.None: return MemoryProtection.NoAccess; - case Memory.MemoryProtection.Read: return MemoryProtection.ReadOnly; - case Memory.MemoryProtection.ReadAndWrite: return MemoryProtection.ReadWrite; - case Memory.MemoryProtection.ReadAndExecute: return MemoryProtection.ExecuteRead; - case Memory.MemoryProtection.Execute: return MemoryProtection.Execute; + case Memory.MemoryProtection.None: return MemoryProtection.NoAccess; + case Memory.MemoryProtection.Read: return MemoryProtection.ReadOnly; + case Memory.MemoryProtection.ReadAndWrite: return MemoryProtection.ReadWrite; + case Memory.MemoryProtection.ReadAndExecute: return MemoryProtection.ExecuteRead; + case Memory.MemoryProtection.ReadWriteExecute: return MemoryProtection.ExecuteReadWrite; + case Memory.MemoryProtection.Execute: return MemoryProtection.Execute; default: throw new ArgumentException($"Invalid permission \"{protection}\"."); } diff --git a/ARMeilleure/Memory/MemoryManager.cs b/ARMeilleure/Memory/MemoryManager.cs index 98390ab063..8cc36fb1c9 100644 --- a/ARMeilleure/Memory/MemoryManager.cs +++ b/ARMeilleure/Memory/MemoryManager.cs @@ -398,23 +398,19 @@ namespace ARMeilleure.Memory return (ulong)position < (ulong)AddressSpaceSize; } - internal bool AtomicCompareExchange2xInt32( - long position, - int expectedLow, - int expectedHigh, - int desiredLow, - int desiredHigh) + internal V128 AtomicLoadInt128(long position) { - long expected = (uint)expectedLow; - long desired = (uint)desiredLow; + if ((position & 0xf) != 0) + { + AbortWithAlignmentFault(position); + } - expected |= (long)expectedHigh << 32; - desired |= (long)desiredHigh << 32; + IntPtr ptr = TranslateWrite(position); - return AtomicCompareExchangeInt64(position, expected, desired); + return MemoryManagerPal.AtomicLoad128(ptr); } - public bool AtomicCompareExchangeByte(long position, byte expected, byte desired) + internal bool AtomicCompareExchangeByte(long position, byte expected, byte desired) { int* ptr = (int*)Translate(position); @@ -426,7 +422,7 @@ namespace ARMeilleure.Memory return Interlocked.CompareExchange(ref *ptr, desired32, expected32) == expected32; } - public bool AtomicCompareExchangeInt16(long position, short expected, short desired) + internal bool AtomicCompareExchangeInt16(long position, short expected, short desired) { if ((position & 1) != 0) { @@ -455,7 +451,7 @@ namespace ARMeilleure.Memory return Interlocked.CompareExchange(ref *ptr, desired, expected) == expected; } - public bool AtomicCompareExchangeInt64(long position, long expected, long desired) + internal bool AtomicCompareExchangeInt64(long position, long expected, long desired) { if ((position & 7) != 0) { @@ -467,12 +463,7 @@ namespace ARMeilleure.Memory return Interlocked.CompareExchange(ref *ptr, desired, expected) == expected; } - internal bool AtomicCompareExchangeInt128( - long position, - ulong expectedLow, - ulong expectedHigh, - ulong desiredLow, - ulong desiredHigh) + internal bool AtomicCompareExchangeInt128(long position, V128 expected, V128 desired) { if ((position & 0xf) != 0) { @@ -481,7 +472,7 @@ namespace ARMeilleure.Memory IntPtr ptr = TranslateWrite(position); - throw new NotImplementedException(); + return MemoryManagerPal.CompareAndSwap128(ptr, expected, desired) == expected; } public int AtomicIncrementInt32(long position) diff --git a/ARMeilleure/Memory/MemoryManagerPal.cs b/ARMeilleure/Memory/MemoryManagerPal.cs new file mode 100644 index 0000000000..b318da8648 --- /dev/null +++ b/ARMeilleure/Memory/MemoryManagerPal.cs @@ -0,0 +1,66 @@ +using ARMeilleure.IntermediateRepresentation; +using ARMeilleure.State; +using ARMeilleure.Translation; +using System; + +namespace ARMeilleure.Memory +{ + static class MemoryManagerPal + { + private delegate V128 CompareExchange128(IntPtr address, V128 expected, V128 desired); + + private static CompareExchange128 _compareExchange128; + + private static object _lock; + + static MemoryManagerPal() + { + _lock = new object(); + } + + public static V128 AtomicLoad128(IntPtr address) + { + return GetCompareAndSwap128()(address, V128.Zero, V128.Zero); + } + + public static V128 CompareAndSwap128(IntPtr address, V128 expected, V128 desired) + { + return GetCompareAndSwap128()(address, expected, desired); + } + + private static CompareExchange128 GetCompareAndSwap128() + { + if (_compareExchange128 == null) + { + GenerateCompareAndSwap128(); + } + + return _compareExchange128; + } + + private static void GenerateCompareAndSwap128() + { + lock (_lock) + { + if (_compareExchange128 != null) + { + return; + } + + EmitterContext context = new EmitterContext(); + + Operand address = context.LoadArgument(OperandType.I64, 0); + Operand expected = context.LoadArgument(OperandType.V128, 1); + Operand desired = context.LoadArgument(OperandType.V128, 2); + + Operand result = context.CompareAndSwap128(address, expected, desired); + + context.Return(result); + + ControlFlowGraph cfg = context.GetControlFlowGraph(); + + _compareExchange128 = Compiler.Compile(cfg, OperandType.V128); + } + } + } +} \ No newline at end of file diff --git a/ARMeilleure/Memory/MemoryProtection.cs b/ARMeilleure/Memory/MemoryProtection.cs index a180d7333d..6bc16f8ea1 100644 --- a/ARMeilleure/Memory/MemoryProtection.cs +++ b/ARMeilleure/Memory/MemoryProtection.cs @@ -10,7 +10,8 @@ namespace ARMeilleure.Memory Write = 1 << 1, Execute = 1 << 2, - ReadAndWrite = Read | Write, - ReadAndExecute = Read | Execute + ReadAndWrite = Read | Write, + ReadAndExecute = Read | Execute, + ReadWriteExecute = Read | Write | Execute } } \ No newline at end of file diff --git a/ARMeilleure/State/V128.cs b/ARMeilleure/State/V128.cs index 412df9ae24..eeb9ff1ca3 100644 --- a/ARMeilleure/State/V128.cs +++ b/ARMeilleure/State/V128.cs @@ -7,6 +7,10 @@ namespace ARMeilleure.State private ulong _e0; private ulong _e1; + private static V128 _zero = new V128(0, 0); + + public static V128 Zero => _zero; + public V128(float value) : this(value, 0, 0, 0) { } public V128(double value) : this(value, 0) { } diff --git a/ARMeilleure/Translation/Compiler.cs b/ARMeilleure/Translation/Compiler.cs new file mode 100644 index 0000000000..a279563458 --- /dev/null +++ b/ARMeilleure/Translation/Compiler.cs @@ -0,0 +1,39 @@ +using ARMeilleure.CodeGen; +using ARMeilleure.CodeGen.X86; +using ARMeilleure.Diagnostics; +using ARMeilleure.IntermediateRepresentation; +using System; +using System.Runtime.InteropServices; + +namespace ARMeilleure.Translation +{ + static class Compiler + { + public static T Compile(ControlFlowGraph cfg, OperandType funcReturnType) + { + IntPtr codePtr = JitCache.Map(Compile(cfg, funcReturnType)); + + return Marshal.GetDelegateForFunctionPointer(codePtr); + } + + public static CompiledFunction Compile(ControlFlowGraph cfg, OperandType funcReturnType) + { + Logger.StartPass(PassName.Dominance); + + Dominance.FindDominators(cfg); + Dominance.FindDominanceFrontiers(cfg); + + Logger.EndPass(PassName.Dominance); + + Logger.StartPass(PassName.SsaConstruction); + + Ssa.Rename(cfg); + + Logger.EndPass(PassName.SsaConstruction, cfg); + + CompilerContext cctx = new CompilerContext(cfg, funcReturnType); + + return CodeGenerator.Generate(cctx); + } + } +} \ No newline at end of file diff --git a/ARMeilleure/Translation/CompilerContext.cs b/ARMeilleure/Translation/CompilerContext.cs new file mode 100644 index 0000000000..3afa51bc0b --- /dev/null +++ b/ARMeilleure/Translation/CompilerContext.cs @@ -0,0 +1,17 @@ +using ARMeilleure.IntermediateRepresentation; + +namespace ARMeilleure.Translation +{ + struct CompilerContext + { + public ControlFlowGraph Cfg { get; } + + public OperandType FuncReturnType { get; } + + public CompilerContext(ControlFlowGraph cfg, OperandType funcReturnType) + { + Cfg = cfg; + FuncReturnType = funcReturnType; + } + } +} \ No newline at end of file diff --git a/ARMeilleure/Translation/EmitterContext.cs b/ARMeilleure/Translation/EmitterContext.cs index a2d5551d2a..cee396ce38 100644 --- a/ARMeilleure/Translation/EmitterContext.cs +++ b/ARMeilleure/Translation/EmitterContext.cs @@ -41,29 +41,29 @@ namespace ARMeilleure.Translation _needsNewBlock = true; } - public Operand Add(Operand a, Operand b) + public Operand Add(Operand op1, Operand op2) { - return Add(Instruction.Add, Local(a.Type), a, b); + return Add(Instruction.Add, Local(op1.Type), op1, op2); } - public Operand BitwiseAnd(Operand a, Operand b) + public Operand BitwiseAnd(Operand op1, Operand op2) { - return Add(Instruction.BitwiseAnd, Local(a.Type), a, b); + return Add(Instruction.BitwiseAnd, Local(op1.Type), op1, op2); } - public Operand BitwiseExclusiveOr(Operand a, Operand b) + public Operand BitwiseExclusiveOr(Operand op1, Operand op2) { - return Add(Instruction.BitwiseExclusiveOr, Local(a.Type), a, b); + return Add(Instruction.BitwiseExclusiveOr, Local(op1.Type), op1, op2); } - public Operand BitwiseNot(Operand a) + public Operand BitwiseNot(Operand op1) { - return Add(Instruction.BitwiseNot, Local(a.Type), a); + return Add(Instruction.BitwiseNot, Local(op1.Type), op1); } - public Operand BitwiseOr(Operand a, Operand b) + public Operand BitwiseOr(Operand op1, Operand op2) { - return Add(Instruction.BitwiseOr, Local(a.Type), a, b); + return Add(Instruction.BitwiseOr, Local(op1.Type), op1, op2); } public void Branch(Operand label) @@ -73,23 +73,23 @@ namespace ARMeilleure.Translation BranchToLabel(label); } - public void BranchIfFalse(Operand label, Operand a) + public void BranchIfFalse(Operand label, Operand op1) { - Add(Instruction.BranchIfFalse, null, a); + Add(Instruction.BranchIfFalse, null, op1); BranchToLabel(label); } - public void BranchIfTrue(Operand label, Operand a) + public void BranchIfTrue(Operand label, Operand op1) { - Add(Instruction.BranchIfTrue, null, a); + Add(Instruction.BranchIfTrue, null, op1); BranchToLabel(label); } - public Operand ByteSwap(Operand a) + public Operand ByteSwap(Operand op1) { - return Add(Instruction.ByteSwap, Local(a.Type), a); + return Add(Instruction.ByteSwap, Local(op1.Type), op1); } public Operand Call(MethodInfo info, params Operand[] callArgs) @@ -154,109 +154,124 @@ namespace ARMeilleure.Translation return Add(Instruction.Call, Local(returnType), callArgs); } - public Operand ConditionalSelect(Operand a, Operand b, Operand c) + public Operand CompareAndSwap128(Operand address, Operand expected, Operand desired) { - return Add(Instruction.ConditionalSelect, Local(b.Type), a, b, c); + return Add(Instruction.CompareAndSwap128, Local(OperandType.V128), address, expected, desired); } - public Operand ConvertI64ToI32(Operand a) + public Operand ConditionalSelect(Operand op1, Operand op2, Operand op3) { - return Add(Instruction.ConvertI64ToI32, Local(OperandType.I32), a); + return Add(Instruction.ConditionalSelect, Local(op2.Type), op1, op2, op3); } - public Operand ConvertToFP(OperandType type, Operand a) + public Operand ConvertI64ToI32(Operand op1) { - return Add(Instruction.ConvertToFP, Local(type), a); + return Add(Instruction.ConvertI64ToI32, Local(OperandType.I32), op1); } - public Operand ConvertToFPUI(OperandType type, Operand a) + public Operand ConvertToFP(OperandType type, Operand op1) { - return Add(Instruction.ConvertToFPUI, Local(type), a); + return Add(Instruction.ConvertToFP, Local(type), op1); } - public Operand Copy(Operand a) + public Operand ConvertToFPUI(OperandType type, Operand op1) { - return Add(Instruction.Copy, Local(a.Type), a); + return Add(Instruction.ConvertToFPUI, Local(type), op1); } - public Operand Copy(Operand d, Operand a) + public Operand Copy(Operand op1) { - return Add(Instruction.Copy, d, a); + return Add(Instruction.Copy, Local(op1.Type), op1); } - public Operand CountLeadingZeros(Operand a) + public Operand Copy(Operand dest, Operand op1) { - return Add(Instruction.CountLeadingZeros, Local(a.Type), a); + return Add(Instruction.Copy, dest, op1); } - public Operand Divide(Operand a, Operand b) + public Operand CountLeadingZeros(Operand op1) { - return Add(Instruction.Divide, Local(a.Type), a, b); + return Add(Instruction.CountLeadingZeros, Local(op1.Type), op1); } - public Operand DivideUI(Operand a, Operand b) + public Operand Divide(Operand op1, Operand op2) { - return Add(Instruction.DivideUI, Local(a.Type), a, b); + return Add(Instruction.Divide, Local(op1.Type), op1, op2); } - public Operand ICompareEqual(Operand a, Operand b) + public Operand DivideUI(Operand op1, Operand op2) { - return Add(Instruction.CompareEqual, Local(OperandType.I32), a, b); + return Add(Instruction.DivideUI, Local(op1.Type), op1, op2); } - public Operand ICompareGreater(Operand a, Operand b) + public Operand ICompareEqual(Operand op1, Operand op2) { - return Add(Instruction.CompareGreater, Local(OperandType.I32), a, b); + return Add(Instruction.CompareEqual, Local(OperandType.I32), op1, op2); } - public Operand ICompareGreaterOrEqual(Operand a, Operand b) + public Operand ICompareGreater(Operand op1, Operand op2) { - return Add(Instruction.CompareGreaterOrEqual, Local(OperandType.I32), a, b); + return Add(Instruction.CompareGreater, Local(OperandType.I32), op1, op2); } - public Operand ICompareGreaterOrEqualUI(Operand a, Operand b) + public Operand ICompareGreaterOrEqual(Operand op1, Operand op2) { - return Add(Instruction.CompareGreaterOrEqualUI, Local(OperandType.I32), a, b); + return Add(Instruction.CompareGreaterOrEqual, Local(OperandType.I32), op1, op2); } - public Operand ICompareGreaterUI(Operand a, Operand b) + public Operand ICompareGreaterOrEqualUI(Operand op1, Operand op2) { - return Add(Instruction.CompareGreaterUI, Local(OperandType.I32), a, b); + return Add(Instruction.CompareGreaterOrEqualUI, Local(OperandType.I32), op1, op2); } - public Operand ICompareLess(Operand a, Operand b) + public Operand ICompareGreaterUI(Operand op1, Operand op2) { - return Add(Instruction.CompareLess, Local(OperandType.I32), a, b); + return Add(Instruction.CompareGreaterUI, Local(OperandType.I32), op1, op2); } - public Operand ICompareLessOrEqual(Operand a, Operand b) + public Operand ICompareLess(Operand op1, Operand op2) { - return Add(Instruction.CompareLessOrEqual, Local(OperandType.I32), a, b); + return Add(Instruction.CompareLess, Local(OperandType.I32), op1, op2); } - public Operand ICompareLessOrEqualUI(Operand a, Operand b) + public Operand ICompareLessOrEqual(Operand op1, Operand op2) { - return Add(Instruction.CompareLessOrEqualUI, Local(OperandType.I32), a, b); + return Add(Instruction.CompareLessOrEqual, Local(OperandType.I32), op1, op2); } - public Operand ICompareLessUI(Operand a, Operand b) + public Operand ICompareLessOrEqualUI(Operand op1, Operand op2) { - return Add(Instruction.CompareLessUI, Local(OperandType.I32), a, b); + return Add(Instruction.CompareLessOrEqualUI, Local(OperandType.I32), op1, op2); } - public Operand ICompareNotEqual(Operand a, Operand b) + public Operand ICompareLessUI(Operand op1, Operand op2) { - return Add(Instruction.CompareNotEqual, Local(OperandType.I32), a, b); + return Add(Instruction.CompareLessUI, Local(OperandType.I32), op1, op2); } - public Operand Load(Operand value, Operand address) + public Operand ICompareNotEqual(Operand op1, Operand op2) { - return Add(Instruction.Load, value, address); + return Add(Instruction.CompareNotEqual, Local(OperandType.I32), op1, op2); } - public Operand LoadFromContext(int offset) + public Operand Load(OperandType type, Operand address) { - return Add(Instruction.LoadFromContext, Local(OperandType.I32), Const(offset)); + return Add(Instruction.Load, Local(type), address); + } + + public Operand Load16(Operand address) + { + return Add(Instruction.Load16, Local(OperandType.I32), address); + } + + public Operand Load8(Operand address) + { + return Add(Instruction.Load8, Local(OperandType.I32), address); + } + + public Operand LoadArgument(OperandType type, int index) + { + return Add(Instruction.LoadArgument, Local(type), Const(index)); } public void LoadFromContext() @@ -266,114 +281,73 @@ namespace ARMeilleure.Translation Add(Instruction.LoadFromContext); } - public void StoreToContext(int offset, Operand value) + public Operand Multiply(Operand op1, Operand op2) { - Add(Instruction.StoreToContext, null, Const(offset), value); + return Add(Instruction.Multiply, Local(op1.Type), op1, op2); } - public void StoreToContext() + public Operand Multiply64HighSI(Operand op1, Operand op2) { - Add(Instruction.StoreToContext); + return Add(Instruction.Multiply64HighSI, Local(OperandType.I64), op1, op2); + } + + public Operand Multiply64HighUI(Operand op1, Operand op2) + { + return Add(Instruction.Multiply64HighUI, Local(OperandType.I64), op1, op2); + } + + public Operand Negate(Operand op1) + { + return Add(Instruction.Negate, Local(op1.Type), op1); + } + + public void Return() + { + Add(Instruction.Return); _needsNewBlock = true; } - public Operand LoadSx16(Operand value, Operand address) + public void Return(Operand op1) { - return Add(Instruction.LoadSx16, value, address); - } - - public Operand LoadSx32(Operand value, Operand address) - { - return Add(Instruction.LoadSx32, value, address); - } - - public Operand LoadSx8(Operand value, Operand address) - { - return Add(Instruction.LoadSx8, value, address); - } - - public Operand LoadZx16(Operand value, Operand address) - { - return Add(Instruction.LoadZx16, value, address); - } - - public Operand LoadZx8(Operand value, Operand address) - { - return Add(Instruction.LoadZx8, value, address); - } - - public Operand Multiply(Operand a, Operand b) - { - return Add(Instruction.Multiply, Local(a.Type), a, b); - } - - public Operand Multiply64HighSI(Operand a, Operand b) - { - return Add(Instruction.Multiply64HighSI, Local(OperandType.I64), a, b); - } - - public Operand Multiply64HighUI(Operand a, Operand b) - { - return Add(Instruction.Multiply64HighUI, Local(OperandType.I64), a, b); - } - - public Operand Negate(Operand a) - { - return Add(Instruction.Negate, Local(a.Type), a); - } - - public Operand Return() - { - Operand returnValue = Add(Instruction.Return); + Add(Instruction.Return, null, op1); _needsNewBlock = true; - - return returnValue; } - public Operand Return(Operand a) + public Operand RotateRight(Operand op1, Operand op2) { - Operand returnValue = Add(Instruction.Return, null, a); - - _needsNewBlock = true; - - return returnValue; + return Add(Instruction.RotateRight, Local(op1.Type), op1, op2); } - public Operand RotateRight(Operand a, Operand b) + public Operand ShiftLeft(Operand op1, Operand op2) { - return Add(Instruction.RotateRight, Local(a.Type), a, b); + return Add(Instruction.ShiftLeft, Local(op1.Type), op1, op2); } - public Operand ShiftLeft(Operand a, Operand b) + public Operand ShiftRightSI(Operand op1, Operand op2) { - return Add(Instruction.ShiftLeft, Local(a.Type), a, b); + return Add(Instruction.ShiftRightSI, Local(op1.Type), op1, op2); } - public Operand ShiftRightSI(Operand a, Operand b) + public Operand ShiftRightUI(Operand op1, Operand op2) { - return Add(Instruction.ShiftRightSI, Local(a.Type), a, b); + return Add(Instruction.ShiftRightUI, Local(op1.Type), op1, op2); } - public Operand ShiftRightUI(Operand a, Operand b) + public Operand SignExtend16(OperandType type, Operand op1) { - return Add(Instruction.ShiftRightUI, Local(a.Type), a, b); + return Add(Instruction.SignExtend16, Local(type), op1); } - public Operand SignExtend16(OperandType type, Operand a) + public Operand SignExtend32(OperandType type, Operand op1) { - return Add(Instruction.SignExtend16, Local(type), a); + return Add(Instruction.SignExtend32, Local(type), op1); } - public Operand SignExtend32(OperandType type, Operand a) + public Operand SignExtend8(OperandType type, Operand op1) { - return Add(Instruction.SignExtend32, Local(type), a); - } - - public Operand SignExtend8(OperandType type, Operand a) - { - return Add(Instruction.SignExtend8, Local(type), a); + return Add(Instruction.SignExtend8, Local(type), op1); } public void Store(Operand address, Operand value) @@ -391,24 +365,31 @@ namespace ARMeilleure.Translation Add(Instruction.Store8, null, address, value); } - public Operand Subtract(Operand a, Operand b) + public void StoreToContext() { - return Add(Instruction.Subtract, Local(a.Type), a, b); + Add(Instruction.StoreToContext); + + _needsNewBlock = true; } - public Operand VectorExtract(Operand vector, Operand value, int index) + public Operand Subtract(Operand op1, Operand op2) { - return Add(Instruction.VectorExtract, value, vector, Const(index)); + return Add(Instruction.Subtract, Local(op1.Type), op1, op2); } - public Operand VectorExtract16(Operand vector, Operand value, int index) + public Operand VectorExtract(OperandType type, Operand vector, int index) { - return Add(Instruction.VectorExtract16, value, vector, Const(index)); + return Add(Instruction.VectorExtract, Local(type), vector, Const(index)); } - public Operand VectorExtract8(Operand vector, Operand value, int index) + public Operand VectorExtract16(Operand vector, int index) { - return Add(Instruction.VectorExtract8, value, vector, Const(index)); + return Add(Instruction.VectorExtract16, Local(OperandType.I32), vector, Const(index)); + } + + public Operand VectorExtract8(Operand vector, int index) + { + return Add(Instruction.VectorExtract8, Local(OperandType.I32), vector, Const(index)); } public Operand VectorInsert(Operand vector, Operand value, int index) @@ -441,19 +422,19 @@ namespace ARMeilleure.Translation return Add(Instruction.VectorZeroUpper96, Local(OperandType.V128), vector); } - public Operand ZeroExtend16(OperandType type, Operand a) + public Operand ZeroExtend16(OperandType type, Operand op1) { - return Add(Instruction.ZeroExtend16, Local(type), a); + return Add(Instruction.ZeroExtend16, Local(type), op1); } - public Operand ZeroExtend32(OperandType type, Operand a) + public Operand ZeroExtend32(OperandType type, Operand op1) { - return Add(Instruction.ZeroExtend32, Local(type), a); + return Add(Instruction.ZeroExtend32, Local(type), op1); } - public Operand ZeroExtend8(OperandType type, Operand a) + public Operand ZeroExtend8(OperandType type, Operand op1) { - return Add(Instruction.ZeroExtend8, Local(type), a); + return Add(Instruction.ZeroExtend8, Local(type), op1); } public Operand AddIntrinsic(Instruction inst, params Operand[] args) diff --git a/ARMeilleure/Translation/JitCache.cs b/ARMeilleure/Translation/JitCache.cs new file mode 100644 index 0000000000..8cbfd8299f --- /dev/null +++ b/ARMeilleure/Translation/JitCache.cs @@ -0,0 +1,100 @@ +using ARMeilleure.CodeGen; +using ARMeilleure.Memory; +using System; +using System.Collections.Generic; + +namespace ARMeilleure.Translation +{ + static class JitCache + { + private const int PageSize = 4 * 1024; + private const int PageMask = PageSize - 1; + + private static uint CacheSize = 512 * 1024 * 1024; + + private static IntPtr _basePointer; + + private static int _offset; + + private static List _cacheEntries; + + static JitCache() + { + _basePointer = MemoryManagement.Allocate(CacheSize); + + JitUnwindWindows.InstallFunctionTableHandler(_basePointer, CacheSize); + + //The first page is used for the table based SEH structs. + _offset = PageSize; + + _cacheEntries = new List(); + } + + public static IntPtr Map(CompiledFunction func) + { + byte[] code = func.Code; + + int funcOffset = Allocate(code.Length); + + IntPtr funcPtr = _basePointer + funcOffset; + + unsafe + { + fixed (byte* codePtr = code) + { + byte* dest = (byte*)funcPtr; + + long size = (long)code.Length; + + Buffer.MemoryCopy(codePtr, dest, size, size); + } + } + + //TODO: W^X. + MemoryManagement.Reprotect(funcPtr, (ulong)code.Length, MemoryProtection.ReadWriteExecute); + + Add(new JitCacheEntry(funcOffset, code.Length, func.UnwindInfo)); + + return funcPtr; + } + + private static int Allocate(int codeSize) + { + int allocOffset = _offset; + + _offset += codeSize; + + if ((ulong)(uint)_offset > CacheSize) + { + throw new OutOfMemoryException(); + } + + return allocOffset; + } + + private static void Add(JitCacheEntry entry) + { + //TODO: Use concurrent collection. + _cacheEntries.Add(entry); + } + + public static bool TryFind(int offset, out JitCacheEntry entry) + { + foreach (JitCacheEntry cacheEntry in _cacheEntries) + { + int endOffset = cacheEntry.Offset + cacheEntry.Size; + + if (offset >= cacheEntry.Offset && offset < endOffset) + { + entry = cacheEntry; + + return true; + } + } + + entry = default(JitCacheEntry); + + return false; + } + } +} \ No newline at end of file diff --git a/ARMeilleure/Translation/JitCacheEntry.cs b/ARMeilleure/Translation/JitCacheEntry.cs new file mode 100644 index 0000000000..87d020e683 --- /dev/null +++ b/ARMeilleure/Translation/JitCacheEntry.cs @@ -0,0 +1,19 @@ +using ARMeilleure.CodeGen.Unwinding; + +namespace ARMeilleure.Translation +{ + struct JitCacheEntry + { + public int Offset { get; } + public int Size { get; } + + public UnwindInfo UnwindInfo { get; } + + public JitCacheEntry(int offset, int size, UnwindInfo unwindInfo) + { + Offset = offset; + Size = size; + UnwindInfo = unwindInfo; + } + } +} \ No newline at end of file diff --git a/ARMeilleure/Translation/JitUnwindWindows.cs b/ARMeilleure/Translation/JitUnwindWindows.cs new file mode 100644 index 0000000000..100525b185 --- /dev/null +++ b/ARMeilleure/Translation/JitUnwindWindows.cs @@ -0,0 +1,148 @@ +using ARMeilleure.IntermediateRepresentation; +using ARMeilleure.Memory; +using System; +using System.Runtime.InteropServices; + +namespace ARMeilleure.Translation +{ + static class JitUnwindWindows + { + private const int MaxUnwindCodesArraySize = 9 + 10 * 2 + 3; + + private struct RuntimeFunction + { + public uint BeginAddress; + public uint EndAddress; + public uint UnwindData; + } + + private struct UnwindInfo + { + public byte VersionAndFlags; + public byte SizeOfProlog; + public byte CountOfUnwindCodes; + public byte FrameRegister; + public unsafe fixed ushort UnwindCodes[MaxUnwindCodesArraySize]; + } + + private enum UnwindOperation + { + PushNonvol = 0, + AllocLarge = 1, + AllocSmall = 2, + SetFpreg = 3, + SaveNonvol = 4, + SaveNonvolFar = 5, + SaveXmm128 = 8, + SaveXmm128Far = 9, + PushMachframe = 10 + } + + private unsafe delegate RuntimeFunction* GetRuntimeFunctionCallback(ulong controlPc, IntPtr context); + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode)] + private static unsafe extern bool RtlInstallFunctionTableCallback( + ulong tableIdentifier, + ulong baseAddress, + uint length, + GetRuntimeFunctionCallback callback, + IntPtr context, + string outOfProcessCallbackDll); + + private static GetRuntimeFunctionCallback _getRuntimeFunctionCallback; + + private static int _sizeOfRuntimeFunction; + + private unsafe static RuntimeFunction* _runtimeFunction; + + private unsafe static UnwindInfo* _unwindInfo; + + public static void InstallFunctionTableHandler(IntPtr codeCachePointer, uint codeCacheLength) + { + ulong codeCachePtr = (ulong)codeCachePointer.ToInt64(); + + _sizeOfRuntimeFunction = Marshal.SizeOf(); + + bool result; + + unsafe + { + _runtimeFunction = (RuntimeFunction*)codeCachePointer; + + _unwindInfo = (UnwindInfo*)(codeCachePointer + _sizeOfRuntimeFunction); + + _getRuntimeFunctionCallback = new GetRuntimeFunctionCallback(FunctionTableHandler); + + result = RtlInstallFunctionTableCallback( + codeCachePtr | 3, + codeCachePtr, + codeCacheLength, + _getRuntimeFunctionCallback, + codeCachePointer, + null); + } + + if (!result) + { + throw new InvalidOperationException("Failure installing function table callback."); + } + } + + private static unsafe RuntimeFunction* FunctionTableHandler(ulong controlPc, IntPtr context) + { + int offset = (int)((long)controlPc - context.ToInt64()); + + if (!JitCache.TryFind(offset, out JitCacheEntry funcEntry)) + { + //Not found. + return null; + } + + var unwindInfo = funcEntry.UnwindInfo; + + _unwindInfo->UnwindCodes[0] = PackUwop(UnwindOperation.AllocLarge, unwindInfo.PrologueSize, 1); + _unwindInfo->UnwindCodes[1] = (ushort)(unwindInfo.FixedAllocSize >> 0); + _unwindInfo->UnwindCodes[2] = (ushort)(unwindInfo.FixedAllocSize >> 16); + + int codeIndex = 3; + + int spOffset = 0; + + for (int index = unwindInfo.PushEntries.Length - 1; index >= 0; index--) + { + var entry = unwindInfo.PushEntries[index]; + + bool isInteger = entry.Type == RegisterType.Integer; + + UnwindOperation uwop = isInteger + ? UnwindOperation.PushNonvol + : UnwindOperation.SaveXmm128; + + _unwindInfo->UnwindCodes[codeIndex++] = PackUwop(uwop, entry.StreamEndOffset, entry.Index); + + if (!isInteger) + { + _unwindInfo->UnwindCodes[codeIndex++] = (ushort)spOffset; + + spOffset -= 16; + } + } + + _unwindInfo->VersionAndFlags = 1; + _unwindInfo->SizeOfProlog = (byte)unwindInfo.PrologueSize; + _unwindInfo->CountOfUnwindCodes = (byte)codeIndex; + _unwindInfo->FrameRegister = 0; + + _runtimeFunction->BeginAddress = (uint)funcEntry.Offset; + _runtimeFunction->EndAddress = (uint)(funcEntry.Offset + funcEntry.Size); + _runtimeFunction->UnwindData = (uint)_sizeOfRuntimeFunction; + + return _runtimeFunction; + } + + private static ushort PackUwop(UnwindOperation uwop, int prologOffset, int opInfo) + { + return (ushort)(prologOffset | ((int)uwop << 8) | (opInfo << 12)); + } + } +} \ No newline at end of file diff --git a/ARMeilleure/Translation/RegisterUsage.cs b/ARMeilleure/Translation/RegisterUsage.cs index f30693c677..bc62c8d0ff 100644 --- a/ARMeilleure/Translation/RegisterUsage.cs +++ b/ARMeilleure/Translation/RegisterUsage.cs @@ -280,6 +280,8 @@ namespace ARMeilleure.Translation private static void LoadLocals(BasicBlock block, long inputs, RegisterType baseType) { + Operand arg0 = Local(OperandType.I64); + for (int bit = 63; bit >= 0; bit--) { long mask = 1L << bit; @@ -291,12 +293,22 @@ namespace ARMeilleure.Translation Operand dest = GetRegFromBit(bit, baseType); - int offset = NativeContext.GetRegisterOffset(dest.GetRegister()); + long offset = NativeContext.GetRegisterOffset(dest.GetRegister()); - Operation operation = new Operation(Instruction.LoadFromContext, dest, Const(offset)); + Operand addr = Local(OperandType.I64); - block.Operations.AddFirst(operation); + Operation loadOp = new Operation(Instruction.Load, dest, addr); + + block.Operations.AddFirst(loadOp); + + Operation calcOffsOp = new Operation(Instruction.Add, addr, arg0, Const(offset)); + + block.Operations.AddFirst(calcOffsOp); } + + Operation loadArg0 = new Operation(Instruction.LoadArgument, arg0, Const(0)); + + block.Operations.AddFirst(loadArg0); } private static void StoreLocals(BasicBlock block, long outputs, RegisterType baseType) @@ -313,6 +325,12 @@ namespace ARMeilleure.Translation } } + Operand arg0 = Local(OperandType.I64); + + Operation loadArg0 = new Operation(Instruction.LoadArgument, arg0, Const(0)); + + block.Append(loadArg0); + for (int bit = 0; bit < 64; bit++) { long mask = 1L << bit; @@ -324,11 +342,17 @@ namespace ARMeilleure.Translation Operand source = GetRegFromBit(bit, baseType); - int offset = NativeContext.GetRegisterOffset(source.GetRegister()); + long offset = NativeContext.GetRegisterOffset(source.GetRegister()); - Operation operation = new Operation(Instruction.StoreToContext, null, Const(offset), source); + Operand addr = Local(OperandType.I64); - block.Append(operation); + Operation calcOffsOp = new Operation(Instruction.Add, addr, arg0, Const(offset)); + + block.Append(calcOffsOp); + + Operation storeOp = new Operation(Instruction.Store, null, addr, source); + + block.Append(storeOp); } } diff --git a/ARMeilleure/Translation/Translator.cs b/ARMeilleure/Translation/Translator.cs index 0c219ba55c..1186709b48 100644 --- a/ARMeilleure/Translation/Translator.cs +++ b/ARMeilleure/Translation/Translator.cs @@ -1,4 +1,3 @@ -using ARMeilleure.CodeGen.X86; using ARMeilleure.Decoders; using ARMeilleure.Diagnostics; using ARMeilleure.Instructions; @@ -17,16 +16,12 @@ namespace ARMeilleure.Translation { private MemoryManager _memory; - private TranslatedCache _cache; - private ConcurrentDictionary _funcs; public Translator(MemoryManager memory) { _memory = memory; - _cache = new TranslatedCache(); - _funcs = new ConcurrentDictionary(); } @@ -73,8 +68,14 @@ namespace ARMeilleure.Translation context.Memory = _memory; + Logger.StartPass(PassName.Decoding); + Block[] blocks = Decoder.DecodeFunction(_memory, address, ExecutionMode.Aarch64); + Logger.EndPass(PassName.Decoding); + + Logger.StartPass(PassName.Translation); + EmitSynchronization(context); if (blocks[0].Address != address) @@ -84,21 +85,17 @@ namespace ARMeilleure.Translation ControlFlowGraph cfg = EmitAndGetCFG(context, blocks); + Logger.EndPass(PassName.Translation); + + Logger.StartPass(PassName.RegisterUsage); + RegisterUsage.RunPass(cfg); - Dominance.FindDominators(cfg); + Logger.EndPass(PassName.RegisterUsage); - Dominance.FindDominanceFrontiers(cfg); + GuestFunction func = Compiler.Compile(cfg, OperandType.I64); - Logger.StartPass(PassName.SsaConstruction); - - Ssa.Rename(cfg); - - Logger.EndPass(PassName.SsaConstruction, cfg); - - byte[] code = CodeGenerator.Generate(cfg, _memory); - - return _cache.CreateFunction(code); + return new TranslatedFunction(func); } private static ControlFlowGraph EmitAndGetCFG(EmitterContext context, Block[] blocks) @@ -164,11 +161,14 @@ namespace ARMeilleure.Translation private static void EmitSynchronization(EmitterContext context) { - int cntOffset = NativeContext.GetCounterOffset(); + long countOffs = NativeContext.GetCounterOffset(); - Operand count = context.LoadFromContext(cntOffset); + Operand countAddr = context.Add(context.LoadArgument(OperandType.I64, 0), Const(countOffs)); + + Operand count = context.Load(OperandType.I32, countAddr); Operand lblNonZero = Label(); + Operand lblExit = Label(); context.BranchIfTrue(lblNonZero, count); @@ -176,11 +176,15 @@ namespace ARMeilleure.Translation context.Call(info); + context.Branch(lblExit); + context.MarkLabel(lblNonZero); count = context.Subtract(count, Const(1)); - context.StoreToContext(cntOffset, count); + context.Store(countAddr, count); + + context.MarkLabel(lblExit); } } } \ No newline at end of file diff --git a/ARMeilleure/Translation/TranslatorCache.cs b/ARMeilleure/Translation/TranslatorCache.cs deleted file mode 100644 index 33306b63ab..0000000000 --- a/ARMeilleure/Translation/TranslatorCache.cs +++ /dev/null @@ -1,41 +0,0 @@ -using ARMeilleure.Memory; -using System; -using System.Runtime.InteropServices; - -namespace ARMeilleure.Translation -{ - class TranslatedCache - { - public TranslatedFunction CreateFunction(byte[] code) - { - IntPtr funcPtr = MapCodeAsExecutable(code); - - GuestFunction func = Marshal.GetDelegateForFunctionPointer(funcPtr); - - return new TranslatedFunction(func); - } - - private static IntPtr MapCodeAsExecutable(byte[] code) - { - ulong codeLength = (ulong)code.Length; - - IntPtr funcPtr = MemoryManagement.Allocate(codeLength); - - unsafe - { - fixed (byte* codePtr = code) - { - byte* dest = (byte*)funcPtr; - - long size = (long)codeLength; - - Buffer.MemoryCopy(codePtr, dest, size, size); - } - } - - MemoryManagement.Reprotect(funcPtr, codeLength, MemoryProtection.Execute); - - return funcPtr; - } - } -} \ No newline at end of file