diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml
index 383bbb1516..399aa039c5 100644
--- a/.github/ISSUE_TEMPLATE/feature_request.yml
+++ b/.github/ISSUE_TEMPLATE/feature_request.yml
@@ -1,6 +1,7 @@
name: Feature Request
description: Suggest a new feature for Ryujinx.
title: "[Feature Request]"
+labels: enhancement
body:
- type: textarea
id: overview
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 9e11302fd0..221c7732e9 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -20,7 +20,7 @@ jobs:
- { name: win-x64, os: windows-latest, zip_os_name: win_x64 }
- { name: linux-x64, os: ubuntu-latest, zip_os_name: linux_x64 }
- { name: linux-arm64, os: ubuntu-latest, zip_os_name: linux_arm64 }
- - { name: osx-x64, os: macOS-latest, zip_os_name: osx_x64 }
+ - { name: osx-x64, os: macos-13, zip_os_name: osx_x64 }
fail-fast: false
steps:
@@ -41,12 +41,12 @@ jobs:
- name: Change config filename
run: sed -r --in-place 's/\%\%RYUJINX_CONFIG_FILE_NAME\%\%/PRConfig\.json/g;' src/Ryujinx.Common/ReleaseInformation.cs
shell: bash
- if: github.event_name == 'pull_request' && matrix.platform.os != 'macOS-latest'
+ if: github.event_name == 'pull_request' && matrix.platform.os != 'macos-13'
- name: Change config filename for macOS
run: sed -r -i '' 's/\%\%RYUJINX_CONFIG_FILE_NAME\%\%/PRConfig\.json/g;' src/Ryujinx.Common/ReleaseInformation.cs
shell: bash
- if: github.event_name == 'pull_request' && matrix.platform.os == 'macOS-latest'
+ if: github.event_name == 'pull_request' && matrix.platform.os == 'macos-13'
- name: Build
run: dotnet build -c "${{ matrix.configuration }}" -p:Version="${{ env.RYUJINX_BASE_VERSION }}" -p:SourceRevisionId="${{ steps.git_short_hash.outputs.result }}" -p:ExtraDefineConstants=DISABLE_UPDATER
@@ -61,15 +61,15 @@ jobs:
- name: Publish Ryujinx
run: dotnet publish -c "${{ matrix.configuration }}" -r "${{ matrix.platform.name }}" -o ./publish -p:Version="${{ env.RYUJINX_BASE_VERSION }}" -p:DebugType=embedded -p:SourceRevisionId="${{ steps.git_short_hash.outputs.result }}" -p:ExtraDefineConstants=DISABLE_UPDATER src/Ryujinx --self-contained true
- if: github.event_name == 'pull_request' && matrix.platform.os != 'macOS-latest'
+ if: github.event_name == 'pull_request' && matrix.platform.os != 'macos-13'
- name: Publish Ryujinx.Headless.SDL2
run: dotnet publish -c "${{ matrix.configuration }}" -r "${{ matrix.platform.name }}" -o ./publish_sdl2_headless -p:Version="${{ env.RYUJINX_BASE_VERSION }}" -p:DebugType=embedded -p:SourceRevisionId="${{ steps.git_short_hash.outputs.result }}" -p:ExtraDefineConstants=DISABLE_UPDATER src/Ryujinx.Headless.SDL2 --self-contained true
- if: github.event_name == 'pull_request' && matrix.platform.os != 'macOS-latest'
+ if: github.event_name == 'pull_request' && matrix.platform.os != 'macos-13'
- name: Publish Ryujinx.Gtk3
run: dotnet publish -c "${{ matrix.configuration }}" -r "${{ matrix.platform.name }}" -o ./publish_gtk -p:Version="${{ env.RYUJINX_BASE_VERSION }}" -p:DebugType=embedded -p:SourceRevisionId="${{ steps.git_short_hash.outputs.result }}" -p:ExtraDefineConstants=DISABLE_UPDATER src/Ryujinx.Gtk3 --self-contained true
- if: github.event_name == 'pull_request' && matrix.platform.os != 'macOS-latest'
+ if: github.event_name == 'pull_request' && matrix.platform.os != 'macos-13'
- name: Set executable bit
run: |
@@ -83,21 +83,21 @@ jobs:
with:
name: ryujinx-${{ matrix.configuration }}-${{ env.RYUJINX_BASE_VERSION }}+${{ steps.git_short_hash.outputs.result }}-${{ matrix.platform.zip_os_name }}
path: publish
- if: github.event_name == 'pull_request' && matrix.platform.os != 'macOS-latest'
+ if: github.event_name == 'pull_request' && matrix.platform.os != 'macos-13'
- name: Upload Ryujinx.Headless.SDL2 artifact
uses: actions/upload-artifact@v4
with:
name: sdl2-ryujinx-headless-${{ matrix.configuration }}-${{ env.RYUJINX_BASE_VERSION }}+${{ steps.git_short_hash.outputs.result }}-${{ matrix.platform.zip_os_name }}
path: publish_sdl2_headless
- if: github.event_name == 'pull_request' && matrix.platform.os != 'macOS-latest'
+ if: github.event_name == 'pull_request' && matrix.platform.os != 'macos-13'
- name: Upload Ryujinx.Gtk3 artifact
uses: actions/upload-artifact@v4
with:
name: gtk-ryujinx-${{ matrix.configuration }}-${{ env.RYUJINX_BASE_VERSION }}+${{ steps.git_short_hash.outputs.result }}-${{ matrix.platform.zip_os_name }}
path: publish_gtk
- if: github.event_name == 'pull_request' && matrix.platform.os != 'macOS-latest'
+ if: github.event_name == 'pull_request' && matrix.platform.os != 'macos-13'
build_macos:
name: macOS Universal (${{ matrix.configuration }})
diff --git a/Directory.Packages.props b/Directory.Packages.props
index c08e943576..d04e237e04 100644
--- a/Directory.Packages.props
+++ b/Directory.Packages.props
@@ -8,19 +8,19 @@
-
-
+
+
-
+
-
-
+
+
@@ -42,7 +42,7 @@
-
+
diff --git a/README.md b/README.md
index f2f3cb001e..7f2294d311 100644
--- a/README.md
+++ b/README.md
@@ -36,8 +36,8 @@
## Compatibility
-As of October 2023, Ryujinx has been tested on approximately 4,200 titles;
-over 4,150 boot past menus and into gameplay, with roughly 3,500 of those being considered playable.
+As of May 2024, Ryujinx has been tested on approximately 4,300 titles;
+over 4,100 boot past menus and into gameplay, with roughly 3,550 of those being considered playable.
You can check out the compatibility list [here](https://github.com/Ryujinx/Ryujinx-Games-List/issues).
diff --git a/Ryujinx.sln.DotSettings b/Ryujinx.sln.DotSettings
index 049bdaf69d..ed7f3e9118 100644
--- a/Ryujinx.sln.DotSettings
+++ b/Ryujinx.sln.DotSettings
@@ -4,6 +4,8 @@
UseExplicitType
UseExplicitType
<Policy Inspect="True" Prefix="" Suffix="" Style="AaBb"><ExtraRule Prefix="I" Suffix="" Style="AaBb" /></Policy>
+ <Policy><Descriptor Staticness="Any" AccessRightKinds="Any" Description="Types and namespaces"><ElementKinds><Kind Name="NAMESPACE" /><Kind Name="CLASS" /><Kind Name="STRUCT" /><Kind Name="ENUM" /><Kind Name="DELEGATE" /></ElementKinds></Descriptor><Policy Inspect="True" Prefix="" Suffix="" Style="AaBb"><ExtraRule Prefix="I" Suffix="" Style="AaBb" /></Policy></Policy>
+ True
True
True
True
diff --git a/docs/README.md b/docs/README.md
index 2213086f67..a22da3c7cf 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -33,8 +33,3 @@ Project Docs
=================
To be added. Many project files will contain basic XML docs for key functions and classes in the meantime.
-
-Other Information
-=================
-
-- N/A
diff --git a/src/ARMeilleure/Instructions/InstEmitAlu32.cs b/src/ARMeilleure/Instructions/InstEmitAlu32.cs
index 3a5e71bccf..028ffbeb13 100644
--- a/src/ARMeilleure/Instructions/InstEmitAlu32.cs
+++ b/src/ARMeilleure/Instructions/InstEmitAlu32.cs
@@ -19,6 +19,12 @@ namespace ARMeilleure.Instructions
Operand n = GetAluN(context);
Operand m = GetAluM(context, setCarry: false);
+ if (op.Rn == RegisterAlias.Aarch32Pc && op is OpCodeT32AluImm12)
+ {
+ // For ADR, PC is always 4 bytes aligned, even in Thumb mode.
+ n = context.BitwiseAnd(n, Const(~3u));
+ }
+
Operand res = context.Add(n, m);
if (ShouldSetFlags(context))
@@ -467,6 +473,12 @@ namespace ARMeilleure.Instructions
Operand n = GetAluN(context);
Operand m = GetAluM(context, setCarry: false);
+ if (op.Rn == RegisterAlias.Aarch32Pc && op is OpCodeT32AluImm12)
+ {
+ // For ADR, PC is always 4 bytes aligned, even in Thumb mode.
+ n = context.BitwiseAnd(n, Const(~3u));
+ }
+
Operand res = context.Subtract(n, m);
if (ShouldSetFlags(context))
diff --git a/src/ARMeilleure/Instructions/InstEmitMemoryHelper.cs b/src/ARMeilleure/Instructions/InstEmitMemoryHelper.cs
index a807eed51c..ace6fe1ce9 100644
--- a/src/ARMeilleure/Instructions/InstEmitMemoryHelper.cs
+++ b/src/ARMeilleure/Instructions/InstEmitMemoryHelper.cs
@@ -157,7 +157,7 @@ namespace ARMeilleure.Instructions
context.Copy(temp, value);
- if (!context.Memory.Type.IsHostMapped())
+ if (!context.Memory.Type.IsHostMappedOrTracked())
{
context.Branch(lblEnd);
@@ -198,7 +198,7 @@ namespace ARMeilleure.Instructions
SetInt(context, rt, value);
- if (!context.Memory.Type.IsHostMapped())
+ if (!context.Memory.Type.IsHostMappedOrTracked())
{
context.Branch(lblEnd);
@@ -265,7 +265,7 @@ namespace ARMeilleure.Instructions
context.Copy(GetVec(rt), value);
- if (!context.Memory.Type.IsHostMapped())
+ if (!context.Memory.Type.IsHostMappedOrTracked())
{
context.Branch(lblEnd);
@@ -312,7 +312,7 @@ namespace ARMeilleure.Instructions
break;
}
- if (!context.Memory.Type.IsHostMapped())
+ if (!context.Memory.Type.IsHostMappedOrTracked())
{
context.Branch(lblEnd);
@@ -385,7 +385,7 @@ namespace ARMeilleure.Instructions
break;
}
- if (!context.Memory.Type.IsHostMapped())
+ if (!context.Memory.Type.IsHostMappedOrTracked())
{
context.Branch(lblEnd);
@@ -403,6 +403,27 @@ namespace ARMeilleure.Instructions
{
return EmitHostMappedPointer(context, address);
}
+ else if (context.Memory.Type.IsHostTracked())
+ {
+ if (address.Type == OperandType.I32)
+ {
+ address = context.ZeroExtend32(OperandType.I64, address);
+ }
+
+ if (context.Memory.Type == MemoryManagerType.HostTracked)
+ {
+ Operand mask = Const(ulong.MaxValue >> (64 - context.Memory.AddressSpaceBits));
+ address = context.BitwiseAnd(address, mask);
+ }
+
+ Operand ptBase = !context.HasPtc
+ ? Const(context.Memory.PageTablePointer.ToInt64())
+ : Const(context.Memory.PageTablePointer.ToInt64(), Ptc.PageTableSymbol);
+
+ Operand ptOffset = context.ShiftRightUI(address, Const(PageBits));
+
+ return context.Add(address, context.Load(OperandType.I64, context.Add(ptBase, context.ShiftLeft(ptOffset, Const(3)))));
+ }
int ptLevelBits = context.Memory.AddressSpaceBits - PageBits;
int ptLevelSize = 1 << ptLevelBits;
diff --git a/src/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs b/src/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs
index 543aab0236..13d9fac683 100644
--- a/src/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs
+++ b/src/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs
@@ -2426,7 +2426,11 @@ namespace ARMeilleure.Instructions
}
else if (Optimizations.FastFP && Optimizations.UseSse41 && sizeF == 0)
{
- Operand res = EmitSse41Round32Exp8OpF(context, context.AddIntrinsic(Intrinsic.X86Rsqrtss, GetVec(op.Rn)), scalar: true);
+ // RSQRTSS handles subnormals as zero, which differs from Arm, so we can't use it here.
+
+ Operand res = context.AddIntrinsic(Intrinsic.X86Sqrtss, GetVec(op.Rn));
+ res = context.AddIntrinsic(Intrinsic.X86Rcpss, res);
+ res = EmitSse41Round32Exp8OpF(context, res, scalar: true);
context.Copy(GetVec(op.Rd), context.VectorZeroUpper96(res));
}
@@ -2451,7 +2455,11 @@ namespace ARMeilleure.Instructions
}
else if (Optimizations.FastFP && Optimizations.UseSse41 && sizeF == 0)
{
- Operand res = EmitSse41Round32Exp8OpF(context, context.AddIntrinsic(Intrinsic.X86Rsqrtps, GetVec(op.Rn)), scalar: false);
+ // RSQRTPS handles subnormals as zero, which differs from Arm, so we can't use it here.
+
+ Operand res = context.AddIntrinsic(Intrinsic.X86Sqrtps, GetVec(op.Rn));
+ res = context.AddIntrinsic(Intrinsic.X86Rcpps, res);
+ res = EmitSse41Round32Exp8OpF(context, res, scalar: false);
if (op.RegisterSize == RegisterSize.Simd64)
{
diff --git a/src/ARMeilleure/Instructions/NativeInterface.cs b/src/ARMeilleure/Instructions/NativeInterface.cs
index d1b2e353c5..0cd3754f7f 100644
--- a/src/ARMeilleure/Instructions/NativeInterface.cs
+++ b/src/ARMeilleure/Instructions/NativeInterface.cs
@@ -91,54 +91,54 @@ namespace ARMeilleure.Instructions
#region "Read"
public static byte ReadByte(ulong address)
{
- return GetMemoryManager().ReadTracked(address);
+ return GetMemoryManager().ReadGuest(address);
}
public static ushort ReadUInt16(ulong address)
{
- return GetMemoryManager().ReadTracked(address);
+ return GetMemoryManager().ReadGuest(address);
}
public static uint ReadUInt32(ulong address)
{
- return GetMemoryManager().ReadTracked(address);
+ return GetMemoryManager().ReadGuest(address);
}
public static ulong ReadUInt64(ulong address)
{
- return GetMemoryManager().ReadTracked(address);
+ return GetMemoryManager().ReadGuest(address);
}
public static V128 ReadVector128(ulong address)
{
- return GetMemoryManager().ReadTracked(address);
+ return GetMemoryManager().ReadGuest(address);
}
#endregion
#region "Write"
public static void WriteByte(ulong address, byte value)
{
- GetMemoryManager().Write(address, value);
+ GetMemoryManager().WriteGuest(address, value);
}
public static void WriteUInt16(ulong address, ushort value)
{
- GetMemoryManager().Write(address, value);
+ GetMemoryManager().WriteGuest(address, value);
}
public static void WriteUInt32(ulong address, uint value)
{
- GetMemoryManager().Write(address, value);
+ GetMemoryManager().WriteGuest(address, value);
}
public static void WriteUInt64(ulong address, ulong value)
{
- GetMemoryManager().Write(address, value);
+ GetMemoryManager().WriteGuest(address, value);
}
public static void WriteVector128(ulong address, V128 value)
{
- GetMemoryManager().Write(address, value);
+ GetMemoryManager().WriteGuest(address, value);
}
#endregion
diff --git a/src/ARMeilleure/Memory/IMemoryManager.cs b/src/ARMeilleure/Memory/IMemoryManager.cs
index 952cd2b4f0..46d4426559 100644
--- a/src/ARMeilleure/Memory/IMemoryManager.cs
+++ b/src/ARMeilleure/Memory/IMemoryManager.cs
@@ -28,6 +28,17 @@ namespace ARMeilleure.Memory
/// The data
T ReadTracked(ulong va) where T : unmanaged;
+ ///
+ /// Reads data from CPU mapped memory, from guest code. (with read tracking)
+ ///
+ /// Type of the data being read
+ /// Virtual address of the data in memory
+ /// The data
+ T ReadGuest(ulong va) where T : unmanaged
+ {
+ return ReadTracked(va);
+ }
+
///
/// Writes data to CPU mapped memory.
///
@@ -36,6 +47,17 @@ namespace ARMeilleure.Memory
/// Data to be written
void Write(ulong va, T value) where T : unmanaged;
+ ///
+ /// Writes data to CPU mapped memory, from guest code.
+ ///
+ /// Type of the data being written
+ /// Virtual address to write the data into
+ /// Data to be written
+ void WriteGuest(ulong va, T value) where T : unmanaged
+ {
+ Write(va, value);
+ }
+
///
/// Gets a read-only span of data from CPU mapped memory.
///
diff --git a/src/ARMeilleure/Memory/MemoryManagerType.cs b/src/ARMeilleure/Memory/MemoryManagerType.cs
index b1cdbb069a..bc8ae26359 100644
--- a/src/ARMeilleure/Memory/MemoryManagerType.cs
+++ b/src/ARMeilleure/Memory/MemoryManagerType.cs
@@ -29,6 +29,18 @@ namespace ARMeilleure.Memory
/// Allows invalid access from JIT code to the rest of the program, but is faster.
///
HostMappedUnsafe,
+
+ ///
+ /// High level implementation using a software flat page table for address translation
+ /// with no support for handling invalid or non-contiguous memory access.
+ ///
+ HostTracked,
+
+ ///
+ /// High level implementation using a software flat page table for address translation
+ /// without masking the address and no support for handling invalid or non-contiguous memory access.
+ ///
+ HostTrackedUnsafe,
}
public static class MemoryManagerTypeExtensions
@@ -37,5 +49,15 @@ namespace ARMeilleure.Memory
{
return type == MemoryManagerType.HostMapped || type == MemoryManagerType.HostMappedUnsafe;
}
+
+ public static bool IsHostTracked(this MemoryManagerType type)
+ {
+ return type == MemoryManagerType.HostTracked || type == MemoryManagerType.HostTrackedUnsafe;
+ }
+
+ public static bool IsHostMappedOrTracked(this MemoryManagerType type)
+ {
+ return type.IsHostMapped() || type.IsHostTracked();
+ }
}
}
diff --git a/src/ARMeilleure/Signal/NativeSignalHandlerGenerator.cs b/src/ARMeilleure/Signal/NativeSignalHandlerGenerator.cs
index c5e708e169..2ec5bc1b38 100644
--- a/src/ARMeilleure/Signal/NativeSignalHandlerGenerator.cs
+++ b/src/ARMeilleure/Signal/NativeSignalHandlerGenerator.cs
@@ -21,10 +21,8 @@ namespace ARMeilleure.Signal
private const uint EXCEPTION_ACCESS_VIOLATION = 0xc0000005;
- private static Operand EmitGenericRegionCheck(EmitterContext context, IntPtr signalStructPtr, Operand faultAddress, Operand isWrite, int rangeStructSize, ulong pageSize)
+ private static Operand EmitGenericRegionCheck(EmitterContext context, IntPtr signalStructPtr, Operand faultAddress, Operand isWrite, int rangeStructSize)
{
- ulong pageMask = pageSize - 1;
-
Operand inRegionLocal = context.AllocateLocal(OperandType.I32);
context.Copy(inRegionLocal, Const(0));
@@ -51,7 +49,7 @@ namespace ARMeilleure.Signal
// Only call tracking if in range.
context.BranchIfFalse(nextLabel, inRange, BasicBlockFrequency.Cold);
- Operand offset = context.BitwiseAnd(context.Subtract(faultAddress, rangeAddress), Const(~pageMask));
+ Operand offset = context.Subtract(faultAddress, rangeAddress);
// Call the tracking action, with the pointer's relative offset to the base address.
Operand trackingActionPtr = context.Load(OperandType.I64, Const((ulong)signalStructPtr + rangeBaseOffset + 20));
@@ -62,8 +60,10 @@ namespace ARMeilleure.Signal
// Tracking action should be non-null to call it, otherwise assume false return.
context.BranchIfFalse(skipActionLabel, trackingActionPtr);
- Operand result = context.Call(trackingActionPtr, OperandType.I32, offset, Const(pageSize), isWrite);
- context.Copy(inRegionLocal, result);
+ Operand result = context.Call(trackingActionPtr, OperandType.I64, offset, Const(1UL), isWrite);
+ context.Copy(inRegionLocal, context.ICompareNotEqual(result, Const(0UL)));
+
+ GenerateFaultAddressPatchCode(context, faultAddress, result);
context.MarkLabel(skipActionLabel);
@@ -155,7 +155,7 @@ namespace ARMeilleure.Signal
throw new PlatformNotSupportedException();
}
- public static byte[] GenerateUnixSignalHandler(IntPtr signalStructPtr, int rangeStructSize, ulong pageSize)
+ public static byte[] GenerateUnixSignalHandler(IntPtr signalStructPtr, int rangeStructSize)
{
EmitterContext context = new();
@@ -168,7 +168,7 @@ namespace ARMeilleure.Signal
Operand isWrite = context.ICompareNotEqual(writeFlag, Const(0L)); // Normalize to 0/1.
- Operand isInRegion = EmitGenericRegionCheck(context, signalStructPtr, faultAddress, isWrite, rangeStructSize, pageSize);
+ Operand isInRegion = EmitGenericRegionCheck(context, signalStructPtr, faultAddress, isWrite, rangeStructSize);
Operand endLabel = Label();
@@ -203,7 +203,7 @@ namespace ARMeilleure.Signal
return Compiler.Compile(cfg, argTypes, OperandType.None, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Code;
}
- public static byte[] GenerateWindowsSignalHandler(IntPtr signalStructPtr, int rangeStructSize, ulong pageSize)
+ public static byte[] GenerateWindowsSignalHandler(IntPtr signalStructPtr, int rangeStructSize)
{
EmitterContext context = new();
@@ -232,7 +232,7 @@ namespace ARMeilleure.Signal
Operand isWrite = context.ICompareNotEqual(writeFlag, Const(0L)); // Normalize to 0/1.
- Operand isInRegion = EmitGenericRegionCheck(context, signalStructPtr, faultAddress, isWrite, rangeStructSize, pageSize);
+ Operand isInRegion = EmitGenericRegionCheck(context, signalStructPtr, faultAddress, isWrite, rangeStructSize);
Operand endLabel = Label();
@@ -256,5 +256,86 @@ namespace ARMeilleure.Signal
return Compiler.Compile(cfg, argTypes, OperandType.I32, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Code;
}
+
+ private static void GenerateFaultAddressPatchCode(EmitterContext context, Operand faultAddress, Operand newAddress)
+ {
+ if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
+ {
+ if (SupportsFaultAddressPatchingForHostOs())
+ {
+ Operand lblSkip = Label();
+
+ context.BranchIf(lblSkip, faultAddress, newAddress, Comparison.Equal);
+
+ Operand ucontextPtr = context.LoadArgument(OperandType.I64, 2);
+ Operand pcCtxAddress = default;
+ ulong baseRegsOffset = 0;
+
+ if (OperatingSystem.IsLinux())
+ {
+ pcCtxAddress = context.Add(ucontextPtr, Const(440UL));
+ baseRegsOffset = 184UL;
+ }
+ else if (OperatingSystem.IsMacOS() || OperatingSystem.IsIOS())
+ {
+ ucontextPtr = context.Load(OperandType.I64, context.Add(ucontextPtr, Const(48UL)));
+
+ pcCtxAddress = context.Add(ucontextPtr, Const(272UL));
+ baseRegsOffset = 16UL;
+ }
+
+ Operand pc = context.Load(OperandType.I64, pcCtxAddress);
+
+ Operand reg = GetAddressRegisterFromArm64Instruction(context, pc);
+ Operand reg64 = context.ZeroExtend32(OperandType.I64, reg);
+ Operand regCtxAddress = context.Add(ucontextPtr, context.Add(context.ShiftLeft(reg64, Const(3)), Const(baseRegsOffset)));
+ Operand regAddress = context.Load(OperandType.I64, regCtxAddress);
+
+ Operand addressDelta = context.Subtract(regAddress, faultAddress);
+
+ context.Store(regCtxAddress, context.Add(newAddress, addressDelta));
+
+ context.MarkLabel(lblSkip);
+ }
+ }
+ }
+
+ private static Operand GetAddressRegisterFromArm64Instruction(EmitterContext context, Operand pc)
+ {
+ Operand inst = context.Load(OperandType.I32, pc);
+ Operand reg = context.AllocateLocal(OperandType.I32);
+
+ Operand isSysInst = context.ICompareEqual(context.BitwiseAnd(inst, Const(0xFFF80000)), Const(0xD5080000));
+
+ Operand lblSys = Label();
+ Operand lblEnd = Label();
+
+ context.BranchIfTrue(lblSys, isSysInst, BasicBlockFrequency.Cold);
+
+ context.Copy(reg, context.BitwiseAnd(context.ShiftRightUI(inst, Const(5)), Const(0x1F)));
+ context.Branch(lblEnd);
+
+ context.MarkLabel(lblSys);
+ context.Copy(reg, context.BitwiseAnd(inst, Const(0x1F)));
+
+ context.MarkLabel(lblEnd);
+
+ return reg;
+ }
+
+ public static bool SupportsFaultAddressPatchingForHost()
+ {
+ return SupportsFaultAddressPatchingForHostArch() && SupportsFaultAddressPatchingForHostOs();
+ }
+
+ private static bool SupportsFaultAddressPatchingForHostArch()
+ {
+ return RuntimeInformation.ProcessArchitecture == Architecture.Arm64;
+ }
+
+ private static bool SupportsFaultAddressPatchingForHostOs()
+ {
+ return OperatingSystem.IsLinux() || OperatingSystem.IsMacOS() || OperatingSystem.IsIOS();
+ }
}
}
diff --git a/src/ARMeilleure/Translation/PTC/Ptc.cs b/src/ARMeilleure/Translation/PTC/Ptc.cs
index 6f6dfcadf3..f56bdce1cd 100644
--- a/src/ARMeilleure/Translation/PTC/Ptc.cs
+++ b/src/ARMeilleure/Translation/PTC/Ptc.cs
@@ -29,7 +29,7 @@ namespace ARMeilleure.Translation.PTC
private const string OuterHeaderMagicString = "PTCohd\0\0";
private const string InnerHeaderMagicString = "PTCihd\0\0";
- private const uint InternalVersion = 5518; //! To be incremented manually for each change to the ARMeilleure project.
+ private const uint InternalVersion = 6634; //! To be incremented manually for each change to the ARMeilleure project.
private const string ActualDir = "0";
private const string BackupDir = "1";
@@ -857,8 +857,14 @@ namespace ARMeilleure.Translation.PTC
Stopwatch sw = Stopwatch.StartNew();
- threads.ForEach((thread) => thread.Start());
- threads.ForEach((thread) => thread.Join());
+ foreach (var thread in threads)
+ {
+ thread.Start();
+ }
+ foreach (var thread in threads)
+ {
+ thread.Join();
+ }
threads.Clear();
diff --git a/src/Ryujinx.Audio.Backends.SDL2/SDL2HardwareDeviceSession.cs b/src/Ryujinx.Audio.Backends.SDL2/SDL2HardwareDeviceSession.cs
index 00188ba58e..62fe5025d6 100644
--- a/src/Ryujinx.Audio.Backends.SDL2/SDL2HardwareDeviceSession.cs
+++ b/src/Ryujinx.Audio.Backends.SDL2/SDL2HardwareDeviceSession.cs
@@ -1,8 +1,10 @@
using Ryujinx.Audio.Backends.Common;
using Ryujinx.Audio.Common;
using Ryujinx.Common.Logging;
+using Ryujinx.Common.Memory;
using Ryujinx.Memory;
using System;
+using System.Buffers;
using System.Collections.Concurrent;
using System.Threading;
@@ -87,7 +89,9 @@ namespace Ryujinx.Audio.Backends.SDL2
return;
}
- byte[] samples = new byte[frameCount * _bytesPerFrame];
+ using IMemoryOwner samplesOwner = ByteMemoryPool.Rent(frameCount * _bytesPerFrame);
+
+ Span samples = samplesOwner.Memory.Span;
_ringBuffer.Read(samples, 0, samples.Length);
diff --git a/src/Ryujinx.Audio.Backends.SoundIo/SoundIoHardwareDeviceSession.cs b/src/Ryujinx.Audio.Backends.SoundIo/SoundIoHardwareDeviceSession.cs
index f60982e303..4011a12142 100644
--- a/src/Ryujinx.Audio.Backends.SoundIo/SoundIoHardwareDeviceSession.cs
+++ b/src/Ryujinx.Audio.Backends.SoundIo/SoundIoHardwareDeviceSession.cs
@@ -1,8 +1,10 @@
using Ryujinx.Audio.Backends.Common;
using Ryujinx.Audio.Backends.SoundIo.Native;
using Ryujinx.Audio.Common;
+using Ryujinx.Common.Memory;
using Ryujinx.Memory;
using System;
+using System.Buffers;
using System.Collections.Concurrent;
using System.Runtime.CompilerServices;
using System.Threading;
@@ -37,7 +39,7 @@ namespace Ryujinx.Audio.Backends.SoundIo
_outputStream = _driver.OpenStream(RequestedSampleFormat, RequestedSampleRate, RequestedChannelCount);
_outputStream.WriteCallback += Update;
_outputStream.Volume = requestedVolume;
- // TODO: Setup other callbacks (errors, ect).
+ // TODO: Setup other callbacks (errors, etc.)
_outputStream.Open();
}
@@ -120,7 +122,9 @@ namespace Ryujinx.Audio.Backends.SoundIo
int channelCount = areas.Length;
- byte[] samples = new byte[frameCount * bytesPerFrame];
+ using IMemoryOwner samplesOwner = ByteMemoryPool.Rent(frameCount * bytesPerFrame);
+
+ Span samples = samplesOwner.Memory.Span;
_ringBuffer.Read(samples, 0, samples.Length);
diff --git a/src/Ryujinx.Audio/Backends/Common/DynamicRingBuffer.cs b/src/Ryujinx.Audio/Backends/Common/DynamicRingBuffer.cs
index 05dd2162a5..b95e5bed14 100644
--- a/src/Ryujinx.Audio/Backends/Common/DynamicRingBuffer.cs
+++ b/src/Ryujinx.Audio/Backends/Common/DynamicRingBuffer.cs
@@ -1,5 +1,7 @@
using Ryujinx.Common;
+using Ryujinx.Common.Memory;
using System;
+using System.Buffers;
namespace Ryujinx.Audio.Backends.Common
{
@@ -12,7 +14,8 @@ namespace Ryujinx.Audio.Backends.Common
private readonly object _lock = new();
- private byte[] _buffer;
+ private IMemoryOwner _bufferOwner;
+ private Memory _buffer;
private int _size;
private int _headOffset;
private int _tailOffset;
@@ -21,7 +24,8 @@ namespace Ryujinx.Audio.Backends.Common
public DynamicRingBuffer(int initialCapacity = RingBufferAlignment)
{
- _buffer = new byte[initialCapacity];
+ _bufferOwner = ByteMemoryPool.RentCleared(initialCapacity);
+ _buffer = _bufferOwner.Memory;
}
public void Clear()
@@ -33,6 +37,11 @@ namespace Ryujinx.Audio.Backends.Common
public void Clear(int size)
{
+ if (size == 0)
+ {
+ return;
+ }
+
lock (_lock)
{
if (size > _size)
@@ -40,11 +49,6 @@ namespace Ryujinx.Audio.Backends.Common
size = _size;
}
- if (size == 0)
- {
- return;
- }
-
_headOffset = (_headOffset + size) % _buffer.Length;
_size -= size;
@@ -58,28 +62,31 @@ namespace Ryujinx.Audio.Backends.Common
private void SetCapacityLocked(int capacity)
{
- byte[] buffer = new byte[capacity];
+ IMemoryOwner newBufferOwner = ByteMemoryPool.RentCleared(capacity);
+ Memory newBuffer = newBufferOwner.Memory;
if (_size > 0)
{
if (_headOffset < _tailOffset)
{
- Buffer.BlockCopy(_buffer, _headOffset, buffer, 0, _size);
+ _buffer.Slice(_headOffset, _size).CopyTo(newBuffer);
}
else
{
- Buffer.BlockCopy(_buffer, _headOffset, buffer, 0, _buffer.Length - _headOffset);
- Buffer.BlockCopy(_buffer, 0, buffer, _buffer.Length - _headOffset, _tailOffset);
+ _buffer[_headOffset..].CopyTo(newBuffer);
+ _buffer[.._tailOffset].CopyTo(newBuffer[(_buffer.Length - _headOffset)..]);
}
}
- _buffer = buffer;
+ _bufferOwner.Dispose();
+
+ _bufferOwner = newBufferOwner;
+ _buffer = newBuffer;
_headOffset = 0;
_tailOffset = _size;
}
-
- public void Write(T[] buffer, int index, int count)
+ public void Write(ReadOnlySpan buffer, int index, int count)
{
if (count == 0)
{
@@ -99,17 +106,17 @@ namespace Ryujinx.Audio.Backends.Common
if (tailLength >= count)
{
- Buffer.BlockCopy(buffer, index, _buffer, _tailOffset, count);
+ buffer.Slice(index, count).CopyTo(_buffer.Span[_tailOffset..]);
}
else
{
- Buffer.BlockCopy(buffer, index, _buffer, _tailOffset, tailLength);
- Buffer.BlockCopy(buffer, index + tailLength, _buffer, 0, count - tailLength);
+ buffer.Slice(index, tailLength).CopyTo(_buffer.Span[_tailOffset..]);
+ buffer.Slice(index + tailLength, count - tailLength).CopyTo(_buffer.Span);
}
}
else
{
- Buffer.BlockCopy(buffer, index, _buffer, _tailOffset, count);
+ buffer.Slice(index, count).CopyTo(_buffer.Span[_tailOffset..]);
}
_size += count;
@@ -117,8 +124,13 @@ namespace Ryujinx.Audio.Backends.Common
}
}
- public int Read(T[] buffer, int index, int count)
+ public int Read(Span buffer, int index, int count)
{
+ if (count == 0)
+ {
+ return 0;
+ }
+
lock (_lock)
{
if (count > _size)
@@ -126,14 +138,9 @@ namespace Ryujinx.Audio.Backends.Common
count = _size;
}
- if (count == 0)
- {
- return 0;
- }
-
if (_headOffset < _tailOffset)
{
- Buffer.BlockCopy(_buffer, _headOffset, buffer, index, count);
+ _buffer.Span.Slice(_headOffset, count).CopyTo(buffer[index..]);
}
else
{
@@ -141,12 +148,12 @@ namespace Ryujinx.Audio.Backends.Common
if (tailLength >= count)
{
- Buffer.BlockCopy(_buffer, _headOffset, buffer, index, count);
+ _buffer.Span.Slice(_headOffset, count).CopyTo(buffer[index..]);
}
else
{
- Buffer.BlockCopy(_buffer, _headOffset, buffer, index, tailLength);
- Buffer.BlockCopy(_buffer, 0, buffer, index + tailLength, count - tailLength);
+ _buffer.Span.Slice(_headOffset, tailLength).CopyTo(buffer[index..]);
+ _buffer.Span[..(count - tailLength)].CopyTo(buffer[(index + tailLength)..]);
}
}
diff --git a/src/Ryujinx.Audio/Renderer/Common/BehaviourParameter.cs b/src/Ryujinx.Audio/Renderer/Common/BehaviourParameter.cs
index b0963c9350..3b8d15dc53 100644
--- a/src/Ryujinx.Audio/Renderer/Common/BehaviourParameter.cs
+++ b/src/Ryujinx.Audio/Renderer/Common/BehaviourParameter.cs
@@ -25,7 +25,7 @@ namespace Ryujinx.Audio.Renderer.Common
public ulong Flags;
///
- /// Represents an error during .
+ /// Represents an error during .
///
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public struct ErrorInfo
diff --git a/src/Ryujinx.Audio/Renderer/Common/UpdateDataHeader.cs b/src/Ryujinx.Audio/Renderer/Common/UpdateDataHeader.cs
index 7efe3b02b4..98b224ebfb 100644
--- a/src/Ryujinx.Audio/Renderer/Common/UpdateDataHeader.cs
+++ b/src/Ryujinx.Audio/Renderer/Common/UpdateDataHeader.cs
@@ -4,7 +4,7 @@ using System.Runtime.CompilerServices;
namespace Ryujinx.Audio.Renderer.Common
{
///
- /// Update data header used for input and output of .
+ /// Update data header used for input and output of .
///
public struct UpdateDataHeader
{
diff --git a/src/Ryujinx.Audio/Renderer/Parameter/BehaviourErrorInfoOutStatus.cs b/src/Ryujinx.Audio/Renderer/Parameter/BehaviourErrorInfoOutStatus.cs
index 5a0565dc61..72438be0e4 100644
--- a/src/Ryujinx.Audio/Renderer/Parameter/BehaviourErrorInfoOutStatus.cs
+++ b/src/Ryujinx.Audio/Renderer/Parameter/BehaviourErrorInfoOutStatus.cs
@@ -8,7 +8,7 @@ namespace Ryujinx.Audio.Renderer.Parameter
///
/// Output information for behaviour.
///
- /// This is used to report errors to the user during processing.
+ /// This is used to report errors to the user during processing.
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public struct BehaviourErrorInfoOutStatus
{
diff --git a/src/Ryujinx.Audio/Renderer/Server/AudioRenderSystem.cs b/src/Ryujinx.Audio/Renderer/Server/AudioRenderSystem.cs
index 7bb8ae5ba7..9b56f5cbdf 100644
--- a/src/Ryujinx.Audio/Renderer/Server/AudioRenderSystem.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/AudioRenderSystem.cs
@@ -386,7 +386,7 @@ namespace Ryujinx.Audio.Renderer.Server
}
}
- public ResultCode Update(Memory output, Memory performanceOutput, ReadOnlyMemory input)
+ public ResultCode Update(Memory output, Memory performanceOutput, ReadOnlySequence input)
{
lock (_lock)
{
@@ -419,14 +419,16 @@ namespace Ryujinx.Audio.Renderer.Server
return result;
}
- result = stateUpdater.UpdateVoices(_voiceContext, _memoryPools);
+ PoolMapper poolMapper = new PoolMapper(_processHandle, _memoryPools, _behaviourContext.IsMemoryPoolForceMappingEnabled());
+
+ result = stateUpdater.UpdateVoices(_voiceContext, poolMapper);
if (result != ResultCode.Success)
{
return result;
}
- result = stateUpdater.UpdateEffects(_effectContext, _isActive, _memoryPools);
+ result = stateUpdater.UpdateEffects(_effectContext, _isActive, poolMapper);
if (result != ResultCode.Success)
{
@@ -450,7 +452,7 @@ namespace Ryujinx.Audio.Renderer.Server
return result;
}
- result = stateUpdater.UpdateSinks(_sinkContext, _memoryPools);
+ result = stateUpdater.UpdateSinks(_sinkContext, poolMapper);
if (result != ResultCode.Success)
{
diff --git a/src/Ryujinx.Audio/Renderer/Server/BehaviourContext.cs b/src/Ryujinx.Audio/Renderer/Server/BehaviourContext.cs
index 3297b5d9fa..fe1dfc4beb 100644
--- a/src/Ryujinx.Audio/Renderer/Server/BehaviourContext.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/BehaviourContext.cs
@@ -1,4 +1,5 @@
using System;
+using System.Buffers;
using System.Diagnostics;
using static Ryujinx.Audio.Renderer.Common.BehaviourParameter;
@@ -211,7 +212,7 @@ namespace Ryujinx.Audio.Renderer.Server
///
/// Check if the audio renderer should fix the GC-ADPCM context not being provided to the DSP.
///
- /// True if if the audio renderer should fix it.
+ /// True if the audio renderer should fix it.
public bool IsAdpcmLoopContextBugFixed()
{
return CheckFeatureSupported(UserRevision, BaseRevisionMagic + Revision2);
@@ -273,7 +274,7 @@ namespace Ryujinx.Audio.Renderer.Server
}
///
- /// Check if the audio renderer should trust the user destination count in .
+ /// Check if the audio renderer should trust the user destination count in .
///
/// True if the audio renderer should trust the user destination count.
public bool IsSplitterBugFixed()
diff --git a/src/Ryujinx.Audio/Renderer/Server/Effect/AuxiliaryBufferEffect.cs b/src/Ryujinx.Audio/Renderer/Server/Effect/AuxiliaryBufferEffect.cs
index 57ca266f4d..74a9baff2a 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Effect/AuxiliaryBufferEffect.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Effect/AuxiliaryBufferEffect.cs
@@ -33,21 +33,21 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
return WorkBuffers[index].GetReference(true);
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion1 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion1 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion2 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion2 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
+ public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
{
- Debug.Assert(IsTypeValid(ref parameter));
+ Debug.Assert(IsTypeValid(in parameter));
- UpdateParameterBase(ref parameter);
+ UpdateParameterBase(in parameter);
Parameter = MemoryMarshal.Cast(parameter.SpecificData)[0];
IsEnabled = parameter.IsEnabled;
diff --git a/src/Ryujinx.Audio/Renderer/Server/Effect/BaseEffect.cs b/src/Ryujinx.Audio/Renderer/Server/Effect/BaseEffect.cs
index a9716db2a5..77d9b5c295 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Effect/BaseEffect.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Effect/BaseEffect.cs
@@ -81,7 +81,7 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
///
/// The user parameter.
/// Returns true if the sent by the user matches the internal .
- public bool IsTypeValid(ref T parameter) where T : unmanaged, IEffectInParameter
+ public bool IsTypeValid(in T parameter) where T : unmanaged, IEffectInParameter
{
return parameter.Type == TargetEffectType;
}
@@ -98,7 +98,7 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
/// Update the internal common parameters from a user parameter.
///
/// The user parameter.
- protected void UpdateParameterBase(ref T parameter) where T : unmanaged, IEffectInParameter
+ protected void UpdateParameterBase(in T parameter) where T : unmanaged, IEffectInParameter
{
MixId = parameter.MixId;
ProcessingOrder = parameter.ProcessingOrder;
@@ -139,7 +139,7 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
///
/// Initialize the given result state.
///
- /// The state to initalize
+ /// The state to initialize
public virtual void InitializeResultState(ref EffectResultState state) { }
///
@@ -155,9 +155,9 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
/// The possible that was generated.
/// The user parameter.
/// The mapper to use.
- public virtual void Update(out ErrorInfo updateErrorInfo, ref EffectInParameterVersion1 parameter, PoolMapper mapper)
+ public virtual void Update(out ErrorInfo updateErrorInfo, in EffectInParameterVersion1 parameter, PoolMapper mapper)
{
- Debug.Assert(IsTypeValid(ref parameter));
+ Debug.Assert(IsTypeValid(in parameter));
updateErrorInfo = new ErrorInfo();
}
@@ -168,9 +168,9 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
/// The possible that was generated.
/// The user parameter.
/// The mapper to use.
- public virtual void Update(out ErrorInfo updateErrorInfo, ref EffectInParameterVersion2 parameter, PoolMapper mapper)
+ public virtual void Update(out ErrorInfo updateErrorInfo, in EffectInParameterVersion2 parameter, PoolMapper mapper)
{
- Debug.Assert(IsTypeValid(ref parameter));
+ Debug.Assert(IsTypeValid(in parameter));
updateErrorInfo = new ErrorInfo();
}
diff --git a/src/Ryujinx.Audio/Renderer/Server/Effect/BiquadFilterEffect.cs b/src/Ryujinx.Audio/Renderer/Server/Effect/BiquadFilterEffect.cs
index b987f7c85e..3b3e1021c4 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Effect/BiquadFilterEffect.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Effect/BiquadFilterEffect.cs
@@ -35,21 +35,21 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
public override EffectType TargetEffectType => EffectType.BiquadFilter;
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion1 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion1 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion2 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion2 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
+ public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
{
- Debug.Assert(IsTypeValid(ref parameter));
+ Debug.Assert(IsTypeValid(in parameter));
- UpdateParameterBase(ref parameter);
+ UpdateParameterBase(in parameter);
Parameter = MemoryMarshal.Cast(parameter.SpecificData)[0];
IsEnabled = parameter.IsEnabled;
diff --git a/src/Ryujinx.Audio/Renderer/Server/Effect/BufferMixEffect.cs b/src/Ryujinx.Audio/Renderer/Server/Effect/BufferMixEffect.cs
index d6cb9cfa39..5d82b5ae87 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Effect/BufferMixEffect.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Effect/BufferMixEffect.cs
@@ -19,21 +19,21 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
public override EffectType TargetEffectType => EffectType.BufferMix;
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion1 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion1 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion2 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion2 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
+ public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
{
- Debug.Assert(IsTypeValid(ref parameter));
+ Debug.Assert(IsTypeValid(in parameter));
- UpdateParameterBase(ref parameter);
+ UpdateParameterBase(in parameter);
Parameter = MemoryMarshal.Cast(parameter.SpecificData)[0];
IsEnabled = parameter.IsEnabled;
diff --git a/src/Ryujinx.Audio/Renderer/Server/Effect/CaptureBufferEffect.cs b/src/Ryujinx.Audio/Renderer/Server/Effect/CaptureBufferEffect.cs
index 5be4b4ed51..6917222f0a 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Effect/CaptureBufferEffect.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Effect/CaptureBufferEffect.cs
@@ -32,21 +32,21 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
return WorkBuffers[index].GetReference(true);
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion1 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion1 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion2 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion2 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
+ public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
{
- Debug.Assert(IsTypeValid(ref parameter));
+ Debug.Assert(IsTypeValid(in parameter));
- UpdateParameterBase(ref parameter);
+ UpdateParameterBase(in parameter);
Parameter = MemoryMarshal.Cast(parameter.SpecificData)[0];
IsEnabled = parameter.IsEnabled;
diff --git a/src/Ryujinx.Audio/Renderer/Server/Effect/CompressorEffect.cs b/src/Ryujinx.Audio/Renderer/Server/Effect/CompressorEffect.cs
index 826c32cb07..eff60e7da8 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Effect/CompressorEffect.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Effect/CompressorEffect.cs
@@ -39,17 +39,17 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
return GetSingleBuffer();
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion1 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion1 parameter, PoolMapper mapper)
{
// Nintendo doesn't do anything here but we still require updateErrorInfo to be initialised.
updateErrorInfo = new BehaviourParameter.ErrorInfo();
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion2 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion2 parameter, PoolMapper mapper)
{
- Debug.Assert(IsTypeValid(ref parameter));
+ Debug.Assert(IsTypeValid(in parameter));
- UpdateParameterBase(ref parameter);
+ UpdateParameterBase(in parameter);
Parameter = MemoryMarshal.Cast(parameter.SpecificData)[0];
IsEnabled = parameter.IsEnabled;
diff --git a/src/Ryujinx.Audio/Renderer/Server/Effect/DelayEffect.cs b/src/Ryujinx.Audio/Renderer/Server/Effect/DelayEffect.cs
index 43cabb7db9..9db1ce465d 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Effect/DelayEffect.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Effect/DelayEffect.cs
@@ -37,19 +37,19 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
return GetSingleBuffer();
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion1 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion1 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion2 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion2 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
+ public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
{
- Debug.Assert(IsTypeValid(ref parameter));
+ Debug.Assert(IsTypeValid(in parameter));
ref DelayParameter delayParameter = ref MemoryMarshal.Cast(parameter.SpecificData)[0];
@@ -57,7 +57,7 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
if (delayParameter.IsChannelCountMaxValid())
{
- UpdateParameterBase(ref parameter);
+ UpdateParameterBase(in parameter);
UsageState oldParameterStatus = Parameter.Status;
diff --git a/src/Ryujinx.Audio/Renderer/Server/Effect/LimiterEffect.cs b/src/Ryujinx.Audio/Renderer/Server/Effect/LimiterEffect.cs
index 3e2f7326d0..d9b3d5666e 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Effect/LimiterEffect.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Effect/LimiterEffect.cs
@@ -39,25 +39,25 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
return GetSingleBuffer();
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion1 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion1 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion2 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion2 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
+ public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
{
- Debug.Assert(IsTypeValid(ref parameter));
+ Debug.Assert(IsTypeValid(in parameter));
ref LimiterParameter limiterParameter = ref MemoryMarshal.Cast(parameter.SpecificData)[0];
updateErrorInfo = new BehaviourParameter.ErrorInfo();
- UpdateParameterBase(ref parameter);
+ UpdateParameterBase(in parameter);
Parameter = limiterParameter;
diff --git a/src/Ryujinx.Audio/Renderer/Server/Effect/Reverb3dEffect.cs b/src/Ryujinx.Audio/Renderer/Server/Effect/Reverb3dEffect.cs
index f9d7f4943c..4b13cfec62 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Effect/Reverb3dEffect.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Effect/Reverb3dEffect.cs
@@ -36,19 +36,19 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
return GetSingleBuffer();
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion1 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion1 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion2 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion2 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
+ public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
{
- Debug.Assert(IsTypeValid(ref parameter));
+ Debug.Assert(IsTypeValid(in parameter));
ref Reverb3dParameter reverbParameter = ref MemoryMarshal.Cast(parameter.SpecificData)[0];
@@ -56,7 +56,7 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
if (reverbParameter.IsChannelCountMaxValid())
{
- UpdateParameterBase(ref parameter);
+ UpdateParameterBase(in parameter);
UsageState oldParameterStatus = Parameter.ParameterStatus;
diff --git a/src/Ryujinx.Audio/Renderer/Server/Effect/ReverbEffect.cs b/src/Ryujinx.Audio/Renderer/Server/Effect/ReverbEffect.cs
index 6fdf8fc23d..aa6e674481 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Effect/ReverbEffect.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Effect/ReverbEffect.cs
@@ -39,19 +39,19 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
return GetSingleBuffer();
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion1 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion1 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref EffectInParameterVersion2 parameter, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in EffectInParameterVersion2 parameter, PoolMapper mapper)
{
- Update(out updateErrorInfo, ref parameter, mapper);
+ Update(out updateErrorInfo, in parameter, mapper);
}
- public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, ref T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
+ public void Update(out BehaviourParameter.ErrorInfo updateErrorInfo, in T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
{
- Debug.Assert(IsTypeValid(ref parameter));
+ Debug.Assert(IsTypeValid(in parameter));
ref ReverbParameter reverbParameter = ref MemoryMarshal.Cast(parameter.SpecificData)[0];
@@ -59,7 +59,7 @@ namespace Ryujinx.Audio.Renderer.Server.Effect
if (reverbParameter.IsChannelCountMaxValid())
{
- UpdateParameterBase(ref parameter);
+ UpdateParameterBase(in parameter);
UsageState oldParameterStatus = Parameter.Status;
diff --git a/src/Ryujinx.Audio/Renderer/Server/MemoryPool/PoolMapper.cs b/src/Ryujinx.Audio/Renderer/Server/MemoryPool/PoolMapper.cs
index 391b80f8db..f67d0c1249 100644
--- a/src/Ryujinx.Audio/Renderer/Server/MemoryPool/PoolMapper.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/MemoryPool/PoolMapper.cs
@@ -249,7 +249,7 @@ namespace Ryujinx.Audio.Renderer.Server.MemoryPool
/// Input user parameter.
/// Output user parameter.
/// Returns the of the operations performed.
- public UpdateResult Update(ref MemoryPoolState memoryPool, ref MemoryPoolInParameter inParameter, ref MemoryPoolOutStatus outStatus)
+ public UpdateResult Update(ref MemoryPoolState memoryPool, in MemoryPoolInParameter inParameter, ref MemoryPoolOutStatus outStatus)
{
MemoryPoolUserState inputState = inParameter.State;
diff --git a/src/Ryujinx.Audio/Renderer/Server/Mix/MixState.cs b/src/Ryujinx.Audio/Renderer/Server/Mix/MixState.cs
index 88ae448314..b90574da92 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Mix/MixState.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Mix/MixState.cs
@@ -195,7 +195,7 @@ namespace Ryujinx.Audio.Renderer.Server.Mix
/// The input parameter of the mix.
/// The splitter context.
/// Return true, new connections were done on the adjacency matrix.
- private bool UpdateConnection(EdgeMatrix edgeMatrix, ref MixParameter parameter, ref SplitterContext splitterContext)
+ private bool UpdateConnection(EdgeMatrix edgeMatrix, in MixParameter parameter, ref SplitterContext splitterContext)
{
bool hasNewConnections;
@@ -259,7 +259,7 @@ namespace Ryujinx.Audio.Renderer.Server.Mix
/// The splitter context.
/// The behaviour context.
/// Return true if the mix was changed.
- public bool Update(EdgeMatrix edgeMatrix, ref MixParameter parameter, EffectContext effectContext, SplitterContext splitterContext, BehaviourContext behaviourContext)
+ public bool Update(EdgeMatrix edgeMatrix, in MixParameter parameter, EffectContext effectContext, SplitterContext splitterContext, BehaviourContext behaviourContext)
{
bool isDirty;
@@ -273,7 +273,7 @@ namespace Ryujinx.Audio.Renderer.Server.Mix
if (behaviourContext.IsSplitterSupported())
{
- isDirty = UpdateConnection(edgeMatrix, ref parameter, ref splitterContext);
+ isDirty = UpdateConnection(edgeMatrix, in parameter, ref splitterContext);
}
else
{
diff --git a/src/Ryujinx.Audio/Renderer/Server/Sink/BaseSink.cs b/src/Ryujinx.Audio/Renderer/Server/Sink/BaseSink.cs
index d36c5e260e..8c65e09bc3 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Sink/BaseSink.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Sink/BaseSink.cs
@@ -59,7 +59,7 @@ namespace Ryujinx.Audio.Renderer.Server.Sink
///
/// The user parameter.
/// Return true, if the sent by the user match the internal .
- public bool IsTypeValid(ref SinkInParameter parameter)
+ public bool IsTypeValid(in SinkInParameter parameter)
{
return parameter.Type == TargetSinkType;
}
@@ -76,7 +76,7 @@ namespace Ryujinx.Audio.Renderer.Server.Sink
/// Update the internal common parameters from user parameter.
///
/// The user parameter.
- protected void UpdateStandardParameter(ref SinkInParameter parameter)
+ protected void UpdateStandardParameter(in SinkInParameter parameter)
{
if (IsUsed != parameter.IsUsed)
{
@@ -92,9 +92,9 @@ namespace Ryujinx.Audio.Renderer.Server.Sink
/// The user parameter.
/// The user output status.
/// The mapper to use.
- public virtual void Update(out ErrorInfo errorInfo, ref SinkInParameter parameter, ref SinkOutStatus outStatus, PoolMapper mapper)
+ public virtual void Update(out ErrorInfo errorInfo, in SinkInParameter parameter, ref SinkOutStatus outStatus, PoolMapper mapper)
{
- Debug.Assert(IsTypeValid(ref parameter));
+ Debug.Assert(IsTypeValid(in parameter));
errorInfo = new ErrorInfo();
}
diff --git a/src/Ryujinx.Audio/Renderer/Server/Sink/CircularBufferSink.cs b/src/Ryujinx.Audio/Renderer/Server/Sink/CircularBufferSink.cs
index 097757988d..f2751cf29b 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Sink/CircularBufferSink.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Sink/CircularBufferSink.cs
@@ -44,18 +44,18 @@ namespace Ryujinx.Audio.Renderer.Server.Sink
public override SinkType TargetSinkType => SinkType.CircularBuffer;
- public override void Update(out BehaviourParameter.ErrorInfo errorInfo, ref SinkInParameter parameter, ref SinkOutStatus outStatus, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo errorInfo, in SinkInParameter parameter, ref SinkOutStatus outStatus, PoolMapper mapper)
{
errorInfo = new BehaviourParameter.ErrorInfo();
outStatus = new SinkOutStatus();
- Debug.Assert(IsTypeValid(ref parameter));
+ Debug.Assert(IsTypeValid(in parameter));
ref CircularBufferParameter inputDeviceParameter = ref MemoryMarshal.Cast(parameter.SpecificData)[0];
if (parameter.IsUsed != IsUsed || ShouldSkip)
{
- UpdateStandardParameter(ref parameter);
+ UpdateStandardParameter(in parameter);
if (parameter.IsUsed)
{
diff --git a/src/Ryujinx.Audio/Renderer/Server/Sink/DeviceSink.cs b/src/Ryujinx.Audio/Renderer/Server/Sink/DeviceSink.cs
index e03fe11d49..afe2d4b1b7 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Sink/DeviceSink.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Sink/DeviceSink.cs
@@ -49,15 +49,15 @@ namespace Ryujinx.Audio.Renderer.Server.Sink
public override SinkType TargetSinkType => SinkType.Device;
- public override void Update(out BehaviourParameter.ErrorInfo errorInfo, ref SinkInParameter parameter, ref SinkOutStatus outStatus, PoolMapper mapper)
+ public override void Update(out BehaviourParameter.ErrorInfo errorInfo, in SinkInParameter parameter, ref SinkOutStatus outStatus, PoolMapper mapper)
{
- Debug.Assert(IsTypeValid(ref parameter));
+ Debug.Assert(IsTypeValid(in parameter));
ref DeviceParameter inputDeviceParameter = ref MemoryMarshal.Cast(parameter.SpecificData)[0];
if (parameter.IsUsed != IsUsed)
{
- UpdateStandardParameter(ref parameter);
+ UpdateStandardParameter(in parameter);
Parameter = inputDeviceParameter;
}
else
diff --git a/src/Ryujinx.Audio/Renderer/Server/Splitter/SplitterContext.cs b/src/Ryujinx.Audio/Renderer/Server/Splitter/SplitterContext.cs
index e408692ab9..3efa783c37 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Splitter/SplitterContext.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Splitter/SplitterContext.cs
@@ -2,10 +2,11 @@ using Ryujinx.Audio.Renderer.Common;
using Ryujinx.Audio.Renderer.Parameter;
using Ryujinx.Audio.Renderer.Utils;
using Ryujinx.Common;
+using Ryujinx.Common.Extensions;
using System;
+using System.Buffers;
using System.Diagnostics;
using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
namespace Ryujinx.Audio.Renderer.Server.Splitter
{
@@ -25,7 +26,7 @@ namespace Ryujinx.Audio.Renderer.Server.Splitter
private Memory _splitterDestinations;
///
- /// If set to true, trust the user destination count in .
+ /// If set to true, trust the user destination count in .
///
public bool IsBugFixed { get; private set; }
@@ -110,7 +111,7 @@ namespace Ryujinx.Audio.Renderer.Server.Splitter
///
/// The storage.
/// The storage.
- /// If set to true, trust the user destination count in .
+ /// If set to true, trust the user destination count in .
private void Setup(Memory splitters, Memory splitterDestinations, bool isBugFixed)
{
_splitters = splitters;
@@ -148,11 +149,11 @@ namespace Ryujinx.Audio.Renderer.Server.Splitter
///
/// The splitter header.
/// The raw data after the splitter header.
- private void UpdateState(scoped ref SplitterInParameterHeader inputHeader, ref ReadOnlySpan input)
+ private void UpdateState(in SplitterInParameterHeader inputHeader, ref SequenceReader input)
{
for (int i = 0; i < inputHeader.SplitterCount; i++)
{
- SplitterInParameter parameter = MemoryMarshal.Read(input);
+ ref readonly SplitterInParameter parameter = ref input.GetRefOrRefToCopy(out _);
Debug.Assert(parameter.IsMagicValid());
@@ -162,10 +163,16 @@ namespace Ryujinx.Audio.Renderer.Server.Splitter
{
ref SplitterState splitter = ref GetState(parameter.Id);
- splitter.Update(this, ref parameter, input[Unsafe.SizeOf()..]);
+ splitter.Update(this, in parameter, ref input);
}
- input = input[(0x1C + parameter.DestinationCount * 4)..];
+ // NOTE: there are 12 bytes of unused/unknown data after the destination IDs array.
+ input.Advance(0xC);
+ }
+ else
+ {
+ input.Rewind(Unsafe.SizeOf());
+ break;
}
}
}
@@ -175,11 +182,11 @@ namespace Ryujinx.Audio.Renderer.Server.Splitter
///
/// The splitter header.
/// The raw data after the splitter header.
- private void UpdateData(scoped ref SplitterInParameterHeader inputHeader, ref ReadOnlySpan input)
+ private void UpdateData(in SplitterInParameterHeader inputHeader, ref SequenceReader input)
{
for (int i = 0; i < inputHeader.SplitterDestinationCount; i++)
{
- SplitterDestinationInParameter parameter = MemoryMarshal.Read(input);
+ ref readonly SplitterDestinationInParameter parameter = ref input.GetRefOrRefToCopy(out _);
Debug.Assert(parameter.IsMagicValid());
@@ -191,8 +198,11 @@ namespace Ryujinx.Audio.Renderer.Server.Splitter
destination.Update(parameter);
}
-
- input = input[Unsafe.SizeOf()..];
+ }
+ else
+ {
+ input.Rewind(Unsafe.SizeOf());
+ break;
}
}
}
@@ -201,36 +211,33 @@ namespace Ryujinx.Audio.Renderer.Server.Splitter
/// Update splitter from user parameters.
///
/// The input raw user data.
- /// The total consumed size.
/// Return true if the update was successful.
- public bool Update(ReadOnlySpan input, out int consumedSize)
+ public bool Update(ref SequenceReader input)
{
if (_splitterDestinations.IsEmpty || _splitters.IsEmpty)
{
- consumedSize = 0;
-
return true;
}
- int originalSize = input.Length;
-
- SplitterInParameterHeader header = SpanIOHelper.Read(ref input);
+ ref readonly SplitterInParameterHeader header = ref input.GetRefOrRefToCopy(out _);
if (header.IsMagicValid())
{
ClearAllNewConnectionFlag();
- UpdateState(ref header, ref input);
- UpdateData(ref header, ref input);
+ UpdateState(in header, ref input);
+ UpdateData(in header, ref input);
- consumedSize = BitUtils.AlignUp(originalSize - input.Length, 0x10);
+ input.SetConsumed(BitUtils.AlignUp(input.Consumed, 0x10));
return true;
}
+ else
+ {
+ input.Rewind(Unsafe.SizeOf());
- consumedSize = 0;
-
- return false;
+ return false;
+ }
}
///
diff --git a/src/Ryujinx.Audio/Renderer/Server/Splitter/SplitterState.cs b/src/Ryujinx.Audio/Renderer/Server/Splitter/SplitterState.cs
index e08ee9ea77..944f092d2c 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Splitter/SplitterState.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Splitter/SplitterState.cs
@@ -1,4 +1,5 @@
using Ryujinx.Audio.Renderer.Parameter;
+using Ryujinx.Common.Extensions;
using System;
using System.Buffers;
using System.Diagnostics;
@@ -122,7 +123,7 @@ namespace Ryujinx.Audio.Renderer.Server.Splitter
/// The splitter context.
/// The user parameter.
/// The raw input data after the .
- public void Update(SplitterContext context, ref SplitterInParameter parameter, ReadOnlySpan input)
+ public void Update(SplitterContext context, in SplitterInParameter parameter, ref SequenceReader input)
{
ClearLinks();
@@ -139,9 +140,9 @@ namespace Ryujinx.Audio.Renderer.Server.Splitter
if (destinationCount > 0)
{
- ReadOnlySpan destinationIds = MemoryMarshal.Cast(input);
+ input.ReadLittleEndian(out int destinationId);
- Memory destination = context.GetDestinationMemory(destinationIds[0]);
+ Memory destination = context.GetDestinationMemory(destinationId);
SetDestination(ref destination.Span[0]);
@@ -149,13 +150,20 @@ namespace Ryujinx.Audio.Renderer.Server.Splitter
for (int i = 1; i < destinationCount; i++)
{
- Memory nextDestination = context.GetDestinationMemory(destinationIds[i]);
+ input.ReadLittleEndian(out destinationId);
+
+ Memory nextDestination = context.GetDestinationMemory(destinationId);
destination.Span[0].Link(ref nextDestination.Span[0]);
destination = nextDestination;
}
}
+ if (destinationCount < parameter.DestinationCount)
+ {
+ input.Advance((parameter.DestinationCount - destinationCount) * sizeof(int));
+ }
+
Debug.Assert(parameter.Id == Id);
if (parameter.Id == Id)
diff --git a/src/Ryujinx.Audio/Renderer/Server/StateUpdater.cs b/src/Ryujinx.Audio/Renderer/Server/StateUpdater.cs
index 22eebc7ccc..f8d87f2d14 100644
--- a/src/Ryujinx.Audio/Renderer/Server/StateUpdater.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/StateUpdater.cs
@@ -9,41 +9,40 @@ using Ryujinx.Audio.Renderer.Server.Sink;
using Ryujinx.Audio.Renderer.Server.Splitter;
using Ryujinx.Audio.Renderer.Server.Voice;
using Ryujinx.Audio.Renderer.Utils;
+using Ryujinx.Common.Extensions;
using Ryujinx.Common.Logging;
using System;
using System.Buffers;
using System.Diagnostics;
using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
using static Ryujinx.Audio.Renderer.Common.BehaviourParameter;
namespace Ryujinx.Audio.Renderer.Server
{
- public class StateUpdater
+ public ref struct StateUpdater
{
- private readonly ReadOnlyMemory _inputOrigin;
+ private SequenceReader _inputReader;
+
private readonly ReadOnlyMemory _outputOrigin;
- private ReadOnlyMemory _input;
private Memory _output;
private readonly uint _processHandle;
private BehaviourContext _behaviourContext;
- private UpdateDataHeader _inputHeader;
+ private readonly ref readonly UpdateDataHeader _inputHeader;
private readonly Memory _outputHeader;
- private ref UpdateDataHeader OutputHeader => ref _outputHeader.Span[0];
+ private readonly ref UpdateDataHeader OutputHeader => ref _outputHeader.Span[0];
- public StateUpdater(ReadOnlyMemory input, Memory output, uint processHandle, BehaviourContext behaviourContext)
+ public StateUpdater(ReadOnlySequence input, Memory output, uint processHandle, BehaviourContext behaviourContext)
{
- _input = input;
- _inputOrigin = _input;
+ _inputReader = new SequenceReader(input);
_output = output;
_outputOrigin = _output;
_processHandle = processHandle;
_behaviourContext = behaviourContext;
- _inputHeader = SpanIOHelper.Read(ref _input);
+ _inputHeader = ref _inputReader.GetRefOrRefToCopy(out _);
_outputHeader = SpanMemoryManager.Cast(_output[..Unsafe.SizeOf()]);
OutputHeader.Initialize(_behaviourContext.UserRevision);
@@ -52,7 +51,7 @@ namespace Ryujinx.Audio.Renderer.Server
public ResultCode UpdateBehaviourContext()
{
- BehaviourParameter parameter = SpanIOHelper.Read(ref _input);
+ ref readonly BehaviourParameter parameter = ref _inputReader.GetRefOrRefToCopy(out _);
if (!BehaviourContext.CheckValidRevision(parameter.UserRevision) || parameter.UserRevision != _behaviourContext.UserRevision)
{
@@ -81,11 +80,11 @@ namespace Ryujinx.Audio.Renderer.Server
foreach (ref MemoryPoolState memoryPool in memoryPools)
{
- MemoryPoolInParameter parameter = SpanIOHelper.Read(ref _input);
+ ref readonly MemoryPoolInParameter parameter = ref _inputReader.GetRefOrRefToCopy(out _);
ref MemoryPoolOutStatus outStatus = ref SpanIOHelper.GetWriteRef(ref _output)[0];
- PoolMapper.UpdateResult updateResult = mapper.Update(ref memoryPool, ref parameter, ref outStatus);
+ PoolMapper.UpdateResult updateResult = mapper.Update(ref memoryPool, in parameter, ref outStatus);
if (updateResult != PoolMapper.UpdateResult.Success &&
updateResult != PoolMapper.UpdateResult.MapError &&
@@ -115,7 +114,7 @@ namespace Ryujinx.Audio.Renderer.Server
for (int i = 0; i < context.GetCount(); i++)
{
- VoiceChannelResourceInParameter parameter = SpanIOHelper.Read(ref _input);
+ ref readonly VoiceChannelResourceInParameter parameter = ref _inputReader.GetRefOrRefToCopy(out _);
ref VoiceChannelResource resource = ref context.GetChannelResource(i);
@@ -127,7 +126,7 @@ namespace Ryujinx.Audio.Renderer.Server
return ResultCode.Success;
}
- public ResultCode UpdateVoices(VoiceContext context, Memory memoryPools)
+ public ResultCode UpdateVoices(VoiceContext context, PoolMapper mapper)
{
if (context.GetCount() * Unsafe.SizeOf() != _inputHeader.VoicesSize)
{
@@ -136,11 +135,7 @@ namespace Ryujinx.Audio.Renderer.Server
int initialOutputSize = _output.Length;
- ReadOnlySpan parameters = MemoryMarshal.Cast(_input[..(int)_inputHeader.VoicesSize].Span);
-
- _input = _input[(int)_inputHeader.VoicesSize..];
-
- PoolMapper mapper = new(_processHandle, memoryPools, _behaviourContext.IsMemoryPoolForceMappingEnabled());
+ long initialInputConsumed = _inputReader.Consumed;
// First make everything not in use.
for (int i = 0; i < context.GetCount(); i++)
@@ -157,7 +152,7 @@ namespace Ryujinx.Audio.Renderer.Server
// Start processing
for (int i = 0; i < context.GetCount(); i++)
{
- VoiceInParameter parameter = parameters[i];
+ ref readonly VoiceInParameter parameter = ref _inputReader.GetRefOrRefToCopy(out _);
voiceUpdateStates.Fill(Memory.Empty);
@@ -181,14 +176,14 @@ namespace Ryujinx.Audio.Renderer.Server
currentVoiceState.Initialize();
}
- currentVoiceState.UpdateParameters(out ErrorInfo updateParameterError, ref parameter, ref mapper, ref _behaviourContext);
+ currentVoiceState.UpdateParameters(out ErrorInfo updateParameterError, in parameter, mapper, ref _behaviourContext);
if (updateParameterError.ErrorCode != ResultCode.Success)
{
_behaviourContext.AppendError(ref updateParameterError);
}
- currentVoiceState.UpdateWaveBuffers(out ErrorInfo[] waveBufferUpdateErrorInfos, ref parameter, voiceUpdateStates, ref mapper, ref _behaviourContext);
+ currentVoiceState.UpdateWaveBuffers(out ErrorInfo[] waveBufferUpdateErrorInfos, in parameter, voiceUpdateStates, mapper, ref _behaviourContext);
foreach (ref ErrorInfo errorInfo in waveBufferUpdateErrorInfos.AsSpan())
{
@@ -198,7 +193,7 @@ namespace Ryujinx.Audio.Renderer.Server
}
}
- currentVoiceState.WriteOutStatus(ref outStatus, ref parameter, voiceUpdateStates);
+ currentVoiceState.WriteOutStatus(ref outStatus, in parameter, voiceUpdateStates);
}
}
@@ -211,10 +206,12 @@ namespace Ryujinx.Audio.Renderer.Server
Debug.Assert((initialOutputSize - currentOutputSize) == OutputHeader.VoicesSize);
+ _inputReader.SetConsumed(initialInputConsumed + _inputHeader.VoicesSize);
+
return ResultCode.Success;
}
- private static void ResetEffect(ref BaseEffect effect, ref T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
+ private static void ResetEffect(ref BaseEffect effect, in T parameter, PoolMapper mapper) where T : unmanaged, IEffectInParameter
{
effect.ForceUnmapBuffers(mapper);
@@ -234,17 +231,17 @@ namespace Ryujinx.Audio.Renderer.Server
};
}
- public ResultCode UpdateEffects(EffectContext context, bool isAudioRendererActive, Memory memoryPools)
+ public ResultCode UpdateEffects(EffectContext context, bool isAudioRendererActive, PoolMapper mapper)
{
if (_behaviourContext.IsEffectInfoVersion2Supported())
{
- return UpdateEffectsVersion2(context, isAudioRendererActive, memoryPools);
+ return UpdateEffectsVersion2(context, isAudioRendererActive, mapper);
}
- return UpdateEffectsVersion1(context, isAudioRendererActive, memoryPools);
+ return UpdateEffectsVersion1(context, isAudioRendererActive, mapper);
}
- public ResultCode UpdateEffectsVersion2(EffectContext context, bool isAudioRendererActive, Memory memoryPools)
+ public ResultCode UpdateEffectsVersion2(EffectContext context, bool isAudioRendererActive, PoolMapper mapper)
{
if (context.GetCount() * Unsafe.SizeOf() != _inputHeader.EffectsSize)
{
@@ -253,26 +250,22 @@ namespace Ryujinx.Audio.Renderer.Server
int initialOutputSize = _output.Length;
- ReadOnlySpan parameters = MemoryMarshal.Cast(_input[..(int)_inputHeader.EffectsSize].Span);
-
- _input = _input[(int)_inputHeader.EffectsSize..];
-
- PoolMapper mapper = new(_processHandle, memoryPools, _behaviourContext.IsMemoryPoolForceMappingEnabled());
+ long initialInputConsumed = _inputReader.Consumed;
for (int i = 0; i < context.GetCount(); i++)
{
- EffectInParameterVersion2 parameter = parameters[i];
+ ref readonly EffectInParameterVersion2 parameter = ref _inputReader.GetRefOrRefToCopy(out _);
ref EffectOutStatusVersion2 outStatus = ref SpanIOHelper.GetWriteRef(ref _output)[0];
ref BaseEffect effect = ref context.GetEffect(i);
- if (!effect.IsTypeValid(ref parameter))
+ if (!effect.IsTypeValid(in parameter))
{
- ResetEffect(ref effect, ref parameter, mapper);
+ ResetEffect(ref effect, in parameter, mapper);
}
- effect.Update(out ErrorInfo updateErrorInfo, ref parameter, mapper);
+ effect.Update(out ErrorInfo updateErrorInfo, in parameter, mapper);
if (updateErrorInfo.ErrorCode != ResultCode.Success)
{
@@ -297,10 +290,12 @@ namespace Ryujinx.Audio.Renderer.Server
Debug.Assert((initialOutputSize - currentOutputSize) == OutputHeader.EffectsSize);
+ _inputReader.SetConsumed(initialInputConsumed + _inputHeader.EffectsSize);
+
return ResultCode.Success;
}
- public ResultCode UpdateEffectsVersion1(EffectContext context, bool isAudioRendererActive, Memory memoryPools)
+ public ResultCode UpdateEffectsVersion1(EffectContext context, bool isAudioRendererActive, PoolMapper mapper)
{
if (context.GetCount() * Unsafe.SizeOf() != _inputHeader.EffectsSize)
{
@@ -309,26 +304,22 @@ namespace Ryujinx.Audio.Renderer.Server
int initialOutputSize = _output.Length;
- ReadOnlySpan parameters = MemoryMarshal.Cast(_input[..(int)_inputHeader.EffectsSize].Span);
-
- _input = _input[(int)_inputHeader.EffectsSize..];
-
- PoolMapper mapper = new(_processHandle, memoryPools, _behaviourContext.IsMemoryPoolForceMappingEnabled());
+ long initialInputConsumed = _inputReader.Consumed;
for (int i = 0; i < context.GetCount(); i++)
{
- EffectInParameterVersion1 parameter = parameters[i];
+ ref readonly EffectInParameterVersion1 parameter = ref _inputReader.GetRefOrRefToCopy(out _);
ref EffectOutStatusVersion1 outStatus = ref SpanIOHelper.GetWriteRef(ref _output)[0];
ref BaseEffect effect = ref context.GetEffect(i);
- if (!effect.IsTypeValid(ref parameter))
+ if (!effect.IsTypeValid(in parameter))
{
- ResetEffect(ref effect, ref parameter, mapper);
+ ResetEffect(ref effect, in parameter, mapper);
}
- effect.Update(out ErrorInfo updateErrorInfo, ref parameter, mapper);
+ effect.Update(out ErrorInfo updateErrorInfo, in parameter, mapper);
if (updateErrorInfo.ErrorCode != ResultCode.Success)
{
@@ -345,38 +336,40 @@ namespace Ryujinx.Audio.Renderer.Server
Debug.Assert((initialOutputSize - currentOutputSize) == OutputHeader.EffectsSize);
+ _inputReader.SetConsumed(initialInputConsumed + _inputHeader.EffectsSize);
+
return ResultCode.Success;
}
public ResultCode UpdateSplitter(SplitterContext context)
{
- if (context.Update(_input.Span, out int consumedSize))
+ if (context.Update(ref _inputReader))
{
- _input = _input[consumedSize..];
-
return ResultCode.Success;
}
return ResultCode.InvalidUpdateInfo;
}
- private static bool CheckMixParametersValidity(MixContext mixContext, uint mixBufferCount, uint inputMixCount, ReadOnlySpan parameters)
+ private static bool CheckMixParametersValidity(MixContext mixContext, uint mixBufferCount, uint inputMixCount, SequenceReader parameters)
{
uint maxMixStateCount = mixContext.GetCount();
uint totalRequiredMixBufferCount = 0;
for (int i = 0; i < inputMixCount; i++)
{
- if (parameters[i].IsUsed)
+ ref readonly MixParameter parameter = ref parameters.GetRefOrRefToCopy(out _);
+
+ if (parameter.IsUsed)
{
- if (parameters[i].DestinationMixId != Constants.UnusedMixId &&
- parameters[i].DestinationMixId > maxMixStateCount &&
- parameters[i].MixId != Constants.FinalMixId)
+ if (parameter.DestinationMixId != Constants.UnusedMixId &&
+ parameter.DestinationMixId > maxMixStateCount &&
+ parameter.MixId != Constants.FinalMixId)
{
return true;
}
- totalRequiredMixBufferCount += parameters[i].BufferCount;
+ totalRequiredMixBufferCount += parameter.BufferCount;
}
}
@@ -391,7 +384,7 @@ namespace Ryujinx.Audio.Renderer.Server
if (_behaviourContext.IsMixInParameterDirtyOnlyUpdateSupported())
{
- MixInParameterDirtyOnlyUpdate parameter = MemoryMarshal.Cast(_input.Span)[0];
+ ref readonly MixInParameterDirtyOnlyUpdate parameter = ref _inputReader.GetRefOrRefToCopy(out _);
mixCount = parameter.MixCount;
@@ -411,25 +404,20 @@ namespace Ryujinx.Audio.Renderer.Server
return ResultCode.InvalidUpdateInfo;
}
- if (_behaviourContext.IsMixInParameterDirtyOnlyUpdateSupported())
- {
- _input = _input[Unsafe.SizeOf()..];
- }
+ long initialInputConsumed = _inputReader.Consumed;
- ReadOnlySpan parameters = MemoryMarshal.Cast(_input.Span[..(int)inputMixSize]);
+ int parameterCount = (int)inputMixSize / Unsafe.SizeOf();
- _input = _input[(int)inputMixSize..];
-
- if (CheckMixParametersValidity(mixContext, mixBufferCount, mixCount, parameters))
+ if (CheckMixParametersValidity(mixContext, mixBufferCount, mixCount, _inputReader))
{
return ResultCode.InvalidUpdateInfo;
}
bool isMixContextDirty = false;
- for (int i = 0; i < parameters.Length; i++)
+ for (int i = 0; i < parameterCount; i++)
{
- MixParameter parameter = parameters[i];
+ ref readonly MixParameter parameter = ref _inputReader.GetRefOrRefToCopy(out _);
int mixId = i;
@@ -454,7 +442,7 @@ namespace Ryujinx.Audio.Renderer.Server
if (mix.IsUsed)
{
- isMixContextDirty |= mix.Update(mixContext.EdgeMatrix, ref parameter, effectContext, splitterContext, _behaviourContext);
+ isMixContextDirty |= mix.Update(mixContext.EdgeMatrix, in parameter, effectContext, splitterContext, _behaviourContext);
}
}
@@ -473,10 +461,12 @@ namespace Ryujinx.Audio.Renderer.Server
}
}
+ _inputReader.SetConsumed(initialInputConsumed + inputMixSize);
+
return ResultCode.Success;
}
- private static void ResetSink(ref BaseSink sink, ref SinkInParameter parameter)
+ private static void ResetSink(ref BaseSink sink, in SinkInParameter parameter)
{
sink.CleanUp();
@@ -489,10 +479,8 @@ namespace Ryujinx.Audio.Renderer.Server
};
}
- public ResultCode UpdateSinks(SinkContext context, Memory memoryPools)
+ public ResultCode UpdateSinks(SinkContext context, PoolMapper mapper)
{
- PoolMapper mapper = new(_processHandle, memoryPools, _behaviourContext.IsMemoryPoolForceMappingEnabled());
-
if (context.GetCount() * Unsafe.SizeOf() != _inputHeader.SinksSize)
{
return ResultCode.InvalidUpdateInfo;
@@ -500,22 +488,20 @@ namespace Ryujinx.Audio.Renderer.Server
int initialOutputSize = _output.Length;
- ReadOnlySpan parameters = MemoryMarshal.Cast(_input[..(int)_inputHeader.SinksSize].Span);
-
- _input = _input[(int)_inputHeader.SinksSize..];
+ long initialInputConsumed = _inputReader.Consumed;
for (int i = 0; i < context.GetCount(); i++)
{
- SinkInParameter parameter = parameters[i];
+ ref readonly SinkInParameter parameter = ref _inputReader.GetRefOrRefToCopy(out _);
ref SinkOutStatus outStatus = ref SpanIOHelper.GetWriteRef(ref _output)[0];
ref BaseSink sink = ref context.GetSink(i);
- if (!sink.IsTypeValid(ref parameter))
+ if (!sink.IsTypeValid(in parameter))
{
- ResetSink(ref sink, ref parameter);
+ ResetSink(ref sink, in parameter);
}
- sink.Update(out ErrorInfo updateErrorInfo, ref parameter, ref outStatus, mapper);
+ sink.Update(out ErrorInfo updateErrorInfo, in parameter, ref outStatus, mapper);
if (updateErrorInfo.ErrorCode != ResultCode.Success)
{
@@ -530,6 +516,8 @@ namespace Ryujinx.Audio.Renderer.Server
Debug.Assert((initialOutputSize - currentOutputSize) == OutputHeader.SinksSize);
+ _inputReader.SetConsumed(initialInputConsumed + _inputHeader.SinksSize);
+
return ResultCode.Success;
}
@@ -540,7 +528,7 @@ namespace Ryujinx.Audio.Renderer.Server
return ResultCode.InvalidUpdateInfo;
}
- PerformanceInParameter parameter = SpanIOHelper.Read(ref _input);
+ ref readonly PerformanceInParameter parameter = ref _inputReader.GetRefOrRefToCopy(out _);
ref PerformanceOutStatus outStatus = ref SpanIOHelper.GetWriteRef(ref _output)[0];
@@ -585,9 +573,9 @@ namespace Ryujinx.Audio.Renderer.Server
return ResultCode.Success;
}
- public ResultCode CheckConsumedSize()
+ public readonly ResultCode CheckConsumedSize()
{
- int consumedInputSize = _inputOrigin.Length - _input.Length;
+ long consumedInputSize = _inputReader.Consumed;
int consumedOutputSize = _outputOrigin.Length - _output.Length;
if (consumedInputSize != _inputHeader.TotalSize)
diff --git a/src/Ryujinx.Audio/Renderer/Server/Voice/VoiceState.cs b/src/Ryujinx.Audio/Renderer/Server/Voice/VoiceState.cs
index 225f7d31b0..040c70e6ce 100644
--- a/src/Ryujinx.Audio/Renderer/Server/Voice/VoiceState.cs
+++ b/src/Ryujinx.Audio/Renderer/Server/Voice/VoiceState.cs
@@ -254,7 +254,7 @@ namespace Ryujinx.Audio.Renderer.Server.Voice
///
/// The user parameter.
/// Return true, if the server voice information needs to be updated.
- private readonly bool ShouldUpdateParameters(ref VoiceInParameter parameter)
+ private readonly bool ShouldUpdateParameters(in VoiceInParameter parameter)
{
if (DataSourceStateAddressInfo.CpuAddress == parameter.DataSourceStateAddress)
{
@@ -273,7 +273,7 @@ namespace Ryujinx.Audio.Renderer.Server.Voice
/// The user parameter.
/// The mapper to use.
/// The behaviour context.
- public void UpdateParameters(out ErrorInfo outErrorInfo, ref VoiceInParameter parameter, ref PoolMapper poolMapper, ref BehaviourContext behaviourContext)
+ public void UpdateParameters(out ErrorInfo outErrorInfo, in VoiceInParameter parameter, PoolMapper poolMapper, ref BehaviourContext behaviourContext)
{
InUse = parameter.InUse;
Id = parameter.Id;
@@ -326,7 +326,7 @@ namespace Ryujinx.Audio.Renderer.Server.Voice
VoiceDropFlag = false;
}
- if (ShouldUpdateParameters(ref parameter))
+ if (ShouldUpdateParameters(in parameter))
{
DataSourceStateUnmapped = !poolMapper.TryAttachBuffer(out outErrorInfo, ref DataSourceStateAddressInfo, parameter.DataSourceStateAddress, parameter.DataSourceStateSize);
}
@@ -380,7 +380,7 @@ namespace Ryujinx.Audio.Renderer.Server.Voice
/// The given user output.
/// The user parameter.
/// The voice states associated to the .
- public void WriteOutStatus(ref VoiceOutStatus outStatus, ref VoiceInParameter parameter, ReadOnlySpan> voiceUpdateStates)
+ public void WriteOutStatus(ref VoiceOutStatus outStatus, in VoiceInParameter parameter, ReadOnlySpan> voiceUpdateStates)
{
#if DEBUG
// Sanity check in debug mode of the internal state
@@ -426,7 +426,12 @@ namespace Ryujinx.Audio.Renderer.Server.Voice
/// The voice states associated to the .
/// The mapper to use.
/// The behaviour context.
- public void UpdateWaveBuffers(out ErrorInfo[] errorInfos, ref VoiceInParameter parameter, ReadOnlySpan> voiceUpdateStates, ref PoolMapper mapper, ref BehaviourContext behaviourContext)
+ public void UpdateWaveBuffers(
+ out ErrorInfo[] errorInfos,
+ in VoiceInParameter parameter,
+ ReadOnlySpan> voiceUpdateStates,
+ PoolMapper mapper,
+ ref BehaviourContext behaviourContext)
{
errorInfos = new ErrorInfo[Constants.VoiceWaveBufferCount * 2];
@@ -444,7 +449,7 @@ namespace Ryujinx.Audio.Renderer.Server.Voice
for (int i = 0; i < Constants.VoiceWaveBufferCount; i++)
{
- UpdateWaveBuffer(errorInfos.AsSpan(i * 2, 2), ref WaveBuffers[i], ref parameter.WaveBuffers[i], parameter.SampleFormat, voiceUpdateState.IsWaveBufferValid[i], ref mapper, ref behaviourContext);
+ UpdateWaveBuffer(errorInfos.AsSpan(i * 2, 2), ref WaveBuffers[i], ref parameter.WaveBuffers[i], parameter.SampleFormat, voiceUpdateState.IsWaveBufferValid[i], mapper, ref behaviourContext);
}
}
@@ -458,7 +463,14 @@ namespace Ryujinx.Audio.Renderer.Server.Voice
/// If set to true, the server side wavebuffer is considered valid.
/// The mapper to use.
/// The behaviour context.
- private void UpdateWaveBuffer(Span errorInfos, ref WaveBuffer waveBuffer, ref WaveBufferInternal inputWaveBuffer, SampleFormat sampleFormat, bool isValid, ref PoolMapper mapper, ref BehaviourContext behaviourContext)
+ private void UpdateWaveBuffer(
+ Span errorInfos,
+ ref WaveBuffer waveBuffer,
+ ref WaveBufferInternal inputWaveBuffer,
+ SampleFormat sampleFormat,
+ bool isValid,
+ PoolMapper mapper,
+ ref BehaviourContext behaviourContext)
{
if (!isValid && waveBuffer.IsSendToAudioProcessor && waveBuffer.BufferAddressInfo.CpuAddress != 0)
{
diff --git a/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeNode.cs b/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeNode.cs
index 8480d51ad6..29d2d0c9a8 100644
--- a/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeNode.cs
+++ b/src/Ryujinx.Common/Collections/IntrusiveRedBlackTreeNode.cs
@@ -5,10 +5,10 @@ namespace Ryujinx.Common.Collections
///
public class IntrusiveRedBlackTreeNode where T : IntrusiveRedBlackTreeNode
{
- internal bool Color = true;
- internal T Left;
- internal T Right;
- internal T Parent;
+ public bool Color = true;
+ public T Left;
+ public T Right;
+ public T Parent;
public T Predecessor => IntrusiveRedBlackTreeImpl.PredecessorOf((T)this);
public T Successor => IntrusiveRedBlackTreeImpl.SuccessorOf((T)this);
diff --git a/src/Ryujinx.Common/Configuration/Hid/KeyboardHotkeys.cs b/src/Ryujinx.Common/Configuration/Hid/KeyboardHotkeys.cs
index e9c163cf20..0cb49ca8ce 100644
--- a/src/Ryujinx.Common/Configuration/Hid/KeyboardHotkeys.cs
+++ b/src/Ryujinx.Common/Configuration/Hid/KeyboardHotkeys.cs
@@ -1,7 +1,5 @@
namespace Ryujinx.Common.Configuration.Hid
{
- // NOTE: Please don't change this to struct.
- // This breaks Avalonia's TwoWay binding, which makes us unable to save new KeyboardHotkeys.
public class KeyboardHotkeys
{
public Key ToggleVsync { get; set; }
diff --git a/src/Ryujinx.Common/Extensions/SequenceReaderExtensions.cs b/src/Ryujinx.Common/Extensions/SequenceReaderExtensions.cs
new file mode 100644
index 0000000000..79b5d743b9
--- /dev/null
+++ b/src/Ryujinx.Common/Extensions/SequenceReaderExtensions.cs
@@ -0,0 +1,181 @@
+using System;
+using System.Buffers;
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+namespace Ryujinx.Common.Extensions
+{
+ public static class SequenceReaderExtensions
+ {
+ ///
+ /// Dumps the entire to a file, restoring its previous location afterward.
+ /// Useful for debugging purposes.
+ ///
+ /// The to write to a file
+ /// The path and name of the file to create and dump to
+ public static void DumpToFile(this ref SequenceReader reader, string fileFullName)
+ {
+ var initialConsumed = reader.Consumed;
+
+ reader.Rewind(initialConsumed);
+
+ using (var fileStream = System.IO.File.Create(fileFullName, 4096, System.IO.FileOptions.None))
+ {
+ while (reader.End == false)
+ {
+ var span = reader.CurrentSpan;
+ fileStream.Write(span);
+ reader.Advance(span.Length);
+ }
+ }
+
+ reader.SetConsumed(initialConsumed);
+ }
+
+ ///
+ /// Returns a reference to the desired value. This ref should always be used. The argument passed in should never be used, as this is only used for storage if the value
+ /// must be copied from multiple segments held by the .
+ ///
+ /// Type to get
+ /// The to read from
+ /// A location used as storage if (and only if) the value to be read spans multiple segments
+ /// A reference to the desired value, either directly to memory in the , or to if it has been used for copying the value in to
+ ///
+ /// DO NOT use after calling this method, as it will only
+ /// contain a value if the value couldn't be referenced directly because it spans multiple segments.
+ /// To discourage use, it is recommended to call this method like the following:
+ ///
+ /// ref readonly MyStruct value = ref sequenceReader.GetRefOrRefToCopy{MyStruct}(out _);
+ ///
+ ///
+ /// The does not contain enough data to read a value of type
+ public static ref readonly T GetRefOrRefToCopy(this scoped ref SequenceReader reader, out T copyDestinationIfRequiredDoNotUse) where T : unmanaged
+ {
+ int lengthRequired = Unsafe.SizeOf();
+
+ ReadOnlySpan span = reader.UnreadSpan;
+ if (lengthRequired <= span.Length)
+ {
+ reader.Advance(lengthRequired);
+
+ copyDestinationIfRequiredDoNotUse = default;
+
+ ReadOnlySpan spanOfT = MemoryMarshal.Cast(span);
+
+ return ref spanOfT[0];
+ }
+ else
+ {
+ copyDestinationIfRequiredDoNotUse = default;
+
+ Span valueSpan = MemoryMarshal.CreateSpan(ref copyDestinationIfRequiredDoNotUse, 1);
+
+ Span valueBytesSpan = MemoryMarshal.AsBytes(valueSpan);
+
+ if (!reader.TryCopyTo(valueBytesSpan))
+ {
+ throw new ArgumentOutOfRangeException(nameof(reader), "The sequence is not long enough to read the desired value.");
+ }
+
+ reader.Advance(lengthRequired);
+
+ return ref valueSpan[0];
+ }
+ }
+
+ ///
+ /// Reads an as little endian.
+ ///
+ /// The to read from
+ /// A location to receive the read value
+ /// Thrown if there wasn't enough data for an
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static void ReadLittleEndian(this ref SequenceReader reader, out int value)
+ {
+ if (!reader.TryReadLittleEndian(out value))
+ {
+ throw new ArgumentOutOfRangeException(nameof(value), "The sequence is not long enough to read the desired value.");
+ }
+ }
+
+ ///
+ /// Reads the desired unmanaged value by copying it to the specified .
+ ///
+ /// Type to read
+ /// The to read from
+ /// The target that will receive the read value
+ /// The does not contain enough data to read a value of type
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static void ReadUnmanaged(this ref SequenceReader reader, out T value) where T : unmanaged
+ {
+ if (!reader.TryReadUnmanaged(out value))
+ {
+ throw new ArgumentOutOfRangeException(nameof(value), "The sequence is not long enough to read the desired value.");
+ }
+ }
+
+ ///
+ /// Sets the reader's position as bytes consumed.
+ ///
+ /// The to set the position
+ /// The number of bytes consumed
+ public static void SetConsumed(ref this SequenceReader reader, long consumed)
+ {
+ reader.Rewind(reader.Consumed);
+ reader.Advance(consumed);
+ }
+
+ ///
+ /// Try to read the given type out of the buffer if possible. Warning: this is dangerous to use with arbitrary
+ /// structs - see remarks for full details.
+ ///
+ /// Type to read
+ ///
+ /// IMPORTANT: The read is a straight copy of bits. If a struct depends on specific state of it's members to
+ /// behave correctly this can lead to exceptions, etc. If reading endian specific integers, use the explicit
+ /// overloads such as
+ ///
+ ///
+ /// True if successful. will be default if failed (due to lack of space).
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static unsafe bool TryReadUnmanaged(ref this SequenceReader reader, out T value) where T : unmanaged
+ {
+ ReadOnlySpan span = reader.UnreadSpan;
+
+ if (span.Length < sizeof(T))
+ {
+ return TryReadUnmanagedMultiSegment(ref reader, out value);
+ }
+
+ value = Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(span));
+
+ reader.Advance(sizeof(T));
+
+ return true;
+ }
+
+ private static unsafe bool TryReadUnmanagedMultiSegment(ref SequenceReader reader, out T value) where T : unmanaged
+ {
+ Debug.Assert(reader.UnreadSpan.Length < sizeof(T));
+
+ // Not enough data in the current segment, try to peek for the data we need.
+ T buffer = default;
+
+ Span tempSpan = new Span(&buffer, sizeof(T));
+
+ if (!reader.TryCopyTo(tempSpan))
+ {
+ value = default;
+ return false;
+ }
+
+ value = Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(tempSpan));
+
+ reader.Advance(sizeof(T));
+
+ return true;
+ }
+ }
+}
diff --git a/src/Ryujinx.Common/Memory/ByteMemoryPool.ByteMemoryPoolBuffer.cs b/src/Ryujinx.Common/Memory/ByteMemoryPool.ByteMemoryPoolBuffer.cs
index df3f8dc93e..05fb29ac71 100644
--- a/src/Ryujinx.Common/Memory/ByteMemoryPool.ByteMemoryPoolBuffer.cs
+++ b/src/Ryujinx.Common/Memory/ByteMemoryPool.ByteMemoryPoolBuffer.cs
@@ -4,7 +4,7 @@ using System.Threading;
namespace Ryujinx.Common.Memory
{
- public sealed partial class ByteMemoryPool
+ public partial class ByteMemoryPool
{
///
/// Represents a that wraps an array rented from
diff --git a/src/Ryujinx.Common/Memory/ByteMemoryPool.cs b/src/Ryujinx.Common/Memory/ByteMemoryPool.cs
index 071f56b136..6fd6a98aa7 100644
--- a/src/Ryujinx.Common/Memory/ByteMemoryPool.cs
+++ b/src/Ryujinx.Common/Memory/ByteMemoryPool.cs
@@ -6,24 +6,8 @@ namespace Ryujinx.Common.Memory
///
/// Provides a pool of re-usable byte array instances.
///
- public sealed partial class ByteMemoryPool
+ public static partial class ByteMemoryPool
{
- private static readonly ByteMemoryPool _shared = new();
-
- ///
- /// Constructs a instance. Private to force access through
- /// the instance.
- ///
- private ByteMemoryPool()
- {
- // No implementation
- }
-
- ///
- /// Retrieves a shared instance.
- ///
- public static ByteMemoryPool Shared => _shared;
-
///
/// Returns the maximum buffer size supported by this pool.
///
@@ -95,6 +79,20 @@ namespace Ryujinx.Common.Memory
return buffer;
}
+ ///
+ /// Copies into a newly rented byte memory buffer.
+ ///
+ /// The byte buffer to copy
+ /// A wrapping the rented memory with copied to it
+ public static IMemoryOwner RentCopy(ReadOnlySpan buffer)
+ {
+ var copy = RentImpl(buffer.Length);
+
+ buffer.CopyTo(copy.Memory.Span);
+
+ return copy;
+ }
+
private static ByteMemoryPoolBuffer RentImpl(int length)
{
if ((uint)length > Array.MaxLength)
diff --git a/src/Ryujinx.Common/Memory/SpanOrArray.cs b/src/Ryujinx.Common/Memory/SpanOrArray.cs
deleted file mode 100644
index 269ac02fd6..0000000000
--- a/src/Ryujinx.Common/Memory/SpanOrArray.cs
+++ /dev/null
@@ -1,89 +0,0 @@
-using System;
-
-namespace Ryujinx.Common.Memory
-{
- ///
- /// A struct that can represent both a Span and Array.
- /// This is useful to keep the Array representation when possible to avoid copies.
- ///
- /// Element Type
- public readonly ref struct SpanOrArray where T : unmanaged
- {
- public readonly T[] Array;
- public readonly ReadOnlySpan Span;
-
- ///
- /// Create a new SpanOrArray from an array.
- ///
- /// Array to store
- public SpanOrArray(T[] array)
- {
- Array = array;
- Span = ReadOnlySpan.Empty;
- }
-
- ///
- /// Create a new SpanOrArray from a readonly span.
- ///
- /// Span to store
- public SpanOrArray(ReadOnlySpan span)
- {
- Array = null;
- Span = span;
- }
-
- ///
- /// Return the contained array, or convert the span if necessary.
- ///
- /// An array containing the data
- public T[] ToArray()
- {
- return Array ?? Span.ToArray();
- }
-
- ///
- /// Return a ReadOnlySpan from either the array or ReadOnlySpan.
- ///
- /// A ReadOnlySpan containing the data
- public ReadOnlySpan AsSpan()
- {
- return Array ?? Span;
- }
-
- ///
- /// Cast an array to a SpanOrArray.
- ///
- /// Source array
- public static implicit operator SpanOrArray(T[] array)
- {
- return new SpanOrArray(array);
- }
-
- ///
- /// Cast a ReadOnlySpan to a SpanOrArray.
- ///
- /// Source ReadOnlySpan
- public static implicit operator SpanOrArray(ReadOnlySpan span)
- {
- return new SpanOrArray(span);
- }
-
- ///
- /// Cast a Span to a SpanOrArray.
- ///
- /// Source Span
- public static implicit operator SpanOrArray(Span span)
- {
- return new SpanOrArray(span);
- }
-
- ///
- /// Cast a SpanOrArray to a ReadOnlySpan
- ///
- /// Source SpanOrArray
- public static implicit operator ReadOnlySpan(SpanOrArray spanOrArray)
- {
- return spanOrArray.AsSpan();
- }
- }
-}
diff --git a/src/Ryujinx.Common/Utilities/EmbeddedResources.cs b/src/Ryujinx.Common/Utilities/EmbeddedResources.cs
index a4facc2e37..e22571c966 100644
--- a/src/Ryujinx.Common/Utilities/EmbeddedResources.cs
+++ b/src/Ryujinx.Common/Utilities/EmbeddedResources.cs
@@ -1,5 +1,6 @@
using Ryujinx.Common.Utilities;
using System;
+using System.Buffers;
using System.IO;
using System.Linq;
using System.Reflection;
@@ -41,6 +42,22 @@ namespace Ryujinx.Common
return StreamUtils.StreamToBytes(stream);
}
+ public static IMemoryOwner ReadFileToRentedMemory(string filename)
+ {
+ var (assembly, path) = ResolveManifestPath(filename);
+
+ return ReadFileToRentedMemory(assembly, path);
+ }
+
+ public static IMemoryOwner ReadFileToRentedMemory(Assembly assembly, string filename)
+ {
+ using var stream = GetStream(assembly, filename);
+
+ return stream is null
+ ? null
+ : StreamUtils.StreamToRentedMemory(stream);
+ }
+
public async static Task ReadAsync(Assembly assembly, string filename)
{
using var stream = GetStream(assembly, filename);
diff --git a/src/Ryujinx.Common/Utilities/StreamUtils.cs b/src/Ryujinx.Common/Utilities/StreamUtils.cs
index 7a20c98e95..74b6af5ecf 100644
--- a/src/Ryujinx.Common/Utilities/StreamUtils.cs
+++ b/src/Ryujinx.Common/Utilities/StreamUtils.cs
@@ -1,4 +1,6 @@
+using Microsoft.IO;
using Ryujinx.Common.Memory;
+using System.Buffers;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
@@ -9,12 +11,50 @@ namespace Ryujinx.Common.Utilities
{
public static byte[] StreamToBytes(Stream input)
{
- using MemoryStream stream = MemoryStreamManager.Shared.GetStream();
+ using RecyclableMemoryStream output = StreamToRecyclableMemoryStream(input);
+ return output.ToArray();
+ }
- input.CopyTo(stream);
+ public static IMemoryOwner StreamToRentedMemory(Stream input)
+ {
+ if (input is MemoryStream inputMemoryStream)
+ {
+ return MemoryStreamToRentedMemory(inputMemoryStream);
+ }
+ else if (input.CanSeek)
+ {
+ long bytesExpected = input.Length;
- return stream.ToArray();
+ IMemoryOwner ownedMemory = ByteMemoryPool.Rent(bytesExpected);
+
+ var destSpan = ownedMemory.Memory.Span;
+
+ int totalBytesRead = 0;
+
+ while (totalBytesRead < bytesExpected)
+ {
+ int bytesRead = input.Read(destSpan[totalBytesRead..]);
+
+ if (bytesRead == 0)
+ {
+ ownedMemory.Dispose();
+
+ throw new IOException($"Tried reading {bytesExpected} but the stream closed after reading {totalBytesRead}.");
+ }
+
+ totalBytesRead += bytesRead;
+ }
+
+ return ownedMemory;
+ }
+ else
+ {
+ // If input is (non-seekable) then copy twice: first into a RecyclableMemoryStream, then to a rented IMemoryOwner.
+ using RecyclableMemoryStream output = StreamToRecyclableMemoryStream(input);
+
+ return MemoryStreamToRentedMemory(output);
+ }
}
public static async Task StreamToBytesAsync(Stream input, CancellationToken cancellationToken = default)
@@ -25,5 +65,26 @@ namespace Ryujinx.Common.Utilities
return stream.ToArray();
}
+
+ private static IMemoryOwner MemoryStreamToRentedMemory(MemoryStream input)
+ {
+ input.Position = 0;
+
+ IMemoryOwner ownedMemory = ByteMemoryPool.Rent(input.Length);
+
+ // Discard the return value because we assume reading a MemoryStream always succeeds completely.
+ _ = input.Read(ownedMemory.Memory.Span);
+
+ return ownedMemory;
+ }
+
+ private static RecyclableMemoryStream StreamToRecyclableMemoryStream(Stream input)
+ {
+ RecyclableMemoryStream stream = MemoryStreamManager.Shared.GetStream();
+
+ input.CopyTo(stream);
+
+ return stream;
+ }
}
}
diff --git a/src/Ryujinx.Cpu/AddressSpace.cs b/src/Ryujinx.Cpu/AddressSpace.cs
index beea14beec..6664ed1345 100644
--- a/src/Ryujinx.Cpu/AddressSpace.cs
+++ b/src/Ryujinx.Cpu/AddressSpace.cs
@@ -1,5 +1,3 @@
-using Ryujinx.Common;
-using Ryujinx.Common.Collections;
using Ryujinx.Memory;
using System;
@@ -7,175 +5,23 @@ namespace Ryujinx.Cpu
{
public class AddressSpace : IDisposable
{
- private const int DefaultBlockAlignment = 1 << 20;
-
- private enum MappingType : byte
- {
- None,
- Private,
- Shared,
- }
-
- private class Mapping : IntrusiveRedBlackTreeNode, IComparable
- {
- public ulong Address { get; private set; }
- public ulong Size { get; private set; }
- public ulong EndAddress => Address + Size;
- public MappingType Type { get; private set; }
-
- public Mapping(ulong address, ulong size, MappingType type)
- {
- Address = address;
- Size = size;
- Type = type;
- }
-
- public Mapping Split(ulong splitAddress)
- {
- ulong leftSize = splitAddress - Address;
- ulong rightSize = EndAddress - splitAddress;
-
- Mapping left = new(Address, leftSize, Type);
-
- Address = splitAddress;
- Size = rightSize;
-
- return left;
- }
-
- public void UpdateState(MappingType newType)
- {
- Type = newType;
- }
-
- public void Extend(ulong sizeDelta)
- {
- Size += sizeDelta;
- }
-
- public int CompareTo(Mapping other)
- {
- if (Address < other.Address)
- {
- return -1;
- }
- else if (Address <= other.EndAddress - 1UL)
- {
- return 0;
- }
- else
- {
- return 1;
- }
- }
- }
-
- private class PrivateMapping : IntrusiveRedBlackTreeNode, IComparable
- {
- public ulong Address { get; private set; }
- public ulong Size { get; private set; }
- public ulong EndAddress => Address + Size;
- public PrivateMemoryAllocation PrivateAllocation { get; private set; }
-
- public PrivateMapping(ulong address, ulong size, PrivateMemoryAllocation privateAllocation)
- {
- Address = address;
- Size = size;
- PrivateAllocation = privateAllocation;
- }
-
- public PrivateMapping Split(ulong splitAddress)
- {
- ulong leftSize = splitAddress - Address;
- ulong rightSize = EndAddress - splitAddress;
-
- (var leftAllocation, PrivateAllocation) = PrivateAllocation.Split(leftSize);
-
- PrivateMapping left = new(Address, leftSize, leftAllocation);
-
- Address = splitAddress;
- Size = rightSize;
-
- return left;
- }
-
- public void Map(MemoryBlock baseBlock, MemoryBlock mirrorBlock, PrivateMemoryAllocation newAllocation)
- {
- baseBlock.MapView(newAllocation.Memory, newAllocation.Offset, Address, Size);
- mirrorBlock.MapView(newAllocation.Memory, newAllocation.Offset, Address, Size);
- PrivateAllocation = newAllocation;
- }
-
- public void Unmap(MemoryBlock baseBlock, MemoryBlock mirrorBlock)
- {
- if (PrivateAllocation.IsValid)
- {
- baseBlock.UnmapView(PrivateAllocation.Memory, Address, Size);
- mirrorBlock.UnmapView(PrivateAllocation.Memory, Address, Size);
- PrivateAllocation.Dispose();
- }
-
- PrivateAllocation = default;
- }
-
- public void Extend(ulong sizeDelta)
- {
- Size += sizeDelta;
- }
-
- public int CompareTo(PrivateMapping other)
- {
- if (Address < other.Address)
- {
- return -1;
- }
- else if (Address <= other.EndAddress - 1UL)
- {
- return 0;
- }
- else
- {
- return 1;
- }
- }
- }
-
private readonly MemoryBlock _backingMemory;
- private readonly PrivateMemoryAllocator _privateMemoryAllocator;
- private readonly IntrusiveRedBlackTree _mappingTree;
- private readonly IntrusiveRedBlackTree _privateTree;
-
- private readonly object _treeLock;
-
- private readonly bool _supports4KBPages;
public MemoryBlock Base { get; }
public MemoryBlock Mirror { get; }
public ulong AddressSpaceSize { get; }
- public AddressSpace(MemoryBlock backingMemory, MemoryBlock baseMemory, MemoryBlock mirrorMemory, ulong addressSpaceSize, bool supports4KBPages)
+ public AddressSpace(MemoryBlock backingMemory, MemoryBlock baseMemory, MemoryBlock mirrorMemory, ulong addressSpaceSize)
{
- if (!supports4KBPages)
- {
- _privateMemoryAllocator = new PrivateMemoryAllocator(DefaultBlockAlignment, MemoryAllocationFlags.Mirrorable | MemoryAllocationFlags.NoMap);
- _mappingTree = new IntrusiveRedBlackTree();
- _privateTree = new IntrusiveRedBlackTree();
- _treeLock = new object();
-
- _mappingTree.Add(new Mapping(0UL, addressSpaceSize, MappingType.None));
- _privateTree.Add(new PrivateMapping(0UL, addressSpaceSize, default));
- }
-
_backingMemory = backingMemory;
- _supports4KBPages = supports4KBPages;
Base = baseMemory;
Mirror = mirrorMemory;
AddressSpaceSize = addressSpaceSize;
}
- public static bool TryCreate(MemoryBlock backingMemory, ulong asSize, bool supports4KBPages, out AddressSpace addressSpace)
+ public static bool TryCreate(MemoryBlock backingMemory, ulong asSize, out AddressSpace addressSpace)
{
addressSpace = null;
@@ -193,7 +39,7 @@ namespace Ryujinx.Cpu
{
baseMemory = new MemoryBlock(addressSpaceSize, AsFlags);
mirrorMemory = new MemoryBlock(addressSpaceSize, AsFlags);
- addressSpace = new AddressSpace(backingMemory, baseMemory, mirrorMemory, addressSpaceSize, supports4KBPages);
+ addressSpace = new AddressSpace(backingMemory, baseMemory, mirrorMemory, addressSpaceSize);
break;
}
@@ -209,289 +55,20 @@ namespace Ryujinx.Cpu
public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags)
{
- if (_supports4KBPages)
- {
- Base.MapView(_backingMemory, pa, va, size);
- Mirror.MapView(_backingMemory, pa, va, size);
-
- return;
- }
-
- lock (_treeLock)
- {
- ulong alignment = MemoryBlock.GetPageSize();
- bool isAligned = ((va | pa | size) & (alignment - 1)) == 0;
-
- if (flags.HasFlag(MemoryMapFlags.Private) && !isAligned)
- {
- Update(va, pa, size, MappingType.Private);
- }
- else
- {
- // The update method assumes that shared mappings are already aligned.
-
- if (!flags.HasFlag(MemoryMapFlags.Private))
- {
- if ((va & (alignment - 1)) != (pa & (alignment - 1)))
- {
- throw new InvalidMemoryRegionException($"Virtual address 0x{va:X} and physical address 0x{pa:X} are misaligned and can't be aligned.");
- }
-
- ulong endAddress = va + size;
- va = BitUtils.AlignDown(va, alignment);
- pa = BitUtils.AlignDown(pa, alignment);
- size = BitUtils.AlignUp(endAddress, alignment) - va;
- }
-
- Update(va, pa, size, MappingType.Shared);
- }
- }
+ Base.MapView(_backingMemory, pa, va, size);
+ Mirror.MapView(_backingMemory, pa, va, size);
}
public void Unmap(ulong va, ulong size)
{
- if (_supports4KBPages)
- {
- Base.UnmapView(_backingMemory, va, size);
- Mirror.UnmapView(_backingMemory, va, size);
-
- return;
- }
-
- lock (_treeLock)
- {
- Update(va, 0UL, size, MappingType.None);
- }
- }
-
- private void Update(ulong va, ulong pa, ulong size, MappingType type)
- {
- Mapping map = _mappingTree.GetNode(new Mapping(va, 1UL, MappingType.None));
-
- Update(map, va, pa, size, type);
- }
-
- private Mapping Update(Mapping map, ulong va, ulong pa, ulong size, MappingType type)
- {
- ulong endAddress = va + size;
-
- for (; map != null; map = map.Successor)
- {
- if (map.Address < va)
- {
- _mappingTree.Add(map.Split(va));
- }
-
- if (map.EndAddress > endAddress)
- {
- Mapping newMap = map.Split(endAddress);
- _mappingTree.Add(newMap);
- map = newMap;
- }
-
- switch (type)
- {
- case MappingType.None:
- if (map.Type == MappingType.Shared)
- {
- ulong startOffset = map.Address - va;
- ulong mapVa = va + startOffset;
- ulong mapSize = Math.Min(size - startOffset, map.Size);
- ulong mapEndAddress = mapVa + mapSize;
- ulong alignment = MemoryBlock.GetPageSize();
-
- mapVa = BitUtils.AlignDown(mapVa, alignment);
- mapEndAddress = BitUtils.AlignUp(mapEndAddress, alignment);
-
- mapSize = mapEndAddress - mapVa;
-
- Base.UnmapView(_backingMemory, mapVa, mapSize);
- Mirror.UnmapView(_backingMemory, mapVa, mapSize);
- }
- else
- {
- UnmapPrivate(va, size);
- }
- break;
- case MappingType.Private:
- if (map.Type == MappingType.Shared)
- {
- throw new InvalidMemoryRegionException($"Private mapping request at 0x{va:X} with size 0x{size:X} overlaps shared mapping at 0x{map.Address:X} with size 0x{map.Size:X}.");
- }
- else
- {
- MapPrivate(va, size);
- }
- break;
- case MappingType.Shared:
- if (map.Type != MappingType.None)
- {
- throw new InvalidMemoryRegionException($"Shared mapping request at 0x{va:X} with size 0x{size:X} overlaps mapping at 0x{map.Address:X} with size 0x{map.Size:X}.");
- }
- else
- {
- ulong startOffset = map.Address - va;
- ulong mapPa = pa + startOffset;
- ulong mapVa = va + startOffset;
- ulong mapSize = Math.Min(size - startOffset, map.Size);
-
- Base.MapView(_backingMemory, mapPa, mapVa, mapSize);
- Mirror.MapView(_backingMemory, mapPa, mapVa, mapSize);
- }
- break;
- }
-
- map.UpdateState(type);
- map = TryCoalesce(map);
-
- if (map.EndAddress >= endAddress)
- {
- break;
- }
- }
-
- return map;
- }
-
- private Mapping TryCoalesce(Mapping map)
- {
- Mapping previousMap = map.Predecessor;
- Mapping nextMap = map.Successor;
-
- if (previousMap != null && CanCoalesce(previousMap, map))
- {
- previousMap.Extend(map.Size);
- _mappingTree.Remove(map);
- map = previousMap;
- }
-
- if (nextMap != null && CanCoalesce(map, nextMap))
- {
- map.Extend(nextMap.Size);
- _mappingTree.Remove(nextMap);
- }
-
- return map;
- }
-
- private static bool CanCoalesce(Mapping left, Mapping right)
- {
- return left.Type == right.Type;
- }
-
- private void MapPrivate(ulong va, ulong size)
- {
- ulong endAddress = va + size;
-
- ulong alignment = MemoryBlock.GetPageSize();
-
- // Expand the range outwards based on page size to ensure that at least the requested region is mapped.
- ulong vaAligned = BitUtils.AlignDown(va, alignment);
- ulong endAddressAligned = BitUtils.AlignUp(endAddress, alignment);
-
- PrivateMapping map = _privateTree.GetNode(new PrivateMapping(va, 1UL, default));
-
- for (; map != null; map = map.Successor)
- {
- if (!map.PrivateAllocation.IsValid)
- {
- if (map.Address < vaAligned)
- {
- _privateTree.Add(map.Split(vaAligned));
- }
-
- if (map.EndAddress > endAddressAligned)
- {
- PrivateMapping newMap = map.Split(endAddressAligned);
- _privateTree.Add(newMap);
- map = newMap;
- }
-
- map.Map(Base, Mirror, _privateMemoryAllocator.Allocate(map.Size, MemoryBlock.GetPageSize()));
- }
-
- if (map.EndAddress >= endAddressAligned)
- {
- break;
- }
- }
- }
-
- private void UnmapPrivate(ulong va, ulong size)
- {
- ulong endAddress = va + size;
-
- ulong alignment = MemoryBlock.GetPageSize();
-
- // Shrink the range inwards based on page size to ensure we won't unmap memory that might be still in use.
- ulong vaAligned = BitUtils.AlignUp(va, alignment);
- ulong endAddressAligned = BitUtils.AlignDown(endAddress, alignment);
-
- if (endAddressAligned <= vaAligned)
- {
- return;
- }
-
- PrivateMapping map = _privateTree.GetNode(new PrivateMapping(va, 1UL, default));
-
- for (; map != null; map = map.Successor)
- {
- if (map.PrivateAllocation.IsValid)
- {
- if (map.Address < vaAligned)
- {
- _privateTree.Add(map.Split(vaAligned));
- }
-
- if (map.EndAddress > endAddressAligned)
- {
- PrivateMapping newMap = map.Split(endAddressAligned);
- _privateTree.Add(newMap);
- map = newMap;
- }
-
- map.Unmap(Base, Mirror);
- map = TryCoalesce(map);
- }
-
- if (map.EndAddress >= endAddressAligned)
- {
- break;
- }
- }
- }
-
- private PrivateMapping TryCoalesce(PrivateMapping map)
- {
- PrivateMapping previousMap = map.Predecessor;
- PrivateMapping nextMap = map.Successor;
-
- if (previousMap != null && CanCoalesce(previousMap, map))
- {
- previousMap.Extend(map.Size);
- _privateTree.Remove(map);
- map = previousMap;
- }
-
- if (nextMap != null && CanCoalesce(map, nextMap))
- {
- map.Extend(nextMap.Size);
- _privateTree.Remove(nextMap);
- }
-
- return map;
- }
-
- private static bool CanCoalesce(PrivateMapping left, PrivateMapping right)
- {
- return !left.PrivateAllocation.IsValid && !right.PrivateAllocation.IsValid;
+ Base.UnmapView(_backingMemory, va, size);
+ Mirror.UnmapView(_backingMemory, va, size);
}
public void Dispose()
{
GC.SuppressFinalize(this);
- _privateMemoryAllocator?.Dispose();
Base.Dispose();
Mirror.Dispose();
}
diff --git a/src/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocator.cs b/src/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocator.cs
index 4e3723d554..86936c5929 100644
--- a/src/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocator.cs
+++ b/src/Ryujinx.Cpu/AppleHv/HvMemoryBlockAllocator.cs
@@ -38,7 +38,7 @@ namespace Ryujinx.Cpu.AppleHv
private readonly HvIpaAllocator _ipaAllocator;
- public HvMemoryBlockAllocator(HvIpaAllocator ipaAllocator, int blockAlignment) : base(blockAlignment, MemoryAllocationFlags.None)
+ public HvMemoryBlockAllocator(HvIpaAllocator ipaAllocator, ulong blockAlignment) : base(blockAlignment, MemoryAllocationFlags.None)
{
_ipaAllocator = ipaAllocator;
}
diff --git a/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs b/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs
index 6e864f4ca6..abdddb31c3 100644
--- a/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs
+++ b/src/Ryujinx.Cpu/AppleHv/HvMemoryManager.cs
@@ -3,12 +3,11 @@ using Ryujinx.Memory;
using Ryujinx.Memory.Range;
using Ryujinx.Memory.Tracking;
using System;
+using System.Buffers;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
using System.Runtime.Versioning;
-using System.Threading;
namespace Ryujinx.Cpu.AppleHv
{
@@ -16,23 +15,8 @@ namespace Ryujinx.Cpu.AppleHv
/// Represents a CPU memory manager which maps guest virtual memory directly onto the Hypervisor page table.
///
[SupportedOSPlatform("macos")]
- public class HvMemoryManager : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock
+ public sealed class HvMemoryManager : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked
{
- public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry.
- public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set.
-
- private enum HostMappedPtBits : ulong
- {
- Unmapped = 0,
- Mapped,
- WriteTracked,
- ReadWriteTracked,
-
- MappedReplicated = 0x5555555555555555,
- WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa,
- ReadWriteTrackedReplicated = ulong.MaxValue,
- }
-
private readonly InvalidAccessHandler _invalidAccessHandler;
private readonly HvAddressSpace _addressSpace;
@@ -42,9 +26,9 @@ namespace Ryujinx.Cpu.AppleHv
private readonly MemoryBlock _backingMemory;
private readonly PageTable _pageTable;
- private readonly ulong[] _pageBitmap;
+ private readonly ManagedPageFlags _pages;
- public bool Supports4KBPages => true;
+ public bool UsesPrivateAllocations => false;
public int AddressSpaceBits { get; }
@@ -84,7 +68,7 @@ namespace Ryujinx.Cpu.AppleHv
AddressSpaceBits = asBits;
- _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))];
+ _pages = new ManagedPageFlags(AddressSpaceBits);
Tracking = new MemoryTracking(this, PageSize, invalidAccessHandler);
}
@@ -95,7 +79,7 @@ namespace Ryujinx.Cpu.AppleHv
PtMap(va, pa, size);
_addressSpace.MapUser(va, pa, size, MemoryPermission.ReadWriteExecute);
- AddMapping(va, size);
+ _pages.AddMapping(va, size);
Tracking.Map(va, size);
}
@@ -112,12 +96,6 @@ namespace Ryujinx.Cpu.AppleHv
}
}
- ///
- public void MapForeign(ulong va, nuint hostPointer, ulong size)
- {
- throw new NotSupportedException();
- }
-
///
public void Unmap(ulong va, ulong size)
{
@@ -126,7 +104,7 @@ namespace Ryujinx.Cpu.AppleHv
UnmapEvent?.Invoke(va, size);
Tracking.Unmap(va, size);
- RemoveMapping(va, size);
+ _pages.RemoveMapping(va, size);
_addressSpace.UnmapUser(va, size);
PtUnmap(va, size);
}
@@ -142,20 +120,11 @@ namespace Ryujinx.Cpu.AppleHv
}
}
- ///
- public T Read(ulong va) where T : unmanaged
- {
- return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0];
- }
-
- ///
- public T ReadTracked(ulong va) where T : unmanaged
+ public override T ReadTracked(ulong va)
{
try
{
- SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false);
-
- return Read(va);
+ return base.ReadTracked(va);
}
catch (InvalidMemoryRegionException)
{
@@ -168,7 +137,6 @@ namespace Ryujinx.Cpu.AppleHv
}
}
- ///
public override void Read(ulong va, Span data)
{
try
@@ -184,101 +152,11 @@ namespace Ryujinx.Cpu.AppleHv
}
}
- ///
- public void Write(ulong va, T value) where T : unmanaged
- {
- Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1)));
- }
-
- ///
- public void Write(ulong va, ReadOnlySpan data)
- {
- if (data.Length == 0)
- {
- return;
- }
-
- SignalMemoryTracking(va, (ulong)data.Length, true);
-
- WriteImpl(va, data);
- }
-
- ///
- public void WriteUntracked(ulong va, ReadOnlySpan data)
- {
- if (data.Length == 0)
- {
- return;
- }
-
- WriteImpl(va, data);
- }
-
- ///
- public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data)
- {
- if (data.Length == 0)
- {
- return false;
- }
-
- SignalMemoryTracking(va, (ulong)data.Length, false);
-
- if (IsContiguousAndMapped(va, data.Length))
- {
- var target = _backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length);
-
- bool changed = !data.SequenceEqual(target);
-
- if (changed)
- {
- data.CopyTo(target);
- }
-
- return changed;
- }
- else
- {
- WriteImpl(va, data);
-
- return true;
- }
- }
-
- private void WriteImpl(ulong va, ReadOnlySpan data)
+ public override void Write(ulong va, ReadOnlySpan data)
{
try
{
- AssertValidAddressAndSize(va, (ulong)data.Length);
-
- if (IsContiguousAndMapped(va, data.Length))
- {
- data.CopyTo(_backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length));
- }
- else
- {
- int offset = 0, size;
-
- if ((va & PageMask) != 0)
- {
- ulong pa = GetPhysicalAddressChecked(va);
-
- size = Math.Min(data.Length, PageSize - (int)(va & PageMask));
-
- data[..size].CopyTo(_backingMemory.GetSpan(pa, size));
-
- offset += size;
- }
-
- for (; offset < data.Length; offset += size)
- {
- ulong pa = GetPhysicalAddressChecked(va + (ulong)offset);
-
- size = Math.Min(data.Length - offset, PageSize);
-
- data.Slice(offset, size).CopyTo(_backingMemory.GetSpan(pa, size));
- }
- }
+ base.Write(va, data);
}
catch (InvalidMemoryRegionException)
{
@@ -289,61 +167,38 @@ namespace Ryujinx.Cpu.AppleHv
}
}
- ///
- public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false)
+ public override void WriteUntracked(ulong va, ReadOnlySpan data)
{
- if (size == 0)
+ try
{
- return ReadOnlySpan.Empty;
+ base.WriteUntracked(va, data);
}
-
- if (tracked)
+ catch (InvalidMemoryRegionException)
{
- SignalMemoryTracking(va, (ulong)size, false);
- }
-
- if (IsContiguousAndMapped(va, size))
- {
- return _backingMemory.GetSpan(GetPhysicalAddressInternal(va), size);
- }
- else
- {
- Span data = new byte[size];
-
- base.Read(va, data);
-
- return data;
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
}
}
- ///
- public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
+ public override ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false)
{
- if (size == 0)
+ try
{
- return new WritableRegion(null, va, Memory.Empty);
+ return base.GetReadOnlySequence(va, size, tracked);
}
-
- if (tracked)
+ catch (InvalidMemoryRegionException)
{
- SignalMemoryTracking(va, (ulong)size, true);
- }
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
- if (IsContiguousAndMapped(va, size))
- {
- return new WritableRegion(null, va, _backingMemory.GetMemory(GetPhysicalAddressInternal(va), size));
- }
- else
- {
- Memory memory = new byte[size];
-
- base.Read(va, memory.Span);
-
- return new WritableRegion(this, va, memory);
+ return ReadOnlySequence.Empty;
}
}
- ///
public ref T GetRef(ulong va) where T : unmanaged
{
if (!IsContiguous(va, Unsafe.SizeOf()))
@@ -356,26 +211,10 @@ namespace Ryujinx.Cpu.AppleHv
return ref _backingMemory.GetRef(GetPhysicalAddressChecked(va));
}
- ///
[MethodImpl(MethodImplOptions.AggressiveInlining)]
- public bool IsMapped(ulong va)
+ public override bool IsMapped(ulong va)
{
- return ValidateAddress(va) && IsMappedImpl(va);
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private bool IsMappedImpl(ulong va)
- {
- ulong page = va >> PageBits;
-
- int bit = (int)((page & 31) << 1);
-
- int pageIndex = (int)(page >> PageToPteShift);
- ref ulong pageRef = ref _pageBitmap[pageIndex];
-
- ulong pte = Volatile.Read(ref pageRef);
-
- return ((pte >> bit) & 3) != 0;
+ return ValidateAddress(va) && _pages.IsMapped(va);
}
///
@@ -383,91 +222,7 @@ namespace Ryujinx.Cpu.AppleHv
{
AssertValidAddressAndSize(va, size);
- return IsRangeMappedImpl(va, size);
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex)
- {
- startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1);
- endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1));
-
- pageIndex = (int)(pageStart >> PageToPteShift);
- pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift);
- }
-
- private bool IsRangeMappedImpl(ulong va, ulong size)
- {
- int pages = GetPagesCount(va, size, out _);
-
- if (pages == 1)
- {
- return IsMappedImpl(va);
- }
-
- ulong pageStart = va >> PageBits;
- ulong pageEnd = pageStart + (ulong)pages;
-
- GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
-
- // Check if either bit in each 2 bit page entry is set.
- // OR the block with itself shifted down by 1, and check the first bit of each entry.
-
- ulong mask = BlockMappedMask & startMask;
-
- while (pageIndex <= pageEndIndex)
- {
- if (pageIndex == pageEndIndex)
- {
- mask &= endMask;
- }
-
- ref ulong pageRef = ref _pageBitmap[pageIndex++];
- ulong pte = Volatile.Read(ref pageRef);
-
- pte |= pte >> 1;
- if ((pte & mask) != mask)
- {
- return false;
- }
-
- mask = BlockMappedMask;
- }
-
- return true;
- }
-
- private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException();
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va);
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private bool IsContiguous(ulong va, int size)
- {
- if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size))
- {
- return false;
- }
-
- int pages = GetPagesCount(va, (uint)size, out va);
-
- for (int page = 0; page < pages - 1; page++)
- {
- if (!ValidateAddress(va + PageSize))
- {
- return false;
- }
-
- if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize))
- {
- return false;
- }
-
- va += PageSize;
- }
-
- return true;
+ return _pages.IsRangeMapped(va, size);
}
///
@@ -546,11 +301,10 @@ namespace Ryujinx.Cpu.AppleHv
return regions;
}
- ///
///
/// This function also validates that the given range is both valid and mapped, and will throw if it is not.
///
- public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
+ public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
{
AssertValidAddressAndSize(va, size);
@@ -560,199 +314,37 @@ namespace Ryujinx.Cpu.AppleHv
return;
}
- // Software table, used for managed memory tracking.
-
- int pages = GetPagesCount(va, size, out _);
- ulong pageStart = va >> PageBits;
-
- if (pages == 1)
- {
- ulong tag = (ulong)(write ? HostMappedPtBits.WriteTracked : HostMappedPtBits.ReadWriteTracked);
-
- int bit = (int)((pageStart & 31) << 1);
-
- int pageIndex = (int)(pageStart >> PageToPteShift);
- ref ulong pageRef = ref _pageBitmap[pageIndex];
-
- ulong pte = Volatile.Read(ref pageRef);
- ulong state = ((pte >> bit) & 3);
-
- if (state >= tag)
- {
- Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
- return;
- }
- else if (state == 0)
- {
- ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
- }
- }
- else
- {
- ulong pageEnd = pageStart + (ulong)pages;
-
- GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
-
- ulong mask = startMask;
-
- ulong anyTrackingTag = (ulong)HostMappedPtBits.WriteTrackedReplicated;
-
- while (pageIndex <= pageEndIndex)
- {
- if (pageIndex == pageEndIndex)
- {
- mask &= endMask;
- }
-
- ref ulong pageRef = ref _pageBitmap[pageIndex++];
-
- ulong pte = Volatile.Read(ref pageRef);
- ulong mappedMask = mask & BlockMappedMask;
-
- ulong mappedPte = pte | (pte >> 1);
- if ((mappedPte & mappedMask) != mappedMask)
- {
- ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
- }
-
- pte &= mask;
- if ((pte & anyTrackingTag) != 0) // Search for any tracking.
- {
- // Writes trigger any tracking.
- // Only trigger tracking from reads if both bits are set on any page.
- if (write || (pte & (pte >> 1) & BlockMappedMask) != 0)
- {
- Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
- break;
- }
- }
-
- mask = ulong.MaxValue;
- }
- }
+ _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId);
}
- ///
- /// Computes the number of pages in a virtual address range.
- ///
- /// Virtual address of the range
- /// Size of the range
- /// The virtual address of the beginning of the first page
- /// This function does not differentiate between allocated and unallocated pages.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static int GetPagesCount(ulong va, ulong size, out ulong startVa)
- {
- // WARNING: Always check if ulong does not overflow during the operations.
- startVa = va & ~(ulong)PageMask;
- ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask;
-
- return (int)(vaSpan / PageSize);
- }
-
- ///
public void Reprotect(ulong va, ulong size, MemoryPermission protection)
{
// TODO
}
///
- public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection)
+ public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest)
{
- // Protection is inverted on software pages, since the default value is 0.
- protection = (~protection) & MemoryPermission.ReadAndWrite;
-
- int pages = GetPagesCount(va, size, out va);
- ulong pageStart = va >> PageBits;
-
- if (pages == 1)
+ if (guest)
{
- ulong protTag = protection switch
- {
- MemoryPermission.None => (ulong)HostMappedPtBits.Mapped,
- MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTracked,
- _ => (ulong)HostMappedPtBits.ReadWriteTracked,
- };
-
- int bit = (int)((pageStart & 31) << 1);
-
- ulong tagMask = 3UL << bit;
- ulong invTagMask = ~tagMask;
-
- ulong tag = protTag << bit;
-
- int pageIndex = (int)(pageStart >> PageToPteShift);
- ref ulong pageRef = ref _pageBitmap[pageIndex];
-
- ulong pte;
-
- do
- {
- pte = Volatile.Read(ref pageRef);
- }
- while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte);
+ _addressSpace.ReprotectUser(va, size, protection);
}
else
{
- ulong pageEnd = pageStart + (ulong)pages;
-
- GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
-
- ulong mask = startMask;
-
- ulong protTag = protection switch
- {
- MemoryPermission.None => (ulong)HostMappedPtBits.MappedReplicated,
- MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTrackedReplicated,
- _ => (ulong)HostMappedPtBits.ReadWriteTrackedReplicated,
- };
-
- while (pageIndex <= pageEndIndex)
- {
- if (pageIndex == pageEndIndex)
- {
- mask &= endMask;
- }
-
- ref ulong pageRef = ref _pageBitmap[pageIndex++];
-
- ulong pte;
- ulong mappedMask;
-
- // Change the protection of all 2 bit entries that are mapped.
- do
- {
- pte = Volatile.Read(ref pageRef);
-
- mappedMask = pte | (pte >> 1);
- mappedMask |= (mappedMask & BlockMappedMask) << 1;
- mappedMask &= mask; // Only update mapped pages within the given range.
- }
- while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte);
-
- mask = ulong.MaxValue;
- }
+ _pages.TrackingReprotect(va, size, protection);
}
-
- protection = protection switch
- {
- MemoryPermission.None => MemoryPermission.ReadAndWrite,
- MemoryPermission.Write => MemoryPermission.Read,
- _ => MemoryPermission.None,
- };
-
- _addressSpace.ReprotectUser(va, size, protection);
}
///
- public RegionHandle BeginTracking(ulong address, ulong size, int id)
+ public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None)
{
- return Tracking.BeginTracking(address, size, id);
+ return Tracking.BeginTracking(address, size, id, flags);
}
///
- public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id)
+ public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None)
{
- return Tracking.BeginGranularTracking(address, size, handles, granularity, id);
+ return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags);
}
///
@@ -761,87 +353,7 @@ namespace Ryujinx.Cpu.AppleHv
return Tracking.BeginSmartGranularTracking(address, size, granularity, id);
}
- ///
- /// Adds the given address mapping to the page table.
- ///
- /// Virtual memory address
- /// Size to be mapped
- private void AddMapping(ulong va, ulong size)
- {
- int pages = GetPagesCount(va, size, out _);
- ulong pageStart = va >> PageBits;
- ulong pageEnd = pageStart + (ulong)pages;
-
- GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
-
- ulong mask = startMask;
-
- while (pageIndex <= pageEndIndex)
- {
- if (pageIndex == pageEndIndex)
- {
- mask &= endMask;
- }
-
- ref ulong pageRef = ref _pageBitmap[pageIndex++];
-
- ulong pte;
- ulong mappedMask;
-
- // Map all 2-bit entries that are unmapped.
- do
- {
- pte = Volatile.Read(ref pageRef);
-
- mappedMask = pte | (pte >> 1);
- mappedMask |= (mappedMask & BlockMappedMask) << 1;
- mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged.
- }
- while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte);
-
- mask = ulong.MaxValue;
- }
- }
-
- ///
- /// Removes the given address mapping from the page table.
- ///
- /// Virtual memory address
- /// Size to be unmapped
- private void RemoveMapping(ulong va, ulong size)
- {
- int pages = GetPagesCount(va, size, out _);
- ulong pageStart = va >> PageBits;
- ulong pageEnd = pageStart + (ulong)pages;
-
- GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
-
- startMask = ~startMask;
- endMask = ~endMask;
-
- ulong mask = startMask;
-
- while (pageIndex <= pageEndIndex)
- {
- if (pageIndex == pageEndIndex)
- {
- mask |= endMask;
- }
-
- ref ulong pageRef = ref _pageBitmap[pageIndex++];
- ulong pte;
-
- do
- {
- pte = Volatile.Read(ref pageRef);
- }
- while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte);
-
- mask = 0;
- }
- }
-
- private ulong GetPhysicalAddressChecked(ulong va)
+ private nuint GetPhysicalAddressChecked(ulong va)
{
if (!IsMapped(va))
{
@@ -851,9 +363,9 @@ namespace Ryujinx.Cpu.AppleHv
return GetPhysicalAddressInternal(va);
}
- private ulong GetPhysicalAddressInternal(ulong va)
+ private nuint GetPhysicalAddressInternal(ulong va)
{
- return _pageTable.Read(va) + (va & PageMask);
+ return (nuint)(_pageTable.Read(va) + (va & PageMask));
}
///
@@ -864,10 +376,17 @@ namespace Ryujinx.Cpu.AppleHv
_addressSpace.Dispose();
}
- protected override Span GetPhysicalAddressSpan(ulong pa, int size)
+ protected override Memory GetPhysicalAddressMemory(nuint pa, int size)
+ => _backingMemory.GetMemory(pa, size);
+
+ protected override Span GetPhysicalAddressSpan(nuint pa, int size)
=> _backingMemory.GetSpan(pa, size);
- protected override ulong TranslateVirtualAddressForRead(ulong va)
+ protected override nuint TranslateVirtualAddressChecked(ulong va)
=> GetPhysicalAddressChecked(va);
+
+ protected override nuint TranslateVirtualAddressUnchecked(ulong va)
+ => GetPhysicalAddressInternal(va);
+
}
}
diff --git a/src/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs b/src/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs
index 199bff240e..e8d11ede5e 100644
--- a/src/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs
+++ b/src/Ryujinx.Cpu/IVirtualMemoryManagerTracked.cs
@@ -28,8 +28,9 @@ namespace Ryujinx.Cpu
/// CPU virtual address of the region
/// Size of the region
/// Handle ID
+ /// Region flags
/// The memory tracking handle
- RegionHandle BeginTracking(ulong address, ulong size, int id);
+ RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None);
///
/// Obtains a memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with.
@@ -39,8 +40,9 @@ namespace Ryujinx.Cpu
/// Handles to inherit state from or reuse. When none are present, provide null
/// Desired granularity of write tracking
/// Handle ID
+ /// Region flags
/// The memory tracking handle
- MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id);
+ MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None);
///
/// Obtains a smart memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with.
diff --git a/src/Ryujinx.Cpu/Jit/HostTracked/AddressIntrusiveRedBlackTree.cs b/src/Ryujinx.Cpu/Jit/HostTracked/AddressIntrusiveRedBlackTree.cs
new file mode 100644
index 0000000000..0e24433035
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/HostTracked/AddressIntrusiveRedBlackTree.cs
@@ -0,0 +1,35 @@
+using Ryujinx.Common.Collections;
+using System;
+
+namespace Ryujinx.Cpu.Jit.HostTracked
+{
+ internal class AddressIntrusiveRedBlackTree : IntrusiveRedBlackTree where T : IntrusiveRedBlackTreeNode, IComparable, IComparable
+ {
+ ///
+ /// Retrieve the node that is considered equal to the specified address by the comparator.
+ ///
+ /// Address to compare with
+ /// Node that is equal to
+ public T GetNode(ulong address)
+ {
+ T node = Root;
+ while (node != null)
+ {
+ int cmp = node.CompareTo(address);
+ if (cmp < 0)
+ {
+ node = node.Left;
+ }
+ else if (cmp > 0)
+ {
+ node = node.Right;
+ }
+ else
+ {
+ return node;
+ }
+ }
+ return null;
+ }
+ }
+}
diff --git a/src/Ryujinx.Cpu/Jit/HostTracked/AddressSpacePartition.cs b/src/Ryujinx.Cpu/Jit/HostTracked/AddressSpacePartition.cs
new file mode 100644
index 0000000000..224c5edc30
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/HostTracked/AddressSpacePartition.cs
@@ -0,0 +1,708 @@
+using Ryujinx.Common;
+using Ryujinx.Common.Collections;
+using Ryujinx.Memory;
+using System;
+using System.Diagnostics;
+using System.Threading;
+
+namespace Ryujinx.Cpu.Jit.HostTracked
+{
+ readonly struct PrivateRange
+ {
+ public readonly MemoryBlock Memory;
+ public readonly ulong Offset;
+ public readonly ulong Size;
+
+ public static PrivateRange Empty => new(null, 0, 0);
+
+ public PrivateRange(MemoryBlock memory, ulong offset, ulong size)
+ {
+ Memory = memory;
+ Offset = offset;
+ Size = size;
+ }
+ }
+
+ class AddressSpacePartition : IDisposable
+ {
+ public const ulong GuestPageSize = 0x1000;
+
+ private const int DefaultBlockAlignment = 1 << 20;
+
+ private enum MappingType : byte
+ {
+ None,
+ Private,
+ }
+
+ private class Mapping : IntrusiveRedBlackTreeNode, IComparable, IComparable
+ {
+ public ulong Address { get; private set; }
+ public ulong Size { get; private set; }
+ public ulong EndAddress => Address + Size;
+ public MappingType Type { get; private set; }
+
+ public Mapping(ulong address, ulong size, MappingType type)
+ {
+ Address = address;
+ Size = size;
+ Type = type;
+ }
+
+ public Mapping Split(ulong splitAddress)
+ {
+ ulong leftSize = splitAddress - Address;
+ ulong rightSize = EndAddress - splitAddress;
+
+ Mapping left = new(Address, leftSize, Type);
+
+ Address = splitAddress;
+ Size = rightSize;
+
+ return left;
+ }
+
+ public void UpdateState(MappingType newType)
+ {
+ Type = newType;
+ }
+
+ public void Extend(ulong sizeDelta)
+ {
+ Size += sizeDelta;
+ }
+
+ public int CompareTo(Mapping other)
+ {
+ if (Address < other.Address)
+ {
+ return -1;
+ }
+ else if (Address <= other.EndAddress - 1UL)
+ {
+ return 0;
+ }
+ else
+ {
+ return 1;
+ }
+ }
+
+ public int CompareTo(ulong address)
+ {
+ if (address < Address)
+ {
+ return -1;
+ }
+ else if (address <= EndAddress - 1UL)
+ {
+ return 0;
+ }
+ else
+ {
+ return 1;
+ }
+ }
+ }
+
+ private class PrivateMapping : IntrusiveRedBlackTreeNode, IComparable, IComparable
+ {
+ public ulong Address { get; private set; }
+ public ulong Size { get; private set; }
+ public ulong EndAddress => Address + Size;
+ public PrivateMemoryAllocation PrivateAllocation { get; private set; }
+
+ public PrivateMapping(ulong address, ulong size, PrivateMemoryAllocation privateAllocation)
+ {
+ Address = address;
+ Size = size;
+ PrivateAllocation = privateAllocation;
+ }
+
+ public PrivateMapping Split(ulong splitAddress)
+ {
+ ulong leftSize = splitAddress - Address;
+ ulong rightSize = EndAddress - splitAddress;
+
+ Debug.Assert(leftSize > 0);
+ Debug.Assert(rightSize > 0);
+
+ (var leftAllocation, PrivateAllocation) = PrivateAllocation.Split(leftSize);
+
+ PrivateMapping left = new(Address, leftSize, leftAllocation);
+
+ Address = splitAddress;
+ Size = rightSize;
+
+ return left;
+ }
+
+ public void Map(AddressSpacePartitionMultiAllocation baseBlock, ulong baseAddress, PrivateMemoryAllocation newAllocation)
+ {
+ baseBlock.MapView(newAllocation.Memory, newAllocation.Offset, Address - baseAddress, Size);
+ PrivateAllocation = newAllocation;
+ }
+
+ public void Unmap(AddressSpacePartitionMultiAllocation baseBlock, ulong baseAddress)
+ {
+ if (PrivateAllocation.IsValid)
+ {
+ baseBlock.UnmapView(PrivateAllocation.Memory, Address - baseAddress, Size);
+ PrivateAllocation.Dispose();
+ }
+
+ PrivateAllocation = default;
+ }
+
+ public void Extend(ulong sizeDelta)
+ {
+ Size += sizeDelta;
+ }
+
+ public int CompareTo(PrivateMapping other)
+ {
+ if (Address < other.Address)
+ {
+ return -1;
+ }
+ else if (Address <= other.EndAddress - 1UL)
+ {
+ return 0;
+ }
+ else
+ {
+ return 1;
+ }
+ }
+
+ public int CompareTo(ulong address)
+ {
+ if (address < Address)
+ {
+ return -1;
+ }
+ else if (address <= EndAddress - 1UL)
+ {
+ return 0;
+ }
+ else
+ {
+ return 1;
+ }
+ }
+ }
+
+ private readonly MemoryBlock _backingMemory;
+ private readonly AddressSpacePartitionMultiAllocation _baseMemory;
+ private readonly PrivateMemoryAllocator _privateMemoryAllocator;
+
+ private readonly AddressIntrusiveRedBlackTree _mappingTree;
+ private readonly AddressIntrusiveRedBlackTree _privateTree;
+
+ private readonly ReaderWriterLockSlim _treeLock;
+
+ private readonly ulong _hostPageSize;
+
+ private ulong? _firstPagePa;
+ private ulong? _lastPagePa;
+ private ulong _cachedFirstPagePa;
+ private ulong _cachedLastPagePa;
+ private MemoryBlock _firstPageMemoryForUnmap;
+ private ulong _firstPageOffsetForLateMap;
+ private MemoryPermission _firstPageMemoryProtection;
+
+ public ulong Address { get; }
+ public ulong Size { get; }
+ public ulong EndAddress => Address + Size;
+
+ public AddressSpacePartition(AddressSpacePartitionAllocation baseMemory, MemoryBlock backingMemory, ulong address, ulong size)
+ {
+ _privateMemoryAllocator = new PrivateMemoryAllocator(DefaultBlockAlignment, MemoryAllocationFlags.Mirrorable);
+ _mappingTree = new AddressIntrusiveRedBlackTree();
+ _privateTree = new AddressIntrusiveRedBlackTree();
+ _treeLock = new ReaderWriterLockSlim();
+
+ _mappingTree.Add(new Mapping(address, size, MappingType.None));
+ _privateTree.Add(new PrivateMapping(address, size, default));
+
+ _hostPageSize = MemoryBlock.GetPageSize();
+
+ _backingMemory = backingMemory;
+ _baseMemory = new(baseMemory);
+
+ _cachedFirstPagePa = ulong.MaxValue;
+ _cachedLastPagePa = ulong.MaxValue;
+
+ Address = address;
+ Size = size;
+ }
+
+ public bool IsEmpty()
+ {
+ _treeLock.EnterReadLock();
+
+ try
+ {
+ Mapping map = _mappingTree.GetNode(Address);
+
+ return map != null && map.Address == Address && map.Size == Size && map.Type == MappingType.None;
+ }
+ finally
+ {
+ _treeLock.ExitReadLock();
+ }
+ }
+
+ public void Map(ulong va, ulong pa, ulong size)
+ {
+ Debug.Assert(va >= Address);
+ Debug.Assert(va + size <= EndAddress);
+
+ if (va == Address)
+ {
+ _firstPagePa = pa;
+ }
+
+ if (va <= EndAddress - GuestPageSize && va + size > EndAddress - GuestPageSize)
+ {
+ _lastPagePa = pa + ((EndAddress - GuestPageSize) - va);
+ }
+
+ Update(va, pa, size, MappingType.Private);
+ }
+
+ public void Unmap(ulong va, ulong size)
+ {
+ Debug.Assert(va >= Address);
+ Debug.Assert(va + size <= EndAddress);
+
+ if (va == Address)
+ {
+ _firstPagePa = null;
+ }
+
+ if (va <= EndAddress - GuestPageSize && va + size > EndAddress - GuestPageSize)
+ {
+ _lastPagePa = null;
+ }
+
+ Update(va, 0UL, size, MappingType.None);
+ }
+
+ public void ReprotectAligned(ulong va, ulong size, MemoryPermission protection)
+ {
+ Debug.Assert(va >= Address);
+ Debug.Assert(va + size <= EndAddress);
+
+ _baseMemory.Reprotect(va - Address, size, protection, false);
+
+ if (va == Address)
+ {
+ _firstPageMemoryProtection = protection;
+ }
+ }
+
+ public void Reprotect(
+ ulong va,
+ ulong size,
+ MemoryPermission protection,
+ AddressSpacePartitioned addressSpace,
+ Action updatePtCallback)
+ {
+ if (_baseMemory.LazyInitMirrorForProtection(addressSpace, Address, Size, protection))
+ {
+ LateMap();
+ }
+
+ updatePtCallback(va, _baseMemory.GetPointerForProtection(va - Address, size, protection), size);
+ }
+
+ public IntPtr GetPointer(ulong va, ulong size)
+ {
+ Debug.Assert(va >= Address);
+ Debug.Assert(va + size <= EndAddress);
+
+ return _baseMemory.GetPointer(va - Address, size);
+ }
+
+ public void InsertBridgeAtEnd(AddressSpacePartition partitionAfter, bool useProtectionMirrors)
+ {
+ ulong firstPagePa = partitionAfter?._firstPagePa ?? ulong.MaxValue;
+ ulong lastPagePa = _lastPagePa ?? ulong.MaxValue;
+
+ if (firstPagePa != _cachedFirstPagePa || lastPagePa != _cachedLastPagePa)
+ {
+ if (partitionAfter != null && partitionAfter._firstPagePa.HasValue)
+ {
+ (MemoryBlock firstPageMemory, ulong firstPageOffset) = partitionAfter.GetFirstPageMemoryAndOffset();
+
+ _baseMemory.MapView(firstPageMemory, firstPageOffset, Size, _hostPageSize);
+
+ if (!useProtectionMirrors)
+ {
+ _baseMemory.Reprotect(Size, _hostPageSize, partitionAfter._firstPageMemoryProtection, throwOnFail: false);
+ }
+
+ _firstPageMemoryForUnmap = firstPageMemory;
+ _firstPageOffsetForLateMap = firstPageOffset;
+ }
+ else
+ {
+ MemoryBlock firstPageMemoryForUnmap = _firstPageMemoryForUnmap;
+
+ if (firstPageMemoryForUnmap != null)
+ {
+ _baseMemory.UnmapView(firstPageMemoryForUnmap, Size, _hostPageSize);
+ _firstPageMemoryForUnmap = null;
+ }
+ }
+
+ _cachedFirstPagePa = firstPagePa;
+ _cachedLastPagePa = lastPagePa;
+ }
+ }
+
+ public void ReprotectBridge(MemoryPermission protection)
+ {
+ if (_firstPageMemoryForUnmap != null)
+ {
+ _baseMemory.Reprotect(Size, _hostPageSize, protection, throwOnFail: false);
+ }
+ }
+
+ private (MemoryBlock, ulong) GetFirstPageMemoryAndOffset()
+ {
+ _treeLock.EnterReadLock();
+
+ try
+ {
+ PrivateMapping map = _privateTree.GetNode(Address);
+
+ if (map != null && map.PrivateAllocation.IsValid)
+ {
+ return (map.PrivateAllocation.Memory, map.PrivateAllocation.Offset + (Address - map.Address));
+ }
+ }
+ finally
+ {
+ _treeLock.ExitReadLock();
+ }
+
+ return (_backingMemory, _firstPagePa.Value);
+ }
+
+ public PrivateRange GetPrivateAllocation(ulong va)
+ {
+ _treeLock.EnterReadLock();
+
+ try
+ {
+ PrivateMapping map = _privateTree.GetNode(va);
+
+ if (map != null && map.PrivateAllocation.IsValid)
+ {
+ return new(map.PrivateAllocation.Memory, map.PrivateAllocation.Offset + (va - map.Address), map.Size - (va - map.Address));
+ }
+ }
+ finally
+ {
+ _treeLock.ExitReadLock();
+ }
+
+ return PrivateRange.Empty;
+ }
+
+ private void Update(ulong va, ulong pa, ulong size, MappingType type)
+ {
+ _treeLock.EnterWriteLock();
+
+ try
+ {
+ Mapping map = _mappingTree.GetNode(va);
+
+ Update(map, va, pa, size, type);
+ }
+ finally
+ {
+ _treeLock.ExitWriteLock();
+ }
+ }
+
+ private Mapping Update(Mapping map, ulong va, ulong pa, ulong size, MappingType type)
+ {
+ ulong endAddress = va + size;
+
+ for (; map != null; map = map.Successor)
+ {
+ if (map.Address < va)
+ {
+ _mappingTree.Add(map.Split(va));
+ }
+
+ if (map.EndAddress > endAddress)
+ {
+ Mapping newMap = map.Split(endAddress);
+ _mappingTree.Add(newMap);
+ map = newMap;
+ }
+
+ switch (type)
+ {
+ case MappingType.None:
+ ulong alignment = _hostPageSize;
+
+ bool unmappedBefore = map.Predecessor == null ||
+ (map.Predecessor.Type == MappingType.None && map.Predecessor.Address <= BitUtils.AlignDown(va, alignment));
+
+ bool unmappedAfter = map.Successor == null ||
+ (map.Successor.Type == MappingType.None && map.Successor.EndAddress >= BitUtils.AlignUp(endAddress, alignment));
+
+ UnmapPrivate(va, size, unmappedBefore, unmappedAfter);
+ break;
+ case MappingType.Private:
+ MapPrivate(va, size);
+ break;
+ }
+
+ map.UpdateState(type);
+ map = TryCoalesce(map);
+
+ if (map.EndAddress >= endAddress)
+ {
+ break;
+ }
+ }
+
+ return map;
+ }
+
+ private Mapping TryCoalesce(Mapping map)
+ {
+ Mapping previousMap = map.Predecessor;
+ Mapping nextMap = map.Successor;
+
+ if (previousMap != null && CanCoalesce(previousMap, map))
+ {
+ previousMap.Extend(map.Size);
+ _mappingTree.Remove(map);
+ map = previousMap;
+ }
+
+ if (nextMap != null && CanCoalesce(map, nextMap))
+ {
+ map.Extend(nextMap.Size);
+ _mappingTree.Remove(nextMap);
+ }
+
+ return map;
+ }
+
+ private static bool CanCoalesce(Mapping left, Mapping right)
+ {
+ return left.Type == right.Type;
+ }
+
+ private void MapPrivate(ulong va, ulong size)
+ {
+ ulong endAddress = va + size;
+
+ ulong alignment = _hostPageSize;
+
+ // Expand the range outwards based on page size to ensure that at least the requested region is mapped.
+ ulong vaAligned = BitUtils.AlignDown(va, alignment);
+ ulong endAddressAligned = BitUtils.AlignUp(endAddress, alignment);
+
+ PrivateMapping map = _privateTree.GetNode(va);
+
+ for (; map != null; map = map.Successor)
+ {
+ if (!map.PrivateAllocation.IsValid)
+ {
+ if (map.Address < vaAligned)
+ {
+ _privateTree.Add(map.Split(vaAligned));
+ }
+
+ if (map.EndAddress > endAddressAligned)
+ {
+ PrivateMapping newMap = map.Split(endAddressAligned);
+ _privateTree.Add(newMap);
+ map = newMap;
+ }
+
+ map.Map(_baseMemory, Address, _privateMemoryAllocator.Allocate(map.Size, _hostPageSize));
+ }
+
+ if (map.EndAddress >= endAddressAligned)
+ {
+ break;
+ }
+ }
+ }
+
+ private void UnmapPrivate(ulong va, ulong size, bool unmappedBefore, bool unmappedAfter)
+ {
+ ulong endAddress = va + size;
+
+ ulong alignment = _hostPageSize;
+
+ // If the adjacent mappings are unmapped, expand the range outwards,
+ // otherwise shrink it inwards. We must ensure we won't unmap pages that might still be in use.
+ ulong vaAligned = unmappedBefore ? BitUtils.AlignDown(va, alignment) : BitUtils.AlignUp(va, alignment);
+ ulong endAddressAligned = unmappedAfter ? BitUtils.AlignUp(endAddress, alignment) : BitUtils.AlignDown(endAddress, alignment);
+
+ if (endAddressAligned <= vaAligned)
+ {
+ return;
+ }
+
+ PrivateMapping map = _privateTree.GetNode(vaAligned);
+
+ for (; map != null; map = map.Successor)
+ {
+ if (map.PrivateAllocation.IsValid)
+ {
+ if (map.Address < vaAligned)
+ {
+ _privateTree.Add(map.Split(vaAligned));
+ }
+
+ if (map.EndAddress > endAddressAligned)
+ {
+ PrivateMapping newMap = map.Split(endAddressAligned);
+ _privateTree.Add(newMap);
+ map = newMap;
+ }
+
+ map.Unmap(_baseMemory, Address);
+ map = TryCoalesce(map);
+ }
+
+ if (map.EndAddress >= endAddressAligned)
+ {
+ break;
+ }
+ }
+ }
+
+ private PrivateMapping TryCoalesce(PrivateMapping map)
+ {
+ PrivateMapping previousMap = map.Predecessor;
+ PrivateMapping nextMap = map.Successor;
+
+ if (previousMap != null && CanCoalesce(previousMap, map))
+ {
+ previousMap.Extend(map.Size);
+ _privateTree.Remove(map);
+ map = previousMap;
+ }
+
+ if (nextMap != null && CanCoalesce(map, nextMap))
+ {
+ map.Extend(nextMap.Size);
+ _privateTree.Remove(nextMap);
+ }
+
+ return map;
+ }
+
+ private static bool CanCoalesce(PrivateMapping left, PrivateMapping right)
+ {
+ return !left.PrivateAllocation.IsValid && !right.PrivateAllocation.IsValid;
+ }
+
+ private void LateMap()
+ {
+ // Map all existing private allocations.
+ // This is necessary to ensure mirrors that are lazily created have the same mappings as the main one.
+
+ PrivateMapping map = _privateTree.GetNode(Address);
+
+ for (; map != null; map = map.Successor)
+ {
+ if (map.PrivateAllocation.IsValid)
+ {
+ _baseMemory.LateMapView(map.PrivateAllocation.Memory, map.PrivateAllocation.Offset, map.Address - Address, map.Size);
+ }
+ }
+
+ MemoryBlock firstPageMemory = _firstPageMemoryForUnmap;
+ ulong firstPageOffset = _firstPageOffsetForLateMap;
+
+ if (firstPageMemory != null)
+ {
+ _baseMemory.LateMapView(firstPageMemory, firstPageOffset, Size, _hostPageSize);
+ }
+ }
+
+ public PrivateRange GetFirstPrivateAllocation(ulong va, ulong size, out ulong nextVa)
+ {
+ _treeLock.EnterReadLock();
+
+ try
+ {
+ PrivateMapping map = _privateTree.GetNode(va);
+
+ nextVa = map.EndAddress;
+
+ if (map != null && map.PrivateAllocation.IsValid)
+ {
+ ulong startOffset = va - map.Address;
+
+ return new(
+ map.PrivateAllocation.Memory,
+ map.PrivateAllocation.Offset + startOffset,
+ Math.Min(map.PrivateAllocation.Size - startOffset, size));
+ }
+ }
+ finally
+ {
+ _treeLock.ExitReadLock();
+ }
+
+ return PrivateRange.Empty;
+ }
+
+ public bool HasPrivateAllocation(ulong va, ulong size, ulong startVa, ulong startSize, ref PrivateRange range)
+ {
+ ulong endVa = va + size;
+
+ _treeLock.EnterReadLock();
+
+ try
+ {
+ for (PrivateMapping map = _privateTree.GetNode(va); map != null && map.Address < endVa; map = map.Successor)
+ {
+ if (map.PrivateAllocation.IsValid)
+ {
+ if (map.Address <= startVa && map.EndAddress >= startVa + startSize)
+ {
+ ulong startOffset = startVa - map.Address;
+
+ range = new(
+ map.PrivateAllocation.Memory,
+ map.PrivateAllocation.Offset + startOffset,
+ Math.Min(map.PrivateAllocation.Size - startOffset, startSize));
+ }
+
+ return true;
+ }
+ }
+ }
+ finally
+ {
+ _treeLock.ExitReadLock();
+ }
+
+ return false;
+ }
+
+ public void Dispose()
+ {
+ GC.SuppressFinalize(this);
+
+ _privateMemoryAllocator.Dispose();
+ _baseMemory.Dispose();
+ }
+ }
+}
diff --git a/src/Ryujinx.Cpu/Jit/HostTracked/AddressSpacePartitionAllocator.cs b/src/Ryujinx.Cpu/Jit/HostTracked/AddressSpacePartitionAllocator.cs
new file mode 100644
index 0000000000..44dedb6404
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/HostTracked/AddressSpacePartitionAllocator.cs
@@ -0,0 +1,202 @@
+using Ryujinx.Common;
+using Ryujinx.Common.Collections;
+using Ryujinx.Memory;
+using Ryujinx.Memory.Tracking;
+using System;
+
+namespace Ryujinx.Cpu.Jit.HostTracked
+{
+ readonly struct AddressSpacePartitionAllocation : IDisposable
+ {
+ private readonly AddressSpacePartitionAllocator _owner;
+ private readonly PrivateMemoryAllocatorImpl.Allocation _allocation;
+
+ public IntPtr Pointer => (IntPtr)((ulong)_allocation.Block.Memory.Pointer + _allocation.Offset);
+
+ public bool IsValid => _owner != null;
+
+ public AddressSpacePartitionAllocation(
+ AddressSpacePartitionAllocator owner,
+ PrivateMemoryAllocatorImpl.Allocation allocation)
+ {
+ _owner = owner;
+ _allocation = allocation;
+ }
+
+ public void RegisterMapping(ulong va, ulong endVa)
+ {
+ _allocation.Block.AddMapping(_allocation.Offset, _allocation.Size, va, endVa);
+ }
+
+ public void MapView(MemoryBlock srcBlock, ulong srcOffset, ulong dstOffset, ulong size)
+ {
+ _allocation.Block.Memory.MapView(srcBlock, srcOffset, _allocation.Offset + dstOffset, size);
+ }
+
+ public void UnmapView(MemoryBlock srcBlock, ulong offset, ulong size)
+ {
+ _allocation.Block.Memory.UnmapView(srcBlock, _allocation.Offset + offset, size);
+ }
+
+ public void Reprotect(ulong offset, ulong size, MemoryPermission permission, bool throwOnFail)
+ {
+ _allocation.Block.Memory.Reprotect(_allocation.Offset + offset, size, permission, throwOnFail);
+ }
+
+ public IntPtr GetPointer(ulong offset, ulong size)
+ {
+ return _allocation.Block.Memory.GetPointer(_allocation.Offset + offset, size);
+ }
+
+ public void Dispose()
+ {
+ _allocation.Block.RemoveMapping(_allocation.Offset, _allocation.Size);
+ _owner.Free(_allocation.Block, _allocation.Offset, _allocation.Size);
+ }
+ }
+
+ class AddressSpacePartitionAllocator : PrivateMemoryAllocatorImpl
+ {
+ private const ulong DefaultBlockAlignment = 1UL << 32; // 4GB
+
+ public class Block : PrivateMemoryAllocator.Block
+ {
+ private readonly MemoryTracking _tracking;
+ private readonly Func _readPtCallback;
+ private readonly MemoryEhMeilleure _memoryEh;
+
+ private class Mapping : IntrusiveRedBlackTreeNode, IComparable, IComparable
+ {
+ public ulong Address { get; }
+ public ulong Size { get; }
+ public ulong EndAddress => Address + Size;
+ public ulong Va { get; }
+ public ulong EndVa { get; }
+
+ public Mapping(ulong address, ulong size, ulong va, ulong endVa)
+ {
+ Address = address;
+ Size = size;
+ Va = va;
+ EndVa = endVa;
+ }
+
+ public int CompareTo(Mapping other)
+ {
+ if (Address < other.Address)
+ {
+ return -1;
+ }
+ else if (Address <= other.EndAddress - 1UL)
+ {
+ return 0;
+ }
+ else
+ {
+ return 1;
+ }
+ }
+
+ public int CompareTo(ulong address)
+ {
+ if (address < Address)
+ {
+ return -1;
+ }
+ else if (address <= EndAddress - 1UL)
+ {
+ return 0;
+ }
+ else
+ {
+ return 1;
+ }
+ }
+ }
+
+ private readonly AddressIntrusiveRedBlackTree _mappingTree;
+ private readonly object _lock;
+
+ public Block(MemoryTracking tracking, Func readPtCallback, MemoryBlock memory, ulong size, object locker) : base(memory, size)
+ {
+ _tracking = tracking;
+ _readPtCallback = readPtCallback;
+ _memoryEh = new(memory, null, tracking, VirtualMemoryEvent);
+ _mappingTree = new();
+ _lock = locker;
+ }
+
+ public void AddMapping(ulong offset, ulong size, ulong va, ulong endVa)
+ {
+ _mappingTree.Add(new(offset, size, va, endVa));
+ }
+
+ public void RemoveMapping(ulong offset, ulong size)
+ {
+ _mappingTree.Remove(_mappingTree.GetNode(offset));
+ }
+
+ private ulong VirtualMemoryEvent(ulong address, ulong size, bool write)
+ {
+ Mapping map;
+
+ lock (_lock)
+ {
+ map = _mappingTree.GetNode(address);
+ }
+
+ if (map == null)
+ {
+ return 0;
+ }
+
+ address -= map.Address;
+
+ ulong addressAligned = BitUtils.AlignDown(address, AddressSpacePartition.GuestPageSize);
+ ulong endAddressAligned = BitUtils.AlignUp(address + size, AddressSpacePartition.GuestPageSize);
+ ulong sizeAligned = endAddressAligned - addressAligned;
+
+ if (!_tracking.VirtualMemoryEvent(map.Va + addressAligned, sizeAligned, write))
+ {
+ return 0;
+ }
+
+ return _readPtCallback(map.Va + address);
+ }
+
+ public override void Destroy()
+ {
+ _memoryEh.Dispose();
+
+ base.Destroy();
+ }
+ }
+
+ private readonly MemoryTracking _tracking;
+ private readonly Func _readPtCallback;
+ private readonly object _lock;
+
+ public AddressSpacePartitionAllocator(
+ MemoryTracking tracking,
+ Func readPtCallback,
+ object locker) : base(DefaultBlockAlignment, MemoryAllocationFlags.Reserve | MemoryAllocationFlags.ViewCompatible)
+ {
+ _tracking = tracking;
+ _readPtCallback = readPtCallback;
+ _lock = locker;
+ }
+
+ public AddressSpacePartitionAllocation Allocate(ulong va, ulong size)
+ {
+ AddressSpacePartitionAllocation allocation = new(this, Allocate(size, MemoryBlock.GetPageSize(), CreateBlock));
+ allocation.RegisterMapping(va, va + size);
+
+ return allocation;
+ }
+
+ private Block CreateBlock(MemoryBlock memory, ulong size)
+ {
+ return new Block(_tracking, _readPtCallback, memory, size, _lock);
+ }
+ }
+}
diff --git a/src/Ryujinx.Cpu/Jit/HostTracked/AddressSpacePartitionMultiAllocation.cs b/src/Ryujinx.Cpu/Jit/HostTracked/AddressSpacePartitionMultiAllocation.cs
new file mode 100644
index 0000000000..3b065583f8
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/HostTracked/AddressSpacePartitionMultiAllocation.cs
@@ -0,0 +1,101 @@
+using Ryujinx.Memory;
+using System;
+using System.Diagnostics;
+
+namespace Ryujinx.Cpu.Jit.HostTracked
+{
+ class AddressSpacePartitionMultiAllocation : IDisposable
+ {
+ private readonly AddressSpacePartitionAllocation _baseMemory;
+ private AddressSpacePartitionAllocation _baseMemoryRo;
+ private AddressSpacePartitionAllocation _baseMemoryNone;
+
+ public AddressSpacePartitionMultiAllocation(AddressSpacePartitionAllocation baseMemory)
+ {
+ _baseMemory = baseMemory;
+ }
+
+ public void MapView(MemoryBlock srcBlock, ulong srcOffset, ulong dstOffset, ulong size)
+ {
+ _baseMemory.MapView(srcBlock, srcOffset, dstOffset, size);
+
+ if (_baseMemoryRo.IsValid)
+ {
+ _baseMemoryRo.MapView(srcBlock, srcOffset, dstOffset, size);
+ _baseMemoryRo.Reprotect(dstOffset, size, MemoryPermission.Read, false);
+ }
+ }
+
+ public void LateMapView(MemoryBlock srcBlock, ulong srcOffset, ulong dstOffset, ulong size)
+ {
+ _baseMemoryRo.MapView(srcBlock, srcOffset, dstOffset, size);
+ _baseMemoryRo.Reprotect(dstOffset, size, MemoryPermission.Read, false);
+ }
+
+ public void UnmapView(MemoryBlock srcBlock, ulong offset, ulong size)
+ {
+ _baseMemory.UnmapView(srcBlock, offset, size);
+
+ if (_baseMemoryRo.IsValid)
+ {
+ _baseMemoryRo.UnmapView(srcBlock, offset, size);
+ }
+ }
+
+ public void Reprotect(ulong offset, ulong size, MemoryPermission permission, bool throwOnFail)
+ {
+ _baseMemory.Reprotect(offset, size, permission, throwOnFail);
+ }
+
+ public IntPtr GetPointer(ulong offset, ulong size)
+ {
+ return _baseMemory.GetPointer(offset, size);
+ }
+
+ public bool LazyInitMirrorForProtection(AddressSpacePartitioned addressSpace, ulong blockAddress, ulong blockSize, MemoryPermission permission)
+ {
+ if (permission == MemoryPermission.None && !_baseMemoryNone.IsValid)
+ {
+ _baseMemoryNone = addressSpace.CreateAsPartitionAllocation(blockAddress, blockSize);
+ }
+ else if (permission == MemoryPermission.Read && !_baseMemoryRo.IsValid)
+ {
+ _baseMemoryRo = addressSpace.CreateAsPartitionAllocation(blockAddress, blockSize);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ public IntPtr GetPointerForProtection(ulong offset, ulong size, MemoryPermission permission)
+ {
+ AddressSpacePartitionAllocation allocation = permission switch
+ {
+ MemoryPermission.ReadAndWrite => _baseMemory,
+ MemoryPermission.Read => _baseMemoryRo,
+ MemoryPermission.None => _baseMemoryNone,
+ _ => throw new ArgumentException($"Invalid protection \"{permission}\"."),
+ };
+
+ Debug.Assert(allocation.IsValid);
+
+ return allocation.GetPointer(offset, size);
+ }
+
+ public void Dispose()
+ {
+ _baseMemory.Dispose();
+
+ if (_baseMemoryRo.IsValid)
+ {
+ _baseMemoryRo.Dispose();
+ }
+
+ if (_baseMemoryNone.IsValid)
+ {
+ _baseMemoryNone.Dispose();
+ }
+ }
+ }
+}
diff --git a/src/Ryujinx.Cpu/Jit/HostTracked/AddressSpacePartitioned.cs b/src/Ryujinx.Cpu/Jit/HostTracked/AddressSpacePartitioned.cs
new file mode 100644
index 0000000000..2cf2c248b2
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/HostTracked/AddressSpacePartitioned.cs
@@ -0,0 +1,407 @@
+using Ryujinx.Common;
+using Ryujinx.Memory;
+using Ryujinx.Memory.Tracking;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+
+namespace Ryujinx.Cpu.Jit.HostTracked
+{
+ class AddressSpacePartitioned : IDisposable
+ {
+ private const int PartitionBits = 25;
+ private const ulong PartitionSize = 1UL << PartitionBits;
+
+ private readonly MemoryBlock _backingMemory;
+ private readonly List _partitions;
+ private readonly AddressSpacePartitionAllocator _asAllocator;
+ private readonly Action _updatePtCallback;
+ private readonly bool _useProtectionMirrors;
+
+ public AddressSpacePartitioned(MemoryTracking tracking, MemoryBlock backingMemory, NativePageTable nativePageTable, bool useProtectionMirrors)
+ {
+ _backingMemory = backingMemory;
+ _partitions = new();
+ _asAllocator = new(tracking, nativePageTable.Read, _partitions);
+ _updatePtCallback = nativePageTable.Update;
+ _useProtectionMirrors = useProtectionMirrors;
+ }
+
+ public void Map(ulong va, ulong pa, ulong size)
+ {
+ ulong endVa = va + size;
+
+ lock (_partitions)
+ {
+ EnsurePartitionsLocked(va, size);
+
+ while (va < endVa)
+ {
+ int partitionIndex = FindPartitionIndexLocked(va);
+ AddressSpacePartition partition = _partitions[partitionIndex];
+
+ (ulong clampedVa, ulong clampedEndVa) = ClampRange(partition, va, endVa);
+
+ partition.Map(clampedVa, pa, clampedEndVa - clampedVa);
+
+ ulong currentSize = clampedEndVa - clampedVa;
+
+ va += currentSize;
+ pa += currentSize;
+
+ InsertOrRemoveBridgeIfNeeded(partitionIndex);
+ }
+ }
+ }
+
+ public void Unmap(ulong va, ulong size)
+ {
+ ulong endVa = va + size;
+
+ while (va < endVa)
+ {
+ AddressSpacePartition partition;
+
+ lock (_partitions)
+ {
+ int partitionIndex = FindPartitionIndexLocked(va);
+ if (partitionIndex < 0)
+ {
+ va += PartitionSize - (va & (PartitionSize - 1));
+
+ continue;
+ }
+
+ partition = _partitions[partitionIndex];
+
+ (ulong clampedVa, ulong clampedEndVa) = ClampRange(partition, va, endVa);
+
+ partition.Unmap(clampedVa, clampedEndVa - clampedVa);
+
+ va += clampedEndVa - clampedVa;
+
+ InsertOrRemoveBridgeIfNeeded(partitionIndex);
+
+ if (partition.IsEmpty())
+ {
+ _partitions.Remove(partition);
+ partition.Dispose();
+ }
+ }
+ }
+ }
+
+ public void Reprotect(ulong va, ulong size, MemoryPermission protection)
+ {
+ ulong endVa = va + size;
+
+ lock (_partitions)
+ {
+ while (va < endVa)
+ {
+ AddressSpacePartition partition = FindPartitionWithIndex(va, out int partitionIndex);
+
+ if (partition == null)
+ {
+ va += PartitionSize - (va & (PartitionSize - 1));
+
+ continue;
+ }
+
+ (ulong clampedVa, ulong clampedEndVa) = ClampRange(partition, va, endVa);
+
+ if (_useProtectionMirrors)
+ {
+ partition.Reprotect(clampedVa, clampedEndVa - clampedVa, protection, this, _updatePtCallback);
+ }
+ else
+ {
+ partition.ReprotectAligned(clampedVa, clampedEndVa - clampedVa, protection);
+
+ if (clampedVa == partition.Address &&
+ partitionIndex > 0 &&
+ _partitions[partitionIndex - 1].EndAddress == partition.Address)
+ {
+ _partitions[partitionIndex - 1].ReprotectBridge(protection);
+ }
+ }
+
+ va += clampedEndVa - clampedVa;
+ }
+ }
+ }
+
+ public PrivateRange GetPrivateAllocation(ulong va)
+ {
+ AddressSpacePartition partition = FindPartition(va);
+
+ if (partition == null)
+ {
+ return PrivateRange.Empty;
+ }
+
+ return partition.GetPrivateAllocation(va);
+ }
+
+ public PrivateRange GetFirstPrivateAllocation(ulong va, ulong size, out ulong nextVa)
+ {
+ AddressSpacePartition partition = FindPartition(va);
+
+ if (partition == null)
+ {
+ nextVa = (va & ~(PartitionSize - 1)) + PartitionSize;
+
+ return PrivateRange.Empty;
+ }
+
+ return partition.GetFirstPrivateAllocation(va, size, out nextVa);
+ }
+
+ public bool HasAnyPrivateAllocation(ulong va, ulong size, out PrivateRange range)
+ {
+ range = PrivateRange.Empty;
+
+ ulong startVa = va;
+ ulong endVa = va + size;
+
+ while (va < endVa)
+ {
+ AddressSpacePartition partition = FindPartition(va);
+
+ if (partition == null)
+ {
+ va += PartitionSize - (va & (PartitionSize - 1));
+
+ continue;
+ }
+
+ (ulong clampedVa, ulong clampedEndVa) = ClampRange(partition, va, endVa);
+
+ if (partition.HasPrivateAllocation(clampedVa, clampedEndVa - clampedVa, startVa, size, ref range))
+ {
+ return true;
+ }
+
+ va += clampedEndVa - clampedVa;
+ }
+
+ return false;
+ }
+
+ private void InsertOrRemoveBridgeIfNeeded(int partitionIndex)
+ {
+ if (partitionIndex > 0)
+ {
+ if (_partitions[partitionIndex - 1].EndAddress == _partitions[partitionIndex].Address)
+ {
+ _partitions[partitionIndex - 1].InsertBridgeAtEnd(_partitions[partitionIndex], _useProtectionMirrors);
+ }
+ else
+ {
+ _partitions[partitionIndex - 1].InsertBridgeAtEnd(null, _useProtectionMirrors);
+ }
+ }
+
+ if (partitionIndex + 1 < _partitions.Count && _partitions[partitionIndex].EndAddress == _partitions[partitionIndex + 1].Address)
+ {
+ _partitions[partitionIndex].InsertBridgeAtEnd(_partitions[partitionIndex + 1], _useProtectionMirrors);
+ }
+ else
+ {
+ _partitions[partitionIndex].InsertBridgeAtEnd(null, _useProtectionMirrors);
+ }
+ }
+
+ public IntPtr GetPointer(ulong va, ulong size)
+ {
+ AddressSpacePartition partition = FindPartition(va);
+
+ return partition.GetPointer(va, size);
+ }
+
+ private static (ulong, ulong) ClampRange(AddressSpacePartition partition, ulong va, ulong endVa)
+ {
+ if (va < partition.Address)
+ {
+ va = partition.Address;
+ }
+
+ if (endVa > partition.EndAddress)
+ {
+ endVa = partition.EndAddress;
+ }
+
+ return (va, endVa);
+ }
+
+ private AddressSpacePartition FindPartition(ulong va)
+ {
+ lock (_partitions)
+ {
+ int index = FindPartitionIndexLocked(va);
+ if (index >= 0)
+ {
+ return _partitions[index];
+ }
+ }
+
+ return null;
+ }
+
+ private AddressSpacePartition FindPartitionWithIndex(ulong va, out int index)
+ {
+ lock (_partitions)
+ {
+ index = FindPartitionIndexLocked(va);
+ if (index >= 0)
+ {
+ return _partitions[index];
+ }
+ }
+
+ return null;
+ }
+
+ private int FindPartitionIndexLocked(ulong va)
+ {
+ int left = 0;
+ int middle;
+ int right = _partitions.Count - 1;
+
+ while (left <= right)
+ {
+ middle = left + ((right - left) >> 1);
+
+ AddressSpacePartition partition = _partitions[middle];
+
+ if (partition.Address <= va && partition.EndAddress > va)
+ {
+ return middle;
+ }
+
+ if (partition.Address >= va)
+ {
+ right = middle - 1;
+ }
+ else
+ {
+ left = middle + 1;
+ }
+ }
+
+ return -1;
+ }
+
+ private void EnsurePartitionsLocked(ulong va, ulong size)
+ {
+ ulong endVa = BitUtils.AlignUp(va + size, PartitionSize);
+ va = BitUtils.AlignDown(va, PartitionSize);
+
+ for (int i = 0; i < _partitions.Count && va < endVa; i++)
+ {
+ AddressSpacePartition partition = _partitions[i];
+
+ if (partition.Address <= va && partition.EndAddress > va)
+ {
+ if (partition.EndAddress >= endVa)
+ {
+ // Fully mapped already.
+ va = endVa;
+
+ break;
+ }
+
+ ulong gapSize;
+
+ if (i + 1 < _partitions.Count)
+ {
+ AddressSpacePartition nextPartition = _partitions[i + 1];
+
+ if (partition.EndAddress == nextPartition.Address)
+ {
+ va = partition.EndAddress;
+
+ continue;
+ }
+
+ gapSize = Math.Min(endVa, nextPartition.Address) - partition.EndAddress;
+ }
+ else
+ {
+ gapSize = endVa - partition.EndAddress;
+ }
+
+ _partitions.Insert(i + 1, CreateAsPartition(partition.EndAddress, gapSize));
+ va = partition.EndAddress + gapSize;
+ i++;
+ }
+ else if (partition.EndAddress > va)
+ {
+ Debug.Assert(partition.Address > va);
+
+ ulong gapSize;
+
+ if (partition.Address < endVa)
+ {
+ gapSize = partition.Address - va;
+ }
+ else
+ {
+ gapSize = endVa - va;
+ }
+
+ _partitions.Insert(i, CreateAsPartition(va, gapSize));
+ va = Math.Min(partition.EndAddress, endVa);
+ i++;
+ }
+ }
+
+ if (va < endVa)
+ {
+ _partitions.Add(CreateAsPartition(va, endVa - va));
+ }
+
+ ValidatePartitionList();
+ }
+
+ [Conditional("DEBUG")]
+ private void ValidatePartitionList()
+ {
+ for (int i = 1; i < _partitions.Count; i++)
+ {
+ Debug.Assert(_partitions[i].Address > _partitions[i - 1].Address);
+ Debug.Assert(_partitions[i].EndAddress > _partitions[i - 1].EndAddress);
+ }
+ }
+
+ private AddressSpacePartition CreateAsPartition(ulong va, ulong size)
+ {
+ return new(CreateAsPartitionAllocation(va, size), _backingMemory, va, size);
+ }
+
+ public AddressSpacePartitionAllocation CreateAsPartitionAllocation(ulong va, ulong size)
+ {
+ return _asAllocator.Allocate(va, size + MemoryBlock.GetPageSize());
+ }
+
+ protected virtual void Dispose(bool disposing)
+ {
+ if (disposing)
+ {
+ foreach (AddressSpacePartition partition in _partitions)
+ {
+ partition.Dispose();
+ }
+
+ _partitions.Clear();
+ _asAllocator.Dispose();
+ }
+ }
+
+ public void Dispose()
+ {
+ Dispose(disposing: true);
+ GC.SuppressFinalize(this);
+ }
+ }
+}
diff --git a/src/Ryujinx.Cpu/Jit/HostTracked/NativePageTable.cs b/src/Ryujinx.Cpu/Jit/HostTracked/NativePageTable.cs
new file mode 100644
index 0000000000..e3174e3fc5
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/HostTracked/NativePageTable.cs
@@ -0,0 +1,223 @@
+using Ryujinx.Cpu.Signal;
+using Ryujinx.Memory;
+using System;
+using System.Diagnostics;
+using System.Numerics;
+using System.Runtime.InteropServices;
+
+namespace Ryujinx.Cpu.Jit.HostTracked
+{
+ sealed class NativePageTable : IDisposable
+ {
+ private delegate ulong TrackingEventDelegate(ulong address, ulong size, bool write);
+
+ private const int PageBits = 12;
+ private const int PageSize = 1 << PageBits;
+ private const int PageMask = PageSize - 1;
+
+ private const int PteSize = 8;
+
+ private readonly int _bitsPerPtPage;
+ private readonly int _entriesPerPtPage;
+ private readonly int _pageCommitmentBits;
+
+ private readonly PageTable _pageTable;
+ private readonly MemoryBlock _nativePageTable;
+ private readonly ulong[] _pageCommitmentBitmap;
+ private readonly ulong _hostPageSize;
+
+ private readonly TrackingEventDelegate _trackingEvent;
+
+ private bool _disposed;
+
+ public IntPtr PageTablePointer => _nativePageTable.Pointer;
+
+ public NativePageTable(ulong asSize)
+ {
+ ulong hostPageSize = MemoryBlock.GetPageSize();
+
+ _entriesPerPtPage = (int)(hostPageSize / sizeof(ulong));
+ _bitsPerPtPage = BitOperations.Log2((uint)_entriesPerPtPage);
+ _pageCommitmentBits = PageBits + _bitsPerPtPage;
+
+ _hostPageSize = hostPageSize;
+ _pageTable = new PageTable();
+ _nativePageTable = new MemoryBlock((asSize / PageSize) * PteSize + _hostPageSize, MemoryAllocationFlags.Reserve);
+ _pageCommitmentBitmap = new ulong[(asSize >> _pageCommitmentBits) / (sizeof(ulong) * 8)];
+
+ ulong ptStart = (ulong)_nativePageTable.Pointer;
+ ulong ptEnd = ptStart + _nativePageTable.Size;
+
+ _trackingEvent = VirtualMemoryEvent;
+
+ bool added = NativeSignalHandler.AddTrackedRegion((nuint)ptStart, (nuint)ptEnd, Marshal.GetFunctionPointerForDelegate(_trackingEvent));
+
+ if (!added)
+ {
+ throw new InvalidOperationException("Number of allowed tracked regions exceeded.");
+ }
+ }
+
+ public void Map(ulong va, ulong pa, ulong size, AddressSpacePartitioned addressSpace, MemoryBlock backingMemory, bool privateMap)
+ {
+ while (size != 0)
+ {
+ _pageTable.Map(va, pa);
+
+ EnsureCommitment(va);
+
+ if (privateMap)
+ {
+ _nativePageTable.Write((va / PageSize) * PteSize, GetPte(va, addressSpace.GetPointer(va, PageSize)));
+ }
+ else
+ {
+ _nativePageTable.Write((va / PageSize) * PteSize, GetPte(va, backingMemory.GetPointer(pa, PageSize)));
+ }
+
+ va += PageSize;
+ pa += PageSize;
+ size -= PageSize;
+ }
+ }
+
+ public void Unmap(ulong va, ulong size)
+ {
+ IntPtr guardPagePtr = GetGuardPagePointer();
+
+ while (size != 0)
+ {
+ _pageTable.Unmap(va);
+ _nativePageTable.Write((va / PageSize) * PteSize, GetPte(va, guardPagePtr));
+
+ va += PageSize;
+ size -= PageSize;
+ }
+ }
+
+ public ulong Read(ulong va)
+ {
+ ulong pte = _nativePageTable.Read((va / PageSize) * PteSize);
+
+ pte += va & ~(ulong)PageMask;
+
+ return pte + (va & PageMask);
+ }
+
+ public void Update(ulong va, IntPtr ptr, ulong size)
+ {
+ ulong remainingSize = size;
+
+ while (remainingSize != 0)
+ {
+ EnsureCommitment(va);
+
+ _nativePageTable.Write((va / PageSize) * PteSize, GetPte(va, ptr));
+
+ va += PageSize;
+ ptr += PageSize;
+ remainingSize -= PageSize;
+ }
+ }
+
+ private void EnsureCommitment(ulong va)
+ {
+ ulong bit = va >> _pageCommitmentBits;
+
+ int index = (int)(bit / (sizeof(ulong) * 8));
+ int shift = (int)(bit % (sizeof(ulong) * 8));
+
+ ulong mask = 1UL << shift;
+
+ ulong oldMask = _pageCommitmentBitmap[index];
+
+ if ((oldMask & mask) == 0)
+ {
+ lock (_pageCommitmentBitmap)
+ {
+ oldMask = _pageCommitmentBitmap[index];
+
+ if ((oldMask & mask) != 0)
+ {
+ return;
+ }
+
+ _nativePageTable.Commit(bit * _hostPageSize, _hostPageSize);
+
+ Span pageSpan = MemoryMarshal.Cast(_nativePageTable.GetSpan(bit * _hostPageSize, (int)_hostPageSize));
+
+ Debug.Assert(pageSpan.Length == _entriesPerPtPage);
+
+ IntPtr guardPagePtr = GetGuardPagePointer();
+
+ for (int i = 0; i < pageSpan.Length; i++)
+ {
+ pageSpan[i] = GetPte((bit << _pageCommitmentBits) | ((ulong)i * PageSize), guardPagePtr);
+ }
+
+ _pageCommitmentBitmap[index] = oldMask | mask;
+ }
+ }
+ }
+
+ private IntPtr GetGuardPagePointer()
+ {
+ return _nativePageTable.GetPointer(_nativePageTable.Size - _hostPageSize, _hostPageSize);
+ }
+
+ private static ulong GetPte(ulong va, IntPtr ptr)
+ {
+ Debug.Assert((va & PageMask) == 0);
+
+ return (ulong)ptr - va;
+ }
+
+ public ulong GetPhysicalAddress(ulong va)
+ {
+ return _pageTable.Read(va) + (va & PageMask);
+ }
+
+ private ulong VirtualMemoryEvent(ulong address, ulong size, bool write)
+ {
+ if (address < _nativePageTable.Size - _hostPageSize)
+ {
+ // Some prefetch instructions do not cause faults with invalid addresses.
+ // Retry if we are hitting a case where the page table is unmapped, the next
+ // run will execute the actual instruction.
+ // The address loaded from the page table will be invalid, and it should hit the else case
+ // if the instruction faults on unmapped or protected memory.
+
+ ulong va = address * (PageSize / sizeof(ulong));
+
+ EnsureCommitment(va);
+
+ return (ulong)_nativePageTable.Pointer + address;
+ }
+ else
+ {
+ throw new InvalidMemoryRegionException();
+ }
+ }
+
+ private void Dispose(bool disposing)
+ {
+ if (!_disposed)
+ {
+ if (disposing)
+ {
+ NativeSignalHandler.RemoveTrackedRegion((nuint)_nativePageTable.Pointer);
+
+ _nativePageTable.Dispose();
+ }
+
+ _disposed = true;
+ }
+ }
+
+ public void Dispose()
+ {
+ Dispose(disposing: true);
+ GC.SuppressFinalize(this);
+ }
+ }
+}
diff --git a/src/Ryujinx.Cpu/Jit/JitCpuContext.cs b/src/Ryujinx.Cpu/Jit/JitCpuContext.cs
index dce0490a41..9893c59b29 100644
--- a/src/Ryujinx.Cpu/Jit/JitCpuContext.cs
+++ b/src/Ryujinx.Cpu/Jit/JitCpuContext.cs
@@ -15,9 +15,9 @@ namespace Ryujinx.Cpu.Jit
_tickSource = tickSource;
_translator = new Translator(new JitMemoryAllocator(forJit: true), memory, for64Bit);
- if (memory.Type.IsHostMapped())
+ if (memory.Type.IsHostMappedOrTracked())
{
- NativeSignalHandler.InitializeSignalHandler(MemoryBlock.GetPageSize());
+ NativeSignalHandler.InitializeSignalHandler();
}
memory.UnmapEvent += UnmapHandler;
diff --git a/src/Ryujinx.Cpu/Jit/MemoryManager.cs b/src/Ryujinx.Cpu/Jit/MemoryManager.cs
index bbfdf536eb..6f594ec2fd 100644
--- a/src/Ryujinx.Cpu/Jit/MemoryManager.cs
+++ b/src/Ryujinx.Cpu/Jit/MemoryManager.cs
@@ -3,6 +3,7 @@ using Ryujinx.Memory;
using Ryujinx.Memory.Range;
using Ryujinx.Memory.Tracking;
using System;
+using System.Buffers;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
@@ -14,7 +15,7 @@ namespace Ryujinx.Cpu.Jit
///
/// Represents a CPU memory manager.
///
- public sealed class MemoryManager : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock
+ public sealed class MemoryManager : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked
{
private const int PteSize = 8;
@@ -24,7 +25,7 @@ namespace Ryujinx.Cpu.Jit
private readonly InvalidAccessHandler _invalidAccessHandler;
///
- public bool Supports4KBPages => true;
+ public bool UsesPrivateAllocations => false;
///
/// Address space width in bits.
@@ -33,6 +34,8 @@ namespace Ryujinx.Cpu.Jit
private readonly MemoryBlock _pageTable;
+ private readonly ManagedPageFlags _pages;
+
///
/// Page table base pointer.
///
@@ -70,6 +73,8 @@ namespace Ryujinx.Cpu.Jit
AddressSpaceSize = asSize;
_pageTable = new MemoryBlock((asSize / PageSize) * PteSize);
+ _pages = new ManagedPageFlags(AddressSpaceBits);
+
Tracking = new MemoryTracking(this, PageSize);
}
@@ -89,15 +94,10 @@ namespace Ryujinx.Cpu.Jit
remainingSize -= PageSize;
}
+ _pages.AddMapping(oVa, size);
Tracking.Map(oVa, size);
}
- ///
- public void MapForeign(ulong va, nuint hostPointer, ulong size)
- {
- throw new NotSupportedException();
- }
-
///
public void Unmap(ulong va, ulong size)
{
@@ -111,6 +111,7 @@ namespace Ryujinx.Cpu.Jit
UnmapEvent?.Invoke(va, size);
Tracking.Unmap(va, size);
+ _pages.RemoveMapping(va, size);
ulong remainingSize = size;
while (remainingSize != 0)
@@ -122,18 +123,29 @@ namespace Ryujinx.Cpu.Jit
}
}
- ///
- public T Read(ulong va) where T : unmanaged
- {
- return MemoryMarshal.Cast(GetSpan(va, Unsafe.SizeOf()))[0];
- }
-
- ///
- public T ReadTracked(ulong va) where T : unmanaged
+ public override T ReadTracked(ulong va)
{
try
{
- SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false);
+ return base.ReadTracked(va);
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+
+ return default;
+ }
+ }
+
+ ///
+ public T ReadGuest(ulong va) where T : unmanaged
+ {
+ try
+ {
+ SignalMemoryTrackingImpl(va, (ulong)Unsafe.SizeOf(), false, true);
return Read(va);
}
@@ -164,107 +176,11 @@ namespace Ryujinx.Cpu.Jit
}
}
- ///
- public void Write(ulong va, T value) where T : unmanaged
- {
- Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1)));
- }
-
- ///
- public void Write(ulong va, ReadOnlySpan data)
- {
- if (data.Length == 0)
- {
- return;
- }
-
- SignalMemoryTracking(va, (ulong)data.Length, true);
-
- WriteImpl(va, data);
- }
-
- ///
- public void WriteUntracked(ulong va, ReadOnlySpan data)
- {
- if (data.Length == 0)
- {
- return;
- }
-
- WriteImpl(va, data);
- }
-
- ///
- public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data)
- {
- if (data.Length == 0)
- {
- return false;
- }
-
- SignalMemoryTracking(va, (ulong)data.Length, false);
-
- if (IsContiguousAndMapped(va, data.Length))
- {
- var target = _backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length);
-
- bool changed = !data.SequenceEqual(target);
-
- if (changed)
- {
- data.CopyTo(target);
- }
-
- return changed;
- }
- else
- {
- WriteImpl(va, data);
-
- return true;
- }
- }
-
- ///
- /// Writes data to CPU mapped memory.
- ///
- /// Virtual address to write the data into
- /// Data to be written
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private void WriteImpl(ulong va, ReadOnlySpan data)
+ public override void Write(ulong va, ReadOnlySpan data)
{
try
{
- AssertValidAddressAndSize(va, (ulong)data.Length);
-
- if (IsContiguousAndMapped(va, data.Length))
- {
- data.CopyTo(_backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length));
- }
- else
- {
- int offset = 0, size;
-
- if ((va & PageMask) != 0)
- {
- ulong pa = GetPhysicalAddressInternal(va);
-
- size = Math.Min(data.Length, PageSize - (int)(va & PageMask));
-
- data[..size].CopyTo(_backingMemory.GetSpan(pa, size));
-
- offset += size;
- }
-
- for (; offset < data.Length; offset += size)
- {
- ulong pa = GetPhysicalAddressInternal(va + (ulong)offset);
-
- size = Math.Min(data.Length - offset, PageSize);
-
- data.Slice(offset, size).CopyTo(_backingMemory.GetSpan(pa, size));
- }
- }
+ base.Write(va, data);
}
catch (InvalidMemoryRegionException)
{
@@ -276,60 +192,47 @@ namespace Ryujinx.Cpu.Jit
}
///
- public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false)
+ public void WriteGuest(ulong va, T value) where T : unmanaged
{
- if (size == 0)
+ Span data = MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1));
+
+ SignalMemoryTrackingImpl(va, (ulong)data.Length, true, true);
+
+ Write(va, data);
+ }
+
+ public override void WriteUntracked(ulong va, ReadOnlySpan data)
+ {
+ try
{
- return ReadOnlySpan.Empty;
+ base.WriteUntracked(va, data);
}
-
- if (tracked)
+ catch (InvalidMemoryRegionException)
{
- SignalMemoryTracking(va, (ulong)size, false);
- }
-
- if (IsContiguousAndMapped(va, size))
- {
- return _backingMemory.GetSpan(GetPhysicalAddressInternal(va), size);
- }
- else
- {
- Span data = new byte[size];
-
- base.Read(va, data);
-
- return data;
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
}
}
- ///
- public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
+ public override ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false)
{
- if (size == 0)
+ try
{
- return new WritableRegion(null, va, Memory.Empty);
+ return base.GetReadOnlySequence(va, size, tracked);
}
-
- if (IsContiguousAndMapped(va, size))
+ catch (InvalidMemoryRegionException)
{
- if (tracked)
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
{
- SignalMemoryTracking(va, (ulong)size, true);
+ throw;
}
- return new WritableRegion(null, va, _backingMemory.GetMemory(GetPhysicalAddressInternal(va), size));
- }
- else
- {
- Memory memory = new byte[size];
-
- GetSpan(va, size).CopyTo(memory.Span);
-
- return new WritableRegion(this, va, memory, tracked);
+ return ReadOnlySequence.Empty;
}
}
- ///
public ref T GetRef(ulong va) where T : unmanaged
{
if (!IsContiguous(va, Unsafe.SizeOf()))
@@ -342,56 +245,6 @@ namespace Ryujinx.Cpu.Jit
return ref _backingMemory.GetRef(GetPhysicalAddressInternal(va));
}
- ///
- /// Computes the number of pages in a virtual address range.
- ///
- /// Virtual address of the range
- /// Size of the range
- /// The virtual address of the beginning of the first page
- /// This function does not differentiate between allocated and unallocated pages.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static int GetPagesCount(ulong va, uint size, out ulong startVa)
- {
- // WARNING: Always check if ulong does not overflow during the operations.
- startVa = va & ~(ulong)PageMask;
- ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask;
-
- return (int)(vaSpan / PageSize);
- }
-
- private static void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException();
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private bool IsContiguousAndMapped(ulong va, int size) => IsContiguous(va, size) && IsMapped(va);
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private bool IsContiguous(ulong va, int size)
- {
- if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size))
- {
- return false;
- }
-
- int pages = GetPagesCount(va, (uint)size, out va);
-
- for (int page = 0; page < pages - 1; page++)
- {
- if (!ValidateAddress(va + PageSize))
- {
- return false;
- }
-
- if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize))
- {
- return false;
- }
-
- va += PageSize;
- }
-
- return true;
- }
-
///
public IEnumerable GetHostRegions(ulong va, ulong size)
{
@@ -496,9 +349,8 @@ namespace Ryujinx.Cpu.Jit
return true;
}
- ///
[MethodImpl(MethodImplOptions.AggressiveInlining)]
- public bool IsMapped(ulong va)
+ public override bool IsMapped(ulong va)
{
if (!ValidateAddress(va))
{
@@ -508,9 +360,9 @@ namespace Ryujinx.Cpu.Jit
return _pageTable.Read((va / PageSize) * PteSize) != 0;
}
- private ulong GetPhysicalAddressInternal(ulong va)
+ private nuint GetPhysicalAddressInternal(ulong va)
{
- return PteToPa(_pageTable.Read((va / PageSize) * PteSize) & ~(0xffffUL << 48)) + (va & PageMask);
+ return (nuint)(PteToPa(_pageTable.Read((va / PageSize) * PteSize) & ~(0xffffUL << 48)) + (va & PageMask));
}
///
@@ -520,50 +372,57 @@ namespace Ryujinx.Cpu.Jit
}
///
- public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection)
+ public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest)
{
AssertValidAddressAndSize(va, size);
- // Protection is inverted on software pages, since the default value is 0.
- protection = (~protection) & MemoryPermission.ReadAndWrite;
-
- long tag = protection switch
+ if (guest)
{
- MemoryPermission.None => 0L,
- MemoryPermission.Write => 2L << PointerTagBit,
- _ => 3L << PointerTagBit,
- };
+ // Protection is inverted on software pages, since the default value is 0.
+ protection = (~protection) & MemoryPermission.ReadAndWrite;
- int pages = GetPagesCount(va, (uint)size, out va);
- ulong pageStart = va >> PageBits;
- long invTagMask = ~(0xffffL << 48);
-
- for (int page = 0; page < pages; page++)
- {
- ref long pageRef = ref _pageTable.GetRef(pageStart * PteSize);
-
- long pte;
-
- do
+ long tag = protection switch
{
- pte = Volatile.Read(ref pageRef);
- }
- while (pte != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte);
+ MemoryPermission.None => 0L,
+ MemoryPermission.Write => 2L << PointerTagBit,
+ _ => 3L << PointerTagBit,
+ };
- pageStart++;
+ int pages = GetPagesCount(va, (uint)size, out va);
+ ulong pageStart = va >> PageBits;
+ long invTagMask = ~(0xffffL << 48);
+
+ for (int page = 0; page < pages; page++)
+ {
+ ref long pageRef = ref _pageTable.GetRef(pageStart * PteSize);
+
+ long pte;
+
+ do
+ {
+ pte = Volatile.Read(ref pageRef);
+ }
+ while (pte != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte);
+
+ pageStart++;
+ }
+ }
+ else
+ {
+ _pages.TrackingReprotect(va, size, protection);
}
}
///
- public RegionHandle BeginTracking(ulong address, ulong size, int id)
+ public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None)
{
- return Tracking.BeginTracking(address, size, id);
+ return Tracking.BeginTracking(address, size, id, flags);
}
///
- public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id)
+ public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None)
{
- return Tracking.BeginGranularTracking(address, size, handles, granularity, id);
+ return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags);
}
///
@@ -572,8 +431,7 @@ namespace Ryujinx.Cpu.Jit
return Tracking.BeginSmartGranularTracking(address, size, granularity, id);
}
- ///
- public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
+ private void SignalMemoryTrackingImpl(ulong va, ulong size, bool write, bool guest, bool precise = false, int? exemptId = null)
{
AssertValidAddressAndSize(va, size);
@@ -583,31 +441,45 @@ namespace Ryujinx.Cpu.Jit
return;
}
- // We emulate guard pages for software memory access. This makes for an easy transition to
- // tracking using host guard pages in future, but also supporting platforms where this is not possible.
+ // If the memory tracking is coming from the guest, use the tag bits in the page table entry.
+ // Otherwise, use the managed page flags.
- // Write tag includes read protection, since we don't have any read actions that aren't performed before write too.
- long tag = (write ? 3L : 1L) << PointerTagBit;
-
- int pages = GetPagesCount(va, (uint)size, out _);
- ulong pageStart = va >> PageBits;
-
- for (int page = 0; page < pages; page++)
+ if (guest)
{
- ref long pageRef = ref _pageTable.GetRef(pageStart * PteSize);
+ // We emulate guard pages for software memory access. This makes for an easy transition to
+ // tracking using host guard pages in future, but also supporting platforms where this is not possible.
- long pte;
+ // Write tag includes read protection, since we don't have any read actions that aren't performed before write too.
+ long tag = (write ? 3L : 1L) << PointerTagBit;
- pte = Volatile.Read(ref pageRef);
+ int pages = GetPagesCount(va, (uint)size, out _);
+ ulong pageStart = va >> PageBits;
- if ((pte & tag) != 0)
+ for (int page = 0; page < pages; page++)
{
- Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
- break;
- }
+ ref long pageRef = ref _pageTable.GetRef(pageStart * PteSize);
- pageStart++;
+ long pte = Volatile.Read(ref pageRef);
+
+ if ((pte & tag) != 0)
+ {
+ Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId, true);
+ break;
+ }
+
+ pageStart++;
+ }
}
+ else
+ {
+ _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId);
+ }
+ }
+
+ ///
+ public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
+ {
+ SignalMemoryTrackingImpl(va, size, write, false, precise, exemptId);
}
private ulong PaToPte(ulong pa)
@@ -625,10 +497,16 @@ namespace Ryujinx.Cpu.Jit
///
protected override void Destroy() => _pageTable.Dispose();
- protected override Span GetPhysicalAddressSpan(ulong pa, int size)
+ protected override Memory GetPhysicalAddressMemory(nuint pa, int size)
+ => _backingMemory.GetMemory(pa, size);
+
+ protected override Span GetPhysicalAddressSpan(nuint pa, int size)
=> _backingMemory.GetSpan(pa, size);
- protected override ulong TranslateVirtualAddressForRead(ulong va)
+ protected override nuint TranslateVirtualAddressChecked(ulong va)
+ => GetPhysicalAddressInternal(va);
+
+ protected override nuint TranslateVirtualAddressUnchecked(ulong va)
=> GetPhysicalAddressInternal(va);
}
}
diff --git a/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs
index 0b6ba260ab..4639ab913d 100644
--- a/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs
+++ b/src/Ryujinx.Cpu/Jit/MemoryManagerHostMapped.cs
@@ -3,33 +3,18 @@ using Ryujinx.Memory;
using Ryujinx.Memory.Range;
using Ryujinx.Memory.Tracking;
using System;
+using System.Buffers;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
-using System.Threading;
namespace Ryujinx.Cpu.Jit
{
///
/// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region.
///
- public sealed class MemoryManagerHostMapped : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked, IWritableBlock
+ public sealed class MemoryManagerHostMapped : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked
{
- public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry.
- public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set.
-
- private enum HostMappedPtBits : ulong
- {
- Unmapped = 0,
- Mapped,
- WriteTracked,
- ReadWriteTracked,
-
- MappedReplicated = 0x5555555555555555,
- WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa,
- ReadWriteTrackedReplicated = ulong.MaxValue,
- }
-
private readonly InvalidAccessHandler _invalidAccessHandler;
private readonly bool _unsafeMode;
@@ -39,10 +24,10 @@ namespace Ryujinx.Cpu.Jit
private readonly MemoryEhMeilleure _memoryEh;
- private readonly ulong[] _pageBitmap;
+ private readonly ManagedPageFlags _pages;
///
- public bool Supports4KBPages => MemoryBlock.GetPageSize() == PageSize;
+ public bool UsesPrivateAllocations => false;
public int AddressSpaceBits { get; }
@@ -81,7 +66,7 @@ namespace Ryujinx.Cpu.Jit
AddressSpaceBits = asBits;
- _pageBitmap = new ulong[1 << (AddressSpaceBits - (PageBits + PageToPteShift))];
+ _pages = new ManagedPageFlags(AddressSpaceBits);
Tracking = new MemoryTracking(this, (int)MemoryBlock.GetPageSize(), invalidAccessHandler);
_memoryEh = new MemoryEhMeilleure(_addressSpace.Base, _addressSpace.Mirror, Tracking);
@@ -94,7 +79,7 @@ namespace Ryujinx.Cpu.Jit
/// Size of the range in bytes
private void AssertMapped(ulong va, ulong size)
{
- if (!ValidateAddressAndSize(va, size) || !IsRangeMappedImpl(va, size))
+ if (!ValidateAddressAndSize(va, size) || !_pages.IsRangeMapped(va, size))
{
throw new InvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
}
@@ -106,18 +91,12 @@ namespace Ryujinx.Cpu.Jit
AssertValidAddressAndSize(va, size);
_addressSpace.Map(va, pa, size, flags);
- AddMapping(va, size);
+ _pages.AddMapping(va, size);
PtMap(va, pa, size);
Tracking.Map(va, size);
}
- ///
- public void MapForeign(ulong va, nuint hostPointer, ulong size)
- {
- throw new NotSupportedException();
- }
-
///
public void Unmap(ulong va, ulong size)
{
@@ -126,7 +105,7 @@ namespace Ryujinx.Cpu.Jit
UnmapEvent?.Invoke(va, size);
Tracking.Unmap(va, size);
- RemoveMapping(va, size);
+ _pages.RemoveMapping(va, size);
PtUnmap(va, size);
_addressSpace.Unmap(va, size);
}
@@ -154,8 +133,7 @@ namespace Ryujinx.Cpu.Jit
}
}
- ///
- public T Read(ulong va) where T : unmanaged
+ public override T Read(ulong va)
{
try
{
@@ -174,14 +152,11 @@ namespace Ryujinx.Cpu.Jit
}
}
- ///
- public T ReadTracked(ulong va) where T : unmanaged
+ public override T ReadTracked(ulong va)
{
try
{
- SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), false);
-
- return Read(va);
+ return base.ReadTracked(va);
}
catch (InvalidMemoryRegionException)
{
@@ -194,7 +169,6 @@ namespace Ryujinx.Cpu.Jit
}
}
- ///
public override void Read(ulong va, Span data)
{
try
@@ -212,9 +186,7 @@ namespace Ryujinx.Cpu.Jit
}
}
-
- ///
- public void Write(ulong va, T value) where T : unmanaged
+ public override void Write(ulong va, T value)
{
try
{
@@ -231,8 +203,7 @@ namespace Ryujinx.Cpu.Jit
}
}
- ///
- public void Write(ulong va, ReadOnlySpan data)
+ public override void Write(ulong va, ReadOnlySpan data)
{
try
{
@@ -249,8 +220,7 @@ namespace Ryujinx.Cpu.Jit
}
}
- ///
- public void WriteUntracked(ulong va, ReadOnlySpan data)
+ public override void WriteUntracked(ulong va, ReadOnlySpan data)
{
try
{
@@ -267,8 +237,7 @@ namespace Ryujinx.Cpu.Jit
}
}
- ///
- public bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data)
+ public override bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data)
{
try
{
@@ -295,8 +264,21 @@ namespace Ryujinx.Cpu.Jit
}
}
- ///
- public ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false)
+ public override ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false)
+ {
+ if (tracked)
+ {
+ SignalMemoryTracking(va, (ulong)size, write: false);
+ }
+ else
+ {
+ AssertMapped(va, (ulong)size);
+ }
+
+ return new ReadOnlySequence(_addressSpace.Mirror.GetMemory(va, size));
+ }
+
+ public override ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false)
{
if (tracked)
{
@@ -310,8 +292,7 @@ namespace Ryujinx.Cpu.Jit
return _addressSpace.Mirror.GetSpan(va, size);
}
- ///
- public WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
+ public override WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
{
if (tracked)
{
@@ -325,7 +306,6 @@ namespace Ryujinx.Cpu.Jit
return _addressSpace.Mirror.GetWritableRegion(va, size);
}
- ///
public ref T GetRef(ulong va) where T : unmanaged
{
SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), true);
@@ -333,26 +313,10 @@ namespace Ryujinx.Cpu.Jit
return ref _addressSpace.Mirror.GetRef(va);
}
- ///
[MethodImpl(MethodImplOptions.AggressiveInlining)]
- public bool IsMapped(ulong va)
+ public override bool IsMapped(ulong va)
{
- return ValidateAddress(va) && IsMappedImpl(va);
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private bool IsMappedImpl(ulong va)
- {
- ulong page = va >> PageBits;
-
- int bit = (int)((page & 31) << 1);
-
- int pageIndex = (int)(page >> PageToPteShift);
- ref ulong pageRef = ref _pageBitmap[pageIndex];
-
- ulong pte = Volatile.Read(ref pageRef);
-
- return ((pte >> bit) & 3) != 0;
+ return ValidateAddress(va) && _pages.IsMapped(va);
}
///
@@ -360,58 +324,7 @@ namespace Ryujinx.Cpu.Jit
{
AssertValidAddressAndSize(va, size);
- return IsRangeMappedImpl(va, size);
- }
-
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex)
- {
- startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1);
- endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1));
-
- pageIndex = (int)(pageStart >> PageToPteShift);
- pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift);
- }
-
- private bool IsRangeMappedImpl(ulong va, ulong size)
- {
- int pages = GetPagesCount(va, size, out _);
-
- if (pages == 1)
- {
- return IsMappedImpl(va);
- }
-
- ulong pageStart = va >> PageBits;
- ulong pageEnd = pageStart + (ulong)pages;
-
- GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
-
- // Check if either bit in each 2 bit page entry is set.
- // OR the block with itself shifted down by 1, and check the first bit of each entry.
-
- ulong mask = BlockMappedMask & startMask;
-
- while (pageIndex <= pageEndIndex)
- {
- if (pageIndex == pageEndIndex)
- {
- mask &= endMask;
- }
-
- ref ulong pageRef = ref _pageBitmap[pageIndex++];
- ulong pte = Volatile.Read(ref pageRef);
-
- pte |= pte >> 1;
- if ((pte & mask) != mask)
- {
- return false;
- }
-
- mask = BlockMappedMask;
- }
-
- return true;
+ return _pages.IsRangeMapped(va, size);
}
///
@@ -472,11 +385,10 @@ namespace Ryujinx.Cpu.Jit
return _pageTable.Read(va) + (va & PageMask);
}
- ///
///
/// This function also validates that the given range is both valid and mapped, and will throw if it is not.
///
- public void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
+ public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
{
AssertValidAddressAndSize(va, size);
@@ -486,93 +398,7 @@ namespace Ryujinx.Cpu.Jit
return;
}
- // Software table, used for managed memory tracking.
-
- int pages = GetPagesCount(va, size, out _);
- ulong pageStart = va >> PageBits;
-
- if (pages == 1)
- {
- ulong tag = (ulong)(write ? HostMappedPtBits.WriteTracked : HostMappedPtBits.ReadWriteTracked);
-
- int bit = (int)((pageStart & 31) << 1);
-
- int pageIndex = (int)(pageStart >> PageToPteShift);
- ref ulong pageRef = ref _pageBitmap[pageIndex];
-
- ulong pte = Volatile.Read(ref pageRef);
- ulong state = ((pte >> bit) & 3);
-
- if (state >= tag)
- {
- Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
- return;
- }
- else if (state == 0)
- {
- ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
- }
- }
- else
- {
- ulong pageEnd = pageStart + (ulong)pages;
-
- GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
-
- ulong mask = startMask;
-
- ulong anyTrackingTag = (ulong)HostMappedPtBits.WriteTrackedReplicated;
-
- while (pageIndex <= pageEndIndex)
- {
- if (pageIndex == pageEndIndex)
- {
- mask &= endMask;
- }
-
- ref ulong pageRef = ref _pageBitmap[pageIndex++];
-
- ulong pte = Volatile.Read(ref pageRef);
- ulong mappedMask = mask & BlockMappedMask;
-
- ulong mappedPte = pte | (pte >> 1);
- if ((mappedPte & mappedMask) != mappedMask)
- {
- ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
- }
-
- pte &= mask;
- if ((pte & anyTrackingTag) != 0) // Search for any tracking.
- {
- // Writes trigger any tracking.
- // Only trigger tracking from reads if both bits are set on any page.
- if (write || (pte & (pte >> 1) & BlockMappedMask) != 0)
- {
- Tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
- break;
- }
- }
-
- mask = ulong.MaxValue;
- }
- }
- }
-
- ///
- /// Computes the number of pages in a virtual address range.
- ///
- /// Virtual address of the range
- /// Size of the range
- /// The virtual address of the beginning of the first page
- /// This function does not differentiate between allocated and unallocated pages.
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- private static int GetPagesCount(ulong va, ulong size, out ulong startVa)
- {
- // WARNING: Always check if ulong does not overflow during the operations.
- startVa = va & ~(ulong)PageMask;
- ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask;
-
- return (int)(vaSpan / PageSize);
+ _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId);
}
///
@@ -582,103 +408,28 @@ namespace Ryujinx.Cpu.Jit
}
///
- public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection)
+ public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest)
{
- // Protection is inverted on software pages, since the default value is 0.
- protection = (~protection) & MemoryPermission.ReadAndWrite;
-
- int pages = GetPagesCount(va, size, out va);
- ulong pageStart = va >> PageBits;
-
- if (pages == 1)
+ if (guest)
{
- ulong protTag = protection switch
- {
- MemoryPermission.None => (ulong)HostMappedPtBits.Mapped,
- MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTracked,
- _ => (ulong)HostMappedPtBits.ReadWriteTracked,
- };
-
- int bit = (int)((pageStart & 31) << 1);
-
- ulong tagMask = 3UL << bit;
- ulong invTagMask = ~tagMask;
-
- ulong tag = protTag << bit;
-
- int pageIndex = (int)(pageStart >> PageToPteShift);
- ref ulong pageRef = ref _pageBitmap[pageIndex];
-
- ulong pte;
-
- do
- {
- pte = Volatile.Read(ref pageRef);
- }
- while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte);
+ _addressSpace.Base.Reprotect(va, size, protection, false);
}
else
{
- ulong pageEnd = pageStart + (ulong)pages;
-
- GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
-
- ulong mask = startMask;
-
- ulong protTag = protection switch
- {
- MemoryPermission.None => (ulong)HostMappedPtBits.MappedReplicated,
- MemoryPermission.Write => (ulong)HostMappedPtBits.WriteTrackedReplicated,
- _ => (ulong)HostMappedPtBits.ReadWriteTrackedReplicated,
- };
-
- while (pageIndex <= pageEndIndex)
- {
- if (pageIndex == pageEndIndex)
- {
- mask &= endMask;
- }
-
- ref ulong pageRef = ref _pageBitmap[pageIndex++];
-
- ulong pte;
- ulong mappedMask;
-
- // Change the protection of all 2 bit entries that are mapped.
- do
- {
- pte = Volatile.Read(ref pageRef);
-
- mappedMask = pte | (pte >> 1);
- mappedMask |= (mappedMask & BlockMappedMask) << 1;
- mappedMask &= mask; // Only update mapped pages within the given range.
- }
- while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte);
-
- mask = ulong.MaxValue;
- }
+ _pages.TrackingReprotect(va, size, protection);
}
-
- protection = protection switch
- {
- MemoryPermission.None => MemoryPermission.ReadAndWrite,
- MemoryPermission.Write => MemoryPermission.Read,
- _ => MemoryPermission.None,
- };
-
- _addressSpace.Base.Reprotect(va, size, protection, false);
}
///
- public RegionHandle BeginTracking(ulong address, ulong size, int id)
+ public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None)
{
- return Tracking.BeginTracking(address, size, id);
+ return Tracking.BeginTracking(address, size, id, flags);
}
///
- public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id)
+ public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None)
{
- return Tracking.BeginGranularTracking(address, size, handles, granularity, id);
+ return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags);
}
///
@@ -687,86 +438,6 @@ namespace Ryujinx.Cpu.Jit
return Tracking.BeginSmartGranularTracking(address, size, granularity, id);
}
- ///
- /// Adds the given address mapping to the page table.
- ///
- /// Virtual memory address
- /// Size to be mapped
- private void AddMapping(ulong va, ulong size)
- {
- int pages = GetPagesCount(va, size, out _);
- ulong pageStart = va >> PageBits;
- ulong pageEnd = pageStart + (ulong)pages;
-
- GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
-
- ulong mask = startMask;
-
- while (pageIndex <= pageEndIndex)
- {
- if (pageIndex == pageEndIndex)
- {
- mask &= endMask;
- }
-
- ref ulong pageRef = ref _pageBitmap[pageIndex++];
-
- ulong pte;
- ulong mappedMask;
-
- // Map all 2-bit entries that are unmapped.
- do
- {
- pte = Volatile.Read(ref pageRef);
-
- mappedMask = pte | (pte >> 1);
- mappedMask |= (mappedMask & BlockMappedMask) << 1;
- mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged.
- }
- while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte);
-
- mask = ulong.MaxValue;
- }
- }
-
- ///
- /// Removes the given address mapping from the page table.
- ///
- /// Virtual memory address
- /// Size to be unmapped
- private void RemoveMapping(ulong va, ulong size)
- {
- int pages = GetPagesCount(va, size, out _);
- ulong pageStart = va >> PageBits;
- ulong pageEnd = pageStart + (ulong)pages;
-
- GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
-
- startMask = ~startMask;
- endMask = ~endMask;
-
- ulong mask = startMask;
-
- while (pageIndex <= pageEndIndex)
- {
- if (pageIndex == pageEndIndex)
- {
- mask |= endMask;
- }
-
- ref ulong pageRef = ref _pageBitmap[pageIndex++];
- ulong pte;
-
- do
- {
- pte = Volatile.Read(ref pageRef);
- }
- while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte);
-
- mask = 0;
- }
- }
-
///
/// Disposes of resources used by the memory manager.
///
@@ -776,10 +447,16 @@ namespace Ryujinx.Cpu.Jit
_memoryEh.Dispose();
}
- protected override Span GetPhysicalAddressSpan(ulong pa, int size)
+ protected override Memory GetPhysicalAddressMemory(nuint pa, int size)
+ => _addressSpace.Mirror.GetMemory(pa, size);
+
+ protected override Span GetPhysicalAddressSpan(nuint pa, int size)
=> _addressSpace.Mirror.GetSpan(pa, size);
- protected override ulong TranslateVirtualAddressForRead(ulong va)
- => va;
+ protected override nuint TranslateVirtualAddressChecked(ulong va)
+ => (nuint)GetPhysicalAddressChecked(va);
+
+ protected override nuint TranslateVirtualAddressUnchecked(ulong va)
+ => (nuint)GetPhysicalAddressInternal(va);
}
}
diff --git a/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs b/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs
new file mode 100644
index 0000000000..663d0aeb15
--- /dev/null
+++ b/src/Ryujinx.Cpu/Jit/MemoryManagerHostTracked.cs
@@ -0,0 +1,634 @@
+using ARMeilleure.Memory;
+using Ryujinx.Common.Memory;
+using Ryujinx.Cpu.Jit.HostTracked;
+using Ryujinx.Cpu.Signal;
+using Ryujinx.Memory;
+using Ryujinx.Memory.Range;
+using Ryujinx.Memory.Tracking;
+using System;
+using System.Buffers;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+
+namespace Ryujinx.Cpu.Jit
+{
+ ///
+ /// Represents a CPU memory manager which maps guest virtual memory directly onto a host virtual region.
+ ///
+ public sealed class MemoryManagerHostTracked : VirtualMemoryManagerRefCountedBase, IMemoryManager, IVirtualMemoryManagerTracked
+ {
+ private readonly InvalidAccessHandler _invalidAccessHandler;
+ private readonly bool _unsafeMode;
+
+ private readonly MemoryBlock _backingMemory;
+
+ public int AddressSpaceBits { get; }
+
+ public MemoryTracking Tracking { get; }
+
+ private readonly NativePageTable _nativePageTable;
+ private readonly AddressSpacePartitioned _addressSpace;
+
+ private readonly ManagedPageFlags _pages;
+
+ protected override ulong AddressSpaceSize { get; }
+
+ ///
+ public bool UsesPrivateAllocations => true;
+
+ public IntPtr PageTablePointer => _nativePageTable.PageTablePointer;
+
+ public MemoryManagerType Type => _unsafeMode ? MemoryManagerType.HostTrackedUnsafe : MemoryManagerType.HostTracked;
+
+ public event Action UnmapEvent;
+
+ ///
+ /// Creates a new instance of the host tracked memory manager.
+ ///
+ /// Physical backing memory where virtual memory will be mapped to
+ /// Size of the address space
+ /// True if unmanaged access should not be masked (unsafe), false otherwise.
+ /// Optional function to handle invalid memory accesses
+ public MemoryManagerHostTracked(MemoryBlock backingMemory, ulong addressSpaceSize, bool unsafeMode, InvalidAccessHandler invalidAccessHandler)
+ {
+ bool useProtectionMirrors = MemoryBlock.GetPageSize() > PageSize;
+
+ Tracking = new MemoryTracking(this, PageSize, invalidAccessHandler, useProtectionMirrors);
+
+ _backingMemory = backingMemory;
+ _invalidAccessHandler = invalidAccessHandler;
+ _unsafeMode = unsafeMode;
+ AddressSpaceSize = addressSpaceSize;
+
+ ulong asSize = PageSize;
+ int asBits = PageBits;
+
+ while (asSize < AddressSpaceSize)
+ {
+ asSize <<= 1;
+ asBits++;
+ }
+
+ AddressSpaceBits = asBits;
+
+ if (useProtectionMirrors && !NativeSignalHandler.SupportsFaultAddressPatching())
+ {
+ // Currently we require being able to change the fault address to something else
+ // in order to "emulate" 4KB granularity protection on systems with larger page size.
+
+ throw new PlatformNotSupportedException();
+ }
+
+ _pages = new ManagedPageFlags(asBits);
+ _nativePageTable = new(asSize);
+ _addressSpace = new(Tracking, backingMemory, _nativePageTable, useProtectionMirrors);
+ }
+
+ public override ReadOnlySequence GetReadOnlySequence(ulong va, int size, bool tracked = false)
+ {
+ if (size == 0)
+ {
+ return ReadOnlySequence.Empty;
+ }
+
+ try
+ {
+ if (tracked)
+ {
+ SignalMemoryTracking(va, (ulong)size, false);
+ }
+ else
+ {
+ AssertValidAddressAndSize(va, (ulong)size);
+ }
+
+ ulong endVa = va + (ulong)size;
+ int offset = 0;
+
+ BytesReadOnlySequenceSegment first = null, last = null;
+
+ while (va < endVa)
+ {
+ (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(size - offset));
+
+ Memory physicalMemory = memory.GetMemory(rangeOffset, (int)copySize);
+
+ if (first is null)
+ {
+ first = last = new BytesReadOnlySequenceSegment(physicalMemory);
+ }
+ else
+ {
+ if (last.IsContiguousWith(physicalMemory, out nuint contiguousStart, out int contiguousSize))
+ {
+ Memory contiguousPhysicalMemory = new NativeMemoryManager(contiguousStart, contiguousSize).Memory;
+
+ last.Replace(contiguousPhysicalMemory);
+ }
+ else
+ {
+ last = last.Append(physicalMemory);
+ }
+ }
+
+ va += copySize;
+ offset += (int)copySize;
+ }
+
+ return new ReadOnlySequence(first, 0, last, (int)(size - last.RunningIndex));
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+
+ return ReadOnlySequence.Empty;
+ }
+ }
+
+ ///
+ public void Map(ulong va, ulong pa, ulong size, MemoryMapFlags flags)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ if (flags.HasFlag(MemoryMapFlags.Private))
+ {
+ _addressSpace.Map(va, pa, size);
+ }
+
+ _pages.AddMapping(va, size);
+ _nativePageTable.Map(va, pa, size, _addressSpace, _backingMemory, flags.HasFlag(MemoryMapFlags.Private));
+
+ Tracking.Map(va, size);
+ }
+
+ ///
+ public void Unmap(ulong va, ulong size)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ _addressSpace.Unmap(va, size);
+
+ UnmapEvent?.Invoke(va, size);
+ Tracking.Unmap(va, size);
+
+ _pages.RemoveMapping(va, size);
+ _nativePageTable.Unmap(va, size);
+ }
+
+ public override T ReadTracked(ulong va)
+ {
+ try
+ {
+ return base.ReadTracked(va);
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+
+ return default;
+ }
+ }
+
+ public override void Read(ulong va, Span data)
+ {
+ if (data.Length == 0)
+ {
+ return;
+ }
+
+ try
+ {
+ AssertValidAddressAndSize(va, (ulong)data.Length);
+
+ ulong endVa = va + (ulong)data.Length;
+ int offset = 0;
+
+ while (va < endVa)
+ {
+ (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset));
+
+ memory.GetSpan(rangeOffset, (int)copySize).CopyTo(data.Slice(offset, (int)copySize));
+
+ va += copySize;
+ offset += (int)copySize;
+ }
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+ }
+ }
+
+ public override bool WriteWithRedundancyCheck(ulong va, ReadOnlySpan data)
+ {
+ if (data.Length == 0)
+ {
+ return false;
+ }
+
+ SignalMemoryTracking(va, (ulong)data.Length, false);
+
+ if (TryGetVirtualContiguous(va, data.Length, out MemoryBlock memoryBlock, out ulong offset))
+ {
+ var target = memoryBlock.GetSpan(offset, data.Length);
+
+ bool changed = !data.SequenceEqual(target);
+
+ if (changed)
+ {
+ data.CopyTo(target);
+ }
+
+ return changed;
+ }
+ else
+ {
+ WriteImpl(va, data);
+
+ return true;
+ }
+ }
+
+ public override ReadOnlySpan GetSpan(ulong va, int size, bool tracked = false)
+ {
+ if (size == 0)
+ {
+ return ReadOnlySpan.Empty;
+ }
+
+ if (tracked)
+ {
+ SignalMemoryTracking(va, (ulong)size, false);
+ }
+
+ if (TryGetVirtualContiguous(va, size, out MemoryBlock memoryBlock, out ulong offset))
+ {
+ return memoryBlock.GetSpan(offset, size);
+ }
+ else
+ {
+ Span data = new byte[size];
+
+ Read(va, data);
+
+ return data;
+ }
+ }
+
+ public override WritableRegion GetWritableRegion(ulong va, int size, bool tracked = false)
+ {
+ if (size == 0)
+ {
+ return new WritableRegion(null, va, Memory.Empty);
+ }
+
+ if (tracked)
+ {
+ SignalMemoryTracking(va, (ulong)size, true);
+ }
+
+ if (TryGetVirtualContiguous(va, size, out MemoryBlock memoryBlock, out ulong offset))
+ {
+ return new WritableRegion(null, va, memoryBlock.GetMemory(offset, size));
+ }
+ else
+ {
+ IMemoryOwner memoryOwner = ByteMemoryPool.Rent(size);
+
+ Read(va, memoryOwner.Memory.Span);
+
+ return new WritableRegion(this, va, memoryOwner);
+ }
+ }
+
+ public ref T GetRef(ulong va) where T : unmanaged
+ {
+ if (!TryGetVirtualContiguous(va, Unsafe.SizeOf(), out MemoryBlock memory, out ulong offset))
+ {
+ ThrowMemoryNotContiguous();
+ }
+
+ SignalMemoryTracking(va, (ulong)Unsafe.SizeOf(), true);
+
+ return ref memory.GetRef(offset);
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public override bool IsMapped(ulong va)
+ {
+ return ValidateAddress(va) && _pages.IsMapped(va);
+ }
+
+ public bool IsRangeMapped(ulong va, ulong size)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ return _pages.IsRangeMapped(va, size);
+ }
+
+ private bool TryGetVirtualContiguous(ulong va, int size, out MemoryBlock memory, out ulong offset)
+ {
+ if (_addressSpace.HasAnyPrivateAllocation(va, (ulong)size, out PrivateRange range))
+ {
+ // If we have a private allocation overlapping the range,
+ // then the access is only considered contiguous if it covers the entire range.
+
+ if (range.Memory != null)
+ {
+ memory = range.Memory;
+ offset = range.Offset;
+
+ return true;
+ }
+
+ memory = null;
+ offset = 0;
+
+ return false;
+ }
+
+ memory = _backingMemory;
+ offset = GetPhysicalAddressInternal(va);
+
+ return IsPhysicalContiguous(va, size);
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private bool IsPhysicalContiguous(ulong va, int size)
+ {
+ if (!ValidateAddress(va) || !ValidateAddressAndSize(va, (ulong)size))
+ {
+ return false;
+ }
+
+ int pages = GetPagesCount(va, (uint)size, out va);
+
+ for (int page = 0; page < pages - 1; page++)
+ {
+ if (!ValidateAddress(va + PageSize))
+ {
+ return false;
+ }
+
+ if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize))
+ {
+ return false;
+ }
+
+ va += PageSize;
+ }
+
+ return true;
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private ulong GetContiguousSize(ulong va, ulong size)
+ {
+ ulong contiguousSize = PageSize - (va & PageMask);
+
+ if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size))
+ {
+ return contiguousSize;
+ }
+
+ int pages = GetPagesCount(va, size, out va);
+
+ for (int page = 0; page < pages - 1; page++)
+ {
+ if (!ValidateAddress(va + PageSize))
+ {
+ return contiguousSize;
+ }
+
+ if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize))
+ {
+ return contiguousSize;
+ }
+
+ va += PageSize;
+ contiguousSize += PageSize;
+ }
+
+ return Math.Min(contiguousSize, size);
+ }
+
+ private (MemoryBlock, ulong, ulong) GetMemoryOffsetAndSize(ulong va, ulong size)
+ {
+ PrivateRange privateRange = _addressSpace.GetFirstPrivateAllocation(va, size, out ulong nextVa);
+
+ if (privateRange.Memory != null)
+ {
+ return (privateRange.Memory, privateRange.Offset, privateRange.Size);
+ }
+
+ ulong physSize = GetContiguousSize(va, Math.Min(size, nextVa - va));
+
+ return (_backingMemory, GetPhysicalAddressChecked(va), physSize);
+ }
+
+ public IEnumerable GetHostRegions(ulong va, ulong size)
+ {
+ if (!ValidateAddressAndSize(va, size))
+ {
+ return null;
+ }
+
+ var regions = new List();
+ ulong endVa = va + size;
+
+ try
+ {
+ while (va < endVa)
+ {
+ (MemoryBlock memory, ulong rangeOffset, ulong rangeSize) = GetMemoryOffsetAndSize(va, endVa - va);
+
+ regions.Add(new((UIntPtr)memory.GetPointer(rangeOffset, rangeSize), rangeSize));
+
+ va += rangeSize;
+ }
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ return null;
+ }
+
+ return regions;
+ }
+
+ public IEnumerable GetPhysicalRegions(ulong va, ulong size)
+ {
+ if (size == 0)
+ {
+ return Enumerable.Empty();
+ }
+
+ return GetPhysicalRegionsImpl(va, size);
+ }
+
+ private List GetPhysicalRegionsImpl(ulong va, ulong size)
+ {
+ if (!ValidateAddress(va) || !ValidateAddressAndSize(va, size))
+ {
+ return null;
+ }
+
+ int pages = GetPagesCount(va, (uint)size, out va);
+
+ var regions = new List();
+
+ ulong regionStart = GetPhysicalAddressInternal(va);
+ ulong regionSize = PageSize;
+
+ for (int page = 0; page < pages - 1; page++)
+ {
+ if (!ValidateAddress(va + PageSize))
+ {
+ return null;
+ }
+
+ ulong newPa = GetPhysicalAddressInternal(va + PageSize);
+
+ if (GetPhysicalAddressInternal(va) + PageSize != newPa)
+ {
+ regions.Add(new MemoryRange(regionStart, regionSize));
+ regionStart = newPa;
+ regionSize = 0;
+ }
+
+ va += PageSize;
+ regionSize += PageSize;
+ }
+
+ regions.Add(new MemoryRange(regionStart, regionSize));
+
+ return regions;
+ }
+
+ ///
+ ///
+ /// This function also validates that the given range is both valid and mapped, and will throw if it is not.
+ ///
+ public override void SignalMemoryTracking(ulong va, ulong size, bool write, bool precise = false, int? exemptId = null)
+ {
+ AssertValidAddressAndSize(va, size);
+
+ if (precise)
+ {
+ Tracking.VirtualMemoryEvent(va, size, write, precise: true, exemptId);
+ return;
+ }
+
+ // Software table, used for managed memory tracking.
+
+ _pages.SignalMemoryTracking(Tracking, va, size, write, exemptId);
+ }
+
+ public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None)
+ {
+ return Tracking.BeginTracking(address, size, id, flags);
+ }
+
+ public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None)
+ {
+ return Tracking.BeginGranularTracking(address, size, handles, granularity, id, flags);
+ }
+
+ public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id)
+ {
+ return Tracking.BeginSmartGranularTracking(address, size, granularity, id);
+ }
+
+ private ulong GetPhysicalAddressChecked(ulong va)
+ {
+ if (!IsMapped(va))
+ {
+ ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}");
+ }
+
+ return GetPhysicalAddressInternal(va);
+ }
+
+ private ulong GetPhysicalAddressInternal(ulong va)
+ {
+ return _nativePageTable.GetPhysicalAddress(va);
+ }
+
+ ///
+ public void Reprotect(ulong va, ulong size, MemoryPermission protection)
+ {
+ // TODO
+ }
+
+ ///
+ public void TrackingReprotect(ulong va, ulong size, MemoryPermission protection, bool guest)
+ {
+ if (guest)
+ {
+ _addressSpace.Reprotect(va, size, protection);
+ }
+ else
+ {
+ _pages.TrackingReprotect(va, size, protection);
+ }
+ }
+
+ ///
+ /// Disposes of resources used by the memory manager.
+ ///
+ protected override void Destroy()
+ {
+ _addressSpace.Dispose();
+ _nativePageTable.Dispose();
+ }
+
+ protected override Memory GetPhysicalAddressMemory(nuint pa, int size)
+ => _backingMemory.GetMemory(pa, size);
+
+ protected override Span GetPhysicalAddressSpan(nuint pa, int size)
+ => _backingMemory.GetSpan(pa, size);
+
+ protected override void WriteImpl(ulong va, ReadOnlySpan data)
+ {
+ try
+ {
+ AssertValidAddressAndSize(va, (ulong)data.Length);
+
+ ulong endVa = va + (ulong)data.Length;
+ int offset = 0;
+
+ while (va < endVa)
+ {
+ (MemoryBlock memory, ulong rangeOffset, ulong copySize) = GetMemoryOffsetAndSize(va, (ulong)(data.Length - offset));
+
+ data.Slice(offset, (int)copySize).CopyTo(memory.GetSpan(rangeOffset, (int)copySize));
+
+ va += copySize;
+ offset += (int)copySize;
+ }
+ }
+ catch (InvalidMemoryRegionException)
+ {
+ if (_invalidAccessHandler == null || !_invalidAccessHandler(va))
+ {
+ throw;
+ }
+ }
+ }
+
+ protected override nuint TranslateVirtualAddressChecked(ulong va)
+ => (nuint)GetPhysicalAddressChecked(va);
+
+ protected override nuint TranslateVirtualAddressUnchecked(ulong va)
+ => (nuint)GetPhysicalAddressInternal(va);
+ }
+}
diff --git a/src/Ryujinx.Cpu/LightningJit/Arm32/Target/Arm64/InstEmitMemory.cs b/src/Ryujinx.Cpu/LightningJit/Arm32/Target/Arm64/InstEmitMemory.cs
index 6ab4b94953..d8caee6e74 100644
--- a/src/Ryujinx.Cpu/LightningJit/Arm32/Target/Arm64/InstEmitMemory.cs
+++ b/src/Ryujinx.Cpu/LightningJit/Arm32/Target/Arm64/InstEmitMemory.cs
@@ -1126,11 +1126,23 @@ namespace Ryujinx.Cpu.LightningJit.Arm32.Target.Arm64
Operand destination64 = new(destination.Kind, OperandType.I64, destination.Value);
Operand basePointer = new(regAlloc.FixedPageTableRegister, RegisterType.Integer, OperandType.I64);
- if (mmType == MemoryManagerType.HostMapped || mmType == MemoryManagerType.HostMappedUnsafe)
- {
- // We don't need to mask the address for the safe mode, since it is already naturally limited to 32-bit
- // and can never reach out of the guest address space.
+ // We don't need to mask the address for the safe mode, since it is already naturally limited to 32-bit
+ // and can never reach out of the guest address space.
+ if (mmType.IsHostTracked())
+ {
+ int tempRegister = regAlloc.AllocateTempGprRegister();
+
+ Operand pte = new(tempRegister, RegisterType.Integer, OperandType.I64);
+
+ asm.Lsr(pte, guestAddress, new Operand(OperandKind.Constant, OperandType.I32, 12));
+ asm.LdrRr(pte, basePointer, pte, ArmExtensionType.Uxtx, true);
+ asm.Add(destination64, pte, guestAddress);
+
+ regAlloc.FreeTempGprRegister(tempRegister);
+ }
+ else if (mmType.IsHostMapped())
+ {
asm.Add(destination64, basePointer, guestAddress);
}
else
diff --git a/src/Ryujinx.Cpu/LightningJit/Arm64/InstName.cs b/src/Ryujinx.Cpu/LightningJit/Arm64/InstName.cs
index 3656406453..3391a2c145 100644
--- a/src/Ryujinx.Cpu/LightningJit/Arm64/InstName.cs
+++ b/src/Ryujinx.Cpu/LightningJit/Arm64/InstName.cs
@@ -1131,5 +1131,37 @@ namespace Ryujinx.Cpu.LightningJit.Arm64
return false;
}
+
+ public static bool IsPartialRegisterUpdateMemory(this InstName name)
+ {
+ switch (name)
+ {
+ case InstName.Ld1AdvsimdSnglAsNoPostIndex:
+ case InstName.Ld1AdvsimdSnglAsPostIndex:
+ case InstName.Ld2AdvsimdSnglAsNoPostIndex:
+ case InstName.Ld2AdvsimdSnglAsPostIndex:
+ case InstName.Ld3AdvsimdSnglAsNoPostIndex:
+ case InstName.Ld3AdvsimdSnglAsPostIndex:
+ case InstName.Ld4AdvsimdSnglAsNoPostIndex:
+ case InstName.Ld4AdvsimdSnglAsPostIndex:
+ return true;
+ }
+
+ return false;
+ }
+
+ public static bool IsPrefetchMemory(this InstName name)
+ {
+ switch (name)
+ {
+ case InstName.PrfmImm:
+ case InstName.PrfmLit:
+ case InstName.PrfmReg:
+ case InstName.Prfum:
+ return true;
+ }
+
+ return false;
+ }
}
}
diff --git a/src/Ryujinx.Cpu/LightningJit/Arm64/RegisterAllocator.cs b/src/Ryujinx.Cpu/LightningJit/Arm64/RegisterAllocator.cs
index c9a932093d..1c6eab0de2 100644
--- a/src/Ryujinx.Cpu/LightningJit/Arm64/RegisterAllocator.cs
+++ b/src/Ryujinx.Cpu/LightningJit/Arm64/RegisterAllocator.cs
@@ -1,15 +1,12 @@
+using ARMeilleure.Memory;
using Ryujinx.Cpu.LightningJit.CodeGen.Arm64;
using System;
-using System.Diagnostics;
using System.Numerics;
namespace Ryujinx.Cpu.LightningJit.Arm64
{
class RegisterAllocator
{
- public const int MaxTemps = 1;
- public const int MaxTempsInclFixed = MaxTemps + 2;
-
private uint _gprMask;
private readonly uint _fpSimdMask;
private readonly uint _pStateMask;
@@ -25,7 +22,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm64
public uint AllFpSimdMask => _fpSimdMask;
public uint AllPStateMask => _pStateMask;
- public RegisterAllocator(uint gprMask, uint fpSimdMask, uint pStateMask, bool hasHostCall)
+ public RegisterAllocator(MemoryManagerType mmType, uint gprMask, uint fpSimdMask, uint pStateMask, bool hasHostCall)
{
_gprMask = gprMask;
_fpSimdMask = fpSimdMask;
@@ -56,7 +53,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm64
BuildRegisterMap(_registerMap);
- Span tempRegisters = stackalloc int[MaxTemps];
+ Span tempRegisters = stackalloc int[CalculateMaxTemps(mmType)];
for (int index = 0; index < tempRegisters.Length; index++)
{
@@ -150,5 +147,15 @@ namespace Ryujinx.Cpu.LightningJit.Arm64
{
mask &= ~(1u << index);
}
+
+ public static int CalculateMaxTemps(MemoryManagerType mmType)
+ {
+ return mmType.IsHostMapped() ? 1 : 2;
+ }
+
+ public static int CalculateMaxTempsInclFixed(MemoryManagerType mmType)
+ {
+ return CalculateMaxTemps(mmType) + 2;
+ }
}
}
diff --git a/src/Ryujinx.Cpu/LightningJit/Arm64/RegisterUtils.cs b/src/Ryujinx.Cpu/LightningJit/Arm64/RegisterUtils.cs
index eb3fc229fe..191e03e7b1 100644
--- a/src/Ryujinx.Cpu/LightningJit/Arm64/RegisterUtils.cs
+++ b/src/Ryujinx.Cpu/LightningJit/Arm64/RegisterUtils.cs
@@ -247,7 +247,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm64
}
}
- if (!flags.HasFlag(InstFlags.ReadRt))
+ if (!flags.HasFlag(InstFlags.ReadRt) || name.IsPartialRegisterUpdateMemory())
{
if (flags.HasFlag(InstFlags.Rt))
{
@@ -281,7 +281,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm64
gprMask |= MaskFromIndex(ExtractRd(flags, encoding));
}
- if (!flags.HasFlag(InstFlags.ReadRt))
+ if (!flags.HasFlag(InstFlags.ReadRt) || name.IsPartialRegisterUpdateMemory())
{
if (flags.HasFlag(InstFlags.Rt))
{
diff --git a/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/Compiler.cs b/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/Compiler.cs
index 7ef3bf49b9..7a6d761e8d 100644
--- a/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/Compiler.cs
+++ b/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/Compiler.cs
@@ -316,7 +316,7 @@ namespace Ryujinx.Cpu.LightningJit.Arm64.Target.Arm64
uint pStateUseMask = multiBlock.GlobalUseMask.PStateMask;
CodeWriter writer = new();
- RegisterAllocator regAlloc = new(gprUseMask, fpSimdUseMask, pStateUseMask, multiBlock.HasHostCall);
+ RegisterAllocator regAlloc = new(memoryManager.Type, gprUseMask, fpSimdUseMask, pStateUseMask, multiBlock.HasHostCall);
RegisterSaveRestore rsr = new(
regAlloc.AllGprMask & AbiConstants.GprCalleeSavedRegsMask,
regAlloc.AllFpSimdMask & AbiConstants.FpSimdCalleeSavedRegsMask,
diff --git a/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/Decoder.cs b/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/Decoder.cs
index 00a1758f29..d5e1eb19c2 100644
--- a/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/Decoder.cs
+++ b/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/Decoder.cs
@@ -274,7 +274,8 @@ namespace Ryujinx.Cpu.LightningJit.Arm64.Target.Arm64
uint tempGprUseMask = gprUseMask | instGprReadMask | instGprWriteMask;
- if (CalculateAvailableTemps(tempGprUseMask) < CalculateRequiredGprTemps(tempGprUseMask) || totalInsts++ >= MaxInstructionsPerFunction)
+ if (CalculateAvailableTemps(tempGprUseMask) < CalculateRequiredGprTemps(memoryManager.Type, tempGprUseMask) ||
+ totalInsts++ >= MaxInstructionsPerFunction)
{
isTruncated = true;
address -= 4UL;
@@ -378,9 +379,9 @@ namespace Ryujinx.Cpu.LightningJit.Arm64.Target.Arm64
return false;
}
- private static int CalculateRequiredGprTemps(uint gprUseMask)
+ private static int CalculateRequiredGprTemps(MemoryManagerType mmType, uint gprUseMask)
{
- return BitOperations.PopCount(gprUseMask & RegisterUtils.ReservedRegsMask) + RegisterAllocator.MaxTempsInclFixed;
+ return BitOperations.PopCount(gprUseMask & RegisterUtils.ReservedRegsMask) + RegisterAllocator.CalculateMaxTempsInclFixed(mmType);
}
private static int CalculateAvailableTemps(uint gprUseMask)
diff --git a/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/InstEmitMemory.cs b/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/InstEmitMemory.cs
index e03d9373a1..790a7de95b 100644
--- a/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/InstEmitMemory.cs
+++ b/src/Ryujinx.Cpu/LightningJit/Arm64/Target/Arm64/InstEmitMemory.cs
@@ -55,6 +55,16 @@ namespace Ryujinx.Cpu.LightningJit.Arm64.Target.Arm64
ulong pc,
uint encoding)
{
+ if (name.IsPrefetchMemory() && mmType == MemoryManagerType.HostTrackedUnsafe)
+ {
+ // Prefetch to invalid addresses do not cause faults, so for memory manager
+ // types where we need to access the page table before doing the prefetch,
+ // we should make sure we won't try to access an out of bounds page table region.
+ // To do this, we force the masked memory manager variant to be used.
+
+ mmType = MemoryManagerType.HostTracked;
+ }
+
switch (addressForm)
{
case AddressForm.OffsetReg:
@@ -511,18 +521,48 @@ namespace Ryujinx.Cpu.LightningJit.Arm64.Target.Arm64
WriteAddressTranslation(asBits, mmType, regAlloc, ref asm, destination, guestAddress);
}
- private static void WriteAddressTranslation(int asBits, MemoryManagerType mmType, RegisterAllocator regAlloc, ref Assembler asm, Operand destination, ulong guestAddress)
+ private static void WriteAddressTranslation(
+ int asBits,
+ MemoryManagerType mmType,
+ RegisterAllocator regAlloc,
+ ref Assembler asm,
+ Operand destination,
+ ulong guestAddress)
{
asm.Mov(destination, guestAddress);
WriteAddressTranslation(asBits, mmType, regAlloc, ref asm, destination, destination);
}
- private static void WriteAddressTranslation(int asBits, MemoryManagerType mmType, RegisterAllocator regAlloc, ref Assembler asm, Operand destination, Operand guestAddress)
+ private static void WriteAddressTranslation(
+ int asBits,
+ MemoryManagerType mmType,
+ RegisterAllocator regAlloc,
+ ref Assembler asm,
+ Operand destination,
+ Operand guestAddress)
{
Operand basePointer = new(regAlloc.FixedPageTableRegister, RegisterType.Integer, OperandType.I64);
- if (mmType == MemoryManagerType.HostMapped || mmType == MemoryManagerType.HostMappedUnsafe)
+ if (mmType.IsHostTracked())
+ {
+ int tempRegister = regAlloc.AllocateTempGprRegister();
+
+ Operand pte = new(tempRegister, RegisterType.Integer, OperandType.I64);
+
+ asm.Lsr(pte, guestAddress, new Operand(OperandKind.Constant, OperandType.I32, 12));
+
+ if (mmType == MemoryManagerType.HostTracked)
+ {
+ asm.And(pte, pte, new Operand(OperandKind.Constant, OperandType.I64, ulong.MaxValue >> (64 - (asBits - 12))));
+ }
+
+ asm.LdrRr(pte, basePointer, pte, ArmExtensionType.Uxtx, true);
+ asm.Add(destination, pte, guestAddress);
+
+ regAlloc.FreeTempGprRegister(tempRegister);
+ }
+ else if (mmType.IsHostMapped())
{
if (mmType == MemoryManagerType.HostMapped)
{
diff --git a/src/Ryujinx.Cpu/LightningJit/Translator.cs b/src/Ryujinx.Cpu/LightningJit/Translator.cs
index c883c1d601..d624102534 100644
--- a/src/Ryujinx.Cpu/LightningJit/Translator.cs
+++ b/src/Ryujinx.Cpu/LightningJit/Translator.cs
@@ -68,9 +68,9 @@ namespace Ryujinx.Cpu.LightningJit
FunctionTable.Fill = (ulong)Stubs.SlowDispatchStub;
- if (memory.Type.IsHostMapped())
+ if (memory.Type.IsHostMappedOrTracked())
{
- NativeSignalHandler.InitializeSignalHandler(MemoryBlock.GetPageSize());
+ NativeSignalHandler.InitializeSignalHandler();
}
}
diff --git a/src/Ryujinx.Cpu/ManagedPageFlags.cs b/src/Ryujinx.Cpu/ManagedPageFlags.cs
new file mode 100644
index 0000000000..a839dae676
--- /dev/null
+++ b/src/Ryujinx.Cpu/ManagedPageFlags.cs
@@ -0,0 +1,389 @@
+using Ryujinx.Memory;
+using Ryujinx.Memory.Tracking;
+using System;
+using System.Runtime.CompilerServices;
+using System.Threading;
+
+namespace Ryujinx.Cpu
+{
+ ///
+ /// A page bitmap that keeps track of mapped state and tracking protection
+ /// for managed memory accesses (not using host page protection).
+ ///
+ internal readonly struct ManagedPageFlags
+ {
+ public const int PageBits = 12;
+ public const int PageSize = 1 << PageBits;
+ public const int PageMask = PageSize - 1;
+
+ private readonly ulong[] _pageBitmap;
+
+ public const int PageToPteShift = 5; // 32 pages (2 bits each) in one ulong page table entry.
+ public const ulong BlockMappedMask = 0x5555555555555555; // First bit of each table entry set.
+
+ private enum ManagedPtBits : ulong
+ {
+ Unmapped = 0,
+ Mapped,
+ WriteTracked,
+ ReadWriteTracked,
+
+ MappedReplicated = 0x5555555555555555,
+ WriteTrackedReplicated = 0xaaaaaaaaaaaaaaaa,
+ ReadWriteTrackedReplicated = ulong.MaxValue,
+ }
+
+ public ManagedPageFlags(int addressSpaceBits)
+ {
+ int bits = Math.Max(0, addressSpaceBits - (PageBits + PageToPteShift));
+ _pageBitmap = new ulong[1 << bits];
+ }
+
+ ///
+ /// Computes the number of pages in a virtual address range.
+ ///
+ /// Virtual address of the range
+ /// Size of the range
+ /// The virtual address of the beginning of the first page
+ /// This function does not differentiate between allocated and unallocated pages.
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static int GetPagesCount(ulong va, ulong size, out ulong startVa)
+ {
+ // WARNING: Always check if ulong does not overflow during the operations.
+ startVa = va & ~(ulong)PageMask;
+ ulong vaSpan = (va - startVa + size + PageMask) & ~(ulong)PageMask;
+
+ return (int)(vaSpan / PageSize);
+ }
+
+ ///
+ /// Checks if the page at a given CPU virtual address is mapped.
+ ///
+ /// Virtual address to check
+ /// True if the address is mapped, false otherwise
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public readonly bool IsMapped(ulong va)
+ {
+ ulong page = va >> PageBits;
+
+ int bit = (int)((page & 31) << 1);
+
+ int pageIndex = (int)(page >> PageToPteShift);
+ ref ulong pageRef = ref _pageBitmap[pageIndex];
+
+ ulong pte = Volatile.Read(ref pageRef);
+
+ return ((pte >> bit) & 3) != 0;
+ }
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static void GetPageBlockRange(ulong pageStart, ulong pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex)
+ {
+ startMask = ulong.MaxValue << ((int)(pageStart & 31) << 1);
+ endMask = ulong.MaxValue >> (64 - ((int)(pageEnd & 31) << 1));
+
+ pageIndex = (int)(pageStart >> PageToPteShift);
+ pageEndIndex = (int)((pageEnd - 1) >> PageToPteShift);
+ }
+
+ ///
+ /// Checks if a memory range is mapped.
+ ///
+ /// Virtual address of the range
+ /// Size of the range in bytes
+ /// True if the entire range is mapped, false otherwise
+ public readonly bool IsRangeMapped(ulong va, ulong size)
+ {
+ int pages = GetPagesCount(va, size, out _);
+
+ if (pages == 1)
+ {
+ return IsMapped(va);
+ }
+
+ ulong pageStart = va >> PageBits;
+ ulong pageEnd = pageStart + (ulong)pages;
+
+ GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
+
+ // Check if either bit in each 2 bit page entry is set.
+ // OR the block with itself shifted down by 1, and check the first bit of each entry.
+
+ ulong mask = BlockMappedMask & startMask;
+
+ while (pageIndex <= pageEndIndex)
+ {
+ if (pageIndex == pageEndIndex)
+ {
+ mask &= endMask;
+ }
+
+ ref ulong pageRef = ref _pageBitmap[pageIndex++];
+ ulong pte = Volatile.Read(ref pageRef);
+
+ pte |= pte >> 1;
+ if ((pte & mask) != mask)
+ {
+ return false;
+ }
+
+ mask = BlockMappedMask;
+ }
+
+ return true;
+ }
+
+ ///
+ /// Reprotect a region of virtual memory for tracking.
+ ///
+ /// Virtual address base
+ /// Size of the region to protect
+ /// Memory protection to set
+ public readonly void TrackingReprotect(ulong va, ulong size, MemoryPermission protection)
+ {
+ // Protection is inverted on software pages, since the default value is 0.
+ protection = (~protection) & MemoryPermission.ReadAndWrite;
+
+ int pages = GetPagesCount(va, size, out va);
+ ulong pageStart = va >> PageBits;
+
+ if (pages == 1)
+ {
+ ulong protTag = protection switch
+ {
+ MemoryPermission.None => (ulong)ManagedPtBits.Mapped,
+ MemoryPermission.Write => (ulong)ManagedPtBits.WriteTracked,
+ _ => (ulong)ManagedPtBits.ReadWriteTracked,
+ };
+
+ int bit = (int)((pageStart & 31) << 1);
+
+ ulong tagMask = 3UL << bit;
+ ulong invTagMask = ~tagMask;
+
+ ulong tag = protTag << bit;
+
+ int pageIndex = (int)(pageStart >> PageToPteShift);
+ ref ulong pageRef = ref _pageBitmap[pageIndex];
+
+ ulong pte;
+
+ do
+ {
+ pte = Volatile.Read(ref pageRef);
+ }
+ while ((pte & tagMask) != 0 && Interlocked.CompareExchange(ref pageRef, (pte & invTagMask) | tag, pte) != pte);
+ }
+ else
+ {
+ ulong pageEnd = pageStart + (ulong)pages;
+
+ GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
+
+ ulong mask = startMask;
+
+ ulong protTag = protection switch
+ {
+ MemoryPermission.None => (ulong)ManagedPtBits.MappedReplicated,
+ MemoryPermission.Write => (ulong)ManagedPtBits.WriteTrackedReplicated,
+ _ => (ulong)ManagedPtBits.ReadWriteTrackedReplicated,
+ };
+
+ while (pageIndex <= pageEndIndex)
+ {
+ if (pageIndex == pageEndIndex)
+ {
+ mask &= endMask;
+ }
+
+ ref ulong pageRef = ref _pageBitmap[pageIndex++];
+
+ ulong pte;
+ ulong mappedMask;
+
+ // Change the protection of all 2 bit entries that are mapped.
+ do
+ {
+ pte = Volatile.Read(ref pageRef);
+
+ mappedMask = pte | (pte >> 1);
+ mappedMask |= (mappedMask & BlockMappedMask) << 1;
+ mappedMask &= mask; // Only update mapped pages within the given range.
+ }
+ while (Interlocked.CompareExchange(ref pageRef, (pte & (~mappedMask)) | (protTag & mappedMask), pte) != pte);
+
+ mask = ulong.MaxValue;
+ }
+ }
+ }
+
+ ///
+ /// Alerts the memory tracking that a given region has been read from or written to.
+ /// This should be called before read/write is performed.
+ ///
+ /// Memory tracking structure to call when pages are protected
+ /// Virtual address of the region
+ /// Size of the region
+ /// True if the region was written, false if read
+ /// Optional ID of the handles that should not be signalled
+ ///
+ /// This function also validates that the given range is both valid and mapped, and will throw if it is not.
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public readonly void SignalMemoryTracking(MemoryTracking tracking, ulong va, ulong size, bool write, int? exemptId = null)
+ {
+ // Software table, used for managed memory tracking.
+
+ int pages = GetPagesCount(va, size, out _);
+ ulong pageStart = va >> PageBits;
+
+ if (pages == 1)
+ {
+ ulong tag = (ulong)(write ? ManagedPtBits.WriteTracked : ManagedPtBits.ReadWriteTracked);
+
+ int bit = (int)((pageStart & 31) << 1);
+
+ int pageIndex = (int)(pageStart >> PageToPteShift);
+ ref ulong pageRef = ref _pageBitmap[pageIndex];
+
+ ulong pte = Volatile.Read(ref pageRef);
+ ulong state = ((pte >> bit) & 3);
+
+ if (state >= tag)
+ {
+ tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
+ return;
+ }
+ else if (state == 0)
+ {
+ ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
+ }
+ }
+ else
+ {
+ ulong pageEnd = pageStart + (ulong)pages;
+
+ GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
+
+ ulong mask = startMask;
+
+ ulong anyTrackingTag = (ulong)ManagedPtBits.WriteTrackedReplicated;
+
+ while (pageIndex <= pageEndIndex)
+ {
+ if (pageIndex == pageEndIndex)
+ {
+ mask &= endMask;
+ }
+
+ ref ulong pageRef = ref _pageBitmap[pageIndex++];
+
+ ulong pte = Volatile.Read(ref pageRef);
+ ulong mappedMask = mask & BlockMappedMask;
+
+ ulong mappedPte = pte | (pte >> 1);
+ if ((mappedPte & mappedMask) != mappedMask)
+ {
+ ThrowInvalidMemoryRegionException($"Not mapped: va=0x{va:X16}, size=0x{size:X16}");
+ }
+
+ pte &= mask;
+ if ((pte & anyTrackingTag) != 0) // Search for any tracking.
+ {
+ // Writes trigger any tracking.
+ // Only trigger tracking from reads if both bits are set on any page.
+ if (write || (pte & (pte >> 1) & BlockMappedMask) != 0)
+ {
+ tracking.VirtualMemoryEvent(va, size, write, precise: false, exemptId);
+ break;
+ }
+ }
+
+ mask = ulong.MaxValue;
+ }
+ }
+ }
+
+ ///
+ /// Adds the given address mapping to the page table.
+ ///
+ /// Virtual memory address
+ /// Size to be mapped
+ public readonly void AddMapping(ulong va, ulong size)
+ {
+ int pages = GetPagesCount(va, size, out _);
+ ulong pageStart = va >> PageBits;
+ ulong pageEnd = pageStart + (ulong)pages;
+
+ GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
+
+ ulong mask = startMask;
+
+ while (pageIndex <= pageEndIndex)
+ {
+ if (pageIndex == pageEndIndex)
+ {
+ mask &= endMask;
+ }
+
+ ref ulong pageRef = ref _pageBitmap[pageIndex++];
+
+ ulong pte;
+ ulong mappedMask;
+
+ // Map all 2-bit entries that are unmapped.
+ do
+ {
+ pte = Volatile.Read(ref pageRef);
+
+ mappedMask = pte | (pte >> 1);
+ mappedMask |= (mappedMask & BlockMappedMask) << 1;
+ mappedMask |= ~mask; // Treat everything outside the range as mapped, thus unchanged.
+ }
+ while (Interlocked.CompareExchange(ref pageRef, (pte & mappedMask) | (BlockMappedMask & (~mappedMask)), pte) != pte);
+
+ mask = ulong.MaxValue;
+ }
+ }
+
+ ///
+ /// Removes the given address mapping from the page table.
+ ///
+ /// Virtual memory address
+ /// Size to be unmapped
+ public readonly void RemoveMapping(ulong va, ulong size)
+ {
+ int pages = GetPagesCount(va, size, out _);
+ ulong pageStart = va >> PageBits;
+ ulong pageEnd = pageStart + (ulong)pages;
+
+ GetPageBlockRange(pageStart, pageEnd, out ulong startMask, out ulong endMask, out int pageIndex, out int pageEndIndex);
+
+ startMask = ~startMask;
+ endMask = ~endMask;
+
+ ulong mask = startMask;
+
+ while (pageIndex <= pageEndIndex)
+ {
+ if (pageIndex == pageEndIndex)
+ {
+ mask |= endMask;
+ }
+
+ ref ulong pageRef = ref _pageBitmap[pageIndex++];
+ ulong pte;
+
+ do
+ {
+ pte = Volatile.Read(ref pageRef);
+ }
+ while (Interlocked.CompareExchange(ref pageRef, pte & mask, pte) != pte);
+
+ mask = 0;
+ }
+ }
+
+ private static void ThrowInvalidMemoryRegionException(string message) => throw new InvalidMemoryRegionException(message);
+ }
+}
diff --git a/src/Ryujinx.Cpu/MemoryEhMeilleure.cs b/src/Ryujinx.Cpu/MemoryEhMeilleure.cs
index f3a5b056bc..379ace9413 100644
--- a/src/Ryujinx.Cpu/MemoryEhMeilleure.cs
+++ b/src/Ryujinx.Cpu/MemoryEhMeilleure.cs
@@ -1,3 +1,4 @@
+using Ryujinx.Common;
using Ryujinx.Cpu.Signal;
using Ryujinx.Memory;
using Ryujinx.Memory.Tracking;
@@ -8,19 +9,27 @@ namespace Ryujinx.Cpu
{
public class MemoryEhMeilleure : IDisposable
{
- private delegate bool TrackingEventDelegate(ulong address, ulong size, bool write);
+ public delegate ulong TrackingEventDelegate(ulong address, ulong size, bool write);
+ private readonly MemoryTracking _tracking;
private readonly TrackingEventDelegate _trackingEvent;
+ private readonly ulong _pageSize;
+
private readonly ulong _baseAddress;
private readonly ulong _mirrorAddress;
- public MemoryEhMeilleure(MemoryBlock addressSpace, MemoryBlock addressSpaceMirror, MemoryTracking tracking)
+ public MemoryEhMeilleure(MemoryBlock addressSpace, MemoryBlock addressSpaceMirror, MemoryTracking tracking, TrackingEventDelegate trackingEvent = null)
{
_baseAddress = (ulong)addressSpace.Pointer;
+
ulong endAddress = _baseAddress + addressSpace.Size;
- _trackingEvent = tracking.VirtualMemoryEvent;
+ _tracking = tracking;
+ _trackingEvent = trackingEvent ?? VirtualMemoryEvent;
+
+ _pageSize = MemoryBlock.GetPageSize();
+
bool added = NativeSignalHandler.AddTrackedRegion((nuint)_baseAddress, (nuint)endAddress, Marshal.GetFunctionPointerForDelegate(_trackingEvent));
if (!added)
@@ -28,7 +37,7 @@ namespace Ryujinx.Cpu
throw new InvalidOperationException("Number of allowed tracked regions exceeded.");
}
- if (OperatingSystem.IsWindows())
+ if (OperatingSystem.IsWindows() && addressSpaceMirror != null)
{
// Add a tracking event with no signal handler for the mirror on Windows.
// The native handler has its own code to check for the partial overlap race when regions are protected by accident,
@@ -46,6 +55,21 @@ namespace Ryujinx.Cpu
}
}
+ private ulong VirtualMemoryEvent(ulong address, ulong size, bool write)
+ {
+ ulong pageSize = _pageSize;
+ ulong addressAligned = BitUtils.AlignDown(address, pageSize);
+ ulong endAddressAligned = BitUtils.AlignUp(address + size, pageSize);
+ ulong sizeAligned = endAddressAligned - addressAligned;
+
+ if (_tracking.VirtualMemoryEvent(addressAligned, sizeAligned, write))
+ {
+ return _baseAddress + address;
+ }
+
+ return 0;
+ }
+
public void Dispose()
{
GC.SuppressFinalize(this);
diff --git a/src/Ryujinx.Cpu/PrivateMemoryAllocator.cs b/src/Ryujinx.Cpu/PrivateMemoryAllocator.cs
index ce8e834198..8db74f1e92 100644
--- a/src/Ryujinx.Cpu/PrivateMemoryAllocator.cs
+++ b/src/Ryujinx.Cpu/PrivateMemoryAllocator.cs
@@ -143,7 +143,7 @@ namespace Ryujinx.Cpu
}
}
- public PrivateMemoryAllocator(int blockAlignment, MemoryAllocationFlags allocationFlags) : base(blockAlignment, allocationFlags)
+ public PrivateMemoryAllocator(ulong blockAlignment, MemoryAllocationFlags allocationFlags) : base(blockAlignment, allocationFlags)
{
}
@@ -180,10 +180,10 @@ namespace Ryujinx.Cpu
private readonly List _blocks;
- private readonly int _blockAlignment;
+ private readonly ulong _blockAlignment;
private readonly MemoryAllocationFlags _allocationFlags;
- public PrivateMemoryAllocatorImpl(int blockAlignment, MemoryAllocationFlags allocationFlags)
+ public PrivateMemoryAllocatorImpl(ulong blockAlignment, MemoryAllocationFlags allocationFlags)
{
_blocks = new List();
_blockAlignment = blockAlignment;
@@ -212,7 +212,7 @@ namespace Ryujinx.Cpu
}
}
- ulong blockAlignedSize = BitUtils.AlignUp(size, (ulong)_blockAlignment);
+ ulong blockAlignedSize = BitUtils.AlignUp(size, _blockAlignment);
var memory = new MemoryBlock(blockAlignedSize, _allocationFlags);
var newBlock = createBlock(memory, blockAlignedSize);
diff --git a/src/Ryujinx.Cpu/Signal/NativeSignalHandler.cs b/src/Ryujinx.Cpu/Signal/NativeSignalHandler.cs
index 5a9d92cc4f..93e6083298 100644
--- a/src/Ryujinx.Cpu/Signal/NativeSignalHandler.cs
+++ b/src/Ryujinx.Cpu/Signal/NativeSignalHandler.cs
@@ -70,7 +70,7 @@ namespace Ryujinx.Cpu.Signal
config = new SignalHandlerConfig();
}
- public static void InitializeSignalHandler(ulong pageSize, Func customSignalHandlerFactory = null)
+ public static void InitializeSignalHandler(Func customSignalHandlerFactory = null)
{
if (_initialized)
{
@@ -90,7 +90,7 @@ namespace Ryujinx.Cpu.Signal
if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS())
{
- _signalHandlerPtr = MapCode(NativeSignalHandlerGenerator.GenerateUnixSignalHandler(_handlerConfig, rangeStructSize, pageSize));
+ _signalHandlerPtr = MapCode(NativeSignalHandlerGenerator.GenerateUnixSignalHandler(_handlerConfig, rangeStructSize));
if (customSignalHandlerFactory != null)
{
@@ -107,7 +107,7 @@ namespace Ryujinx.Cpu.Signal
config.StructAddressOffset = 40; // ExceptionInformation1
config.StructWriteOffset = 32; // ExceptionInformation0
- _signalHandlerPtr = MapCode(NativeSignalHandlerGenerator.GenerateWindowsSignalHandler(_handlerConfig, rangeStructSize, pageSize));
+ _signalHandlerPtr = MapCode(NativeSignalHandlerGenerator.GenerateWindowsSignalHandler(_handlerConfig, rangeStructSize));
if (customSignalHandlerFactory != null)
{
@@ -175,5 +175,10 @@ namespace Ryujinx.Cpu.Signal
return false;
}
+
+ public static bool SupportsFaultAddressPatching()
+ {
+ return NativeSignalHandlerGenerator.SupportsFaultAddressPatchingForHost();
+ }
}
}
diff --git a/src/Ryujinx.Cpu/VirtualMemoryManagerRefCountedBase.cs b/src/Ryujinx.Cpu/VirtualMemoryManagerRefCountedBase.cs
index c2d8cfb1a0..3c7b338055 100644
--- a/src/Ryujinx.Cpu/VirtualMemoryManagerRefCountedBase.cs
+++ b/src/Ryujinx.Cpu/VirtualMemoryManagerRefCountedBase.cs
@@ -1,13 +1,10 @@
using Ryujinx.Memory;
using System.Diagnostics;
-using System.Numerics;
using System.Threading;
namespace Ryujinx.Cpu
{
- public abstract class VirtualMemoryManagerRefCountedBase : VirtualMemoryManagerBase, IRefCounted
- where TVirtual : IBinaryInteger
- where TPhysical : IBinaryInteger
+ public abstract class VirtualMemoryManagerRefCountedBase : VirtualMemoryManagerBase, IRefCounted
{
private int _referenceCount;
diff --git a/src/Ryujinx.Graphics.Device/DeviceMemoryManager.cs b/src/Ryujinx.Graphics.Device/DeviceMemoryManager.cs
index d64ed30954..fc075a2643 100644
--- a/src/Ryujinx.Graphics.Device/DeviceMemoryManager.cs
+++ b/src/Ryujinx.Graphics.Device/DeviceMemoryManager.cs
@@ -1,5 +1,7 @@
+using Ryujinx.Common.Memory;
using Ryujinx.Memory;
using System;
+using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
@@ -143,11 +145,11 @@ namespace Ryujinx.Graphics.Device
}
else
{
- Memory memory = new byte[size];
+ IMemoryOwner memoryOwner = ByteMemoryPool.Rent(size);
- GetSpan(va, size).CopyTo(memory.Span);
+ GetSpan(va, size).CopyTo(memoryOwner.Memory.Span);
- return new WritableRegion(this, va, memory, tracked: true);
+ return new WritableRegion(this, va, memoryOwner, tracked: true);
}
}
diff --git a/src/Ryujinx.Graphics.GAL/Capabilities.cs b/src/Ryujinx.Graphics.GAL/Capabilities.cs
index dc927eaba1..779ce5b5dc 100644
--- a/src/Ryujinx.Graphics.GAL/Capabilities.cs
+++ b/src/Ryujinx.Graphics.GAL/Capabilities.cs
@@ -36,6 +36,8 @@ namespace Ryujinx.Graphics.GAL
public readonly bool SupportsMismatchingViewFormat;
public readonly bool SupportsCubemapView;
public readonly bool SupportsNonConstantTextureOffset;
+ public readonly bool SupportsQuads;
+ public readonly bool SupportsSeparateSampler;
public readonly bool SupportsShaderBallot;
public readonly bool SupportsShaderBarrierDivergence;
public readonly bool SupportsShaderFloat64;
@@ -92,6 +94,8 @@ namespace Ryujinx.Graphics.GAL
bool supportsMismatchingViewFormat,
bool supportsCubemapView,
bool supportsNonConstantTextureOffset,
+ bool supportsQuads,
+ bool supportsSeparateSampler,
bool supportsShaderBallot,
bool supportsShaderBarrierDivergence,
bool supportsShaderFloat64,
@@ -144,6 +148,8 @@ namespace Ryujinx.Graphics.GAL
SupportsMismatchingViewFormat = supportsMismatchingViewFormat;
SupportsCubemapView = supportsCubemapView;
SupportsNonConstantTextureOffset = supportsNonConstantTextureOffset;
+ SupportsQuads = supportsQuads;
+ SupportsSeparateSampler = supportsSeparateSampler;
SupportsShaderBallot = supportsShaderBallot;
SupportsShaderBarrierDivergence = supportsShaderBarrierDivergence;
SupportsShaderFloat64 = supportsShaderFloat64;
diff --git a/src/Ryujinx.Graphics.GAL/IImageArray.cs b/src/Ryujinx.Graphics.GAL/IImageArray.cs
new file mode 100644
index 0000000000..30cff50b15
--- /dev/null
+++ b/src/Ryujinx.Graphics.GAL/IImageArray.cs
@@ -0,0 +1,8 @@
+namespace Ryujinx.Graphics.GAL
+{
+ public interface IImageArray
+ {
+ void SetFormats(int index, Format[] imageFormats);
+ void SetImages(int index, ITexture[] images);
+ }
+}
diff --git a/src/Ryujinx.Graphics.GAL/IPipeline.cs b/src/Ryujinx.Graphics.GAL/IPipeline.cs
index 3ba084aa5b..9efb9e3e8f 100644
--- a/src/Ryujinx.Graphics.GAL/IPipeline.cs
+++ b/src/Ryujinx.Graphics.GAL/IPipeline.cs
@@ -59,6 +59,7 @@ namespace Ryujinx.Graphics.GAL
void SetIndexBuffer(BufferRange buffer, IndexType type);
void SetImage(ShaderStage stage, int binding, ITexture texture, Format imageFormat);
+ void SetImageArray(ShaderStage stage, int binding, IImageArray array);
void SetLineParameters(float width, bool smooth);
@@ -89,6 +90,7 @@ namespace Ryujinx.Graphics.GAL
void SetStorageBuffers(ReadOnlySpan buffers);
void SetTextureAndSampler(ShaderStage stage, int binding, ITexture texture, ISampler sampler);
+ void SetTextureArray(ShaderStage stage, int binding, ITextureArray array);
void SetTransformFeedbackBuffers(ReadOnlySpan buffers);
void SetUniformBuffers(ReadOnlySpan buffers);
diff --git a/src/Ryujinx.Graphics.GAL/IRenderer.cs b/src/Ryujinx.Graphics.GAL/IRenderer.cs
index 3bf56465eb..a3466e3966 100644
--- a/src/Ryujinx.Graphics.GAL/IRenderer.cs
+++ b/src/Ryujinx.Graphics.GAL/IRenderer.cs
@@ -21,10 +21,14 @@ namespace Ryujinx.Graphics.GAL
BufferHandle CreateBuffer(nint pointer, int size);
BufferHandle CreateBufferSparse(ReadOnlySpan storageBuffers);
+ IImageArray CreateImageArray(int size, bool isBuffer);
+
IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info);
ISampler CreateSampler(SamplerCreateInfo info);
ITexture CreateTexture(TextureCreateInfo info);
+ ITextureArray CreateTextureArray(int size, bool isBuffer);
+
bool PrepareHostMapping(nint address, ulong size);
void CreateSync(ulong id, bool strict);
diff --git a/src/Ryujinx.Graphics.GAL/ITexture.cs b/src/Ryujinx.Graphics.GAL/ITexture.cs
index 5a4623a66d..2d9c6b7990 100644
--- a/src/Ryujinx.Graphics.GAL/ITexture.cs
+++ b/src/Ryujinx.Graphics.GAL/ITexture.cs
@@ -1,4 +1,4 @@
-using Ryujinx.Common.Memory;
+using System.Buffers;
namespace Ryujinx.Graphics.GAL
{
@@ -17,10 +17,34 @@ namespace Ryujinx.Graphics.GAL
PinnedSpan GetData();
PinnedSpan GetData(int layer, int level);
- void SetData(SpanOrArray data);
- void SetData(SpanOrArray data, int layer, int level);
- void SetData(SpanOrArray data, int layer, int level, Rectangle region);
+ ///
+ /// Sets the texture data. The data passed as a