Add support for RGBA8 output format on the video image composer

Ryujinx.Graphics/VideoDecoding/FFmpeg.cs

@@ -9,6 +9,10 @@ namespace Ryujinx.Graphics.VideoDecoding
         private static AVCodec*        Codec;
         private static AVCodecContext* Context;
         private static AVFrame*        Frame;
+        private static SwsContext*     ScalerCtx;
+
+        private static int ScalerWidth;
+        private static int ScalerHeight;
 
         public static bool IsInitialized { get; private set; }
 
@@ -29,9 +33,9 @@ namespace Ryujinx.Graphics.VideoDecoding
                 Uninitialize();
             }
 
-            Codec   = ffmpeg.avcodec_find_decoder(CodecId);
+            Codec     = ffmpeg.avcodec_find_decoder(CodecId);
-            Context = ffmpeg.avcodec_alloc_context3(Codec);
+            Context   = ffmpeg.avcodec_alloc_context3(Codec);
-            Frame   = ffmpeg.av_frame_alloc();
+            Frame     = ffmpeg.av_frame_alloc();
 
             ffmpeg.avcodec_open2(Context, Codec, null);
 
@@ -82,6 +86,61 @@ namespace Ryujinx.Graphics.VideoDecoding
             };
         }
 
+        public static FFmpegFrame GetFrameRgba()
+        {
+            if (!IsInitialized)
+            {
+                throw new InvalidOperationException("Tried to use uninitialized codec!");
+            }
+
+            AVFrame ManagedFrame = Marshal.PtrToStructure<AVFrame>((IntPtr)Frame);
+
+            EnsureScalerSetup(ManagedFrame.width, ManagedFrame.height);
+
+            byte*[] Data = ManagedFrame.data.ToArray();
+
+            int[] LineSizes = ManagedFrame.linesize.ToArray();
+
+            byte[] Dst = new byte[ManagedFrame.width * ManagedFrame.height * 4];
+
+            fixed (byte* Ptr = Dst)
+            {
+                byte*[] DstData = new byte*[] { Ptr };
+
+                int[] DstLineSizes = new int[] { ManagedFrame.width * 4 };
+
+                ffmpeg.sws_scale(ScalerCtx, Data, LineSizes, 0, ManagedFrame.height, DstData, DstLineSizes);
+            }
+
+            return new FFmpegFrame()
+            {
+                Width  = ManagedFrame.width,
+                Height = ManagedFrame.height,
+
+                Data = Dst
+            };
+        }
+
+        private static void EnsureScalerSetup(int Width, int Height)
+        {
+            if (Width == 0 || Height == 0)
+            {
+                return;
+            }
+
+            if (ScalerCtx == null || ScalerWidth != Width || ScalerHeight != Height)
+            {
+                FreeScaler();
+
+                ScalerCtx = ffmpeg.sws_getContext(
+                    Width, Height, AVPixelFormat.AV_PIX_FMT_YUV420P,
+                    Width, Height, AVPixelFormat.AV_PIX_FMT_RGBA, 0, null, null, null);
+
+                ScalerWidth  = Width;
+                ScalerHeight = Height;
+            }
+        }
+
         public static void Uninitialize()
         {
             if (IsInitialized)
@@ -90,8 +149,20 @@ namespace Ryujinx.Graphics.VideoDecoding
                 ffmpeg.av_free(Frame);
                 ffmpeg.avcodec_close(Context);
 
+                FreeScaler();
+
                 IsInitialized = false;
             }
         }
+
+        private static void FreeScaler()
+        {
+            if (ScalerCtx != null)
+            {
+                ffmpeg.sws_freeContext(ScalerCtx);
+
+                ScalerCtx = null;
+            }
+        }
     }
 }

Ryujinx.Graphics/VideoDecoding/FFmpegFrame.cs

@@ -8,5 +8,7 @@ namespace Ryujinx.Graphics.VideoDecoding
         public byte* LumaPtr;
         public byte* ChromaBPtr;
         public byte* ChromaRPtr;
+
+        public byte[] Data;
     }
 }

Ryujinx.Graphics/VideoDecoding/H264Decoder.cs

@@ -4,7 +4,7 @@ namespace Ryujinx.Graphics.VideoDecoding
 {
     class H264Decoder
     {
-        private int    MaxPicOrderCntLsbMinus4;
+        private int    Log2MaxPicOrderCntLsbMinus4;
         private bool   DeltaPicOrderAlwaysZeroFlag;
         private bool   FrameMbsOnlyFlag;
         private int    PicWidthInMbs;
@@ -35,9 +35,7 @@ namespace Ryujinx.Graphics.VideoDecoding
 
         public void Decode(H264ParameterSets Params, H264Matrices Matrices, byte[] FrameData)
         {
-            int LastFrameNumber = FrameNumber;
+            Log2MaxPicOrderCntLsbMinus4           = Params.Log2MaxPicOrderCntLsbMinus4;
-
-            MaxPicOrderCntLsbMinus4               = Params.MaxPicOrderCntLsbMinus4;
             DeltaPicOrderAlwaysZeroFlag           = Params.DeltaPicOrderAlwaysZeroFlag;
             FrameMbsOnlyFlag                      = Params.FrameMbsOnlyFlag;
             PicWidthInMbs                         = Params.PicWidthInMbs;
@@ -117,7 +115,7 @@ namespace Ryujinx.Graphics.VideoDecoding
 
                 if (PicOrderCntType == 0)
                 {
-                    Writer.WriteUe(MaxPicOrderCntLsbMinus4);
+                    Writer.WriteUe(Log2MaxPicOrderCntLsbMinus4);
                 }
                 else if (PicOrderCntType == 1)
                 {
@@ -130,7 +128,7 @@ namespace Ryujinx.Graphics.VideoDecoding
 
                 int PicHeightInMbs = PicHeightInMapUnits / (FrameMbsOnlyFlag ? 1 : 2);
 
-                Writer.WriteUe(4);
+                Writer.WriteUe(16);
                 Writer.WriteBit(false);
                 Writer.WriteUe(PicWidthInMbs - 1);
                 Writer.WriteUe(PicHeightInMbs - 1);

Ryujinx.Graphics/VideoDecoding/H264ParameterSets.cs

@@ -5,7 +5,7 @@ namespace Ryujinx.Graphics.VideoDecoding
     [StructLayout(LayoutKind.Sequential, Pack = 4)]
     struct H264ParameterSets
     {
-        public int  MaxPicOrderCntLsbMinus4;
+        public int  Log2MaxPicOrderCntLsbMinus4;
         public bool DeltaPicOrderAlwaysZeroFlag;
         public bool FrameMbsOnlyFlag;
         public int  PicWidthInMbs;

Ryujinx.Graphics/VideoDecoding/VideoDecoder.cs

@@ -1,4 +1,7 @@
+using Ryujinx.Graphics.Gal;
 using Ryujinx.Graphics.Memory;
+using Ryujinx.Graphics.Texture;
+using Ryujinx.Graphics.VideoImageComposition;
 using System;
 using System.Runtime.InteropServices;
 
@@ -195,30 +198,81 @@ namespace Ryujinx.Graphics.VideoDecoding
             return (long)(uint)Arguments[0] << 8;
         }
 
-        internal void CopyPlanes(NvGpuVmm Vmm, long LumaPlaneAddress, long ChromaPlaneAddress)
+        internal void CopyPlanes(NvGpuVmm Vmm, SurfaceOutputConfig OutputConfig)
+        {
+            switch (OutputConfig.PixelFormat)
+            {
+                case SurfacePixelFormat.RGBA8:   CopyPlanesRgba8  (Vmm, OutputConfig); break;
+                case SurfacePixelFormat.YUV420P: CopyPlanesYuv420p(Vmm, OutputConfig); break;
+
+                default: ThrowUnimplementedPixelFormat(OutputConfig.PixelFormat); break;
+            }
+        }
+
+        private void CopyPlanesRgba8(NvGpuVmm Vmm, SurfaceOutputConfig OutputConfig)
+        {
+            FFmpegFrame Frame = FFmpegWrapper.GetFrameRgba();
+
+            if ((Frame.Width | Frame.Height) == 0)
+            {
+                return;
+            }
+
+            GalImage Image = new GalImage(
+                OutputConfig.SurfaceWidth,
+                OutputConfig.SurfaceHeight, 1,
+                OutputConfig.GobBlockHeight,
+                GalMemoryLayout.BlockLinear,
+                GalImageFormat.RGBA8 | GalImageFormat.Unorm);
+
+            ImageUtils.WriteTexture(Vmm, Image, Vmm.GetPhysicalAddress(OutputConfig.SurfaceLumaAddress), Frame.Data);
+        }
+
+        private void CopyPlanesYuv420p(NvGpuVmm Vmm, SurfaceOutputConfig OutputConfig)
         {
             FFmpegFrame Frame = FFmpegWrapper.GetFrame();
 
-            int HalfWidth  = Frame.Width  / 2;
+            if ((Frame.Width | Frame.Height) == 0)
-            int HalfHeight = Frame.Height / 2;
+            {
+                return;
+            }
 
-            int AlignedWidth = (Frame.Width + 0xff) & ~0xff;
+            if ((uint)OutputConfig.SurfaceWidth  > (uint)Frame.Width ||
+                (uint)OutputConfig.SurfaceHeight > (uint)Frame.Height)
+            {
+                string Msg = "Surface and frame resolution mismatch!";
 
-            byte* LumaPtr   = (byte*)Vmm.GetHostAddress(LumaPlaneAddress,   AlignedWidth * Frame.Height);
+                Msg += $" Surface resolution is {OutputConfig.SurfaceWidth}x{OutputConfig.SurfaceHeight},";
-            byte* ChromaPtr = (byte*)Vmm.GetHostAddress(ChromaPlaneAddress, AlignedWidth * HalfHeight);
+
+                Msg += $" Frame resolution is {Frame.Width}x{Frame.Height}.";
+
+                throw new InvalidOperationException(Msg);
+            }
+
+            int HalfSrcWidth = Frame.Width / 2;
+
+            int HalfWidth  = OutputConfig.SurfaceWidth  / 2;
+            int HalfHeight = OutputConfig.SurfaceHeight / 2;
+
+            int AlignedWidth = (OutputConfig.SurfaceWidth + 0xff) & ~0xff;
+
+            byte* LumaPtr   = (byte*)Vmm.GetHostAddress(OutputConfig.SurfaceLumaAddress,    AlignedWidth * Frame.Height);
+            byte* ChromaPtr = (byte*)Vmm.GetHostAddress(OutputConfig.SurfaceChromaUAddress, AlignedWidth * HalfHeight);
 
             for (int Y = 0; Y < Frame.Height; Y++)
             {
                 int Src = Y * Frame.Width;
                 int Dst = Y * AlignedWidth;
 
-                Buffer.MemoryCopy(Frame.LumaPtr + Src, LumaPtr + Dst, Frame.Width, Frame.Width);
+                int Size = OutputConfig.SurfaceWidth;
+
+                Buffer.MemoryCopy(Frame.LumaPtr + Src, LumaPtr + Dst, Size, Size);
             }
 
             //Copy chroma data from both channels with interleaving.
             for (int Y = 0; Y < HalfHeight; Y++)
             {
-                int Src = Y * HalfWidth;
+                int Src = Y * HalfSrcWidth;
                 int Dst = Y * AlignedWidth;
 
                 for (int X = 0; X < HalfWidth; X++)
@@ -233,5 +287,10 @@ namespace Ryujinx.Graphics.VideoDecoding
         {
             throw new NotImplementedException("Codec \"" + CurrentVideoCodec + "\" is not supported!");
         }
+
+        private void ThrowUnimplementedPixelFormat(SurfacePixelFormat PixelFormat)
+        {
+            throw new NotImplementedException("Pixel format \"" + PixelFormat + "\" is not supported!");
+        }
     }
 }

Ryujinx.Graphics/VideoImageComposition/StructUnpacker.cs

@@ -0,0 +1,69 @@
+using Ryujinx.Graphics.Memory;
+using System;
+
+namespace Ryujinx.Graphics.VideoImageComposition
+{
+    class StructUnpacker
+    {
+        private NvGpuVmm Vmm;
+
+        private long Position;
+
+        private ulong Buffer;
+        private int   BuffPos;
+
+        public StructUnpacker(NvGpuVmm Vmm, long Position)
+        {
+            this.Vmm      = Vmm;
+            this.Position = Position;
+
+            BuffPos = 64;
+        }
+
+        public int Read(int Bits)
+        {
+            if ((uint)Bits > 32)
+            {
+                throw new ArgumentOutOfRangeException(nameof(Bits));
+            }
+
+            int Value = 0;
+
+            while (Bits > 0)
+            {
+                RefillBufferIfNeeded();
+
+                int ReadBits = Bits;
+
+                int MaxReadBits = 64 - BuffPos;
+
+                if (ReadBits > MaxReadBits)
+                {
+                    ReadBits = MaxReadBits;
+                }
+
+                Value <<= ReadBits;
+
+                Value |= (int)(Buffer >> BuffPos) & (int)(0xffffffff >> (32 - ReadBits));
+
+                BuffPos += ReadBits;
+
+                Bits -= ReadBits;
+            }
+
+            return Value;
+        }
+
+        private void RefillBufferIfNeeded()
+        {
+            if (BuffPos >= 64)
+            {
+                Buffer = Vmm.ReadUInt64(Position);
+
+                Position += 8;
+
+                BuffPos = 0;
+            }
+        }
+    }
+}

Ryujinx.Graphics/VideoImageComposition/SurfaceOutputConfig.cs

@@ -0,0 +1,33 @@
+namespace Ryujinx.Graphics.VideoImageComposition
+{
+    struct SurfaceOutputConfig
+    {
+        public SurfacePixelFormat PixelFormat;
+
+        public int SurfaceWidth;
+        public int SurfaceHeight;
+        public int GobBlockHeight;
+
+        public long SurfaceLumaAddress;
+        public long SurfaceChromaUAddress;
+        public long SurfaceChromaVAddress;
+
+        public SurfaceOutputConfig(
+            SurfacePixelFormat PixelFormat,
+            int                SurfaceWidth,
+            int                SurfaceHeight,
+            int                GobBlockHeight,
+            long               OutputSurfaceLumaAddress,
+            long               OutputSurfaceChromaUAddress,
+            long               OutputSurfaceChromaVAddress)
+        {
+            this.PixelFormat           = PixelFormat;
+            this.SurfaceWidth          = SurfaceWidth;
+            this.SurfaceHeight         = SurfaceHeight;
+            this.GobBlockHeight        = GobBlockHeight;
+            this.SurfaceLumaAddress    = OutputSurfaceLumaAddress;
+            this.SurfaceChromaUAddress = OutputSurfaceChromaUAddress;
+            this.SurfaceChromaVAddress = OutputSurfaceChromaVAddress;
+        }
+    }
+}

Ryujinx.Graphics/VideoImageComposition/SurfacePixelFormat.cs

@@ -0,0 +1,8 @@
+namespace Ryujinx.Graphics.VideoImageComposition
+{
+    enum SurfacePixelFormat
+    {
+        RGBA8   = 0x1f,
+        YUV420P = 0x44
+    }
+}

Ryujinx.Graphics/VideoImageComposition/VideoImageComposer.cs

@@ -6,8 +6,10 @@ namespace Ryujinx.Graphics.VideoImageComposition
     {
         private NvGpu Gpu;
 
-        private long LumaPlaneAddress;
+        private long ConfigStructAddress;
-        private long ChromaPlaneAddress;
+        private long OutputSurfaceLumaAddress;
+        private long OutputSurfaceChromaUAddress;
+        private long OutputSurfaceChromaVAddress;
 
         public VideoImageComposer(NvGpu Gpu)
         {
@@ -20,25 +22,81 @@ namespace Ryujinx.Graphics.VideoImageComposition
 
             switch (Method)
             {
-                case VideoImageComposerMeth.Execute:                Execute               (Vmm, Arguments); break;
+                case VideoImageComposerMeth.Execute:
-                case VideoImageComposerMeth.SetVDecLumaPlaneAddr:   SetVDecLumaPlaneAddr  (Vmm, Arguments); break;
+                    Execute(Vmm, Arguments);
-                case VideoImageComposerMeth.SetVDecChromaPlaneAddr: SetVDecChromaPlaneAddr(Vmm, Arguments); break;
+                    break;
+
+                case VideoImageComposerMeth.SetConfigStructOffset:
+                    SetConfigStructOffset(Vmm, Arguments);
+                    break;
+
+                case VideoImageComposerMeth.SetOutputSurfaceLumaOffset:
+                    SetOutputSurfaceLumaOffset(Vmm, Arguments);
+                    break;
+
+                case VideoImageComposerMeth.SetOutputSurfaceChromaUOffset:
+                    SetOutputSurfaceChromaUOffset(Vmm, Arguments);
+                    break;
+
+                case VideoImageComposerMeth.SetOutputSurfaceChromaVOffset:
+                    SetOutputSurfaceChromaVOffset(Vmm, Arguments);
+                    break;
             }
         }
 
         private void Execute(NvGpuVmm Vmm, int[] Arguments)
         {
-            Gpu.VideoDecoder.CopyPlanes(Vmm, LumaPlaneAddress, ChromaPlaneAddress);
+            StructUnpacker Unpacker = new StructUnpacker(Vmm, ConfigStructAddress + 0x20);
+
+            SurfacePixelFormat PixelFormat = (SurfacePixelFormat)Unpacker.Read(7);
+
+            int ChromaLocHoriz = Unpacker.Read(2);
+            int ChromaLocVert  = Unpacker.Read(2);
+
+            int BlockLinearKind       = Unpacker.Read(4);
+            int BlockLinearHeightLog2 = Unpacker.Read(4);
+
+            int Reserved0 = Unpacker.Read(3);
+            int Reserved1 = Unpacker.Read(10);
+
+            int SurfaceWidthMinus1  = Unpacker.Read(14);
+            int SurfaceHeightMinus1 = Unpacker.Read(14);
+
+            int GobBlockHeight = 1 << BlockLinearHeightLog2;
+
+            int SurfaceWidth  = SurfaceWidthMinus1  + 1;
+            int SurfaceHeight = SurfaceHeightMinus1 + 1;
+
+            SurfaceOutputConfig OutputConfig = new SurfaceOutputConfig(
+                PixelFormat,
+                SurfaceWidth,
+                SurfaceHeight,
+                GobBlockHeight,
+                OutputSurfaceLumaAddress,
+                OutputSurfaceChromaUAddress,
+                OutputSurfaceChromaVAddress);
+
+            Gpu.VideoDecoder.CopyPlanes(Vmm, OutputConfig);
         }
 
-        private void SetVDecLumaPlaneAddr(NvGpuVmm Vmm, int[] Arguments)
+        private void SetConfigStructOffset(NvGpuVmm Vmm, int[] Arguments)
         {
-            LumaPlaneAddress = GetAddress(Arguments);
+            ConfigStructAddress = GetAddress(Arguments);
         }
 
-        private void SetVDecChromaPlaneAddr(NvGpuVmm Vmm, int[] Arguments)
+        private void SetOutputSurfaceLumaOffset(NvGpuVmm Vmm, int[] Arguments)
         {
-            ChromaPlaneAddress = GetAddress(Arguments);
+            OutputSurfaceLumaAddress = GetAddress(Arguments);
+        }
+
+        private void SetOutputSurfaceChromaUOffset(NvGpuVmm Vmm, int[] Arguments)
+        {
+            OutputSurfaceChromaUAddress = GetAddress(Arguments);
+        }
+
+        private void SetOutputSurfaceChromaVOffset(NvGpuVmm Vmm, int[] Arguments)
+        {
+            OutputSurfaceChromaVAddress = GetAddress(Arguments);
         }
 
         private static long GetAddress(int[] Arguments)

Ryujinx.Graphics/VideoImageComposition/VideoImageComposerMeth.cs

@@ -2,8 +2,11 @@ namespace Ryujinx.Graphics.VideoImageComposition
 {
     enum VideoImageComposerMeth
     {
-        Execute                = 0xc0,
+        Execute                       = 0xc0,
-        SetVDecLumaPlaneAddr   = 0x1c8,
+        SetControlParams              = 0x1c1,
-        SetVDecChromaPlaneAddr = 0x1c9
+        SetConfigStructOffset         = 0x1c2,
+        SetOutputSurfaceLumaOffset    = 0x1c8,
+        SetOutputSurfaceChromaUOffset = 0x1c9,
+        SetOutputSurfaceChromaVOffset = 0x1ca
     }
 }