TAA Phase 2: motion-vector reprojection (self-contained Lucas-Kanade)

The history is now read at the reprojected position (prevPos = loc + mv) instead
of the fixed pixel, so it tracks moving geometry and the Phase 1 ghosting
collapses. TAA reconstructs its own motion field (the DLSS Lucas-Kanade pass +
3x3 median filter, reused) so it works with DLSS off. Off-screen reprojection
(disocclusion) falls back to the current frame. MV sign calibrated in game to +1
(default; RYUJINX_TAA_MV_SIGN=-1 flips). Residual ghosting from imperfect
reconstructed vectors is cleaned by the Phase 3 neighborhood clamp.

src/Ryujinx.Graphics.Vulkan/Effects/Shaders/Temporal.glsl

@@ -1,14 +1,16 @@
 // Temporal Anti-Aliasing (clean-room).
 //
-// Phase 1: fixed-weight temporal accumulation with ping-pong history. No motion vectors yet, so the
+// Phase 2: motion-vector reprojection. The history is no longer read at the fixed pixel but at the
-// history is read at the SAME pixel -- static and slow-moving content converges to a rock-steady image
+// position the pixel's content occupied last frame, following our reconstructed motion field:
-// (cascades/scenery freeze like marble), while fast camera motion ghosts. That ghosting is expected and
-// goes away in Phase 2, which reprojects the history with our reconstructed motion vectors.
 //
-//   result = mix(current, history, blend)   // blend = history weight (e.g. 0.90)
+//   prevPos = loc + motionSign * mv        // motionSign = +1 (calibrated): follow the field to the history
+//   history = bilinear sample of HistoryRead at prevPos
+//   result  = mix(current, history, blend)
 //
-// The result is written both to the present output (input format) and to the write-side history target
+// This makes the accumulated history track moving geometry, so the massive ghosting from Phase 1
-// (rgba16f), which becomes next frame's read-side history.
+// collapses while the camera turns. Pixels whose previous position falls off-screen (disocclusion)
+// fall back to the current frame so freshly revealed areas do not smear. Neighborhood clamping for
+// the remaining ghosting/fireflies comes in Phase 3.
 
 #version 430 core
 
@@ -16,14 +18,17 @@ layout (local_size_x = 16, local_size_y = 16) in;
 
 layout (rgba8,   binding = 0, set = 3) uniform image2D imgOutput;
 layout (rgba16f, binding = 1, set = 3) uniform image2D imgHistoryWrite;
-layout (binding = 1, set = 2) uniform sampler2D Source;
+layout (binding = 1, set = 2) uniform sampler2D Source;      // current color
-layout (binding = 3, set = 2) uniform sampler2D HistoryRead;
+layout (binding = 3, set = 2) uniform sampler2D HistoryRead; // previous accumulated result
+layout (binding = 5, set = 2) uniform sampler2D Motion;      // rg16f motion field, render-res pixels
 
 layout (binding = 2) uniform params {
     float width;
     float height;
-    float blend;      // history weight in [0,1]; current weight is (1 - blend)
+    float blend;       // history weight in [0,1]; current weight is (1 - blend)
-    float hasHistory; // >0.5 once the read-side history holds a valid previous frame
+    float hasHistory;  // >0.5 once the read-side history holds a valid previous frame
+    float motionSign;  // +1 (calibrated) maps a pixel back to its previous position (prevPos = loc + mv)
+    float hasMotion;   // >0.5 once a previous frame exists to estimate motion from
 };
 
 void main()
@@ -36,10 +41,17 @@ void main()
     }
 
     vec4 current = texelFetch(Source, loc, 0);
-    vec4 history = texelFetch(HistoryRead, loc, 0);
 
-    // First frame after (re)allocation has no valid history: pass current through and seed the history.
+    // This pixel's motion (in render-resolution pixels); reproject the history read by it.
-    float w = hasHistory > 0.5 ? blend : 0.0;
+    vec2 mv = hasMotion > 0.5 ? texelFetch(Motion, loc, 0).xy : vec2(0.0);
+    vec2 prevPos = vec2(loc) + 0.5 + motionSign * mv;
+    vec2 uv = prevPos / vec2(width, height);
+
+    bool onScreen = all(greaterThanEqual(uv, vec2(0.0))) && all(lessThanEqual(uv, vec2(1.0)));
+    vec4 history = texture(HistoryRead, uv); // bilinear: prevPos is sub-pixel
+
+    // No history yet, or reprojected off-screen (disocclusion): take the current frame, no ghost.
+    float w = (hasHistory > 0.5 && onScreen) ? blend : 0.0;
     vec4 result = mix(current, history, w);
 
     imageStore(imgOutput, loc, result);

src/Ryujinx.Graphics.Vulkan/Effects/TemporalFilter.cs

@@ -14,14 +14,16 @@ namespace Ryujinx.Graphics.Vulkan.Effects
     /// <summary>
     /// Clean-room native Temporal Anti-Aliasing.
     ///
-    /// Phase 1: fixed-weight temporal accumulation. Each frame blends the current frame with the previous
+    /// Phase 2: fixed-weight temporal accumulation with motion-vector reprojection. Each frame it
-    /// accumulated frame (<c>result = mix(current, history, blend)</c>) using two ping-pong rgba16f history
+    /// reconstructs a motion field from the current and previous color (the same Lucas-Kanade pass + 3x3
-    /// targets (one read, one written, swapped every frame). With no motion vectors yet the history is read
+    /// median filter used by the DLSS path -- self-contained, so TAA works with DLSS off), then blends the
-    /// at the same pixel, so static/slow content converges to a rock-steady image while fast motion ghosts
+    /// current frame with the history sampled at the reprojected position (<c>prevPos = loc - mv</c>). Static
-    /// (resolved in Phase 2 by reprojecting the history with our reconstructed motion vectors).
+    /// content stays rock-steady and the Phase 1 ghosting collapses while the camera moves. Neighborhood
+    /// clamping for the remaining ghosting/fireflies is Phase 3.
     ///
     /// Gated entirely on RYUJINX_TAA=1: when unset the present path never constructs or runs this, so the
-    /// default render is byte-identical to before. RYUJINX_TAA_BLEND overrides the history weight (default 0.90).
+    /// default render is byte-identical. RYUJINX_TAA_BLEND overrides the history weight (default 0.90);
+    /// RYUJINX_TAA_MV_SIGN flips the reprojection direction (default +1) for sign calibration.
     /// </summary>
     internal class TemporalFilter : IPostProcessingEffect
     {
@@ -29,13 +31,20 @@ namespace Ryujinx.Graphics.Vulkan.Effects
         public static readonly bool IsEnabled =
             Environment.GetEnvironmentVariable("RYUJINX_TAA") is "1" or "true" or "TRUE" or "True";
 
-        // History weight (fraction of the accumulated history kept each frame). Tunable for bring-up.
+        private const float MaxMotion = 32f;  // motion-vector clamp, in render-resolution pixels
+        private const int CounterCount = 9;   // uints in the motion-pass SSBO (unused by TAA, required by the shader)
+
         private static readonly float HistoryBlend = ParseBlend();
+        private static readonly float MotionSign = ParseSign();
 
         private readonly VulkanRenderer _renderer;
-        private readonly PipelineHelperShader _pipeline;
+        private readonly PipelineHelperShader _pipeline;       // accumulation/blend pass
+        private readonly PipelineHelperShader _motionPipeline; // motion estimate + median filter passes
         private ISampler _sampler;
         private ShaderCollection _program;
+        private ShaderCollection _motionProgram;
+        private ShaderCollection _motionFilterProgram;
+        private BufferHandle _sceneChangeBuffer;
 
         // Output keeps the input format so the downstream blit/scaling filter sees an identical texture.
         private TextureView _output;
@@ -46,12 +55,19 @@ namespace Ryujinx.Graphics.Vulkan.Effects
         private bool _readFromHistory0;
         private bool _hasHistory;
 
+        // Optical-flow inputs/outputs: previous color (input format), raw and median-filtered motion (rg16f).
+        private TextureView _prevColor;
+        private TextureView _motion;
+        private TextureView _motionFiltered;
+        private bool _hasPrev;
+
         private bool _activeLogged;
 
         public TemporalFilter(VulkanRenderer renderer, Device device)
         {
             _renderer = renderer;
             _pipeline = new PipelineHelperShader(renderer, device);
+            _motionPipeline = new PipelineHelperShader(renderer, device);
 
             Initialize();
         }
@@ -69,24 +85,62 @@ namespace Ryujinx.Graphics.Vulkan.Effects
             return 0.90f;
         }
 
+        private static float ParseSign()
+        {
+            // Default +1 (prevPos = loc + mv): calibrated in game. The Lucas-Kanade field is oriented so that
+            // following +mv lands the read on the history; -1 doubled the image. RYUJINX_TAA_MV_SIGN=-1 flips back.
+            return Environment.GetEnvironmentVariable("RYUJINX_TAA_MV_SIGN") is "-1" ? -1f : 1f;
+        }
+
         private void Initialize()
         {
             _pipeline.Initialize();
-
+            _motionPipeline.Initialize();
-            byte[] shader = EmbeddedResources.Read("Ryujinx.Graphics.Vulkan/Effects/Shaders/Temporal.spv");
-
-            ResourceLayout layout = new ResourceLayoutBuilder()
-                .Add(ResourceStages.Compute, ResourceType.UniformBuffer, 2)
-                .Add(ResourceStages.Compute, ResourceType.TextureAndSampler, 1) // current color
-                .Add(ResourceStages.Compute, ResourceType.TextureAndSampler, 3) // history (read)
-                .Add(ResourceStages.Compute, ResourceType.Image, 0, true)        // present output
-                .Add(ResourceStages.Compute, ResourceType.Image, 1, true).Build(); // history (write)
 
             _sampler = _renderer.CreateSampler(SamplerCreateInfo.Create(MinFilter.Linear, MagFilter.Linear));
 
+            // Accumulation/blend pass: current color (b1), history read (b3), motion (b5), params (b2),
+            // present output image (b0/set3), history write image (b1/set3).
+            byte[] blendShader = EmbeddedResources.Read("Ryujinx.Graphics.Vulkan/Effects/Shaders/Temporal.spv");
+            ResourceLayout blendLayout = new ResourceLayoutBuilder()
+                .Add(ResourceStages.Compute, ResourceType.UniformBuffer, 2)
+                .Add(ResourceStages.Compute, ResourceType.TextureAndSampler, 1)
+                .Add(ResourceStages.Compute, ResourceType.TextureAndSampler, 3)
+                .Add(ResourceStages.Compute, ResourceType.TextureAndSampler, 5)
+                .Add(ResourceStages.Compute, ResourceType.Image, 0, true)
+                .Add(ResourceStages.Compute, ResourceType.Image, 1, true).Build();
+
             _program = _renderer.CreateProgramWithMinimalLayout([
-                new ShaderSource(shader, ShaderStage.Compute, TargetLanguage.Spirv)
+                new ShaderSource(blendShader, ShaderStage.Compute, TargetLanguage.Spirv)
-            ], layout);
+            ], blendLayout);
+
+            // Motion estimate (Lucas-Kanade): current (b1), previous (b3), params (b2), stats SSBO (b0/set1),
+            // motion image (b0/set3). Reused verbatim from the DLSS path.
+            byte[] motionShader = EmbeddedResources.Read("Ryujinx.Graphics.Vulkan/Effects/Shaders/MotionVectors.spv");
+            ResourceLayout motionLayout = new ResourceLayoutBuilder()
+                .Add(ResourceStages.Compute, ResourceType.UniformBuffer, 2)
+                .Add(ResourceStages.Compute, ResourceType.StorageBuffer, 0)
+                .Add(ResourceStages.Compute, ResourceType.TextureAndSampler, 1)
+                .Add(ResourceStages.Compute, ResourceType.TextureAndSampler, 3)
+                .Add(ResourceStages.Compute, ResourceType.Image, 0, true).Build();
+
+            _motionProgram = _renderer.CreateProgramWithMinimalLayout([
+                new ShaderSource(motionShader, ShaderStage.Compute, TargetLanguage.Spirv)
+            ], motionLayout);
+
+            // 3x3 median outlier filter: params (b2), stats SSBO (b0/set1), filtered out (b0/set3), raw in (b1/set3).
+            byte[] filterShader = EmbeddedResources.Read("Ryujinx.Graphics.Vulkan/Effects/Shaders/MotionFilter.spv");
+            ResourceLayout filterLayout = new ResourceLayoutBuilder()
+                .Add(ResourceStages.Compute, ResourceType.UniformBuffer, 2)
+                .Add(ResourceStages.Compute, ResourceType.StorageBuffer, 0)
+                .Add(ResourceStages.Compute, ResourceType.Image, 0, true)
+                .Add(ResourceStages.Compute, ResourceType.Image, 1, true).Build();
+
+            _motionFilterProgram = _renderer.CreateProgramWithMinimalLayout([
+                new ShaderSource(filterShader, ShaderStage.Compute, TargetLanguage.Spirv)
+            ], filterLayout);
+
+            _sceneChangeBuffer = _renderer.BufferManager.CreateWithHandle(_renderer, CounterCount * sizeof(uint));
         }
 
         private static TextureCreateInfo MakeInfo(TextureCreateInfo b, int w, int h, Format format, int bytesPerPixel)
@@ -119,14 +173,21 @@ namespace Ryujinx.Graphics.Vulkan.Effects
             _output?.Dispose();
             _history0?.Dispose();
             _history1?.Dispose();
+            _prevColor?.Dispose();
+            _motion?.Dispose();
+            _motionFiltered?.Dispose();
 
             _output = _renderer.CreateTexture(view.Info) as TextureView;
             _history0 = _renderer.CreateTexture(MakeInfo(view.Info, view.Width, view.Height, Format.R16G16B16A16Float, 8)) as TextureView;
             _history1 = _renderer.CreateTexture(MakeInfo(view.Info, view.Width, view.Height, Format.R16G16B16A16Float, 8)) as TextureView;
+            _prevColor = _renderer.CreateTexture(MakeInfo(view.Info, view.Width, view.Height, view.Info.Format, view.Info.BytesPerPixel)) as TextureView;
+            _motion = _renderer.CreateTexture(MakeInfo(view.Info, view.Width, view.Height, Format.R16G16Float, 4)) as TextureView;
+            _motionFiltered = _renderer.CreateTexture(MakeInfo(view.Info, view.Width, view.Height, Format.R16G16Float, 4)) as TextureView;
 
-            // Fresh history: the first frame has nothing to blend against.
+            // Fresh resources: nothing to blend or estimate motion against yet.
             _readFromHistory0 = true;
             _hasHistory = false;
+            _hasPrev = false;
         }
 
         public TextureView Run(TextureView view, CommandBufferScoped cbs, int width, int height)
@@ -136,7 +197,15 @@ namespace Ryujinx.Graphics.Vulkan.Effects
             if (!_activeLogged)
             {
                 _activeLogged = true;
-                Logger.Info?.Print(LogClass.Gpu, $"TAA: active (accumulation, history blend {HistoryBlend:0.00}).");
+                Logger.Info?.Print(LogClass.Gpu,
+                    $"TAA: active (reprojection, history blend {HistoryBlend:0.00}, mv sign {MotionSign:+0;-0}).");
+            }
+
+            // Reconstruct this frame's motion field from (current, previous), then median-filter it.
+            if (_hasPrev)
+            {
+                RunMotionPass(view, cbs);
+                RunMotionFilterPass(view, cbs);
             }
 
             // Ping-pong: read last frame's result, write this frame's result to the other target.
@@ -147,8 +216,17 @@ namespace Ryujinx.Graphics.Vulkan.Effects
             _pipeline.SetProgram(_program);
             _pipeline.SetTextureAndSampler(ShaderStage.Compute, 1, view, _sampler);
             _pipeline.SetTextureAndSampler(ShaderStage.Compute, 3, historyRead, _sampler);
+            _pipeline.SetTextureAndSampler(ShaderStage.Compute, 5, _motionFiltered, _sampler);
 
-            ReadOnlySpan<float> paramsBuffer = [view.Width, view.Height, HistoryBlend, _hasHistory ? 1f : 0f];
+            ReadOnlySpan<float> paramsBuffer =
+            [
+                view.Width,
+                view.Height,
+                HistoryBlend,
+                _hasHistory ? 1f : 0f,
+                MotionSign,
+                _hasPrev ? 1f : 0f,
+            ];
             int rangeSize = paramsBuffer.Length * sizeof(float);
             using ScopedTemporaryBuffer buffer = _renderer.BufferManager.ReserveOrCreate(_renderer, cbs, rangeSize);
             buffer.Holder.SetDataUnchecked(buffer.Offset, paramsBuffer);
@@ -164,21 +242,100 @@ namespace Ryujinx.Graphics.Vulkan.Effects
 
             _pipeline.Finish();
 
-            // Next frame reads the target we just wrote.
+            // Keep this frame's color as the "previous" frame for next time's motion estimate.
+            CopyToPrev(view, cbs);
+
+            // Next frame reads the target we just wrote and has both a history and a previous frame.
             _readFromHistory0 = !_readFromHistory0;
             _hasHistory = true;
+            _hasPrev = true;
 
             return _output;
         }
 
+        private void RunMotionPass(TextureView input, CommandBufferScoped cbs)
+        {
+            _motionPipeline.SetCommandBuffer(cbs);
+            _motionPipeline.SetProgram(_motionProgram);
+            _motionPipeline.SetTextureAndSampler(ShaderStage.Compute, 1, input, _sampler);
+            _motionPipeline.SetTextureAndSampler(ShaderStage.Compute, 3, _prevColor, _sampler);
+
+            // width/height/maxMotion/metrics + dejitterX/Y + 2 pad. No clip-space jitter on the TAA input
+            // (that is a DLSS-only feature), so de-jitter is zero; metrics off.
+            ReadOnlySpan<float> p = [input.Width, input.Height, MaxMotion, 0f, 0f, 0f, 0f, 0f];
+            using ScopedTemporaryBuffer buffer = _renderer.BufferManager.ReserveOrCreate(_renderer, cbs, p.Length * sizeof(float));
+            buffer.Holder.SetDataUnchecked(buffer.Offset, p);
+
+            _motionPipeline.SetUniformBuffers([new BufferAssignment(2, buffer.Range)]);
+            _motionPipeline.SetStorageBuffers([new BufferAssignment(0, new BufferRange(_sceneChangeBuffer, 0, CounterCount * sizeof(uint), true))]);
+            _motionPipeline.SetImage(0, _motion.GetImageView());
+
+            _motionPipeline.DispatchCompute(BitUtils.DivRoundUp(input.Width, 16), BitUtils.DivRoundUp(input.Height, 16), 1);
+            _motionPipeline.ComputeBarrier();
+            _motionPipeline.Finish();
+        }
+
+        private void RunMotionFilterPass(TextureView input, CommandBufferScoped cbs)
+        {
+            _motionPipeline.SetCommandBuffer(cbs);
+            _motionPipeline.SetProgram(_motionFilterProgram);
+
+            ReadOnlySpan<float> p = [input.Width, input.Height, MaxMotion, 0f, 0f, 0f, 0f, 0f];
+            using ScopedTemporaryBuffer buffer = _renderer.BufferManager.ReserveOrCreate(_renderer, cbs, p.Length * sizeof(float));
+            buffer.Holder.SetDataUnchecked(buffer.Offset, p);
+
+            _motionPipeline.SetUniformBuffers([new BufferAssignment(2, buffer.Range)]);
+            _motionPipeline.SetStorageBuffers([new BufferAssignment(0, new BufferRange(_sceneChangeBuffer, 0, CounterCount * sizeof(uint), true))]);
+            _motionPipeline.SetImage(0, _motionFiltered.GetImageView());
+            _motionPipeline.SetImage(1, _motion.GetImageView());
+
+            _motionPipeline.DispatchCompute(BitUtils.DivRoundUp(input.Width, 16), BitUtils.DivRoundUp(input.Height, 16), 1);
+            _motionPipeline.ComputeBarrier();
+            _motionPipeline.Finish();
+        }
+
+        private void CopyToPrev(TextureView input, CommandBufferScoped cbs)
+        {
+            ImageSubresourceLayers layers = new()
+            {
+                AspectMask = ImageAspectFlags.ColorBit,
+                MipLevel = 0,
+                BaseArrayLayer = 0,
+                LayerCount = 1,
+            };
+
+            ImageCopy region = new()
+            {
+                SrcSubresource = layers,
+                DstSubresource = layers,
+                Extent = new Extent3D((uint)input.Width, (uint)input.Height, 1),
+            };
+
+            _renderer.Api.CmdCopyImage(
+                cbs.CommandBuffer,
+                input.GetImage().Get(cbs).Value,
+                ImageLayout.General,
+                _prevColor.GetImage().Get(cbs).Value,
+                ImageLayout.General,
+                1,
+                in region);
+        }
+
         public void Dispose()
         {
             _pipeline.Dispose();
+            _motionPipeline.Dispose();
             _program.Dispose();
+            _motionProgram.Dispose();
+            _motionFilterProgram.Dispose();
             _sampler.Dispose();
             _output?.Dispose();
             _history0?.Dispose();
             _history1?.Dispose();
+            _prevColor?.Dispose();
+            _motion?.Dispose();
+            _motionFiltered?.Dispose();
+            _renderer.BufferManager.Delete(_sceneChangeBuffer);
         }
     }
 }