fix a bunch of shit

src/GPU3D.cpp

@@ -299,6 +299,7 @@ void GPU3D::Reset() noexcept
     NumPolygons = 0;
     CurRAMBank = 0;
 
+    ShadowSent = true;
     FlushRequest = 0;
     FlushAttributes = 0;
 
@@ -358,6 +359,7 @@ void GPU3D::DoSavestate(Savestate* file) noexcept
     file->Var32(&RenderClearAttr2);
     
     file->Bool32(&RenderRasterRev);
+    file->Bool32(&ShadowSent);
 
     file->Var16(&RenderXPos);
 
@@ -463,6 +465,7 @@ void GPU3D::DoSavestate(Savestate* file) noexcept
         file->Bool32(&poly->FacingView);
         file->Bool32(&poly->Translucent);
 
+        file->Bool32(&poly->ClearStencil);
         file->Bool32(&poly->IsShadowMask);
         file->Bool32(&poly->IsShadow);
 
@@ -1225,6 +1228,18 @@ void GPU3D::SubmitPolygon() noexcept
 
     poly->IsShadowMask = ((CurPolygonAttr & 0x3F000030) == 0x00000030);
     poly->IsShadow = ((CurPolygonAttr & 0x30) == 0x30) && !poly->IsShadowMask;
+            
+    // yes we need specifically the rasterizer bit, not the gx bit
+    poly->ClearStencil = false;
+    if (NDS.GetSCFGRasterBit() && (FlushAttributes & 1) && poly->Translucent)
+    {
+        if (poly->IsShadowMask && ShadowSent)
+        {
+            ShadowSent = false;
+            poly->ClearStencil = true;
+        }
+    }
+    if (poly->IsShadow) ShadowSent = true;
 
     if (!poly->Translucent) NumOpaquePolygons++;
 
@@ -2056,6 +2071,7 @@ void GPU3D::ExecuteCommand() noexcept
 
         case 0x50: // flush
             VertexPipelineCmdDelayed4();
+            ShadowSent = true;
             FlushRequest = 1;
             FlushAttributes = entry.Param & 0x3;
             CycleCount = 325;

src/GPU3D.h

@@ -68,6 +68,7 @@ struct Polygon
     bool FacingView;
     bool Translucent;
 
+    bool ClearStencil; // used by the dsi's revised rasterizer mode
     bool IsShadowMask;
     bool IsShadow;
 
@@ -328,6 +329,8 @@ public:
     // used to fix stencil buffer's frame-to-frame persistence not working properly under extreme misuse of shadow masks/shadows
     bool ForceRerender = false;
     bool DontRerenderLoop = false;
+    
+    bool ShadowSent = false; // used to track whether the clear stencil flag should be set when the raster scfg bit is active
 
     u32 FlushRequest = 0;
     u32 FlushAttributes = 0;

src/GPU3D_Soft.cpp

@@ -125,6 +125,8 @@ void SoftRenderer::Reset(GPU& gpu)
 
     ShadowRendered[0] = false;
     ShadowRendered[1] = false;
+    ShadowRenderedi[0] = false;
+    ShadowRenderedi[1] = false;
     SetupRenderThread(gpu);
     EnableRenderThread();
 }
@@ -762,10 +764,9 @@ void SoftRenderer::RenderShadowMaskScanline(const GPU3D& gpu3d, RendererPolygon*
     bool (*fnDepthTest)(s32 dstz, s32 z, u32 attr, u32 dstattr, u8 flags);
 
     // stencil buffer is only cleared when beginning a shadow mask after a shadow polygon is rendered
-    // the "Revised" Rasterizer Circuit bugs out stencil buffer clearing
-    // TODO: under some circumstances that i am struggling to reproduce it is possible for this bit to actually *fix* bugs with misused shadow masks
-    // while this is in effect, toggling the scfg bit appears to glitch the stencil buffer for a frame?
-    if (ShadowRendered[y&0x1] && !gpu3d.RenderRasterRev)
+    // the state of whether a polygon was or wasn't rendered can persist between frames
+    // the Revised Rasterizer Circuit handles clearing the stencil buffer elsewhere
+    if (ShadowRendered[y&0x1] && (!gpu3d.RenderRasterRev || polygon->ClearStencil))
     {
         StencilCleared = true;
         memset(&StencilBuffer[256 * (y&0x1)], 0, 256);
@@ -961,7 +962,7 @@ void SoftRenderer::RenderPolygonScanline(GPU& gpu, RendererPolygon* rp, s32 y)
 
     if (polygon->IsShadow)
     {
-        ShadowRendered[y&0x1] = true;
+        if (!gpu.GPU3D.RenderRasterRev || !polygon->Translucent) ShadowRendered[y&0x1] = true;
         if (wireframe) return; // TODO: this probably still counts towards timings.
         if (!StencilCleared)
         {
@@ -1593,6 +1594,22 @@ void SoftRenderer::RenderScanline(GPU& gpu, s32 y, int npolys)
     {
         RendererPolygon* rp = &PolygonList[i];
         Polygon* polygon = rp->PolyData;
+        
+        //we actually handle clearing the stencil buffer here when the revision bit is set, this allows for a polygon to clear it on every scanline, even ones it isn't part of.
+        if (gpu.GPU3D.RenderRasterRev)
+        {
+            if (polygon->ClearStencil && polygon->Translucent && ShadowRenderedi[(y&0x1)])
+            {
+                StencilCleared = true;
+                memset(&StencilBuffer[256 * (y&0x1)], 0, 256);
+                ShadowRenderedi[(y&0x1)] = false;
+
+            }
+            else if (polygon->IsShadow && polygon->Translucent)
+            {
+                ShadowRenderedi[(y&0x1)] = true;
+            }
+        }
 
         if (y >= polygon->YTop && (y < polygon->YBottom || (y == polygon->YTop && polygon->YBottom == polygon->YTop)))
         {

src/GPU3D_Soft.h

@@ -510,6 +510,7 @@ private:
 
     u8 StencilBuffer[256*2];
     bool ShadowRendered[2];
+    bool ShadowRenderedi[2];
     bool StencilCleared;
 
     bool Enabled;