rasterizer: Fixed a depth testing bug.

src/video_core/pica.h

@@ -452,9 +452,7 @@ struct Regs {
         D24S8  = 3
     };
 
-    /*
+    // Returns the number of bytes in the specified depth format
-     * Returns the number of bytes in the specified depth format
-     */
     static u32 BytesPerDepthPixel(DepthFormat format) {
         switch (format) {
         case DepthFormat::D16:
@@ -469,6 +467,20 @@ struct Regs {
         }
     }
 
+    // Returns the number of bits per depth component of the specified depth format
+    static u32 DepthBitsPerPixel(DepthFormat format) {
+        switch (format) {
+        case DepthFormat::D16:
+            return 16;
+        case DepthFormat::D24:
+        case DepthFormat::D24S8:
+            return 24;
+        default:
+            LOG_CRITICAL(HW_GPU, "Unknown depth format %u", format);
+            UNIMPLEMENTED();
+        }
+    }
+
     struct {
         // Components are laid out in reverse byte order, most significant bits first.
         enum ColorFormat : u32 {

src/video_core/rasterizer.cpp

@@ -643,9 +643,10 @@ static void ProcessTriangleInternal(const VertexShader::OutputVertex& v0,
 
             // TODO: Does depth indeed only get written even if depth testing is enabled?
             if (registers.output_merger.depth_test_enable) {
-                u16 z = (u16)((v0.screenpos[2].ToFloat32() * w0 +
+                unsigned num_bits = Pica::Regs::DepthBitsPerPixel(registers.framebuffer.depth_format);
-                            v1.screenpos[2].ToFloat32() * w1 +
+                u32 z = (u32)((v0.screenpos[2].ToFloat32() * w0 +
-                            v2.screenpos[2].ToFloat32() * w2) * 65535.f / wsum);
+                               v1.screenpos[2].ToFloat32() * w1 +
+                               v2.screenpos[2].ToFloat32() * w2) * ((1 << num_bits) - 1) / wsum);
                 u32 ref_z = GetDepth(x >> 4, y >> 4);
 
                 bool pass = false;