Fix

port/fast3d/gfx_opengl.cpp

@@ -34,9 +34,9 @@ struct ShaderProgram {
     GLint noise_scale_location;
     GLint three_point_filter_locations[2];
     GLint decal_mode_location;
-    GLint decal_depth_location;
+    GLint decal_scale_location;
     int decal_mode;
-    float decal_depth[2];
+    float decal_scale;
 };
 
 struct Framebuffer {
@@ -66,8 +66,10 @@ static MipmapFilteringMode current_mipmap_filter_mode = MIPMAP_LINEAR;
 static bool current_textures_linear_filter[2] = {false, false};
 static struct ShaderProgram* current_shader_program;
 
-// Decal depth proximity: decal pixels are discarded if the surface below them is much further away
+// N64 decal depth mode: pixels only pass when close to the stored depth, using a copy of the depth buffer
 #define DECAL_DEPTH_TEXTURE_UNIT 2
+// Height of the N64 frame; the RDP's depth slope tolerance is measured in N64 pixels
+#define DECAL_N64_SCREEN_HEIGHT 240.0f
 static bool depth_copy_supported;
 static bool depth_copy_dirty = true;
 static size_t depth_copy_source;
@@ -75,8 +77,6 @@ static GLuint depth_copy_fbo, depth_copy_tex;
 static uint32_t depth_copy_width, depth_copy_height;
 static bool current_depth_test;
 static bool current_depth_decal;
-static bool decal_projection_valid;
-static float decal_depth_params[2];
 
 static int gl_glsl_version = 130;
 static char gl_glsl_version_str[16] = "130";
@@ -403,7 +403,7 @@ static struct ShaderProgram* gfx_opengl_create_and_load_new_shader(uint64_t shad
     append_line(fs_buf, &fs_len, "uniform int frame_count;");
     append_line(fs_buf, &fs_len, "uniform float noise_scale;");
     append_line(fs_buf, &fs_len, "uniform int uDecalMode;");
-    append_line(fs_buf, &fs_len, "uniform highp vec2 uDecalDepth;");
+    append_line(fs_buf, &fs_len, "uniform highp float uDecalPixelScale;");
     append_line(fs_buf, &fs_len, "uniform highp sampler2D uDepthTex;");
 
     append_line(fs_buf, &fs_len, "float random(in vec3 value) {");
@@ -465,13 +465,13 @@ static struct ShaderProgram* gfx_opengl_create_and_load_new_shader(uint64_t shad
 
     append_line(fs_buf, &fs_len, "void main() {");
 
-    // Window depth is c0 + c1 / w, so the stored depth can be converted back to the clip-space W of the surface.
-    // Reject decal pixels that hang past an edge over a surface that is further away than a small tolerance.
+    // RDP decal Z mode: pass only if |z - stored z| <= dz (the per-pixel depth slope), and never over cleared depth.
+    // The slope is computed before anything can discard, since derivatives are undefined after a neighbor discards.
+    append_line(fs_buf, &fs_len, "    highp float decalSlope = fwidth(gl_FragCoord.z);");
     append_line(fs_buf, &fs_len, "    if (uDecalMode != 0) {");
-    append_line(fs_buf, &fs_len, "        highp float fragW = 1.0 / gl_FragCoord.w;");
-    append_line(fs_buf, &fs_len, "        highp float surfaceZ = texelFetch(uDepthTex, ivec2(gl_FragCoord.xy), 0).r;");
-    append_line(fs_buf, &fs_len, "        highp float surfaceW = uDecalDepth.y / (surfaceZ - uDecalDepth.x);");
-    append_line(fs_buf, &fs_len, "        if (surfaceW - fragW > fragW * 0.03 + 2.0 * fwidth(fragW)) discard;");
+    append_line(fs_buf, &fs_len, "        highp float storedZ = texelFetch(uDepthTex, ivec2(gl_FragCoord.xy), 0).r;");
+    append_line(fs_buf, &fs_len, "        highp float dz = max(decalSlope * uDecalPixelScale, 1.0 / 262144.0);");
+    append_line(fs_buf, &fs_len, "        if (storedZ >= 1.0 || abs(gl_FragCoord.z - storedZ) > dz) discard;");
     append_line(fs_buf, &fs_len, "    }");
 
     for (int i = 0; i < 2; i++) {
@@ -685,9 +685,9 @@ static struct ShaderProgram* gfx_opengl_create_and_load_new_shader(uint64_t shad
     prg->three_point_filter_locations[0] = glGetUniformLocation(shader_program, "three_point_filter0");
     prg->three_point_filter_locations[1] = glGetUniformLocation(shader_program, "three_point_filter1");
     prg->decal_mode_location = glGetUniformLocation(shader_program, "uDecalMode");
-    prg->decal_depth_location = glGetUniformLocation(shader_program, "uDecalDepth");
+    prg->decal_scale_location = glGetUniformLocation(shader_program, "uDecalPixelScale");
     prg->decal_mode = 0;
-    prg->decal_depth[0] = prg->decal_depth[1] = 0.0f;
+    prg->decal_scale = 0.0f;
     if (prg->decal_mode_location >= 0) {
         glUniform1i(prg->decal_mode_location, 0);
         glUniform1i(glGetUniformLocation(shader_program, "uDepthTex"), DECAL_DEPTH_TEXTURE_UNIT);
@@ -854,14 +854,6 @@ static void gfx_opengl_set_use_alpha(bool use_alpha, bool modulate) {
     }
 }
 
-static void gfx_opengl_set_decal_projection(bool valid, float alpha, float beta) {
-    // Clip-space Z is alpha * W + beta; convert that to window depth, including the no-depth-clamp Z scale hack
-    const float zscale = GLAD_GL_ARB_depth_clamp ? 1.0f : 0.3f;
-    decal_projection_valid = valid;
-    decal_depth_params[0] = 0.5f + 0.5f * zscale * alpha;
-    decal_depth_params[1] = 0.5f * zscale * beta;
-}
-
 static bool gfx_opengl_update_depth_copy(void) {
     if (!depth_copy_supported) {
         return false;
@@ -942,18 +934,29 @@ static void gfx_opengl_set_decal_uniforms(struct ShaderProgram* prg, bool enable
         glUniform1i(prg->decal_mode_location, mode);
         prg->decal_mode = mode;
     }
-    if (enabled &&
-        (prg->decal_depth[0] != decal_depth_params[0] || prg->decal_depth[1] != decal_depth_params[1])) {
-        glUniform2f(prg->decal_depth_location, decal_depth_params[0], decal_depth_params[1]);
-        prg->decal_depth[0] = decal_depth_params[0];
-        prg->decal_depth[1] = decal_depth_params[1];
+
+    const float scale = framebuffers[current_framebuffer].height / DECAL_N64_SCREEN_HEIGHT;
+    if (enabled && prg->decal_scale != scale) {
+        glUniform1f(prg->decal_scale_location, scale);
+        prg->decal_scale = scale;
     }
 }
 
 static void gfx_opengl_draw_triangles(float buf_vbo[], size_t buf_vbo_len, size_t buf_vbo_num_tris) {
     // printf("flushing %d tris\n", buf_vbo_num_tris);
-    const bool decal = current_depth_decal && decal_projection_valid && gfx_opengl_update_depth_copy();
-    gfx_opengl_set_decal_uniforms(current_shader_program, decal);
+    if (current_depth_decal) {
+        // The shader replaces the depth test like the RDP does; fall back to polygon offset if depth can't be copied
+        const bool decal = gfx_opengl_update_depth_copy();
+        glDepthFunc(decal ? GL_ALWAYS : GL_LEQUAL);
+        if (decal) {
+            glDisable(GL_POLYGON_OFFSET_FILL);
+        } else {
+            glEnable(GL_POLYGON_OFFSET_FILL);
+        }
+        gfx_opengl_set_decal_uniforms(current_shader_program, decal);
+    } else {
+        gfx_opengl_set_decal_uniforms(current_shader_program, false);
+    }
 
     glBufferData(GL_ARRAY_BUFFER, sizeof(float) * buf_vbo_len, buf_vbo, GL_STREAM_DRAW);
     glDrawArrays(GL_TRIANGLES, 0, 3 * buf_vbo_num_tris);
@@ -1472,6 +1475,5 @@ struct GfxRenderingAPI gfx_opengl_api = {
     gfx_opengl_get_texture_filter,
     gfx_opengl_set_mipmap_filter,
     gfx_opengl_set_anisotropy_level,
-    gfx_opengl_get_max_anisotropy_level,
-    gfx_opengl_set_decal_projection
+    gfx_opengl_get_max_anisotropy_level
 };

port/fast3d/gfx_pc.cpp

@@ -204,8 +204,6 @@ static struct RenderingState {
     struct XYWidthHeight viewport, scissor;
     struct ShaderProgram* shader_program;
     TextureCacheNode* textures[SHADER_MAX_TEXTURES];
-    bool decal_projection_valid;
-    float decal_projection[2];
 } rendering_state;
 
 struct GfxDimensions gfx_current_window_dimensions;
@@ -1276,39 +1274,6 @@ static void gfx_sp_tri1(uint8_t vtx1_idx, uint8_t vtx2_idx, uint8_t vtx3_idx, bo
         rendering_state.depth_mode = depth_mode;
     }
 
-    if (depth_test && depth_compare && zmode == ZMODE_DEC) {
-        // For perspective projections the clip-space Z column is alpha * W column + beta,
-        // which lets the renderer turn stored depth back into distance for decal proximity tests
-        int k = 0;
-        for (int i = 1; i < 3; i++) {
-            if (fabsf(rsp.P_matrix[i][3]) > fabsf(rsp.P_matrix[k][3])) {
-                k = i;
-            }
-        }
-
-        bool valid = fabsf(rsp.P_matrix[k][3]) > 1e-6f;
-        float alpha = 0.0f, beta = 0.0f;
-        if (valid) {
-            alpha = rsp.P_matrix[k][2] / rsp.P_matrix[k][3];
-            beta = rsp.P_matrix[3][2] - alpha * rsp.P_matrix[3][3];
-            for (int i = 0; i < 3; i++) {
-                if (fabsf(rsp.P_matrix[i][2] - alpha * rsp.P_matrix[i][3]) > 1e-4f * (1.0f + fabsf(rsp.P_matrix[i][2]))) {
-                    valid = false;
-                }
-            }
-            valid = valid && fabsf(beta) > 1e-6f;
-        }
-
-        if (valid != rendering_state.decal_projection_valid || alpha != rendering_state.decal_projection[0] ||
-            beta != rendering_state.decal_projection[1]) {
-            gfx_flush();
-            gfx_rapi->set_decal_projection(valid, alpha, beta);
-            rendering_state.decal_projection_valid = valid;
-            rendering_state.decal_projection[0] = alpha;
-            rendering_state.decal_projection[1] = beta;
-        }
-    }
-
     if (rdp.viewport_or_scissor_changed) {
         if (memcmp(&rdp.viewport, &rendering_state.viewport, sizeof(rdp.viewport)) != 0) {
             gfx_flush();

port/fast3d/gfx_rendering_api.h

@@ -56,7 +56,6 @@ struct GfxRenderingAPI {
     void (*set_mipmap_filter)(enum MipmapFilteringMode mode);
 	void (*set_anisotropy_level)(int);
 	int (*get_max_anisotropy_level)(void);
-    void (*set_decal_projection)(bool valid, float alpha, float beta);
 };
 
 #endif