opengl: simplify cm pipeline

fixes a few mistakes, and skips the CM shader in cpu instead of adding a costly branch

ref #9641

3 files changed, 73 insertions, 80 deletions

diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index 91a1df9f..137e4a8f 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -875,7 +875,6 @@ void CHyprOpenGLImpl::initShaders() {
         m_RenderData.pCurrentMonData->m_shCM.proj              = glGetUniformLocation(prog, "proj");
         m_RenderData.pCurrentMonData->m_shCM.tex               = glGetUniformLocation(prog, "tex");
         m_RenderData.pCurrentMonData->m_shCM.texType           = glGetUniformLocation(prog, "texType");
-        m_RenderData.pCurrentMonData->m_shCM.skipCM            = glGetUniformLocation(prog, "skipCM");
         m_RenderData.pCurrentMonData->m_shCM.sourceTF          = glGetUniformLocation(prog, "sourceTF");
         m_RenderData.pCurrentMonData->m_shCM.targetTF          = glGetUniformLocation(prog, "targetTF");
         m_RenderData.pCurrentMonData->m_shCM.sourcePrimaries   = glGetUniformLocation(prog, "sourcePrimaries");
@@ -1352,27 +1351,18 @@ void CHyprOpenGLImpl::renderTextureInternalWithDamage(SP<CTexture> tex, const CB
             shader           = &m_RenderData.pCurrentMonData->m_shPASSTHRURGBA;
             usingFinalShader = true;
         } else {
-#ifndef GLES2
-            if (m_bCMSupported)
-                shader = &m_RenderData.pCurrentMonData->m_shCM;
-            else
-#endif
-                switch (tex->m_iType) {
-                    case TEXTURE_RGBA: shader = &m_RenderData.pCurrentMonData->m_shRGBA; break;
-                    case TEXTURE_RGBX: shader = &m_RenderData.pCurrentMonData->m_shRGBX; break;
+            switch (tex->m_iType) {
+                case TEXTURE_RGBA: shader = &m_RenderData.pCurrentMonData->m_shRGBA; break;
+                case TEXTURE_RGBX: shader = &m_RenderData.pCurrentMonData->m_shRGBX; break;
 
-                    case TEXTURE_EXTERNAL: shader = &m_RenderData.pCurrentMonData->m_shEXT; break; // might be unused
-                    default: RASSERT(false, "tex->m_iTarget unsupported!");
-                }
+                case TEXTURE_EXTERNAL: shader = &m_RenderData.pCurrentMonData->m_shEXT; break; // might be unused
+                default: RASSERT(false, "tex->m_iTarget unsupported!");
+            }
         }
     }
 
-    if (m_RenderData.currentWindow && m_RenderData.currentWindow->m_sWindowData.RGBX.valueOrDefault()) {
-#ifdef GLES2
-        shader = &m_RenderData.pCurrentMonData->m_shRGBX;
-#endif
+    if (m_RenderData.currentWindow && m_RenderData.currentWindow->m_sWindowData.RGBX.valueOrDefault())
         texType = TEXTURE_RGBX;
-    }
 
     glActiveTexture(GL_TEXTURE0);
     glBindTexture(tex->m_iTarget, tex->m_iTexID);
@@ -1388,58 +1378,61 @@ void CHyprOpenGLImpl::renderTextureInternalWithDamage(SP<CTexture> tex, const CB
         glTexParameteri(tex->m_iTarget, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
     }
 
+    const bool skipCM = !m_RenderData.surface /* No surface - no point in CM */
+        || !m_bCMSupported                    /* CM unsupported - hw failed to compile the shader probably */
+        || (*PPASS && m_RenderData.pMonitor->activeWorkspace && m_RenderData.pMonitor->activeWorkspace->m_bHasFullscreenWindow &&
+            m_RenderData.pMonitor->activeWorkspace->m_efFullscreenMode == FSMODE_FULLSCREEN) /* Fullscreen window with pass cm enabled */;
+
     glUseProgram(shader->program);
 
 #ifndef GLES2
+    if (!skipCM && !usingFinalShader && (texType == TEXTURE_RGBA || texType == TEXTURE_RGBX)) {
+        shader = &m_RenderData.pCurrentMonData->m_shCM;
+        glUseProgram(shader->program);
+        glUniform1i(shader->texType, texType);
+        const auto imageDescription =
+            m_RenderData.surface.valid() && m_RenderData.surface->colorManagement.valid() ? m_RenderData.surface->colorManagement->imageDescription() : SImageDescription{};
+        glUniform1i(shader->sourceTF, imageDescription.transferFunction);
+        glUniform1i(shader->targetTF, m_RenderData.pMonitor->imageDescription.transferFunction);
+        const auto sourcePrimaries =
+            imageDescription.primariesNameSet || imageDescription.primaries == SPCPRimaries{} ? getPrimaries(imageDescription.primariesNamed) : imageDescription.primaries;
+        const auto    targetPrimaries = m_RenderData.pMonitor->imageDescription.primariesNameSet || m_RenderData.pMonitor->imageDescription.primaries == SPCPRimaries{} ?
+               getPrimaries(m_RenderData.pMonitor->imageDescription.primariesNamed) :
+               m_RenderData.pMonitor->imageDescription.primaries;
+
+        const GLfloat glSourcePrimaries[8] = {
+            sourcePrimaries.red.x,  sourcePrimaries.red.y,  sourcePrimaries.green.x, sourcePrimaries.green.y,
+            sourcePrimaries.blue.x, sourcePrimaries.blue.y, sourcePrimaries.white.x, sourcePrimaries.white.y,
+        };
+        const GLfloat glTargetPrimaries[8] = {
+            targetPrimaries.red.x,  targetPrimaries.red.y,  targetPrimaries.green.x, targetPrimaries.green.y,
+            targetPrimaries.blue.x, targetPrimaries.blue.y, targetPrimaries.white.x, targetPrimaries.white.y,
+        };
+        glUniformMatrix4x2fv(shader->sourcePrimaries, 1, false, glSourcePrimaries);
+        glUniformMatrix4x2fv(shader->targetPrimaries, 1, false, glTargetPrimaries);
+
+        const float maxLuminance = imageDescription.luminances.max > 0 ? imageDescription.luminances.max : imageDescription.luminances.reference;
+        glUniform1f(shader->maxLuminance, maxLuminance * m_RenderData.pMonitor->imageDescription.luminances.reference / imageDescription.luminances.reference);
+        glUniform1f(shader->dstMaxLuminance, m_RenderData.pMonitor->imageDescription.luminances.max > 0 ? m_RenderData.pMonitor->imageDescription.luminances.max : 10000);
+        glUniform1f(shader->dstRefLuminance, m_RenderData.pMonitor->imageDescription.luminances.reference);
+        glUniform1f(shader->sdrSaturation,
+                    m_RenderData.pMonitor->sdrSaturation > 0 && m_RenderData.pMonitor->imageDescription.transferFunction == NColorManagement::CM_TRANSFER_FUNCTION_ST2084_PQ ?
+                        m_RenderData.pMonitor->sdrSaturation :
+                        1.0f);
+        glUniform1f(shader->sdrBrightness,
+                    m_RenderData.pMonitor->sdrBrightness > 0 && m_RenderData.pMonitor->imageDescription.transferFunction == NColorManagement::CM_TRANSFER_FUNCTION_ST2084_PQ ?
+                        m_RenderData.pMonitor->sdrBrightness :
+                        1.0f);
+    }
+#endif
+
+#ifndef GLES2
     glUniformMatrix3fv(shader->proj, 1, GL_TRUE, glMatrix.getMatrix().data());
 #else
     glMatrix.transpose();
     glUniformMatrix3fv(shader->proj, 1, GL_FALSE, glMatrix.getMatrix().data());
 #endif
     glUniform1i(shader->tex, 0);
-#ifndef GLES2
-    if (shader == &m_RenderData.pCurrentMonData->m_shCM && !usingFinalShader && (texType == TEXTURE_RGBA || texType == TEXTURE_RGBX)) {
-        const bool skipCM = *PPASS && m_RenderData.pMonitor->activeWorkspace && m_RenderData.pMonitor->activeWorkspace->m_bHasFullscreenWindow &&
-            m_RenderData.pMonitor->activeWorkspace->m_efFullscreenMode == FSMODE_FULLSCREEN;
-        glUniform1i(shader->texType, texType);
-        glUniform1i(shader->skipCM, skipCM);
-        if (!skipCM) {
-            const auto imageDescription =
-                m_RenderData.surface.valid() && m_RenderData.surface->colorManagement.valid() ? m_RenderData.surface->colorManagement->imageDescription() : SImageDescription{};
-            glUniform1i(shader->sourceTF, imageDescription.transferFunction);
-            glUniform1i(shader->targetTF, m_RenderData.pMonitor->imageDescription.transferFunction);
-            const auto sourcePrimaries =
-                imageDescription.primariesNameSet || imageDescription.primaries == SPCPRimaries{} ? getPrimaries(imageDescription.primariesNamed) : imageDescription.primaries;
-            const auto    targetPrimaries = m_RenderData.pMonitor->imageDescription.primariesNameSet || m_RenderData.pMonitor->imageDescription.primaries == SPCPRimaries{} ?
-                   getPrimaries(m_RenderData.pMonitor->imageDescription.primariesNamed) :
-                   m_RenderData.pMonitor->imageDescription.primaries;
-
-            const GLfloat glSourcePrimaries[8] = {
-                sourcePrimaries.red.x,  sourcePrimaries.red.y,  sourcePrimaries.green.x, sourcePrimaries.green.y,
-                sourcePrimaries.blue.x, sourcePrimaries.blue.y, sourcePrimaries.white.x, sourcePrimaries.white.y,
-            };
-            const GLfloat glTargetPrimaries[8] = {
-                targetPrimaries.red.x,  targetPrimaries.red.y,  targetPrimaries.green.x, targetPrimaries.green.y,
-                targetPrimaries.blue.x, targetPrimaries.blue.y, targetPrimaries.white.x, targetPrimaries.white.y,
-            };
-            glUniformMatrix4x2fv(shader->sourcePrimaries, 1, false, glSourcePrimaries);
-            glUniformMatrix4x2fv(shader->targetPrimaries, 1, false, glTargetPrimaries);
-
-            const float maxLuminance = imageDescription.luminances.max > 0 ? imageDescription.luminances.max : imageDescription.luminances.reference;
-            glUniform1f(shader->maxLuminance, maxLuminance * m_RenderData.pMonitor->imageDescription.luminances.reference / imageDescription.luminances.reference);
-            glUniform1f(shader->dstMaxLuminance, m_RenderData.pMonitor->imageDescription.luminances.max > 0 ? m_RenderData.pMonitor->imageDescription.luminances.max : 10000);
-            glUniform1f(shader->dstRefLuminance, m_RenderData.pMonitor->imageDescription.luminances.reference);
-            glUniform1f(shader->sdrSaturation,
-                        m_RenderData.pMonitor->sdrSaturation > 0 && m_RenderData.pMonitor->imageDescription.transferFunction == NColorManagement::CM_TRANSFER_FUNCTION_ST2084_PQ ?
-                            m_RenderData.pMonitor->sdrSaturation :
-                            1.0f);
-            glUniform1f(shader->sdrBrightness,
-                        m_RenderData.pMonitor->sdrBrightness > 0 && m_RenderData.pMonitor->imageDescription.transferFunction == NColorManagement::CM_TRANSFER_FUNCTION_ST2084_PQ ?
-                            m_RenderData.pMonitor->sdrBrightness :
-                            1.0f);
-        }
-    }
-#endif
 
     if ((usingFinalShader && *PDT == 0) || CRASHING) {
         glUniform1f(shader->time, m_tGlobalTimer.getSeconds() - shader->initialTime);
diff --git a/src/render/Shader.hpp b/src/render/Shader.hpp
index bcc25268..413f3ef0 100644
--- a/src/render/Shader.hpp
+++ b/src/render/Shader.hpp
@@ -12,7 +12,6 @@ class CShader {
     GLint   color             = -1;
     GLint   alphaMatte        = -1;
     GLint   texType           = -1;
-    GLint   skipCM            = -1;
     GLint   sourceTF          = -1;
     GLint   targetTF          = -1;
     GLint   sourcePrimaries   = -1;
diff --git a/src/render/shaders/CM.frag b/src/render/shaders/CM.frag
index dd695d44..aa26f97d 100644
--- a/src/render/shaders/CM.frag
+++ b/src/render/shaders/CM.frag
@@ -8,7 +8,6 @@ uniform sampler2D tex;
 //uniform samplerExternalOES texture0;
 
 uniform int texType; // eTextureType: 0 - rgba, 1 - rgbx, 2 - ext
-uniform int skipCM;
 uniform int sourceTF; // eTransferFunction
 uniform int targetTF; // eTransferFunction
 uniform mat4x2 sourcePrimaries;
@@ -408,26 +407,28 @@ void main() {
     if (discardAlpha == 1 && pixColor[3] <= discardAlphaValue)
         discard;
 
-    if (skipCM == 0) {
-        pixColor.rgb /= max(pixColor.a, 0.001);
-        pixColor.rgb = toLinearRGB(pixColor.rgb, sourceTF);
-        mat3 srcxyz = primaries2xyz(sourcePrimaries);
-        mat3 dstxyz;
-        if (sourcePrimaries == targetPrimaries)
-            dstxyz = srcxyz;
-        else {
-            dstxyz = primaries2xyz(targetPrimaries);
-            pixColor = convertPrimaries(pixColor, srcxyz, sourcePrimaries[3], dstxyz, targetPrimaries[3]);
-        }
-        pixColor = toNit(pixColor, sourceTF);
-        pixColor.rgb *= pixColor.a;
-        pixColor = tonemap(pixColor, dstxyz);
-        if (sourceTF == CM_TRANSFER_FUNCTION_SRGB && targetTF == CM_TRANSFER_FUNCTION_ST2084_PQ)
-            pixColor = saturate(pixColor, srcxyz, sdrSaturation);
-            pixColor *= sdrBrightnessMultiplier;
-        pixColor = fromLinearNit(pixColor, targetTF);
+    pixColor.rgb /= max(pixColor.a, 0.001);
+    pixColor.rgb = toLinearRGB(pixColor.rgb, sourceTF);
+    mat3 srcxyz = primaries2xyz(sourcePrimaries);
+    mat3 dstxyz;
+
+    if (sourcePrimaries == targetPrimaries)
+        dstxyz = srcxyz;
+    else {
+        dstxyz = primaries2xyz(targetPrimaries);
+        pixColor = convertPrimaries(pixColor, srcxyz, sourcePrimaries[3], dstxyz, targetPrimaries[3]);
     }
 
+    pixColor = toNit(pixColor, sourceTF);
+    pixColor.rgb *= pixColor.a;
+    pixColor = tonemap(pixColor, dstxyz);
+
+    if (sourceTF == CM_TRANSFER_FUNCTION_SRGB && targetTF == CM_TRANSFER_FUNCTION_ST2084_PQ)
+        pixColor = saturate(pixColor, srcxyz, sdrSaturation);
+
+    pixColor *= sdrBrightnessMultiplier;
+    pixColor = fromLinearNit(pixColor, targetTF);
+
     if (applyTint == 1)
         pixColor = vec4(pixColor.rgb * tint.rgb, pixColor[3]);