opengl: simplify cm pipeline

fixes a few mistakes, and skips the CM shader in cpu instead of adding a costly branch

ref #9641

1 files changed, 53 insertions, 60 deletions

diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index 91a1df9f..137e4a8f 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -875,7 +875,6 @@ void CHyprOpenGLImpl::initShaders() {
         m_RenderData.pCurrentMonData->m_shCM.proj              = glGetUniformLocation(prog, "proj");
         m_RenderData.pCurrentMonData->m_shCM.tex               = glGetUniformLocation(prog, "tex");
         m_RenderData.pCurrentMonData->m_shCM.texType           = glGetUniformLocation(prog, "texType");
-        m_RenderData.pCurrentMonData->m_shCM.skipCM            = glGetUniformLocation(prog, "skipCM");
         m_RenderData.pCurrentMonData->m_shCM.sourceTF          = glGetUniformLocation(prog, "sourceTF");
         m_RenderData.pCurrentMonData->m_shCM.targetTF          = glGetUniformLocation(prog, "targetTF");
         m_RenderData.pCurrentMonData->m_shCM.sourcePrimaries   = glGetUniformLocation(prog, "sourcePrimaries");
@@ -1352,27 +1351,18 @@ void CHyprOpenGLImpl::renderTextureInternalWithDamage(SP<CTexture> tex, const CB
             shader           = &m_RenderData.pCurrentMonData->m_shPASSTHRURGBA;
             usingFinalShader = true;
         } else {
-#ifndef GLES2
-            if (m_bCMSupported)
-                shader = &m_RenderData.pCurrentMonData->m_shCM;
-            else
-#endif
-                switch (tex->m_iType) {
-                    case TEXTURE_RGBA: shader = &m_RenderData.pCurrentMonData->m_shRGBA; break;
-                    case TEXTURE_RGBX: shader = &m_RenderData.pCurrentMonData->m_shRGBX; break;
+            switch (tex->m_iType) {
+                case TEXTURE_RGBA: shader = &m_RenderData.pCurrentMonData->m_shRGBA; break;
+                case TEXTURE_RGBX: shader = &m_RenderData.pCurrentMonData->m_shRGBX; break;
 
-                    case TEXTURE_EXTERNAL: shader = &m_RenderData.pCurrentMonData->m_shEXT; break; // might be unused
-                    default: RASSERT(false, "tex->m_iTarget unsupported!");
-                }
+                case TEXTURE_EXTERNAL: shader = &m_RenderData.pCurrentMonData->m_shEXT; break; // might be unused
+                default: RASSERT(false, "tex->m_iTarget unsupported!");
+            }
         }
     }
 
-    if (m_RenderData.currentWindow && m_RenderData.currentWindow->m_sWindowData.RGBX.valueOrDefault()) {
-#ifdef GLES2
-        shader = &m_RenderData.pCurrentMonData->m_shRGBX;
-#endif
+    if (m_RenderData.currentWindow && m_RenderData.currentWindow->m_sWindowData.RGBX.valueOrDefault())
         texType = TEXTURE_RGBX;
-    }
 
     glActiveTexture(GL_TEXTURE0);
     glBindTexture(tex->m_iTarget, tex->m_iTexID);
@@ -1388,58 +1378,61 @@ void CHyprOpenGLImpl::renderTextureInternalWithDamage(SP<CTexture> tex, const CB
         glTexParameteri(tex->m_iTarget, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
     }
 
+    const bool skipCM = !m_RenderData.surface /* No surface - no point in CM */
+        || !m_bCMSupported                    /* CM unsupported - hw failed to compile the shader probably */
+        || (*PPASS && m_RenderData.pMonitor->activeWorkspace && m_RenderData.pMonitor->activeWorkspace->m_bHasFullscreenWindow &&
+            m_RenderData.pMonitor->activeWorkspace->m_efFullscreenMode == FSMODE_FULLSCREEN) /* Fullscreen window with pass cm enabled */;
+
     glUseProgram(shader->program);
 
 #ifndef GLES2
+    if (!skipCM && !usingFinalShader && (texType == TEXTURE_RGBA || texType == TEXTURE_RGBX)) {
+        shader = &m_RenderData.pCurrentMonData->m_shCM;
+        glUseProgram(shader->program);
+        glUniform1i(shader->texType, texType);
+        const auto imageDescription =
+            m_RenderData.surface.valid() && m_RenderData.surface->colorManagement.valid() ? m_RenderData.surface->colorManagement->imageDescription() : SImageDescription{};
+        glUniform1i(shader->sourceTF, imageDescription.transferFunction);
+        glUniform1i(shader->targetTF, m_RenderData.pMonitor->imageDescription.transferFunction);
+        const auto sourcePrimaries =
+            imageDescription.primariesNameSet || imageDescription.primaries == SPCPRimaries{} ? getPrimaries(imageDescription.primariesNamed) : imageDescription.primaries;
+        const auto    targetPrimaries = m_RenderData.pMonitor->imageDescription.primariesNameSet || m_RenderData.pMonitor->imageDescription.primaries == SPCPRimaries{} ?
+               getPrimaries(m_RenderData.pMonitor->imageDescription.primariesNamed) :
+               m_RenderData.pMonitor->imageDescription.primaries;
+
+        const GLfloat glSourcePrimaries[8] = {
+            sourcePrimaries.red.x,  sourcePrimaries.red.y,  sourcePrimaries.green.x, sourcePrimaries.green.y,
+            sourcePrimaries.blue.x, sourcePrimaries.blue.y, sourcePrimaries.white.x, sourcePrimaries.white.y,
+        };
+        const GLfloat glTargetPrimaries[8] = {
+            targetPrimaries.red.x,  targetPrimaries.red.y,  targetPrimaries.green.x, targetPrimaries.green.y,
+            targetPrimaries.blue.x, targetPrimaries.blue.y, targetPrimaries.white.x, targetPrimaries.white.y,
+        };
+        glUniformMatrix4x2fv(shader->sourcePrimaries, 1, false, glSourcePrimaries);
+        glUniformMatrix4x2fv(shader->targetPrimaries, 1, false, glTargetPrimaries);
+
+        const float maxLuminance = imageDescription.luminances.max > 0 ? imageDescription.luminances.max : imageDescription.luminances.reference;
+        glUniform1f(shader->maxLuminance, maxLuminance * m_RenderData.pMonitor->imageDescription.luminances.reference / imageDescription.luminances.reference);
+        glUniform1f(shader->dstMaxLuminance, m_RenderData.pMonitor->imageDescription.luminances.max > 0 ? m_RenderData.pMonitor->imageDescription.luminances.max : 10000);
+        glUniform1f(shader->dstRefLuminance, m_RenderData.pMonitor->imageDescription.luminances.reference);
+        glUniform1f(shader->sdrSaturation,
+                    m_RenderData.pMonitor->sdrSaturation > 0 && m_RenderData.pMonitor->imageDescription.transferFunction == NColorManagement::CM_TRANSFER_FUNCTION_ST2084_PQ ?
+                        m_RenderData.pMonitor->sdrSaturation :
+                        1.0f);
+        glUniform1f(shader->sdrBrightness,
+                    m_RenderData.pMonitor->sdrBrightness > 0 && m_RenderData.pMonitor->imageDescription.transferFunction == NColorManagement::CM_TRANSFER_FUNCTION_ST2084_PQ ?
+                        m_RenderData.pMonitor->sdrBrightness :
+                        1.0f);
+    }
+#endif
+
+#ifndef GLES2
     glUniformMatrix3fv(shader->proj, 1, GL_TRUE, glMatrix.getMatrix().data());
 #else
     glMatrix.transpose();
     glUniformMatrix3fv(shader->proj, 1, GL_FALSE, glMatrix.getMatrix().data());
 #endif
     glUniform1i(shader->tex, 0);
-#ifndef GLES2
-    if (shader == &m_RenderData.pCurrentMonData->m_shCM && !usingFinalShader && (texType == TEXTURE_RGBA || texType == TEXTURE_RGBX)) {
-        const bool skipCM = *PPASS && m_RenderData.pMonitor->activeWorkspace && m_RenderData.pMonitor->activeWorkspace->m_bHasFullscreenWindow &&
-            m_RenderData.pMonitor->activeWorkspace->m_efFullscreenMode == FSMODE_FULLSCREEN;
-        glUniform1i(shader->texType, texType);
-        glUniform1i(shader->skipCM, skipCM);
-        if (!skipCM) {
-            const auto imageDescription =
-                m_RenderData.surface.valid() && m_RenderData.surface->colorManagement.valid() ? m_RenderData.surface->colorManagement->imageDescription() : SImageDescription{};
-            glUniform1i(shader->sourceTF, imageDescription.transferFunction);
-            glUniform1i(shader->targetTF, m_RenderData.pMonitor->imageDescription.transferFunction);
-            const auto sourcePrimaries =
-                imageDescription.primariesNameSet || imageDescription.primaries == SPCPRimaries{} ? getPrimaries(imageDescription.primariesNamed) : imageDescription.primaries;
-            const auto    targetPrimaries = m_RenderData.pMonitor->imageDescription.primariesNameSet || m_RenderData.pMonitor->imageDescription.primaries == SPCPRimaries{} ?
-                   getPrimaries(m_RenderData.pMonitor->imageDescription.primariesNamed) :
-                   m_RenderData.pMonitor->imageDescription.primaries;
-
-            const GLfloat glSourcePrimaries[8] = {
-                sourcePrimaries.red.x,  sourcePrimaries.red.y,  sourcePrimaries.green.x, sourcePrimaries.green.y,
-                sourcePrimaries.blue.x, sourcePrimaries.blue.y, sourcePrimaries.white.x, sourcePrimaries.white.y,
-            };
-            const GLfloat glTargetPrimaries[8] = {
-                targetPrimaries.red.x,  targetPrimaries.red.y,  targetPrimaries.green.x, targetPrimaries.green.y,
-                targetPrimaries.blue.x, targetPrimaries.blue.y, targetPrimaries.white.x, targetPrimaries.white.y,
-            };
-            glUniformMatrix4x2fv(shader->sourcePrimaries, 1, false, glSourcePrimaries);
-            glUniformMatrix4x2fv(shader->targetPrimaries, 1, false, glTargetPrimaries);
-
-            const float maxLuminance = imageDescription.luminances.max > 0 ? imageDescription.luminances.max : imageDescription.luminances.reference;
-            glUniform1f(shader->maxLuminance, maxLuminance * m_RenderData.pMonitor->imageDescription.luminances.reference / imageDescription.luminances.reference);
-            glUniform1f(shader->dstMaxLuminance, m_RenderData.pMonitor->imageDescription.luminances.max > 0 ? m_RenderData.pMonitor->imageDescription.luminances.max : 10000);
-            glUniform1f(shader->dstRefLuminance, m_RenderData.pMonitor->imageDescription.luminances.reference);
-            glUniform1f(shader->sdrSaturation,
-                        m_RenderData.pMonitor->sdrSaturation > 0 && m_RenderData.pMonitor->imageDescription.transferFunction == NColorManagement::CM_TRANSFER_FUNCTION_ST2084_PQ ?
-                            m_RenderData.pMonitor->sdrSaturation :
-                            1.0f);
-            glUniform1f(shader->sdrBrightness,
-                        m_RenderData.pMonitor->sdrBrightness > 0 && m_RenderData.pMonitor->imageDescription.transferFunction == NColorManagement::CM_TRANSFER_FUNCTION_ST2084_PQ ?
-                            m_RenderData.pMonitor->sdrBrightness :
-                            1.0f);
-        }
-    }
-#endif
 
     if ((usingFinalShader && *PDT == 0) || CRASHING) {
         glUniform1f(shader->time, m_tGlobalTimer.getSeconds() - shader->initialTime);