renderer/gl: add internal gl formats and reduce internal driver format conversions (#12879)

* format: add internal formats for drm formats

cross referenced with weston and added internal formats and types for a
lot of missing ones. also added a isFormatYUV helper.

* framebuffer: ensure we use right internalformat

ensure we use the right internal format to avoid internal driver
blitting, also since we only attach the GL_STENCIL_ATTACHMENT we might
just aswell only use the GL_STENCIL_INDEX8 to not confuse drivers that
we want a depth aswell.

* texture: use external on yuv or non linear mods

using external makes us use the gpu's internal detiler.
and this is makes intel a lot happier then having to format convert it
to a linear format internally.

* shaders: add external support to CM frag

add external support to CM frag, and correct ext.frag typo.

* formats: remove duplicates and fix a typo in cm.frag

remove duplicate formats and a typo in cm.frag

* formats: add swizzle logic to all formats

add swizzle logic from weston for all formats and use it in shm texture
paths.

* format: more format changes

use monitor drm format instead of forcing something different.

* shader: remove external from cm.frag

drivers want this resolved at compiletime cant use both
samplerExternalOES and sampler2d and then runtime branch it.

* screencopy: swizzle textures in screencopy

swizzle textures in screencopy, to get the right colors when copying.

* screencopy: restore old behaviour

try restore old behaviour before the gles3 format changes.
glReadPixels had the wrong format, so i went to far trying to mitigate
it. should be like before now.

1 files changed, 9 insertions, 7 deletions

diff --git a/src/render/Framebuffer.cpp b/src/render/Framebuffer.cpp
index 48e44570..cfafd4be 100644
--- a/src/render/Framebuffer.cpp
+++ b/src/render/Framebuffer.cpp
@@ -9,10 +9,8 @@ bool CFramebuffer::alloc(int w, int h, uint32_t drmFormat) {
     bool firstAlloc = false;
     RASSERT((w > 0 && h > 0), "cannot alloc a FB with negative / zero size! (attempted {}x{})", w, h);
 
-    const uint32_t glFormat      = NFormatUtils::drmFormatToGL(drmFormat);
-    const uint32_t glType        = NFormatUtils::glFormatToType(glFormat);
-    const bool     sizeChanged   = (m_size != Vector2D(w, h));
-    const bool     formatChanged = (drmFormat != m_drmFormat);
+    const bool sizeChanged   = (m_size != Vector2D(w, h));
+    const bool formatChanged = (drmFormat != m_drmFormat);
 
     if (!m_tex) {
         m_tex = makeShared<CTexture>();
@@ -32,14 +30,15 @@ bool CFramebuffer::alloc(int w, int h, uint32_t drmFormat) {
     }
 
     if (firstAlloc || sizeChanged || formatChanged) {
+        const auto format = NFormatUtils::getPixelFormatFromDRM(drmFormat);
         m_tex->bind();
-        glTexImage2D(GL_TEXTURE_2D, 0, glFormat, w, h, 0, GL_RGBA, glType, nullptr);
+        glTexImage2D(GL_TEXTURE_2D, 0, format->glInternalFormat ? format->glInternalFormat : format->glFormat, w, h, 0, format->glFormat, format->glType, nullptr);
         glBindFramebuffer(GL_FRAMEBUFFER, m_fb);
         glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, m_tex->m_texID, 0);
 
         if (m_stencilTex) {
             m_stencilTex->bind();
-            glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH24_STENCIL8, w, h, 0, GL_DEPTH_STENCIL, GL_UNSIGNED_INT_24_8, nullptr);
+            glTexImage2D(GL_TEXTURE_2D, 0, GL_STENCIL_INDEX8, w, h, 0, GL_STENCIL_INDEX, GL_UNSIGNED_BYTE, nullptr);
             glFramebufferTexture2D(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, GL_TEXTURE_2D, m_stencilTex->m_texID, 0);
         }
 
@@ -59,9 +58,12 @@ bool CFramebuffer::alloc(int w, int h, uint32_t drmFormat) {
 }
 
 void CFramebuffer::addStencil(SP<CTexture> tex) {
+    if (m_stencilTex == tex)
+        return;
+
     m_stencilTex = tex;
     m_stencilTex->bind();
-    glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH24_STENCIL8, m_size.x, m_size.y, 0, GL_DEPTH_STENCIL, GL_UNSIGNED_INT_24_8, nullptr);
+    glTexImage2D(GL_TEXTURE_2D, 0, GL_STENCIL_INDEX8, m_size.x, m_size.y, 0, GL_STENCIL_INDEX, GL_UNSIGNED_BYTE, nullptr);
 
     glBindFramebuffer(GL_FRAMEBUFFER, m_fb);