renderer/gl: add internal gl formats and reduce internal driver format conversions (#12879)

* format: add internal formats for drm formats

cross referenced with weston and added internal formats and types for a
lot of missing ones. also added a isFormatYUV helper.

* framebuffer: ensure we use right internalformat

ensure we use the right internal format to avoid internal driver
blitting, also since we only attach the GL_STENCIL_ATTACHMENT we might
just aswell only use the GL_STENCIL_INDEX8 to not confuse drivers that
we want a depth aswell.

* texture: use external on yuv or non linear mods

using external makes us use the gpu's internal detiler.
and this is makes intel a lot happier then having to format convert it
to a linear format internally.

* shaders: add external support to CM frag

add external support to CM frag, and correct ext.frag typo.

* formats: remove duplicates and fix a typo in cm.frag

remove duplicate formats and a typo in cm.frag

* formats: add swizzle logic to all formats

add swizzle logic from weston for all formats and use it in shm texture
paths.

* format: more format changes

use monitor drm format instead of forcing something different.

* shader: remove external from cm.frag

drivers want this resolved at compiletime cant use both
samplerExternalOES and sampler2d and then runtime branch it.

* screencopy: swizzle textures in screencopy

swizzle textures in screencopy, to get the right colors when copying.

* screencopy: restore old behaviour

try restore old behaviour before the gles3 format changes.
glReadPixels had the wrong format, so i went to far trying to mitigate
it. should be like before now.

6 files changed, 37 insertions, 24 deletions

diff --git a/src/render/Framebuffer.cpp b/src/render/Framebuffer.cpp
index 48e44570..cfafd4be 100644
--- a/src/render/Framebuffer.cpp
+++ b/src/render/Framebuffer.cpp
@@ -9,10 +9,8 @@ bool CFramebuffer::alloc(int w, int h, uint32_t drmFormat) {
     bool firstAlloc = false;
     RASSERT((w > 0 && h > 0), "cannot alloc a FB with negative / zero size! (attempted {}x{})", w, h);
 
-    const uint32_t glFormat      = NFormatUtils::drmFormatToGL(drmFormat);
-    const uint32_t glType        = NFormatUtils::glFormatToType(glFormat);
-    const bool     sizeChanged   = (m_size != Vector2D(w, h));
-    const bool     formatChanged = (drmFormat != m_drmFormat);
+    const bool sizeChanged   = (m_size != Vector2D(w, h));
+    const bool formatChanged = (drmFormat != m_drmFormat);
 
     if (!m_tex) {
         m_tex = makeShared<CTexture>();
@@ -32,14 +30,15 @@ bool CFramebuffer::alloc(int w, int h, uint32_t drmFormat) {
     }
 
     if (firstAlloc || sizeChanged || formatChanged) {
+        const auto format = NFormatUtils::getPixelFormatFromDRM(drmFormat);
         m_tex->bind();
-        glTexImage2D(GL_TEXTURE_2D, 0, glFormat, w, h, 0, GL_RGBA, glType, nullptr);
+        glTexImage2D(GL_TEXTURE_2D, 0, format->glInternalFormat ? format->glInternalFormat : format->glFormat, w, h, 0, format->glFormat, format->glType, nullptr);
         glBindFramebuffer(GL_FRAMEBUFFER, m_fb);
         glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, m_tex->m_texID, 0);
 
         if (m_stencilTex) {
             m_stencilTex->bind();
-            glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH24_STENCIL8, w, h, 0, GL_DEPTH_STENCIL, GL_UNSIGNED_INT_24_8, nullptr);
+            glTexImage2D(GL_TEXTURE_2D, 0, GL_STENCIL_INDEX8, w, h, 0, GL_STENCIL_INDEX, GL_UNSIGNED_BYTE, nullptr);
             glFramebufferTexture2D(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, GL_TEXTURE_2D, m_stencilTex->m_texID, 0);
         }
 
@@ -59,9 +58,12 @@ bool CFramebuffer::alloc(int w, int h, uint32_t drmFormat) {
 }
 
 void CFramebuffer::addStencil(SP<CTexture> tex) {
+    if (m_stencilTex == tex)
+        return;
+
     m_stencilTex = tex;
     m_stencilTex->bind();
-    glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH24_STENCIL8, m_size.x, m_size.y, 0, GL_DEPTH_STENCIL, GL_UNSIGNED_INT_24_8, nullptr);
+    glTexImage2D(GL_TEXTURE_2D, 0, GL_STENCIL_INDEX8, m_size.x, m_size.y, 0, GL_STENCIL_INDEX, GL_UNSIGNED_BYTE, nullptr);
 
     glBindFramebuffer(GL_FRAMEBUFFER, m_fb);
 
diff --git a/src/render/Framebuffer.hpp b/src/render/Framebuffer.hpp
index 6dbedfee..e6c93876 100644
--- a/src/render/Framebuffer.hpp
+++ b/src/render/Framebuffer.hpp
@@ -3,13 +3,14 @@
 #include "../defines.hpp"
 #include "../helpers/Format.hpp"
 #include "Texture.hpp"
+#include <drm_fourcc.h>
 
 class CFramebuffer {
   public:
     CFramebuffer();
     ~CFramebuffer();
 
-    bool         alloc(int w, int h, uint32_t format = GL_RGBA);
+    bool         alloc(int w, int h, uint32_t format = DRM_FORMAT_ARGB8888);
     void         addStencil(SP<CTexture> tex);
     void         bind();
     void         unbind();
diff --git a/src/render/Texture.cpp b/src/render/Texture.cpp
index f1704afa..5e8c5d40 100644
--- a/src/render/Texture.cpp
+++ b/src/render/Texture.cpp
@@ -73,10 +73,8 @@ void CTexture::createFromShm(uint32_t drmFormat, uint8_t* pixels, uint32_t strid
     setTexParameter(GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
     setTexParameter(GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 
-    if (format->flipRB) {
-        setTexParameter(GL_TEXTURE_SWIZZLE_R, GL_BLUE);
-        setTexParameter(GL_TEXTURE_SWIZZLE_B, GL_RED);
-    }
+    if (format->swizzle.has_value())
+        swizzle(format->swizzle.value());
 
     GLCALL(glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, stride / format->bytesPerBlock));
     GLCALL(glTexImage2D(GL_TEXTURE_2D, 0, format->glInternalFormat ? format->glInternalFormat : format->glFormat, size_.x, size_.y, 0, format->glFormat, format->glType, pixels));
@@ -96,10 +94,18 @@ void CTexture::createFromDma(const Aquamarine::SDMABUFAttrs& attrs, void* image)
     }
 
     m_opaque = NFormatUtils::isFormatOpaque(attrs.format);
+
+    // #TODO external only formats should be external aswell.
+    // also needs a seperate color shader.
+    /*if (NFormatUtils::isFormatYUV(attrs.format)) {
+        m_target = GL_TEXTURE_EXTERNAL_OES;
+        m_type   = TEXTURE_EXTERNAL;
+    } else {*/
     m_target = GL_TEXTURE_2D;
-    m_type   = TEXTURE_RGBA;
-    m_size   = attrs.size;
     m_type   = NFormatUtils::isFormatOpaque(attrs.format) ? TEXTURE_RGBX : TEXTURE_RGBA;
+    //}
+
+    m_size = attrs.size;
     allocate();
     m_eglImage = image;
 
@@ -121,10 +127,8 @@ void CTexture::update(uint32_t drmFormat, uint8_t* pixels, uint32_t stride, cons
 
     bind();
 
-    if (format->flipRB) {
-        setTexParameter(GL_TEXTURE_SWIZZLE_R, GL_BLUE);
-        setTexParameter(GL_TEXTURE_SWIZZLE_B, GL_RED);
-    }
+    if (format->swizzle.has_value())
+        swizzle(format->swizzle.value());
 
     damage.copy().intersect(CBox{{}, m_size}).forEachRect([&format, &stride, &pixels](const auto& rect) {
         GLCALL(glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, stride / format->bytesPerBlock));
@@ -205,3 +209,10 @@ void CTexture::setTexParameter(GLenum pname, GLint param) {
     m_cachedStates[idx] = param;
     GLCALL(glTexParameteri(m_target, pname, param));
 }
+
+void CTexture::swizzle(const std::array<GLint, 4>& colors) {
+    setTexParameter(GL_TEXTURE_SWIZZLE_R, colors.at(0));
+    setTexParameter(GL_TEXTURE_SWIZZLE_G, colors.at(1));
+    setTexParameter(GL_TEXTURE_SWIZZLE_B, colors.at(2));
+    setTexParameter(GL_TEXTURE_SWIZZLE_A, colors.at(3));
+}
diff --git a/src/render/Texture.hpp b/src/render/Texture.hpp
index b9811230..8ee2cab0 100644
--- a/src/render/Texture.hpp
+++ b/src/render/Texture.hpp
@@ -37,6 +37,7 @@ class CTexture {
     void                        bind();
     void                        unbind();
     void                        setTexParameter(GLenum pname, GLint param);
+    void                        swizzle(const std::array<GLint, 4>& colors);
 
     eTextureType                m_type          = TEXTURE_RGBA;
     GLenum                      m_target        = GL_TEXTURE_2D;
diff --git a/src/render/shaders/glsl/CM.frag b/src/render/shaders/glsl/CM.frag
index 031fe7f3..7f075b82 100644
--- a/src/render/shaders/glsl/CM.frag
+++ b/src/render/shaders/glsl/CM.frag
@@ -1,11 +1,9 @@
 #version 300 es
-//#extension GL_OES_EGL_image_external : require
 #extension GL_ARB_shading_language_include : enable
 
 precision highp float;
 in vec2 v_texcoord;
 uniform sampler2D tex;
-//uniform samplerExternalOES texture0;
 
 uniform int texType; // eTextureType: 0 - rgba, 1 - rgbx, 2 - ext
 // uniform int skipCM;
@@ -30,8 +28,8 @@ void main() {
     vec4 pixColor;
     if (texType == 1)
         pixColor = vec4(texture(tex, v_texcoord).rgb, 1.0);
-//    else if (texType == 2)
-//        pixColor = texture(texture0, v_texcoord);
+    //else if (texType == 2)
+    //    discard; // this shouldnt happen.
     else // assume rgba
         pixColor = texture(tex, v_texcoord);
 
diff --git a/src/render/shaders/glsl/ext.frag b/src/render/shaders/glsl/ext.frag
index f540a9f9..e855a832 100644
--- a/src/render/shaders/glsl/ext.frag
+++ b/src/render/shaders/glsl/ext.frag
@@ -5,7 +5,7 @@
 
 precision highp float;
 in vec2 v_texcoord;
-uniform samplerExternalOES texture0;
+uniform samplerExternalOES tex;
 uniform float alpha;
 
 #include "rounding.glsl"
@@ -20,7 +20,7 @@ uniform vec3 tint;
 layout(location = 0) out vec4 fragColor;
 void main() {
 
-    vec4 pixColor = texture(texture0, v_texcoord);
+    vec4 pixColor = texture(tex, v_texcoord);
 
     if (discardOpaque == 1 && pixColor[3] * alpha == 1.0)
 	discard;