renderer/gl: add internal gl formats and reduce internal driver format conversions (#12879)

* format: add internal formats for drm formats

cross referenced with weston and added internal formats and types for a
lot of missing ones. also added a isFormatYUV helper.

* framebuffer: ensure we use right internalformat

ensure we use the right internal format to avoid internal driver
blitting, also since we only attach the GL_STENCIL_ATTACHMENT we might
just aswell only use the GL_STENCIL_INDEX8 to not confuse drivers that
we want a depth aswell.

* texture: use external on yuv or non linear mods

using external makes us use the gpu's internal detiler.
and this is makes intel a lot happier then having to format convert it
to a linear format internally.

* shaders: add external support to CM frag

add external support to CM frag, and correct ext.frag typo.

* formats: remove duplicates and fix a typo in cm.frag

remove duplicate formats and a typo in cm.frag

* formats: add swizzle logic to all formats

add swizzle logic from weston for all formats and use it in shm texture
paths.

* format: more format changes

use monitor drm format instead of forcing something different.

* shader: remove external from cm.frag

drivers want this resolved at compiletime cant use both
samplerExternalOES and sampler2d and then runtime branch it.

* screencopy: swizzle textures in screencopy

swizzle textures in screencopy, to get the right colors when copying.

* screencopy: restore old behaviour

try restore old behaviour before the gles3 format changes.
glReadPixels had the wrong format, so i went to far trying to mitigate
it. should be like before now.

1 files changed, 182 insertions, 130 deletions

diff --git a/src/helpers/Format.cpp b/src/helpers/Format.cpp
index 37f77d78..0054c25a 100644
--- a/src/helpers/Format.cpp
+++ b/src/helpers/Format.cpp
@@ -6,158 +6,186 @@
 #include <xf86drm.h>
 #include <drm_fourcc.h>
 
-/*
-    DRM formats are LE, while OGL is BE. The two primary formats
-    will be flipped, so we will set flipRB which will later use swizzle
-    to flip the red and blue channels.
-    This will not work on GLES2, but I want to drop support for it one day anyways.
-*/
 inline const std::vector<SPixelFormat> GLES3_FORMATS = {
     {
-        .drmFormat     = DRM_FORMAT_ARGB8888,
-        .flipRB        = true,
-        .glFormat      = GL_RGBA,
-        .glType        = GL_UNSIGNED_BYTE,
-        .withAlpha     = true,
-        .alphaStripped = DRM_FORMAT_XRGB8888,
-        .bytesPerBlock = 4,
+        .drmFormat        = DRM_FORMAT_ARGB8888,
+        .glInternalFormat = GL_RGBA8,
+        .glFormat         = GL_RGBA,
+        .glType           = GL_UNSIGNED_BYTE,
+        .withAlpha        = true,
+        .alphaStripped    = DRM_FORMAT_XRGB8888,
+        .bytesPerBlock    = 4,
+        .swizzle          = {SWIZZLE_BGRA},
     },
     {
-        .drmFormat     = DRM_FORMAT_XRGB8888,
-        .flipRB        = true,
-        .glFormat      = GL_RGBA,
-        .glType        = GL_UNSIGNED_BYTE,
-        .withAlpha     = false,
-        .alphaStripped = DRM_FORMAT_XRGB8888,
-        .bytesPerBlock = 4,
+        .drmFormat        = DRM_FORMAT_XRGB8888,
+        .glInternalFormat = GL_RGBA8,
+        .glFormat         = GL_RGBA,
+        .glType           = GL_UNSIGNED_BYTE,
+        .withAlpha        = false,
+        .alphaStripped    = DRM_FORMAT_XRGB8888,
+        .bytesPerBlock    = 4,
+        .swizzle          = {SWIZZLE_BGR1},
     },
     {
-        .drmFormat     = DRM_FORMAT_XBGR8888,
-        .glFormat      = GL_RGBA,
-        .glType        = GL_UNSIGNED_BYTE,
-        .withAlpha     = false,
-        .alphaStripped = DRM_FORMAT_XBGR8888,
-        .bytesPerBlock = 4,
+        .drmFormat        = DRM_FORMAT_XBGR8888,
+        .glInternalFormat = GL_RGBA8,
+        .glFormat         = GL_RGBA,
+        .glType           = GL_UNSIGNED_BYTE,
+        .withAlpha        = false,
+        .alphaStripped    = DRM_FORMAT_XBGR8888,
+        .bytesPerBlock    = 4,
+        .swizzle          = {SWIZZLE_RGB1},
     },
     {
-        .drmFormat     = DRM_FORMAT_ABGR8888,
-        .glFormat      = GL_RGBA,
-        .glType        = GL_UNSIGNED_BYTE,
-        .withAlpha     = true,
-        .alphaStripped = DRM_FORMAT_XBGR8888,
-        .bytesPerBlock = 4,
+        .drmFormat        = DRM_FORMAT_ABGR8888,
+        .glInternalFormat = GL_RGBA8,
+        .glFormat         = GL_RGBA,
+        .glType           = GL_UNSIGNED_BYTE,
+        .withAlpha        = true,
+        .alphaStripped    = DRM_FORMAT_XBGR8888,
+        .bytesPerBlock    = 4,
+        .swizzle          = {SWIZZLE_RGBA},
     },
     {
-        .drmFormat     = DRM_FORMAT_BGR888,
-        .glFormat      = GL_RGB,
-        .glType        = GL_UNSIGNED_BYTE,
-        .withAlpha     = false,
-        .alphaStripped = DRM_FORMAT_BGR888,
-        .bytesPerBlock = 3,
+        .drmFormat        = DRM_FORMAT_BGR888,
+        .glInternalFormat = GL_RGB8,
+        .glFormat         = GL_RGB,
+        .glType           = GL_UNSIGNED_BYTE,
+        .withAlpha        = false,
+        .alphaStripped    = DRM_FORMAT_BGR888,
+        .bytesPerBlock    = 3,
+        .swizzle          = {SWIZZLE_RGB1},
     },
     {
-        .drmFormat     = DRM_FORMAT_RGBX4444,
-        .glFormat      = GL_RGBA,
-        .glType        = GL_UNSIGNED_SHORT_4_4_4_4,
-        .withAlpha     = false,
-        .alphaStripped = DRM_FORMAT_RGBX4444,
-        .bytesPerBlock = 2,
+        .drmFormat        = DRM_FORMAT_RGBX4444,
+        .glInternalFormat = GL_RGBA4,
+        .glFormat         = GL_RGBA,
+        .glType           = GL_UNSIGNED_SHORT_4_4_4_4,
+        .withAlpha        = false,
+        .alphaStripped    = DRM_FORMAT_RGBX4444,
+        .bytesPerBlock    = 2,
+        .swizzle          = {SWIZZLE_RGB1},
     },
     {
-        .drmFormat     = DRM_FORMAT_RGBA4444,
-        .glFormat      = GL_RGBA,
-        .glType        = GL_UNSIGNED_SHORT_4_4_4_4,
-        .withAlpha     = true,
-        .alphaStripped = DRM_FORMAT_RGBX4444,
-        .bytesPerBlock = 2,
+        .drmFormat        = DRM_FORMAT_RGBA4444,
+        .glInternalFormat = GL_RGBA4,
+        .glFormat         = GL_RGBA,
+        .glType           = GL_UNSIGNED_SHORT_4_4_4_4,
+        .withAlpha        = true,
+        .alphaStripped    = DRM_FORMAT_RGBX4444,
+        .bytesPerBlock    = 2,
+        .swizzle          = {SWIZZLE_RGBA},
     },
     {
-        .drmFormat     = DRM_FORMAT_RGBX5551,
-        .glFormat      = GL_RGBA,
-        .glType        = GL_UNSIGNED_SHORT_5_5_5_1,
-        .withAlpha     = false,
-        .alphaStripped = DRM_FORMAT_RGBX5551,
-        .bytesPerBlock = 2,
+        .drmFormat        = DRM_FORMAT_RGBX5551,
+        .glInternalFormat = GL_RGB5_A1,
+        .glFormat         = GL_RGBA,
+        .glType           = GL_UNSIGNED_SHORT_5_5_5_1,
+        .withAlpha        = false,
+        .alphaStripped    = DRM_FORMAT_RGBX5551,
+        .bytesPerBlock    = 2,
+        .swizzle          = {SWIZZLE_RGB1},
     },
     {
-        .drmFormat     = DRM_FORMAT_RGBA5551,
-        .glFormat      = GL_RGBA,
-        .glType        = GL_UNSIGNED_SHORT_5_5_5_1,
-        .withAlpha     = true,
-        .alphaStripped = DRM_FORMAT_RGBX5551,
-        .bytesPerBlock = 2,
+        .drmFormat        = DRM_FORMAT_RGBA5551,
+        .glInternalFormat = GL_RGB5_A1,
+        .glFormat         = GL_RGBA,
+        .glType           = GL_UNSIGNED_SHORT_5_5_5_1,
+        .withAlpha        = true,
+        .alphaStripped    = DRM_FORMAT_RGBX5551,
+        .bytesPerBlock    = 2,
+        .swizzle          = {SWIZZLE_RGBA},
     },
     {
-        .drmFormat     = DRM_FORMAT_RGB565,
-        .glFormat      = GL_RGB,
-        .glType        = GL_UNSIGNED_SHORT_5_6_5,
-        .withAlpha     = false,
-        .alphaStripped = DRM_FORMAT_RGB565,
-        .bytesPerBlock = 2,
+        .drmFormat        = DRM_FORMAT_RGB565,
+        .glInternalFormat = GL_RGB565,
+        .glFormat         = GL_RGB,
+        .glType           = GL_UNSIGNED_SHORT_5_6_5,
+        .withAlpha        = false,
+        .alphaStripped    = DRM_FORMAT_RGB565,
+        .bytesPerBlock    = 2,
+        .swizzle          = {SWIZZLE_RGB1},
     },
     {
-        .drmFormat     = DRM_FORMAT_XBGR2101010,
-        .glFormat      = GL_RGBA,
-        .glType        = GL_UNSIGNED_INT_2_10_10_10_REV,
-        .withAlpha     = false,
-        .alphaStripped = DRM_FORMAT_XBGR2101010,
-        .bytesPerBlock = 4,
+        .drmFormat        = DRM_FORMAT_XBGR2101010,
+        .glInternalFormat = GL_RGB10_A2,
+        .glFormat         = GL_RGBA,
+        .glType           = GL_UNSIGNED_INT_2_10_10_10_REV,
+        .withAlpha        = false,
+        .alphaStripped    = DRM_FORMAT_XBGR2101010,
+        .bytesPerBlock    = 4,
+        .swizzle          = {SWIZZLE_RGB1},
     },
     {
-        .drmFormat     = DRM_FORMAT_ABGR2101010,
-        .glFormat      = GL_RGBA,
-        .glType        = GL_UNSIGNED_INT_2_10_10_10_REV,
-        .withAlpha     = true,
-        .alphaStripped = DRM_FORMAT_XBGR2101010,
-        .bytesPerBlock = 4,
+        .drmFormat        = DRM_FORMAT_ABGR2101010,
+        .glInternalFormat = GL_RGB10_A2,
+        .glFormat         = GL_RGBA,
+        .glType           = GL_UNSIGNED_INT_2_10_10_10_REV,
+        .withAlpha        = true,
+        .alphaStripped    = DRM_FORMAT_XBGR2101010,
+        .bytesPerBlock    = 4,
+        .swizzle          = {SWIZZLE_RGBA},
     },
     {
-        .drmFormat     = DRM_FORMAT_XRGB2101010,
-        .glFormat      = GL_RGBA,
-        .glType        = GL_UNSIGNED_INT_2_10_10_10_REV,
-        .withAlpha     = false,
-        .alphaStripped = DRM_FORMAT_XRGB2101010,
-        .bytesPerBlock = 4,
+        .drmFormat        = DRM_FORMAT_XRGB2101010,
+        .glInternalFormat = GL_RGB10_A2,
+        .glFormat         = GL_RGBA,
+        .glType           = GL_UNSIGNED_INT_2_10_10_10_REV,
+        .withAlpha        = false,
+        .alphaStripped    = DRM_FORMAT_XRGB2101010,
+        .bytesPerBlock    = 4,
+        .swizzle          = {SWIZZLE_BGR1},
     },
     {
-        .drmFormat     = DRM_FORMAT_ARGB2101010,
-        .glFormat      = GL_RGBA,
-        .glType        = GL_UNSIGNED_INT_2_10_10_10_REV,
-        .withAlpha     = true,
-        .alphaStripped = DRM_FORMAT_XRGB2101010,
-        .bytesPerBlock = 4,
+        .drmFormat        = DRM_FORMAT_ARGB2101010,
+        .glInternalFormat = GL_RGB10_A2,
+        .glFormat         = GL_RGBA,
+        .glType           = GL_UNSIGNED_INT_2_10_10_10_REV,
+        .withAlpha        = true,
+        .alphaStripped    = DRM_FORMAT_XRGB2101010,
+        .bytesPerBlock    = 4,
+        .swizzle          = {SWIZZLE_BGRA},
     },
     {
-        .drmFormat     = DRM_FORMAT_XBGR16161616F,
-        .glFormat      = GL_RGBA,
-        .glType        = GL_HALF_FLOAT,
-        .withAlpha     = false,
-        .alphaStripped = DRM_FORMAT_XBGR16161616F,
-        .bytesPerBlock = 8,
+        .drmFormat        = DRM_FORMAT_XBGR16161616F,
+        .glInternalFormat = GL_RGBA16F,
+        .glFormat         = GL_RGBA,
+        .glType           = GL_HALF_FLOAT,
+        .withAlpha        = false,
+        .alphaStripped    = DRM_FORMAT_XBGR16161616F,
+        .bytesPerBlock    = 8,
+        .swizzle          = {SWIZZLE_RGB1},
     },
     {
-        .drmFormat     = DRM_FORMAT_ABGR16161616F,
-        .glFormat      = GL_RGBA,
-        .glType        = GL_HALF_FLOAT,
-        .withAlpha     = true,
-        .alphaStripped = DRM_FORMAT_XBGR16161616F,
-        .bytesPerBlock = 8,
+        .drmFormat        = DRM_FORMAT_ABGR16161616F,
+        .glInternalFormat = GL_RGBA16F,
+        .glFormat         = GL_RGBA,
+        .glType           = GL_HALF_FLOAT,
+        .withAlpha        = true,
+        .alphaStripped    = DRM_FORMAT_XBGR16161616F,
+        .bytesPerBlock    = 8,
+        .swizzle          = {SWIZZLE_RGBA},
     },
     {
-        .drmFormat     = DRM_FORMAT_XBGR16161616,
-        .glFormat      = GL_RGBA16UI,
-        .glType        = GL_UNSIGNED_SHORT,
-        .withAlpha     = false,
-        .alphaStripped = DRM_FORMAT_XBGR16161616,
-        .bytesPerBlock = 8,
+        .drmFormat        = DRM_FORMAT_XBGR16161616,
+        .glInternalFormat = GL_RGBA16UI,
+        .glFormat         = GL_RGBA_INTEGER,
+        .glType           = GL_UNSIGNED_SHORT,
+        .withAlpha        = false,
+        .alphaStripped    = DRM_FORMAT_XBGR16161616,
+        .bytesPerBlock    = 8,
+        .swizzle          = {SWIZZLE_RGBA},
     },
     {
-        .drmFormat     = DRM_FORMAT_ABGR16161616,
-        .glFormat      = GL_RGBA16UI,
-        .glType        = GL_UNSIGNED_SHORT,
-        .withAlpha     = true,
-        .alphaStripped = DRM_FORMAT_XBGR16161616,
-        .bytesPerBlock = 8,
+        .drmFormat        = DRM_FORMAT_ABGR16161616,
+        .glInternalFormat = GL_RGBA16UI,
+        .glFormat         = GL_RGBA_INTEGER,
+        .glType           = GL_UNSIGNED_SHORT,
+        .withAlpha        = true,
+        .alphaStripped    = DRM_FORMAT_XBGR16161616,
+        .bytesPerBlock    = 8,
+        .swizzle          = {SWIZZLE_RGBA},
     },
     {
         .drmFormat     = DRM_FORMAT_YVYU,
@@ -170,24 +198,28 @@ inline const std::vector<SPixelFormat> GLES3_FORMATS = {
         .blockSize     = {2, 1},
     },
     {
-        .drmFormat     = DRM_FORMAT_R8,
-        .bytesPerBlock = 1,
+        .drmFormat        = DRM_FORMAT_R8,
+        .glInternalFormat = GL_R8,
+        .glFormat         = GL_RED,
+        .glType           = GL_UNSIGNED_BYTE,
+        .bytesPerBlock    = 1,
+        .swizzle          = {SWIZZLE_R001},
     },
     {
-        .drmFormat     = DRM_FORMAT_GR88,
-        .bytesPerBlock = 2,
+        .drmFormat        = DRM_FORMAT_GR88,
+        .glInternalFormat = GL_RG8,
+        .glFormat         = GL_RG,
+        .glType           = GL_UNSIGNED_BYTE,
+        .bytesPerBlock    = 2,
+        .swizzle          = {SWIZZLE_RG01},
     },
     {
-        .drmFormat     = DRM_FORMAT_RGB888,
-        .bytesPerBlock = 3,
-    },
-    {
-        .drmFormat     = DRM_FORMAT_BGR888,
-        .bytesPerBlock = 3,
-    },
-    {
-        .drmFormat     = DRM_FORMAT_RGBX4444,
-        .bytesPerBlock = 2,
+        .drmFormat        = DRM_FORMAT_RGB888,
+        .glInternalFormat = GL_RGB8,
+        .glFormat         = GL_RGB,
+        .glType           = GL_UNSIGNED_BYTE,
+        .bytesPerBlock    = 3,
+        .swizzle          = {SWIZZLE_BGR1},
     },
 };
 
@@ -229,6 +261,26 @@ const SPixelFormat* NFormatUtils::getPixelFormatFromGL(uint32_t glFormat, uint32
     return nullptr;
 }
 
+bool NFormatUtils::isFormatYUV(uint32_t drmFormat) {
+    switch (drmFormat) {
+        case DRM_FORMAT_YUYV:
+        case DRM_FORMAT_YVYU:
+        case DRM_FORMAT_UYVY:
+        case DRM_FORMAT_VYUY:
+        case DRM_FORMAT_AYUV:
+        case DRM_FORMAT_NV12:
+        case DRM_FORMAT_NV21:
+        case DRM_FORMAT_NV16:
+        case DRM_FORMAT_NV61:
+        case DRM_FORMAT_YUV410:
+        case DRM_FORMAT_YUV411:
+        case DRM_FORMAT_YUV420:
+        case DRM_FORMAT_YUV422:
+        case DRM_FORMAT_YUV444: return true;
+        default: return false;
+    }
+}
+
 bool NFormatUtils::isFormatOpaque(DRMFormat drm) {
     const auto FMT = NFormatUtils::getPixelFormatFromDRM(drm);
     if (!FMT)