screencopy: fix detecting gl shm formats

ref #4029

3 files changed, 147 insertions, 187 deletions

diff --git a/src/protocols/Screencopy.cpp b/src/protocols/Screencopy.cpp
index 0913f02e..19744ac5 100644
--- a/src/protocols/Screencopy.cpp
+++ b/src/protocols/Screencopy.cpp
@@ -210,6 +210,7 @@ void CScreencopyProtocolManager::captureOutput(wl_client* client, wl_resource* r
     PFRAME->client = PCLIENT;
     PCLIENT->ref++;
 
+    g_pHyprRenderer->makeEGLCurrent();
     PFRAME->shmFormat = g_pHyprOpenGL->getPreferredReadFormat(PFRAME->pMonitor);
     if (PFRAME->shmFormat == DRM_FORMAT_INVALID) {
         Debug::log(ERR, "No format supported by renderer in capture output");
@@ -241,7 +242,7 @@ void CScreencopyProtocolManager::captureOutput(wl_client* client, wl_resource* r
     wlr_output_effective_resolution(PFRAME->pMonitor->output, &ow, &oh);
     PFRAME->box.transform(PFRAME->pMonitor->transform, ow, oh).scale(PFRAME->pMonitor->scale).round();
 
-    PFRAME->shmStride = (PSHMINFO->bpp / 8) * PFRAME->box.width;
+    PFRAME->shmStride = pixel_format_info_min_stride(PSHMINFO, PFRAME->box.w);
 
     zwlr_screencopy_frame_v1_send_buffer(PFRAME->resource, convert_drm_format_to_wl_shm(PFRAME->shmFormat), PFRAME->box.width, PFRAME->box.height, PFRAME->shmStride);
 
@@ -456,8 +457,13 @@ bool CScreencopyProtocolManager::copyFrameShm(SScreencopyFrame* frame, timespec*
 
     glBindFramebuffer(GL_READ_FRAMEBUFFER, fb.m_iFb);
 
-    const auto GLFORMAT = drmFormatToGL(frame->pMonitor->drmFormat);
-    const auto GLTYPE   = glFormatToType(GLFORMAT);
+    GLint glf, glt;
+    glGetIntegerv(GL_IMPLEMENTATION_COLOR_READ_FORMAT, &glf);
+    glGetIntegerv(GL_IMPLEMENTATION_COLOR_READ_TYPE, &glt);
+    if (glf == 0 || glt == 0) {
+        glf = drmFormatToGL(frame->pMonitor->drmFormat);
+        glt = glFormatToType(glf);
+    }
 
     g_pHyprRenderer->endRender();
 
@@ -465,7 +471,20 @@ bool CScreencopyProtocolManager::copyFrameShm(SScreencopyFrame* frame, timespec*
     g_pHyprOpenGL->m_RenderData.pMonitor = frame->pMonitor;
     fb.bind();
 
-    glReadPixels(0, 0, frame->box.w, frame->box.h, GL_RGBA, GLTYPE, data);
+    glPixelStorei(GL_PACK_ALIGNMENT, 1);
+
+    const auto                   FMT        = g_pHyprOpenGL->getPreferredReadFormat(frame->pMonitor);
+    const wlr_pixel_format_info* drmFmtWlr  = drm_get_pixel_format_info(FMT);
+    uint32_t                     packStride = pixel_format_info_min_stride(drmFmtWlr, frame->box.w);
+
+    if (packStride == stride) {
+        glReadPixels(frame->box.x, frame->box.y, frame->box.w, frame->box.h, glf, glt, data);
+    } else {
+        for (size_t i = 0; i < frame->box.h; ++i) {
+            uint32_t y = frame->box.x + i;
+            glReadPixels(frame->box.x, y, frame->box.w, 1, glf, glt, ((unsigned char*)data) + i * stride);
+        }
+    }
 
     g_pHyprOpenGL->m_RenderData.pMonitor = nullptr;
 
diff --git a/src/protocols/ToplevelExport.cpp b/src/protocols/ToplevelExport.cpp
index d68aa616..73c3ab06 100644
--- a/src/protocols/ToplevelExport.cpp
+++ b/src/protocols/ToplevelExport.cpp
@@ -177,6 +177,7 @@ void CToplevelExportProtocolManager::captureToplevel(wl_client* client, wl_resou
 
     const auto PMONITOR = g_pCompositor->getMonitorFromID(PFRAME->pWindow->m_iMonitorID);
 
+    g_pHyprRenderer->makeEGLCurrent();
     PFRAME->shmFormat = g_pHyprOpenGL->getPreferredReadFormat(PFRAME->pMonitor);
     if (PFRAME->shmFormat == DRM_FORMAT_INVALID) {
         Debug::log(ERR, "No format supported by renderer in capture toplevel");
@@ -204,7 +205,7 @@ void CToplevelExportProtocolManager::captureToplevel(wl_client* client, wl_resou
     wlr_output_effective_resolution(PMONITOR->output, &ow, &oh);
     PFRAME->box.transform(PMONITOR->transform, ow, oh).round();
 
-    PFRAME->shmStride = (PSHMINFO->bpp / 8) * PFRAME->box.width;
+    PFRAME->shmStride = pixel_format_info_min_stride(PSHMINFO, PFRAME->box.w);
 
     hyprland_toplevel_export_frame_v1_send_buffer(PFRAME->resource, convert_drm_format_to_wl_shm(PFRAME->shmFormat), PFRAME->box.width, PFRAME->box.height, PFRAME->shmStride);
 
@@ -386,12 +387,19 @@ bool CToplevelExportProtocolManager::copyFrameShm(SScreencopyFrame* frame, times
     if (frame->overlayCursor)
         g_pHyprRenderer->renderSoftwareCursors(PMONITOR, fakeDamage, g_pInputManager->getMouseCoordsInternal() - frame->pWindow->m_vRealPosition.vec());
 
-    const auto GLFORMAT = drmFormatToGL(frame->pMonitor->drmFormat);
-    const auto GLTYPE   = glFormatToType(GLFORMAT);
+    GLint glf, glt;
+    glGetIntegerv(GL_IMPLEMENTATION_COLOR_READ_FORMAT, &glf);
+    glGetIntegerv(GL_IMPLEMENTATION_COLOR_READ_TYPE, &glt);
+    if (glf == 0 || glt == 0) {
+        glf = drmFormatToGL(frame->pMonitor->drmFormat);
+        glt = glFormatToType(glf);
+    }
 
     g_pHyprOpenGL->m_RenderData.mainFB->bind();
 
-    glReadPixels(0, 0, frame->box.width, frame->box.height, GL_RGBA, GLTYPE, data);
+    glPixelStorei(GL_PACK_ALIGNMENT, 1);
+
+    glReadPixels(0, 0, frame->box.width, frame->box.height, glf, glt, data);
 
     g_pHyprRenderer->endRender();
 
diff --git a/src/protocols/ToplevelExportWlrFuncs.hpp b/src/protocols/ToplevelExportWlrFuncs.hpp
index b580d3f0..b6f387e8 100644
--- a/src/protocols/ToplevelExportWlrFuncs.hpp
+++ b/src/protocols/ToplevelExportWlrFuncs.hpp
@@ -11,8 +11,10 @@ struct wlr_pixel_format_info {
 	 */
     uint32_t opaque_substitute;
 
-    /* Bits per pixels */
-    uint32_t bpp;
+    /* Bytes per block (including padding) */
+    uint32_t bytes_per_block;
+    /* Size of a block in pixels (zero for 1×1) */
+    uint32_t block_width, block_height;
 
     /* True if the format has an alpha channel */
     bool has_alpha;
@@ -20,143 +22,171 @@ struct wlr_pixel_format_info {
 
 static const struct wlr_pixel_format_info pixel_format_info[] = {
     {
-        .drm_format        = DRM_FORMAT_XRGB8888,
-        .opaque_substitute = DRM_FORMAT_INVALID,
-        .bpp               = 32,
-        .has_alpha         = false,
+        .drm_format      = DRM_FORMAT_XRGB8888,
+        .bytes_per_block = 4,
     },
     {
         .drm_format        = DRM_FORMAT_ARGB8888,
         .opaque_substitute = DRM_FORMAT_XRGB8888,
-        .bpp               = 32,
+        .bytes_per_block   = 4,
         .has_alpha         = true,
     },
     {
-        .drm_format        = DRM_FORMAT_XBGR8888,
-        .opaque_substitute = DRM_FORMAT_INVALID,
-        .bpp               = 32,
-        .has_alpha         = false,
+        .drm_format      = DRM_FORMAT_XBGR8888,
+        .bytes_per_block = 4,
     },
     {
         .drm_format        = DRM_FORMAT_ABGR8888,
         .opaque_substitute = DRM_FORMAT_XBGR8888,
-        .bpp               = 32,
+        .bytes_per_block   = 4,
         .has_alpha         = true,
     },
     {
-        .drm_format        = DRM_FORMAT_RGBX8888,
-        .opaque_substitute = DRM_FORMAT_INVALID,
-        .bpp               = 32,
-        .has_alpha         = false,
+        .drm_format      = DRM_FORMAT_RGBX8888,
+        .bytes_per_block = 4,
     },
     {
         .drm_format        = DRM_FORMAT_RGBA8888,
         .opaque_substitute = DRM_FORMAT_RGBX8888,
-        .bpp               = 32,
+        .bytes_per_block   = 4,
         .has_alpha         = true,
     },
     {
-        .drm_format        = DRM_FORMAT_BGRX8888,
-        .opaque_substitute = DRM_FORMAT_INVALID,
-        .bpp               = 32,
-        .has_alpha         = false,
+        .drm_format      = DRM_FORMAT_BGRX8888,
+        .bytes_per_block = 4,
     },
     {
         .drm_format        = DRM_FORMAT_BGRA8888,
         .opaque_substitute = DRM_FORMAT_BGRX8888,
-        .bpp               = 32,
+        .bytes_per_block   = 4,
         .has_alpha         = true,
     },
     {
-        .drm_format        = DRM_FORMAT_BGR888,
-        .opaque_substitute = DRM_FORMAT_INVALID,
-        .bpp               = 24,
-        .has_alpha         = false,
+        .drm_format      = DRM_FORMAT_R8,
+        .bytes_per_block = 1,
     },
     {
-        .drm_format        = DRM_FORMAT_RGBX4444,
-        .opaque_substitute = DRM_FORMAT_INVALID,
-        .bpp               = 16,
-        .has_alpha         = false,
+        .drm_format      = DRM_FORMAT_GR88,
+        .bytes_per_block = 2,
+    },
+    {
+        .drm_format      = DRM_FORMAT_RGB888,
+        .bytes_per_block = 3,
+    },
+    {
+        .drm_format      = DRM_FORMAT_BGR888,
+        .bytes_per_block = 3,
+    },
+    {
+        .drm_format      = DRM_FORMAT_RGBX4444,
+        .bytes_per_block = 2,
     },
     {
         .drm_format        = DRM_FORMAT_RGBA4444,
         .opaque_substitute = DRM_FORMAT_RGBX4444,
-        .bpp               = 16,
+        .bytes_per_block   = 2,
         .has_alpha         = true,
     },
     {
-        .drm_format        = DRM_FORMAT_RGBX5551,
-        .opaque_substitute = DRM_FORMAT_INVALID,
-        .bpp               = 16,
-        .has_alpha         = false,
+        .drm_format      = DRM_FORMAT_BGRX4444,
+        .bytes_per_block = 2,
+    },
+    {
+        .drm_format        = DRM_FORMAT_BGRA4444,
+        .opaque_substitute = DRM_FORMAT_BGRX4444,
+        .bytes_per_block   = 2,
+        .has_alpha         = true,
+    },
+    {
+        .drm_format      = DRM_FORMAT_RGBX5551,
+        .bytes_per_block = 2,
     },
     {
         .drm_format        = DRM_FORMAT_RGBA5551,
         .opaque_substitute = DRM_FORMAT_RGBX5551,
-        .bpp               = 16,
+        .bytes_per_block   = 2,
+        .has_alpha         = true,
+    },
+    {
+        .drm_format      = DRM_FORMAT_BGRX5551,
+        .bytes_per_block = 2,
+    },
+    {
+        .drm_format        = DRM_FORMAT_BGRA5551,
+        .opaque_substitute = DRM_FORMAT_BGRX5551,
+        .bytes_per_block   = 2,
+        .has_alpha         = true,
+    },
+    {
+        .drm_format      = DRM_FORMAT_XRGB1555,
+        .bytes_per_block = 2,
+    },
+    {
+        .drm_format        = DRM_FORMAT_ARGB1555,
+        .opaque_substitute = DRM_FORMAT_XRGB1555,
+        .bytes_per_block   = 2,
         .has_alpha         = true,
     },
     {
-        .drm_format        = DRM_FORMAT_RGB565,
-        .opaque_substitute = DRM_FORMAT_INVALID,
-        .bpp               = 16,
-        .has_alpha         = false,
+        .drm_format      = DRM_FORMAT_RGB565,
+        .bytes_per_block = 2,
     },
     {
-        .drm_format        = DRM_FORMAT_BGR565,
-        .opaque_substitute = DRM_FORMAT_INVALID,
-        .bpp               = 16,
-        .has_alpha         = false,
+        .drm_format      = DRM_FORMAT_BGR565,
+        .bytes_per_block = 2,
     },
     {
-        .drm_format        = DRM_FORMAT_XRGB2101010,
-        .opaque_substitute = DRM_FORMAT_INVALID,
-        .bpp               = 32,
-        .has_alpha         = false,
+        .drm_format      = DRM_FORMAT_XRGB2101010,
+        .bytes_per_block = 4,
     },
     {
         .drm_format        = DRM_FORMAT_ARGB2101010,
         .opaque_substitute = DRM_FORMAT_XRGB2101010,
-        .bpp               = 32,
+        .bytes_per_block   = 4,
         .has_alpha         = true,
     },
     {
-        .drm_format        = DRM_FORMAT_XBGR2101010,
-        .opaque_substitute = DRM_FORMAT_INVALID,
-        .bpp               = 32,
-        .has_alpha         = false,
+        .drm_format      = DRM_FORMAT_XBGR2101010,
+        .bytes_per_block = 4,
     },
     {
         .drm_format        = DRM_FORMAT_ABGR2101010,
         .opaque_substitute = DRM_FORMAT_XBGR2101010,
-        .bpp               = 32,
+        .bytes_per_block   = 4,
         .has_alpha         = true,
     },
     {
-        .drm_format        = DRM_FORMAT_XBGR16161616F,
-        .opaque_substitute = DRM_FORMAT_INVALID,
-        .bpp               = 64,
-        .has_alpha         = false,
+        .drm_format      = DRM_FORMAT_XBGR16161616F,
+        .bytes_per_block = 8,
     },
     {
         .drm_format        = DRM_FORMAT_ABGR16161616F,
         .opaque_substitute = DRM_FORMAT_XBGR16161616F,
-        .bpp               = 64,
+        .bytes_per_block   = 8,
         .has_alpha         = true,
     },
     {
-        .drm_format        = DRM_FORMAT_XBGR16161616,
-        .opaque_substitute = DRM_FORMAT_INVALID,
-        .bpp               = 64,
-        .has_alpha         = false,
+        .drm_format      = DRM_FORMAT_XBGR16161616,
+        .bytes_per_block = 8,
     },
     {
         .drm_format        = DRM_FORMAT_ABGR16161616,
         .opaque_substitute = DRM_FORMAT_XBGR16161616,
-        .bpp               = 64,
+        .bytes_per_block   = 8,
         .has_alpha         = true,
     },
+    {
+        .drm_format      = DRM_FORMAT_YVYU,
+        .bytes_per_block = 4,
+        .block_width     = 2,
+        .block_height    = 1,
+    },
+    {
+        .drm_format      = DRM_FORMAT_VYUY,
+        .bytes_per_block = 4,
+        .block_width     = 2,
+        .block_height    = 1,
+    },
 };
 
 static const size_t                        pixel_format_info_size = sizeof(pixel_format_info) / sizeof(pixel_format_info[0]);
@@ -187,124 +217,27 @@ static enum wl_shm_format convert_drm_format_to_wl_shm(uint32_t fmt) {
     }
 }
 
-struct wlr_gles2_pixel_format {
-    uint32_t drm_format;
-    // optional field, if empty then internalformat = format
-    GLint gl_internalformat;
-    GLint gl_format, gl_type;
-    bool  has_alpha;
-};
+static uint32_t pixel_format_info_pixels_per_block(const struct wlr_pixel_format_info* info) {
+    uint32_t pixels = info->block_width * info->block_height;
+    return pixels > 0 ? pixels : 1;
+}
 
-static const struct wlr_gles2_pixel_format formats[] = {
-    {
-        .drm_format = DRM_FORMAT_ARGB8888,
-        .gl_format  = GL_BGRA_EXT,
-        .gl_type    = GL_UNSIGNED_BYTE,
-        .has_alpha  = true,
-    },
-    {
-        .drm_format = DRM_FORMAT_XRGB8888,
-        .gl_format  = GL_BGRA_EXT,
-        .gl_type    = GL_UNSIGNED_BYTE,
-        .has_alpha  = false,
-    },
-    {
-        .drm_format = DRM_FORMAT_XBGR8888,
-        .gl_format  = GL_RGBA,
-        .gl_type    = GL_UNSIGNED_BYTE,
-        .has_alpha  = false,
-    },
-    {
-        .drm_format = DRM_FORMAT_ABGR8888,
-        .gl_format  = GL_RGBA,
-        .gl_type    = GL_UNSIGNED_BYTE,
-        .has_alpha  = true,
-    },
-    {
-        .drm_format = DRM_FORMAT_BGR888,
-        .gl_format  = GL_RGB,
-        .gl_type    = GL_UNSIGNED_BYTE,
-        .has_alpha  = false,
-    },
-#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-    {
-        .drm_format = DRM_FORMAT_RGBX4444,
-        .gl_format  = GL_RGBA,
-        .gl_type    = GL_UNSIGNED_SHORT_4_4_4_4,
-        .has_alpha  = false,
-    },
-    {
-        .drm_format = DRM_FORMAT_RGBA4444,
-        .gl_format  = GL_RGBA,
-        .gl_type    = GL_UNSIGNED_SHORT_4_4_4_4,
-        .has_alpha  = true,
-    },
-    {
-        .drm_format = DRM_FORMAT_RGBX5551,
-        .gl_format  = GL_RGBA,
-        .gl_type    = GL_UNSIGNED_SHORT_5_5_5_1,
-        .has_alpha  = false,
-    },
-    {
-        .drm_format = DRM_FORMAT_RGBA5551,
-        .gl_format  = GL_RGBA,
-        .gl_type    = GL_UNSIGNED_SHORT_5_5_5_1,
-        .has_alpha  = true,
-    },
-    {
-        .drm_format = DRM_FORMAT_RGB565,
-        .gl_format  = GL_RGB,
-        .gl_type    = GL_UNSIGNED_SHORT_5_6_5,
-        .has_alpha  = false,
-    },
-    {
-        .drm_format = DRM_FORMAT_XBGR2101010,
-        .gl_format  = GL_RGBA,
-        .gl_type    = GL_UNSIGNED_INT_2_10_10_10_REV_EXT,
-        .has_alpha  = false,
-    },
-    {
-        .drm_format = DRM_FORMAT_ABGR2101010,
-        .gl_format  = GL_RGBA,
-        .gl_type    = GL_UNSIGNED_INT_2_10_10_10_REV_EXT,
-        .has_alpha  = true,
-    },
-    {
-        .drm_format = DRM_FORMAT_XBGR16161616F,
-        .gl_format  = GL_RGBA,
-        .gl_type    = GL_HALF_FLOAT_OES,
-        .has_alpha  = false,
-    },
-    {
-        .drm_format = DRM_FORMAT_ABGR16161616F,
-        .gl_format  = GL_RGBA,
-        .gl_type    = GL_HALF_FLOAT_OES,
-        .has_alpha  = true,
-    },
-    {
-        .drm_format        = DRM_FORMAT_XBGR16161616,
-        .gl_internalformat = GL_RGBA16_EXT,
-        .gl_format         = GL_RGBA,
-        .gl_type           = GL_UNSIGNED_SHORT,
-        .has_alpha         = false,
-    },
-    {
-        .drm_format        = DRM_FORMAT_ABGR16161616,
-        .gl_internalformat = GL_RGBA16_EXT,
-        .gl_format         = GL_RGBA,
-        .gl_type           = GL_UNSIGNED_SHORT,
-        .has_alpha         = true,
-    },
-#endif
-};
+static int32_t div_round_up(int32_t dividend, int32_t divisor) {
+    int32_t quotient = dividend / divisor;
+    if (dividend % divisor != 0) {
+        quotient++;
+    }
+    return quotient;
+}
 
-inline const struct wlr_gles2_pixel_format* gles2FromDRM(uint32_t fmt) {
-    for (size_t i = 0; i < sizeof(formats) / sizeof(*formats); ++i) {
-        if (formats[i].drm_format == fmt) {
-            return &formats[i];
-        }
+static int32_t pixel_format_info_min_stride(const wlr_pixel_format_info* fmt, int32_t width) {
+    int32_t pixels_per_block = (int32_t)pixel_format_info_pixels_per_block(fmt);
+    int32_t bytes_per_block  = (int32_t)fmt->bytes_per_block;
+    if (width > INT32_MAX / bytes_per_block) {
+        wlr_log(WLR_DEBUG, "Invalid width %d (overflow)", width);
+        return 0;
     }
-    return NULL;
+    return div_round_up(width * bytes_per_block, pixels_per_block);
 }
 
 #endif
\ No newline at end of file