screencopy: fix shm exports with 10-bit

fixes #4019

7 files changed, 37 insertions, 30 deletions

diff --git a/src/helpers/MiscFunctions.cpp b/src/helpers/MiscFunctions.cpp
index 71263748..16650e1b 100644
--- a/src/helpers/MiscFunctions.cpp
+++ b/src/helpers/MiscFunctions.cpp
@@ -781,4 +781,14 @@ uint32_t drmFormatToGL(uint32_t drm) {
     }
     UNREACHABLE();
     return GL_RGBA;
+}
+
+uint32_t glFormatToType(uint32_t gl) {
+    return gl != GL_RGBA ?
+#ifdef GLES2
+        GL_UNSIGNED_INT_2_10_10_10_REV_EXT :
+#else
+        GL_UNSIGNED_INT_2_10_10_10_REV :
+#endif
+        GL_UNSIGNED_BYTE;
 }
\ No newline at end of file
diff --git a/src/helpers/MiscFunctions.hpp b/src/helpers/MiscFunctions.hpp
index 90479121..d8070c76 100644
--- a/src/helpers/MiscFunctions.hpp
+++ b/src/helpers/MiscFunctions.hpp
@@ -34,6 +34,7 @@ std::string                      replaceInString(std::string subject, const std:
 std::vector<SCallstackFrameInfo> getBacktrace();
 void                             throwError(const std::string& err);
 uint32_t                         drmFormatToGL(uint32_t drm);
+uint32_t                         glFormatToType(uint32_t gl);
 
 template <typename... Args>
 [[deprecated("use std::format instead")]] std::string getFormat(std::format_string<Args...> fmt, Args&&... args) {
diff --git a/src/protocols/Screencopy.cpp b/src/protocols/Screencopy.cpp
index 7240fd37..6c7178c0 100644
--- a/src/protocols/Screencopy.cpp
+++ b/src/protocols/Screencopy.cpp
@@ -210,7 +210,7 @@ void CScreencopyProtocolManager::captureOutput(wl_client* client, wl_resource* r
     PFRAME->client = PCLIENT;
     PCLIENT->ref++;
 
-    PFRAME->shmFormat = g_pHyprRenderer->isNvidia() ? DRM_FORMAT_XRGB8888 : PFRAME->pMonitor->drmFormat;
+    PFRAME->shmFormat = g_pHyprOpenGL->getPreferredReadFormat(PFRAME->pMonitor);
     if (PFRAME->shmFormat == DRM_FORMAT_INVALID) {
         Debug::log(ERR, "No format supported by renderer in capture output");
         zwlr_screencopy_frame_v1_send_failed(PFRAME->resource);
@@ -441,7 +441,7 @@ bool CScreencopyProtocolManager::copyFrameShm(SScreencopyFrame* frame, timespec*
     g_pHyprRenderer->makeEGLCurrent();
 
     CFramebuffer fb;
-    fb.alloc(frame->box.w, frame->box.h); // 8bit only
+    fb.alloc(frame->box.w, frame->box.h, frame->pMonitor->drmFormat);
 
     if (!g_pHyprRenderer->beginRender(frame->pMonitor, fakeDamage, RENDER_MODE_FULL_FAKE, nullptr, &fb)) {
         wlr_texture_destroy(sourceTex);
@@ -456,13 +456,8 @@ bool CScreencopyProtocolManager::copyFrameShm(SScreencopyFrame* frame, timespec*
 
     glBindFramebuffer(GL_READ_FRAMEBUFFER, fb.m_iFb);
 
-    const auto PFORMAT = gles2FromDRM(format);
-    if (!PFORMAT) {
-        Debug::log(ERR, "[sc] Cannot read pixels, unsupported format {:x}", (uintptr_t)PFORMAT);
-        wlr_texture_destroy(sourceTex);
-        wlr_buffer_end_data_ptr_access(frame->buffer);
-        return false;
-    }
+    const auto GLFORMAT = drmFormatToGL(frame->pMonitor->drmFormat);
+    const auto GLTYPE   = glFormatToType(GLFORMAT);
 
     g_pHyprRenderer->endRender();
 
@@ -470,7 +465,7 @@ bool CScreencopyProtocolManager::copyFrameShm(SScreencopyFrame* frame, timespec*
     g_pHyprOpenGL->m_RenderData.pMonitor = frame->pMonitor;
     fb.bind();
 
-    glReadPixels(0, 0, frame->box.w, frame->box.h, PFORMAT->gl_format, PFORMAT->gl_type, data);
+    glReadPixels(0, 0, frame->box.w, frame->box.h, GL_RGBA, GLTYPE, data);
 
     g_pHyprOpenGL->m_RenderData.pMonitor = nullptr;
 
diff --git a/src/protocols/ToplevelExport.cpp b/src/protocols/ToplevelExport.cpp
index aa035b6e..d68aa616 100644
--- a/src/protocols/ToplevelExport.cpp
+++ b/src/protocols/ToplevelExport.cpp
@@ -177,7 +177,7 @@ void CToplevelExportProtocolManager::captureToplevel(wl_client* client, wl_resou
 
     const auto PMONITOR = g_pCompositor->getMonitorFromID(PFRAME->pWindow->m_iMonitorID);
 
-    PFRAME->shmFormat = g_pHyprRenderer->isNvidia() ? DRM_FORMAT_XRGB8888 : PMONITOR->drmFormat;
+    PFRAME->shmFormat = g_pHyprOpenGL->getPreferredReadFormat(PFRAME->pMonitor);
     if (PFRAME->shmFormat == DRM_FORMAT_INVALID) {
         Debug::log(ERR, "No format supported by renderer in capture toplevel");
         hyprland_toplevel_export_frame_v1_send_failed(resource);
@@ -366,7 +366,7 @@ bool CToplevelExportProtocolManager::copyFrameShm(SScreencopyFrame* frame, times
     g_pHyprRenderer->makeEGLCurrent();
 
     CFramebuffer outFB;
-    outFB.alloc(PMONITOR->vecPixelSize.x, PMONITOR->vecPixelSize.y);
+    outFB.alloc(PMONITOR->vecPixelSize.x, PMONITOR->vecPixelSize.y, PMONITOR->drmFormat);
 
     if (!g_pHyprRenderer->beginRender(PMONITOR, fakeDamage, RENDER_MODE_FULL_FAKE, nullptr, &outFB)) {
         wlr_buffer_end_data_ptr_access(frame->buffer);
@@ -386,20 +386,12 @@ bool CToplevelExportProtocolManager::copyFrameShm(SScreencopyFrame* frame, times
     if (frame->overlayCursor)
         g_pHyprRenderer->renderSoftwareCursors(PMONITOR, fakeDamage, g_pInputManager->getMouseCoordsInternal() - frame->pWindow->m_vRealPosition.vec());
 
-    // copy pixels
-    const auto PFORMAT = gles2FromDRM(format);
-    if (!PFORMAT) {
-        Debug::log(ERR, "[toplevel_export] Cannot read pixels, unsupported format {:x}", (uintptr_t)PFORMAT);
-        g_pHyprRenderer->endRender();
-        wlr_buffer_end_data_ptr_access(frame->buffer);
-        if (frame->overlayCursor)
-            wlr_output_lock_software_cursors(PMONITOR->output, false);
-        return false;
-    }
+    const auto GLFORMAT = drmFormatToGL(frame->pMonitor->drmFormat);
+    const auto GLTYPE   = glFormatToType(GLFORMAT);
 
     g_pHyprOpenGL->m_RenderData.mainFB->bind();
 
-    glReadPixels(0, 0, frame->box.width, frame->box.height, PFORMAT->gl_format, PFORMAT->gl_type, data);
+    glReadPixels(0, 0, frame->box.width, frame->box.height, GL_RGBA, GLTYPE, data);
 
     g_pHyprRenderer->endRender();
 
diff --git a/src/render/Framebuffer.cpp b/src/render/Framebuffer.cpp
index 94b3f64a..eed23699 100644
--- a/src/render/Framebuffer.cpp
+++ b/src/render/Framebuffer.cpp
@@ -6,13 +6,7 @@ bool CFramebuffer::alloc(int w, int h, uint32_t drmFormat) {
     RASSERT((w > 1 && h > 1), "cannot alloc a FB with negative / zero size! (attempted {}x{})", w, h);
 
     uint32_t glFormat = drmFormatToGL(drmFormat);
-    uint32_t glType   = glFormat != GL_RGBA ?
-#ifdef GLES2
-        GL_UNSIGNED_INT_2_10_10_10_REV_EXT :
-#else
-        GL_UNSIGNED_INT_2_10_10_10_REV :
-#endif
-        GL_UNSIGNED_BYTE;
+    uint32_t glType   = glFormatToType(glFormat);
 
     if (m_iFb == (uint32_t)-1) {
         firstAlloc = true;
diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index dbdabdad..4eb45d8f 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -2064,3 +2064,16 @@ void CHyprOpenGLImpl::bindBackOnMain() {
 void CHyprOpenGLImpl::setMonitorTransformEnabled(bool enabled) {
     m_bEndFrame = !enabled;
 }
+
+uint32_t CHyprOpenGLImpl::getPreferredReadFormat(CMonitor* pMonitor) {
+    if (g_pHyprRenderer->isNvidia())
+        return DRM_FORMAT_XRGB8888;
+
+    if (pMonitor->drmFormat == DRM_FORMAT_XRGB8888)
+        return DRM_FORMAT_XRGB8888;
+    if (pMonitor->drmFormat == DRM_FORMAT_XBGR8888)
+        return DRM_FORMAT_XBGR8888;
+    if (pMonitor->drmFormat == DRM_FORMAT_XRGB2101010 || pMonitor->drmFormat == DRM_FORMAT_XBGR2101010)
+        return DRM_FORMAT_XBGR2101010;
+    return DRM_FORMAT_INVALID;
+}
diff --git a/src/render/OpenGL.hpp b/src/render/OpenGL.hpp
index f4661e1a..2830870b 100644
--- a/src/render/OpenGL.hpp
+++ b/src/render/OpenGL.hpp
@@ -158,6 +158,8 @@ class CHyprOpenGLImpl {
     void               renderOffToMain(CFramebuffer* off);
     void               bindBackOnMain();
 
+    uint32_t           getPreferredReadFormat(CMonitor* pMonitor);
+
     SCurrentRenderData m_RenderData;
 
     GLint              m_iCurrentOutputFb = 0;