protocols/toplevelExport: Support transparency in toplevel export (#12824)

3 files changed, 21 insertions, 4 deletions

diff --git a/src/helpers/Format.cpp b/src/helpers/Format.cpp
index 0054c25a..a4efb948 100644
--- a/src/helpers/Format.cpp
+++ b/src/helpers/Format.cpp
@@ -326,3 +326,17 @@ std::string NFormatUtils::drmModifierName(uint64_t mod) {
     free(n); // NOLINT(cppcoreguidelines-no-malloc,-warnings-as-errors)
     return name;
 }
+
+DRMFormat NFormatUtils::alphaFormat(DRMFormat prevFormat) {
+    switch (prevFormat) {
+        case DRM_FORMAT_XRGB8888: return DRM_FORMAT_ARGB8888;
+        case DRM_FORMAT_XBGR8888: return DRM_FORMAT_ABGR8888;
+        case DRM_FORMAT_BGRX8888: return DRM_FORMAT_BGRA8888;
+        case DRM_FORMAT_RGBX8888: return DRM_FORMAT_RGBA8888;
+        case DRM_FORMAT_XRGB2101010: return DRM_FORMAT_ARGB2101010;
+        case DRM_FORMAT_XBGR2101010: return DRM_FORMAT_ABGR2101010;
+        case DRM_FORMAT_RGBX1010102: return DRM_FORMAT_RGBA1010102;
+        case DRM_FORMAT_BGRX1010102: return DRM_FORMAT_BGRA1010102;
+        default: return 0;
+    }
+}
diff --git a/src/helpers/Format.hpp b/src/helpers/Format.hpp
index 917fe3cb..ce5d8b40 100644
--- a/src/helpers/Format.hpp
+++ b/src/helpers/Format.hpp
@@ -57,4 +57,5 @@ namespace NFormatUtils {
     uint32_t            glFormatToType(uint32_t gl);
     std::string         drmFormatName(DRMFormat drm);
     std::string         drmModifierName(uint64_t mod);
+    DRMFormat           alphaFormat(DRMFormat prevFormat);
 };
diff --git a/src/protocols/ToplevelExport.cpp b/src/protocols/ToplevelExport.cpp
index 7b7c0a31..b223f778 100644
--- a/src/protocols/ToplevelExport.cpp
+++ b/src/protocols/ToplevelExport.cpp
@@ -100,7 +100,9 @@ CToplevelExportFrame::CToplevelExportFrame(SP<CHyprlandToplevelExportFrameV1> re
 
     g_pHyprRenderer->makeEGLCurrent();
 
-    m_shmFormat = g_pHyprOpenGL->getPreferredReadFormat(PMONITOR);
+    m_shmFormat = NFormatUtils::alphaFormat(g_pHyprOpenGL->getPreferredReadFormat(PMONITOR));
+    LOGM(Log::DEBUG, "Format {:x}", m_shmFormat);
+    //m_shmFormat = NFormatUtils::alphaFormat(m_shmFormat);
     if UNLIKELY (m_shmFormat == DRM_FORMAT_INVALID) {
         LOGM(Log::ERR, "No format supported by renderer in capture toplevel");
         m_resource->sendFailed();
@@ -114,7 +116,7 @@ CToplevelExportFrame::CToplevelExportFrame(SP<CHyprlandToplevelExportFrameV1> re
         return;
     }
 
-    m_dmabufFormat = g_pHyprOpenGL->getPreferredReadFormat(PMONITOR);
+    m_dmabufFormat = NFormatUtils::alphaFormat(g_pHyprOpenGL->getPreferredReadFormat(PMONITOR));
 
     m_box = {0, 0, sc<int>(m_window->m_realSize->value().x * PMONITOR->m_scale), sc<int>(m_window->m_realSize->value().y * PMONITOR->m_scale)};
 
@@ -253,7 +255,7 @@ bool CToplevelExportFrame::copyShm(const Time::steady_tp& now) {
     if (!g_pHyprRenderer->beginRender(PMONITOR, fakeDamage, RENDER_MODE_FULL_FAKE, nullptr, &outFB))
         return false;
 
-    g_pHyprOpenGL->clear(CHyprColor(0, 0, 0, 1.0));
+    g_pHyprOpenGL->clear(CHyprColor(0, 0, 0, 0));
 
     // render client at 0,0
     if (PERM == PERMISSION_RULE_ALLOW_MODE_ALLOW) {
@@ -356,7 +358,7 @@ bool CToplevelExportFrame::copyDmabuf(const Time::steady_tp& now) {
     if (!g_pHyprRenderer->beginRender(PMONITOR, fakeDamage, RENDER_MODE_TO_BUFFER, m_buffer.m_buffer))
         return false;
 
-    g_pHyprOpenGL->clear(CHyprColor(0, 0, 0, 1.0));
+    g_pHyprOpenGL->clear(CHyprColor(0, 0, 0, 0));
     if (PERM == PERMISSION_RULE_ALLOW_MODE_ALLOW) {
         if (!m_window->m_ruleApplicator->noScreenShare().valueOrDefault()) {
             g_pHyprRenderer->m_bBlockSurfaceFeedback = g_pHyprRenderer->shouldRenderWindow(m_window); // block the feedback to avoid spamming the surface if it's visible