screencopy: add force 8 bit to fix 10b screensharing (#11623)

ref https://github.com/hyprwm/xdg-desktop-portal-hyprland/issues/270

6 files changed, 22 insertions, 3 deletions

diff --git a/src/config/ConfigDescriptions.hpp b/src/config/ConfigDescriptions.hpp
index 0308dde0..ee8fa240 100644
--- a/src/config/ConfigDescriptions.hpp
+++ b/src/config/ConfigDescriptions.hpp
@@ -1321,6 +1321,12 @@ inline static const std::vector<SConfigOptionDescription> CONFIG_OPTIONS = {
         .type        = CONFIG_OPTION_INT,
         .data        = SConfigOptionDescription::SRangeData{1, 1, 10},
     },
+    SConfigOptionDescription{
+        .value       = "misc:screencopy_force_8b",
+        .description = "forces 8 bit screencopy (fixes apps that don't understand 10bit)",
+        .type        = CONFIG_OPTION_BOOL,
+        .data        = SConfigOptionDescription::SBoolData{true},
+    },
 
     /*
      * binds:
diff --git a/src/config/ConfigManager.cpp b/src/config/ConfigManager.cpp
index 16183498..ac789c19 100644
--- a/src/config/ConfigManager.cpp
+++ b/src/config/ConfigManager.cpp
@@ -521,6 +521,7 @@ CConfigManager::CConfigManager() {
     registerConfigVar("misc:lockdead_screen_delay", Hyprlang::INT{1000});
     registerConfigVar("misc:enable_anr_dialog", Hyprlang::INT{1});
     registerConfigVar("misc:anr_missed_pings", Hyprlang::INT{1});
+    registerConfigVar("misc:screencopy_force_8b", Hyprlang::INT{1});
 
     registerConfigVar("group:insert_after_current", Hyprlang::INT{1});
     registerConfigVar("group:focus_removed_window", Hyprlang::INT{1});
diff --git a/src/protocols/Screencopy.cpp b/src/protocols/Screencopy.cpp
index 804e144c..3a2e4b4a 100644
--- a/src/protocols/Screencopy.cpp
+++ b/src/protocols/Screencopy.cpp
@@ -60,7 +60,7 @@ CScreencopyFrame::CScreencopyFrame(SP<CZwlrScreencopyFrameV1> resource_, int32_t
         return;
     }
 
-    m_dmabufFormat = m_monitor->m_output->state->state().drmFormat;
+    m_dmabufFormat = g_pHyprOpenGL->getPreferredReadFormat(m_monitor.lock());
 
     if (box_.width == 0 && box_.height == 0)
         m_box = {0, 0, sc<int>(m_monitor->m_size.x), sc<int>(m_monitor->m_size.y)};
diff --git a/src/protocols/Screencopy.hpp b/src/protocols/Screencopy.hpp
index a44628fd..54b0d28c 100644
--- a/src/protocols/Screencopy.hpp
+++ b/src/protocols/Screencopy.hpp
@@ -108,6 +108,8 @@ class CScreencopyProtocol : public IWaylandProtocol {
     bool                               copyFrameDmabuf(CScreencopyFrame* frame);
     bool                               copyFrameShm(CScreencopyFrame* frame, const Time::steady_tp& now);
 
+    uint32_t                           drmFormatForMonitor(PHLMONITOR pMonitor);
+
     friend class CScreencopyFrame;
     friend class CScreencopyClient;
 };
diff --git a/src/protocols/ToplevelExport.cpp b/src/protocols/ToplevelExport.cpp
index 37e90b6f..eb0a39aa 100644
--- a/src/protocols/ToplevelExport.cpp
+++ b/src/protocols/ToplevelExport.cpp
@@ -114,7 +114,7 @@ CToplevelExportFrame::CToplevelExportFrame(SP<CHyprlandToplevelExportFrameV1> re
         return;
     }
 
-    m_dmabufFormat = PMONITOR->m_output->state->state().drmFormat;
+    m_dmabufFormat = g_pHyprOpenGL->getPreferredReadFormat(PMONITOR);
 
     m_box = {0, 0, sc<int>(m_window->m_realSize->value().x * PMONITOR->m_scale), sc<int>(m_window->m_realSize->value().y * PMONITOR->m_scale)};
 
diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index 0780adf9..d7ea5842 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -3113,7 +3113,17 @@ void CHyprOpenGLImpl::setCapStatus(int cap, bool status) {
 }
 
 uint32_t CHyprOpenGLImpl::getPreferredReadFormat(PHLMONITOR pMonitor) {
-    return pMonitor->m_output->state->state().drmFormat;
+    static const auto PFORCE8BIT = CConfigValue<Hyprlang::INT>("misc:screencopy_force_8b");
+
+    if (!*PFORCE8BIT)
+        return pMonitor->m_output->state->state().drmFormat;
+
+    auto fmt = pMonitor->m_output->state->state().drmFormat;
+
+    if (fmt == DRM_FORMAT_BGRA1010102 || fmt == DRM_FORMAT_ARGB2101010 || fmt == DRM_FORMAT_XRGB2101010 || fmt == DRM_FORMAT_BGRX1010102)
+        return DRM_FORMAT_XRGB8888;
+
+    return fmt;
 }
 
 bool CHyprOpenGLImpl::explicitSyncSupported() {