renderer,config: add custom DRM modeline support (#2254)

This allows specifying custom display resolutions for the DRM backend.
This is useful for display overclocking, working around broken EDIDs,
etc. To use this feature, specify a modeline instead of a resolution
in the config, for example:

    monitor = DP-1, modeline 1071.101 3840 3848 3880 3920 2160 2263 2271 2277 +hsync -vsync, 0x0, 1

This example is a custom 3840x2160@120Hz mode with tightened timings.
I use it because the standard timings don't work with my monitor and GPU
combination (M28U with RX580).

The syntax is compatible with Sway and Xorg.

4 files changed, 103 insertions, 18 deletions

diff --git a/src/config/ConfigManager.cpp b/src/config/ConfigManager.cpp
index 27b42171..833e17e4 100644
--- a/src/config/ConfigManager.cpp
+++ b/src/config/ConfigManager.cpp
@@ -470,6 +470,58 @@ void CConfigManager::handleRawExec(const std::string& command, const std::string
     g_pKeybindManager->spawn(args);
 }
 
+static bool parseModeLine(const std::string& modeline, drmModeModeInfo& mode) {
+    auto args = CVarList(modeline, 0, 's');
+
+    auto keyword = args[0];
+    std::transform(keyword.begin(), keyword.end(), keyword.begin(), ::tolower);
+
+    if (keyword != "modeline")
+        return false;
+
+    if (args.size() < 10) {
+        Debug::log(ERR, "modeline parse error: expected at least 9 arguments, got %i", args.size() - 1);
+        return false;
+    }
+
+    int argno = 1;
+
+    mode.type        = DRM_MODE_TYPE_USERDEF;
+    mode.clock       = std::stof(args[argno++]) * 1000;
+    mode.hdisplay    = std::stoi(args[argno++]);
+    mode.hsync_start = std::stoi(args[argno++]);
+    mode.hsync_end   = std::stoi(args[argno++]);
+    mode.htotal      = std::stoi(args[argno++]);
+    mode.vdisplay    = std::stoi(args[argno++]);
+    mode.vsync_start = std::stoi(args[argno++]);
+    mode.vsync_end   = std::stoi(args[argno++]);
+    mode.vtotal      = std::stoi(args[argno++]);
+    mode.vrefresh    = mode.clock * 1000.0 * 1000.0 / mode.htotal / mode.vtotal;
+
+    static std::unordered_map<std::string, uint32_t> flagsmap = {
+        {"+hsync", DRM_MODE_FLAG_PHSYNC},
+        {"-hsync", DRM_MODE_FLAG_NHSYNC},
+        {"+vsync", DRM_MODE_FLAG_PVSYNC},
+        {"-vsync", DRM_MODE_FLAG_NVSYNC},
+    };
+
+    for (; argno < static_cast<int>(args.size()); argno++) {
+        auto key = args[argno];
+        std::transform(key.begin(), key.end(), key.begin(), ::tolower);
+
+        auto it = flagsmap.find(key);
+
+        if (it != flagsmap.end())
+            mode.flags |= it->second;
+        else
+            Debug::log(ERR, "invalid flag %s in modeline", it->first.c_str());
+    }
+
+    snprintf(mode.name, sizeof(mode.name), "%dx%d@%d", mode.hdisplay, mode.vdisplay, mode.vrefresh / 1000);
+
+    return true;
+}
+
 void CConfigManager::handleMonitor(const std::string& command, const std::string& args) {
 
     // get the monitor config
@@ -531,6 +583,9 @@ void CConfigManager::handleMonitor(const std::string& command, const std::string
         newrule.resolution = Vector2D(-1, -1);
     } else if (ARGS[1].find("highres") == 0) {
         newrule.resolution = Vector2D(-1, -2);
+    } else if (parseModeLine(ARGS[1], newrule.drmMode)) {
+        newrule.resolution  = Vector2D(newrule.drmMode.hdisplay, newrule.drmMode.vdisplay);
+        newrule.refreshRate = newrule.drmMode.vrefresh / 1000;
     } else {
         newrule.resolution.x = stoi(ARGS[1].substr(0, ARGS[1].find_first_of('x')));
         newrule.resolution.y = stoi(ARGS[1].substr(ARGS[1].find_first_of('x') + 1, ARGS[1].find_first_of('@')));
diff --git a/src/config/ConfigManager.hpp b/src/config/ConfigManager.hpp
index 1d35e9dd..ea824a32 100644
--- a/src/config/ConfigManager.hpp
+++ b/src/config/ConfigManager.hpp
@@ -11,6 +11,7 @@
 #include <algorithm>
 #include <regex>
 #include <optional>
+#include <xf86drmMode.h>
 #include "../Window.hpp"
 #include "../helpers/WLClasses.hpp"
 
@@ -44,6 +45,7 @@ struct SMonitorRule {
     wl_output_transform transform   = WL_OUTPUT_TRANSFORM_NORMAL;
     std::string         mirrorOf    = "";
     bool                enable10bit = false;
+    drmModeModeInfo     drmMode     = {};
 };
 
 struct SWorkspaceRule {
diff --git a/src/helpers/Monitor.hpp b/src/helpers/Monitor.hpp
index 1f41c691..28dad111 100644
--- a/src/helpers/Monitor.hpp
+++ b/src/helpers/Monitor.hpp
@@ -6,6 +6,7 @@
 #include <vector>
 #include <array>
 #include <memory>
+#include <xf86drmMode.h>
 #include "Timer.hpp"
 
 struct SMonitorRule;
@@ -15,21 +16,23 @@ class CMonitor {
     CMonitor();
     ~CMonitor();
 
-    Vector2D    vecPosition        = Vector2D(-1, -1); // means unset
-    Vector2D    vecSize            = Vector2D(0, 0);
-    Vector2D    vecPixelSize       = Vector2D(0, 0);
-    Vector2D    vecTransformedSize = Vector2D(0, 0);
+    Vector2D        vecPosition        = Vector2D(-1, -1); // means unset
+    Vector2D        vecSize            = Vector2D(0, 0);
+    Vector2D        vecPixelSize       = Vector2D(0, 0);
+    Vector2D        vecTransformedSize = Vector2D(0, 0);
 
-    bool        primary = false;
+    bool            primary = false;
 
-    uint64_t    ID              = -1;
-    int         activeWorkspace = -1;
-    float       scale           = 1;
+    uint64_t        ID              = -1;
+    int             activeWorkspace = -1;
+    float           scale           = 1;
 
-    std::string szName = "";
+    std::string     szName = "";
 
-    Vector2D    vecReservedTopLeft     = Vector2D(0, 0);
-    Vector2D    vecReservedBottomRight = Vector2D(0, 0);
+    Vector2D        vecReservedTopLeft     = Vector2D(0, 0);
+    Vector2D        vecReservedBottomRight = Vector2D(0, 0);
+
+    drmModeModeInfo customDrmMode = {};
 
     // WLR stuff
     wlr_damage_ring     damage;
diff --git a/src/render/Renderer.cpp b/src/render/Renderer.cpp
index 8b88c997..bcecf2c8 100644
--- a/src/render/Renderer.cpp
+++ b/src/render/Renderer.cpp
@@ -1520,12 +1520,16 @@ bool CHyprRenderer::applyMonitorRule(CMonitor* pMonitor, SMonitorRule* pMonitorR
         DELTALESSTHAN(pMonitor->refreshRate, pMonitorRule->refreshRate, 1) && pMonitor->scale == pMonitorRule->scale &&
         ((DELTALESSTHAN(pMonitor->vecPosition.x, pMonitorRule->offset.x, 1) && DELTALESSTHAN(pMonitor->vecPosition.y, pMonitorRule->offset.y, 1)) ||
          pMonitorRule->offset == Vector2D(-1, -1)) &&
-        pMonitor->transform == pMonitorRule->transform && pMonitorRule->enable10bit == pMonitor->enabled10bit) {
+        pMonitor->transform == pMonitorRule->transform && pMonitorRule->enable10bit == pMonitor->enabled10bit &&
+        !memcmp(&pMonitor->customDrmMode, &pMonitorRule->drmMode, sizeof(pMonitor->customDrmMode))) {
 
         Debug::log(LOG, "Not applying a new rule to %s because it's already applied!", pMonitor->szName.c_str());
         return true;
     }
 
+    // Needed in case we are switching from a custom modeline to a standard mode
+    pMonitor->customDrmMode = {};
+
     if (pMonitorRule->scale > 0.1) {
         wlr_output_set_scale(pMonitor->output, pMonitorRule->scale);
         pMonitor->scale = pMonitorRule->scale;
@@ -1540,7 +1544,7 @@ bool CHyprRenderer::applyMonitorRule(CMonitor* pMonitor, SMonitorRule* pMonitorR
 
     // loop over modes and choose an appropriate one.
     if (pMonitorRule->resolution != Vector2D() && pMonitorRule->resolution != Vector2D(-1, -1) && pMonitorRule->resolution != Vector2D(-1, -2)) {
-        if (!wl_list_empty(&pMonitor->output->modes)) {
+        if (!wl_list_empty(&pMonitor->output->modes) && pMonitorRule->drmMode.type != DRM_MODE_TYPE_USERDEF) {
             wlr_output_mode* mode;
             bool             found = false;
 
@@ -1599,17 +1603,37 @@ bool CHyprRenderer::applyMonitorRule(CMonitor* pMonitor, SMonitorRule* pMonitorR
                 }
             }
         } else {
-            wlr_output_set_custom_mode(pMonitor->output, (int)pMonitorRule->resolution.x, (int)pMonitorRule->resolution.y, (int)pMonitorRule->refreshRate * 1000);
+            // custom resolution
+            bool fail = false;
+
+            if (pMonitorRule->drmMode.type == DRM_MODE_TYPE_USERDEF) {
+                if (!wlr_output_is_drm(pMonitor->output)) {
+                    Debug::log(ERR, "Tried to set custom modeline on non-DRM output");
+                    fail = true;
+                } else {
+                    auto* mode = wlr_drm_connector_add_mode(pMonitor->output, &pMonitorRule->drmMode);
+                    if (mode) {
+                        wlr_output_set_mode(pMonitor->output, mode);
+                        pMonitor->customDrmMode = pMonitorRule->drmMode;
+                    } else {
+                        Debug::log(ERR, "wlr_drm_connector_add_mode failed");
+                        fail = true;
+                    }
+                }
+            } else {
+                wlr_output_set_custom_mode(pMonitor->output, (int)pMonitorRule->resolution.x, (int)pMonitorRule->resolution.y, (int)pMonitorRule->refreshRate * 1000);
+            }
+
             pMonitor->vecSize     = pMonitorRule->resolution;
             pMonitor->refreshRate = pMonitorRule->refreshRate;
 
-            if (!wlr_output_test(pMonitor->output)) {
+            if (fail || !wlr_output_test(pMonitor->output)) {
                 Debug::log(ERR, "Custom resolution FAILED, falling back to preferred");
 
                 const auto PREFERREDMODE = wlr_output_preferred_mode(pMonitor->output);
 
                 if (!PREFERREDMODE) {
-                    Debug::log(ERR, "Monitor %s has NO PREFERRED MODE, and an INVALID one was requested: %ix%i@%2f", (int)pMonitorRule->resolution.x,
+                    Debug::log(ERR, "Monitor %s has NO PREFERRED MODE, and an INVALID one was requested: %ix%i@%2f", pMonitor->output->name, (int)pMonitorRule->resolution.x,
                                (int)pMonitorRule->resolution.y, (float)pMonitorRule->refreshRate);
                     return true;
                 }
@@ -1621,8 +1645,9 @@ bool CHyprRenderer::applyMonitorRule(CMonitor* pMonitor, SMonitorRule* pMonitorR
                            (int)pMonitorRule->resolution.x, (int)pMonitorRule->resolution.y, (float)pMonitorRule->refreshRate, PREFERREDMODE->width, PREFERREDMODE->height,
                            PREFERREDMODE->refresh / 1000.f);
 
-                pMonitor->refreshRate = PREFERREDMODE->refresh / 1000.f;
-                pMonitor->vecSize     = Vector2D(PREFERREDMODE->width, PREFERREDMODE->height);
+                pMonitor->refreshRate   = PREFERREDMODE->refresh / 1000.f;
+                pMonitor->vecSize       = Vector2D(PREFERREDMODE->width, PREFERREDMODE->height);
+                pMonitor->customDrmMode = {};
             } else {
                 Debug::log(LOG, "Set a custom mode %ix%i@%2f (mode not found in monitor modes)", (int)pMonitorRule->resolution.x, (int)pMonitorRule->resolution.y,
                            (float)pMonitorRule->refreshRate);