renderer,config: add custom DRM modeline support (#2254)

This allows specifying custom display resolutions for the DRM backend.
This is useful for display overclocking, working around broken EDIDs,
etc. To use this feature, specify a modeline instead of a resolution
in the config, for example:

    monitor = DP-1, modeline 1071.101 3840 3848 3880 3920 2160 2263 2271 2277 +hsync -vsync, 0x0, 1

This example is a custom 3840x2160@120Hz mode with tightened timings.
I use it because the standard timings don't work with my monitor and GPU
combination (M28U with RX580).

The syntax is compatible with Sway and Xorg.

1 files changed, 32 insertions, 7 deletions

diff --git a/src/render/Renderer.cpp b/src/render/Renderer.cpp
index 8b88c997..bcecf2c8 100644
--- a/src/render/Renderer.cpp
+++ b/src/render/Renderer.cpp
@@ -1520,12 +1520,16 @@ bool CHyprRenderer::applyMonitorRule(CMonitor* pMonitor, SMonitorRule* pMonitorR
         DELTALESSTHAN(pMonitor->refreshRate, pMonitorRule->refreshRate, 1) && pMonitor->scale == pMonitorRule->scale &&
         ((DELTALESSTHAN(pMonitor->vecPosition.x, pMonitorRule->offset.x, 1) && DELTALESSTHAN(pMonitor->vecPosition.y, pMonitorRule->offset.y, 1)) ||
          pMonitorRule->offset == Vector2D(-1, -1)) &&
-        pMonitor->transform == pMonitorRule->transform && pMonitorRule->enable10bit == pMonitor->enabled10bit) {
+        pMonitor->transform == pMonitorRule->transform && pMonitorRule->enable10bit == pMonitor->enabled10bit &&
+        !memcmp(&pMonitor->customDrmMode, &pMonitorRule->drmMode, sizeof(pMonitor->customDrmMode))) {
 
         Debug::log(LOG, "Not applying a new rule to %s because it's already applied!", pMonitor->szName.c_str());
         return true;
     }
 
+    // Needed in case we are switching from a custom modeline to a standard mode
+    pMonitor->customDrmMode = {};
+
     if (pMonitorRule->scale > 0.1) {
         wlr_output_set_scale(pMonitor->output, pMonitorRule->scale);
         pMonitor->scale = pMonitorRule->scale;
@@ -1540,7 +1544,7 @@ bool CHyprRenderer::applyMonitorRule(CMonitor* pMonitor, SMonitorRule* pMonitorR
 
     // loop over modes and choose an appropriate one.
     if (pMonitorRule->resolution != Vector2D() && pMonitorRule->resolution != Vector2D(-1, -1) && pMonitorRule->resolution != Vector2D(-1, -2)) {
-        if (!wl_list_empty(&pMonitor->output->modes)) {
+        if (!wl_list_empty(&pMonitor->output->modes) && pMonitorRule->drmMode.type != DRM_MODE_TYPE_USERDEF) {
             wlr_output_mode* mode;
             bool             found = false;
 
@@ -1599,17 +1603,37 @@ bool CHyprRenderer::applyMonitorRule(CMonitor* pMonitor, SMonitorRule* pMonitorR
                 }
             }
         } else {
-            wlr_output_set_custom_mode(pMonitor->output, (int)pMonitorRule->resolution.x, (int)pMonitorRule->resolution.y, (int)pMonitorRule->refreshRate * 1000);
+            // custom resolution
+            bool fail = false;
+
+            if (pMonitorRule->drmMode.type == DRM_MODE_TYPE_USERDEF) {
+                if (!wlr_output_is_drm(pMonitor->output)) {
+                    Debug::log(ERR, "Tried to set custom modeline on non-DRM output");
+                    fail = true;
+                } else {
+                    auto* mode = wlr_drm_connector_add_mode(pMonitor->output, &pMonitorRule->drmMode);
+                    if (mode) {
+                        wlr_output_set_mode(pMonitor->output, mode);
+                        pMonitor->customDrmMode = pMonitorRule->drmMode;
+                    } else {
+                        Debug::log(ERR, "wlr_drm_connector_add_mode failed");
+                        fail = true;
+                    }
+                }
+            } else {
+                wlr_output_set_custom_mode(pMonitor->output, (int)pMonitorRule->resolution.x, (int)pMonitorRule->resolution.y, (int)pMonitorRule->refreshRate * 1000);
+            }
+
             pMonitor->vecSize     = pMonitorRule->resolution;
             pMonitor->refreshRate = pMonitorRule->refreshRate;
 
-            if (!wlr_output_test(pMonitor->output)) {
+            if (fail || !wlr_output_test(pMonitor->output)) {
                 Debug::log(ERR, "Custom resolution FAILED, falling back to preferred");
 
                 const auto PREFERREDMODE = wlr_output_preferred_mode(pMonitor->output);
 
                 if (!PREFERREDMODE) {
-                    Debug::log(ERR, "Monitor %s has NO PREFERRED MODE, and an INVALID one was requested: %ix%i@%2f", (int)pMonitorRule->resolution.x,
+                    Debug::log(ERR, "Monitor %s has NO PREFERRED MODE, and an INVALID one was requested: %ix%i@%2f", pMonitor->output->name, (int)pMonitorRule->resolution.x,
                                (int)pMonitorRule->resolution.y, (float)pMonitorRule->refreshRate);
                     return true;
                 }
@@ -1621,8 +1645,9 @@ bool CHyprRenderer::applyMonitorRule(CMonitor* pMonitor, SMonitorRule* pMonitorR
                            (int)pMonitorRule->resolution.x, (int)pMonitorRule->resolution.y, (float)pMonitorRule->refreshRate, PREFERREDMODE->width, PREFERREDMODE->height,
                            PREFERREDMODE->refresh / 1000.f);
 
-                pMonitor->refreshRate = PREFERREDMODE->refresh / 1000.f;
-                pMonitor->vecSize     = Vector2D(PREFERREDMODE->width, PREFERREDMODE->height);
+                pMonitor->refreshRate   = PREFERREDMODE->refresh / 1000.f;
+                pMonitor->vecSize       = Vector2D(PREFERREDMODE->width, PREFERREDMODE->height);
+                pMonitor->customDrmMode = {};
             } else {
                 Debug::log(LOG, "Set a custom mode %ix%i@%2f (mode not found in monitor modes)", (int)pMonitorRule->resolution.x, (int)pMonitorRule->resolution.y,
                            (float)pMonitorRule->refreshRate);