xdgoutput: separate logic for zero scaling positions

if a scaled monitor was directly to the left of another monitor, we'd change the size to have zero scaling without pushing other monitors out of the way, creating overlaps in xwayland.

2 files changed, 45 insertions, 3 deletions

diff --git a/src/protocols/XDGOutput.cpp b/src/protocols/XDGOutput.cpp
index 99d3c34a..c6635def 100644
--- a/src/protocols/XDGOutput.cpp
+++ b/src/protocols/XDGOutput.cpp
@@ -97,7 +97,7 @@ void CXDGOutputProtocol::onManagerGetXDGOutput(wl_client* client, wl_resource* r
     if (XDGVER >= ZXDG_OUTPUT_V1_DESCRIPTION_SINCE_VERSION && PMONITOR->output->description)
         zxdg_output_v1_send_description(pXDGOutput->resource->resource(), PMONITOR->output->description);
 
-    updateOutputDetails(pXDGOutput);
+    updateAllOutputs();
 
     const auto OUTPUTVER = wl_resource_get_version(outputResource);
     if (OUTPUTVER >= WL_OUTPUT_DONE_SINCE_VERSION && XDGVER >= OUTPUT_DONE_DEPRECATED_SINCE_VERSION)
@@ -107,10 +107,11 @@ void CXDGOutputProtocol::onManagerGetXDGOutput(wl_client* client, wl_resource* r
 void CXDGOutputProtocol::updateOutputDetails(SXDGOutput* pOutput) {
     static auto* const PXWLFORCESCALEZERO = &g_pConfigManager->getConfigValuePtr("xwayland:force_zero_scaling")->intValue;
 
-    zxdg_output_v1_send_logical_position(pOutput->resource->resource(), pOutput->monitor->vecPosition.x, pOutput->monitor->vecPosition.y);
+    const auto         POS = pOutput->overridePosition.value_or(pOutput->monitor->vecPosition);
+    zxdg_output_v1_send_logical_position(pOutput->resource->resource(), POS.x, POS.y);
 
     if (*PXWLFORCESCALEZERO && pOutput->isXWayland)
-        zxdg_output_v1_send_logical_size(pOutput->resource->resource(), pOutput->monitor->vecPixelSize.x, pOutput->monitor->vecPixelSize.y);
+        zxdg_output_v1_send_logical_size(pOutput->resource->resource(), pOutput->monitor->vecTransformedSize.x, pOutput->monitor->vecTransformedSize.y);
     else
         zxdg_output_v1_send_logical_size(pOutput->resource->resource(), pOutput->monitor->vecSize.x, pOutput->monitor->vecSize.y);
 
@@ -119,9 +120,47 @@ void CXDGOutputProtocol::updateOutputDetails(SXDGOutput* pOutput) {
 }
 
 void CXDGOutputProtocol::updateAllOutputs() {
+    static auto* const       PXWLFORCESCALEZERO = &g_pConfigManager->getConfigValuePtr("xwayland:force_zero_scaling")->intValue;
+    std::vector<SXDGOutput*> xwlOutputs;
+
     for (auto& o : m_vXDGOutputs) {
+        if (o->isXWayland && *PXWLFORCESCALEZERO) {
+            xwlOutputs.push_back(o.get());
+            o->overridePosition.reset();
+            continue;
+        }
+
         updateOutputDetails(o.get());
 
         wlr_output_schedule_done(o->monitor->output);
     }
+
+    // all of this will be noop if force_zero_scaling is off
+    std::sort(xwlOutputs.begin(), xwlOutputs.end(), [&](const auto& a, const auto& b) { return a->monitor->vecPosition < b->monitor->vecPosition; });
+
+    for (auto& o : xwlOutputs) {
+        Vector2D suggestedPos = o->monitor->vecPosition;
+
+        // check for overlaps
+        for (auto& o1 : xwlOutputs) {
+            if (!o1->overridePosition.has_value())
+                continue;
+
+            const auto OPOS = o1->overridePosition.value();
+
+            if (OPOS.x + o1->monitor->vecTransformedSize.x > suggestedPos.x && /* Y overlap */
+                !(suggestedPos.y + o->monitor->vecTransformedSize.y <= OPOS.y || OPOS.y + o1->monitor->vecTransformedSize.y <= suggestedPos.y))
+                suggestedPos.x = OPOS.x + o1->monitor->vecTransformedSize.x;
+
+            if (OPOS.y + o1->monitor->vecTransformedSize.y > suggestedPos.y && /* X overlap */
+                !(suggestedPos.x + o->monitor->vecTransformedSize.x <= OPOS.x || OPOS.x + o1->monitor->vecTransformedSize.x <= suggestedPos.x))
+                suggestedPos.y = OPOS.y + o1->monitor->vecTransformedSize.y;
+        }
+
+        o->overridePosition = suggestedPos;
+
+        updateOutputDetails(o);
+
+        wlr_output_schedule_done(o->monitor->output);
+    }
 }
diff --git a/src/protocols/XDGOutput.hpp b/src/protocols/XDGOutput.hpp
index 13b7029c..2469e600 100644
--- a/src/protocols/XDGOutput.hpp
+++ b/src/protocols/XDGOutput.hpp
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "WaylandProtocol.hpp"
+#include <optional>
 
 class CMonitor;
 
@@ -8,6 +9,8 @@ struct SXDGOutput {
     CMonitor*                         monitor = nullptr;
     std::unique_ptr<CWaylandResource> resource;
 
+    std::optional<Vector2D>           overridePosition;
+
     wl_client*                        client     = nullptr;
     bool                              isXWayland = false;
 };