fixed tiled popups going behind windows

5 files changed, 100 insertions, 32 deletions

diff --git a/src/Compositor.cpp b/src/Compositor.cpp
index 076c8095..fb44b541 100644
--- a/src/Compositor.cpp
+++ b/src/Compositor.cpp
@@ -427,6 +427,15 @@ CWindow* CCompositor::vectorToWindowTiled(const Vector2D& pos) {
     return nullptr;
 }
 
+void findExtensionForVector2D(wlr_surface* surface, int x, int y, void* data) {
+    const auto DATA = (SExtensionFindingData*)data;
+
+    wlr_box box = {DATA->origin.x + x, DATA->origin.y + y, surface->current.width, surface->current.height};
+
+    if (wlr_box_contains_point(&box, DATA->vec.x, DATA->vec.y))
+        *DATA->found = surface;
+}
+
 CWindow* CCompositor::vectorToWindowIdeal(const Vector2D& pos) {
     const auto PMONITOR = getMonitorFromVector(pos);
 
@@ -448,10 +457,34 @@ CWindow* CCompositor::vectorToWindowIdeal(const Vector2D& pos) {
     // first loop over floating cuz they're above, m_lWindows should be sorted bottom->top, for tiled it doesn't matter.
     for (auto w = m_vWindows.rbegin(); w != m_vWindows.rend(); w++) {
         wlr_box box = {(*w)->m_vRealPosition.vec().x, (*w)->m_vRealPosition.vec().y, (*w)->m_vRealSize.vec().x, (*w)->m_vRealSize.vec().y};
-        if ((*w)->m_bIsFloating && (*w)->m_bIsMapped && wlr_box_contains_point(&box, m_sWLRCursor->x, m_sWLRCursor->y) && isWorkspaceVisible((*w)->m_iWorkspaceID) && !(*w)->m_bHidden && (*w)->m_iX11Type != 2)
-            return w->get();
+        if ((*w)->m_bIsFloating && (*w)->m_bIsMapped && isWorkspaceVisible((*w)->m_iWorkspaceID) && !(*w)->m_bHidden && (*w)->m_iX11Type != 2) {
+            if (wlr_box_contains_point(&box, m_sWLRCursor->x, m_sWLRCursor->y))
+                return w->get();
+
+            if (!(*w)->m_bIsX11) {
+                wlr_surface* resultSurf = nullptr;
+                Vector2D origin =(*w)->m_vRealPosition.vec();
+                SExtensionFindingData data = {origin, pos, &resultSurf};
+                wlr_xdg_surface_for_each_popup_surface((*w)->m_uSurface.xdg, findExtensionForVector2D, &data);
+
+                if (resultSurf)
+                    return w->get();
+            }
+        } 
     }
 
+    // for windows, we need to check their extensions too, first.
+    for (auto& w : m_vWindows) {
+        if (!w->m_bIsX11 && !w->m_bIsFloating && w->m_bIsMapped && w->m_iWorkspaceID == PMONITOR->activeWorkspace && !w->m_bHidden && w->m_iX11Type != 2) {
+            wlr_surface* resultSurf = nullptr;
+            Vector2D origin = w->m_vRealPosition.vec();
+            SExtensionFindingData data = {origin, pos, &resultSurf};
+            wlr_xdg_surface_for_each_popup_surface(w->m_uSurface.xdg, findExtensionForVector2D, &data);
+
+            if (resultSurf)
+                return w.get();
+        }
+    }
     for (auto& w : m_vWindows) {
         wlr_box box = {w->m_vPosition.x, w->m_vPosition.y, w->m_vSize.x, w->m_vSize.y};
         if (!w->m_bIsFloating && w->m_bIsMapped && wlr_box_contains_point(&box, pos.x, pos.y) && w->m_iWorkspaceID == PMONITOR->activeWorkspace && !w->m_bHidden && w->m_iX11Type != 2)
diff --git a/src/helpers/WLClasses.hpp b/src/helpers/WLClasses.hpp
index ed10b2fc..ce2d7248 100644
--- a/src/helpers/WLClasses.hpp
+++ b/src/helpers/WLClasses.hpp
@@ -72,6 +72,12 @@ struct SRenderData {
     bool squishOversized = false;
 };
 
+struct SExtensionFindingData {
+    Vector2D origin;
+    Vector2D vec;
+    wlr_surface** found;
+};
+
 struct SStringRuleNames {
     std::string layout = "";
     std::string model = "";
diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index bfe3a9ed..4b2fabea 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -717,7 +717,7 @@ void CHyprOpenGLImpl::makeWindowSnapshot(CWindow* pWindow) {
     const auto BLURVAL = g_pConfigManager->getInt("decoration:blur");
     g_pConfigManager->setInt("decoration:blur", 0);
 
-    g_pHyprRenderer->renderWindow(pWindow, PMONITOR, &now, !pWindow->m_bX11DoesntWantBorders);
+    g_pHyprRenderer->renderWindow(pWindow, PMONITOR, &now, !pWindow->m_bX11DoesntWantBorders, RENDER_PASS_ALL);
 
     g_pConfigManager->setInt("decoration:blur", BLURVAL);
 
diff --git a/src/render/Renderer.cpp b/src/render/Renderer.cpp
index be36509b..3f43f343 100644
--- a/src/render/Renderer.cpp
+++ b/src/render/Renderer.cpp
@@ -111,7 +111,7 @@ void CHyprRenderer::renderWorkspaceWithFullscreenWindow(SMonitor* pMonitor, CWor
             continue;
 
         // found it!
-        renderWindow(w.get(), pMonitor, time, pWorkspace->m_efFullscreenMode != FULLSCREEN_FULL);
+        renderWindow(w.get(), pMonitor, time, pWorkspace->m_efFullscreenMode != FULLSCREEN_FULL, RENDER_PASS_ALL);
 
         pWorkspaceWindow = w.get();
     }
@@ -121,7 +121,7 @@ void CHyprRenderer::renderWorkspaceWithFullscreenWindow(SMonitor* pMonitor, CWor
         if (w->m_iWorkspaceID != pWorkspaceWindow->m_iWorkspaceID || !w->m_bCreatedOverFullscreen || !w->m_bIsMapped)
             continue;
 
-        renderWindow(w.get(), pMonitor, time, true);
+        renderWindow(w.get(), pMonitor, time, true, RENDER_PASS_ALL);
     }
 
     // and then special windows
@@ -136,7 +136,7 @@ void CHyprRenderer::renderWorkspaceWithFullscreenWindow(SMonitor* pMonitor, CWor
             continue;
 
         // render the bad boy
-        renderWindow(w.get(), pMonitor, time, true);
+        renderWindow(w.get(), pMonitor, time, true, RENDER_PASS_ALL);
     }
 
     // and the overlay layers
@@ -158,7 +158,7 @@ void CHyprRenderer::renderWorkspaceWithFullscreenWindow(SMonitor* pMonitor, CWor
         g_pHyprError->draw();
 }
 
-void CHyprRenderer::renderWindow(CWindow* pWindow, SMonitor* pMonitor, timespec* time, bool decorate) {
+void CHyprRenderer::renderWindow(CWindow* pWindow, SMonitor* pMonitor, timespec* time, bool decorate, eRenderPassMode mode) {
     if (pWindow->m_bHidden)
         return;
 
@@ -193,35 +193,40 @@ void CHyprRenderer::renderWindow(CWindow* pWindow, SMonitor* pMonitor, timespec*
     g_pHyprOpenGL->m_pCurrentWindow = pWindow;
 
     // render window decorations first, if not fullscreen full
-    if (!pWindow->m_bIsFullscreen || PWORKSPACE->m_efFullscreenMode != FULLSCREEN_FULL) for (auto& wd : pWindow->m_dWindowDecorations)
-        wd->draw(pMonitor, renderdata.alpha * renderdata.fadeAlpha / 255.f);
 
-    if (!pWindow->m_bIsX11) {
-        wlr_box geom;
-        wlr_xdg_surface_get_geometry(pWindow->m_uSurface.xdg, &geom);
+    if (mode == RENDER_PASS_ALL || mode == RENDER_PASS_MAIN) {
+        if (!pWindow->m_bIsFullscreen || PWORKSPACE->m_efFullscreenMode != FULLSCREEN_FULL) for (auto& wd : pWindow->m_dWindowDecorations)
+                wd->draw(pMonitor, renderdata.alpha * renderdata.fadeAlpha / 255.f);
 
-        g_pHyprOpenGL->m_RenderData.primarySurfaceUVTopLeft = Vector2D((double)geom.x / (double)pWindow->m_uSurface.xdg->surface->current.width, (double)geom.y / (double)pWindow->m_uSurface.xdg->surface->current.height);
-        g_pHyprOpenGL->m_RenderData.primarySurfaceUVBottomRight = Vector2D((double)(geom.width + geom.x) / (double)pWindow->m_uSurface.xdg->surface->current.width, (double)(geom.y + geom.height) / (double)pWindow->m_uSurface.xdg->surface->current.height);
-        
-        if (g_pHyprOpenGL->m_RenderData.primarySurfaceUVTopLeft == Vector2D() && g_pHyprOpenGL->m_RenderData.primarySurfaceUVBottomRight == Vector2D(1, 1)) {
-            // No special UV mods needed
+        if (!pWindow->m_bIsX11) {
+            wlr_box geom;
+            wlr_xdg_surface_get_geometry(pWindow->m_uSurface.xdg, &geom);
+
+            g_pHyprOpenGL->m_RenderData.primarySurfaceUVTopLeft = Vector2D((double)geom.x / (double)pWindow->m_uSurface.xdg->surface->current.width, (double)geom.y / (double)pWindow->m_uSurface.xdg->surface->current.height);
+            g_pHyprOpenGL->m_RenderData.primarySurfaceUVBottomRight = Vector2D((double)(geom.width + geom.x) / (double)pWindow->m_uSurface.xdg->surface->current.width, (double)(geom.y + geom.height) / (double)pWindow->m_uSurface.xdg->surface->current.height);
+
+            if (g_pHyprOpenGL->m_RenderData.primarySurfaceUVTopLeft == Vector2D() && g_pHyprOpenGL->m_RenderData.primarySurfaceUVBottomRight == Vector2D(1, 1)) {
+                // No special UV mods needed
+                g_pHyprOpenGL->m_RenderData.primarySurfaceUVTopLeft = Vector2D(-1, -1);
+                g_pHyprOpenGL->m_RenderData.primarySurfaceUVBottomRight = Vector2D(-1, -1);
+            }
+        } else {
             g_pHyprOpenGL->m_RenderData.primarySurfaceUVTopLeft = Vector2D(-1, -1);
             g_pHyprOpenGL->m_RenderData.primarySurfaceUVBottomRight = Vector2D(-1, -1);
         }
-    } else {
+
+        wlr_surface_for_each_surface(g_pXWaylandManager->getWindowSurface(pWindow), renderSurface, &renderdata);
+
         g_pHyprOpenGL->m_RenderData.primarySurfaceUVTopLeft = Vector2D(-1, -1);
         g_pHyprOpenGL->m_RenderData.primarySurfaceUVBottomRight = Vector2D(-1, -1);
     }
 
-    wlr_surface_for_each_surface(g_pXWaylandManager->getWindowSurface(pWindow), renderSurface, &renderdata);
-
-    g_pHyprOpenGL->m_RenderData.primarySurfaceUVTopLeft = Vector2D(-1, -1);
-    g_pHyprOpenGL->m_RenderData.primarySurfaceUVBottomRight = Vector2D(-1, -1);
-
-    if (!pWindow->m_bIsX11) {
-        renderdata.dontRound = false; // restore dontround
-        renderdata.pMonitor = pMonitor;
-        wlr_xdg_surface_for_each_popup_surface(pWindow->m_uSurface.xdg, renderSurface, &renderdata);
+    if (mode == RENDER_PASS_ALL || mode == RENDER_PASS_POPUP) {
+        if (!pWindow->m_bIsX11) {
+            renderdata.dontRound = false;  // restore dontround
+            renderdata.pMonitor = pMonitor;
+            wlr_xdg_surface_for_each_popup_surface(pWindow->m_uSurface.xdg, renderSurface, &renderdata);
+        }
     }
 
     g_pHyprOpenGL->m_pCurrentWindow = nullptr;
@@ -266,7 +271,25 @@ void CHyprRenderer::renderAllClientsForMonitor(const int& ID, timespec* time) {
         return;
     }
 
-    // Non-floating
+    // Non-floating main
+    for (auto& w : g_pCompositor->m_vWindows) {
+        if (w->m_bHidden && !w->m_bIsMapped && !w->m_bFadingOut)
+            continue;
+
+        if (w->m_bIsFloating)
+            continue;  // floating are in the second pass
+
+        if (w->m_iWorkspaceID == SPECIAL_WORKSPACE_ID)
+            continue;  // special are in the third pass
+
+        if (!shouldRenderWindow(w.get(), PMONITOR))
+            continue;
+
+        // render the bad boy
+        renderWindow(w.get(), PMONITOR, time, true, RENDER_PASS_MAIN);
+    }
+
+    // Non-floating popup
     for (auto& w : g_pCompositor->m_vWindows) {
         if (w->m_bHidden && !w->m_bIsMapped && !w->m_bFadingOut)
             continue;
@@ -281,7 +304,7 @@ void CHyprRenderer::renderAllClientsForMonitor(const int& ID, timespec* time) {
             continue;
 
         // render the bad boy
-        renderWindow(w.get(), PMONITOR, time, true);
+        renderWindow(w.get(), PMONITOR, time, true, RENDER_PASS_POPUP);
     }
 
     // floating on top
@@ -299,7 +322,7 @@ void CHyprRenderer::renderAllClientsForMonitor(const int& ID, timespec* time) {
             continue;
 
         // render the bad boy
-        renderWindow(w.get(), PMONITOR, time, true);
+        renderWindow(w.get(), PMONITOR, time, true, RENDER_PASS_ALL);
     }
 
     // and then special
@@ -314,7 +337,7 @@ void CHyprRenderer::renderAllClientsForMonitor(const int& ID, timespec* time) {
             continue;
 
         // render the bad boy
-        renderWindow(w.get(), PMONITOR, time, true);
+        renderWindow(w.get(), PMONITOR, time, true, RENDER_PASS_ALL);
     }
 
     // Render surfaces above windows for monitor
diff --git a/src/render/Renderer.hpp b/src/render/Renderer.hpp
index ff9fd4ea..c03c7117 100644
--- a/src/render/Renderer.hpp
+++ b/src/render/Renderer.hpp
@@ -17,6 +17,12 @@ enum DAMAGETRACKINGMODES {
     DAMAGE_TRACKING_FULL
 };
 
+enum eRenderPassMode {
+    RENDER_PASS_ALL = 0,
+    RENDER_PASS_MAIN,
+    RENDER_PASS_POPUP
+};
+
 class CHyprRenderer {
 public:
 
@@ -41,7 +47,7 @@ public:
 private:
     void                arrangeLayerArray(SMonitor*, const std::list<SLayerSurface*>&, bool, wlr_box*);
     void                renderWorkspaceWithFullscreenWindow(SMonitor*, CWorkspace*, timespec*);
-    void                renderWindow(CWindow*, SMonitor*, timespec*, bool);
+    void                renderWindow(CWindow*, SMonitor*, timespec*, bool, eRenderPassMode);
     void                renderLayer(SLayerSurface*, SMonitor*, timespec*);
     void                renderDragIcon(SMonitor*, timespec*);