groups: fix swallowing (#8223)

* fix swallowing for groups

* remove unnecessary check

* clang-format

* clarify comment

* make variables consistent

* make aditional variables consistent

4 files changed, 32 insertions, 39 deletions

diff --git a/src/desktop/Window.hpp b/src/desktop/Window.hpp
index b2a8e763..9867a8c9 100644
--- a/src/desktop/Window.hpp
+++ b/src/desktop/Window.hpp
@@ -357,6 +357,7 @@ class CWindow {
 
     // swallowing
     PHLWINDOWREF m_pSwallowed;
+    bool         m_bGroupSwallowed = false;
 
     // focus stuff
     bool m_bStayFocused = false;
diff --git a/src/events/Windows.cpp b/src/events/Windows.cpp
index 57a28948..e8abe30d 100644
--- a/src/events/Windows.cpp
+++ b/src/events/Windows.cpp
@@ -44,8 +44,6 @@ void Events::listener_mapWindow(void* owner, void* data) {
     static auto PINACTIVEALPHA     = CConfigValue<Hyprlang::FLOAT>("decoration:inactive_opacity");
     static auto PACTIVEALPHA       = CConfigValue<Hyprlang::FLOAT>("decoration:active_opacity");
     static auto PDIMSTRENGTH       = CConfigValue<Hyprlang::FLOAT>("decoration:dim_strength");
-    static auto PSWALLOW           = CConfigValue<Hyprlang::INT>("misc:enable_swallow");
-    static auto PSWALLOWREGEX      = CConfigValue<std::string>("misc:swallow_regex");
     static auto PNEWTAKESOVERFS    = CConfigValue<Hyprlang::INT>("misc:new_window_takes_over_fullscreen");
     static auto PINITIALWSTRACKING = CConfigValue<Hyprlang::INT>("misc:initial_workspace_tracking");
 
@@ -322,6 +320,10 @@ void Events::listener_mapWindow(void* owner, void* data) {
 
     PWINDOW->updateWindowData();
 
+    // Verify window swallowing. Get the swallower before calling onWindowCreated(PWINDOW) because getSwallower() wouldn't get it after if PWINDOW gets auto grouped.
+    const auto SWALLOWER  = PWINDOW->getSwallower();
+    PWINDOW->m_pSwallowed = SWALLOWER;
+
     if (PWINDOW->m_bIsFloating) {
         g_pLayoutManager->getCurrentLayout()->onWindowCreated(PWINDOW);
         PWINDOW->m_bCreatedOverFullscreen = true;
@@ -567,20 +569,12 @@ void Events::listener_mapWindow(void* owner, void* data) {
             g_pCompositor->focusWindow(nullptr);
     }
 
-    // verify swallowing
-    if (*PSWALLOW && std::string{*PSWALLOWREGEX} != STRVAL_EMPTY) {
-        const auto SWALLOWER = PWINDOW->getSwallower();
-
-        if (SWALLOWER) {
-            // swallow
-            PWINDOW->m_pSwallowed = SWALLOWER;
-
-            g_pLayoutManager->getCurrentLayout()->onWindowRemoved(SWALLOWER);
-
-            SWALLOWER->setHidden(true);
-
-            g_pLayoutManager->getCurrentLayout()->recalculateMonitor(PWINDOW->m_iMonitorID);
-        }
+    // swallow
+    if (SWALLOWER) {
+        g_pLayoutManager->getCurrentLayout()->onWindowRemoved(SWALLOWER);
+        g_pHyprRenderer->damageWindow(SWALLOWER);
+        SWALLOWER->setHidden(true);
+        g_pLayoutManager->getCurrentLayout()->recalculateMonitor(PWINDOW->m_iMonitorID);
     }
 
     PWINDOW->m_bFirstMap = false;
@@ -662,7 +656,13 @@ void Events::listener_unmapWindow(void* owner, void* data) {
     // swallowing
     if (valid(PWINDOW->m_pSwallowed)) {
         PWINDOW->m_pSwallowed->setHidden(false);
+
+        if (PWINDOW->m_sGroupData.pNextWindow.lock())
+            PWINDOW->m_pSwallowed->m_bGroupSwallowed = true; // flag for the swallowed window to be created into the group where it belongs when auto_group = false.
+
         g_pLayoutManager->getCurrentLayout()->onWindowCreated(PWINDOW->m_pSwallowed.lock());
+
+        PWINDOW->m_pSwallowed->m_bGroupSwallowed = false;
         PWINDOW->m_pSwallowed.reset();
     }
 
diff --git a/src/layout/IHyprLayout.cpp b/src/layout/IHyprLayout.cpp
index 655f2341..454a6132 100644
--- a/src/layout/IHyprLayout.cpp
+++ b/src/layout/IHyprLayout.cpp
@@ -184,22 +184,20 @@ void IHyprLayout::onWindowCreatedFloating(PHLWINDOW pWindow) {
 }
 
 bool IHyprLayout::onWindowCreatedAutoGroup(PHLWINDOW pWindow) {
-    static auto PAUTOGROUP = CConfigValue<Hyprlang::INT>("group:auto_group");
-    PHLWINDOW   OPENINGON  = g_pCompositor->m_pLastWindow.lock() && g_pCompositor->m_pLastWindow->m_pWorkspace == pWindow->m_pWorkspace ?
-           g_pCompositor->m_pLastWindow.lock() :
-           g_pCompositor->getFirstWindowOnWorkspace(pWindow->workspaceID());
-
-    bool        denied = false;
-    if (pWindow->m_bIsFloating && !OPENINGON->m_bIsFloating)
-        denied = true;
-
-    if (*PAUTOGROUP                                      // check if auto_group is enabled.
+    static auto     PAUTOGROUP       = CConfigValue<Hyprlang::INT>("group:auto_group");
+    const PHLWINDOW OPENINGON        = g_pCompositor->m_pLastWindow.lock() && g_pCompositor->m_pLastWindow->m_pWorkspace == pWindow->m_pWorkspace ?
+               g_pCompositor->m_pLastWindow.lock() :
+               g_pCompositor->getFirstWindowOnWorkspace(pWindow->workspaceID());
+    const bool      FLOATEDINTOTILED = pWindow->m_bIsFloating && !OPENINGON->m_bIsFloating;
+    const bool      SWALLOWING       = pWindow->m_pSwallowed || pWindow->m_bGroupSwallowed;
+
+    if ((*PAUTOGROUP || SWALLOWING)                      // continue if auto_group is enabled or if dealing with window swallowing.
         && OPENINGON                                     // this shouldn't be 0, but honestly, better safe than sorry.
         && OPENINGON != pWindow                          // prevent freeze when the "group set" window rule makes the new window to be already a group.
         && OPENINGON->m_sGroupData.pNextWindow.lock()    // check if OPENINGON is a group.
         && pWindow->canBeGroupedInto(OPENINGON)          // check if the new window can be grouped into OPENINGON.
-        && !g_pXWaylandManager->shouldBeFloated(pWindow) // don't group child windows. Fix for floated groups. Tiled groups don't need this because we check if !denied.
-        && !denied) {                                    // don't group a new floated window into a tiled group (for convenience).
+        && !g_pXWaylandManager->shouldBeFloated(pWindow) // don't group child windows. Fix for floated groups. Tiled groups don't need this because we check if !FLOATEDINTOTILED.
+        && !FLOATEDINTOTILED) {                          // don't group a new floated window into a tiled group (for convenience).
 
         pWindow->m_bIsFloating = OPENINGON->m_bIsFloating; // match the floating state. Needed to autogroup a new tiled window into a floated group.
 
@@ -341,12 +339,9 @@ void IHyprLayout::onEndDragWindow() {
             if (pWindow->checkInputOnDecos(INPUT_TYPE_DRAG_END, MOUSECOORDS, DRAGGINGWINDOW))
                 return;
 
-            bool denied = false;
-            if (!pWindow->m_bIsFloating && !DRAGGINGWINDOW->m_bDraggingTiled)
-                denied = true;
-
-            static auto PDRAGINTOGROUP = CConfigValue<Hyprlang::INT>("group:drag_into_group");
-            if (pWindow->m_sGroupData.pNextWindow.lock() && DRAGGINGWINDOW->canBeGroupedInto(pWindow) && *PDRAGINTOGROUP == 1 && !denied) {
+            const bool  FLOATEDINTOTILED = !pWindow->m_bIsFloating && !DRAGGINGWINDOW->m_bDraggingTiled;
+            static auto PDRAGINTOGROUP   = CConfigValue<Hyprlang::INT>("group:drag_into_group");
+            if (pWindow->m_sGroupData.pNextWindow.lock() && DRAGGINGWINDOW->canBeGroupedInto(pWindow) && *PDRAGINTOGROUP == 1 && !FLOATEDINTOTILED) {
                 if (DRAGGINGWINDOW->m_bDraggingTiled) {
                     changeWindowFloatingMode(DRAGGINGWINDOW);
                     DRAGGINGWINDOW->m_vLastFloatingSize = m_vDraggingWindowOriginalFloatSize;
diff --git a/src/render/decorations/CHyprGroupBarDecoration.cpp b/src/render/decorations/CHyprGroupBarDecoration.cpp
index 4219a5a8..1868bb5c 100644
--- a/src/render/decorations/CHyprGroupBarDecoration.cpp
+++ b/src/render/decorations/CHyprGroupBarDecoration.cpp
@@ -405,12 +405,9 @@ bool CHyprGroupBarDecoration::onEndWindowDragOnDeco(const Vector2D& pos, PHLWIND
     static auto PSTACKED                         = CConfigValue<Hyprlang::INT>("group:groupbar:stacked");
     static auto PDRAGINTOGROUP                   = CConfigValue<Hyprlang::INT>("group:drag_into_group");
     static auto PMERGEFLOATEDINTOTILEDONGROUPBAR = CConfigValue<Hyprlang::INT>("group:merge_floated_into_tiled_on_groupbar");
+    const bool  FLOATEDINTOTILED                 = !m_pWindow->m_bIsFloating && !pDraggedWindow->m_bDraggingTiled;
 
-    bool        denied = false;
-    if (!m_pWindow->m_bIsFloating && !pDraggedWindow->m_bDraggingTiled && !*PMERGEFLOATEDINTOTILEDONGROUPBAR)
-        denied = true;
-
-    if (!pDraggedWindow->canBeGroupedInto(m_pWindow.lock()) || (*PDRAGINTOGROUP != 1 && *PDRAGINTOGROUP != 2) || denied)
+    if (!pDraggedWindow->canBeGroupedInto(m_pWindow.lock()) || (*PDRAGINTOGROUP != 1 && *PDRAGINTOGROUP != 2) || (FLOATEDINTOTILED && !*PMERGEFLOATEDINTOTILEDONGROUPBAR))
         return false;
 
     const float BARRELATIVE = *PSTACKED ? pos.y - assignedBoxGlobal().y - (m_fBarHeight + BAR_PADDING_OUTER_VERT) / 2 : pos.x - assignedBoxGlobal().x - m_fBarWidth / 2;