subcompositor/renderer: fixup handling of subsurfaces below the main one

some apps (notably vlc 4) place a subsurface below the main surface (which is kinda cursed) but we have to accomodate for that

3 files changed, 60 insertions, 13 deletions

diff --git a/src/protocols/core/Compositor.cpp b/src/protocols/core/Compositor.cpp
index db6e3258..165da6ee 100644
--- a/src/protocols/core/Compositor.cpp
+++ b/src/protocols/core/Compositor.cpp
@@ -277,8 +277,26 @@ void CWLSurfaceResource::resetRole() {
 }
 
 void CWLSurfaceResource::bfHelper(std::vector<SP<CWLSurfaceResource>> nodes, std::function<void(SP<CWLSurfaceResource>, const Vector2D&, void*)> fn, void* data) {
+
+    std::vector<SP<CWLSurfaceResource>> nodes2;
+
+    // first, gather all nodes below
     for (auto& n : nodes) {
+        std::erase_if(n->subsurfaces, [](const auto& e) { return e.expired(); });
+        // subsurfaces is sorted lowest -> highest
+        for (auto& c : n->subsurfaces) {
+            if (c->zIndex >= 0)
+                break;
+            nodes2.push_back(c->surface.lock());
+        }
+    }
 
+    if (!nodes2.empty())
+        bfHelper(nodes2, fn, data);
+
+    nodes2.clear();
+
+    for (auto& n : nodes) {
         Vector2D offset = {};
         if (n->role->role() == SURFACE_ROLE_SUBSURFACE) {
             auto subsurface = (CWLSubsurfaceResource*)n->role.get();
@@ -288,11 +306,10 @@ void CWLSurfaceResource::bfHelper(std::vector<SP<CWLSurfaceResource>> nodes, std
         fn(n, offset, data);
     }
 
-    std::vector<SP<CWLSurfaceResource>> nodes2;
-
     for (auto& n : nodes) {
-        std::erase_if(n->subsurfaces, [](const auto& e) { return e.expired(); });
         for (auto& c : n->subsurfaces) {
+            if (c->zIndex < 0)
+                continue;
             nodes2.push_back(c->surface.lock());
         }
     }
diff --git a/src/protocols/core/Subcompositor.cpp b/src/protocols/core/Subcompositor.cpp
index d407636c..cbc4063a 100644
--- a/src/protocols/core/Subcompositor.cpp
+++ b/src/protocols/core/Subcompositor.cpp
@@ -23,15 +23,29 @@ CWLSubsurfaceResource::CWLSubsurfaceResource(SP<CWlSubsurface> resource_, SP<CWL
         if (!parent)
             return;
 
-        std::erase(parent->subsurfaces, self.lock());
+        auto pushAboveIndex = [this](int idx) -> void {
+            for (auto& c : parent->subsurfaces) {
+                if (c->zIndex >= idx)
+                    c->zIndex++;
+            }
+        };
+
+        std::erase_if(parent->subsurfaces, [this](const auto& e) { return e == self || !e; });
 
         auto it = std::find(parent->subsurfaces.begin(), parent->subsurfaces.end(), SURF);
 
         if (it == parent->subsurfaces.end()) {
-            LOGM(ERR, "Invalid surface reference in placeAbove");
-            parent->subsurfaces.emplace_back(self.lock());
-        } else
-            parent->subsurfaces.insert(it, self.lock());
+            LOGM(ERR, "Invalid surface reference in placeAbove, likely parent");
+            pushAboveIndex(1);
+            parent->subsurfaces.emplace_back(self);
+            zIndex = 1;
+        } else {
+            pushAboveIndex((*it)->zIndex);
+            zIndex = (*it)->zIndex;
+            parent->subsurfaces.emplace_back(self);
+        }
+
+        std::sort(parent->subsurfaces.begin(), parent->subsurfaces.end(), [](const auto& a, const auto& b) { return a->zIndex < b->zIndex; });
     });
 
     resource->setPlaceBelow([this](CWlSubsurface* r, wl_resource* surf) {
@@ -40,15 +54,29 @@ CWLSubsurfaceResource::CWLSubsurfaceResource(SP<CWlSubsurface> resource_, SP<CWL
         if (!parent)
             return;
 
-        std::erase(parent->subsurfaces, self.lock());
+        auto pushBelowIndex = [this](int idx) -> void {
+            for (auto& c : parent->subsurfaces) {
+                if (c->zIndex <= idx)
+                    c->zIndex--;
+            }
+        };
+
+        std::erase_if(parent->subsurfaces, [this](const auto& e) { return e == self || !e; });
 
         auto it = std::find(parent->subsurfaces.begin(), parent->subsurfaces.end(), SURF);
 
         if (it == parent->subsurfaces.end()) {
-            LOGM(ERR, "Invalid surface reference in placeBelow");
-            parent->subsurfaces.emplace_back(self.lock());
-        } else
-            parent->subsurfaces.insert(it--, self.lock());
+            LOGM(ERR, "Invalid surface reference in placeBelow, likely parent");
+            pushBelowIndex(-1);
+            parent->subsurfaces.emplace_back(self);
+            zIndex = -1;
+        } else {
+            pushBelowIndex((*it)->zIndex);
+            zIndex = (*it)->zIndex;
+            parent->subsurfaces.emplace_back(self);
+        }
+
+        std::sort(parent->subsurfaces.begin(), parent->subsurfaces.end(), [](const auto& a, const auto& b) { return a->zIndex < b->zIndex; });
     });
 
     listeners.commitSurface = surface->events.commit.registerListener([this](std::any d) {
diff --git a/src/protocols/core/Subcompositor.hpp b/src/protocols/core/Subcompositor.hpp
index abcfbf6d..824f0ffc 100644
--- a/src/protocols/core/Subcompositor.hpp
+++ b/src/protocols/core/Subcompositor.hpp
@@ -35,6 +35,8 @@ class CWLSubsurfaceResource : public ISurfaceRole {
 
     WP<CWLSubsurfaceResource> self;
 
+    int                       zIndex = 1; // by default, it's above
+
     struct {
         CSignal destroy;
     } events;