Core: Move to aquamarine (#6608)

Moves Hyprland from wlroots to aquamarine for the backend.

---------

Signed-off-by: Vaxry <vaxry@vaxry.net>
Co-authored-by: Mihai Fufezan <mihai@fufexan.net>
Co-authored-by: Jan Beich <jbeich@FreeBSD.org>
Co-authored-by: vaxerski <vaxerski@users.noreply.github.com>
Co-authored-by: UjinT34 <41110182+UjinT34@users.noreply.github.com>
Co-authored-by: Tom Englund <tomenglund26@gmail.com>
Co-authored-by: Ikalco <73481042+ikalco@users.noreply.github.com>
Co-authored-by: diniamo <diniamo53@gmail.com>

1 files changed, 235 insertions, 76 deletions

diff --git a/src/protocols/LinuxDMABUF.cpp b/src/protocols/LinuxDMABUF.cpp
index cf8f8730..0fbf832e 100644
--- a/src/protocols/LinuxDMABUF.cpp
+++ b/src/protocols/LinuxDMABUF.cpp
@@ -6,6 +6,7 @@
 #include <sys/mman.h>
 #include <xf86drm.h>
 #include <fcntl.h>
+#include <sys/stat.h>
 #include "core/Compositor.hpp"
 #include "types/DMABuffer.hpp"
 #include "types/WLBuffer.hpp"
@@ -22,21 +23,66 @@ static std::optional<dev_t> devIDFromFD(int fd) {
     return stat.st_rdev;
 }
 
-CCompiledDMABUFFeedback::CCompiledDMABUFFeedback(dev_t device, std::vector<SDMABufTranche> tranches_) {
+CDMABUFFormatTable::CDMABUFFormatTable(SDMABUFTranche _rendererTranche, std::vector<std::pair<SP<CMonitor>, SDMABUFTranche>> tranches_) :
+    rendererTranche(_rendererTranche), monitorTranches(tranches_) {
+
+    std::vector<SDMABUFFormatTableEntry>    formatsVec;
     std::set<std::pair<uint32_t, uint64_t>> formats;
-    for (auto& t : tranches_) {
-        for (auto& fmt : t.formats) {
-            for (auto& mod : fmt.mods) {
-                formats.insert(std::make_pair<>(fmt.format, mod));
+
+    // insert formats into vec if they got inserted into set, meaning they're unique
+    size_t i = 0;
+
+    rendererTranche.indicies.clear();
+    for (auto& fmt : rendererTranche.formats) {
+        for (auto& mod : fmt.modifiers) {
+            auto format        = std::make_pair<>(fmt.drmFormat, mod);
+            auto [_, inserted] = formats.insert(format);
+            if (inserted) {
+                // if it was inserted into set, then its unique and will have a new index in vec
+                rendererTranche.indicies.push_back(i++);
+                formatsVec.push_back(SDMABUFFormatTableEntry{
+                    .fmt      = fmt.drmFormat,
+                    .modifier = mod,
+                });
+            } else {
+                // if it wasn't inserted then find its index in vec
+                auto it =
+                    std::find_if(formatsVec.begin(), formatsVec.end(), [fmt, mod](const SDMABUFFormatTableEntry& oth) { return oth.fmt == fmt.drmFormat && oth.modifier == mod; });
+                rendererTranche.indicies.push_back(it - formatsVec.begin());
             }
         }
     }
 
-    tableLen   = formats.size() * sizeof(SDMABUFFeedbackTableEntry);
+    for (auto& [monitor, tranche] : monitorTranches) {
+        tranche.indicies.clear();
+        for (auto& fmt : tranche.formats) {
+            for (auto& mod : fmt.modifiers) {
+                // apparently these can implode on planes, so dont use them
+                if (mod == DRM_FORMAT_MOD_INVALID || mod == DRM_FORMAT_MOD_LINEAR)
+                    continue;
+                auto format        = std::make_pair<>(fmt.drmFormat, mod);
+                auto [_, inserted] = formats.insert(format);
+                if (inserted) {
+                    tranche.indicies.push_back(i++);
+                    formatsVec.push_back(SDMABUFFormatTableEntry{
+                        .fmt      = fmt.drmFormat,
+                        .modifier = mod,
+                    });
+                } else {
+                    auto it = std::find_if(formatsVec.begin(), formatsVec.end(),
+                                           [fmt, mod](const SDMABUFFormatTableEntry& oth) { return oth.fmt == fmt.drmFormat && oth.modifier == mod; });
+                    tranche.indicies.push_back(it - formatsVec.begin());
+                }
+            }
+        }
+    }
+
+    tableSize = formatsVec.size() * sizeof(SDMABUFFormatTableEntry);
+
     int fds[2] = {0};
-    allocateSHMFilePair(tableLen, &fds[0], &fds[1]);
+    allocateSHMFilePair(tableSize, &fds[0], &fds[1]);
 
-    auto arr = (SDMABUFFeedbackTableEntry*)mmap(nullptr, tableLen, PROT_READ | PROT_WRITE, MAP_SHARED, fds[0], 0);
+    auto arr = (SDMABUFFormatTableEntry*)mmap(nullptr, tableSize, PROT_READ | PROT_WRITE, MAP_SHARED, fds[0], 0);
 
     if (arr == MAP_FAILED) {
         LOGM(ERR, "mmap failed");
@@ -47,33 +93,18 @@ CCompiledDMABUFFeedback::CCompiledDMABUFFeedback(dev_t device, std::vector<SDMAB
 
     close(fds[0]);
 
-    std::vector<std::pair<uint32_t, uint64_t>> formatsVec;
-    for (auto& f : formats) {
-        formatsVec.push_back(f);
-    }
-
-    size_t i = 0;
-    for (auto& [fmt, mod] : formatsVec) {
-        arr[i++] = SDMABUFFeedbackTableEntry{
-            .fmt      = fmt,
-            .modifier = mod,
-        };
-    }
+    std::copy(formatsVec.begin(), formatsVec.end(), arr);
 
-    munmap(arr, tableLen);
+    munmap(arr, tableSize);
 
-    mainDevice = device;
-    tableFD    = fds[1];
-    tranches   = formatsVec;
-
-    // TODO: maybe calculate indices? currently we send all as available which could be wrong? I ain't no kernel dev tho.
+    tableFD = fds[1];
 }
 
-CCompiledDMABUFFeedback::~CCompiledDMABUFFeedback() {
+CDMABUFFormatTable::~CDMABUFFormatTable() {
     close(tableFD);
 }
 
-CLinuxDMABuffer::CLinuxDMABuffer(uint32_t id, wl_client* client, SDMABUFAttrs attrs) {
+CLinuxDMABuffer::CLinuxDMABuffer(uint32_t id, wl_client* client, Aquamarine::SDMABUFAttrs attrs) {
     buffer = makeShared<CDMABuffer>(id, client, attrs);
 
     buffer->resource->buffer = buffer;
@@ -103,7 +134,7 @@ CLinuxDMABBUFParamsResource::CLinuxDMABBUFParamsResource(SP<CZwpLinuxBufferParam
     resource->setOnDestroy([this](CZwpLinuxBufferParamsV1* r) { PROTO::linuxDma->destroyResource(this); });
     resource->setDestroy([this](CZwpLinuxBufferParamsV1* r) { PROTO::linuxDma->destroyResource(this); });
 
-    attrs = makeShared<SDMABUFAttrs>();
+    attrs = makeShared<Aquamarine::SDMABUFAttrs>();
 
     attrs->success = true;
 
@@ -190,7 +221,7 @@ void CLinuxDMABBUFParamsResource::create(uint32_t id) {
         return;
     }
 
-    LOGM(LOG, "Creating a dmabuf, with id {}: size {}, fmt {}, planes {}", id, attrs->size, attrs->format, attrs->planes);
+    LOGM(LOG, "Creating a dmabuf, with id {}: size {}, fmt {}, planes {}", id, attrs->size, FormatUtils::drmFormatName(attrs->format), attrs->planes);
     for (int i = 0; i < attrs->planes; ++i) {
         LOGM(LOG, " | plane {}: mod {} fd {} stride {} offset {}", i, attrs->modifier, attrs->fds[i], attrs->strides[i], attrs->offsets[i]);
     }
@@ -277,40 +308,52 @@ CLinuxDMABUFFeedbackResource::CLinuxDMABUFFeedbackResource(SP<CZwpLinuxDmabufFee
     resource->setOnDestroy([this](CZwpLinuxDmabufFeedbackV1* r) { PROTO::linuxDma->destroyResource(this); });
     resource->setDestroy([this](CZwpLinuxDmabufFeedbackV1* r) { PROTO::linuxDma->destroyResource(this); });
 
-    if (surface)
-        LOGM(ERR, "FIXME: surface feedback stub");
+    auto& formatTable = PROTO::linuxDma->formatTable;
+    resource->sendFormatTable(formatTable->tableFD, formatTable->tableSize);
+    sendDefaultFeedback();
+}
 
-    auto* feedback = PROTO::linuxDma->defaultFeedback.get();
+CLinuxDMABUFFeedbackResource::~CLinuxDMABUFFeedbackResource() {
+    ;
+}
 
-    resource->sendFormatTable(feedback->tableFD, feedback->tableLen);
+bool CLinuxDMABUFFeedbackResource::good() {
+    return resource->resource();
+}
 
-    // send default feedback
+void CLinuxDMABUFFeedbackResource::sendTranche(SDMABUFTranche& tranche) {
     struct wl_array deviceArr = {
-        .size = sizeof(feedback->mainDevice),
-        .data = (void*)&feedback->mainDevice,
+        .size = sizeof(tranche.device),
+        .data = (void*)&tranche.device,
     };
-    resource->sendMainDevice(&deviceArr);
     resource->sendTrancheTargetDevice(&deviceArr);
-    resource->sendTrancheFlags((zwpLinuxDmabufFeedbackV1TrancheFlags)0);
 
-    wl_array indices;
-    wl_array_init(&indices);
-    for (size_t i = 0; i < feedback->tranches.size(); ++i) {
-        *((uint16_t*)wl_array_add(&indices, sizeof(uint16_t))) = i;
-    }
+    resource->sendTrancheFlags((zwpLinuxDmabufFeedbackV1TrancheFlags)tranche.flags);
+
+    wl_array indices = {
+        .size = tranche.indicies.size() * sizeof(tranche.indicies.at(0)),
+        .data = tranche.indicies.data(),
+    };
     resource->sendTrancheFormats(&indices);
-    wl_array_release(&indices);
     resource->sendTrancheDone();
-
-    resource->sendDone();
 }
 
-CLinuxDMABUFFeedbackResource::~CLinuxDMABUFFeedbackResource() {
-    ;
-}
+// default tranche is based on renderer (egl)
+void CLinuxDMABUFFeedbackResource::sendDefaultFeedback() {
+    auto            mainDevice  = PROTO::linuxDma->mainDevice;
+    auto&           formatTable = PROTO::linuxDma->formatTable;
 
-bool CLinuxDMABUFFeedbackResource::good() {
-    return resource->resource();
+    struct wl_array deviceArr = {
+        .size = sizeof(mainDevice),
+        .data = (void*)&mainDevice,
+    };
+    resource->sendMainDevice(&deviceArr);
+
+    sendTranche(formatTable->rendererTranche);
+
+    resource->sendDone();
+
+    lastFeedbackWasScanout = false;
 }
 
 CLinuxDMABUFResource::CLinuxDMABUFResource(SP<CZwpLinuxDmabufV1> resource_) : resource(resource_) {
@@ -361,48 +404,81 @@ bool CLinuxDMABUFResource::good() {
 }
 
 void CLinuxDMABUFResource::sendMods() {
-    for (auto& [fmt, mod] : PROTO::linuxDma->defaultFeedback->tranches) {
-        if (resource->version() < 3) {
-            if (mod == DRM_FORMAT_MOD_INVALID)
-                resource->sendFormat(fmt);
-            continue;
-        }
+    for (auto& fmt : PROTO::linuxDma->formatTable->rendererTranche.formats) {
+        for (auto& mod : fmt.modifiers) {
+            if (resource->version() < 3) {
+                if (mod == DRM_FORMAT_MOD_INVALID || mod == DRM_FORMAT_MOD_LINEAR)
+                    resource->sendFormat(fmt.drmFormat);
+                continue;
+            }
 
-        // TODO: https://gitlab.freedesktop.org/xorg/xserver/-/issues/1166
+            // TODO: https://gitlab.freedesktop.org/xorg/xserver/-/issues/1166
 
-        resource->sendModifier(fmt, mod >> 32, mod & 0xFFFFFFFF);
+            resource->sendModifier(fmt.drmFormat, mod >> 32, mod & 0xFFFFFFFF);
+        }
     }
 }
 
 CLinuxDMABufV1Protocol::CLinuxDMABufV1Protocol(const wl_interface* iface, const int& ver, const std::string& name) : IWaylandProtocol(iface, ver, name) {
     static auto P = g_pHookSystem->hookDynamic("ready", [this](void* self, SCallbackInfo& info, std::any d) {
-        int  rendererFD = wlr_renderer_get_drm_fd(g_pCompositor->m_sWLRRenderer);
+        int  rendererFD = g_pCompositor->m_iDRMFD;
         auto dev        = devIDFromFD(rendererFD);
 
         if (!dev.has_value()) {
-            LOGM(ERR, "failed to get drm dev");
-            PROTO::linuxDma.reset();
+            protoLog(ERR, "failed to get drm dev, disabling linux dmabuf");
+            removeGlobal();
             return;
         }
 
         mainDevice = *dev;
 
-        auto           fmts = g_pHyprOpenGL->getDRMFormats();
-
-        SDMABufTranche tranche = {
-            .device  = *dev,
-            .formats = fmts,
+        SDMABUFTranche eglTranche = {
+            .device  = mainDevice,
+            .flags   = 0, // renderer isnt for ds so dont set flag.
+            .formats = g_pHyprOpenGL->getDRMFormats(),
         };
 
-        std::vector<SDMABufTranche> tches;
-        tches.push_back(tranche);
+        std::vector<std::pair<SP<CMonitor>, SDMABUFTranche>> tches;
+
+        if (g_pCompositor->m_pAqBackend->hasSession()) {
+            // this assumes there's only 1 device used for both scanout and rendering
+            // also that each monitor never changes its primary plane
+
+            for (auto& mon : g_pCompositor->m_vMonitors) {
+                auto tranche = SDMABUFTranche{
+                    .device  = mainDevice,
+                    .flags   = ZWP_LINUX_DMABUF_FEEDBACK_V1_TRANCHE_FLAGS_SCANOUT,
+                    .formats = mon->output->getRenderFormats(),
+                };
+                tches.push_back(std::make_pair<>(mon, tranche));
+            }
+
+            static auto monitorAdded = g_pHookSystem->hookDynamic("monitorAdded", [this](void* self, SCallbackInfo& info, std::any param) {
+                auto pMonitor = std::any_cast<CMonitor*>(param);
+                auto mon      = pMonitor->self.lock();
+                auto tranche  = SDMABUFTranche{
+                     .device  = mainDevice,
+                     .flags   = ZWP_LINUX_DMABUF_FEEDBACK_V1_TRANCHE_FLAGS_SCANOUT,
+                     .formats = mon->output->getRenderFormats(),
+                };
+                formatTable->monitorTranches.push_back(std::make_pair<>(mon, tranche));
+                resetFormatTable();
+            });
+
+            static auto monitorRemoved = g_pHookSystem->hookDynamic("monitorRemoved", [this](void* self, SCallbackInfo& info, std::any param) {
+                auto pMonitor = std::any_cast<CMonitor*>(param);
+                auto mon      = pMonitor->self.lock();
+                std::erase_if(formatTable->monitorTranches, [mon](std::pair<SP<CMonitor>, SDMABUFTranche> pair) { return pair.first == mon; });
+                resetFormatTable();
+            });
+        }
 
-        defaultFeedback = std::make_unique<CCompiledDMABUFFeedback>(*dev, tches);
+        formatTable = std::make_unique<CDMABUFFormatTable>(eglTranche, tches);
 
         drmDevice* device = nullptr;
         if (drmGetDeviceFromDevId(mainDevice, 0, &device) != 0) {
-            LOGM(ERR, "failed to get drm dev");
-            PROTO::linuxDma.reset();
+            protoLog(ERR, "failed to get drm dev, disabling linux dmabuf");
+            removeGlobal();
             return;
         }
 
@@ -411,17 +487,51 @@ CLinuxDMABufV1Protocol::CLinuxDMABufV1Protocol(const wl_interface* iface, const
             mainDeviceFD     = open(name, O_RDWR | O_CLOEXEC);
             drmFreeDevice(&device);
             if (mainDeviceFD < 0) {
-                LOGM(ERR, "failed to open drm dev");
-                PROTO::linuxDma.reset();
+                protoLog(ERR, "failed to open drm dev, disabling linux dmabuf");
+                removeGlobal();
                 return;
             }
         } else {
-            LOGM(ERR, "DRM device {} has no render node!!", device->nodes[DRM_NODE_PRIMARY]);
+            protoLog(ERR, "DRM device {} has no render node, disabling linux dmabuf", device->nodes[DRM_NODE_PRIMARY] ? device->nodes[DRM_NODE_PRIMARY] : "null");
             drmFreeDevice(&device);
+            removeGlobal();
         }
     });
 }
 
+void CLinuxDMABufV1Protocol::resetFormatTable() {
+    if (!formatTable)
+        return;
+
+    LOGM(LOG, "Resetting format table");
+
+    // this might be a big copy
+    auto newFormatTable = std::make_unique<CDMABUFFormatTable>(formatTable->rendererTranche, formatTable->monitorTranches);
+
+    for (auto& feedback : m_vFeedbacks) {
+        feedback->resource->sendFormatTable(newFormatTable->tableFD, newFormatTable->tableSize);
+        if (feedback->lastFeedbackWasScanout) {
+            SP<CMonitor> mon;
+            auto         HLSurface = CWLSurface::fromResource(feedback->surface);
+            if (auto w = HLSurface->getWindow(); w)
+                if (auto m = g_pCompositor->getMonitorFromID(w->m_iMonitorID); m)
+                    mon = m->self.lock();
+
+            if (!mon) {
+                feedback->sendDefaultFeedback();
+                return;
+            }
+
+            updateScanoutTranche(feedback->surface, mon);
+        } else {
+            feedback->sendDefaultFeedback();
+        }
+    }
+
+    // delete old table after we sent new one
+    formatTable = std::move(newFormatTable);
+}
+
 CLinuxDMABufV1Protocol::~CLinuxDMABufV1Protocol() {
     if (mainDeviceFD >= 0)
         close(mainDeviceFD);
@@ -452,3 +562,52 @@ void CLinuxDMABufV1Protocol::destroyResource(CLinuxDMABBUFParamsResource* resour
 void CLinuxDMABufV1Protocol::destroyResource(CLinuxDMABuffer* resource) {
     std::erase_if(m_vBuffers, [&](const auto& other) { return other.get() == resource; });
 }
+
+void CLinuxDMABufV1Protocol::updateScanoutTranche(SP<CWLSurfaceResource> surface, SP<CMonitor> pMonitor) {
+    SP<CLinuxDMABUFFeedbackResource> feedbackResource;
+    for (auto& f : m_vFeedbacks) {
+        if (f->surface != surface)
+            continue;
+
+        feedbackResource = f;
+        break;
+    }
+
+    if (!feedbackResource) {
+        LOGM(LOG, "updateScanoutTranche: surface has no dmabuf_feedback");
+        return;
+    }
+
+    if (!pMonitor) {
+        LOGM(LOG, "updateScanoutTranche: resetting feedback");
+        feedbackResource->sendDefaultFeedback();
+        return;
+    }
+
+    const auto& monitorTranchePair = std::find_if(formatTable->monitorTranches.begin(), formatTable->monitorTranches.end(),
+                                                  [pMonitor](std::pair<SP<CMonitor>, SDMABUFTranche> pair) { return pair.first == pMonitor; });
+
+    if (monitorTranchePair == formatTable->monitorTranches.end()) {
+        LOGM(LOG, "updateScanoutTranche: monitor has no tranche");
+        return;
+    }
+
+    auto& monitorTranche = (*monitorTranchePair).second;
+
+    LOGM(LOG, "updateScanoutTranche: sending a scanout tranche");
+
+    struct wl_array deviceArr = {
+        .size = sizeof(mainDevice),
+        .data = (void*)&mainDevice,
+    };
+    feedbackResource->resource->sendMainDevice(&deviceArr);
+
+    // prioritize scnaout tranche but have renderer fallback tranche
+    // also yes formats can be duped here because different tranche flags (ds and no ds)
+    feedbackResource->sendTranche(monitorTranche);
+    feedbackResource->sendTranche(formatTable->rendererTranche);
+
+    feedbackResource->resource->sendDone();
+
+    feedbackResource->lastFeedbackWasScanout = true;
+}