From b0bcb60801354e1008df12311fd9915913e2d65e Mon Sep 17 00:00:00 2001 From: wokron Date: Tue, 1 Sep 2026 20:28:37 +0800 Subject: [PATCH 1/7] add missing dmabuf_fd --- include/condy/zcrx.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index a9d8ca22..45f17358 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -145,6 +145,7 @@ class ZeroCopyRxBufferPool { area_reg.addr = area.offset; area_reg.len = area.size; area_reg.flags = IORING_ZCRX_AREA_DMABUF; + area_reg.dmabuf_fd = area.dmabuf_fd; register_ifq_(if_idx, if_rxq, rq_entries, area_reg, sysconf(_SC_PAGESIZE)); From 5dcbeff0e5a762512733fe9a168e3267399a32c4 Mon Sep 17 00:00:00 2001 From: wokron Date: Tue, 1 Sep 2026 20:34:06 +0800 Subject: [PATCH 2/7] remove not supported offset field --- include/condy/zcrx.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 45f17358..ef4cb979 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -55,8 +55,6 @@ struct ZeroCopyRxArea { struct ZeroCopyRxDMABufArea { /** @brief File descriptor of the DMA-BUF to use. */ int dmabuf_fd; - /** @brief Offset into the DMA-BUF where the buffer area starts. */ - size_t offset; /** @brief Size of the buffer area within the DMA-BUF, in bytes. */ size_t size; }; @@ -142,7 +140,6 @@ class ZeroCopyRxBufferPool { area_ptr_ = nullptr; io_uring_zcrx_area_reg area_reg = {}; - area_reg.addr = area.offset; area_reg.len = area.size; area_reg.flags = IORING_ZCRX_AREA_DMABUF; area_reg.dmabuf_fd = area.dmabuf_fd; From e08486e761a8ef812501fe0f7c95d23cc0b140b3 Mon Sep 17 00:00:00 2001 From: wokron Date: Wed, 2 Sep 2026 12:59:58 +0800 Subject: [PATCH 3/7] remove ManagedBuffer --- include/condy/detail/buffers.hpp | 79 ------------------------------ include/condy/provided_buffers.hpp | 65 ++++++++++++++++++++++-- include/condy/zcrx.hpp | 64 ++++++++++++++++++++++-- 3 files changed, 120 insertions(+), 88 deletions(-) delete mode 100644 include/condy/detail/buffers.hpp diff --git a/include/condy/detail/buffers.hpp b/include/condy/detail/buffers.hpp deleted file mode 100644 index 59904ba8..00000000 --- a/include/condy/detail/buffers.hpp +++ /dev/null @@ -1,79 +0,0 @@ -/** - * @file buffers.hpp - * @brief Basic buffer types and conversion utilities. - * @details This file defines basic buffer types and conversion functions. - * Buffer types are primarily used in asynchronous operations. - */ - -#pragma once - -#include "condy/detail/utils.hpp" -#include -#include -#include -#include - -namespace condy { -namespace detail { - -template struct ManagedBuffer { -public: - using CondyBuffer = void; - - ManagedBuffer() = default; - ManagedBuffer(void *data, size_t size, BufferPool *pool) - : data_(data), size_(size), pool_(pool) {} - ManagedBuffer(ManagedBuffer &&other) noexcept - : data_(std::exchange(other.data_, nullptr)), - size_(std::exchange(other.size_, 0)), - pool_(std::exchange(other.pool_, nullptr)) {} - ManagedBuffer &operator=(ManagedBuffer &&other) noexcept { - if (this != &other) { - reset(); - data_ = std::exchange(other.data_, nullptr); - size_ = std::exchange(other.size_, 0); - pool_ = std::exchange(other.pool_, nullptr); - } - return *this; - } - - ~ManagedBuffer() { reset(); } - - CONDY_DELETE_COPY(ManagedBuffer); - -public: - /** - * @brief Get the data pointer of the buffer - */ - void *data() const noexcept { return data_; } - - /** * - * @brief Get the size of the buffer - */ - size_t size() const noexcept { return size_; } - - /** - * @brief Reset the buffer, returning it to the pool if owned - */ - void reset() noexcept { - if (pool_ != nullptr) { - pool_->add_buffer_back(data_, size_); - } - data_ = nullptr; - size_ = 0; - pool_ = nullptr; - } - - /** - * @brief Check if the buffer owns a buffer from a pool. - */ - bool owns_buffer() const noexcept { return pool_ != nullptr; } - -private: - void *data_ = nullptr; - size_t size_ = 0; - BufferPool *pool_ = nullptr; -}; - -} // namespace detail -} // namespace condy \ No newline at end of file diff --git a/include/condy/provided_buffers.hpp b/include/condy/provided_buffers.hpp index 8774b0ac..21fc0641 100644 --- a/include/condy/provided_buffers.hpp +++ b/include/condy/provided_buffers.hpp @@ -9,7 +9,6 @@ #include "condy/concepts.hpp" #include "condy/condy_uring.hpp" -#include "condy/detail/buffers.hpp" #include "condy/detail/context.hpp" #include "condy/detail/ring.hpp" #include "condy/detail/utils.hpp" @@ -22,6 +21,7 @@ #include #include #include +#include #include namespace condy { @@ -244,11 +244,57 @@ class BundledProvidedBufferPool; * @note The lifetime of the provided buffer must not exceed the lifetime of the * provided buffer pool it is associated with. */ -class ProvidedBuffer - : public detail::ManagedBuffer { +class ProvidedBuffer { public: - using Base = detail::ManagedBuffer; - using Base::Base; + using CondyBuffer = void; + + ProvidedBuffer() = default; + ProvidedBuffer(void *data, size_t size, + detail::BundledProvidedBufferPool *pool) + : data_(data), size_(size), pool_(pool) {} + ProvidedBuffer(ProvidedBuffer &&other) noexcept + : data_(std::exchange(other.data_, nullptr)), + size_(std::exchange(other.size_, 0)), + pool_(std::exchange(other.pool_, nullptr)) {} + ProvidedBuffer &operator=(ProvidedBuffer &&other) noexcept { + if (this != &other) { + reset(); + data_ = std::exchange(other.data_, nullptr); + size_ = std::exchange(other.size_, 0); + pool_ = std::exchange(other.pool_, nullptr); + } + return *this; + } + + ~ProvidedBuffer() noexcept { reset(); } + + CONDY_DELETE_COPY(ProvidedBuffer); + +public: + /** + * @brief Get the data pointer of the buffer + */ + void *data() const noexcept { return data_; } + + /** + * @brief Get the size of the buffer + */ + size_t size() const noexcept { return size_; } + + /** + * @brief Reset the buffer, returning it to the pool if owned + */ + void reset() noexcept; + + /** + * @brief Check if the buffer owns a buffer from a pool. + */ + bool owns_buffer() const noexcept { return pool_ != nullptr; } + +private: + void *data_ = nullptr; + size_t size_ = 0; + detail::BundledProvidedBufferPool *pool_ = nullptr; }; namespace detail { @@ -411,6 +457,15 @@ class BundledProvidedBufferPool { } // namespace detail +inline void ProvidedBuffer::reset() noexcept { + if (pool_ != nullptr) { + pool_->add_buffer_back(data_, size_); + } + data_ = nullptr; + size_ = 0; + pool_ = nullptr; +} + /** * @brief Provided buffer pool. * @details A provided buffer pool manages a pool of buffers that can be used in diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index ef4cb979..fe12c3a4 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -10,13 +10,14 @@ #pragma once -#include "condy/detail/buffers.hpp" #include "condy/detail/context.hpp" #include "condy/detail/ring.hpp" #include "condy/detail/utils.hpp" #include "condy/runtime.hpp" #include +#include #include +#include namespace condy { @@ -32,10 +33,56 @@ class ZeroCopyRxBufferPool; * @note The lifetime of the buffer must not exceed the lifetime of the * ZeroCopyRxBufferPool it is associated with. */ -class ZeroCopyRxBuffer : public detail::ManagedBuffer { +class ZeroCopyRxBuffer { public: - using Base = detail::ManagedBuffer; - using Base::Base; + using CondyBuffer = void; + + ZeroCopyRxBuffer() = default; + ZeroCopyRxBuffer(void *data, size_t size, ZeroCopyRxBufferPool *pool) + : data_(data), size_(size), pool_(pool) {} + ZeroCopyRxBuffer(ZeroCopyRxBuffer &&other) noexcept + : data_(std::exchange(other.data_, nullptr)), + size_(std::exchange(other.size_, 0)), + pool_(std::exchange(other.pool_, nullptr)) {} + ZeroCopyRxBuffer &operator=(ZeroCopyRxBuffer &&other) noexcept { + if (this != &other) { + reset(); + data_ = std::exchange(other.data_, nullptr); + size_ = std::exchange(other.size_, 0); + pool_ = std::exchange(other.pool_, nullptr); + } + return *this; + } + + ~ZeroCopyRxBuffer() noexcept { reset(); } + + CONDY_DELETE_COPY(ZeroCopyRxBuffer); + +public: + /** + * @brief Get the data pointer of the buffer + */ + void *data() const noexcept { return data_; } + + /** + * @brief Get the size of the buffer + */ + size_t size() const noexcept { return size_; } + + /** + * @brief Reset the buffer, returning it to the pool if owned + */ + void reset() noexcept; + + /** + * @brief Check if the buffer owns a buffer from a pool. + */ + bool owns_buffer() const noexcept { return pool_ != nullptr; } + +private: + void *data_ = nullptr; + size_t size_ = 0; + ZeroCopyRxBufferPool *pool_ = nullptr; }; /** @@ -326,6 +373,15 @@ class ZeroCopyRxBufferPool { uint32_t flags_; }; +inline void ZeroCopyRxBuffer::reset() noexcept { + if (pool_ != nullptr) { + pool_->add_buffer_back(data_, size_); + } + data_ = nullptr; + size_ = 0; + pool_ = nullptr; +} + #endif } // namespace condy \ No newline at end of file From 4d25dc93f61302d905fcc84f1dd581612f312e32 Mon Sep 17 00:00:00 2001 From: wokron Date: Wed, 2 Sep 2026 13:03:30 +0800 Subject: [PATCH 4/7] ZeroCopyRxBuffer takes area_token --- include/condy/zcrx.hpp | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index fe12c3a4..13247810 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -38,18 +38,21 @@ class ZeroCopyRxBuffer { using CondyBuffer = void; ZeroCopyRxBuffer() = default; - ZeroCopyRxBuffer(void *data, size_t size, ZeroCopyRxBufferPool *pool) - : data_(data), size_(size), pool_(pool) {} + ZeroCopyRxBuffer(void *data, size_t size, ZeroCopyRxBufferPool *pool, + uint64_t area_token) + : data_(data), size_(size), pool_(pool), area_token_(area_token) {} ZeroCopyRxBuffer(ZeroCopyRxBuffer &&other) noexcept : data_(std::exchange(other.data_, nullptr)), size_(std::exchange(other.size_, 0)), - pool_(std::exchange(other.pool_, nullptr)) {} + pool_(std::exchange(other.pool_, nullptr)), + area_token_(std::exchange(other.area_token_, 0)) {} ZeroCopyRxBuffer &operator=(ZeroCopyRxBuffer &&other) noexcept { if (this != &other) { reset(); data_ = std::exchange(other.data_, nullptr); size_ = std::exchange(other.size_, 0); pool_ = std::exchange(other.pool_, nullptr); + area_token_ = std::exchange(other.area_token_, 0); } return *this; } @@ -83,6 +86,7 @@ class ZeroCopyRxBuffer { void *data_ = nullptr; size_t size_ = 0; ZeroCopyRxBufferPool *pool_ = nullptr; + uint64_t area_token_ = 0; }; /** @@ -258,11 +262,11 @@ class ZeroCopyRxBufferPool { void *data = static_cast(area_ptr_) + (rcqe->off & ~IORING_ZCRX_AREA_MASK); size_t size = static_cast(cqe->res); - return ZeroCopyRxBuffer(data, size, this); + return ZeroCopyRxBuffer(data, size, this, area_token_); } - void add_buffer_back(void *ptr, size_t size) noexcept { - rq_enqueue_(ptr, size); + void add_buffer_back(void *ptr, size_t size, uint64_t area_token) noexcept { + rq_enqueue_(ptr, size, area_token); maybe_flush_rq_(); } @@ -335,13 +339,13 @@ class ZeroCopyRxBufferPool { return rq_ring_.rq_tail - io_uring_smp_load_acquire(rq_ring_.khead); } - void rq_enqueue_(void *ptr, size_t size) noexcept { + void rq_enqueue_(void *ptr, size_t size, uint64_t area_token) noexcept { assert(rq_nr_queued_() < rq_ring_.ring_entries); io_uring_zcrx_rqe *rqe; unsigned rq_mask = rq_ring_.ring_entries - 1; rqe = &rq_ring_.rqes[rq_ring_.rq_tail & rq_mask]; rqe->off = (static_cast(ptr) - static_cast(area_ptr_)) | - area_token_; + area_token; rqe->len = static_cast(size); io_uring_smp_store_release(rq_ring_.ktail, ++rq_ring_.rq_tail); } @@ -375,11 +379,12 @@ class ZeroCopyRxBufferPool { inline void ZeroCopyRxBuffer::reset() noexcept { if (pool_ != nullptr) { - pool_->add_buffer_back(data_, size_); + pool_->add_buffer_back(data_, size_, area_token_); } data_ = nullptr; size_ = 0; pool_ = nullptr; + area_token_ = 0; } #endif From 778ee878ee4d8b1171fc902d3cf63380529e195c Mon Sep 17 00:00:00 2001 From: wokron Date: Wed, 2 Sep 2026 13:10:22 +0800 Subject: [PATCH 5/7] define Area struct --- include/condy/zcrx.hpp | 51 ++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 13247810..677368bb 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -187,9 +187,6 @@ class ZeroCopyRxBufferPool { } }); - area_size_ = 0; - area_ptr_ = nullptr; - io_uring_zcrx_area_reg area_reg = {}; area_reg.len = area.size; area_reg.flags = IORING_ZCRX_AREA_DMABUF; @@ -219,26 +216,25 @@ class ZeroCopyRxBufferPool { const size_t page_size = sysconf(_SC_PAGESIZE); if (area.addr == nullptr) { - area_size_ = detail::align_up(area.size, page_size); - area_ptr_ = mmap(nullptr, area_size_, PROT_READ | PROT_WRITE, + area_.size = detail::align_up(area.size, page_size); + area_.ptr = mmap(nullptr, area_.size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, 0, 0); - if (area_ptr_ == MAP_FAILED) { + if (area_.ptr == MAP_FAILED) { throw detail::make_system_error("mmap"); } io_uring_zcrx_area_reg area_reg = {}; - area_reg.addr = reinterpret_cast(area_ptr_); - area_reg.len = area_size_; + area_reg.addr = reinterpret_cast(area_.ptr); + area_reg.len = area_.size; area_reg.flags = 0; register_ifq_(if_idx, if_rxq, rq_entries, area_reg, page_size); } else { // Not owned, so we don't track the size for unmapping - area_size_ = 0; - area_ptr_ = area.addr; + area_.ptr = area.addr; io_uring_zcrx_area_reg area_reg = {}; - area_reg.addr = reinterpret_cast(area_ptr_); + area_reg.addr = reinterpret_cast(area_.ptr); area_reg.len = area.size; area_reg.flags = 0; @@ -259,10 +255,10 @@ class ZeroCopyRxBufferPool { } io_uring_zcrx_cqe *rcqe = reinterpret_cast(cqe->big_cqe); - void *data = static_cast(area_ptr_) + + void *data = static_cast(area_.ptr) + (rcqe->off & ~IORING_ZCRX_AREA_MASK); size_t size = static_cast(cqe->res); - return ZeroCopyRxBuffer(data, size, this, area_token_); + return ZeroCopyRxBuffer(data, size, this, area_.token); } void add_buffer_back(void *ptr, size_t size, uint64_t area_token) noexcept { @@ -272,13 +268,9 @@ class ZeroCopyRxBufferPool { private: void cleanup_() noexcept { - [[maybe_unused]] int r; - if (area_ptr_ != nullptr && area_size_ > 0) { - r = munmap(area_ptr_, area_size_); - assert(r == 0); - } + area_.maybe_cleanup(); if (rq_ring_.ring_ptr != nullptr) { - r = munmap(rq_ring_.ring_ptr, ring_size_); + [[maybe_unused]] int r = munmap(rq_ring_.ring_ptr, ring_size_); assert(r == 0); } // TODO: Unregister ifq. For now there's no way to unregister ifq, so we @@ -324,7 +316,7 @@ class ZeroCopyRxBufferPool { rq_ring_.ring_ptr = ring_ptr; zcrx_id_ = reg.zcrx_id; - area_token_ = area_reg.rq_area_token; + area_.token = area_reg.rq_area_token; } static size_t get_refill_ring_size_(uint32_t rq_entries, @@ -344,7 +336,7 @@ class ZeroCopyRxBufferPool { io_uring_zcrx_rqe *rqe; unsigned rq_mask = rq_ring_.ring_entries - 1; rqe = &rq_ring_.rqes[rq_ring_.rq_tail & rq_mask]; - rqe->off = (static_cast(ptr) - static_cast(area_ptr_)) | + rqe->off = (static_cast(ptr) - static_cast(area_.ptr)) | area_token; rqe->len = static_cast(size); io_uring_smp_store_release(rq_ring_.ktail, ++rq_ring_.rq_tail); @@ -367,13 +359,24 @@ class ZeroCopyRxBufferPool { } private: + struct Area { + void *ptr = nullptr; + size_t size = 0; + uint64_t token = 0; + + void maybe_cleanup() noexcept { + if (ptr != nullptr && size > 0) { + [[maybe_unused]] int r = munmap(ptr, size); + assert(r == 0); + } + } + }; + detail::Ring *ring_; - size_t area_size_ = 0; - void *area_ptr_ = nullptr; + Area area_; size_t ring_size_ = 0; io_uring_zcrx_rq rq_ring_ = {}; uint32_t zcrx_id_; - uint64_t area_token_; uint32_t flags_; }; From 715cec56f8faffaa5f9448488f867fd66d302b71 Mon Sep 17 00:00:00 2001 From: wokron Date: Wed, 2 Sep 2026 13:21:28 +0800 Subject: [PATCH 6/7] add other_areas_ field --- include/condy/zcrx.hpp | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 677368bb..72077226 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #include namespace condy { @@ -255,10 +256,13 @@ class ZeroCopyRxBufferPool { } io_uring_zcrx_cqe *rcqe = reinterpret_cast(cqe->big_cqe); - void *data = static_cast(area_.ptr) + + const uint64_t token = rcqe->off & IORING_ZCRX_AREA_MASK; + Area *area = find_area_(token); + assert(area != nullptr); + void *data = static_cast(area->ptr) + (rcqe->off & ~IORING_ZCRX_AREA_MASK); size_t size = static_cast(cqe->res); - return ZeroCopyRxBuffer(data, size, this, area_.token); + return ZeroCopyRxBuffer(data, size, this, token); } void add_buffer_back(void *ptr, size_t size, uint64_t area_token) noexcept { @@ -269,6 +273,9 @@ class ZeroCopyRxBufferPool { private: void cleanup_() noexcept { area_.maybe_cleanup(); + for (auto &&[_, area] : other_areas_) { + area.maybe_cleanup(); + } if (rq_ring_.ring_ptr != nullptr) { [[maybe_unused]] int r = munmap(rq_ring_.ring_ptr, ring_size_); assert(r == 0); @@ -332,12 +339,14 @@ class ZeroCopyRxBufferPool { } void rq_enqueue_(void *ptr, size_t size, uint64_t area_token) noexcept { + Area *area = find_area_(area_token); + assert(area != nullptr); assert(rq_nr_queued_() < rq_ring_.ring_entries); io_uring_zcrx_rqe *rqe; unsigned rq_mask = rq_ring_.ring_entries - 1; rqe = &rq_ring_.rqes[rq_ring_.rq_tail & rq_mask]; - rqe->off = (static_cast(ptr) - static_cast(area_.ptr)) | - area_token; + rqe->off = (static_cast(ptr) - static_cast(area->ptr)) | + area->token; rqe->len = static_cast(size); io_uring_smp_store_release(rq_ring_.ktail, ++rq_ring_.rq_tail); } @@ -372,8 +381,20 @@ class ZeroCopyRxBufferPool { } }; + Area *find_area_(uint64_t area_token) noexcept { + if (area_.token == area_token) { + return &area_; + } + auto it = other_areas_.find(area_token); + if (it != other_areas_.end()) { + return &it->second; + } + return nullptr; + } + detail::Ring *ring_; Area area_; + std::unordered_map other_areas_; size_t ring_size_ = 0; io_uring_zcrx_rq rq_ring_ = {}; uint32_t zcrx_id_; From 2f265fb195053c6cbf5549f596d9e535d8e1853d Mon Sep 17 00:00:00 2001 From: wokron Date: Wed, 2 Sep 2026 13:39:54 +0800 Subject: [PATCH 7/7] add todo --- include/condy/zcrx.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/condy/zcrx.hpp b/include/condy/zcrx.hpp index 72077226..8b81944d 100644 --- a/include/condy/zcrx.hpp +++ b/include/condy/zcrx.hpp @@ -245,6 +245,11 @@ class ZeroCopyRxBufferPool { ok = true; } +public: + // TODO: Implement this after liburing sync kernel uapi + // void add_area(const ZeroCopyRxArea &area); + // void add_area(const ZeroCopyRxDMABufArea &area); + public: uint32_t zcrx_id() const noexcept { return zcrx_id_; }