From ae8a2ab652403469b5f3ada69d0af47e6d8a85b1 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Thu, 21 Jun 2018 22:17:23 +0200 Subject: [PATCH] util/SparseBuffer: new utility class --- Makefile.am | 1 + src/util/SparseBuffer.cxx | 113 ++++++++++++++++++++++++++++++ src/util/SparseBuffer.hxx | 141 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 255 insertions(+) create mode 100644 src/util/SparseBuffer.cxx create mode 100644 src/util/SparseBuffer.hxx diff --git a/Makefile.am b/Makefile.am index 8f3f57b7d..4f3a6229c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -526,6 +526,7 @@ libutil_a_SOURCES = \ src/util/SliceBuffer.hxx \ src/util/HugeAllocator.cxx src/util/HugeAllocator.hxx \ src/util/PeakBuffer.cxx src/util/PeakBuffer.hxx \ + src/util/SparseBuffer.cxx src/util/SparseBuffer.hxx \ src/util/OptionParser.cxx src/util/OptionParser.hxx \ src/util/OptionDef.hxx \ src/util/ByteReverse.cxx src/util/ByteReverse.hxx \ diff --git a/src/util/SparseBuffer.cxx b/src/util/SparseBuffer.cxx new file mode 100644 index 000000000..aadf39732 --- /dev/null +++ b/src/util/SparseBuffer.cxx @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2013-2018 Max Kellermann + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "SparseBuffer.hxx" + +SparseMap::CheckResult +SparseMap::Check(size_type offset) const noexcept +{ + assert(!map.empty()); + assert(offset < GetEndOffset()); + + auto i = map.lower_bound(offset); + if (i == map.end()) + /* from here up until the end of the file is + defined */ + return {0, std::prev(i)->second - offset}; + + assert(i->first >= offset); + + if (offset == i->first) + /* at the beginning of this chunk: the whole chunk can + be read */ + return {0, i->second - offset}; + + if (i == map.begin()) + /* before the very first chunk: there's a hole, and + after the hole, the whole chunk can be read */ + return {i->first - offset, i->second - i->first}; + + auto p = std::prev(i); + assert(p->first < offset); + + if (offset >= p->second) + /* between two chunks */ + return {i->first - offset, i->second - i->first}; + + /* inside a chunk: the rest of the chunk can be read */ + return {0, p->second - offset}; +} + +void +SparseMap::Commit(size_type start_offset, size_type end_offset) noexcept +{ + assert(start_offset < end_offset); + + auto e = map.emplace(start_offset, end_offset); + if (!e.second && end_offset > e.first->second) + e.first->second = end_offset; + + CheckCollapseNext(CheckCollapsePrevious(e.first)); +} + +inline SparseMap::Iterator +SparseMap::CheckCollapsePrevious(Iterator i) noexcept +{ + assert(i != map.end()); + + while (i != map.begin()) { + auto previous = std::prev(i); + if (previous->second < i->first) + break; + + if (i->second > previous->second) + previous->second = i->second; + + map.erase(i); + i = previous; + } + + return i; +} + +inline SparseMap::Iterator +SparseMap::CheckCollapseNext(Iterator i) noexcept +{ + assert(i != map.end()); + + for (auto next = std::next(i); + next != map.end() && i->second >= next->first;) { + if (next->second > i->second) + i->second = next->second; + + next = map.erase(next); + } + + return i; +} diff --git a/src/util/SparseBuffer.hxx b/src/util/SparseBuffer.hxx new file mode 100644 index 000000000..5d72f34da --- /dev/null +++ b/src/util/SparseBuffer.hxx @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2013-2018 Max Kellermann + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef SPARSE_BUFFER_HXX +#define SPARSE_BUFFER_HXX + +#include "HugeAllocator.hxx" +#include "ConstBuffer.hxx" +#include "WritableBuffer.hxx" + +#include + +#include + +/** + * Helper class for #SparseBuffer which describes which portions of + * the buffer have "known" data. + */ +class SparseMap { + using size_type = std::size_t; + + /** + * Key is start offset, value is end offset. + */ + using Map = std::map; + using Iterator = typename Map::iterator; + + Map map; + +public: + explicit SparseMap(size_type size) noexcept + :map{{size, size}} { + assert(size > 0); + } + + size_type size() const noexcept { + return GetEndOffset(); + } + + struct CheckResult { + size_type undefined_size; + size_type defined_size; + }; + + /** + * Check and classify the given offset. Returns a structure + * which tells you how much data is undefined, and how much + * data follows which is defined. + */ + CheckResult Check(size_type offset) const noexcept; + + /** + * Commit a write: mark the given range in the buffer as + * "defined". + */ + void Commit(size_type start_offset, size_type end_offset) noexcept; + +private: + size_type GetEndOffset() const noexcept { + return std::prev(map.end())->second; + } + + Iterator CheckCollapsePrevious(Iterator i) noexcept; + Iterator CheckCollapseNext(Iterator i) noexcept; +}; + +#endif + +/** + * A buffer which caches the contents of a "huge" array, and remembers + * which chunks are available. + */ +template +class SparseBuffer { + using Buffer = HugeArray; + using size_type = typename Buffer::size_type; + + Buffer buffer; + + SparseMap map; + +public: + explicit SparseBuffer(size_type size) noexcept + :buffer(size), map(size) { + buffer.ForkCow(false); + } + + size_type size() const noexcept { + return map.size(); + } + + struct ReadResult { + size_type undefined_size; + ConstBuffer defined_buffer; + + constexpr bool HasData() const noexcept { + return undefined_size == 0 && + !defined_buffer.empty(); + } + }; + + ReadResult Read(size_type offset) const noexcept { + auto c = map.Check(offset); + return {c.undefined_size, {&buffer.front() + offset + c.undefined_size, c.defined_size}}; + } + + WritableBuffer Write(size_type offset) noexcept { + auto c = map.Check(offset); + return {&buffer.front() + offset, c.undefined_size}; + } + + void Commit(size_type start_offset, size_type end_offset) noexcept { + map.Commit(start_offset, end_offset); + } +};