util/SparseBuffer: new utility class

Makefile.am

@@ -526,6 +526,7 @@ libutil_a_SOURCES = \
 	src/util/SliceBuffer.hxx \
 	src/util/HugeAllocator.cxx src/util/HugeAllocator.hxx \
 	src/util/PeakBuffer.cxx src/util/PeakBuffer.hxx \
+	src/util/SparseBuffer.cxx src/util/SparseBuffer.hxx \
 	src/util/OptionParser.cxx src/util/OptionParser.hxx \
 	src/util/OptionDef.hxx \
 	src/util/ByteReverse.cxx src/util/ByteReverse.hxx \

src/util/SparseBuffer.cxx

@@ -0,0 +1,113 @@
+/*
+ * Copyright (C) 2013-2018 Max Kellermann <max.kellermann@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "SparseBuffer.hxx"
+
+SparseMap::CheckResult
+SparseMap::Check(size_type offset) const noexcept
+{
+	assert(!map.empty());
+	assert(offset < GetEndOffset());
+
+	auto i = map.lower_bound(offset);
+	if (i == map.end())
+		/* from here up until the end of the file is
+		   defined */
+		return {0, std::prev(i)->second - offset};
+
+	assert(i->first >= offset);
+
+	if (offset == i->first)
+		/* at the beginning of this chunk: the whole chunk can
+		   be read */
+		return {0, i->second - offset};
+
+	if (i == map.begin())
+		/* before the very first chunk: there's a hole, and
+		   after the hole, the whole chunk can be read */
+		return {i->first - offset, i->second - i->first};
+
+	auto p = std::prev(i);
+	assert(p->first < offset);
+
+	if (offset >= p->second)
+		/* between two chunks */
+		return {i->first - offset, i->second - i->first};
+
+	/* inside a chunk: the rest of the chunk can be read */
+	return {0, p->second - offset};
+}
+
+void
+SparseMap::Commit(size_type start_offset, size_type end_offset) noexcept
+{
+	assert(start_offset < end_offset);
+
+	auto e = map.emplace(start_offset, end_offset);
+	if (!e.second && end_offset > e.first->second)
+		e.first->second = end_offset;
+
+	CheckCollapseNext(CheckCollapsePrevious(e.first));
+}
+
+inline SparseMap::Iterator
+SparseMap::CheckCollapsePrevious(Iterator i) noexcept
+{
+	assert(i != map.end());
+
+	while (i != map.begin()) {
+		auto previous = std::prev(i);
+		if (previous->second < i->first)
+			break;
+
+		if (i->second > previous->second)
+			previous->second = i->second;
+
+		map.erase(i);
+		i = previous;
+	}
+
+	return i;
+}
+
+inline SparseMap::Iterator
+SparseMap::CheckCollapseNext(Iterator i) noexcept
+{
+	assert(i != map.end());
+
+	for (auto next = std::next(i);
+	     next != map.end() && i->second >= next->first;) {
+		if (next->second > i->second)
+			i->second = next->second;
+
+		next = map.erase(next);
+	}
+
+	return i;
+}

src/util/SparseBuffer.hxx

@@ -0,0 +1,141 @@
+/*
+ * Copyright (C) 2013-2018 Max Kellermann <max.kellermann@gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ * OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef SPARSE_BUFFER_HXX
+#define SPARSE_BUFFER_HXX
+
+#include "HugeAllocator.hxx"
+#include "ConstBuffer.hxx"
+#include "WritableBuffer.hxx"
+
+#include <map>
+
+#include <assert.h>
+
+/**
+ * Helper class for #SparseBuffer which describes which portions of
+ * the buffer have "known" data.
+ */
+class SparseMap {
+	using size_type = std::size_t;
+
+	/**
+	 * Key is start offset, value is end offset.
+	 */
+	using Map = std::map<size_type, size_type>;
+	using Iterator = typename Map::iterator;
+
+	Map map;
+
+public:
+	explicit SparseMap(size_type size) noexcept
+		:map{{size, size}} {
+		assert(size > 0);
+	}
+
+	size_type size() const noexcept {
+		return GetEndOffset();
+	}
+
+	struct CheckResult {
+		size_type undefined_size;
+		size_type defined_size;
+	};
+
+	/**
+	 * Check and classify the given offset.  Returns a structure
+	 * which tells you how much data is undefined, and how much
+	 * data follows which is defined.
+	 */
+	CheckResult Check(size_type offset) const noexcept;
+
+	/**
+	 * Commit a write: mark the given range in the buffer as
+	 * "defined".
+	 */
+	void Commit(size_type start_offset, size_type end_offset) noexcept;
+
+private:
+	size_type GetEndOffset() const noexcept {
+		return std::prev(map.end())->second;
+	}
+
+	Iterator CheckCollapsePrevious(Iterator i) noexcept;
+	Iterator CheckCollapseNext(Iterator i) noexcept;
+};
+
+#endif
+
+/**
+ * A buffer which caches the contents of a "huge" array, and remembers
+ * which chunks are available.
+ */
+template<typename T>
+class SparseBuffer {
+	using Buffer = HugeArray<T>;
+	using size_type = typename Buffer::size_type;
+
+	Buffer buffer;
+
+	SparseMap map;
+
+public:
+	explicit SparseBuffer(size_type size) noexcept
+		:buffer(size), map(size) {
+		buffer.ForkCow(false);
+	}
+
+	size_type size() const noexcept {
+		return map.size();
+	}
+
+	struct ReadResult {
+		size_type undefined_size;
+		ConstBuffer<T> defined_buffer;
+
+		constexpr bool HasData() const noexcept {
+			return undefined_size == 0 &&
+				!defined_buffer.empty();
+		}
+	};
+
+	ReadResult Read(size_type offset) const noexcept {
+		auto c = map.Check(offset);
+		return {c.undefined_size, {&buffer.front() + offset + c.undefined_size, c.defined_size}};
+	}
+
+	WritableBuffer<T> Write(size_type offset) noexcept {
+		auto c = map.Check(offset);
+		return {&buffer.front() + offset, c.undefined_size};
+	}
+
+	void Commit(size_type start_offset, size_type end_offset) noexcept {
+		map.Commit(start_offset, end_offset);
+	}
+};