From 58fb36bdb9efc058061fa5b35bb910a759be4381 Mon Sep 17 00:00:00 2001
From: Max Kellermann <max@musicpd.org>
Date: Tue, 3 Jan 2017 07:25:14 +0100
Subject: [PATCH] storage/http: new storage plugin
---
Makefile.am | 12 +
NEWS | 2 +
configure.ac | 13 +
doc/user.xml | 12 +
src/lib/curl/Global.hxx | 2 +
src/storage/Registry.cxx | 4 +
src/storage/plugins/CurlStorage.cxx | 590 ++++++++++++++++++++++++++++
src/storage/plugins/CurlStorage.hxx | 29 ++
8 files changed, 664 insertions(+)
create mode 100644 src/storage/plugins/CurlStorage.cxx
create mode 100644 src/storage/plugins/CurlStorage.hxx
diff --git a/Makefile.am b/Makefile.am
index de02ebaa2..cdc621d57 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -699,6 +699,8 @@ libstorage_a_CPPFLAGS = $(AM_CPPFLAGS) \
STORAGE_LIBS = \
libstorage.a \
+ $(CURL_LIBS) \
+ $(EXPAT_LIBS) \
$(NFS_LIBS) \
$(SMBCLIENT_LIBS)
@@ -714,6 +716,12 @@ libstorage_a_SOURCES += \
src/storage/plugins/NfsStorage.cxx src/storage/plugins/NfsStorage.hxx
endif
+if ENABLE_WEBDAV
+libstorage_a_SOURCES += \
+ src/lib/expat/ExpatParser.cxx \
+ src/storage/plugins/CurlStorage.cxx src/storage/plugins/CurlStorage.hxx
+endif
+
endif
# neighbor plugins
@@ -1768,6 +1776,10 @@ test_run_storage_SOURCES = \
test/ScopeIOThread.hxx \
test/run_storage.cxx
+if ENABLE_WEBDAV
+test_run_storage_SOURCES += $(CURL_SOURCES)
+endif
+
endif
test_run_input_LDADD = \
diff --git a/NEWS b/NEWS
index 11b832967..d03044778 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,8 @@
ver 0.20.1 (not yet released)
* input
- curl: fix crash bug
+* storage
+ - curl: new storage plugin for WebDAV (work in progress)
* mixer
- alsa: normalize displayed volume according to human perception
* fix crash with volume_normalization enabled
diff --git a/configure.ac b/configure.ac
index 420458b6f..4f9e3c540 100644
--- a/configure.ac
+++ b/configure.ac
@@ -720,6 +720,19 @@ fi
MPD_ENABLE_AUTO_PKG(mms, MMS, [libmms >= 0.4],
[libmms mms:// protocol support], [libmms not found])
+dnl ---------------------------------------------------------------------------
+dnl Storage Plugins
+dnl ---------------------------------------------------------------------------
+
+MPD_ENABLE_AUTO(webdav, WEBDAV, [WebDAV storage plugin],
+ [WebDAV requires libcurl and libexpat],
+ [auto],
+ [if test x$enable_curl = xyes && test x$enable_expat = xyes; then
+ found_webdav=yes
+ else
+ found_webdav=no
+ fi])
+
dnl ---------------------------------------------------------------------------
dnl Playlist Plugins
dnl ---------------------------------------------------------------------------
diff --git a/doc/user.xml b/doc/user.xml
index a27b6be88..d7b4a74cd 100644
--- a/doc/user.xml
+++ b/doc/user.xml
@@ -1868,6 +1868,18 @@ run</programlisting>
</para>
</section>
+ <section id="curl_storage">
+ <title><varname>curl</varname></title>
+
+ <para>
+ A WebDAV client using <filename>libcurl</filename>. It used
+ used when <varname>music_directory</varname> contains a
+ <parameter>http://</parameter> or
+ <parameter>https://</parameter> URI, for example
+ "<parameter>https://the.server/dav/</parameter>".
+ </para>
+ </section>
+
<section id="smbclient_storage">
<title><varname>smbclient</varname></title>
diff --git a/src/lib/curl/Global.hxx b/src/lib/curl/Global.hxx
index 9591b3f80..8669234c2 100644
--- a/src/lib/curl/Global.hxx
+++ b/src/lib/curl/Global.hxx
@@ -46,6 +46,8 @@ class CurlGlobal final : TimeoutMonitor, DeferredMonitor {
public:
explicit CurlGlobal(EventLoop &_loop);
+ using TimeoutMonitor::GetEventLoop;
+
void Add(CURL *easy, CurlRequest &request);
void Remove(CURL *easy);
diff --git a/src/storage/Registry.cxx b/src/storage/Registry.cxx
index 4901da6a8..d0104359c 100644
--- a/src/storage/Registry.cxx
+++ b/src/storage/Registry.cxx
@@ -23,6 +23,7 @@
#include "plugins/LocalStorage.hxx"
#include "plugins/SmbclientStorage.hxx"
#include "plugins/NfsStorage.hxx"
+#include "plugins/CurlStorage.hxx"
#include <assert.h>
#include <string.h>
@@ -34,6 +35,9 @@ const StoragePlugin *const storage_plugins[] = {
#endif
#ifdef ENABLE_NFS
&nfs_storage_plugin,
+#endif
+#ifdef ENABLE_WEBDAV
+ &curl_storage_plugin,
#endif
nullptr
};
diff --git a/src/storage/plugins/CurlStorage.cxx b/src/storage/plugins/CurlStorage.cxx
new file mode 100644
index 000000000..cc65badfd
--- /dev/null
+++ b/src/storage/plugins/CurlStorage.cxx
@@ -0,0 +1,590 @@
+/*
+ * Copyright 2003-2016 The Music Player Daemon Project
+ * http://www.musicpd.org
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include "config.h"
+#include "CurlStorage.hxx"
+#include "storage/StoragePlugin.hxx"
+#include "storage/StorageInterface.hxx"
+#include "storage/FileInfo.hxx"
+#include "storage/MemoryDirectoryReader.hxx"
+#include "lib/curl/Global.hxx"
+#include "lib/curl/Slist.hxx"
+#include "lib/curl/Request.hxx"
+#include "lib/curl/Handler.hxx"
+#include "lib/expat/ExpatParser.hxx"
+#include "fs/Traits.hxx"
+#include "event/Call.hxx"
+#include "event/DeferredMonitor.hxx"
+#include "thread/Mutex.hxx"
+#include "thread/Cond.hxx"
+#include "util/RuntimeError.hxx"
+#include "util/StringCompare.hxx"
+#include "util/TimeParser.hxx"
+#include "util/UriUtil.hxx"
+
+#include <algorithm>
+#include <memory>
+#include <string>
+#include <list>
+
+#include <assert.h>
+
+class CurlStorage final : public Storage {
+ const std::string base;
+
+ CurlGlobal *const curl;
+
+public:
+ CurlStorage(EventLoop &_loop, const char *_base)
+ :base(_base),
+ curl(new CurlGlobal(_loop)) {}
+
+ ~CurlStorage() {
+ BlockingCall(curl->GetEventLoop(), [this](){ delete curl; });
+ }
+
+ /* virtual methods from class Storage */
+ StorageFileInfo GetInfo(const char *uri_utf8, bool follow) override;
+
+ StorageDirectoryReader *OpenDirectory(const char *uri_utf8) override;
+
+ std::string MapUTF8(const char *uri_utf8) const override;
+
+ const char *MapToRelativeUTF8(const char *uri_utf8) const override;
+};
+
+std::string
+CurlStorage::MapUTF8(const char *uri_utf8) const
+{
+ assert(uri_utf8 != nullptr);
+
+ if (StringIsEmpty(uri_utf8))
+ return base;
+
+ // TODO: escape the given URI
+
+ return PathTraitsUTF8::Build(base.c_str(), uri_utf8);
+}
+
+const char *
+CurlStorage::MapToRelativeUTF8(const char *uri_utf8) const
+{
+ // TODO: escape/unescape?
+
+ return PathTraitsUTF8::Relative(base.c_str(), uri_utf8);
+}
+
+class BlockingHttpRequest : protected CurlResponseHandler, DeferredMonitor {
+ std::exception_ptr postponed_error;
+
+ bool done = false;
+
+protected:
+ CurlRequest request;
+
+ Mutex mutex;
+ Cond cond;
+
+public:
+ BlockingHttpRequest(CurlGlobal &curl, const char *uri)
+ :DeferredMonitor(curl.GetEventLoop()),
+ request(curl, uri, *this) {
+ // TODO: use CurlInputStream's configuration
+
+ /* start the transfer inside the IOThread */
+ DeferredMonitor::Schedule();
+ }
+
+ void Wait() {
+ const std::lock_guard<Mutex> lock(mutex);
+ while (!done)
+ cond.wait(mutex);
+
+ if (postponed_error)
+ std::rethrow_exception(postponed_error);
+ }
+
+protected:
+ void SetDone() {
+ assert(!done);
+
+ request.Stop();
+ done = true;
+ cond.signal();
+ }
+
+ void LockSetDone() {
+ const std::lock_guard<Mutex> lock(mutex);
+ SetDone();
+ }
+
+private:
+ /* virtual methods from DeferredMonitor */
+ void RunDeferred() final {
+ assert(!done);
+
+ request.Start();
+ }
+
+ /* virtual methods from CurlResponseHandler */
+ void OnError(std::exception_ptr e) final {
+ const std::lock_guard<Mutex> lock(mutex);
+ postponed_error = std::move(e);
+ SetDone();
+ }
+};
+
+/**
+ * A helper class which feeds a (foreign) memory buffer into the
+ * CURLOPT_READFUNCTION.
+ */
+class CurlRequestBody {
+ ConstBuffer<char> data;
+
+public:
+ explicit CurlRequestBody(ConstBuffer<void> _data)
+ :data(ConstBuffer<char>::FromVoid(_data)) {}
+
+ explicit constexpr CurlRequestBody(StringView _data)
+ :data(_data) {}
+
+ template<typename T>
+ CurlRequestBody(CurlRequest &request, T _data)
+ :CurlRequestBody(_data) {
+ request.SetOption(CURLOPT_READFUNCTION, Callback);
+ request.SetOption(CURLOPT_READDATA, this);
+ }
+
+private:
+ size_t Read(char *buffer, size_t size) {
+ size_t n = std::min(size, data.size);
+ std::copy_n(data.begin(), n, buffer);
+ return n;
+ }
+
+ static size_t Callback(char *buffer, size_t size, size_t nitems,
+ void *instream) {
+ auto &rb = *(CurlRequestBody *)instream;
+ return rb.Read(buffer, size * nitems);
+ }
+};
+
+/**
+ * The (relevant) contents of a "<D:response>" element.
+ */
+struct DavResponse {
+ std::string href;
+ unsigned status = 0;
+ bool collection = false;
+ std::chrono::system_clock::time_point mtime =
+ std::chrono::system_clock::time_point::min();
+ uint64_t length = 0;
+
+ bool Check() const {
+ return !href.empty();
+ }
+};
+
+static unsigned
+ParseStatus(const char *s)
+{
+ /* skip the "HTTP/1.1" prefix */
+ const char *space = strchr(s, ' ');
+ if (space == nullptr)
+ return 0;
+
+ return strtoul(space + 1, nullptr, 10);
+}
+
+static unsigned
+ParseStatus(const char *s, size_t length)
+{
+ return ParseStatus(std::string(s, length).c_str());
+}
+
+static std::chrono::system_clock::time_point
+ParseTimeStamp(const char *s)
+{
+ try {
+ // TODO: make this more robust
+ return ParseTimePoint(s, "%a, %d %b %Y %T %Z");
+ } catch (const std::runtime_error &) {
+ return std::chrono::system_clock::time_point::min();
+ }
+}
+
+static std::chrono::system_clock::time_point
+ParseTimeStamp(const char *s, size_t length)
+{
+ return ParseTimeStamp(std::string(s, length).c_str());
+}
+
+static uint64_t
+ParseU64(const char *s)
+{
+ return strtoull(s, nullptr, 10);
+}
+
+static uint64_t
+ParseU64(const char *s, size_t length)
+{
+ return ParseU64(std::string(s, length).c_str());
+}
+
+/**
+ * A WebDAV PROPFIND request. Each "response" element will be passed
+ * to OnDavResponse() (to be implemented by a derived class).
+ */
+class PropfindOperation : BlockingHttpRequest, CommonExpatParser {
+ CurlSlist request_headers;
+ CurlRequestBody request_body;
+
+ enum class State {
+ ROOT,
+ RESPONSE,
+ HREF,
+ STATUS,
+ TYPE,
+ MTIME,
+ LENGTH,
+ } state = State::ROOT;
+
+ DavResponse response;
+
+public:
+ PropfindOperation(CurlGlobal &_curl, const char *_uri, unsigned depth)
+ :BlockingHttpRequest(_curl, _uri),
+ CommonExpatParser(ExpatNamespaceSeparator{'|'}),
+ request_body(request,
+ "<?xml version=\"1.0\"?>\n"
+ "<a:propfind xmlns:a=\"DAV:\">"
+ "<a:prop><a:getcontenttype/></a:prop>"
+ "<a:prop><a:getcontentlength/></a:prop>"
+ "</a:propfind>")
+ {
+ request.SetOption(CURLOPT_CUSTOMREQUEST, "PROPFIND");
+
+ char buffer[40];
+ sprintf(buffer, "depth: %u", depth);
+ request_headers.Append(buffer);
+
+ request.SetOption(CURLOPT_HTTPHEADER, request_headers.Get());
+
+ // TODO: send request body
+ }
+
+ using BlockingHttpRequest::Wait;
+
+protected:
+ virtual void OnDavResponse(DavResponse &&r) = 0;
+
+private:
+ void FinishResponse() {
+ if (response.Check())
+ OnDavResponse(std::move(response));
+ response = DavResponse();
+ }
+
+ /* virtual methods from CurlResponseHandler */
+ void OnHeaders(unsigned status,
+ std::multimap<std::string, std::string> &&headers) final {
+ if (status != 207)
+ throw FormatRuntimeError("Status %d from WebDAV server; expected \"207 Multi-Status\"",
+ status);
+
+ auto i = headers.find("content-type");
+ if (i == headers.end() ||
+ strncmp(i->second.c_str(), "text/xml", 8) != 0)
+ throw std::runtime_error("Unexpected Content-Type from WebDAV server");
+ }
+
+ void OnData(ConstBuffer<void> _data) final {
+ const auto data = ConstBuffer<char>::FromVoid(_data);
+ Parse(data.data, data.size, false);
+ }
+
+ void OnEnd() final {
+ Parse("", 0, true);
+ LockSetDone();
+ }
+
+ /* virtual methods from CommonExpatParser */
+ void StartElement(const XML_Char *name,
+ gcc_unused const XML_Char **attrs) final {
+ switch (state) {
+ case State::ROOT:
+ if (strcmp(name, "DAV:|response") == 0)
+ state = State::RESPONSE;
+ break;
+
+ case State::RESPONSE:
+ if (strcmp(name, "DAV:|href") == 0)
+ state = State::HREF;
+ else if (strcmp(name, "DAV:|status") == 0)
+ state = State::STATUS;
+ else if (strcmp(name, "DAV:|resourcetype") == 0)
+ state = State::TYPE;
+ else if (strcmp(name, "DAV:|getlastmodified") == 0)
+ state = State::MTIME;
+ else if (strcmp(name, "DAV:|getcontentlength") == 0)
+ state = State::LENGTH;
+ break;
+
+ case State::TYPE:
+ if (strcmp(name, "DAV:|collection") == 0)
+ response.collection = true;
+ break;
+
+ case State::HREF:
+ case State::STATUS:
+ case State::LENGTH:
+ case State::MTIME:
+ break;
+ }
+ }
+
+ void EndElement(const XML_Char *name) final {
+ switch (state) {
+ case State::ROOT:
+ break;
+
+ case State::RESPONSE:
+ if (strcmp(name, "DAV:|response") == 0) {
+ FinishResponse();
+ state = State::ROOT;
+ }
+
+ break;
+
+ case State::HREF:
+ if (strcmp(name, "DAV:|href") == 0)
+ state = State::RESPONSE;
+ break;
+
+ case State::STATUS:
+ if (strcmp(name, "DAV:|status") == 0)
+ state = State::RESPONSE;
+ break;
+
+ case State::TYPE:
+ if (strcmp(name, "DAV:|resourcetype") == 0)
+ state = State::RESPONSE;
+ break;
+
+ case State::MTIME:
+ if (strcmp(name, "DAV:|getlastmodified") == 0)
+ state = State::RESPONSE;
+ break;
+
+ case State::LENGTH:
+ if (strcmp(name, "DAV:|getcontentlength") == 0)
+ state = State::RESPONSE;
+ break;
+ }
+ }
+
+ void CharacterData(const XML_Char *s, int len) final {
+ switch (state) {
+ case State::ROOT:
+ case State::RESPONSE:
+ case State::TYPE:
+ break;
+
+ case State::HREF:
+ response.href.assign(s, len);
+ break;
+
+ case State::STATUS:
+ response.status = ParseStatus(s, len);
+ break;
+
+ case State::MTIME:
+ response.mtime = ParseTimeStamp(s, len);
+ break;
+
+ case State::LENGTH:
+ response.length = ParseU64(s, len);
+ break;
+ }
+ }
+};
+
+/**
+ * Obtain information about a single file using WebDAV PROPFIND.
+ */
+class HttpGetInfoOperation final : public PropfindOperation {
+ StorageFileInfo info;
+
+public:
+ HttpGetInfoOperation(CurlGlobal &curl, const char *uri)
+ :PropfindOperation(curl, uri, 0) {
+ info.type = StorageFileInfo::Type::OTHER;
+ info.size = 0;
+ info.mtime = 0;
+ info.device = info.inode = 0;
+ }
+
+ const StorageFileInfo &Perform() {
+ Wait();
+ return info;
+ }
+
+protected:
+ /* virtual methods from PropfindOperation */
+ void OnDavResponse(DavResponse &&r) override {
+ if (r.status != 200)
+ return;
+
+ info.type = r.collection
+ ? StorageFileInfo::Type::DIRECTORY
+ : StorageFileInfo::Type::REGULAR;
+ info.size = r.length;
+ info.mtime = r.mtime > std::chrono::system_clock::time_point()
+ ? std::chrono::system_clock::to_time_t(r.mtime)
+ : 0;
+ info.device = info.inode = 0;
+ }
+};
+
+StorageFileInfo
+CurlStorage::GetInfo(const char *uri_utf8, gcc_unused bool follow)
+{
+ // TODO: escape the given URI
+
+ std::string uri = base;
+ uri += uri_utf8;
+
+ return HttpGetInfoOperation(*curl, uri.c_str()).Perform();
+}
+
+gcc_pure
+static const char *
+UriPathOrSlash(const char *uri)
+{
+ const char *path = uri_get_path(uri);
+ if (path == nullptr)
+ path = "/";
+ return path;
+}
+
+/**
+ * Obtain a directory listing using WebDAV PROPFIND.
+ */
+class HttpListDirectoryOperation final : public PropfindOperation {
+ const std::string base_path;
+
+ MemoryStorageDirectoryReader::List entries;
+
+public:
+ HttpListDirectoryOperation(CurlGlobal &curl, const char *uri)
+ :PropfindOperation(curl, uri, 1),
+ base_path(UriPathOrSlash(uri)) {}
+
+ StorageDirectoryReader *Perform() {
+ Wait();
+ return ToReader();
+ }
+
+private:
+ StorageDirectoryReader *ToReader() {
+ return new MemoryStorageDirectoryReader(std::move(entries));
+ }
+
+ /**
+ * Convert a "href" attribute (which may be an absolute URI)
+ * to the base file name.
+ */
+ gcc_pure
+ StringView HrefToEscapedName(const char *href) const {
+ const char *path = uri_get_path(href);
+ if (path == nullptr)
+ return nullptr;
+
+ path = StringAfterPrefix(path, base_path.c_str());
+ if (path == nullptr || *path == 0)
+ return nullptr;
+
+ const char *slash = strchr(path, '/');
+ if (slash == nullptr)
+ /* regular file */
+ return path;
+ else if (slash[1] == 0)
+ /* trailing slash: collection; strip the slash */
+ return {path, slash};
+ else
+ /* strange, better ignore it */
+ return nullptr;
+ }
+
+protected:
+ /* virtual methods from PropfindOperation */
+ void OnDavResponse(DavResponse &&r) override {
+ if (r.status != 200)
+ return;
+
+ const auto escaped_name = HrefToEscapedName(r.href.c_str());
+ if (escaped_name.IsNull())
+ return;
+
+ // TODO: unescape
+ const auto name = escaped_name;
+
+ entries.emplace_front(std::string(name.data, name.size));
+
+ auto &info = entries.front().info;
+ info.type = r.collection
+ ? StorageFileInfo::Type::DIRECTORY
+ : StorageFileInfo::Type::REGULAR;
+ info.size = r.length;
+ info.mtime = r.mtime > std::chrono::system_clock::time_point()
+ ? std::chrono::system_clock::to_time_t(r.mtime)
+ : 0;
+ info.device = info.inode = 0;
+ }
+};
+
+StorageDirectoryReader *
+CurlStorage::OpenDirectory(const char *uri_utf8)
+{
+ // TODO: escape the given URI
+
+ std::string uri = base;
+ uri += uri_utf8;
+
+ /* collection URIs must end with a slash */
+ if (uri.back() != '/')
+ uri.push_back('/');
+
+ return HttpListDirectoryOperation(*curl, uri.c_str()).Perform();
+}
+
+static Storage *
+CreateCurlStorageURI(EventLoop &event_loop, const char *uri)
+{
+ if (strncmp(uri, "http://", 7) != 0 &&
+ strncmp(uri, "https://", 8) != 0)
+ return nullptr;
+
+ return new CurlStorage(event_loop, uri);
+}
+
+const StoragePlugin curl_storage_plugin = {
+ "curl",
+ CreateCurlStorageURI,
+};
diff --git a/src/storage/plugins/CurlStorage.hxx b/src/storage/plugins/CurlStorage.hxx
new file mode 100644
index 000000000..de63ef595
--- /dev/null
+++ b/src/storage/plugins/CurlStorage.hxx
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2003-2016 The Music Player Daemon Project
+ * http://www.musicpd.org
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef MPD_STORAGE_CURL_HXX
+#define MPD_STORAGE_CURL_HXX
+
+#include "check.h"
+
+struct StoragePlugin;
+
+extern const StoragePlugin curl_storage_plugin;
+
+#endif