From 58fb36bdb9efc058061fa5b35bb910a759be4381 Mon Sep 17 00:00:00 2001 From: Max Kellermann Date: Tue, 3 Jan 2017 07:25:14 +0100 Subject: [PATCH] storage/http: new storage plugin --- Makefile.am | 12 + NEWS | 2 + configure.ac | 13 + doc/user.xml | 12 + src/lib/curl/Global.hxx | 2 + src/storage/Registry.cxx | 4 + src/storage/plugins/CurlStorage.cxx | 590 ++++++++++++++++++++++++++++ src/storage/plugins/CurlStorage.hxx | 29 ++ 8 files changed, 664 insertions(+) create mode 100644 src/storage/plugins/CurlStorage.cxx create mode 100644 src/storage/plugins/CurlStorage.hxx diff --git a/Makefile.am b/Makefile.am index de02ebaa2..cdc621d57 100644 --- a/Makefile.am +++ b/Makefile.am @@ -699,6 +699,8 @@ libstorage_a_CPPFLAGS = $(AM_CPPFLAGS) \ STORAGE_LIBS = \ libstorage.a \ + $(CURL_LIBS) \ + $(EXPAT_LIBS) \ $(NFS_LIBS) \ $(SMBCLIENT_LIBS) @@ -714,6 +716,12 @@ libstorage_a_SOURCES += \ src/storage/plugins/NfsStorage.cxx src/storage/plugins/NfsStorage.hxx endif +if ENABLE_WEBDAV +libstorage_a_SOURCES += \ + src/lib/expat/ExpatParser.cxx \ + src/storage/plugins/CurlStorage.cxx src/storage/plugins/CurlStorage.hxx +endif + endif # neighbor plugins @@ -1768,6 +1776,10 @@ test_run_storage_SOURCES = \ test/ScopeIOThread.hxx \ test/run_storage.cxx +if ENABLE_WEBDAV +test_run_storage_SOURCES += $(CURL_SOURCES) +endif + endif test_run_input_LDADD = \ diff --git a/NEWS b/NEWS index 11b832967..d03044778 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ ver 0.20.1 (not yet released) * input - curl: fix crash bug +* storage + - curl: new storage plugin for WebDAV (work in progress) * mixer - alsa: normalize displayed volume according to human perception * fix crash with volume_normalization enabled diff --git a/configure.ac b/configure.ac index 420458b6f..4f9e3c540 100644 --- a/configure.ac +++ b/configure.ac @@ -720,6 +720,19 @@ fi MPD_ENABLE_AUTO_PKG(mms, MMS, [libmms >= 0.4], [libmms mms:// protocol support], [libmms not found]) +dnl --------------------------------------------------------------------------- +dnl Storage Plugins +dnl --------------------------------------------------------------------------- + +MPD_ENABLE_AUTO(webdav, WEBDAV, [WebDAV storage plugin], + [WebDAV requires libcurl and libexpat], + [auto], + [if test x$enable_curl = xyes && test x$enable_expat = xyes; then + found_webdav=yes + else + found_webdav=no + fi]) + dnl --------------------------------------------------------------------------- dnl Playlist Plugins dnl --------------------------------------------------------------------------- diff --git a/doc/user.xml b/doc/user.xml index a27b6be88..d7b4a74cd 100644 --- a/doc/user.xml +++ b/doc/user.xml @@ -1868,6 +1868,18 @@ run +
+ <varname>curl</varname> + + + A WebDAV client using libcurl. It used + used when music_directory contains a + http:// or + https:// URI, for example + "https://the.server/dav/". + +
+
<varname>smbclient</varname> diff --git a/src/lib/curl/Global.hxx b/src/lib/curl/Global.hxx index 9591b3f80..8669234c2 100644 --- a/src/lib/curl/Global.hxx +++ b/src/lib/curl/Global.hxx @@ -46,6 +46,8 @@ class CurlGlobal final : TimeoutMonitor, DeferredMonitor { public: explicit CurlGlobal(EventLoop &_loop); + using TimeoutMonitor::GetEventLoop; + void Add(CURL *easy, CurlRequest &request); void Remove(CURL *easy); diff --git a/src/storage/Registry.cxx b/src/storage/Registry.cxx index 4901da6a8..d0104359c 100644 --- a/src/storage/Registry.cxx +++ b/src/storage/Registry.cxx @@ -23,6 +23,7 @@ #include "plugins/LocalStorage.hxx" #include "plugins/SmbclientStorage.hxx" #include "plugins/NfsStorage.hxx" +#include "plugins/CurlStorage.hxx" #include #include @@ -34,6 +35,9 @@ const StoragePlugin *const storage_plugins[] = { #endif #ifdef ENABLE_NFS &nfs_storage_plugin, +#endif +#ifdef ENABLE_WEBDAV + &curl_storage_plugin, #endif nullptr }; diff --git a/src/storage/plugins/CurlStorage.cxx b/src/storage/plugins/CurlStorage.cxx new file mode 100644 index 000000000..cc65badfd --- /dev/null +++ b/src/storage/plugins/CurlStorage.cxx @@ -0,0 +1,590 @@ +/* + * Copyright 2003-2016 The Music Player Daemon Project + * http://www.musicpd.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "config.h" +#include "CurlStorage.hxx" +#include "storage/StoragePlugin.hxx" +#include "storage/StorageInterface.hxx" +#include "storage/FileInfo.hxx" +#include "storage/MemoryDirectoryReader.hxx" +#include "lib/curl/Global.hxx" +#include "lib/curl/Slist.hxx" +#include "lib/curl/Request.hxx" +#include "lib/curl/Handler.hxx" +#include "lib/expat/ExpatParser.hxx" +#include "fs/Traits.hxx" +#include "event/Call.hxx" +#include "event/DeferredMonitor.hxx" +#include "thread/Mutex.hxx" +#include "thread/Cond.hxx" +#include "util/RuntimeError.hxx" +#include "util/StringCompare.hxx" +#include "util/TimeParser.hxx" +#include "util/UriUtil.hxx" + +#include +#include +#include +#include + +#include + +class CurlStorage final : public Storage { + const std::string base; + + CurlGlobal *const curl; + +public: + CurlStorage(EventLoop &_loop, const char *_base) + :base(_base), + curl(new CurlGlobal(_loop)) {} + + ~CurlStorage() { + BlockingCall(curl->GetEventLoop(), [this](){ delete curl; }); + } + + /* virtual methods from class Storage */ + StorageFileInfo GetInfo(const char *uri_utf8, bool follow) override; + + StorageDirectoryReader *OpenDirectory(const char *uri_utf8) override; + + std::string MapUTF8(const char *uri_utf8) const override; + + const char *MapToRelativeUTF8(const char *uri_utf8) const override; +}; + +std::string +CurlStorage::MapUTF8(const char *uri_utf8) const +{ + assert(uri_utf8 != nullptr); + + if (StringIsEmpty(uri_utf8)) + return base; + + // TODO: escape the given URI + + return PathTraitsUTF8::Build(base.c_str(), uri_utf8); +} + +const char * +CurlStorage::MapToRelativeUTF8(const char *uri_utf8) const +{ + // TODO: escape/unescape? + + return PathTraitsUTF8::Relative(base.c_str(), uri_utf8); +} + +class BlockingHttpRequest : protected CurlResponseHandler, DeferredMonitor { + std::exception_ptr postponed_error; + + bool done = false; + +protected: + CurlRequest request; + + Mutex mutex; + Cond cond; + +public: + BlockingHttpRequest(CurlGlobal &curl, const char *uri) + :DeferredMonitor(curl.GetEventLoop()), + request(curl, uri, *this) { + // TODO: use CurlInputStream's configuration + + /* start the transfer inside the IOThread */ + DeferredMonitor::Schedule(); + } + + void Wait() { + const std::lock_guard lock(mutex); + while (!done) + cond.wait(mutex); + + if (postponed_error) + std::rethrow_exception(postponed_error); + } + +protected: + void SetDone() { + assert(!done); + + request.Stop(); + done = true; + cond.signal(); + } + + void LockSetDone() { + const std::lock_guard lock(mutex); + SetDone(); + } + +private: + /* virtual methods from DeferredMonitor */ + void RunDeferred() final { + assert(!done); + + request.Start(); + } + + /* virtual methods from CurlResponseHandler */ + void OnError(std::exception_ptr e) final { + const std::lock_guard lock(mutex); + postponed_error = std::move(e); + SetDone(); + } +}; + +/** + * A helper class which feeds a (foreign) memory buffer into the + * CURLOPT_READFUNCTION. + */ +class CurlRequestBody { + ConstBuffer data; + +public: + explicit CurlRequestBody(ConstBuffer _data) + :data(ConstBuffer::FromVoid(_data)) {} + + explicit constexpr CurlRequestBody(StringView _data) + :data(_data) {} + + template + CurlRequestBody(CurlRequest &request, T _data) + :CurlRequestBody(_data) { + request.SetOption(CURLOPT_READFUNCTION, Callback); + request.SetOption(CURLOPT_READDATA, this); + } + +private: + size_t Read(char *buffer, size_t size) { + size_t n = std::min(size, data.size); + std::copy_n(data.begin(), n, buffer); + return n; + } + + static size_t Callback(char *buffer, size_t size, size_t nitems, + void *instream) { + auto &rb = *(CurlRequestBody *)instream; + return rb.Read(buffer, size * nitems); + } +}; + +/** + * The (relevant) contents of a "" element. + */ +struct DavResponse { + std::string href; + unsigned status = 0; + bool collection = false; + std::chrono::system_clock::time_point mtime = + std::chrono::system_clock::time_point::min(); + uint64_t length = 0; + + bool Check() const { + return !href.empty(); + } +}; + +static unsigned +ParseStatus(const char *s) +{ + /* skip the "HTTP/1.1" prefix */ + const char *space = strchr(s, ' '); + if (space == nullptr) + return 0; + + return strtoul(space + 1, nullptr, 10); +} + +static unsigned +ParseStatus(const char *s, size_t length) +{ + return ParseStatus(std::string(s, length).c_str()); +} + +static std::chrono::system_clock::time_point +ParseTimeStamp(const char *s) +{ + try { + // TODO: make this more robust + return ParseTimePoint(s, "%a, %d %b %Y %T %Z"); + } catch (const std::runtime_error &) { + return std::chrono::system_clock::time_point::min(); + } +} + +static std::chrono::system_clock::time_point +ParseTimeStamp(const char *s, size_t length) +{ + return ParseTimeStamp(std::string(s, length).c_str()); +} + +static uint64_t +ParseU64(const char *s) +{ + return strtoull(s, nullptr, 10); +} + +static uint64_t +ParseU64(const char *s, size_t length) +{ + return ParseU64(std::string(s, length).c_str()); +} + +/** + * A WebDAV PROPFIND request. Each "response" element will be passed + * to OnDavResponse() (to be implemented by a derived class). + */ +class PropfindOperation : BlockingHttpRequest, CommonExpatParser { + CurlSlist request_headers; + CurlRequestBody request_body; + + enum class State { + ROOT, + RESPONSE, + HREF, + STATUS, + TYPE, + MTIME, + LENGTH, + } state = State::ROOT; + + DavResponse response; + +public: + PropfindOperation(CurlGlobal &_curl, const char *_uri, unsigned depth) + :BlockingHttpRequest(_curl, _uri), + CommonExpatParser(ExpatNamespaceSeparator{'|'}), + request_body(request, + "\n" + "" + "" + "" + "") + { + request.SetOption(CURLOPT_CUSTOMREQUEST, "PROPFIND"); + + char buffer[40]; + sprintf(buffer, "depth: %u", depth); + request_headers.Append(buffer); + + request.SetOption(CURLOPT_HTTPHEADER, request_headers.Get()); + + // TODO: send request body + } + + using BlockingHttpRequest::Wait; + +protected: + virtual void OnDavResponse(DavResponse &&r) = 0; + +private: + void FinishResponse() { + if (response.Check()) + OnDavResponse(std::move(response)); + response = DavResponse(); + } + + /* virtual methods from CurlResponseHandler */ + void OnHeaders(unsigned status, + std::multimap &&headers) final { + if (status != 207) + throw FormatRuntimeError("Status %d from WebDAV server; expected \"207 Multi-Status\"", + status); + + auto i = headers.find("content-type"); + if (i == headers.end() || + strncmp(i->second.c_str(), "text/xml", 8) != 0) + throw std::runtime_error("Unexpected Content-Type from WebDAV server"); + } + + void OnData(ConstBuffer _data) final { + const auto data = ConstBuffer::FromVoid(_data); + Parse(data.data, data.size, false); + } + + void OnEnd() final { + Parse("", 0, true); + LockSetDone(); + } + + /* virtual methods from CommonExpatParser */ + void StartElement(const XML_Char *name, + gcc_unused const XML_Char **attrs) final { + switch (state) { + case State::ROOT: + if (strcmp(name, "DAV:|response") == 0) + state = State::RESPONSE; + break; + + case State::RESPONSE: + if (strcmp(name, "DAV:|href") == 0) + state = State::HREF; + else if (strcmp(name, "DAV:|status") == 0) + state = State::STATUS; + else if (strcmp(name, "DAV:|resourcetype") == 0) + state = State::TYPE; + else if (strcmp(name, "DAV:|getlastmodified") == 0) + state = State::MTIME; + else if (strcmp(name, "DAV:|getcontentlength") == 0) + state = State::LENGTH; + break; + + case State::TYPE: + if (strcmp(name, "DAV:|collection") == 0) + response.collection = true; + break; + + case State::HREF: + case State::STATUS: + case State::LENGTH: + case State::MTIME: + break; + } + } + + void EndElement(const XML_Char *name) final { + switch (state) { + case State::ROOT: + break; + + case State::RESPONSE: + if (strcmp(name, "DAV:|response") == 0) { + FinishResponse(); + state = State::ROOT; + } + + break; + + case State::HREF: + if (strcmp(name, "DAV:|href") == 0) + state = State::RESPONSE; + break; + + case State::STATUS: + if (strcmp(name, "DAV:|status") == 0) + state = State::RESPONSE; + break; + + case State::TYPE: + if (strcmp(name, "DAV:|resourcetype") == 0) + state = State::RESPONSE; + break; + + case State::MTIME: + if (strcmp(name, "DAV:|getlastmodified") == 0) + state = State::RESPONSE; + break; + + case State::LENGTH: + if (strcmp(name, "DAV:|getcontentlength") == 0) + state = State::RESPONSE; + break; + } + } + + void CharacterData(const XML_Char *s, int len) final { + switch (state) { + case State::ROOT: + case State::RESPONSE: + case State::TYPE: + break; + + case State::HREF: + response.href.assign(s, len); + break; + + case State::STATUS: + response.status = ParseStatus(s, len); + break; + + case State::MTIME: + response.mtime = ParseTimeStamp(s, len); + break; + + case State::LENGTH: + response.length = ParseU64(s, len); + break; + } + } +}; + +/** + * Obtain information about a single file using WebDAV PROPFIND. + */ +class HttpGetInfoOperation final : public PropfindOperation { + StorageFileInfo info; + +public: + HttpGetInfoOperation(CurlGlobal &curl, const char *uri) + :PropfindOperation(curl, uri, 0) { + info.type = StorageFileInfo::Type::OTHER; + info.size = 0; + info.mtime = 0; + info.device = info.inode = 0; + } + + const StorageFileInfo &Perform() { + Wait(); + return info; + } + +protected: + /* virtual methods from PropfindOperation */ + void OnDavResponse(DavResponse &&r) override { + if (r.status != 200) + return; + + info.type = r.collection + ? StorageFileInfo::Type::DIRECTORY + : StorageFileInfo::Type::REGULAR; + info.size = r.length; + info.mtime = r.mtime > std::chrono::system_clock::time_point() + ? std::chrono::system_clock::to_time_t(r.mtime) + : 0; + info.device = info.inode = 0; + } +}; + +StorageFileInfo +CurlStorage::GetInfo(const char *uri_utf8, gcc_unused bool follow) +{ + // TODO: escape the given URI + + std::string uri = base; + uri += uri_utf8; + + return HttpGetInfoOperation(*curl, uri.c_str()).Perform(); +} + +gcc_pure +static const char * +UriPathOrSlash(const char *uri) +{ + const char *path = uri_get_path(uri); + if (path == nullptr) + path = "/"; + return path; +} + +/** + * Obtain a directory listing using WebDAV PROPFIND. + */ +class HttpListDirectoryOperation final : public PropfindOperation { + const std::string base_path; + + MemoryStorageDirectoryReader::List entries; + +public: + HttpListDirectoryOperation(CurlGlobal &curl, const char *uri) + :PropfindOperation(curl, uri, 1), + base_path(UriPathOrSlash(uri)) {} + + StorageDirectoryReader *Perform() { + Wait(); + return ToReader(); + } + +private: + StorageDirectoryReader *ToReader() { + return new MemoryStorageDirectoryReader(std::move(entries)); + } + + /** + * Convert a "href" attribute (which may be an absolute URI) + * to the base file name. + */ + gcc_pure + StringView HrefToEscapedName(const char *href) const { + const char *path = uri_get_path(href); + if (path == nullptr) + return nullptr; + + path = StringAfterPrefix(path, base_path.c_str()); + if (path == nullptr || *path == 0) + return nullptr; + + const char *slash = strchr(path, '/'); + if (slash == nullptr) + /* regular file */ + return path; + else if (slash[1] == 0) + /* trailing slash: collection; strip the slash */ + return {path, slash}; + else + /* strange, better ignore it */ + return nullptr; + } + +protected: + /* virtual methods from PropfindOperation */ + void OnDavResponse(DavResponse &&r) override { + if (r.status != 200) + return; + + const auto escaped_name = HrefToEscapedName(r.href.c_str()); + if (escaped_name.IsNull()) + return; + + // TODO: unescape + const auto name = escaped_name; + + entries.emplace_front(std::string(name.data, name.size)); + + auto &info = entries.front().info; + info.type = r.collection + ? StorageFileInfo::Type::DIRECTORY + : StorageFileInfo::Type::REGULAR; + info.size = r.length; + info.mtime = r.mtime > std::chrono::system_clock::time_point() + ? std::chrono::system_clock::to_time_t(r.mtime) + : 0; + info.device = info.inode = 0; + } +}; + +StorageDirectoryReader * +CurlStorage::OpenDirectory(const char *uri_utf8) +{ + // TODO: escape the given URI + + std::string uri = base; + uri += uri_utf8; + + /* collection URIs must end with a slash */ + if (uri.back() != '/') + uri.push_back('/'); + + return HttpListDirectoryOperation(*curl, uri.c_str()).Perform(); +} + +static Storage * +CreateCurlStorageURI(EventLoop &event_loop, const char *uri) +{ + if (strncmp(uri, "http://", 7) != 0 && + strncmp(uri, "https://", 8) != 0) + return nullptr; + + return new CurlStorage(event_loop, uri); +} + +const StoragePlugin curl_storage_plugin = { + "curl", + CreateCurlStorageURI, +}; diff --git a/src/storage/plugins/CurlStorage.hxx b/src/storage/plugins/CurlStorage.hxx new file mode 100644 index 000000000..de63ef595 --- /dev/null +++ b/src/storage/plugins/CurlStorage.hxx @@ -0,0 +1,29 @@ +/* + * Copyright 2003-2016 The Music Player Daemon Project + * http://www.musicpd.org + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef MPD_STORAGE_CURL_HXX +#define MPD_STORAGE_CURL_HXX + +#include "check.h" + +struct StoragePlugin; + +extern const StoragePlugin curl_storage_plugin; + +#endif