tokenizer: convert to C++

Makefile.am

@@ -98,7 +98,6 @@ mpd_headers = \
 	src/tag_ape.h \
 	src/tag_id3.h \
 	src/tag_rva2.h \
-	src/tokenizer.h \
 	src/utils.h \
 	src/string_util.h \
 	src/timer.h \
@@ -242,7 +241,6 @@ src_mpd_SOURCES = \
 	src/TagSave.cxx src/TagSave.hxx \
 	src/tag_handler.c src/tag_handler.h \
 	src/TagFile.cxx src/TagFile.hxx \
-	src/tokenizer.c \
 	src/TextFile.cxx src/TextFile.hxx \
 	src/text_input_stream.c \
 	src/utils.c \
@@ -290,6 +288,7 @@ endif
 # Generic utility library
 
 libutil_a_SOURCES = \
+	src/util/Tokenizer.cxx src/util/Tokenizer.hxx \
 	src/util/UriUtil.cxx src/util/UriUtil.hxx \
 	src/util/Manual.hxx \
 	src/util/RefCount.hxx \
@@ -1055,10 +1054,11 @@ endif
 
 test_read_conf_LDADD = \
 	libconf.a \
+	libutil.a \
 	libfs.a \
 	$(GLIB_LIBS)
 test_read_conf_SOURCES = test/read_conf.cxx \
-	src/tokenizer.c src/utils.c src/string_util.c
+	src/utils.c src/string_util.c
 
 test_run_resolver_LDADD = \
 	$(GLIB_LIBS)
@@ -1081,7 +1081,7 @@ test_DumpDatabase_SOURCES = test/DumpDatabase.cxx \
 	src/Tag.cxx src/TagNames.c src/TagPool.cxx src/TagSave.cxx \
 	src/SongFilter.cxx \
 	src/TextFile.cxx \
-	src/tokenizer.c src/utils.c src/string_util.c
+	src/utils.c src/string_util.c
 
 test_run_input_LDADD = \
 	$(INPUT_LIBS) \
@@ -1094,7 +1094,7 @@ test_run_input_LDADD = \
 test_run_input_SOURCES = test/run_input.cxx \
 	test/stdbin.h \
 	src/IOThread.cxx \
-	src/tokenizer.c src/utils.c src/string_util.c\
+	src/utils.c src/string_util.c\
 	src/Tag.cxx src/TagNames.c src/TagPool.cxx src/TagSave.cxx \
 	src/fd_util.c
 
@@ -1111,7 +1111,7 @@ test_visit_archive_LDADD = \
 test_visit_archive_SOURCES = test/visit_archive.cxx \
 	src/IOThread.cxx \
 	src/InputStream.cxx \
-	src/tokenizer.c src/utils.c src/string_util.c \
+	src/utils.c src/string_util.c \
 	src/Tag.cxx src/TagNames.c src/TagPool.cxx \
 	src/fd_util.c
 
@@ -1132,7 +1132,7 @@ test_dump_text_file_LDADD = \
 test_dump_text_file_SOURCES = test/dump_text_file.cxx \
 	test/stdbin.h \
 	src/IOThread.cxx \
-	src/tokenizer.c src/utils.c src/string_util.c\
+	src/utils.c src/string_util.c\
 	src/Tag.cxx src/TagNames.c src/TagPool.cxx \
 	src/text_input_stream.c \
 	src/fd_util.c
@@ -1152,7 +1152,7 @@ test_dump_playlist_LDADD = \
 test_dump_playlist_SOURCES = test/dump_playlist.cxx \
 	$(DECODER_SRC) \
 	src/IOThread.cxx \
-	src/tokenizer.c src/utils.c src/string_util.c\
+	src/utils.c src/string_util.c\
 	src/Song.cxx src/Tag.cxx src/TagNames.c src/TagPool.cxx src/TagSave.cxx \
 	src/tag_handler.c src/TagFile.cxx \
 	src/audio_check.c src/pcm_buffer.c \
@@ -1180,7 +1180,7 @@ test_run_decoder_LDADD = \
 test_run_decoder_SOURCES = test/run_decoder.cxx \
 	test/stdbin.h \
 	src/IOThread.cxx \
-	src/tokenizer.c src/utils.c src/string_util.c \
+	src/utils.c src/string_util.c \
 	src/Tag.cxx src/TagNames.c src/TagPool.cxx src/tag_handler.c \
 	src/ReplayGainInfo.cxx \
 	src/fd_util.c \
@@ -1204,7 +1204,7 @@ test_read_tags_LDADD = \
 	$(GLIB_LIBS)
 test_read_tags_SOURCES = test/read_tags.cxx \
 	src/IOThread.cxx \
-	src/tokenizer.c src/utils.c src/string_util.c \
+	src/utils.c src/string_util.c \
 	src/Tag.cxx src/TagNames.c src/TagPool.cxx src/tag_handler.c \
 	src/ReplayGainInfo.cxx \
 	src/fd_util.c \
@@ -1225,13 +1225,14 @@ endif
 test_run_filter_LDADD = \
 	$(FILTER_LIBS) \
 	libconf.a \
+	libutil.a \
 	libfs.a \
 	$(GLIB_LIBS)
 test_run_filter_SOURCES = test/run_filter.cxx \
 	test/FakeReplayGainConfig.cxx \
 	test/stdbin.h \
 	src/FilterPlugin.cxx src/FilterRegistry.cxx \
-	src/tokenizer.c src/utils.c src/string_util.c \
+	src/utils.c src/string_util.c \
 	src/audio_check.c \
 	src/audio_format.c \
 	src/AudioParser.cxx \
@@ -1250,7 +1251,7 @@ if ENABLE_ENCODER
 noinst_PROGRAMS += test/run_encoder
 test_run_encoder_SOURCES = test/run_encoder.cxx \
 	test/stdbin.h \
-	src/tokenizer.c src/utils.c src/string_util.c \
+	src/utils.c src/string_util.c \
 	src/Tag.cxx src/TagNames.c src/TagPool.cxx \
 	src/audio_check.c \
 	src/audio_format.c \
@@ -1269,7 +1270,7 @@ if ENABLE_VORBIS_ENCODER
 noinst_PROGRAMS += test/test_vorbis_encoder
 test_test_vorbis_encoder_SOURCES = test/test_vorbis_encoder.cxx \
 	test/stdbin.h \
-	src/tokenizer.c src/utils.c src/string_util.c \
+	src/utils.c src/string_util.c \
 	src/Tag.cxx src/TagNames.c src/TagPool.cxx \
 	src/audio_check.c \
 	src/audio_format.c \
@@ -1325,7 +1326,7 @@ test_run_output_LDADD = $(MPD_LIBS) \
 test_run_output_SOURCES = test/run_output.cxx \
 	test/FakeReplayGainConfig.cxx \
 	test/stdbin.h \
-	src/tokenizer.c src/utils.c src/string_util.c \
+	src/utils.c src/string_util.c \
 	src/IOThread.cxx \
 	src/audio_check.c \
 	src/audio_format.c \
@@ -1351,11 +1352,12 @@ test_read_mixer_LDADD = \
 	libmixer_plugins.a \
 	$(OUTPUT_LIBS) \
 	libconf.a \
+	libutil.a \
 	libevent.a \
 	libfs.a \
 	$(GLIB_LIBS)
 test_read_mixer_SOURCES = test/read_mixer.cxx \
-	src/tokenizer.c src/utils.c src/string_util.c \
+	src/utils.c src/string_util.c \
 	src/MixerControl.cxx \
 	src/MixerInternal.cxx \
 	src/FilterPlugin.cxx \

src/AllCommands.cxx

@@ -31,10 +31,7 @@
 #include "tag.h"
 #include "protocol/Result.hxx"
 #include "Client.hxx"
-
+#include "util/Tokenizer.hxx"
-extern "C" {
-#include "tokenizer.h"
-}
 
 #ifdef ENABLE_SQLITE
 #include "StickerCommands.hxx"
@@ -329,10 +326,11 @@ command_process(Client *client, unsigned num, char *line)
 
 	/* get the command name (first word on the line) */
 
-	argv[0] = tokenizer_next_word(&line, &error);
+	Tokenizer tokenizer(line);
+	argv[0] = tokenizer.NextWord(&error);
 	if (argv[0] == NULL) {
 		current_command = "";
-		if (*line == 0)
+		if (tokenizer.IsEnd())
 			command_error(client, ACK_ERROR_UNKNOWN,
 				      "No command given");
 		else {
@@ -351,7 +349,7 @@ command_process(Client *client, unsigned num, char *line)
 
 	while (argc < (int)G_N_ELEMENTS(argv) &&
 	       (argv[argc] =
-		tokenizer_next_param(&line, &error)) != NULL)
+		tokenizer.NextParam(&error)) != NULL)
 		++argc;
 
 	/* some error checks; we have to set current_command because
@@ -365,7 +363,7 @@ command_process(Client *client, unsigned num, char *line)
 		return COMMAND_RETURN_ERROR;
 	}
 
-	if (*line != 0) {
+	if (!tokenizer.IsEnd()) {
 		command_error(client, ACK_ERROR_ARG,
 			      "%s", error->message);
 		current_command = NULL;

src/ConfigFile.cxx

@@ -23,10 +23,10 @@
 #include "ConfigData.hxx"
 #include "ConfigTemplates.hxx"
 #include "conf.h"
+#include "util/Tokenizer.hxx"
 
 extern "C" {
 #include "string_util.h"
-#include "tokenizer.h"
 }
 
 #include "fs/Path.hxx"
@@ -50,15 +50,17 @@ static bool
 config_read_name_value(struct config_param *param, char *input, unsigned line,
 		       GError **error_r)
 {
-	const char *name = tokenizer_next_word(&input, error_r);
+	Tokenizer tokenizer(input);
+
+	const char *name = tokenizer.NextWord(error_r);
 	if (name == NULL) {
-		assert(*input != 0);
+		assert(!tokenizer.IsEnd());
 		return false;
 	}
 
-	const char *value = tokenizer_next_string(&input, error_r);
+	const char *value = tokenizer.NextString(error_r);
 	if (value == NULL) {
-		if (*input == 0) {
+		if (tokenizer.IsEnd()) {
 			assert(error_r == NULL || *error_r == NULL);
 			g_set_error(error_r, config_quark(), 0,
 				    "Value missing");
@@ -69,7 +71,7 @@ config_read_name_value(struct config_param *param, char *input, unsigned line,
 		return false;
 	}
 
-	if (*input != 0 && *input != CONF_COMMENT) {
+	if (!tokenizer.IsEnd() && tokenizer.CurrentChar() != CONF_COMMENT) {
 		g_set_error(error_r, config_quark(), 0,
 			    "Unknown tokens after value");
 		return false;
@@ -173,9 +175,10 @@ ReadConfigFile(ConfigData &config_data, FILE *fp, GError **error_r)
 		/* the first token in each line is the name, followed
 		   by either the value or '{' */
 
-		name = tokenizer_next_word(&line, &error);
+		Tokenizer tokenizer(line);
+		name = tokenizer.NextWord(&error);
 		if (name == NULL) {
-			assert(*line != 0);
+			assert(!tokenizer.IsEnd());
 			g_propagate_prefixed_error(error_r, error,
 						   "line %i: ", count);
 			return false;
@@ -210,13 +213,13 @@ ReadConfigFile(ConfigData &config_data, FILE *fp, GError **error_r)
 		if (option.block) {
 			/* it's a block, call config_read_block() */
 
-			if (*line != '{') {
+			if (tokenizer.CurrentChar() != '{') {
 				g_set_error(error_r, config_quark(), 0,
 					    "line %i: '{' expected", count);
 				return false;
 			}
 
-			line = strchug_fast(line + 1);
+			line = strchug_fast(tokenizer.Rest() + 1);
 			if (*line != 0 && *line != CONF_COMMENT) {
 				g_set_error(error_r, config_quark(), 0,
 					    "line %i: Unknown tokens after '{'",
@@ -231,9 +234,9 @@ ReadConfigFile(ConfigData &config_data, FILE *fp, GError **error_r)
 		} else {
 			/* a string value */
 
-			value = tokenizer_next_string(&line, &error);
+			value = tokenizer.NextString(&error);
 			if (value == NULL) {
-				if (*line == 0)
+				if (tokenizer.IsEnd())
 					g_set_error(error_r, config_quark(), 0,
 						    "line %i: Value missing",
 						    count);
@@ -247,7 +250,8 @@ ReadConfigFile(ConfigData &config_data, FILE *fp, GError **error_r)
 				return false;
 			}
 
-			if (*line != 0 && *line != CONF_COMMENT) {
+			if (!tokenizer.IsEnd() &&
+			    tokenizer.CurrentChar() != CONF_COMMENT) {
 				g_set_error(error_r, config_quark(), 0,
 					    "line %i: Unknown tokens after value",
 					    count);

src/tokenizer.h

@@ -1,83 +0,0 @@
-/*
- * Copyright (C) 2003-2011 The Music Player Daemon Project
- * http://www.musicpd.org
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- */
-
-#ifndef MPD_TOKENIZER_H
-#define MPD_TOKENIZER_H
-
-#include "gerror.h"
-
-/**
- * Reads the next word from the input string.  This function modifies
- * the input string.
- *
- * @param input_p the input string; this function returns a pointer to
- * the first non-whitespace character of the following token
- * @param error_r if this function returns NULL and **input_p!=0, it
- * optionally provides a GError object in this argument
- * @return a pointer to the null-terminated word, or NULL on error or
- * end of line
- */
-char *
-tokenizer_next_word(char **input_p, GError **error_r);
-
-/**
- * Reads the next unquoted word from the input string.  This function
- * modifies the input string.
- *
- * @param input_p the input string; this function returns a pointer to
- * the first non-whitespace character of the following token
- * @param error_r if this function returns NULL and **input_p!=0, it
- * optionally provides a GError object in this argument
- * @return a pointer to the null-terminated word, or NULL on error or
- * end of line
- */
-char *
-tokenizer_next_unquoted(char **input_p, GError **error_r);
-
-/**
- * Reads the next quoted string from the input string.  A backslash
- * escapes the following character.  This function modifies the input
- * string.
- *
- * @param input_p the input string; this function returns a pointer to
- * the first non-whitespace character of the following token
- * @param error_r if this function returns NULL and **input_p!=0, it
- * optionally provides a GError object in this argument
- * @return a pointer to the null-terminated string, or NULL on error
- * or end of line
- */
-char *
-tokenizer_next_string(char **input_p, GError **error_r);
-
-/**
- * Reads the next unquoted word or quoted string from the input.  This
- * is a wrapper for tokenizer_next_unquoted() and
- * tokenizer_next_string().
- *
- * @param input_p the input string; this function returns a pointer to
- * the first non-whitespace character of the following token
- * @param error_r if this function returns NULL and **input_p!=0, it
- * optionally provides a GError object in this argument
- * @return a pointer to the null-terminated string, or NULL on error
- * or end of line
- */
-char *
-tokenizer_next_param(char **input_p, GError **error_r);
-
-#endif

src/util/Tokenizer.cxx

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2003-2011 The Music Player Daemon Project
+ * Copyright (C) 2003-2013 The Music Player Daemon Project
  * http://www.musicpd.org
  *
  * This program is free software; you can redistribute it and/or modify
@@ -18,7 +18,7 @@
  */
 
 #include "config.h"
-#include "tokenizer.h"
+#include "Tokenizer.hxx"
 #include "string_util.h"
 
 #include <glib.h>
@@ -47,24 +47,19 @@ valid_word_char(char ch)
 }
 
 char *
-tokenizer_next_word(char **input_p, GError **error_r)
+Tokenizer::NextWord(GError **error_r)
 {
-	char *word, *input;
+	char *const word = input;
-
-	assert(input_p != NULL);
-	assert(*input_p != NULL);
-
-	word = input = *input_p;
 
 	if (*input == 0)
-		return NULL;
+		return nullptr;
 
 	/* check the first character */
 
 	if (!valid_word_first_char(*input)) {
 		g_set_error(error_r, tokenizer_quark(), 0,
 			    "Letter expected");
-		return NULL;
+		return nullptr;
 	}
 
 	/* now iterate over the other characters until we find a
@@ -80,17 +75,15 @@ tokenizer_next_word(char **input_p, GError **error_r)
 		}
 
 		if (!valid_word_char(*input)) {
-			*input_p = input;
 			g_set_error(error_r, tokenizer_quark(), 0,
 				    "Invalid word character");
-			return NULL;
+			return nullptr;
 		}
 	}
 
 	/* end of string: the string is already null-terminated
 	   here */
 
-	*input_p = input;
 	return word;
 }
 
@@ -101,24 +94,19 @@ valid_unquoted_char(char ch)
 }
 
 char *
-tokenizer_next_unquoted(char **input_p, GError **error_r)
+Tokenizer::NextUnquoted(GError **error_r)
 {
-	char *word, *input;
+	char *const word = input;
-
-	assert(input_p != NULL);
-	assert(*input_p != NULL);
-
-	word = input = *input_p;
 
 	if (*input == 0)
-		return NULL;
+		return nullptr;
 
 	/* check the first character */
 
 	if (!valid_unquoted_char(*input)) {
 		g_set_error(error_r, tokenizer_quark(), 0,
 			    "Invalid unquoted character");
-		return NULL;
+		return nullptr;
 	}
 
 	/* now iterate over the other characters until we find a
@@ -134,40 +122,33 @@ tokenizer_next_unquoted(char **input_p, GError **error_r)
 		}
 
 		if (!valid_unquoted_char(*input)) {
-			*input_p = input;
 			g_set_error(error_r, tokenizer_quark(), 0,
 				    "Invalid unquoted character");
-			return NULL;
+			return nullptr;
 		}
 	}
 
 	/* end of string: the string is already null-terminated
 	   here */
 
-	*input_p = input;
 	return word;
 }
 
 char *
-tokenizer_next_string(char **input_p, GError **error_r)
+Tokenizer::NextString(GError **error_r)
 {
-	char *word, *dest, *input;
+	char *const word = input, *dest = input;
-
-	assert(input_p != NULL);
-	assert(*input_p != NULL);
-
-	word = dest = input = *input_p;
 
 	if (*input == 0)
 		/* end of line */
-		return NULL;
+		return nullptr;
 
 	/* check for the opening " */
 
 	if (*input != '"') {
 		g_set_error(error_r, tokenizer_quark(), 0,
 			    "'\"' expected");
-		return NULL;
+		return nullptr;
 	}
 
 	++input;
@@ -184,10 +165,10 @@ tokenizer_next_string(char **input_p, GError **error_r)
 			/* return input-1 so the caller can see the
 			   difference between "end of line" and
 			   "error" */
-			*input_p = input - 1;
+			--input;
 			g_set_error(error_r, tokenizer_quark(), 0,
 				    "Missing closing '\"'");
-			return NULL;
+			return nullptr;
 		}
 
 		/* copy one character */
@@ -199,27 +180,23 @@ tokenizer_next_string(char **input_p, GError **error_r)
 
 	++input;
 	if (*input != 0 && !g_ascii_isspace(*input)) {
-		*input_p = input;
 		g_set_error(error_r, tokenizer_quark(), 0,
 			    "Space expected after closing '\"'");
-		return NULL;
+		return nullptr;
 	}
 
 	/* finish the string and return it */
 
 	*dest = 0;
-	*input_p = strchug_fast(input);
+	input = strchug_fast(input);
 	return word;
 }
 
 char *
-tokenizer_next_param(char **input_p, GError **error_r)
+Tokenizer::NextParam(GError **error_r)
 {
-	assert(input_p != NULL);
+	if (*input == '"')
-	assert(*input_p != NULL);
+		return NextString(error_r);
-
-	if (**input_p == '"')
-		return tokenizer_next_string(input_p, error_r);
 	else
-		return tokenizer_next_unquoted(input_p, error_r);
+		return NextUnquoted(error_r);
 }

src/util/Tokenizer.hxx

@@ -0,0 +1,99 @@
+/*
+ * Copyright (C) 2003-2013 The Music Player Daemon Project
+ * http://www.musicpd.org
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef MPD_TOKENIZER_HXX
+#define MPD_TOKENIZER_HXX
+
+#include "gerror.h"
+
+class Tokenizer {
+	char *input;
+
+public:
+	/**
+	 * @param _input the input string; the contents will be
+	 * modified by this class
+	 */
+	constexpr Tokenizer(char *_input):input(_input) {}
+
+	Tokenizer(const Tokenizer &) = delete;
+	Tokenizer &operator=(const Tokenizer &) = delete;
+
+	char *Rest() {
+		return input;
+	}
+
+	char CurrentChar() const {
+		return *input;
+	}
+
+	bool IsEnd() const {
+		return CurrentChar() == 0;
+	}
+
+	/**
+	 * Reads the next word.
+	 *
+	 * @param error_r if this function returns nullptr and
+	 * **input_p!=0, it optionally provides a GError object in
+	 * this argument
+	 * @return a pointer to the null-terminated word, or nullptr
+	 * on error or end of line
+	 */
+	char *NextWord(GError **error_r);
+
+	/**
+	 * Reads the next unquoted word from the input string.
+	 *
+	 * @param error_r if this function returns nullptr and **input_p!=0, it
+	 * optionally provides a GError object in this argument
+	 * @return a pointer to the null-terminated word, or nullptr
+	 * on error or end of line
+	 */
+	char *NextUnquoted(GError **error_r);
+
+	/**
+	 * Reads the next quoted string from the input string.  A backslash
+	 * escapes the following character.  This function modifies the input
+	 * string.
+	 *
+	 * @param input_p the input string; this function returns a pointer to
+	 * the first non-whitespace character of the following token
+	 * @param error_r if this function returns nullptr and **input_p!=0, it
+	 * optionally provides a GError object in this argument
+	 * @return a pointer to the null-terminated string, or nullptr on error
+	 * or end of line
+	 */
+	char *NextString(GError **error_r);
+
+	/**
+	 * Reads the next unquoted word or quoted string from the
+	 * input.  This is a wrapper for NextUnquoted() and
+	 * NextString().
+	 *
+	 * @param error_r if this function returns nullptr and
+	 * **input_p!=0, it optionally provides a GError object in
+	 * this argument
+	 * @return a pointer to the null-terminated string, or nullptr
+	 * on error or end of line
+	 */
+	char *NextParam(GError **error_r);
+};
+
+#endif