Pluggable libheimbase interface for DBs and misc libheimbase enhancements

[Code reviewed by Love Hörnquist Åstrand <lha@kth.se>]

    Added heim_db_*() entry points for dealing with databases, and
    make krb5_aname_to_localname() use it.

    The following enhancements to libheimbase are included:

     - Add heim_data_t and heim_string_t "reference" variants to
       avoid memory copies of potentially large data/strings.

       See heim_data_ref_create() and heim_string_ref_create().

     - Added enhancements to heim_array_t to allow their use for
       queues and stacks, and to improve performance.  See
       heim_array_insert_value().

     - Added XPath-like accessors for heim_object_t.  See
       heim_path_get(), heim_path_copy(), heim_path_create(), and
       heim_path_delete().  These are used extensively in the DB
       framework's generic composition of ACID support and in the
       test_base program

     - Made libheimbase more consistent with Core Foundation naming
       conventions.  See heim_{dict, array}_{get, copy}_value() and
       heim_path_{get, copy}().

     - Added functionality to and fixed bugs in base/json.c:
        - heim_serialize();
        - depth limit for JSON parsing (for DoS protection);
        - pretty-printing;
        - JSON compliance (see below);
        - flag options for parsing and serializing; these are needed
          because of impedance mismatches between heim_object_t and
          JSON (e.g., heim_dict_t allows non-string keys, but JSON
          does not; heimbase supports binary data, while JSON does
          not).

     - Added heim_error_enomem().

     - Enhanced the test_base program to test new functionality and
       to use heim_path*() to better test JSON encoding.  This
       includes some fuzz testing of JSON parsing, and running the
       test under valgrind.

     - Started to add doxygen documentation for libheimbase (but doc
       build for libheimbase is still incomplete).

    Note that there's still some incomplete JSON support:

     - JSON string quoting is not fully implemented;

     - libheimbase lacks support for real numbers, while JSON has
       it -- otherwise libheimbase is a superset of JSON,
       specifically in that any heim_object_t can be a key for an
       associative array.

    The following DB backends are supported natively:

     - "sorted-text", a binary search of sorted (in C locale), flat
       text files;

     - "json", a backend that stores DB contents serialized as JSON
       (this is intended for configuration-like contents).

    The DB framework supports:

     - multiple key/value tables per-DB
     - ACID transactions

    The DB framework also natively implements ACID transactions for
    any DB backends that a) do not provide transactions natively, b)
    do provide lock/unlock/sync methods (even on Windows).  This
    includes autocommit of DB updates outside transactions.

    Future DB enhancements may include:

     - add backends for various DB types (BDB, CDB, MDB, ...);

     - make libhdb use heim_db_t;

     - add a command-line tool for interfacing to databases via
       libheimbase (e.g., to get/set/delete values, create/copy/
       backup DBs, inspect history, check integrity);

     - framework-level transaction logging (with redo and undo
       logging), for generic incremental replication;

     - framework-level DB integrity checking.

       We could store a MAC of the XOR of a hash function applied to
       {key, value} for every entry in the DB, then use this to check
       DB integrity incrementally during incremental replication, as
       well as for the whole DB.

base/heimbase.c

@@ -76,7 +76,7 @@ struct heim_auto_release {
 
 
 /**
- * Retain object
+ * Retain object (i.e., take a reference)
  *
  * @param object to be released, NULL is ok
  *
@@ -100,7 +100,7 @@ heim_retain(void *ptr)
 }
 
 /**
- * Release object, free is reference count reaches zero
+ * Release object, free if reference count reaches zero
  *
  * @param object to be released
  */
@@ -257,6 +257,18 @@ struct heim_type_data memory_object = {
     NULL
 };
 
+/**
+ * Allocate memory for an object of anonymous type
+ *
+ * @param size size of object to be allocated
+ * @param name name of ad-hoc type
+ * @param dealloc destructor function
+ *
+ * Objects allocated with this interface do not serialize.
+ *
+ * @return allocated object
+ */
+
 void *
 heim_alloc(size_t size, const char *name, heim_type_dealloc dealloc)
 {
@@ -310,6 +322,18 @@ _heim_alloc_object(heim_type_t type, size_t size)
     return BASE2PTR(p);
 }
 
+void *
+_heim_get_isaextra(heim_object_t ptr, size_t idx)
+{
+    struct heim_base *p = (struct heim_base *)PTR2BASE(ptr);
+
+    heim_assert(ptr != NULL, "internal error");
+    if (p->isa == &memory_object)
+	return NULL;
+    heim_assert(idx < 3, "invalid private heim_base extra data index");
+    return &p->isaextra[idx];
+}
+
 heim_tid_t
 _heim_type_get_tid(heim_type_t type)
 {
@@ -489,7 +513,11 @@ static struct heim_type_data _heim_autorel_object = {
 };
 
 /**
+ * Create thread-specific object auto-release pool
  *
+ * Objects placed on the per-thread auto-release pool (with
+ * heim_auto_release()) can be released in one fell swoop by calling
+ * heim_auto_release_drain().
  */
 
 heim_auto_release_t
@@ -515,7 +543,9 @@ heim_auto_release_create(void)
 }
 
 /**
- * Mark the current object as a
+ * Place the current object on the thread's auto-release pool
+ *
+ * @param ptr object
  */
 
 void
@@ -546,7 +576,7 @@ heim_auto_release(heim_object_t ptr)
 }
 
 /**
- *
+ * Release all objects on the given auto-release pool
  */
 
 void
@@ -565,3 +595,369 @@ heim_auto_release_drain(heim_auto_release_t autorel)
     }
     HEIMDAL_MUTEX_unlock(&autorel->pool_mutex);
 }
+
+/*
+ * Helper for heim_path_vget() and heim_path_delete().  On success
+ * outputs the node named by the path and the parent node and key
+ * (useful for heim_path_delete()).
+ */
+
+static heim_object_t
+heim_path_vget2(heim_object_t ptr, heim_object_t *parent, heim_object_t *key,
+		heim_error_t *error, va_list ap)
+{
+    heim_object_t path_element;
+    heim_object_t node, next_node;
+    heim_tid_t node_type;
+
+    *parent = NULL;
+    *key = NULL;
+    if (ptr == NULL)
+	return NULL;
+
+    for (node = ptr; node != NULL; ) {
+	path_element = va_arg(ap, heim_object_t);
+	if (path_element == NULL) {
+	    *parent = node;
+	    *key = path_element;
+	    return node;
+	}
+
+	node_type = heim_get_tid(node);
+	switch (node_type) {
+	case HEIM_TID_ARRAY:
+	case HEIM_TID_DICT:
+	case HEIM_TID_DB:
+	    break;
+	default:
+	    if (node == ptr)
+		heim_abort("heim_path_get() only operates on container types");
+	    return NULL;
+	}
+
+	if (node_type == HEIM_TID_DICT) {
+	    next_node = heim_dict_get_value(node, path_element);
+	} else if (node_type == HEIM_TID_DB) {
+	    next_node = _heim_db_get_value(node, NULL, path_element, NULL);
+	} else if (node_type == HEIM_TID_ARRAY) {
+	    int idx = -1;
+
+	    if (heim_get_tid(path_element) == HEIM_TID_NUMBER)
+		idx = heim_number_get_int(path_element);
+	    if (idx < 0) {
+		if (error)
+		    *error = heim_error_create(EINVAL,
+					       "heim_path_get() path elements "
+					       "for array nodes must be "
+					       "numeric and positive");
+		return NULL;
+	    }
+	    next_node = heim_array_get_value(node, idx);
+	} else {
+	    if (error)
+		*error = heim_error_create(EINVAL,
+					   "heim_path_get() node in path "
+					   "not a container type");
+	    return NULL;
+	}
+	node = next_node;
+    }
+    return NULL;
+}
+
+/**
+ * Get a node in a heim_object tree by path
+ *
+ * @param ptr tree
+ * @param error error (output)
+ * @param ap NULL-terminated va_list of heim_object_ts that form a path
+ *
+ * @return object (not retained) if found
+ *
+ * @addtogroup heimbase
+ */
+
+heim_object_t
+heim_path_vget(heim_object_t ptr, heim_error_t *error, va_list ap)
+{
+    heim_object_t p, k;
+
+    return heim_path_vget2(ptr, &p, &k, error, ap);
+}
+
+/**
+ * Get a node in a tree by path, with retained reference
+ *
+ * @param ptr tree
+ * @param error error (output)
+ * @param ap NULL-terminated va_list of heim_object_ts that form a path
+ *
+ * @return retained object if found
+ *
+ * @addtogroup heimbase
+ */
+
+heim_object_t
+heim_path_vcopy(heim_object_t ptr, heim_error_t *error, va_list ap)
+{
+    heim_object_t p, k;
+
+    return heim_retain(heim_path_vget2(ptr, &p, &k, error, ap));
+}
+
+/**
+ * Get a node in a tree by path
+ *
+ * @param ptr tree
+ * @param error error (output)
+ * @param ... NULL-terminated va_list of heim_object_ts that form a path
+ *
+ * @return object (not retained) if found
+ *
+ * @addtogroup heimbase
+ */
+
+heim_object_t
+heim_path_get(heim_object_t ptr, heim_error_t *error, ...)
+{
+    heim_object_t o;
+    heim_object_t p, k;
+    va_list ap;
+
+    if (ptr == NULL)
+	return NULL;
+
+    va_start(ap, error);
+    o = heim_path_vget2(ptr, &p, &k, error, ap);
+    va_end(ap);
+    return o;
+}
+
+/**
+ * Get a node in a tree by path, with retained reference
+ *
+ * @param ptr tree
+ * @param error error (output)
+ * @param ... NULL-terminated va_list of heim_object_ts that form a path
+ *
+ * @return retained object if found
+ *
+ * @addtogroup heimbase
+ */
+
+heim_object_t
+heim_path_copy(heim_object_t ptr, heim_error_t *error, ...)
+{
+    heim_object_t o;
+    heim_object_t p, k;
+    va_list ap;
+
+    if (ptr == NULL)
+	return NULL;
+
+    va_start(ap, error);
+    o = heim_retain(heim_path_vget2(ptr, &p, &k, error, ap));
+    va_end(ap);
+    return o;
+}
+
+/**
+ * Create a path in a heim_object_t tree
+ *
+ * @param ptr the tree
+ * @param size the size of the heim_dict_t nodes to be created
+ * @param leaf leaf node to be added, if any
+ * @param error error (output)
+ * @param ap NULL-terminated of path component objects
+ *
+ * Create a path of heim_dict_t interior nodes in a given heim_object_t
+ * tree, as necessary, and set/replace a leaf, if given (if leaf is NULL
+ * then the leaf is not deleted).
+ *
+ * @return 0 on success, else a system error
+ *
+ * @addtogroup heimbase
+ */
+
+int
+heim_path_vcreate(heim_object_t ptr, size_t size, heim_object_t leaf,
+		  heim_error_t *error, va_list ap)
+{
+    heim_object_t path_element = va_arg(ap, heim_object_t);
+    heim_object_t next_path_element = NULL;
+    heim_object_t node = ptr;
+    heim_object_t next_node = NULL;
+    heim_tid_t node_type;
+    int ret;
+
+    if (ptr == NULL)
+	heim_abort("heim_path_vcreate() does not create root nodes");
+
+    while (path_element != NULL) {
+	next_path_element = va_arg(ap, heim_object_t);
+	node_type = heim_get_tid(node);
+
+	if (node_type == HEIM_TID_DICT) {
+	    next_node = heim_dict_get_value(node, path_element);
+	} else if (node_type == HEIM_TID_ARRAY) {
+	    int idx = -1;
+
+	    if (heim_get_tid(path_element) == HEIM_TID_NUMBER)
+		idx = heim_number_get_int(path_element);
+	    if (idx < 0) {
+		if (error)
+		    *error = heim_error_create(EINVAL,
+					       "heim_path() path elements for "
+					       "array nodes must be numeric "
+					       "and positive");
+		return EINVAL;
+	    }
+	    if (idx < heim_array_get_length(node))
+		next_node = heim_array_get_value(node, idx);
+	    else
+		next_node = NULL;
+	} else if (node_type == HEIM_TID_DB && next_path_element != NULL) {
+	    if (error)
+		*error = heim_error_create(EINVAL, "Interior node is a DB");
+	    return EINVAL;
+	}
+
+	if (next_path_element == NULL)
+	    break;
+
+	/* Create missing interior node */
+	if (next_node == NULL) {
+	    next_node = heim_dict_create(size); /* no arrays or DBs, just dicts */
+	    if (next_node == NULL) {
+		ret = ENOMEM;
+		goto err;
+	    }
+
+	    if (node_type == HEIM_TID_DICT) {
+		ret = heim_dict_set_value(node, path_element, next_node);
+	    } else if (node_type == HEIM_TID_ARRAY &&
+		heim_number_get_int(path_element) <= heim_array_get_length(node)) {
+		ret = heim_array_insert_value(node,
+					      heim_number_get_int(path_element),
+					      next_node);
+	    } else {
+		ret = EINVAL;
+		if (error)
+		    *error = heim_error_create(ret, "Node in path not a "
+					       "container");
+		goto err;
+	    }
+	    heim_release(next_node);
+	    if (ret)
+		goto err;
+	}
+
+	path_element = next_path_element;
+	node = next_node;
+	next_node = NULL;
+    }
+
+    if (path_element == NULL)
+	goto err;
+
+    /* Add the leaf */
+    if (leaf != NULL) {
+	if (node_type == HEIM_TID_DICT)
+	    ret = heim_dict_set_value(node, path_element, leaf);
+	else
+	    ret = heim_array_insert_value(node,
+					  heim_number_get_int(path_element),
+					  leaf);
+    }
+    return 0;
+
+err:
+    if (error && !*error) {
+	if (ret == ENOMEM)
+	    *error = heim_error_enomem();
+	else
+	    *error = heim_error_create(ret, "Could not set "
+				       "dict value");
+    }
+    return ret;
+}
+
+/**
+ * Create a path in a heim_object_t tree
+ *
+ * @param ptr the tree
+ * @param size the size of the heim_dict_t nodes to be created
+ * @param leaf leaf node to be added, if any
+ * @param error error (output)
+ * @param ... NULL-terminated list of path component objects
+ *
+ * Create a path of heim_dict_t interior nodes in a given heim_object_t
+ * tree, as necessary, and set/replace a leaf, if given (if leaf is NULL
+ * then the leaf is not deleted).
+ *
+ * @return 0 on success, else a system error
+ *
+ * @addtogroup heimbase
+ */
+
+int
+heim_path_create(heim_object_t ptr, size_t size, heim_object_t leaf,
+		 heim_error_t *error, ...)
+{
+    va_list ap;
+    int ret;
+
+    va_start(ap, error);
+    ret = heim_path_vcreate(ptr, size, leaf, error, ap);
+    va_end(ap);
+    return ret;
+}
+
+/**
+ * Delete leaf node named by a path in a heim_object_t tree
+ *
+ * @param ptr the tree
+ * @param error error (output)
+ * @param ap NULL-terminated list of path component objects
+ *
+ * @addtogroup heimbase
+ */
+
+void
+heim_path_vdelete(heim_object_t ptr, heim_error_t *error, va_list ap)
+{
+    heim_object_t parent, key, child;
+
+    child = heim_path_vget2(ptr, &parent, &key, error, ap);
+    if (child != NULL) {
+	if (heim_get_tid(parent) == HEIM_TID_DICT)
+	    heim_dict_delete_key(parent, key);
+	else if (heim_get_tid(parent) == HEIM_TID_DB)
+	    heim_db_delete_key(parent, NULL, key, error);
+	else if (heim_get_tid(parent) == HEIM_TID_ARRAY)
+	    heim_array_delete_value(parent, heim_number_get_int(key));
+	heim_release(child);
+    }
+}
+
+/**
+ * Delete leaf node named by a path in a heim_object_t tree
+ *
+ * @param ptr the tree
+ * @param error error (output)
+ * @param ap NULL-terminated list of path component objects
+ *
+ * @addtogroup heimbase
+ */
+
+void
+heim_path_delete(heim_object_t ptr, heim_error_t *error, ...)
+{
+    va_list ap;
+
+    va_start(ap, error);
+    heim_path_vdelete(ptr, error, ap);
+    va_end(ap);
+    return;
+}
+