json: Add json encoder/decoder

Using the jsmn library for decoding.
Based on the original implementation by Svyatoslav Mishyn <juef@openmailbox.org>

Signed-off-by: Steve Bennett <steveb@workware.net.au>

6 files changed, 750 insertions, 8 deletions

diff --git a/auto.def b/auto.def
index e80618a..6ac056f 100644
--- a/auto.def
+++ b/auto.def
@@ -55,6 +55,7 @@ options {
         binary    - Tcl-compatible 'binary' command
         tclprefix - Support for the tcl::prefix command
         zlib      - Interface to zlib
+        json      - JSON encode/decode
 
         These are disabled unless explicitly enabled:
 
@@ -337,6 +338,8 @@ dict set extdb attrs {
     glob      { tcl }
     history   {}
     interp    { }
+    json      { optional }
+    jsonencode { tcl optional }
     load      { static }
     mk        { cpp off }
     namespace { static }
@@ -372,6 +375,7 @@ dict set extdb info {
     load     { check {[have-feature dlopen-compat] || [cc-check-function-in-lib dlopen dl]} libdep lib_dlopen }
     mk       { check {[check-metakit]} libdep lib_mk }
     namespace { dep nshelper }
+    json     { dep jsonencode extrasrcs jsmn/jsmn.c }
     posix    { check {[have-feature waitpid]} }
     readdir  { check {[have-feature opendir]} }
     readline { pkg-config readline check {[cc-check-function-in-lib readline readline]} libdep lib_readline}
@@ -460,6 +464,14 @@ if {$jimregexp || [opt-bool jim-regexp]} {
     }
 }
 
+foreach mod $extinfo(static-c) {
+    if {[dict exists $extdb info $mod extrasrcs]} {
+        foreach src [dict get $extdb info $mod extrasrcs] {
+            lappend extra_objs {*}[file rootname $src].o
+        }
+    }
+}
+
 # poor-man's signals
 if {"signal" ni $extinfo(static-c)} {
     lappend extra_objs jim-nosignal.o
@@ -501,13 +513,19 @@ set lines {}
 foreach mod $extinfo(module-c) {
     set objs {}
     set libs [get-define LDLIBS_$mod]
-    set src jim-$mod.c
-    lappend lines "$mod.so: $src"
-    set obj [file rootname $src].o
-    lappend lines "\t\$(ECHO)\t\"\tCC\t$obj\""
-    lappend lines "\t\$(Q)\$(CC) \$(CFLAGS) \$(SHOBJ_CFLAGS) -c -o $obj $src"
+    set srcs jim-$mod.c
+    if {[dict exists $extdb info $mod extrasrcs]} {
+        lappend srcs {*}[dict get $extdb info $mod extrasrcs]
+    }
+    lappend lines "$mod.so: $srcs"
+    foreach src $srcs {
+        set obj [file rootname $src].o
+        lappend objs $obj
+        lappend lines "\t\$(ECHO)\t\"\tCC\t$obj\""
+        lappend lines "\t\$(Q)\$(CC) \$(CFLAGS) \$(SHOBJ_CFLAGS) -c -o $obj $src"
+    }
     lappend lines "\t\$(ECHO)\t\"\tLDSO\t\$@\""
-    lappend lines "\t\$(Q)\$(CC) \$(CFLAGS) \$(LDFLAGS) \$(SHOBJ_LDFLAGS) -o \$@ $obj \$(SH_LIBJIM) $libs"
+    lappend lines "\t\$(Q)\$(CC) \$(CFLAGS) \$(LDFLAGS) \$(SHOBJ_LDFLAGS) -o \$@ $objs \$(SH_LIBJIM) $libs"
     lappend lines ""
 }
 define BUILD_SHOBJS [join $lines \n]
diff --git a/jim-json.c b/jim-json.c
new file mode 100644
index 0000000..c75ac74
--- /dev/null
+++ b/jim-json.c
@@ -0,0 +1,414 @@
+/*
+ * Copyright (c) 2015 - 2016 Svyatoslav Mishyn <juef@openmailbox.org>
+ * Copyright (c) 2019 Steve Bennett <steveb@workware.net.au>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for
+ * any purpose with or without fee is hereby granted, provided that the
+ * above copyright notice and this permission notice appear in all
+ * copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
+ * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
+ * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
+ * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
+ * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <assert.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <jim.h>
+
+#include "jsmn/jsmn.h"
+
+/* These are all the schema types we support */
+typedef enum {
+	JSON_BOOL,
+	JSON_OBJ,
+	JSON_LIST,
+	JSON_MIXED,
+	JSON_STR,
+	JSON_NUM,
+	JSON_MAX_TYPE,
+} json_schema_t;
+
+struct json_state {
+	Jim_Obj *nullObj;
+	const char *json;
+	jsmntok_t *tok;
+	int need_subst;
+	/* The following are used for -schema */
+	int enable_schema;
+	Jim_Obj *schemaObj;
+	Jim_Obj *schemaTypeObj[JSON_MAX_TYPE];
+};
+
+static void json_decode_dump_value(Jim_Interp *interp, struct json_state *state, Jim_Obj *list);
+
+/**
+ * Start a new subschema. Returns the previous schemaObj.
+ * Does nothing and returns NULL if -schema is not enabled.
+ */
+static Jim_Obj *json_decode_schema_push(Jim_Interp *interp, struct json_state *state)
+{
+	Jim_Obj *prevSchemaObj = NULL;
+	if (state->enable_schema) {
+		prevSchemaObj = state->schemaObj;
+		state->schemaObj = Jim_NewListObj(interp, NULL, 0);
+		Jim_IncrRefCount(state->schemaObj);
+	}
+	return prevSchemaObj;
+}
+
+/**
+ * Combines the current schema with the previous schema, prevSchemaObj
+ * returned by json_decode_schema_push().
+ * Does nothing if -schema is not enabled.
+ */
+static void json_decode_schema_pop(Jim_Interp *interp, struct json_state *state, Jim_Obj *prevSchemaObj)
+{
+	if (state->enable_schema) {
+		Jim_ListAppendElement(interp, prevSchemaObj, state->schemaObj);
+		Jim_DecrRefCount(interp, state->schemaObj);
+		state->schemaObj = prevSchemaObj;
+	}
+}
+
+/**
+ * Appends the schema type to state->schemaObj based on 'type'
+ */
+static void json_decode_add_schema_type(Jim_Interp *interp, struct json_state *state, json_schema_t type)
+{
+	static const char * const schema_names[] = {
+		"bool",
+		"obj",
+		"list",
+		"mixed",
+		"str",
+		"num",
+	};
+	assert(type >= 0 && type < JSON_MAX_TYPE);
+	/* Share multiple instances of the same type */
+	if (state->schemaTypeObj[type] == NULL) {
+		state->schemaTypeObj[type] = Jim_NewStringObj(interp, schema_names[type], -1);
+	}
+	Jim_ListAppendElement(interp, state->schemaObj, state->schemaTypeObj[type]);
+}
+
+/**
+ * Returns the schema type for the given token.
+ * There is a one-to-one correspondence except for JSMN_PRIMITIVE
+ * which will return JSON_BOOL for true, false and JSON_NUM otherise.
+ */
+static json_schema_t json_decode_get_type(const jsmntok_t *tok, const char *json)
+{
+	switch (tok->type) {
+		case JSMN_PRIMITIVE:
+			assert(json);
+			if (json[tok->start] == 't' || json[tok->start] == 'f') {
+				return JSON_BOOL;
+			}
+			return JSON_NUM;
+		case JSMN_OBJECT:
+			return JSON_OBJ;
+		case JSMN_ARRAY:
+			/* Return mixed by default - need other checks to select list instead */
+			return JSON_MIXED;
+		case JSMN_STRING:
+		default:
+			return JSON_STR;
+	}
+}
+
+/**
+ * Returns the current object (state->tok) as a Tcl list.
+ *
+ * state->tok is incremented to just past the object that was dumped.
+ */
+static Jim_Obj *
+json_decode_dump_container(Jim_Interp *interp, struct json_state *state)
+{
+	int i;
+	Jim_Obj *list = Jim_NewListObj(interp, NULL, 0);
+	int size = state->tok->size;
+	int type = state->tok->type;
+	json_schema_t container_type = JSON_OBJ; /* JSON_LIST, JSON_MIXED or JSON_OBJ */
+
+	if (state->schemaObj) {
+		json_schema_t list_type;
+		/* Figure out the type to use for the container */
+		if (type == JSMN_ARRAY) {
+			/* If every element of the array is of the same primitive schema type (str, bool or num),
+			 * we can use "list", otherwise need to use "mixed"
+			 */
+			container_type = JSON_LIST;
+			if (size) {
+				list_type = json_decode_get_type(&state->tok[1], state->json);
+
+				if (list_type == JSON_BOOL || list_type == JSON_STR || list_type == JSON_NUM) {
+					for (i = 2; i <= size; i++) {
+						if (json_decode_get_type(state->tok + i, state->json) != list_type) {
+							/* Can't use list */
+							container_type = JSON_MIXED;
+							break;
+						}
+					}
+				}
+			}
+		}
+		json_decode_add_schema_type(interp, state, container_type);
+		if (container_type == JSON_LIST && size) {
+			json_decode_add_schema_type(interp, state, list_type);
+		}
+	}
+
+	state->tok++;
+
+	for (i = 0; i < size; i++) {
+		if (type == JSMN_OBJECT) {
+			/* Dump the object key */
+			if (state->enable_schema) {
+				const char *p = state->json + state->tok->start;
+				int len = state->tok->end - state->tok->start;
+				Jim_ListAppendElement(interp, state->schemaObj, Jim_NewStringObj(interp, p, len));
+			}
+			json_decode_dump_value(interp, state, list);
+		}
+
+		if (state->schemaObj && container_type != JSON_LIST) {
+			if (state->tok->type == JSMN_STRING || state->tok->type == JSMN_PRIMITIVE) {
+				json_decode_add_schema_type(interp, state, json_decode_get_type(state->tok, state->json));
+			}
+		}
+
+		/* Dump the array or object value */
+		json_decode_dump_value(interp, state, list);
+	}
+
+	return list;
+}
+
+/**
+ * Appends the value at state->tok to 'list' and increments state->tok to just
+ * past that token.
+ *
+ * Also appends to the schema if state->enable_schema is set.
+ */
+static void
+json_decode_dump_value(Jim_Interp *interp, struct json_state *state, Jim_Obj *list)
+{
+	const jsmntok_t *t = state->tok;
+
+	if (t->type == JSMN_STRING || t->type == JSMN_PRIMITIVE) {
+		Jim_Obj	*elem;
+		int len = t->end - t->start;
+		const char *p = state->json + t->start;
+		if (t->type == JSMN_STRING) {
+			/* Do we need to process backslash escapes? */
+			if (state->need_subst == 0 && memchr(p, '\\', len) != NULL) {
+				state->need_subst = 1;
+			}
+			elem = Jim_NewStringObj(interp, p, len);
+		} else if (p[0] == 'n') {	/* null */
+			elem = state->nullObj;
+		} else if (p[0] == 'I') {
+			elem = Jim_NewStringObj(interp, "Inf", -1);
+		} else if (p[0] == '-' && p[1] == 'I') {
+			elem = Jim_NewStringObj(interp, "-Inf", -1);
+		} else {		/* number, true or false */
+			elem = Jim_NewStringObj(interp, p, len);
+		}
+
+		Jim_ListAppendElement(interp, list, elem);
+		state->tok++;
+	}
+	else {
+		Jim_Obj *prevSchemaObj = json_decode_schema_push(interp, state);
+		Jim_Obj *newList = json_decode_dump_container(interp, state);
+		Jim_ListAppendElement(interp, list, newList);
+		json_decode_schema_pop(interp, state, prevSchemaObj);
+	}
+}
+
+/* Parses the options ?-null string? ?-schema? *state.
+ * Any options not present are not set.
+ *
+ * Returns JIM_OK or JIM_ERR and sets an error result.
+ */
+static int parse_json_decode_options(Jim_Interp *interp, int argc, Jim_Obj *const argv[], struct json_state *state)
+{
+	static const char * const options[] = { "-null", "-schema", NULL };
+	enum { OPT_NULL, OPT_SCHEMA, };
+	int i;
+
+	for (i = 1; i < argc - 1; i++) {
+		int option;
+		if (Jim_GetEnum(interp, argv[i], options, &option, NULL, JIM_ERRMSG | JIM_ENUM_ABBREV) != JIM_OK) {
+			return JIM_ERR;
+		}
+		switch (option) {
+			case OPT_NULL:
+				i++;
+				Jim_IncrRefCount(argv[i]);
+				Jim_DecrRefCount(interp, state->nullObj);
+				state->nullObj = argv[i];
+				break;
+
+			case OPT_SCHEMA:
+				state->enable_schema = 1;
+				break;
+		}
+	}
+
+	if (i != argc - 1) {
+		Jim_WrongNumArgs(interp, 1, argv,
+			"?-null nullvalue? ?-schema? json");
+		return JIM_ERR;
+	}
+
+	return JIM_OK;
+}
+
+/**
+ * Use jsmn to tokenise the JSON string 'json' of length 'len'
+ *
+ * Returns an allocated array of tokens or NULL on error (and sets an error result)
+ */
+static jsmntok_t *
+json_decode_tokenize(Jim_Interp *interp, const char *json, size_t len)
+{
+	jsmntok_t	*t;
+	jsmn_parser	 parser;
+	int n;
+
+	/* Parse once just to find the number of tokens */
+	jsmn_init(&parser);
+	n = jsmn_parse(&parser, json, len, NULL, 0);
+
+error:
+	switch (n) {
+		case JSMN_ERROR_INVAL:
+			Jim_SetResultString(interp, "invalid JSON string", -1);
+			return NULL;
+
+		case JSMN_ERROR_PART:
+			Jim_SetResultString(interp, "truncated JSON string", -1);
+			return NULL;
+
+		case 0:
+			Jim_SetResultString(interp, "root element must be an object or an array", -1);
+			return NULL;
+
+		default:
+			break;
+	}
+
+	if (n < 0) {
+		return NULL;
+	}
+
+	t = Jim_Alloc(n * sizeof(*t));
+
+	jsmn_init(&parser);
+	n = jsmn_parse(&parser, json, len, t, n);
+	if (t->type != JSMN_OBJECT && t->type != JSMN_ARRAY) {
+		n = 0;
+	}
+	if (n <= 0) {
+		Jim_Free(t);
+		goto error;
+	}
+
+	return t;
+}
+
+/**
+ * json::decode returns the decoded data structure.
+ *
+ * If -schema is specified, returns a list of {data schema}
+ */
+static int
+json_decode(Jim_Interp *interp, int argc, Jim_Obj *const argv[])
+{
+	Jim_Obj *list;
+	jsmntok_t *tokens;
+	int len;
+	int ret = JIM_ERR;
+	struct json_state state;
+
+	memset(&state, 0, sizeof(state));
+
+	state.nullObj = Jim_NewStringObj(interp, "null", -1);
+	Jim_IncrRefCount(state.nullObj);
+
+	if (parse_json_decode_options(interp, argc, argv, &state) != JIM_OK) {
+		goto done;
+	}
+
+	state.json = Jim_GetString(argv[argc - 1], &len);
+
+	if (!len) {
+		Jim_SetResultString(interp, "empty JSON string", -1);
+		goto done;
+	}
+	if ((tokens = json_decode_tokenize(interp, state.json, len)) == NULL) {
+		goto done;
+	}
+	state.tok = tokens;
+	json_decode_schema_push(interp, &state);
+
+	list = json_decode_dump_container(interp, &state);
+	Jim_Free(tokens);
+	ret = JIM_OK;
+
+	/* Make sure the refcount doesn't go to 0 during Jim_SubstObj() */
+	Jim_IncrRefCount(list);
+
+	if (state.need_subst) {
+		/* Subsitute backslashes in the returned dictionary.
+		 * Need to be careful of refcounts.
+		 * Note that Jim_SubstObj() supports a few more escapes than
+		 * JSON requires, but should give the same result for all legal escapes.
+		 */
+		Jim_Obj *newList;
+		Jim_SubstObj(interp, list, &newList, JIM_SUBST_FLAG | JIM_SUBST_NOCMD | JIM_SUBST_NOVAR);
+		Jim_IncrRefCount(newList);
+		Jim_DecrRefCount(interp, list);
+		list = newList;
+	}
+
+	if (state.schemaObj) {
+		Jim_Obj *resultObj = Jim_NewListObj(interp, NULL, 0);
+		Jim_ListAppendElement(interp, resultObj, list);
+		Jim_ListAppendElement(interp, resultObj, state.schemaObj);
+		Jim_SetResult(interp, resultObj);
+		Jim_DecrRefCount(interp, state.schemaObj);
+	}
+	else {
+		Jim_SetResult(interp, list);
+	}
+	Jim_DecrRefCount(interp, list);
+
+done:
+	Jim_DecrRefCount(interp, state.nullObj);
+
+	return ret;
+}
+
+int
+Jim_jsonInit(Jim_Interp *interp)
+{
+	if (Jim_PackageProvide(interp, "json", "1.0", JIM_ERRMSG) != JIM_OK) {
+		return JIM_ERR;
+	}
+
+	Jim_CreateCommand(interp, "json::decode", json_decode, NULL, NULL);
+	/* Load the Tcl implementation of the json encoder if possible */
+	Jim_PackageRequire(interp, "jsonencode", 0);
+	return JIM_OK;
+}
diff --git a/jim_tcl.txt b/jim_tcl.txt
index 754210f..2ad1f25 100644
--- a/jim_tcl.txt
+++ b/jim_tcl.txt
@@ -59,6 +59,7 @@ Changes since 0.78
 3. Add support for `aio lock -wait`
 4. Add `signal block` to prevent delivery of signals
 5. Add support for `file split`
+6. Add support for `json::encode` and `json::decode`
 
 Changes between 0.77 and 0.78
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -5331,6 +5332,69 @@ independently (but synchronously) of the main interpreter.
     alias for +'parentcmd'+ in the parent interpreter, with the given, fixed arguments.
     The alias may be deleted in the child with 'rename'.
 
+json::encode
+~~~~~~~~~~~~
+
+The Tcl -> JSON encoder is part of the optional 'json' package.
+
++*json::encode* 'value ?schema?'+::
+
+Encode a Tcl value as JSON according to the schema (defaults to +'str'+). The following schema types are supported:
+* 'str' - Tcl string -> JSON string
+* 'num' - Tcl value -> bare numeric value or null
+* 'bool' - Tcl boolean value -> true, false
+* 'obj ?name subschema ...?' - Tcl dict -> JSON object. For each dict key matching 'name', the corresponding 'subschema'
+is applied. The special name +'*'+ matches any keys not otherwise matched, otherwise the default +'str'+ is used.
+* 'list ?subschema?' - Tcl list -> JSON array. The 'subschema' (default +'str'+) is applied for each element of the list/array.
+* 'mixed ?subschema ...?' = Tcl list -> JSON array. Each 'subschema' is applied for the corresponding element of the list/array.
+  ::
+The following are examples:
+----
+    . json::encode {1 2 true false null 5.0} list
+    [ "1", "2", "true", "false", "null", "5.0" ]
+    . json::encode {1 2 true false null 5.0} {list num}
+    [ 1, 2, true, false, null, 5.0 ]
+    . json::encode {0 1 2 true false 5.0 off} {list bool}
+    [ false, true, true, true, false, true, false ]
+    . json::encode {a 1 b hello c {3 4}} obj
+    { "a":"1", "b":"hello", "c":"3 4" }
+    . json::encode {a 1 b hello c {3 4}} {obj a num c {list num}}
+    { "a":1, "b":"hello", "c":[ 3, 4 ] }
+    . json::encode {true true {abc def}} {mixed str num obj}
+    [ "true", true, { "abc":"def" } ]
+    . json::encode {a 1 b 3.0 c hello d null} {obj c str * num}
+    { "a":1, "b":3.0, "c":"hello", "d":null }
+----
+
+json::decode
+~~~~~~~~~~~~
+
+The JSON -> Tcl decoder is part of the optional 'json' package.
+
++*json::decode* ?*-null* 'string'? ?*-schema*? 'json-string'+::
+
+Decodes the given JSON string (must be array or object) into a Tcl data structure. If '+-schema+' is specified, returns a
+list of +'{data schema}'+ where the schema is compatible with `json::encode`. Otherwise just returns the data.
+Decoding is as follows (with schema types listed in parentheses):
+* object -> dict ('obj')
+* array -> list ('mixed' or 'list')
+* number -> as-is ('num')
+* boolean -> as-is ('bool')
+* string -> string ('str')
+* null -> supplied null string or the default +'"null"'+ ('num')
+ ::
+ Note that an object decoded into a dict will return the keys in the same order as the original string.
+----
+    . json::decode {[1, 2]}
+    {1 2}
+    . json::decode -schema {[1, 2]}
+    {1 2} {list num}
+    . json::decode -schema {{"a":1, "b":2}}
+    {a 1 b 2} {obj a num b num}
+    . json::decode -schema {[1, 2, {a:"b", c:false}, "hello"]}
+    {1 2 {a b c false} hello} {mixed num num {obj a str c bool} str}
+----
+
 [[BuiltinVariables]]
 BUILT-IN VARIABLES
 ------------------
diff --git a/jsonencode.tcl b/jsonencode.tcl
new file mode 100644
index 0000000..107ab1a
--- /dev/null
+++ b/jsonencode.tcl
@@ -0,0 +1,100 @@
+# Implements 'json::encode'
+#
+# (c) 2019 Steve Bennett <steveb@workware.net.au>
+#
+# See LICENCE in this directory for licensing.
+
+# Encode Tcl objects as JSON
+# dict -> object
+# list -> array
+# numeric -> number
+# string -> string
+#
+# The schema provides the type information for the value.
+# str = string
+# num = numeric (or null)
+# bool = boolean
+# obj ... = object. parameters are 'name' 'subschema' where the name matches the dict.
+# list ... = array. parameters are 'subschema' for the elements of the list/array.
+# mixed ... = array of mixed types. parameters are types for each element of the list/array.
+
+# Top level JSON encoder which encodes the given
+# value based on the schema
+proc json::encode {value {schema str}} {
+	json::encode.[lindex $schema 0] $value [lrange $schema 1 end]
+}
+
+# Encode a string
+proc json::encode.str {value {dummy {}}} {
+	# Strictly we should be converting \x00 through \x1F to unicode escapes
+	# And anything outside the BMP to a UTF-16 surrogate pair
+	return \"[string map [list \\ \\\\ \" \\" \f \\f \n \\n / \\/ \b \\b \r \\r \t \\t] $value]\"
+}
+
+# If no type is given, also encode as a string
+proc json::encode. {args} {
+	tailcall json::encode.str {*}$args
+}
+
+# Encode a number
+proc json::encode.num {value {dummy {}}} {
+	if {$value in {Inf -Inf}} {
+		append value inity
+	}
+	return $value
+}
+
+# Encode a boolean
+proc json::encode.bool {value {dummy {}}} {
+	if {$value} {
+		return true
+	}
+	return false
+}
+
+# Encode an object (dictionary)
+proc json::encode.obj {obj {schema {}}} {
+	set result "\{"
+	set sep " "
+	foreach k [lsort [dict keys $obj]] {
+		if {[dict exists $schema $k]} {
+			set type [dict get $schema $k]
+		} elseif {[dict exists $schema *]} {
+			set type [dict get $schema *]
+		} else {
+			set type str
+		}
+		append result $sep\"$k\":
+
+		append result [json::encode.[lindex $type 0] [dict get $obj $k] [lrange $type 1 end]]
+		set sep ", "
+	}
+	append result " \}"
+}
+
+# Encode an array (list)
+proc json::encode.list {list {type str}} {
+	set result "\["
+	set sep " "
+	foreach l $list {
+		append result $sep
+		append result [json::encode.[lindex $type 0] $l [lrange $type 1 end]]
+		set sep ", "
+	}
+	append result " \]"
+}
+
+# Encode a mixed-type array (list)
+# Must be as many types as there are elements of the list
+proc json::encode.mixed {list types} {
+	set result "\["
+	set sep " "
+	foreach l $list type $types {
+		append result $sep
+		append result [json::encode.[lindex $type 0] $l [lrange $type 1 end]]
+		set sep ", "
+	}
+	append result " \]"
+}
+
+# vim: se ts=4:
diff --git a/make-index b/make-index
index 1908d00..5071d88 100755
--- a/make-index
+++ b/make-index
@@ -16,13 +16,13 @@ while {[gets $f buf] >= 0} {
 			incr c
 			set target cmd_$c
 			set lines [linsert $lines end-1 "\[\[$target\]\]"]
-			set prevlist [split $prev ":, "]
+			set prevlist [split $prev ", "]
 		} else {
 			set target _[string map {:: _} $prev]
 			set prevlist [list $prev]
 		}
 		foreach cmd $prevlist {
-			set cmd [string trim $cmd]
+			set cmd [string trim $cmd :]
 			if {[regexp {^[a-z.:]+$} $cmd]} {
 				lappend commands [list $cmd $target]
 				set cdict($cmd) $target
diff --git a/tests/json.test b/tests/json.test
new file mode 100644
index 0000000..066cc0f
--- /dev/null
+++ b/tests/json.test
@@ -0,0 +1,146 @@
+source [file dirname [info script]]/testing.tcl
+
+needs cmd json::decode json
+needs cmd json::encode json
+
+set json {
+{
+	"fossil":"9c65b5432e4aeecf3556e5550c338ce93fd861cc",
+	"timestamp":1435827337,
+	"command":"timeline/checkin",
+	"procTimeUs":3333,
+	"procTimeMs":3,
+	"homepage":null,
+	"payload":{
+		"limit":1,
+		"timeline":[{
+			"type":"checkin",
+			"uuid":"f8b17edee7ff4f16517601c40eb713602aed7a52",
+			"isLeaf":true,
+			"timestamp":1435318826,
+			"user":"juef",
+			"comment":"adwaita-icon-theme: update to 3.17.3",
+			"parents":["de628be645cc62429d630f9234c56d1fddfdc2a3"],
+			"tags":["trunk"]
+		}]
+	}
+}}
+
+test json-decode-001 {top level keys} {
+	lsort [dict keys [json::decode $json]]
+} {command fossil homepage payload procTimeMs procTimeUs timestamp}
+
+# Note that the decode will return the keys/values in order
+test json-decode-002 {object value} {
+	dict get [json::decode $json] payload
+} {limit 1 timeline {{type checkin uuid f8b17edee7ff4f16517601c40eb713602aed7a52 isLeaf true timestamp 1435318826 user juef comment {adwaita-icon-theme: update to 3.17.3} parents de628be645cc62429d630f9234c56d1fddfdc2a3 tags trunk}}}
+
+test json-decode-003 {object nested value} {
+	dict get [json::decode $json] payload timeline
+} {{type checkin uuid f8b17edee7ff4f16517601c40eb713602aed7a52 isLeaf true timestamp 1435318826 user juef comment {adwaita-icon-theme: update to 3.17.3} parents de628be645cc62429d630f9234c56d1fddfdc2a3 tags trunk}}
+
+test json-decode-004 {array entry from nested value} {
+	lindex [dict get [json::decode $json] payload timeline] 0
+} {type checkin uuid f8b17edee7ff4f16517601c40eb713602aed7a52 isLeaf true timestamp 1435318826 user juef comment {adwaita-icon-theme: update to 3.17.3} parents de628be645cc62429d630f9234c56d1fddfdc2a3 tags trunk}
+
+test json-decode-005 {object value from child array entry} {
+	dict get [lindex [dict get [json::decode $json] payload timeline] 0] comment
+} {adwaita-icon-theme: update to 3.17.3}
+
+test json-decode-006 {unicode escape} {
+	dict get [json::decode {{"key":"\u2022"}}] key
+} \u2022
+
+test json-decode-011 {null subsitution} {
+	dict get [json::decode -null NULL $json] homepage
+} {NULL}
+
+test json-decode-012 {default null value} {
+	dict get [json::decode $json] homepage
+} {null}
+
+test json-decode-1.1 {Number forms} {
+	json::decode {[ 1, 2, 3.0, 4, Infinity, NaN, -Infinity, -0.0, 1e5, -1e-5 ]}
+} {1 2 3.0 4 Inf NaN -Inf -0.0 1e5 -1e-5}
+
+test json-2.1 {schema tests} {
+	lindex [json::decode -schema {[]}] 1
+} {list}
+
+test json-2.2 {schema tests} {
+	lindex [json::decode -schema {[1, 2]}] 1
+} {list num}
+
+test json-2.3 {schema tests} {
+	lindex [json::decode -schema {[1, 2, [3, 4], 4, 6]}] 1
+} {mixed num num {list num} num num}
+
+test json-2.4 {schema tests} {
+	lindex [json::decode -schema {{"a":1, "b":2}}] 1
+} {obj a num b num}
+
+test json-2.5 {schema tests} {
+	lindex [json::decode -schema {[1, 2, {a:"b", c:false}, "hello"]}] 1
+} {mixed num num {obj a str c bool} str}
+
+test json-2.6 {schema tests} {
+	lindex [json::decode -schema {[1, 2, {a:["b", 1, true, Infinity]}]}] 1
+} {mixed num num {obj a {mixed str num bool num}}}
+
+test json-2.7 {schema tests} {
+	lindex [json::decode -schema {[1, 2, {a:["b", 1, true, ["d", "e", "f"]]}]}] 1
+} {mixed num num {obj a {mixed str num bool {list str}}}}
+
+test json-2.8 {schema tests} {
+	lindex [json::decode -schema {[1, 2, true, false]}] 1
+} {mixed num num bool bool}
+
+
+unset -nocomplain json
+
+test json-encode-1.1 {String with backslashes}  {
+	json::encode {A "quoted string containing \backslashes\"}
+} {"A \"quoted string containing \\backslashes\\\""}
+
+test json-encode-1.2 {String with special chars} {
+	json::encode "Various \n special \b characters \t and /slash/ \r too"
+} {"Various \n special \b characters \t and \/slash\/ \r too"}
+
+test json-encode-1.3 {Array of numbers} {
+	json::encode {1 2 3.0 4 Inf NaN -Inf -0.0 1e5 -1e-5} {list num}
+} {[ 1, 2, 3.0, 4, Infinity, NaN, -Infinity, -0.0, 1e5, -1e-5 ]}
+
+test json-encode-1.4 {Array of strings} {
+	json::encode {1 2 3.0 4} list
+} {[ "1", "2", "3.0", "4" ]}
+
+test json-encode-1.5 {Array of objects} {
+	json::encode {{state NY city {New York} postalCode 10021 streetAddress {21 2nd Street}} {state CA city {Los Angeles} postalCode 10345 streetAddress {15 Hale St}}} {list obj postalCode num}
+} {[ { "city":"New York", "postalCode":10021, "state":"NY", "streetAddress":"21 2nd Street" }, { "city":"Los Angeles", "postalCode":10345, "state":"CA", "streetAddress":"15 Hale St" } ]}
+
+test json-encode-1.6 {Simple typeless object} {
+	json::encode {home {212 555-1234} fax {646 555-4567}} obj
+} {{ "fax":"646 555-4567", "home":"212 555-1234" }}
+
+test json-encode-1.7 {Primitives as num} {
+	json::encode {a false b null c true} {obj a num b num c num}
+} {{ "a":false, "b":null, "c":true }}
+
+test json-encode-1.8 {Complex schema} {
+	json::encode {Person {firstName John age 25 lastName Smith years {1972 1980 1995 2002} PhoneNumbers {home {212 555-1234} fax {646 555-4567}} Address {state NY city {New York} postalCode 10021 streetAddress {21 2nd Street}}}} {obj Person {obj age num Address {obj postalCode num} PhoneNumbers obj years {list num}}}
+} {{ "Person":{ "Address":{ "city":"New York", "postalCode":10021, "state":"NY", "streetAddress":"21 2nd Street" }, "PhoneNumbers":{ "fax":"646 555-4567", "home":"212 555-1234" }, "age":25, "firstName":"John", "lastName":"Smith", "years":[ 1972, 1980, 1995, 2002 ] } }}
+
+test json-encode-1.9 {Array of mixed types} {
+	json::encode {{a b c d} 44} {mixed list num}
+} {[ [ "a", "b", "c", "d" ], 44 ]}
+
+test json-encode-1.10 {Array of objects} {
+	json::encode {{state NY city {New York} postalCode 10021 streetAddress {21 2nd Street}} {state CA city {Los Angeles} postalCode 10345 streetAddress {15 Hale St}}} {list obj postalCode num}
+} {[ { "city":"New York", "postalCode":10021, "state":"NY", "streetAddress":"21 2nd Street" }, { "city":"Los Angeles", "postalCode":10345, "state":"CA", "streetAddress":"15 Hale St" } ]}
+
+test json-encode-1.11 {Forms of boolean} {
+	json::encode {-5 4 1 0 yes no true false} {list bool}
+} {[ true, true, true, false, true, false, true, false ]}
+
+
+testreport