From 62815d85aed71eff7b6c3a524705180fb04f5d30 Mon Sep 17 00:00:00 2001 From: Markus Armbruster Date: Thu, 23 Aug 2018 18:40:01 +0200 Subject: json: Redesign the callback to consume JSON values The classical way to structure parser and lexer is to have the client call the parser to get an abstract syntax tree, the parser call the lexer to get the next token, and the lexer call some function to get input characters. Another way to structure them would be to have the client feed characters to the lexer, the lexer feed tokens to the parser, and the parser feed abstract syntax trees to some callback provided by the client. This way is more easily integrated into an event loop that dispatches input characters as they arrive. Our JSON parser is kind of between the two. The lexer feeds tokens to a "streamer" instead of a real parser. The streamer accumulates tokens until it got the sequence of tokens that comprise a single JSON value (it counts curly braces and square brackets to decide). It feeds those token sequences to a callback provided by the client. The callback passes each token sequence to the parser, and gets back an abstract syntax tree. I figure it was done that way to make a straightforward recursive descent parser possible. "Get next token" becomes "pop the first token off the token sequence". Drawback: we need to store a complete token sequence. Each token eats 13 + input characters + malloc overhead bytes. Observations: 1. This is not the only way to use recursive descent. If we replaced "get next token" by a coroutine yield, we could do without a streamer. 2. The lexer reports errors by passing a JSON_ERROR token to the streamer. This communicates the offending input characters and their location, but no more. 3. The streamer reports errors by passing a null token sequence to the callback. The (already poor) lexical error information is thrown away. 4. Having the callback receive a token sequence duplicates the code to convert token sequence to abstract syntax tree in every callback. 5. Known bug: the streamer silently drops incomplete token sequences. This commit rectifies 4. by lifting the call of the parser from the callbacks into the streamer. Later commits will address 3. and 5. The lifting removes a bug from qjson.c's parse_json(): it passed a pointer to a non-null Error * in certain cases, as demonstrated by check-qjson.c. json_parser_parse() is now unused. It's a stupid wrapper around json_parser_parse_err(). Drop it, and rename json_parser_parse_err() to json_parser_parse(). Signed-off-by: Markus Armbruster Reviewed-by: Eric Blake Message-Id: <20180823164025.12553-35-armbru@redhat.com> --- monitor.c | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'monitor.c') diff --git a/monitor.c b/monitor.c index 94f6735..08f799a 100644 --- a/monitor.c +++ b/monitor.c @@ -59,7 +59,6 @@ #include "qapi/qmp/qstring.h" #include "qapi/qmp/qjson.h" #include "qapi/qmp/json-streamer.h" -#include "qapi/qmp/json-parser.h" #include "qapi/qmp/qlist.h" #include "qom/object_interfaces.h" #include "trace-root.h" @@ -4256,18 +4255,15 @@ static void monitor_qmp_bh_dispatcher(void *data) #define QMP_REQ_QUEUE_LEN_MAX (8) -static void handle_qmp_command(JSONMessageParser *parser, GQueue *tokens) +static void handle_qmp_command(void *opaque, QObject *req, Error *err) { - QObject *req, *id = NULL; + Monitor *mon = opaque; + QObject *id = NULL; QDict *qdict; - MonitorQMP *mon_qmp = container_of(parser, MonitorQMP, parser); - Monitor *mon = container_of(mon_qmp, Monitor, qmp); - Error *err = NULL; QMPRequest *req_obj; - req = json_parser_parse_err(tokens, NULL, &err); if (!req && !err) { - /* json_parser_parse_err() sucks: can fail without setting @err */ + /* json_parser_parse() sucks: can fail without setting @err */ error_setg(&err, QERR_JSON_PARSING); } @@ -4465,7 +4461,8 @@ static void monitor_qmp_event(void *opaque, int event) monitor_qmp_response_flush(mon); monitor_qmp_cleanup_queues(mon); json_message_parser_destroy(&mon->qmp.parser); - json_message_parser_init(&mon->qmp.parser, handle_qmp_command); + json_message_parser_init(&mon->qmp.parser, handle_qmp_command, + mon, NULL); mon_refcount--; monitor_fdsets_cleanup(); break; @@ -4683,7 +4680,8 @@ void monitor_init(Chardev *chr, int flags) if (monitor_is_qmp(mon)) { qemu_chr_fe_set_echo(&mon->chr, true); - json_message_parser_init(&mon->qmp.parser, handle_qmp_command); + json_message_parser_init(&mon->qmp.parser, handle_qmp_command, + mon, NULL); if (mon->use_io_thread) { /* * Make sure the old iowatch is gone. It's possible when -- cgit v1.1 From 84a56f38b23440cb3127eaffe4e495826a29f18c Mon Sep 17 00:00:00 2001 From: Markus Armbruster Date: Thu, 23 Aug 2018 18:40:06 +0200 Subject: json: Pass lexical errors and limit violations to callback The callback to consume JSON values takes QObject *json, Error *err. If both are null, the callback is supposed to make up an error by itself. This sucks. qjson.c's consume_json() neglects to do so, which makes qobject_from_json() null instead of failing. I consider that a bug. The culprit is json_message_process_token(): it passes two null pointers when it runs into a lexical error or a limit violation. Fix it to pass a proper Error object then. Update the callbacks: * monitor.c's handle_qmp_command(): the code to make up an error is now dead, drop it. * qga/main.c's process_event(): lumps the "both null" case together with the "not a JSON object" case. The former is now gone. The error message "Invalid JSON syntax" is misleading for the latter. Improve it to "Input must be a JSON object". * qobject/qjson.c's consume_json(): no update; check-qjson demonstrates qobject_from_json() now sets an error on lexical errors, but still doesn't on some other errors. * tests/libqtest.c's qmp_response(): the Error object is now reliable, so use it to improve the error message. Signed-off-by: Markus Armbruster Reviewed-by: Eric Blake Message-Id: <20180823164025.12553-40-armbru@redhat.com> --- monitor.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'monitor.c') diff --git a/monitor.c b/monitor.c index 08f799a..3dbdcb5 100644 --- a/monitor.c +++ b/monitor.c @@ -4262,10 +4262,7 @@ static void handle_qmp_command(void *opaque, QObject *req, Error *err) QDict *qdict; QMPRequest *req_obj; - if (!req && !err) { - /* json_parser_parse() sucks: can fail without setting @err */ - error_setg(&err, QERR_JSON_PARSING); - } + assert(!req != !err); qdict = qobject_to(QDict, req); if (qdict) { -- cgit v1.1 From 86cdf9ec8dec2763702cc52fa412d108a5dc9608 Mon Sep 17 00:00:00 2001 From: Markus Armbruster Date: Thu, 23 Aug 2018 18:40:20 +0200 Subject: json: Clean up headers The JSON parser has three public headers, json-lexer.h, json-parser.h, json-streamer.h. They all contain stuff that is of no interest outside qobject/json-*.c. Collect the public interface in include/qapi/qmp/json-parser.h, and everything else in qobject/json-parser-int.h. Signed-off-by: Markus Armbruster Reviewed-by: Eric Blake Message-Id: <20180823164025.12553-54-armbru@redhat.com> --- monitor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'monitor.c') diff --git a/monitor.c b/monitor.c index 3dbdcb5..021c11b 100644 --- a/monitor.c +++ b/monitor.c @@ -58,7 +58,7 @@ #include "qapi/qmp/qnum.h" #include "qapi/qmp/qstring.h" #include "qapi/qmp/qjson.h" -#include "qapi/qmp/json-streamer.h" +#include "qapi/qmp/json-parser.h" #include "qapi/qmp/qlist.h" #include "qom/object_interfaces.h" #include "trace-root.h" -- cgit v1.1