helper: Make unhexify() robust on invalid data

The current implementation is not suitable for user provided data
because it does not detect invalid inputs in many cases. For example,
the string "aa0xbb" is successfully converted to the 3 bytes: 0xaa,
0x00 and 0xbb. An other example is "aabi" which is successfully
converted to the 2 bytes: 0xaa and 0x0b. Both are obviously incorrect.

Make unhexify() robust on invalid data and use more appropriate data
types for its parameters. Also, add a small documentation for the
function.

Change-Id: Idb799beb86fc608b066c8a76365021ed44c7f890
Signed-off-by: Marc Schink <openocd-dev@marcschink.de>
Reviewed-on: http://openocd.zylin.com/3792
Tested-by: jenkins
Reviewed-by: Tomas Vanek <vanekt@fbl.cz>

2 files changed, 33 insertions, 8 deletions

diff --git a/src/helper/binarybuffer.c b/src/helper/binarybuffer.c
index c1e6322..26aa8cc 100644
--- a/src/helper/binarybuffer.c
+++ b/src/helper/binarybuffer.c
@@ -369,17 +369,42 @@ void bit_copy_discard(struct bit_copy_queue *q)
 	}
 }
 
-int unhexify(char *bin, const char *hex, int count)
+/**
+ * Convert a string of hexadecimal pairs into its binary
+ * representation.
+ *
+ * @param[out] bin Buffer to store binary representation. The buffer size must
+ *                 be at least @p count.
+ * @param[in] hex String with hexadecimal pairs to convert into its binary
+ *                representation.
+ * @param[in] count Number of hexadecimal pairs to convert.
+ *
+ * @return The number of converted hexadecimal pairs.
+ */
+size_t unhexify(uint8_t *bin, const char *hex, size_t count)
 {
-	int i, tmp;
+	size_t i;
+	char tmp;
 
-	for (i = 0; i < count; i++) {
-		if (sscanf(hex + (2 * i), "%02x", &tmp) != 1)
-			return i;
-		bin[i] = tmp;
+	if (!bin || !hex)
+		return 0;
+
+	memset(bin, 0, count);
+
+	for (i = 0; i < 2 * count; i++) {
+		if (hex[i] >= 'a' && hex[i] <= 'f')
+			tmp = hex[i] - 'a' + 10;
+		else if (hex[i] >= 'A' && hex[i] <= 'F')
+			tmp = hex[i] - 'A' + 10;
+		else if (hex[i] >= '0' && hex[i] <= '9')
+			tmp = hex[i] - '0';
+		else
+			return i / 2;
+
+		bin[i / 2] |= tmp << (4 * ((i + 1) % 2));
 	}
 
-	return i;
+	return i / 2;
 }
 
 int hexify(char *hex, const char *bin, int count, int out_maxlen)
diff --git a/src/helper/binarybuffer.h b/src/helper/binarybuffer.h
index dd0d275..b035779 100644
--- a/src/helper/binarybuffer.h
+++ b/src/helper/binarybuffer.h
@@ -234,7 +234,7 @@ void bit_copy_discard(struct bit_copy_queue *q);
 
 /* functions to convert to/from hex encoded buffer
  * used in ti-icdi driver and gdb server */
-int unhexify(char *bin, const char *hex, int count);
+size_t unhexify(uint8_t *bin, const char *hex, size_t count);
 int hexify(char *hex, const char *bin, int count, int out_maxlen);
 void buffer_shr(void *_buf, unsigned buf_len, unsigned count);