diff --git a/src/common/type/json.c b/src/common/type/json.c index 46c9538e2c..ef25a7c837 100644 --- a/src/common/type/json.c +++ b/src/common/type/json.c @@ -819,12 +819,12 @@ jsonReadSkipStr(JsonRead *const this) { this->json++; - // Expect four digits + // Expect four hex digits unsigned int digitIdx = 0; for (; digitIdx < 4; digitIdx++) { - if (!isdigit(this->json[digitIdx])) + if (!isxdigit(this->json[digitIdx])) break; } @@ -937,6 +937,60 @@ jsonReadSkip(JsonRead *const this) FUNCTION_TEST_RETURN_VOID(); } +/**********************************************************************************************************************************/ +FN_EXTERN void +jsonDecodeUnicode(JsonRead *const this, String *const result) +{ + this->json++; // get past /u + const unsigned int high = cvtZSubNToUIntBase(this->json, 0, 4, 16); + unsigned int codepoint = high; + this->json += 4; + + // Check for supplementary planes + if (high >= 0xD800 && high <= 0xDBFF) + { + if (*this->json != '\\' || *(this->json + 1) != 'u') + THROW_FMT(JsonFormatError, "missing low surrogate for high surrogate at: %s", this->json - 6); + + this->json += 2; + const unsigned int low = cvtZSubNToUIntBase(this->json, 0, 4, 16); + this->json += 4; + + if (low < 0xDC00 || low > 0xDFFF) + THROW_FMT(JsonFormatError, "invalid low surrogate in unicode escape at: %s", this->json - 6); + + codepoint = 0x10000 + ((high - 0xD800) << 10) + (low - 0xDC00); + } + else if (high >= 0xDC00 && high <= 0xDFFF) + THROW_FMT(JsonFormatError, "unexpected low surrogate without high surrogate at: %s", this->json - 4); + + // Append 1-4 bytes to the result + if (codepoint <= 0x7F) + { + strCatChr(result, (char)codepoint); + } + else if (codepoint <= 0x7FF) + { + strCatChr(result, (char)(0xC0 | (codepoint >> 6))); + strCatChr(result, (char)(0x80 | (codepoint & 0x3F))); + } + else if (codepoint <= 0xFFFF) + { + strCatChr(result, (char)(0xE0 | (codepoint >> 12))); + strCatChr(result, (char)(0x80 | ((codepoint >> 6) & 0x3F))); + strCatChr(result, (char)(0x80 | (codepoint & 0x3F))); + } + else + { + strCatChr(result, (char)(0xF0 | (codepoint >> 18))); + strCatChr(result, (char)(0x80 | ((codepoint >> 12) & 0x3F))); + strCatChr(result, (char)(0x80 | ((codepoint >> 6) & 0x3F))); + strCatChr(result, (char)(0x80 | (codepoint & 0x3F))); + } + + this->json--; +} + /**********************************************************************************************************************************/ FN_EXTERN String * jsonReadStr(JsonRead *const this) @@ -1016,17 +1070,7 @@ jsonReadStr(JsonRead *const this) case 'u': { - this->json++; - - // We don't know how to decode anything except ASCII so fail if it looks like Unicode - if (strncmp(this->json, "00", 2) != 0) - THROW_FMT(JsonFormatError, "unable to decode at: %s", this->json - 2); - - // Decode char - this->json += 2; - strCatChr(result, (char)cvtZSubNToUIntBase(this->json, 0, 2, 16)); - this->json++; - + jsonDecodeUnicode(this, result); break; } diff --git a/src/common/type/json.h b/src/common/type/json.h index e8650e7a8b..b9c9ef19c1 100644 --- a/src/common/type/json.h +++ b/src/common/type/json.h @@ -89,6 +89,9 @@ FN_EXTERN Variant *jsonReadVar(JsonRead *this); /*********************************************************************************************************************************** Read Helper Functions ***********************************************************************************************************************************/ +// Unicode escape +FN_EXTERN void jsonDecodeUnicode(JsonRead *this, String *result); + // Convert JSON to Variant FN_EXTERN Variant *jsonToVar(const String *json); diff --git a/test/src/module/common/typeJsonTest.c b/test/src/module/common/typeJsonTest.c index e6a6317f83..18049a5fdb 100644 --- a/test/src/module/common/typeJsonTest.c +++ b/test/src/module/common/typeJsonTest.c @@ -57,14 +57,42 @@ testRun(void) JsonFormatError, "invalid number at: 9999999999999999999999999999999999999999999999999999999999999999"); // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("error on invalid ascii encoding"); + TEST_TITLE("error on invalid escape"); - TEST_ERROR(jsonReadStr(jsonReadNew(STRDEF("\"\\u1111"))), JsonFormatError, "unable to decode at: \\u1111"); + TEST_ERROR(jsonReadStr(jsonReadNew(STRDEF("\"\\xy"))), JsonFormatError, "invalid escape character at: \\xy"); // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("error on invalid escape"); + TEST_TITLE("unicode escape followed by escape"); - TEST_ERROR(jsonReadStr(jsonReadNew(STRDEF("\"\\xy"))), JsonFormatError, "invalid escape character at: \\xy"); + TEST_RESULT_STR_Z(jsonReadStr(jsonReadNew(STRDEF("\"\\u0076\\n\""))), "v\n", "unicode then newline"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error on high surrogate without low surrogate"); + + TEST_ERROR( + jsonReadStr(jsonReadNew(STRDEF("\"\\uD83D\""))), JsonFormatError, + "missing low surrogate for high surrogate at: \\uD83D\""); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error on high surrogate followed by non-unicode escape"); + + TEST_ERROR( + jsonReadStr(jsonReadNew(STRDEF("\"\\uD83D\\n\""))), JsonFormatError, + "missing low surrogate for high surrogate at: \\uD83D\\n\""); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error on invalid low surrogate"); + + TEST_ERROR( + jsonReadStr(jsonReadNew(STRDEF("\"\\uD83D\\u0041\""))), JsonFormatError, + "invalid low surrogate in unicode escape at: \\u0041\""); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error on low surrogate without high surrogate"); + + TEST_ERROR( + jsonReadStr(jsonReadNew(STRDEF("\"\\uDE00\""))), JsonFormatError, + "unexpected low surrogate without high surrogate at: DE00\""); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("error on unterminated string");