From de82c8af0a7626cae95762db8780ec3d1b175bc0 Mon Sep 17 00:00:00 2001 From: Grigorii Heifetz Date: Wed, 28 Oct 2026 15:35:05 +0300 Subject: [PATCH 1/5] Initial patch Try the first solution. --- src/common/type/json.c | 67 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 9 deletions(-) diff --git a/src/common/type/json.c b/src/common/type/json.c index 46c9538e2c..54c8d4f43c 100644 --- a/src/common/type/json.c +++ b/src/common/type/json.c @@ -819,12 +819,12 @@ jsonReadSkipStr(JsonRead *const this) { this->json++; - // Expect four digits + // Expect four hex digits (0-9, A-F, a-f) unsigned int digitIdx = 0; for (; digitIdx < 4; digitIdx++) { - if (!isdigit(this->json[digitIdx])) + if (!isxdigit(this->json[digitIdx])) // <-- Use isxdigit instead of isdigit break; } @@ -1018,14 +1018,63 @@ jsonReadStr(JsonRead *const this) { this->json++; - // We don't know how to decode anything except ASCII so fail if it looks like Unicode - if (strncmp(this->json, "00", 2) != 0) - THROW_FMT(JsonFormatError, "unable to decode at: %s", this->json - 2); + unsigned int high = cvtZSubNToUIntBase(this->json, 0, 4, 16); + this->json += 4; + unsigned int codepoint = high; - // Decode char - this->json += 2; - strCatChr(result, (char)cvtZSubNToUIntBase(this->json, 0, 2, 16)); - this->json++; + // Handle UTF-16 surrogate pairs + if (high >= 0xD800 && high <= 0xDBFF) + { + // Expecting the low surrogate to immediately follow as \uXXXX + if (*this->json == '\\' && *(this->json + 1) == 'u') + { + this->json += 2; // skip \u + unsigned int low = cvtZSubNToUIntBase(this->json, 0, 4, 16); + this->json += 4; + + if (low >= 0xDC00 && low <= 0xDFFF) + { + // Combine high and low surrogates into a single 32-bit Unicode code point + codepoint = 0x10000 + ((high - 0xD800) << 10) + (low - 0xDC00); + } + else + { + THROW_FMT(JsonFormatError, "invalid low surrogate in unicode escape at: %s", this->json - 6); + } + } + else + { + THROW_FMT(JsonFormatError, "missing low surrogate for high surrogate at: %s", this->json - 4); + } + } + else if (high >= 0xDC00 && high <= 0xDFFF) + { + THROW_FMT(JsonFormatError, "unexpected low surrogate without high surrogate at: %s", this->json - 4); + } + + // Encode the final Unicode code point into UTF-8 bytes and append them + if (codepoint <= 0x7F) + { + strCatChr(result, (char)codepoint); + } + else if (codepoint <= 0x7FF) + { + strCatChr(result, (char)(0xC0 | (codepoint >> 6))); + strCatChr(result, (char)(0x80 | (codepoint & 0x3F))); + } + else if (codepoint <= 0xFFFF) + { + strCatChr(result, (char)(0xE0 | (codepoint >> 12))); + strCatChr(result, (char)(0x80 | ((codepoint >> 6) & 0x3F))); + strCatChr(result, (char)(0x80 | (codepoint & 0x3F))); + } + else + { + strCatChr(result, (char)(0xF0 | (codepoint >> 18))); + strCatChr(result, (char)(0x80 | ((codepoint >> 12) & 0x3F))); + strCatChr(result, (char)(0x80 | ((codepoint >> 6) & 0x3F))); + strCatChr(result, (char)(0x80 | (codepoint & 0x3F))); + } break; } From 83aafa3b99766eba348fc8db9055cfefe9feda2e Mon Sep 17 00:00:00 2001 From: Grigorii Heifetz Date: Thu, 29 Oct 2026 15:00:57 +0300 Subject: [PATCH 2/5] Remove test for unknown encoding This test now passes, it is not needed. --- test/src/module/common/typeJsonTest.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/src/module/common/typeJsonTest.c b/test/src/module/common/typeJsonTest.c index e6a6317f83..7ba3ebb995 100644 --- a/test/src/module/common/typeJsonTest.c +++ b/test/src/module/common/typeJsonTest.c @@ -56,11 +56,6 @@ testRun(void) jsonReadNumber(jsonReadNew(STRDEF("9999999999999999999999999999999999999999999999999999999999999999"))), JsonFormatError, "invalid number at: 9999999999999999999999999999999999999999999999999999999999999999"); - // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("error on invalid ascii encoding"); - - TEST_ERROR(jsonReadStr(jsonReadNew(STRDEF("\"\\u1111"))), JsonFormatError, "unable to decode at: \\u1111"); - // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("error on invalid escape"); From 1fbd4bb551e79937c1fa966187ebabbd1bb845e8 Mon Sep 17 00:00:00 2001 From: Grigorii Heifetz Date: Thu, 29 Oct 2026 15:15:49 +0300 Subject: [PATCH 3/5] Fix increment bug Was wrong this->json++. --- src/common/type/json.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/common/type/json.c b/src/common/type/json.c index 54c8d4f43c..138b51f3a9 100644 --- a/src/common/type/json.c +++ b/src/common/type/json.c @@ -819,12 +819,12 @@ jsonReadSkipStr(JsonRead *const this) { this->json++; - // Expect four hex digits (0-9, A-F, a-f) + // Expect four hex digits unsigned int digitIdx = 0; for (; digitIdx < 4; digitIdx++) { - if (!isxdigit(this->json[digitIdx])) // <-- Use isxdigit instead of isdigit + if (!isxdigit(this->json[digitIdx])) break; } @@ -1016,8 +1016,6 @@ jsonReadStr(JsonRead *const this) case 'u': { - this->json++; - unsigned int high = cvtZSubNToUIntBase(this->json, 0, 4, 16); this->json += 4; unsigned int codepoint = high; From c8b48a4b61d38a57342d7461e1a28869f4ebf29e Mon Sep 17 00:00:00 2001 From: Grigorii Heifetz Date: Thu, 29 Oct 2026 15:21:20 +0300 Subject: [PATCH 4/5] Fix increment bug again Get back this->json++, add this->json--. --- src/common/type/json.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/common/type/json.c b/src/common/type/json.c index 138b51f3a9..f5bc5c3acc 100644 --- a/src/common/type/json.c +++ b/src/common/type/json.c @@ -1016,6 +1016,8 @@ jsonReadStr(JsonRead *const this) case 'u': { + this->json++; + unsigned int high = cvtZSubNToUIntBase(this->json, 0, 4, 16); this->json += 4; unsigned int codepoint = high; @@ -1074,6 +1076,8 @@ jsonReadStr(JsonRead *const this) strCatChr(result, (char)(0x80 | (codepoint & 0x3F))); } + this->json--; + break; } From 5f3279814828dc9f1f8e871ca333f875456d72e5 Mon Sep 17 00:00:00 2001 From: Grigorii Heifetz Date: Thu, 29 Oct 2026 17:01:48 +0300 Subject: [PATCH 5/5] Refactor and add tests Make a separate function, add tests for error branches. --- src/common/type/json.c | 117 ++++++++++++-------------- src/common/type/json.h | 3 + test/src/module/common/typeJsonTest.c | 33 ++++++++ 3 files changed, 91 insertions(+), 62 deletions(-) diff --git a/src/common/type/json.c b/src/common/type/json.c index f5bc5c3acc..ef25a7c837 100644 --- a/src/common/type/json.c +++ b/src/common/type/json.c @@ -937,6 +937,60 @@ jsonReadSkip(JsonRead *const this) FUNCTION_TEST_RETURN_VOID(); } +/**********************************************************************************************************************************/ +FN_EXTERN void +jsonDecodeUnicode(JsonRead *const this, String *const result) +{ + this->json++; // get past /u + const unsigned int high = cvtZSubNToUIntBase(this->json, 0, 4, 16); + unsigned int codepoint = high; + this->json += 4; + + // Check for supplementary planes + if (high >= 0xD800 && high <= 0xDBFF) + { + if (*this->json != '\\' || *(this->json + 1) != 'u') + THROW_FMT(JsonFormatError, "missing low surrogate for high surrogate at: %s", this->json - 6); + + this->json += 2; + const unsigned int low = cvtZSubNToUIntBase(this->json, 0, 4, 16); + this->json += 4; + + if (low < 0xDC00 || low > 0xDFFF) + THROW_FMT(JsonFormatError, "invalid low surrogate in unicode escape at: %s", this->json - 6); + + codepoint = 0x10000 + ((high - 0xD800) << 10) + (low - 0xDC00); + } + else if (high >= 0xDC00 && high <= 0xDFFF) + THROW_FMT(JsonFormatError, "unexpected low surrogate without high surrogate at: %s", this->json - 4); + + // Append 1-4 bytes to the result + if (codepoint <= 0x7F) + { + strCatChr(result, (char)codepoint); + } + else if (codepoint <= 0x7FF) + { + strCatChr(result, (char)(0xC0 | (codepoint >> 6))); + strCatChr(result, (char)(0x80 | (codepoint & 0x3F))); + } + else if (codepoint <= 0xFFFF) + { + strCatChr(result, (char)(0xE0 | (codepoint >> 12))); + strCatChr(result, (char)(0x80 | ((codepoint >> 6) & 0x3F))); + strCatChr(result, (char)(0x80 | (codepoint & 0x3F))); + } + else + { + strCatChr(result, (char)(0xF0 | (codepoint >> 18))); + strCatChr(result, (char)(0x80 | ((codepoint >> 12) & 0x3F))); + strCatChr(result, (char)(0x80 | ((codepoint >> 6) & 0x3F))); + strCatChr(result, (char)(0x80 | (codepoint & 0x3F))); + } + + this->json--; +} + /**********************************************************************************************************************************/ FN_EXTERN String * jsonReadStr(JsonRead *const this) @@ -1016,68 +1070,7 @@ jsonReadStr(JsonRead *const this) case 'u': { - this->json++; - - unsigned int high = cvtZSubNToUIntBase(this->json, 0, 4, 16); - this->json += 4; - unsigned int codepoint = high; - - // Handle UTF-16 surrogate pairs - if (high >= 0xD800 && high <= 0xDBFF) - { - // Expecting the low surrogate to immediately follow as \uXXXX - if (*this->json == '\\' && *(this->json + 1) == 'u') - { - this->json += 2; // skip \u - unsigned int low = cvtZSubNToUIntBase(this->json, 0, 4, 16); - this->json += 4; - - if (low >= 0xDC00 && low <= 0xDFFF) - { - // Combine high and low surrogates into a single 32-bit Unicode code point - codepoint = 0x10000 + ((high - 0xD800) << 10) + (low - 0xDC00); - } - else - { - THROW_FMT(JsonFormatError, "invalid low surrogate in unicode escape at: %s", this->json - 6); - } - } - else - { - THROW_FMT(JsonFormatError, "missing low surrogate for high surrogate at: %s", this->json - 4); - } - } - else if (high >= 0xDC00 && high <= 0xDFFF) - { - THROW_FMT(JsonFormatError, "unexpected low surrogate without high surrogate at: %s", this->json - 4); - } - - // Encode the final Unicode code point into UTF-8 bytes and append them - if (codepoint <= 0x7F) - { - strCatChr(result, (char)codepoint); - } - else if (codepoint <= 0x7FF) - { - strCatChr(result, (char)(0xC0 | (codepoint >> 6))); - strCatChr(result, (char)(0x80 | (codepoint & 0x3F))); - } - else if (codepoint <= 0xFFFF) - { - strCatChr(result, (char)(0xE0 | (codepoint >> 12))); - strCatChr(result, (char)(0x80 | ((codepoint >> 6) & 0x3F))); - strCatChr(result, (char)(0x80 | (codepoint & 0x3F))); - } - else - { - strCatChr(result, (char)(0xF0 | (codepoint >> 18))); - strCatChr(result, (char)(0x80 | ((codepoint >> 12) & 0x3F))); - strCatChr(result, (char)(0x80 | ((codepoint >> 6) & 0x3F))); - strCatChr(result, (char)(0x80 | (codepoint & 0x3F))); - } - - this->json--; - + jsonDecodeUnicode(this, result); break; } diff --git a/src/common/type/json.h b/src/common/type/json.h index e8650e7a8b..b9c9ef19c1 100644 --- a/src/common/type/json.h +++ b/src/common/type/json.h @@ -89,6 +89,9 @@ FN_EXTERN Variant *jsonReadVar(JsonRead *this); /*********************************************************************************************************************************** Read Helper Functions ***********************************************************************************************************************************/ +// Unicode escape +FN_EXTERN void jsonDecodeUnicode(JsonRead *this, String *result); + // Convert JSON to Variant FN_EXTERN Variant *jsonToVar(const String *json); diff --git a/test/src/module/common/typeJsonTest.c b/test/src/module/common/typeJsonTest.c index 7ba3ebb995..18049a5fdb 100644 --- a/test/src/module/common/typeJsonTest.c +++ b/test/src/module/common/typeJsonTest.c @@ -61,6 +61,39 @@ testRun(void) TEST_ERROR(jsonReadStr(jsonReadNew(STRDEF("\"\\xy"))), JsonFormatError, "invalid escape character at: \\xy"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("unicode escape followed by escape"); + + TEST_RESULT_STR_Z(jsonReadStr(jsonReadNew(STRDEF("\"\\u0076\\n\""))), "v\n", "unicode then newline"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error on high surrogate without low surrogate"); + + TEST_ERROR( + jsonReadStr(jsonReadNew(STRDEF("\"\\uD83D\""))), JsonFormatError, + "missing low surrogate for high surrogate at: \\uD83D\""); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error on high surrogate followed by non-unicode escape"); + + TEST_ERROR( + jsonReadStr(jsonReadNew(STRDEF("\"\\uD83D\\n\""))), JsonFormatError, + "missing low surrogate for high surrogate at: \\uD83D\\n\""); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error on invalid low surrogate"); + + TEST_ERROR( + jsonReadStr(jsonReadNew(STRDEF("\"\\uD83D\\u0041\""))), JsonFormatError, + "invalid low surrogate in unicode escape at: \\u0041\""); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error on low surrogate without high surrogate"); + + TEST_ERROR( + jsonReadStr(jsonReadNew(STRDEF("\"\\uDE00\""))), JsonFormatError, + "unexpected low surrogate without high surrogate at: DE00\""); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("error on unterminated string");