diff --git a/src/unicode.cpp b/src/unicode.cpp index 8d8843e0e4..6c834cbced 100644 --- a/src/unicode.cpp +++ b/src/unicode.cpp @@ -42,6 +42,9 @@ uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) { throw std::invalid_argument("invalid character"); } auto result = ((utf8[offset + 0] & 0x1f) << 6) | (utf8[offset + 1] & 0x3f); + if (result < 0x80) { + throw std::invalid_argument("invalid character"); + } offset += 2; return result; } @@ -50,6 +53,9 @@ uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) { throw std::invalid_argument("invalid character"); } auto result = ((utf8[offset + 0] & 0x0f) << 12) | ((utf8[offset + 1] & 0x3f) << 6) | (utf8[offset + 2] & 0x3f); + if (result < 0x800) { + throw std::invalid_argument("invalid character"); + } offset += 3; return result; } @@ -58,6 +64,9 @@ uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) { throw std::invalid_argument("invalid character"); } auto result = ((utf8[offset + 0] & 0x07) << 18) | ((utf8[offset + 1] & 0x3f) << 12) | ((utf8[offset + 2] & 0x3f) << 6) | (utf8[offset + 3] & 0x3f); + if (result < 0x10000 || result > 0x10ffff) { + throw std::invalid_argument("invalid character"); + } offset += 4; return result; } diff --git a/tests/test-unicode.cpp b/tests/test-unicode.cpp index 561ff3e545..5ced7a28a7 100644 --- a/tests/test-unicode.cpp +++ b/tests/test-unicode.cpp @@ -6,6 +6,7 @@ #include int main() { + bool success = true; const std::vector, std::vector>> normalization = { {{}, {}}, {{0x65, 0x301}, {0xE9}}, @@ -23,7 +24,46 @@ int main() { for (const auto & test : normalization) { if (unicode_cpts_normalize_nfc(test.first) != test.second || unicode_cpts_normalize_nfc(test.second) != test.second) { fprintf(stderr, "unexpected NFC normalization\n"); - return 1; + success = false; + break; + } + } + + const struct { + std::string text; + std::vector expected; + } utf8_tests[] = { + { "\x7f", { 0x7f } }, + { "\xc2\x80", { 0x80 } }, + { "\xdf\xbf", { 0x7ff } }, + { "\xe0\xa0\x80", { 0x800 } }, + { "\xef\xbf\xbf", { 0xffff } }, + { "\xf0\x90\x80\x80", { 0x10000 } }, + { "\xf4\x8f\xbf\xbf", { 0x10ffff } }, + // Invalid-input cases preserve the existing recovery behavior: one U+FFFD per byte. + { "\xc0\xa0", { 0xfffd, 0xfffd } }, + { "\xc1\xbf", { 0xfffd, 0xfffd } }, + { "\xe0\x80\xa0", { 0xfffd, 0xfffd, 0xfffd } }, + { "\xe0\x9f\xbf", { 0xfffd, 0xfffd, 0xfffd } }, + { "\xf0\x80\x80\xa0", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } }, + { "\xf0\x8f\xbf\xbf", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } }, + { "\xf4\x90\x80\x80", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } }, + { "\xf5\x80\x80\x80", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } }, + { "\xf7\xbf\xbf\xbf", { 0xfffd, 0xfffd, 0xfffd, 0xfffd } }, + { "\xe2\x80", { 0xfffd, 0xfffd } }, + }; + + for (const auto & test : utf8_tests) { + auto expected = test.expected; + expected.insert(expected.begin(), 'a'); + expected.push_back('b'); + if (unicode_cpts_from_utf8("a" + test.text + "b") != expected) { + fprintf(stderr, "unexpected UTF-8 decoding for bytes:"); + for (const unsigned char byte : test.text) { + fprintf(stderr, " %02x", byte); + } + fprintf(stderr, "\n"); + success = false; } } @@ -39,8 +79,8 @@ int main() { fprintf(stderr, " [%s]", piece.c_str()); } fprintf(stderr, "\n"); - return 1; + success = false; } - return 0; + return success ? 0 : 1; }