unicode : reject overlong UTF-8 and out-of-range codepoints (#30485)

* unicode : reject overlong UTF-8 and out-of-range codepoints

* Update tests/test-unicode.cpp

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

* tests: document byte-wise UTF-8 error recovery

* break out of first test on failure

---------

Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>

src/unicode.cpp

@@ -42,6 +42,9 @@ uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {
             throw std::invalid_argument("invalid character");
         }
         auto result = ((utf8[offset + 0] & 0x1f) << 6) | (utf8[offset + 1] & 0x3f);
+        if (result < 0x80) {
+            throw std::invalid_argument("invalid character");
+        }
         offset += 2;
         return result;
     }
@@ -50,6 +53,9 @@ uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {
             throw std::invalid_argument("invalid character");
         }
         auto result = ((utf8[offset + 0] & 0x0f) << 12) | ((utf8[offset + 1] & 0x3f) << 6) | (utf8[offset + 2] & 0x3f);
+        if (result < 0x800) {
+            throw std::invalid_argument("invalid character");
+        }
         offset += 3;
         return result;
     }
@@ -58,6 +64,9 @@ uint32_t unicode_cpt_from_utf8(const std::string & utf8, size_t & offset) {
             throw std::invalid_argument("invalid character");
         }
         auto result = ((utf8[offset + 0] & 0x07) << 18) | ((utf8[offset + 1] & 0x3f) << 12) | ((utf8[offset + 2] & 0x3f) << 6) | (utf8[offset + 3] & 0x3f);
+        if (result < 0x10000 || result > 0x10ffff) {
+            throw std::invalid_argument("invalid character");
+        }
         offset += 4;
         return result;
     }

tests/test-unicode.cpp

@@ -6,6 +6,7 @@
 #include <vector>
 
 int main() {
+    bool success = true;
     const std::vector<std::pair<std::vector<uint32_t>, std::vector<uint32_t>>> normalization = {
         {{}, {}},
         {{0x65, 0x301}, {0xE9}},
@@ -23,7 +24,46 @@ int main() {
     for (const auto & test : normalization) {
         if (unicode_cpts_normalize_nfc(test.first) != test.second || unicode_cpts_normalize_nfc(test.second) != test.second) {
             fprintf(stderr, "unexpected NFC normalization\n");
-            return 1;
+            success = false;
+            break;
+        }
+    }
+
+    const struct {
+        std::string text;
+        std::vector<uint32_t> expected;
+    } utf8_tests[] = {
+        { "\x7f",                 { 0x7f } },
+        { "\xc2\x80",             { 0x80 } },
+        { "\xdf\xbf",             { 0x7ff } },
+        { "\xe0\xa0\x80",         { 0x800 } },
+        { "\xef\xbf\xbf",         { 0xffff } },
+        { "\xf0\x90\x80\x80",     { 0x10000 } },
+        { "\xf4\x8f\xbf\xbf",     { 0x10ffff } },
+        // Invalid-input cases preserve the existing recovery behavior: one U+FFFD per byte.
+        { "\xc0\xa0",             { 0xfffd, 0xfffd } },
+        { "\xc1\xbf",             { 0xfffd, 0xfffd } },
+        { "\xe0\x80\xa0",         { 0xfffd, 0xfffd, 0xfffd } },
+        { "\xe0\x9f\xbf",         { 0xfffd, 0xfffd, 0xfffd } },
+        { "\xf0\x80\x80\xa0",     { 0xfffd, 0xfffd, 0xfffd, 0xfffd } },
+        { "\xf0\x8f\xbf\xbf",     { 0xfffd, 0xfffd, 0xfffd, 0xfffd } },
+        { "\xf4\x90\x80\x80",     { 0xfffd, 0xfffd, 0xfffd, 0xfffd } },
+        { "\xf5\x80\x80\x80",     { 0xfffd, 0xfffd, 0xfffd, 0xfffd } },
+        { "\xf7\xbf\xbf\xbf",     { 0xfffd, 0xfffd, 0xfffd, 0xfffd } },
+        { "\xe2\x80",             { 0xfffd, 0xfffd } },
+    };
+
+    for (const auto & test : utf8_tests) {
+        auto expected = test.expected;
+        expected.insert(expected.begin(), 'a');
+        expected.push_back('b');
+        if (unicode_cpts_from_utf8("a" + test.text + "b") != expected) {
+            fprintf(stderr, "unexpected UTF-8 decoding for bytes:");
+            for (const unsigned char byte : test.text) {
+                fprintf(stderr, " %02x", byte);
+            }
+            fprintf(stderr, "\n");
+            success = false;
         }
     }
 
@@ -39,8 +79,8 @@ int main() {
             fprintf(stderr, " [%s]", piece.c_str());
         }
         fprintf(stderr, "\n");
-        return 1;
+        success = false;
     }
 
-    return 0;
+    return success ? 0 : 1;
 }