123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129 |
- /*
- * Copyright 2018-present Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- #include <folly/Unicode.h>
- #include <initializer_list>
- #include <stdexcept>
- #include <folly/Range.h>
- #include <folly/portability/GTest.h>
- using folly::utf8ToCodePoint;
- void testValid(std::initializer_list<unsigned char> data, char32_t expected) {
- {
- const unsigned char* p = data.begin();
- const unsigned char* e = data.end();
- EXPECT_EQ(utf8ToCodePoint(p, e, /* skipOnError */ false), expected)
- << folly::StringPiece(
- (const char*)data.begin(), (const char*)data.end());
- }
- {
- const unsigned char* p = data.begin();
- const unsigned char* e = data.end();
- EXPECT_EQ(utf8ToCodePoint(p, e, /* skipOnError */ true), expected)
- << folly::StringPiece(
- (const char*)data.begin(), (const char*)data.end());
- }
- }
- void testInvalid(std::initializer_list<unsigned char> data) {
- {
- const unsigned char* p = data.begin();
- const unsigned char* e = data.end();
- EXPECT_THROW(
- utf8ToCodePoint(p, e, /* skipOnError */ false), std::runtime_error)
- << folly::StringPiece(
- (const char*)data.begin(), (const char*)data.end());
- }
- {
- const unsigned char* p = data.begin();
- const unsigned char* e = data.end();
- EXPECT_EQ(utf8ToCodePoint(p, e, /* skipOnError */ true), 0xfffd)
- << folly::StringPiece(
- (const char*)data.begin(), (const char*)data.end());
- }
- }
- TEST(InvalidUtf8ToCodePoint, rfc3629Overlong) {
- // https://tools.ietf.org/html/rfc3629
- // Implementations of the decoding algorithm above MUST protect against
- // decoding invalid sequences. For instance, a naive implementation may
- // decode the overlong UTF-8 sequence C0 80 into the character U+0000 [...]
- // Decoding invalid sequences may have security consequences or cause other
- // problems.
- testInvalid({0xC0, 0x80});
- }
- TEST(InvalidUtf8ToCodePoint, rfc3629SurrogatePair) {
- // https://tools.ietf.org/html/rfc3629
- // Implementations of the decoding algorithm above MUST protect against
- // decoding invalid sequences. For instance, a naive implementation may
- // decode [...] the surrogate pair ED A1 8C ED BE B4 into U+233B4.
- // Decoding invalid sequences may have security consequences or cause other
- // problems.
- testInvalid({0xED, 0xA1, 0x8C, 0xED, 0xBE, 0xB4});
- }
- TEST(InvalidUtf8ToCodePoint, MarkusKuhnSingleUTF16Surrogates) {
- // https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
- // 5.1.1 U+D800 = ed a0 80
- // 5.1.2 U+DB7F = ed ad bf
- // 5.1.3 U+DB80 = ed ae 80
- // 5.1.4 U+DBFF = ed af bf
- // 5.1.5 U+DC00 = ed b0 80
- // 5.1.6 U+DF80 = ed be 80
- // 5.1.7 U+DFFF = ed bf bf
- testInvalid({0xed, 0xa0, 0x80});
- testInvalid({0xed, 0xad, 0xbf});
- testInvalid({0xed, 0xae, 0x80});
- testInvalid({0xed, 0xaf, 0xbf});
- testInvalid({0xed, 0xb0, 0x80});
- testInvalid({0xed, 0xbe, 0x80});
- testInvalid({0xed, 0xbf, 0xbf});
- }
- TEST(InvalidUtf8ToCodePoint, MarkusKuhnPairedUTF16Surrogates) {
- // https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
- // 5.2.1 U+D800 U+DC00 = ed a0 80 ed b0 80
- // 5.2.2 U+D800 U+DFFF = ed a0 80 ed bf bf
- // 5.2.3 U+DB7F U+DC00 = ed ad bf ed b0 80
- // 5.2.4 U+DB7F U+DFFF = ed ad bf ed bf bf
- // 5.2.5 U+DB80 U+DC00 = ed ae 80 ed b0 80
- // 5.2.6 U+DB80 U+DFFF = ed ae 80 ed bf bf
- // 5.2.7 U+DBFF U+DC00 = ed af bf ed b0 80
- // 5.2.8 U+DBFF U+DFFF = ed af bf ed bf bf
- testInvalid({0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80});
- testInvalid({0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf});
- testInvalid({0xed, 0xad, 0xbf, 0xed, 0xb0, 0x80});
- testInvalid({0xed, 0xad, 0xbf, 0xed, 0xbf, 0xbf});
- testInvalid({0xed, 0xae, 0x80, 0xed, 0xb0, 0x80});
- testInvalid({0xed, 0xae, 0x80, 0xed, 0xbf, 0xbf});
- testInvalid({0xed, 0xaf, 0xbf, 0xed, 0xb0, 0x80});
- testInvalid({0xed, 0xaf, 0xbf, 0xed, 0xbf, 0xbf});
- }
- TEST(ValidUtf8ToCodePoint, FourCloverLeaf) {
- testValid({0xF0, 0x9F, 0x8D, 0x80}, 0x1F340); // u8"\U0001F340";
- }
- TEST(InvalidUtf8ToCodePoint, FourCloverLeafAsSurrogates) {
- testInvalid({0xd8, 0x3c, 0xdf, 0x40}); // u8"\U0001F340";
- }
- TEST(ValidUtf8ToCodePoint, LastCodePoint) {
- testValid({0xF4, 0x8F, 0xBF, 0xBF}, 0x10FFFF); // u8"\U0010FFFF";
- }
|