nuclear@3: /************************************************************************** nuclear@3: nuclear@3: Filename : OVR_UTF8Util.cpp nuclear@3: Content : UTF8 Unicode character encoding/decoding support nuclear@3: Created : September 19, 2012 nuclear@3: Notes : nuclear@3: Notes : Much useful info at "UTF-8 and Unicode FAQ" nuclear@3: http://www.cl.cam.ac.uk/~mgk25/unicode.html nuclear@3: nuclear@3: Copyright : Copyright 2012 Oculus VR, Inc. All Rights reserved. nuclear@3: nuclear@3: Use of this software is subject to the terms of the Oculus license nuclear@3: agreement provided at the time of installation or download, or which nuclear@3: otherwise accompanies this software in either electronic or hard copy form. nuclear@3: nuclear@3: ************************************************************************************/ nuclear@3: nuclear@3: #include "OVR_UTF8Util.h" nuclear@3: nuclear@3: namespace OVR { namespace UTF8Util { nuclear@3: nuclear@3: SPInt OVR_STDCALL GetLength(const char* buf, SPInt buflen) nuclear@3: { nuclear@3: const char* p = buf; nuclear@3: SPInt length = 0; nuclear@3: nuclear@3: if (buflen != -1) nuclear@3: { nuclear@3: while (p - buf < buflen) nuclear@3: { nuclear@3: // We should be able to have ASStrings with 0 in the middle. nuclear@3: UTF8Util::DecodeNextChar_Advance0(&p); nuclear@3: length++; nuclear@3: } nuclear@3: } nuclear@3: else nuclear@3: { nuclear@3: while (UTF8Util::DecodeNextChar_Advance0(&p)) nuclear@3: length++; nuclear@3: } nuclear@3: nuclear@3: return length; nuclear@3: } nuclear@3: nuclear@3: UInt32 OVR_STDCALL GetCharAt(SPInt index, const char* putf8str, SPInt length) nuclear@3: { nuclear@3: const char* buf = putf8str; nuclear@3: UInt32 c = 0; nuclear@3: nuclear@3: if (length != -1) nuclear@3: { nuclear@3: while (buf - putf8str < length) nuclear@3: { nuclear@3: c = UTF8Util::DecodeNextChar_Advance0(&buf); nuclear@3: if (index == 0) nuclear@3: return c; nuclear@3: index--; nuclear@3: } nuclear@3: nuclear@3: return c; nuclear@3: } nuclear@3: nuclear@3: do nuclear@3: { nuclear@3: c = UTF8Util::DecodeNextChar_Advance0(&buf); nuclear@3: index--; nuclear@3: nuclear@3: if (c == 0) nuclear@3: { nuclear@3: // We've hit the end of the string; don't go further. nuclear@3: OVR_ASSERT(index == 0); nuclear@3: return c; nuclear@3: } nuclear@3: } while (index >= 0); nuclear@3: nuclear@3: return c; nuclear@3: } nuclear@3: nuclear@3: SPInt OVR_STDCALL GetByteIndex(SPInt index, const char *putf8str, SPInt length) nuclear@3: { nuclear@3: const char* buf = putf8str; nuclear@3: nuclear@3: if (length != -1) nuclear@3: { nuclear@3: while ((buf - putf8str) < length && index > 0) nuclear@3: { nuclear@3: UTF8Util::DecodeNextChar_Advance0(&buf); nuclear@3: index--; nuclear@3: } nuclear@3: nuclear@3: return buf-putf8str; nuclear@3: } nuclear@3: nuclear@3: while (index > 0) nuclear@3: { nuclear@3: UInt32 c = UTF8Util::DecodeNextChar_Advance0(&buf); nuclear@3: index--; nuclear@3: nuclear@3: if (c == 0) nuclear@3: return buf-putf8str; nuclear@3: }; nuclear@3: nuclear@3: return buf-putf8str; nuclear@3: } nuclear@3: nuclear@3: int OVR_STDCALL GetEncodeCharSize(UInt32 ucs_character) nuclear@3: { nuclear@3: if (ucs_character <= 0x7F) nuclear@3: return 1; nuclear@3: else if (ucs_character <= 0x7FF) nuclear@3: return 2; nuclear@3: else if (ucs_character <= 0xFFFF) nuclear@3: return 3; nuclear@3: else if (ucs_character <= 0x1FFFFF) nuclear@3: return 4; nuclear@3: else if (ucs_character <= 0x3FFFFFF) nuclear@3: return 5; nuclear@3: else if (ucs_character <= 0x7FFFFFFF) nuclear@3: return 6; nuclear@3: else nuclear@3: return 0; nuclear@3: } nuclear@3: nuclear@3: UInt32 OVR_STDCALL DecodeNextChar_Advance0(const char** putf8Buffer) nuclear@3: { nuclear@3: UInt32 uc; nuclear@3: char c; nuclear@3: nuclear@3: // Security considerations: nuclear@3: // nuclear@3: // Changed, this is now only the case for DecodeNextChar: nuclear@3: // - If we hit a zero byte, we want to return 0 without stepping nuclear@3: // the buffer pointer past the 0. th nuclear@3: // nuclear@3: // If we hit an "overlong sequence"; i.e. a character encoded nuclear@3: // in a longer multibyte string than is necessary, then we nuclear@3: // need to discard the character. This is so attackers can't nuclear@3: // disguise dangerous characters or character sequences -- nuclear@3: // there is only one valid encoding for each character. nuclear@3: // nuclear@3: // If we decode characters { 0xD800 .. 0xDFFF } or { 0xFFFE, nuclear@3: // 0xFFFF } then we ignore them; they are not valid in UTF-8. nuclear@3: nuclear@3: // This isn't actually an invalid character; it's a valid char that nuclear@3: // looks like an inverted question mark. nuclear@3: #define INVALID_CHAR 0x0FFFD nuclear@3: nuclear@3: #define FIRST_BYTE(mask, shift) \ nuclear@3: uc = (c & (mask)) << (shift); nuclear@3: nuclear@3: #define NEXT_BYTE(shift) \ nuclear@3: c = **putf8Buffer; \ nuclear@3: if (c == 0) return 0; /* end of buffer, do not advance */ \ nuclear@3: if ((c & 0xC0) != 0x80) return INVALID_CHAR; /* standard check */ \ nuclear@3: (*putf8Buffer)++; \ nuclear@3: uc |= (c & 0x3F) << shift; nuclear@3: nuclear@3: c = **putf8Buffer; nuclear@3: (*putf8Buffer)++; nuclear@3: if (c == 0) nuclear@3: return 0; // End of buffer. nuclear@3: nuclear@3: if ((c & 0x80) == 0) return (UInt32) c; // Conventional 7-bit ASCII. nuclear@3: nuclear@3: // Multi-byte sequences. nuclear@3: if ((c & 0xE0) == 0xC0) nuclear@3: { nuclear@3: // Two-byte sequence. nuclear@3: FIRST_BYTE(0x1F, 6); nuclear@3: NEXT_BYTE(0); nuclear@3: if (uc < 0x80) return INVALID_CHAR; // overlong nuclear@3: return uc; nuclear@3: } nuclear@3: else if ((c & 0xF0) == 0xE0) nuclear@3: { nuclear@3: // Three-byte sequence. nuclear@3: FIRST_BYTE(0x0F, 12); nuclear@3: NEXT_BYTE(6); nuclear@3: NEXT_BYTE(0); nuclear@3: if (uc < 0x800) return INVALID_CHAR; // overlong nuclear@3: // Not valid ISO 10646, but Flash requires these to work nuclear@3: // see AS3 test e15_5_3_2_3 for String.fromCharCode().charCodeAt(0) nuclear@3: // if (uc >= 0x0D800 && uc <= 0x0DFFF) return INVALID_CHAR; nuclear@3: // if (uc == 0x0FFFE || uc == 0x0FFFF) return INVALID_CHAR; // not valid ISO 10646 nuclear@3: return uc; nuclear@3: } nuclear@3: else if ((c & 0xF8) == 0xF0) nuclear@3: { nuclear@3: // Four-byte sequence. nuclear@3: FIRST_BYTE(0x07, 18); nuclear@3: NEXT_BYTE(12); nuclear@3: NEXT_BYTE(6); nuclear@3: NEXT_BYTE(0); nuclear@3: if (uc < 0x010000) return INVALID_CHAR; // overlong nuclear@3: return uc; nuclear@3: } nuclear@3: else if ((c & 0xFC) == 0xF8) nuclear@3: { nuclear@3: // Five-byte sequence. nuclear@3: FIRST_BYTE(0x03, 24); nuclear@3: NEXT_BYTE(18); nuclear@3: NEXT_BYTE(12); nuclear@3: NEXT_BYTE(6); nuclear@3: NEXT_BYTE(0); nuclear@3: if (uc < 0x0200000) return INVALID_CHAR; // overlong nuclear@3: return uc; nuclear@3: } nuclear@3: else if ((c & 0xFE) == 0xFC) nuclear@3: { nuclear@3: // Six-byte sequence. nuclear@3: FIRST_BYTE(0x01, 30); nuclear@3: NEXT_BYTE(24); nuclear@3: NEXT_BYTE(18); nuclear@3: NEXT_BYTE(12); nuclear@3: NEXT_BYTE(6); nuclear@3: NEXT_BYTE(0); nuclear@3: if (uc < 0x04000000) return INVALID_CHAR; // overlong nuclear@3: return uc; nuclear@3: } nuclear@3: else nuclear@3: { nuclear@3: // Invalid. nuclear@3: return INVALID_CHAR; nuclear@3: } nuclear@3: } nuclear@3: nuclear@3: nuclear@3: void OVR_STDCALL EncodeChar(char* pbuffer, SPInt* pindex, UInt32 ucs_character) nuclear@3: { nuclear@3: if (ucs_character <= 0x7F) nuclear@3: { nuclear@3: // Plain single-byte ASCII. nuclear@3: pbuffer[(*pindex)++] = (char) ucs_character; nuclear@3: } nuclear@3: else if (ucs_character <= 0x7FF) nuclear@3: { nuclear@3: // Two bytes. nuclear@3: pbuffer[(*pindex)++] = 0xC0 | (char)(ucs_character >> 6); nuclear@3: pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 0) & 0x3F); nuclear@3: } nuclear@3: else if (ucs_character <= 0xFFFF) nuclear@3: { nuclear@3: // Three bytes. nuclear@3: pbuffer[(*pindex)++] = 0xE0 | (char)(ucs_character >> 12); nuclear@3: pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 6) & 0x3F); nuclear@3: pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 0) & 0x3F); nuclear@3: } nuclear@3: else if (ucs_character <= 0x1FFFFF) nuclear@3: { nuclear@3: // Four bytes. nuclear@3: pbuffer[(*pindex)++] = 0xF0 | (char)(ucs_character >> 18); nuclear@3: pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 12) & 0x3F); nuclear@3: pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 6) & 0x3F); nuclear@3: pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 0) & 0x3F); nuclear@3: } nuclear@3: else if (ucs_character <= 0x3FFFFFF) nuclear@3: { nuclear@3: // Five bytes. nuclear@3: pbuffer[(*pindex)++] = 0xF8 | (char)(ucs_character >> 24); nuclear@3: pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 18) & 0x3F); nuclear@3: pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 12) & 0x3F); nuclear@3: pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 6) & 0x3F); nuclear@3: pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 0) & 0x3F); nuclear@3: } nuclear@3: else if (ucs_character <= 0x7FFFFFFF) nuclear@3: { nuclear@3: // Six bytes. nuclear@3: pbuffer[(*pindex)++] = 0xFC | (char)(ucs_character >> 30); nuclear@3: pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 24) & 0x3F); nuclear@3: pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 18) & 0x3F); nuclear@3: pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 12) & 0x3F); nuclear@3: pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 6) & 0x3F); nuclear@3: pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 0) & 0x3F); nuclear@3: } nuclear@3: else nuclear@3: { nuclear@3: // Invalid char; don't encode anything. nuclear@3: } nuclear@3: } nuclear@3: nuclear@3: SPInt OVR_STDCALL GetEncodeStringSize(const wchar_t* pchar, SPInt length) nuclear@3: { nuclear@3: SPInt len = 0; nuclear@3: if (length != -1) nuclear@3: for (int i = 0; i < length; i++) nuclear@3: { nuclear@3: len += GetEncodeCharSize(pchar[i]); nuclear@3: } nuclear@3: else nuclear@3: for (int i = 0;; i++) nuclear@3: { nuclear@3: if (pchar[i] == 0) nuclear@3: return len; nuclear@3: len += GetEncodeCharSize(pchar[i]); nuclear@3: } nuclear@3: return len; nuclear@3: } nuclear@3: nuclear@3: void OVR_STDCALL EncodeString(char *pbuff, const wchar_t* pchar, SPInt length) nuclear@3: { nuclear@3: SPInt ofs = 0; nuclear@3: if (length != -1) nuclear@3: { nuclear@3: for (int i = 0; i < length; i++) nuclear@3: { nuclear@3: EncodeChar(pbuff, &ofs, pchar[i]); nuclear@3: } nuclear@3: } nuclear@3: else nuclear@3: { nuclear@3: for (int i = 0;; i++) nuclear@3: { nuclear@3: if (pchar[i] == 0) nuclear@3: break; nuclear@3: EncodeChar(pbuff, &ofs, pchar[i]); nuclear@3: } nuclear@3: } nuclear@3: pbuff[ofs] = 0; nuclear@3: } nuclear@3: nuclear@3: UPInt OVR_STDCALL DecodeString(wchar_t *pbuff, const char* putf8str, SPInt bytesLen) nuclear@3: { nuclear@3: wchar_t *pbegin = pbuff; nuclear@3: if (bytesLen == -1) nuclear@3: { nuclear@3: while (1) nuclear@3: { nuclear@3: UInt32 ch = DecodeNextChar_Advance0(&putf8str); nuclear@3: if (ch == 0) nuclear@3: break; nuclear@3: else if (ch >= 0xFFFF) nuclear@3: ch = 0xFFFD; nuclear@3: *pbuff++ = wchar_t(ch); nuclear@3: } nuclear@3: } nuclear@3: else nuclear@3: { nuclear@3: const char* p = putf8str; nuclear@3: while ((p - putf8str) < bytesLen) nuclear@3: { nuclear@3: UInt32 ch = DecodeNextChar_Advance0(&p); nuclear@3: if (ch >= 0xFFFF) nuclear@3: ch = 0xFFFD; nuclear@3: *pbuff++ = wchar_t(ch); nuclear@3: } nuclear@3: } nuclear@3: nuclear@3: *pbuff = 0; nuclear@3: return pbuff - pbegin; nuclear@3: } nuclear@3: nuclear@3: nuclear@3: #ifdef UTF8_UNIT_TEST nuclear@3: nuclear@3: // Compile this test case with something like: nuclear@3: // nuclear@3: // gcc utf8.cpp -g -I.. -DUTF8_UNIT_TEST -lstdc++ -o utf8_test nuclear@3: // nuclear@3: // or nuclear@3: // nuclear@3: // cl utf8.cpp -Zi -Od -DUTF8_UNIT_TEST -I.. nuclear@3: // nuclear@3: // If possible, try running the test program with the first arg nuclear@3: // pointing at the file: nuclear@3: // nuclear@3: // http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt nuclear@3: // nuclear@3: // and examine the results by eye to make sure they are acceptable to nuclear@3: // you. nuclear@3: nuclear@3: nuclear@3: #include "base/utility.h" nuclear@3: #include nuclear@3: nuclear@3: nuclear@3: bool check_equal(const char* utf8_in, const UInt32* ucs_in) nuclear@3: { nuclear@3: for (;;) nuclear@3: { nuclear@3: UInt32 next_ucs = *ucs_in++; nuclear@3: UInt32 next_ucs_from_utf8 = utf8::decode_next_unicode_character(&utf8_in); nuclear@3: if (next_ucs != next_ucs_from_utf8) nuclear@3: { nuclear@3: return false; nuclear@3: } nuclear@3: if (next_ucs == 0) nuclear@3: { nuclear@3: OVR_ASSERT(next_ucs_from_utf8 == 0); nuclear@3: break; nuclear@3: } nuclear@3: } nuclear@3: nuclear@3: return true; nuclear@3: } nuclear@3: nuclear@3: nuclear@3: void log_ascii(const char* line) nuclear@3: { nuclear@3: for (;;) nuclear@3: { nuclear@3: unsigned char c = (unsigned char) *line++; nuclear@3: if (c == 0) nuclear@3: { nuclear@3: // End of line. nuclear@3: return; nuclear@3: } nuclear@3: else if (c != '\n' nuclear@3: && (c < 32 || c > 127)) nuclear@3: { nuclear@3: // Non-printable as plain ASCII. nuclear@3: printf("<0x%02X>", (int) c); nuclear@3: } nuclear@3: else nuclear@3: { nuclear@3: printf("%c", c); nuclear@3: } nuclear@3: } nuclear@3: } nuclear@3: nuclear@3: nuclear@3: void log_ucs(const UInt32* line) nuclear@3: { nuclear@3: for (;;) nuclear@3: { nuclear@3: UInt32 uc = *line++; nuclear@3: if (uc == 0) nuclear@3: { nuclear@3: // End of line. nuclear@3: return; nuclear@3: } nuclear@3: else if (uc != '\n' nuclear@3: && (uc < 32 || uc > 127)) nuclear@3: { nuclear@3: // Non-printable as plain ASCII. nuclear@3: printf("", uc); nuclear@3: } nuclear@3: else nuclear@3: { nuclear@3: printf("%c", (char) uc); nuclear@3: } nuclear@3: } nuclear@3: } nuclear@3: nuclear@3: nuclear@3: // Simple canned test. nuclear@3: int main(int argc, const char* argv[]) nuclear@3: { nuclear@3: { nuclear@3: const char* test8 = "Ignacio CastaƱo"; nuclear@3: const UInt32 test32[] = nuclear@3: { nuclear@3: 0x49, 0x67, 0x6E, 0x61, 0x63, nuclear@3: 0x69, 0x6F, 0x20, 0x43, 0x61, nuclear@3: 0x73, 0x74, 0x61, 0xF1, 0x6F, nuclear@3: 0x00 nuclear@3: }; nuclear@3: nuclear@3: OVR_ASSERT(check_equal(test8, test32)); nuclear@3: } nuclear@3: nuclear@3: // If user passed an arg, try reading the file as UTF-8 encoded text. nuclear@3: if (argc > 1) nuclear@3: { nuclear@3: const char* filename = argv[1]; nuclear@3: FILE* fp = fopen(filename, "rb"); nuclear@3: if (fp == NULL) nuclear@3: { nuclear@3: printf("Can't open file '%s'\n", filename); nuclear@3: return 1; nuclear@3: } nuclear@3: nuclear@3: // Read lines from the file, encode/decode them, and highlight discrepancies. nuclear@3: const int LINE_SIZE = 200; // max line size nuclear@3: char line_buffer_utf8[LINE_SIZE]; nuclear@3: char reencoded_utf8[6 * LINE_SIZE]; nuclear@3: UInt32 line_buffer_ucs[LINE_SIZE]; nuclear@3: nuclear@3: int byte_counter = 0; nuclear@3: for (;;) nuclear@3: { nuclear@3: int c = fgetc(fp); nuclear@3: if (c == EOF) nuclear@3: { nuclear@3: // Done. nuclear@3: break; nuclear@3: } nuclear@3: line_buffer_utf8[byte_counter++] = c; nuclear@3: if (c == '\n' || byte_counter >= LINE_SIZE - 2) nuclear@3: { nuclear@3: // End of line. Process the line. nuclear@3: line_buffer_utf8[byte_counter++] = 0; // terminate. nuclear@3: nuclear@3: // Decode into UCS. nuclear@3: const char* p = line_buffer_utf8; nuclear@3: UInt32* q = line_buffer_ucs; nuclear@3: for (;;) nuclear@3: { nuclear@3: UInt32 uc = UTF8Util::DecodeNextChar(&p); nuclear@3: *q++ = uc; nuclear@3: nuclear@3: OVR_ASSERT(q < line_buffer_ucs + LINE_SIZE); nuclear@3: OVR_ASSERT(p < line_buffer_utf8 + LINE_SIZE); nuclear@3: nuclear@3: if (uc == 0) break; nuclear@3: } nuclear@3: nuclear@3: // Encode back into UTF-8. nuclear@3: q = line_buffer_ucs; nuclear@3: int index = 0; nuclear@3: for (;;) nuclear@3: { nuclear@3: UInt32 uc = *q++; nuclear@3: OVR_ASSERT(index < LINE_SIZE * 6 - 6); nuclear@3: int last_index = index; nuclear@3: UTF8Util::EncodeChar(reencoded_utf8, &index, uc); nuclear@3: OVR_ASSERT(index <= last_index + 6); nuclear@3: if (uc == 0) break; nuclear@3: } nuclear@3: nuclear@3: // This can be useful for debugging. nuclear@3: #if 0 nuclear@3: // Show the UCS and the re-encoded UTF-8. nuclear@3: log_ucs(line_buffer_ucs); nuclear@3: log_ascii(reencoded_utf8); nuclear@3: #endif // 0 nuclear@3: nuclear@3: OVR_ASSERT(check_equal(line_buffer_utf8, line_buffer_ucs)); nuclear@3: OVR_ASSERT(check_equal(reencoded_utf8, line_buffer_ucs)); nuclear@3: nuclear@3: // Start next line. nuclear@3: byte_counter = 0; nuclear@3: } nuclear@3: } nuclear@3: nuclear@3: fclose(fp); nuclear@3: } nuclear@3: nuclear@3: return 0; nuclear@3: } nuclear@3: nuclear@3: nuclear@3: #endif // UTF8_UNIT_TEST nuclear@3: nuclear@3: }} // namespace UTF8Util::OVR nuclear@3: