nuclear@3: /**************************************************************************
nuclear@3: 
nuclear@3: Filename    :   OVR_UTF8Util.cpp
nuclear@3: Content     :   UTF8 Unicode character encoding/decoding support
nuclear@3: Created     :   September 19, 2012
nuclear@3: Notes       : 
nuclear@3: Notes       :   Much useful info at "UTF-8 and Unicode FAQ"
nuclear@3:                 http://www.cl.cam.ac.uk/~mgk25/unicode.html
nuclear@3: 
nuclear@3: Copyright   :   Copyright 2012 Oculus VR, Inc. All Rights reserved.
nuclear@3: 
nuclear@3: Use of this software is subject to the terms of the Oculus license
nuclear@3: agreement provided at the time of installation or download, or which
nuclear@3: otherwise accompanies this software in either electronic or hard copy form.
nuclear@3: 
nuclear@3: ************************************************************************************/
nuclear@3: 
nuclear@3: #include "OVR_UTF8Util.h"
nuclear@3: 
nuclear@3: namespace OVR { namespace UTF8Util {
nuclear@3: 
nuclear@3: SPInt OVR_STDCALL GetLength(const char* buf, SPInt buflen)
nuclear@3: {
nuclear@3:     const char* p = buf;
nuclear@3:     SPInt length = 0;
nuclear@3: 
nuclear@3:     if (buflen != -1)
nuclear@3:     {
nuclear@3:         while (p - buf < buflen)
nuclear@3:         {
nuclear@3:             // We should be able to have ASStrings with 0 in the middle.
nuclear@3:             UTF8Util::DecodeNextChar_Advance0(&p);
nuclear@3:             length++;
nuclear@3:         }
nuclear@3:     }
nuclear@3:     else
nuclear@3:     {
nuclear@3:         while (UTF8Util::DecodeNextChar_Advance0(&p))
nuclear@3:             length++;
nuclear@3:     }
nuclear@3:     
nuclear@3:     return length;
nuclear@3: }
nuclear@3: 
nuclear@3: UInt32 OVR_STDCALL GetCharAt(SPInt index, const char* putf8str, SPInt length)
nuclear@3: {
nuclear@3:     const char* buf = putf8str;
nuclear@3:     UInt32  c = 0;
nuclear@3: 
nuclear@3:     if (length != -1)
nuclear@3:     {
nuclear@3:         while (buf - putf8str < length)
nuclear@3:         {           
nuclear@3:             c = UTF8Util::DecodeNextChar_Advance0(&buf);
nuclear@3:             if (index == 0)
nuclear@3:                 return c;
nuclear@3:             index--;
nuclear@3:         }
nuclear@3: 
nuclear@3:         return c;
nuclear@3:     }
nuclear@3: 
nuclear@3:     do 
nuclear@3:     {
nuclear@3:         c = UTF8Util::DecodeNextChar_Advance0(&buf);
nuclear@3:         index--;
nuclear@3: 
nuclear@3:         if (c == 0)
nuclear@3:         {
nuclear@3:             // We've hit the end of the string; don't go further.
nuclear@3:             OVR_ASSERT(index == 0);
nuclear@3:             return c;
nuclear@3:         }
nuclear@3:     } while (index >= 0);
nuclear@3: 
nuclear@3:     return c;
nuclear@3: }
nuclear@3: 
nuclear@3: SPInt OVR_STDCALL GetByteIndex(SPInt index, const char *putf8str, SPInt length)
nuclear@3: {
nuclear@3:     const char* buf = putf8str;
nuclear@3: 
nuclear@3:     if (length != -1)
nuclear@3:     {
nuclear@3:         while ((buf - putf8str) < length && index > 0)
nuclear@3:         {
nuclear@3:             UTF8Util::DecodeNextChar_Advance0(&buf);
nuclear@3:             index--;
nuclear@3:         }
nuclear@3: 
nuclear@3:         return buf-putf8str;
nuclear@3:     }
nuclear@3: 
nuclear@3:     while (index > 0) 
nuclear@3:     {
nuclear@3:         UInt32 c = UTF8Util::DecodeNextChar_Advance0(&buf);
nuclear@3:         index--;
nuclear@3: 
nuclear@3:         if (c == 0)
nuclear@3:             return buf-putf8str;
nuclear@3:     };
nuclear@3: 
nuclear@3:     return buf-putf8str;
nuclear@3: }
nuclear@3: 
nuclear@3: int OVR_STDCALL GetEncodeCharSize(UInt32 ucs_character)
nuclear@3: {
nuclear@3:     if (ucs_character <= 0x7F)
nuclear@3:         return 1;
nuclear@3:     else if (ucs_character <= 0x7FF)
nuclear@3:         return 2;
nuclear@3:     else if (ucs_character <= 0xFFFF)
nuclear@3:         return 3;
nuclear@3:     else if (ucs_character <= 0x1FFFFF)
nuclear@3:         return 4;
nuclear@3:     else if (ucs_character <= 0x3FFFFFF)
nuclear@3:         return 5;
nuclear@3:     else if (ucs_character <= 0x7FFFFFFF)
nuclear@3:         return 6;
nuclear@3:     else
nuclear@3:         return 0;
nuclear@3: }
nuclear@3: 
nuclear@3: UInt32 OVR_STDCALL DecodeNextChar_Advance0(const char** putf8Buffer)
nuclear@3: {
nuclear@3:     UInt32  uc;
nuclear@3:     char    c;
nuclear@3:     
nuclear@3:     // Security considerations:
nuclear@3:     //
nuclear@3:     // Changed, this is now only the case for DecodeNextChar:
nuclear@3:     //  - If we hit a zero byte, we want to return 0 without stepping
nuclear@3:     //    the buffer pointer past the 0. th
nuclear@3:     //
nuclear@3:     // If we hit an "overlong sequence"; i.e. a character encoded
nuclear@3:     // in a longer multibyte string than is necessary, then we
nuclear@3:     // need to discard the character.  This is so attackers can't
nuclear@3:     // disguise dangerous characters or character sequences --
nuclear@3:     // there is only one valid encoding for each character.
nuclear@3:     //
nuclear@3:     // If we decode characters { 0xD800 .. 0xDFFF } or { 0xFFFE,
nuclear@3:     // 0xFFFF } then we ignore them; they are not valid in UTF-8.
nuclear@3:     
nuclear@3:     // This isn't actually an invalid character; it's a valid char that
nuclear@3:     // looks like an inverted question mark.
nuclear@3: #define INVALID_CHAR 0x0FFFD
nuclear@3:     
nuclear@3: #define FIRST_BYTE(mask, shift)     \
nuclear@3:     uc = (c & (mask)) << (shift);
nuclear@3:     
nuclear@3: #define NEXT_BYTE(shift) \
nuclear@3:     c = **putf8Buffer;   \
nuclear@3:     if (c == 0) return 0; /* end of buffer, do not advance */   \
nuclear@3:     if ((c & 0xC0) != 0x80) return INVALID_CHAR; /* standard check */  \
nuclear@3:     (*putf8Buffer)++;    \
nuclear@3:     uc |= (c & 0x3F) << shift;
nuclear@3:     
nuclear@3:     c = **putf8Buffer;
nuclear@3:     (*putf8Buffer)++;
nuclear@3:     if (c == 0)
nuclear@3:         return 0;   // End of buffer.
nuclear@3:     
nuclear@3:     if ((c & 0x80) == 0) return (UInt32) c; // Conventional 7-bit ASCII.
nuclear@3:     
nuclear@3:     // Multi-byte sequences.
nuclear@3:     if ((c & 0xE0) == 0xC0)
nuclear@3:     {
nuclear@3:         // Two-byte sequence.
nuclear@3:         FIRST_BYTE(0x1F, 6);
nuclear@3:         NEXT_BYTE(0);
nuclear@3:         if (uc < 0x80) return INVALID_CHAR;  // overlong
nuclear@3:         return uc;
nuclear@3:     }
nuclear@3:     else if ((c & 0xF0) == 0xE0)
nuclear@3:     {
nuclear@3:         // Three-byte sequence.
nuclear@3:         FIRST_BYTE(0x0F, 12);
nuclear@3:         NEXT_BYTE(6);
nuclear@3:         NEXT_BYTE(0);
nuclear@3:         if (uc < 0x800) return INVALID_CHAR; // overlong
nuclear@3:         // Not valid ISO 10646, but Flash requires these to work
nuclear@3:         // see AS3 test e15_5_3_2_3 for String.fromCharCode().charCodeAt(0)
nuclear@3:         // if (uc >= 0x0D800 && uc <= 0x0DFFF) return INVALID_CHAR;
nuclear@3:         // if (uc == 0x0FFFE || uc == 0x0FFFF) return INVALID_CHAR; // not valid ISO 10646
nuclear@3:         return uc;
nuclear@3:     }
nuclear@3:     else if ((c & 0xF8) == 0xF0)
nuclear@3:     {
nuclear@3:         // Four-byte sequence.
nuclear@3:         FIRST_BYTE(0x07, 18);
nuclear@3:         NEXT_BYTE(12);
nuclear@3:         NEXT_BYTE(6);
nuclear@3:         NEXT_BYTE(0);
nuclear@3:         if (uc < 0x010000) return INVALID_CHAR;  // overlong
nuclear@3:         return uc;
nuclear@3:     }
nuclear@3:     else if ((c & 0xFC) == 0xF8)
nuclear@3:     {
nuclear@3:         // Five-byte sequence.
nuclear@3:         FIRST_BYTE(0x03, 24);
nuclear@3:         NEXT_BYTE(18);
nuclear@3:         NEXT_BYTE(12);
nuclear@3:         NEXT_BYTE(6);
nuclear@3:         NEXT_BYTE(0);
nuclear@3:         if (uc < 0x0200000) return INVALID_CHAR; // overlong
nuclear@3:         return uc;
nuclear@3:     }
nuclear@3:     else if ((c & 0xFE) == 0xFC)
nuclear@3:     {
nuclear@3:         // Six-byte sequence.
nuclear@3:         FIRST_BYTE(0x01, 30);
nuclear@3:         NEXT_BYTE(24);
nuclear@3:         NEXT_BYTE(18);
nuclear@3:         NEXT_BYTE(12);
nuclear@3:         NEXT_BYTE(6);
nuclear@3:         NEXT_BYTE(0);
nuclear@3:         if (uc < 0x04000000) return INVALID_CHAR;    // overlong
nuclear@3:         return uc;
nuclear@3:     }
nuclear@3:     else
nuclear@3:     {
nuclear@3:         // Invalid.
nuclear@3:         return INVALID_CHAR;
nuclear@3:     }
nuclear@3: }
nuclear@3: 
nuclear@3: 
nuclear@3: void OVR_STDCALL EncodeChar(char* pbuffer, SPInt* pindex, UInt32 ucs_character)
nuclear@3: {
nuclear@3:     if (ucs_character <= 0x7F)
nuclear@3:     {
nuclear@3:         // Plain single-byte ASCII.
nuclear@3:         pbuffer[(*pindex)++] = (char) ucs_character;
nuclear@3:     }
nuclear@3:     else if (ucs_character <= 0x7FF)
nuclear@3:     {
nuclear@3:         // Two bytes.
nuclear@3:         pbuffer[(*pindex)++] = 0xC0 | (char)(ucs_character >> 6);
nuclear@3:         pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 0) & 0x3F);
nuclear@3:     }
nuclear@3:     else if (ucs_character <= 0xFFFF)
nuclear@3:     {
nuclear@3:         // Three bytes.
nuclear@3:         pbuffer[(*pindex)++] = 0xE0 | (char)(ucs_character >> 12);
nuclear@3:         pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 6) & 0x3F);
nuclear@3:         pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 0) & 0x3F);
nuclear@3:     }
nuclear@3:     else if (ucs_character <= 0x1FFFFF)
nuclear@3:     {
nuclear@3:         // Four bytes.
nuclear@3:         pbuffer[(*pindex)++] = 0xF0 | (char)(ucs_character >> 18);
nuclear@3:         pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 12) & 0x3F);
nuclear@3:         pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 6) & 0x3F);
nuclear@3:         pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 0) & 0x3F);
nuclear@3:     }
nuclear@3:     else if (ucs_character <= 0x3FFFFFF)
nuclear@3:     {
nuclear@3:         // Five bytes.
nuclear@3:         pbuffer[(*pindex)++] = 0xF8 | (char)(ucs_character >> 24);
nuclear@3:         pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 18) & 0x3F);
nuclear@3:         pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 12) & 0x3F);
nuclear@3:         pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 6) & 0x3F);
nuclear@3:         pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 0) & 0x3F);
nuclear@3:     }
nuclear@3:     else if (ucs_character <= 0x7FFFFFFF)
nuclear@3:     {
nuclear@3:         // Six bytes.
nuclear@3:         pbuffer[(*pindex)++] = 0xFC | (char)(ucs_character >> 30);
nuclear@3:         pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 24) & 0x3F);
nuclear@3:         pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 18) & 0x3F);
nuclear@3:         pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 12) & 0x3F);
nuclear@3:         pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 6) & 0x3F);
nuclear@3:         pbuffer[(*pindex)++] = 0x80 | (char)((ucs_character >> 0) & 0x3F);
nuclear@3:     }
nuclear@3:     else
nuclear@3:     {
nuclear@3:         // Invalid char; don't encode anything.
nuclear@3:     }
nuclear@3: }
nuclear@3: 
nuclear@3: SPInt OVR_STDCALL GetEncodeStringSize(const wchar_t* pchar, SPInt length)
nuclear@3: {
nuclear@3:     SPInt len = 0;
nuclear@3:     if (length != -1)
nuclear@3:         for (int i = 0; i < length; i++)
nuclear@3:         {
nuclear@3:             len += GetEncodeCharSize(pchar[i]);
nuclear@3:         }
nuclear@3:     else
nuclear@3:         for (int i = 0;; i++)
nuclear@3:         {
nuclear@3:             if (pchar[i] == 0)
nuclear@3:                 return len;
nuclear@3:             len += GetEncodeCharSize(pchar[i]);
nuclear@3:         }
nuclear@3:     return len;
nuclear@3: }
nuclear@3: 
nuclear@3: void OVR_STDCALL EncodeString(char *pbuff, const wchar_t* pchar, SPInt length)
nuclear@3: {
nuclear@3:     SPInt ofs = 0;
nuclear@3:     if (length != -1)
nuclear@3:     {
nuclear@3:         for (int i = 0; i < length; i++)
nuclear@3:         {            
nuclear@3:             EncodeChar(pbuff, &ofs, pchar[i]);
nuclear@3:         }
nuclear@3:     }
nuclear@3:     else
nuclear@3:     {
nuclear@3:         for (int i = 0;; i++)
nuclear@3:         {
nuclear@3:             if (pchar[i] == 0)
nuclear@3:                 break;
nuclear@3:             EncodeChar(pbuff, &ofs, pchar[i]);
nuclear@3:         }
nuclear@3:     }
nuclear@3:     pbuff[ofs] = 0;
nuclear@3: }
nuclear@3: 
nuclear@3: UPInt OVR_STDCALL DecodeString(wchar_t *pbuff, const char* putf8str, SPInt bytesLen)
nuclear@3: {
nuclear@3:     wchar_t *pbegin = pbuff;
nuclear@3:     if (bytesLen == -1)
nuclear@3:     {
nuclear@3:         while (1)
nuclear@3:         {
nuclear@3:             UInt32 ch = DecodeNextChar_Advance0(&putf8str);
nuclear@3:             if (ch == 0)
nuclear@3:                 break;
nuclear@3:             else if (ch >= 0xFFFF)
nuclear@3:                 ch = 0xFFFD;
nuclear@3:             *pbuff++ = wchar_t(ch);
nuclear@3:         }
nuclear@3:     }
nuclear@3:     else
nuclear@3:     {
nuclear@3:         const char* p = putf8str;
nuclear@3:         while ((p - putf8str) < bytesLen)
nuclear@3:         {
nuclear@3:             UInt32 ch = DecodeNextChar_Advance0(&p);
nuclear@3:             if (ch >= 0xFFFF)
nuclear@3:                 ch = 0xFFFD;
nuclear@3:             *pbuff++ = wchar_t(ch);
nuclear@3:         }
nuclear@3:     }
nuclear@3: 
nuclear@3:     *pbuff = 0;
nuclear@3:     return pbuff - pbegin;
nuclear@3: }
nuclear@3: 
nuclear@3: 
nuclear@3: #ifdef UTF8_UNIT_TEST
nuclear@3: 
nuclear@3: // Compile this test case with something like:
nuclear@3: //
nuclear@3: // gcc utf8.cpp -g -I.. -DUTF8_UNIT_TEST -lstdc++ -o utf8_test
nuclear@3: //
nuclear@3: //    or
nuclear@3: //
nuclear@3: // cl utf8.cpp -Zi -Od -DUTF8_UNIT_TEST -I..
nuclear@3: //
nuclear@3: // If possible, try running the test program with the first arg
nuclear@3: // pointing at the file:
nuclear@3: //
nuclear@3: // http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
nuclear@3: // 
nuclear@3: // and examine the results by eye to make sure they are acceptable to
nuclear@3: // you.
nuclear@3: 
nuclear@3: 
nuclear@3: #include "base/utility.h"
nuclear@3: #include <stdio.h>
nuclear@3: 
nuclear@3: 
nuclear@3: bool    check_equal(const char* utf8_in, const UInt32* ucs_in)
nuclear@3: {
nuclear@3:     for (;;)
nuclear@3:     {
nuclear@3:         UInt32  next_ucs = *ucs_in++;
nuclear@3:         UInt32  next_ucs_from_utf8 = utf8::decode_next_unicode_character(&utf8_in);
nuclear@3:         if (next_ucs != next_ucs_from_utf8)
nuclear@3:         {
nuclear@3:             return false;
nuclear@3:         }
nuclear@3:         if (next_ucs == 0)
nuclear@3:         {
nuclear@3:             OVR_ASSERT(next_ucs_from_utf8 == 0);
nuclear@3:             break;
nuclear@3:         }
nuclear@3:     }
nuclear@3:     
nuclear@3:     return true;
nuclear@3: }
nuclear@3: 
nuclear@3: 
nuclear@3: void    log_ascii(const char* line)
nuclear@3: {
nuclear@3:     for (;;)
nuclear@3:     {
nuclear@3:         unsigned char   c = (unsigned char) *line++;
nuclear@3:         if (c == 0)
nuclear@3:         {
nuclear@3:             // End of line.
nuclear@3:             return;
nuclear@3:         }
nuclear@3:         else if (c != '\n'
nuclear@3:             && (c < 32 || c > 127))
nuclear@3:         {
nuclear@3:             // Non-printable as plain ASCII.
nuclear@3:             printf("<0x%02X>", (int) c);
nuclear@3:         }
nuclear@3:         else
nuclear@3:         {
nuclear@3:             printf("%c", c);
nuclear@3:         }
nuclear@3:     }
nuclear@3: }
nuclear@3: 
nuclear@3: 
nuclear@3: void    log_ucs(const UInt32* line)
nuclear@3: {
nuclear@3:     for (;;)
nuclear@3:     {
nuclear@3:         UInt32  uc = *line++;
nuclear@3:         if (uc == 0)
nuclear@3:         {
nuclear@3:             // End of line.
nuclear@3:             return;
nuclear@3:         }
nuclear@3:         else if (uc != '\n'
nuclear@3:             && (uc < 32 || uc > 127))
nuclear@3:         {
nuclear@3:             // Non-printable as plain ASCII.
nuclear@3:             printf("<U-%04X>", uc);
nuclear@3:         }
nuclear@3:         else
nuclear@3:         {
nuclear@3:             printf("%c", (char) uc);
nuclear@3:         }
nuclear@3:     }
nuclear@3: }
nuclear@3: 
nuclear@3: 
nuclear@3: // Simple canned test.
nuclear@3: int main(int argc, const char* argv[])
nuclear@3: {
nuclear@3:     {
nuclear@3:         const char* test8 = "Ignacio Castaño";
nuclear@3:         const UInt32    test32[] =
nuclear@3:         {
nuclear@3:             0x49, 0x67, 0x6E, 0x61, 0x63,
nuclear@3:                 0x69, 0x6F, 0x20, 0x43, 0x61,
nuclear@3:                 0x73, 0x74, 0x61, 0xF1, 0x6F,
nuclear@3:                 0x00
nuclear@3:         };
nuclear@3:         
nuclear@3:         OVR_ASSERT(check_equal(test8, test32));
nuclear@3:     }
nuclear@3:         
nuclear@3:         // If user passed an arg, try reading the file as UTF-8 encoded text.
nuclear@3:         if (argc > 1)
nuclear@3:         {
nuclear@3:             const char* filename = argv[1];
nuclear@3:             FILE*   fp = fopen(filename, "rb");
nuclear@3:             if (fp == NULL)
nuclear@3:             {
nuclear@3:                 printf("Can't open file '%s'\n", filename);
nuclear@3:                 return 1;
nuclear@3:             }
nuclear@3:             
nuclear@3:             // Read lines from the file, encode/decode them, and highlight discrepancies.
nuclear@3:             const int LINE_SIZE = 200;  // max line size
nuclear@3:             char    line_buffer_utf8[LINE_SIZE];
nuclear@3:             char    reencoded_utf8[6 * LINE_SIZE];
nuclear@3:             UInt32  line_buffer_ucs[LINE_SIZE];
nuclear@3:             
nuclear@3:             int byte_counter = 0;
nuclear@3:             for (;;)
nuclear@3:             {
nuclear@3:                 int c = fgetc(fp);
nuclear@3:                 if (c == EOF)
nuclear@3:                 {
nuclear@3:                     // Done.
nuclear@3:                     break;
nuclear@3:                 }
nuclear@3:                 line_buffer_utf8[byte_counter++] = c;
nuclear@3:                 if (c == '\n' || byte_counter >= LINE_SIZE - 2)
nuclear@3:                 {
nuclear@3:                     // End of line.  Process the line.
nuclear@3:                     line_buffer_utf8[byte_counter++] = 0;   // terminate.
nuclear@3:                     
nuclear@3:                     // Decode into UCS.
nuclear@3:                     const char* p = line_buffer_utf8;
nuclear@3:                     UInt32* q = line_buffer_ucs;
nuclear@3:                     for (;;)
nuclear@3:                     {
nuclear@3:                         UInt32  uc = UTF8Util::DecodeNextChar(&p);
nuclear@3:                         *q++ = uc;
nuclear@3:                         
nuclear@3:                         OVR_ASSERT(q < line_buffer_ucs + LINE_SIZE);
nuclear@3:                         OVR_ASSERT(p < line_buffer_utf8 + LINE_SIZE);
nuclear@3:                         
nuclear@3:                         if (uc == 0) break;
nuclear@3:                     }
nuclear@3:                     
nuclear@3:                     // Encode back into UTF-8.
nuclear@3:                     q = line_buffer_ucs;
nuclear@3:                     int index = 0;
nuclear@3:                     for (;;)
nuclear@3:                     {
nuclear@3:                         UInt32  uc = *q++;
nuclear@3:                         OVR_ASSERT(index < LINE_SIZE * 6 - 6);
nuclear@3:                         int last_index = index;
nuclear@3:                         UTF8Util::EncodeChar(reencoded_utf8, &index, uc);
nuclear@3:                         OVR_ASSERT(index <= last_index + 6);
nuclear@3:                         if (uc == 0) break;
nuclear@3:                     }
nuclear@3:                     
nuclear@3:                     // This can be useful for debugging.
nuclear@3: #if 0
nuclear@3:                     // Show the UCS and the re-encoded UTF-8.
nuclear@3:                     log_ucs(line_buffer_ucs);
nuclear@3:                     log_ascii(reencoded_utf8);
nuclear@3: #endif // 0
nuclear@3:                     
nuclear@3:                     OVR_ASSERT(check_equal(line_buffer_utf8, line_buffer_ucs));
nuclear@3:                     OVR_ASSERT(check_equal(reencoded_utf8, line_buffer_ucs));
nuclear@3:                     
nuclear@3:                     // Start next line.
nuclear@3:                     byte_counter = 0;
nuclear@3:                 }
nuclear@3:             }
nuclear@3:             
nuclear@3:             fclose(fp);
nuclear@3:         }
nuclear@3:         
nuclear@3:         return 0;
nuclear@3: }
nuclear@3: 
nuclear@3: 
nuclear@3: #endif // UTF8_UNIT_TEST
nuclear@3: 
nuclear@3: }} // namespace UTF8Util::OVR
nuclear@3: