vrshoot

annotate libs/assimp/ConvertUTF/ConvertUTF.c @ 1:e7ca128b8713

find changesets by author, revision, files, or words in the commit message
looks nice :)
author John Tsiombikas <nuclear@member.fsf.org>
date Sun, 02 Feb 2014 00:35:22 +0200
parents
children
rev   line source
nuclear@0 1 /*
nuclear@0 2 * Copyright 2001-2004 Unicode, Inc.
nuclear@0 3 *
nuclear@0 4 * Disclaimer
nuclear@0 5 *
nuclear@0 6 * This source code is provided as is by Unicode, Inc. No claims are
nuclear@0 7 * made as to fitness for any particular purpose. No warranties of any
nuclear@0 8 * kind are expressed or implied. The recipient agrees to determine
nuclear@0 9 * applicability of information provided. If this file has been
nuclear@0 10 * purchased on magnetic or optical media from Unicode, Inc., the
nuclear@0 11 * sole remedy for any claim will be exchange of defective media
nuclear@0 12 * within 90 days of receipt.
nuclear@0 13 *
nuclear@0 14 * Limitations on Rights to Redistribute This Code
nuclear@0 15 *
nuclear@0 16 * Unicode, Inc. hereby grants the right to freely use the information
nuclear@0 17 * supplied in this file in the creation of products supporting the
nuclear@0 18 * Unicode Standard, and to make copies of this file in any form
nuclear@0 19 * for internal or external distribution as long as this notice
nuclear@0 20 * remains attached.
nuclear@0 21 */
nuclear@0 22
nuclear@0 23 /* ---------------------------------------------------------------------
nuclear@0 24
nuclear@0 25 Conversions between UTF32, UTF-16, and UTF-8. Source code file.
nuclear@0 26 Author: Mark E. Davis, 1994.
nuclear@0 27 Rev History: Rick McGowan, fixes & updates May 2001.
nuclear@0 28 Sept 2001: fixed const & error conditions per
nuclear@0 29 mods suggested by S. Parent & A. Lillich.
nuclear@0 30 June 2002: Tim Dodd added detection and handling of incomplete
nuclear@0 31 source sequences, enhanced error detection, added casts
nuclear@0 32 to eliminate compiler warnings.
nuclear@0 33 July 2003: slight mods to back out aggressive FFFE detection.
nuclear@0 34 Jan 2004: updated switches in from-UTF8 conversions.
nuclear@0 35 Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.
nuclear@0 36
nuclear@0 37 See the header file "ConvertUTF.h" for complete documentation.
nuclear@0 38
nuclear@0 39 ------------------------------------------------------------------------ */
nuclear@0 40
nuclear@0 41
nuclear@0 42 #include "ConvertUTF.h"
nuclear@0 43 #ifdef CVTUTF_DEBUG
nuclear@0 44 #include <stdio.h>
nuclear@0 45 #endif
nuclear@0 46
nuclear@0 47 static const int halfShift = 10; /* used for shifting by 10 bits */
nuclear@0 48
nuclear@0 49 static const UTF32 halfBase = 0x0010000UL;
nuclear@0 50 static const UTF32 halfMask = 0x3FFUL;
nuclear@0 51
nuclear@0 52 #define UNI_SUR_HIGH_START (UTF32)0xD800
nuclear@0 53 #define UNI_SUR_HIGH_END (UTF32)0xDBFF
nuclear@0 54 #define UNI_SUR_LOW_START (UTF32)0xDC00
nuclear@0 55 #define UNI_SUR_LOW_END (UTF32)0xDFFF
nuclear@0 56 #define false 0
nuclear@0 57 #define true 1
nuclear@0 58
nuclear@0 59 /* --------------------------------------------------------------------- */
nuclear@0 60
nuclear@0 61 ConversionResult ConvertUTF32toUTF16 (
nuclear@0 62 const UTF32** sourceStart, const UTF32* sourceEnd,
nuclear@0 63 UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
nuclear@0 64 ConversionResult result = conversionOK;
nuclear@0 65 const UTF32* source = *sourceStart;
nuclear@0 66 UTF16* target = *targetStart;
nuclear@0 67 while (source < sourceEnd) {
nuclear@0 68 UTF32 ch;
nuclear@0 69 if (target >= targetEnd) {
nuclear@0 70 result = targetExhausted; break;
nuclear@0 71 }
nuclear@0 72 ch = *source++;
nuclear@0 73 if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
nuclear@0 74 /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */
nuclear@0 75 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
nuclear@0 76 if (flags == strictConversion) {
nuclear@0 77 --source; /* return to the illegal value itself */
nuclear@0 78 result = sourceIllegal;
nuclear@0 79 break;
nuclear@0 80 } else {
nuclear@0 81 *target++ = UNI_REPLACEMENT_CHAR;
nuclear@0 82 }
nuclear@0 83 } else {
nuclear@0 84 *target++ = (UTF16)ch; /* normal case */
nuclear@0 85 }
nuclear@0 86 } else if (ch > UNI_MAX_LEGAL_UTF32) {
nuclear@0 87 if (flags == strictConversion) {
nuclear@0 88 result = sourceIllegal;
nuclear@0 89 } else {
nuclear@0 90 *target++ = UNI_REPLACEMENT_CHAR;
nuclear@0 91 }
nuclear@0 92 } else {
nuclear@0 93 /* target is a character in range 0xFFFF - 0x10FFFF. */
nuclear@0 94 if (target + 1 >= targetEnd) {
nuclear@0 95 --source; /* Back up source pointer! */
nuclear@0 96 result = targetExhausted; break;
nuclear@0 97 }
nuclear@0 98 ch -= halfBase;
nuclear@0 99 *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
nuclear@0 100 *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
nuclear@0 101 }
nuclear@0 102 }
nuclear@0 103 *sourceStart = source;
nuclear@0 104 *targetStart = target;
nuclear@0 105 return result;
nuclear@0 106 }
nuclear@0 107
nuclear@0 108 /* --------------------------------------------------------------------- */
nuclear@0 109
nuclear@0 110 ConversionResult ConvertUTF16toUTF32 (
nuclear@0 111 const UTF16** sourceStart, const UTF16* sourceEnd,
nuclear@0 112 UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
nuclear@0 113 ConversionResult result = conversionOK;
nuclear@0 114 const UTF16* source = *sourceStart;
nuclear@0 115 UTF32* target = *targetStart;
nuclear@0 116 UTF32 ch, ch2;
nuclear@0 117 while (source < sourceEnd) {
nuclear@0 118 const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
nuclear@0 119 ch = *source++;
nuclear@0 120 /* If we have a surrogate pair, convert to UTF32 first. */
nuclear@0 121 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
nuclear@0 122 /* If the 16 bits following the high surrogate are in the source buffer... */
nuclear@0 123 if (source < sourceEnd) {
nuclear@0 124 ch2 = *source;
nuclear@0 125 /* If it's a low surrogate, convert to UTF32. */
nuclear@0 126 if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
nuclear@0 127 ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
nuclear@0 128 + (ch2 - UNI_SUR_LOW_START) + halfBase;
nuclear@0 129 ++source;
nuclear@0 130 } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
nuclear@0 131 --source; /* return to the illegal value itself */
nuclear@0 132 result = sourceIllegal;
nuclear@0 133 break;
nuclear@0 134 }
nuclear@0 135 } else { /* We don't have the 16 bits following the high surrogate. */
nuclear@0 136 --source; /* return to the high surrogate */
nuclear@0 137 result = sourceExhausted;
nuclear@0 138 break;
nuclear@0 139 }
nuclear@0 140 } else if (flags == strictConversion) {
nuclear@0 141 /* UTF-16 surrogate values are illegal in UTF-32 */
nuclear@0 142 if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
nuclear@0 143 --source; /* return to the illegal value itself */
nuclear@0 144 result = sourceIllegal;
nuclear@0 145 break;
nuclear@0 146 }
nuclear@0 147 }
nuclear@0 148 if (target >= targetEnd) {
nuclear@0 149 source = oldSource; /* Back up source pointer! */
nuclear@0 150 result = targetExhausted; break;
nuclear@0 151 }
nuclear@0 152 *target++ = ch;
nuclear@0 153 }
nuclear@0 154 *sourceStart = source;
nuclear@0 155 *targetStart = target;
nuclear@0 156 #ifdef CVTUTF_DEBUG
nuclear@0 157 if (result == sourceIllegal) {
nuclear@0 158 fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2);
nuclear@0 159 fflush(stderr);
nuclear@0 160 }
nuclear@0 161 #endif
nuclear@0 162 return result;
nuclear@0 163 }
nuclear@0 164
nuclear@0 165 /* --------------------------------------------------------------------- */
nuclear@0 166
nuclear@0 167 /*
nuclear@0 168 * Index into the table below with the first byte of a UTF-8 sequence to
nuclear@0 169 * get the number of trailing bytes that are supposed to follow it.
nuclear@0 170 * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
nuclear@0 171 * left as-is for anyone who may want to do such conversion, which was
nuclear@0 172 * allowed in earlier algorithms.
nuclear@0 173 */
nuclear@0 174 static const char trailingBytesForUTF8[256] = {
nuclear@0 175 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
nuclear@0 176 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
nuclear@0 177 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
nuclear@0 178 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
nuclear@0 179 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
nuclear@0 180 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
nuclear@0 181 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
nuclear@0 182 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
nuclear@0 183 };
nuclear@0 184
nuclear@0 185 /*
nuclear@0 186 * Magic values subtracted from a buffer value during UTF8 conversion.
nuclear@0 187 * This table contains as many values as there might be trailing bytes
nuclear@0 188 * in a UTF-8 sequence.
nuclear@0 189 */
nuclear@0 190 static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL,
nuclear@0 191 0x03C82080UL, 0xFA082080UL, 0x82082080UL };
nuclear@0 192
nuclear@0 193 /*
nuclear@0 194 * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
nuclear@0 195 * into the first byte, depending on how many bytes follow. There are
nuclear@0 196 * as many entries in this table as there are UTF-8 sequence types.
nuclear@0 197 * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
nuclear@0 198 * for *legal* UTF-8 will be 4 or fewer bytes total.
nuclear@0 199 */
nuclear@0 200 static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
nuclear@0 201
nuclear@0 202 /* --------------------------------------------------------------------- */
nuclear@0 203
nuclear@0 204 /* The interface converts a whole buffer to avoid function-call overhead.
nuclear@0 205 * Constants have been gathered. Loops & conditionals have been removed as
nuclear@0 206 * much as possible for efficiency, in favor of drop-through switches.
nuclear@0 207 * (See "Note A" at the bottom of the file for equivalent code.)
nuclear@0 208 * If your compiler supports it, the "isLegalUTF8" call can be turned
nuclear@0 209 * into an inline function.
nuclear@0 210 */
nuclear@0 211
nuclear@0 212 /* --------------------------------------------------------------------- */
nuclear@0 213
nuclear@0 214 ConversionResult ConvertUTF16toUTF8 (
nuclear@0 215 const UTF16** sourceStart, const UTF16* sourceEnd,
nuclear@0 216 UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
nuclear@0 217 ConversionResult result = conversionOK;
nuclear@0 218 const UTF16* source = *sourceStart;
nuclear@0 219 UTF8* target = *targetStart;
nuclear@0 220 while (source < sourceEnd) {
nuclear@0 221 UTF32 ch;
nuclear@0 222 unsigned short bytesToWrite = 0;
nuclear@0 223 const UTF32 byteMask = 0xBF;
nuclear@0 224 const UTF32 byteMark = 0x80;
nuclear@0 225 const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */
nuclear@0 226 ch = *source++;
nuclear@0 227 /* If we have a surrogate pair, convert to UTF32 first. */
nuclear@0 228 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
nuclear@0 229 /* If the 16 bits following the high surrogate are in the source buffer... */
nuclear@0 230 if (source < sourceEnd) {
nuclear@0 231 UTF32 ch2 = *source;
nuclear@0 232 /* If it's a low surrogate, convert to UTF32. */
nuclear@0 233 if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
nuclear@0 234 ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
nuclear@0 235 + (ch2 - UNI_SUR_LOW_START) + halfBase;
nuclear@0 236 ++source;
nuclear@0 237 } else if (flags == strictConversion) { /* it's an unpaired high surrogate */
nuclear@0 238 --source; /* return to the illegal value itself */
nuclear@0 239 result = sourceIllegal;
nuclear@0 240 break;
nuclear@0 241 }
nuclear@0 242 } else { /* We don't have the 16 bits following the high surrogate. */
nuclear@0 243 --source; /* return to the high surrogate */
nuclear@0 244 result = sourceExhausted;
nuclear@0 245 break;
nuclear@0 246 }
nuclear@0 247 } else if (flags == strictConversion) {
nuclear@0 248 /* UTF-16 surrogate values are illegal in UTF-32 */
nuclear@0 249 if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) {
nuclear@0 250 --source; /* return to the illegal value itself */
nuclear@0 251 result = sourceIllegal;
nuclear@0 252 break;
nuclear@0 253 }
nuclear@0 254 }
nuclear@0 255 /* Figure out how many bytes the result will require */
nuclear@0 256 if (ch < (UTF32)0x80) { bytesToWrite = 1;
nuclear@0 257 } else if (ch < (UTF32)0x800) { bytesToWrite = 2;
nuclear@0 258 } else if (ch < (UTF32)0x10000) { bytesToWrite = 3;
nuclear@0 259 } else if (ch < (UTF32)0x110000) { bytesToWrite = 4;
nuclear@0 260 } else { bytesToWrite = 3;
nuclear@0 261 ch = UNI_REPLACEMENT_CHAR;
nuclear@0 262 }
nuclear@0 263
nuclear@0 264 target += bytesToWrite;
nuclear@0 265 if (target > targetEnd) {
nuclear@0 266 source = oldSource; /* Back up source pointer! */
nuclear@0 267 target -= bytesToWrite; result = targetExhausted; break;
nuclear@0 268 }
nuclear@0 269 switch (bytesToWrite) { /* note: everything falls through. */
nuclear@0 270 case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
nuclear@0 271 case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
nuclear@0 272 case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
nuclear@0 273 case 1: *--target = (UTF8)(ch | firstByteMark[bytesToWrite]);
nuclear@0 274 }
nuclear@0 275 target += bytesToWrite;
nuclear@0 276 }
nuclear@0 277 *sourceStart = source;
nuclear@0 278 *targetStart = target;
nuclear@0 279 return result;
nuclear@0 280 }
nuclear@0 281
nuclear@0 282 /* --------------------------------------------------------------------- */
nuclear@0 283
nuclear@0 284 /*
nuclear@0 285 * Utility routine to tell whether a sequence of bytes is legal UTF-8.
nuclear@0 286 * This must be called with the length pre-determined by the first byte.
nuclear@0 287 * If not calling this from ConvertUTF8to*, then the length can be set by:
nuclear@0 288 * length = trailingBytesForUTF8[*source]+1;
nuclear@0 289 * and the sequence is illegal right away if there aren't that many bytes
nuclear@0 290 * available.
nuclear@0 291 * If presented with a length > 4, this returns false. The Unicode
nuclear@0 292 * definition of UTF-8 goes up to 4-byte sequences.
nuclear@0 293 */
nuclear@0 294
nuclear@0 295 static Boolean isLegalUTF8(const UTF8 *source, int length) {
nuclear@0 296 UTF8 a;
nuclear@0 297 const UTF8 *srcptr = source+length;
nuclear@0 298 switch (length) {
nuclear@0 299 default: return false;
nuclear@0 300 /* Everything else falls through when "true"... */
nuclear@0 301 case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
nuclear@0 302 case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
nuclear@0 303 case 2: if ((a = (*--srcptr)) > 0xBF) return false;
nuclear@0 304
nuclear@0 305 switch (*source) {
nuclear@0 306 /* no fall-through in this inner switch */
nuclear@0 307 case 0xE0: if (a < 0xA0) return false; break;
nuclear@0 308 case 0xED: if (a > 0x9F) return false; break;
nuclear@0 309 case 0xF0: if (a < 0x90) return false; break;
nuclear@0 310 case 0xF4: if (a > 0x8F) return false; break;
nuclear@0 311 default: if (a < 0x80) return false;
nuclear@0 312 }
nuclear@0 313
nuclear@0 314 case 1: if (*source >= 0x80 && *source < 0xC2) return false;
nuclear@0 315 }
nuclear@0 316 if (*source > 0xF4) return false;
nuclear@0 317 return true;
nuclear@0 318 }
nuclear@0 319
nuclear@0 320 /* --------------------------------------------------------------------- */
nuclear@0 321
nuclear@0 322 /*
nuclear@0 323 * Exported function to return whether a UTF-8 sequence is legal or not.
nuclear@0 324 * This is not used here; it's just exported.
nuclear@0 325 */
nuclear@0 326 Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
nuclear@0 327 int length = trailingBytesForUTF8[*source]+1;
nuclear@0 328 if (source+length > sourceEnd) {
nuclear@0 329 return false;
nuclear@0 330 }
nuclear@0 331 return isLegalUTF8(source, length);
nuclear@0 332 }
nuclear@0 333
nuclear@0 334 /* --------------------------------------------------------------------- */
nuclear@0 335
nuclear@0 336 ConversionResult ConvertUTF8toUTF16 (
nuclear@0 337 const UTF8** sourceStart, const UTF8* sourceEnd,
nuclear@0 338 UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) {
nuclear@0 339 ConversionResult result = conversionOK;
nuclear@0 340 const UTF8* source = *sourceStart;
nuclear@0 341 UTF16* target = *targetStart;
nuclear@0 342 while (source < sourceEnd) {
nuclear@0 343 UTF32 ch = 0;
nuclear@0 344 unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
nuclear@0 345 if (source + extraBytesToRead >= sourceEnd) {
nuclear@0 346 result = sourceExhausted; break;
nuclear@0 347 }
nuclear@0 348 /* Do this check whether lenient or strict */
nuclear@0 349 if (! isLegalUTF8(source, extraBytesToRead+1)) {
nuclear@0 350 result = sourceIllegal;
nuclear@0 351 break;
nuclear@0 352 }
nuclear@0 353 /*
nuclear@0 354 * The cases all fall through. See "Note A" below.
nuclear@0 355 */
nuclear@0 356 switch (extraBytesToRead) {
nuclear@0 357 case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
nuclear@0 358 case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */
nuclear@0 359 case 3: ch += *source++; ch <<= 6;
nuclear@0 360 case 2: ch += *source++; ch <<= 6;
nuclear@0 361 case 1: ch += *source++; ch <<= 6;
nuclear@0 362 case 0: ch += *source++;
nuclear@0 363 }
nuclear@0 364 ch -= offsetsFromUTF8[extraBytesToRead];
nuclear@0 365
nuclear@0 366 if (target >= targetEnd) {
nuclear@0 367 source -= (extraBytesToRead+1); /* Back up source pointer! */
nuclear@0 368 result = targetExhausted; break;
nuclear@0 369 }
nuclear@0 370 if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */
nuclear@0 371 /* UTF-16 surrogate values are illegal in UTF-32 */
nuclear@0 372 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
nuclear@0 373 if (flags == strictConversion) {
nuclear@0 374 source -= (extraBytesToRead+1); /* return to the illegal value itself */
nuclear@0 375 result = sourceIllegal;
nuclear@0 376 break;
nuclear@0 377 } else {
nuclear@0 378 *target++ = UNI_REPLACEMENT_CHAR;
nuclear@0 379 }
nuclear@0 380 } else {
nuclear@0 381 *target++ = (UTF16)ch; /* normal case */
nuclear@0 382 }
nuclear@0 383 } else if (ch > UNI_MAX_UTF16) {
nuclear@0 384 if (flags == strictConversion) {
nuclear@0 385 result = sourceIllegal;
nuclear@0 386 source -= (extraBytesToRead+1); /* return to the start */
nuclear@0 387 break; /* Bail out; shouldn't continue */
nuclear@0 388 } else {
nuclear@0 389 *target++ = UNI_REPLACEMENT_CHAR;
nuclear@0 390 }
nuclear@0 391 } else {
nuclear@0 392 /* target is a character in range 0xFFFF - 0x10FFFF. */
nuclear@0 393 if (target + 1 >= targetEnd) {
nuclear@0 394 source -= (extraBytesToRead+1); /* Back up source pointer! */
nuclear@0 395 result = targetExhausted; break;
nuclear@0 396 }
nuclear@0 397 ch -= halfBase;
nuclear@0 398 *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
nuclear@0 399 *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
nuclear@0 400 }
nuclear@0 401 }
nuclear@0 402 *sourceStart = source;
nuclear@0 403 *targetStart = target;
nuclear@0 404 return result;
nuclear@0 405 }
nuclear@0 406
nuclear@0 407 /* --------------------------------------------------------------------- */
nuclear@0 408
nuclear@0 409 ConversionResult ConvertUTF32toUTF8 (
nuclear@0 410 const UTF32** sourceStart, const UTF32* sourceEnd,
nuclear@0 411 UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) {
nuclear@0 412 ConversionResult result = conversionOK;
nuclear@0 413 const UTF32* source = *sourceStart;
nuclear@0 414 UTF8* target = *targetStart;
nuclear@0 415 while (source < sourceEnd) {
nuclear@0 416 UTF32 ch;
nuclear@0 417 unsigned short bytesToWrite = 0;
nuclear@0 418 const UTF32 byteMask = 0xBF;
nuclear@0 419 const UTF32 byteMark = 0x80;
nuclear@0 420 ch = *source++;
nuclear@0 421 if (flags == strictConversion ) {
nuclear@0 422 /* UTF-16 surrogate values are illegal in UTF-32 */
nuclear@0 423 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
nuclear@0 424 --source; /* return to the illegal value itself */
nuclear@0 425 result = sourceIllegal;
nuclear@0 426 break;
nuclear@0 427 }
nuclear@0 428 }
nuclear@0 429 /*
nuclear@0 430 * Figure out how many bytes the result will require. Turn any
nuclear@0 431 * illegally large UTF32 things (> Plane 17) into replacement chars.
nuclear@0 432 */
nuclear@0 433 if (ch < (UTF32)0x80) { bytesToWrite = 1;
nuclear@0 434 } else if (ch < (UTF32)0x800) { bytesToWrite = 2;
nuclear@0 435 } else if (ch < (UTF32)0x10000) { bytesToWrite = 3;
nuclear@0 436 } else if (ch <= UNI_MAX_LEGAL_UTF32) { bytesToWrite = 4;
nuclear@0 437 } else { bytesToWrite = 3;
nuclear@0 438 ch = UNI_REPLACEMENT_CHAR;
nuclear@0 439 result = sourceIllegal;
nuclear@0 440 }
nuclear@0 441
nuclear@0 442 target += bytesToWrite;
nuclear@0 443 if (target > targetEnd) {
nuclear@0 444 --source; /* Back up source pointer! */
nuclear@0 445 target -= bytesToWrite; result = targetExhausted; break;
nuclear@0 446 }
nuclear@0 447 switch (bytesToWrite) { /* note: everything falls through. */
nuclear@0 448 case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
nuclear@0 449 case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
nuclear@0 450 case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
nuclear@0 451 case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]);
nuclear@0 452 }
nuclear@0 453 target += bytesToWrite;
nuclear@0 454 }
nuclear@0 455 *sourceStart = source;
nuclear@0 456 *targetStart = target;
nuclear@0 457 return result;
nuclear@0 458 }
nuclear@0 459
nuclear@0 460 /* --------------------------------------------------------------------- */
nuclear@0 461
nuclear@0 462 ConversionResult ConvertUTF8toUTF32 (
nuclear@0 463 const UTF8** sourceStart, const UTF8* sourceEnd,
nuclear@0 464 UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) {
nuclear@0 465 ConversionResult result = conversionOK;
nuclear@0 466 const UTF8* source = *sourceStart;
nuclear@0 467 UTF32* target = *targetStart;
nuclear@0 468 while (source < sourceEnd) {
nuclear@0 469 UTF32 ch = 0;
nuclear@0 470 unsigned short extraBytesToRead = trailingBytesForUTF8[*source];
nuclear@0 471 if (source + extraBytesToRead >= sourceEnd) {
nuclear@0 472 result = sourceExhausted; break;
nuclear@0 473 }
nuclear@0 474 /* Do this check whether lenient or strict */
nuclear@0 475 if (! isLegalUTF8(source, extraBytesToRead+1)) {
nuclear@0 476 result = sourceIllegal;
nuclear@0 477 break;
nuclear@0 478 }
nuclear@0 479 /*
nuclear@0 480 * The cases all fall through. See "Note A" below.
nuclear@0 481 */
nuclear@0 482 switch (extraBytesToRead) {
nuclear@0 483 case 5: ch += *source++; ch <<= 6;
nuclear@0 484 case 4: ch += *source++; ch <<= 6;
nuclear@0 485 case 3: ch += *source++; ch <<= 6;
nuclear@0 486 case 2: ch += *source++; ch <<= 6;
nuclear@0 487 case 1: ch += *source++; ch <<= 6;
nuclear@0 488 case 0: ch += *source++;
nuclear@0 489 }
nuclear@0 490 ch -= offsetsFromUTF8[extraBytesToRead];
nuclear@0 491
nuclear@0 492 if (target >= targetEnd) {
nuclear@0 493 source -= (extraBytesToRead+1); /* Back up the source pointer! */
nuclear@0 494 result = targetExhausted; break;
nuclear@0 495 }
nuclear@0 496 if (ch <= UNI_MAX_LEGAL_UTF32) {
nuclear@0 497 /*
nuclear@0 498 * UTF-16 surrogate values are illegal in UTF-32, and anything
nuclear@0 499 * over Plane 17 (> 0x10FFFF) is illegal.
nuclear@0 500 */
nuclear@0 501 if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) {
nuclear@0 502 if (flags == strictConversion) {
nuclear@0 503 source -= (extraBytesToRead+1); /* return to the illegal value itself */
nuclear@0 504 result = sourceIllegal;
nuclear@0 505 break;
nuclear@0 506 } else {
nuclear@0 507 *target++ = UNI_REPLACEMENT_CHAR;
nuclear@0 508 }
nuclear@0 509 } else {
nuclear@0 510 *target++ = ch;
nuclear@0 511 }
nuclear@0 512 } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */
nuclear@0 513 result = sourceIllegal;
nuclear@0 514 *target++ = UNI_REPLACEMENT_CHAR;
nuclear@0 515 }
nuclear@0 516 }
nuclear@0 517 *sourceStart = source;
nuclear@0 518 *targetStart = target;
nuclear@0 519 return result;
nuclear@0 520 }
nuclear@0 521
nuclear@0 522 /* ---------------------------------------------------------------------
nuclear@0 523
nuclear@0 524 Note A.
nuclear@0 525 The fall-through switches in UTF-8 reading code save a
nuclear@0 526 temp variable, some decrements & conditionals. The switches
nuclear@0 527 are equivalent to the following loop:
nuclear@0 528 {
nuclear@0 529 int tmpBytesToRead = extraBytesToRead+1;
nuclear@0 530 do {
nuclear@0 531 ch += *source++;
nuclear@0 532 --tmpBytesToRead;
nuclear@0 533 if (tmpBytesToRead) ch <<= 6;
nuclear@0 534 } while (tmpBytesToRead > 0);
nuclear@0 535 }
nuclear@0 536 In UTF-8 writing code, the switches on "bytesToWrite" are
nuclear@0 537 similarly unrolled loops.
nuclear@0 538
nuclear@0 539 --------------------------------------------------------------------- */