vrshoot

annotate libs/assimp/FBXTokenizer.cpp @ 0:b2f14e535253

find changesets by author, revision, files, or words in the commit message
initial commit
author John Tsiombikas <nuclear@member.fsf.org>
date Sat, 01 Feb 2014 19:58:19 +0200
parents
children
rev   line source
nuclear@0 1 /*
nuclear@0 2 Open Asset Import Library (assimp)
nuclear@0 3 ----------------------------------------------------------------------
nuclear@0 4
nuclear@0 5 Copyright (c) 2006-2012, assimp team
nuclear@0 6 All rights reserved.
nuclear@0 7
nuclear@0 8 Redistribution and use of this software in source and binary forms,
nuclear@0 9 with or without modification, are permitted provided that the
nuclear@0 10 following conditions are met:
nuclear@0 11
nuclear@0 12 * Redistributions of source code must retain the above
nuclear@0 13 copyright notice, this list of conditions and the
nuclear@0 14 following disclaimer.
nuclear@0 15
nuclear@0 16 * Redistributions in binary form must reproduce the above
nuclear@0 17 copyright notice, this list of conditions and the
nuclear@0 18 following disclaimer in the documentation and/or other
nuclear@0 19 materials provided with the distribution.
nuclear@0 20
nuclear@0 21 * Neither the name of the assimp team, nor the names of its
nuclear@0 22 contributors may be used to endorse or promote products
nuclear@0 23 derived from this software without specific prior
nuclear@0 24 written permission of the assimp team.
nuclear@0 25
nuclear@0 26 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
nuclear@0 27 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
nuclear@0 28 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
nuclear@0 29 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
nuclear@0 30 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
nuclear@0 31 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
nuclear@0 32 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
nuclear@0 33 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
nuclear@0 34 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
nuclear@0 35 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
nuclear@0 36 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
nuclear@0 37
nuclear@0 38 ----------------------------------------------------------------------
nuclear@0 39 */
nuclear@0 40
nuclear@0 41 /** @file FBXTokenizer.cpp
nuclear@0 42 * @brief Implementation of the FBX broadphase lexer
nuclear@0 43 */
nuclear@0 44 #include "AssimpPCH.h"
nuclear@0 45
nuclear@0 46 #ifndef ASSIMP_BUILD_NO_FBX_IMPORTER
nuclear@0 47
nuclear@0 48 // tab width for logging columns
nuclear@0 49 #define ASSIMP_FBX_TAB_WIDTH 4
nuclear@0 50
nuclear@0 51 #include "ParsingUtils.h"
nuclear@0 52
nuclear@0 53 #include "FBXTokenizer.h"
nuclear@0 54 #include "FBXUtil.h"
nuclear@0 55
nuclear@0 56 namespace Assimp {
nuclear@0 57 namespace FBX {
nuclear@0 58
nuclear@0 59 // ------------------------------------------------------------------------------------------------
nuclear@0 60 Token::Token(const char* sbegin, const char* send, TokenType type, unsigned int line, unsigned int column)
nuclear@0 61 : sbegin(sbegin)
nuclear@0 62 , send(send)
nuclear@0 63 , type(type)
nuclear@0 64 , line(line)
nuclear@0 65 , column(column)
nuclear@0 66 #ifdef DEBUG
nuclear@0 67 , contents(sbegin, static_cast<size_t>(send-sbegin))
nuclear@0 68 #endif
nuclear@0 69 {
nuclear@0 70 ai_assert(sbegin);
nuclear@0 71 ai_assert(send);
nuclear@0 72
nuclear@0 73 // tokens must be of non-zero length
nuclear@0 74 ai_assert(static_cast<size_t>(send-sbegin) > 0);
nuclear@0 75 }
nuclear@0 76
nuclear@0 77
nuclear@0 78 // ------------------------------------------------------------------------------------------------
nuclear@0 79 Token::~Token()
nuclear@0 80 {
nuclear@0 81 }
nuclear@0 82
nuclear@0 83
nuclear@0 84 namespace {
nuclear@0 85
nuclear@0 86 // ------------------------------------------------------------------------------------------------
nuclear@0 87 // signal tokenization error, this is always unrecoverable. Throws DeadlyImportError.
nuclear@0 88 void TokenizeError(const std::string& message, unsigned int line, unsigned int column)
nuclear@0 89 {
nuclear@0 90 throw DeadlyImportError(Util::AddLineAndColumn("FBX-Tokenize",message,line,column));
nuclear@0 91 }
nuclear@0 92
nuclear@0 93
nuclear@0 94 // process a potential data token up to 'cur', adding it to 'output_tokens'.
nuclear@0 95 // ------------------------------------------------------------------------------------------------
nuclear@0 96 void ProcessDataToken( TokenList& output_tokens, const char*& start, const char*& end,
nuclear@0 97 unsigned int line,
nuclear@0 98 unsigned int column,
nuclear@0 99 TokenType type = TokenType_DATA,
nuclear@0 100 bool must_have_token = false)
nuclear@0 101 {
nuclear@0 102 if (start && end) {
nuclear@0 103 // sanity check:
nuclear@0 104 // tokens should have no whitespace outside quoted text and [start,end] should
nuclear@0 105 // properly delimit the valid range.
nuclear@0 106 bool in_double_quotes = false;
nuclear@0 107 for (const char* c = start; c != end + 1; ++c) {
nuclear@0 108 if (*c == '\"') {
nuclear@0 109 in_double_quotes = !in_double_quotes;
nuclear@0 110 }
nuclear@0 111
nuclear@0 112 if (!in_double_quotes && IsSpaceOrNewLine(*c)) {
nuclear@0 113 TokenizeError("unexpected whitespace in token", line, column);
nuclear@0 114 }
nuclear@0 115 }
nuclear@0 116
nuclear@0 117 if (in_double_quotes) {
nuclear@0 118 TokenizeError("non-terminated double quotes", line, column);
nuclear@0 119 }
nuclear@0 120
nuclear@0 121 output_tokens.push_back(new_Token(start,end + 1,type,line,column));
nuclear@0 122 }
nuclear@0 123 else if (must_have_token) {
nuclear@0 124 TokenizeError("unexpected character, expected data token", line, column);
nuclear@0 125 }
nuclear@0 126
nuclear@0 127 start = end = NULL;
nuclear@0 128 }
nuclear@0 129
nuclear@0 130 }
nuclear@0 131
nuclear@0 132 // ------------------------------------------------------------------------------------------------
nuclear@0 133 void Tokenize(TokenList& output_tokens, const char* input)
nuclear@0 134 {
nuclear@0 135 ai_assert(input);
nuclear@0 136
nuclear@0 137 // line and column numbers numbers are one-based
nuclear@0 138 unsigned int line = 1;
nuclear@0 139 unsigned int column = 1;
nuclear@0 140
nuclear@0 141 bool comment = false;
nuclear@0 142 bool in_double_quotes = false;
nuclear@0 143 bool pending_data_token = false;
nuclear@0 144
nuclear@0 145 const char* token_begin = NULL, *token_end = NULL;
nuclear@0 146 for (const char* cur = input;*cur;column += (*cur == '\t' ? ASSIMP_FBX_TAB_WIDTH : 1), ++cur) {
nuclear@0 147 const char c = *cur;
nuclear@0 148
nuclear@0 149 if (IsLineEnd(c)) {
nuclear@0 150 comment = false;
nuclear@0 151
nuclear@0 152 column = 0;
nuclear@0 153 ++line;
nuclear@0 154 }
nuclear@0 155
nuclear@0 156 if(comment) {
nuclear@0 157 continue;
nuclear@0 158 }
nuclear@0 159
nuclear@0 160 if(in_double_quotes) {
nuclear@0 161 if (c == '\"') {
nuclear@0 162 in_double_quotes = false;
nuclear@0 163 token_end = cur;
nuclear@0 164
nuclear@0 165 ProcessDataToken(output_tokens,token_begin,token_end,line,column);
nuclear@0 166 pending_data_token = false;
nuclear@0 167 }
nuclear@0 168 continue;
nuclear@0 169 }
nuclear@0 170
nuclear@0 171 switch(c)
nuclear@0 172 {
nuclear@0 173 case '\"':
nuclear@0 174 if (token_begin) {
nuclear@0 175 TokenizeError("unexpected double-quote", line, column);
nuclear@0 176 }
nuclear@0 177 token_begin = cur;
nuclear@0 178 in_double_quotes = true;
nuclear@0 179 continue;
nuclear@0 180
nuclear@0 181 case ';':
nuclear@0 182 ProcessDataToken(output_tokens,token_begin,token_end,line,column);
nuclear@0 183 comment = true;
nuclear@0 184 continue;
nuclear@0 185
nuclear@0 186 case '{':
nuclear@0 187 ProcessDataToken(output_tokens,token_begin,token_end, line, column);
nuclear@0 188 output_tokens.push_back(new_Token(cur,cur+1,TokenType_OPEN_BRACKET,line,column));
nuclear@0 189 continue;
nuclear@0 190
nuclear@0 191 case '}':
nuclear@0 192 ProcessDataToken(output_tokens,token_begin,token_end,line,column);
nuclear@0 193 output_tokens.push_back(new_Token(cur,cur+1,TokenType_CLOSE_BRACKET,line,column));
nuclear@0 194 continue;
nuclear@0 195
nuclear@0 196 case ',':
nuclear@0 197 if (pending_data_token) {
nuclear@0 198 ProcessDataToken(output_tokens,token_begin,token_end,line,column,TokenType_DATA,true);
nuclear@0 199 }
nuclear@0 200 output_tokens.push_back(new_Token(cur,cur+1,TokenType_COMMA,line,column));
nuclear@0 201 continue;
nuclear@0 202
nuclear@0 203 case ':':
nuclear@0 204 if (pending_data_token) {
nuclear@0 205 ProcessDataToken(output_tokens,token_begin,token_end,line,column,TokenType_KEY,true);
nuclear@0 206 }
nuclear@0 207 else {
nuclear@0 208 TokenizeError("unexpected colon", line, column);
nuclear@0 209 }
nuclear@0 210 continue;
nuclear@0 211 }
nuclear@0 212
nuclear@0 213 if (IsSpaceOrNewLine(c)) {
nuclear@0 214
nuclear@0 215 if (token_begin) {
nuclear@0 216 // peek ahead and check if the next token is a colon in which
nuclear@0 217 // case this counts as KEY token.
nuclear@0 218 TokenType type = TokenType_DATA;
nuclear@0 219 for (const char* peek = cur; *peek && IsSpaceOrNewLine(*peek); ++peek) {
nuclear@0 220 if (*peek == ':') {
nuclear@0 221 type = TokenType_KEY;
nuclear@0 222 cur = peek;
nuclear@0 223 break;
nuclear@0 224 }
nuclear@0 225 }
nuclear@0 226
nuclear@0 227 ProcessDataToken(output_tokens,token_begin,token_end,line,column,type);
nuclear@0 228 }
nuclear@0 229
nuclear@0 230 pending_data_token = false;
nuclear@0 231 }
nuclear@0 232 else {
nuclear@0 233 token_end = cur;
nuclear@0 234 if (!token_begin) {
nuclear@0 235 token_begin = cur;
nuclear@0 236 }
nuclear@0 237
nuclear@0 238 pending_data_token = true;
nuclear@0 239 }
nuclear@0 240 }
nuclear@0 241 }
nuclear@0 242
nuclear@0 243 } // !FBX
nuclear@0 244 } // !Assimp
nuclear@0 245
nuclear@0 246 #endif