1.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     1.2 +++ b/libs/assimp/FBXTokenizer.cpp	Sat Feb 01 19:58:19 2014 +0200
     1.3 @@ -0,0 +1,246 @@
     1.4 +/*
     1.5 +Open Asset Import Library (assimp)
     1.6 +----------------------------------------------------------------------
     1.7 +
     1.8 +Copyright (c) 2006-2012, assimp team
     1.9 +All rights reserved.
    1.10 +
    1.11 +Redistribution and use of this software in source and binary forms, 
    1.12 +with or without modification, are permitted provided that the 
    1.13 +following conditions are met:
    1.14 +
    1.15 +* Redistributions of source code must retain the above
    1.16 +  copyright notice, this list of conditions and the
    1.17 +  following disclaimer.
    1.18 +
    1.19 +* Redistributions in binary form must reproduce the above
    1.20 +  copyright notice, this list of conditions and the
    1.21 +  following disclaimer in the documentation and/or other
    1.22 +  materials provided with the distribution.
    1.23 +
    1.24 +* Neither the name of the assimp team, nor the names of its
    1.25 +  contributors may be used to endorse or promote products
    1.26 +  derived from this software without specific prior
    1.27 +  written permission of the assimp team.
    1.28 +
    1.29 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
    1.30 +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
    1.31 +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
    1.32 +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 
    1.33 +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
    1.34 +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
    1.35 +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
    1.36 +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
    1.37 +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
    1.38 +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
    1.39 +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    1.40 +
    1.41 +----------------------------------------------------------------------
    1.42 +*/
    1.43 +
    1.44 +/** @file  FBXTokenizer.cpp
    1.45 + *  @brief Implementation of the FBX broadphase lexer
    1.46 + */
    1.47 +#include "AssimpPCH.h"
    1.48 +
    1.49 +#ifndef ASSIMP_BUILD_NO_FBX_IMPORTER
    1.50 +
    1.51 +// tab width for logging columns
    1.52 +#define ASSIMP_FBX_TAB_WIDTH 4
    1.53 +
    1.54 +#include "ParsingUtils.h"
    1.55 +
    1.56 +#include "FBXTokenizer.h"
    1.57 +#include "FBXUtil.h"
    1.58 +
    1.59 +namespace Assimp {
    1.60 +namespace FBX {
    1.61 +
    1.62 +// ------------------------------------------------------------------------------------------------
    1.63 +Token::Token(const char* sbegin, const char* send, TokenType type, unsigned int line, unsigned int column)
    1.64 +	: sbegin(sbegin)
    1.65 +	, send(send)
    1.66 +	, type(type)
    1.67 +	, line(line)
    1.68 +	, column(column)
    1.69 +#ifdef DEBUG
    1.70 +	, contents(sbegin, static_cast<size_t>(send-sbegin))
    1.71 +#endif
    1.72 +{
    1.73 +	ai_assert(sbegin);
    1.74 +	ai_assert(send);
    1.75 +
    1.76 +	// tokens must be of non-zero length
    1.77 +	ai_assert(static_cast<size_t>(send-sbegin) > 0);
    1.78 +}
    1.79 +
    1.80 +
    1.81 +// ------------------------------------------------------------------------------------------------
    1.82 +Token::~Token()
    1.83 +{
    1.84 +}
    1.85 +
    1.86 +
    1.87 +namespace {
    1.88 +
    1.89 +// ------------------------------------------------------------------------------------------------
    1.90 +// signal tokenization error, this is always unrecoverable. Throws DeadlyImportError.
    1.91 +void TokenizeError(const std::string& message, unsigned int line, unsigned int column)
    1.92 +{
    1.93 +	throw DeadlyImportError(Util::AddLineAndColumn("FBX-Tokenize",message,line,column));
    1.94 +}
    1.95 +
    1.96 +
    1.97 +// process a potential data token up to 'cur', adding it to 'output_tokens'. 
    1.98 +// ------------------------------------------------------------------------------------------------
    1.99 +void ProcessDataToken( TokenList& output_tokens, const char*& start, const char*& end,
   1.100 +					  unsigned int line, 
   1.101 +					  unsigned int column, 
   1.102 +					  TokenType type = TokenType_DATA,
   1.103 +					  bool must_have_token = false)
   1.104 +{
   1.105 +	if (start && end) {
   1.106 +		// sanity check:
   1.107 +		// tokens should have no whitespace outside quoted text and [start,end] should
   1.108 +		// properly delimit the valid range.
   1.109 +		bool in_double_quotes = false;
   1.110 +		for (const char* c = start; c != end + 1; ++c) {
   1.111 +			if (*c == '\"') {
   1.112 +				in_double_quotes = !in_double_quotes;
   1.113 +			}
   1.114 +
   1.115 +			if (!in_double_quotes && IsSpaceOrNewLine(*c)) {
   1.116 +				TokenizeError("unexpected whitespace in token", line, column);
   1.117 +			}
   1.118 +		}
   1.119 +
   1.120 +		if (in_double_quotes) {
   1.121 +			TokenizeError("non-terminated double quotes", line, column);
   1.122 +		}
   1.123 +
   1.124 +		output_tokens.push_back(new_Token(start,end + 1,type,line,column));
   1.125 +	}
   1.126 +	else if (must_have_token) {
   1.127 +		TokenizeError("unexpected character, expected data token", line, column);
   1.128 +	}
   1.129 +
   1.130 +	start = end = NULL;
   1.131 +}
   1.132 +
   1.133 +}
   1.134 +
   1.135 +// ------------------------------------------------------------------------------------------------
   1.136 +void Tokenize(TokenList& output_tokens, const char* input)
   1.137 +{
   1.138 +	ai_assert(input);
   1.139 +
   1.140 +	// line and column numbers numbers are one-based
   1.141 +	unsigned int line = 1;
   1.142 +	unsigned int column = 1;
   1.143 +
   1.144 +	bool comment = false;
   1.145 +	bool in_double_quotes = false;
   1.146 +	bool pending_data_token = false;
   1.147 +	
   1.148 +	const char* token_begin = NULL, *token_end = NULL;
   1.149 +	for (const char* cur = input;*cur;column += (*cur == '\t' ? ASSIMP_FBX_TAB_WIDTH : 1), ++cur) {
   1.150 +		const char c = *cur;
   1.151 +
   1.152 +		if (IsLineEnd(c)) {
   1.153 +			comment = false;
   1.154 +
   1.155 +			column = 0;
   1.156 +			++line;
   1.157 +		}
   1.158 +
   1.159 +		if(comment) {
   1.160 +			continue;
   1.161 +		}
   1.162 +
   1.163 +		if(in_double_quotes) {
   1.164 +			if (c == '\"') {
   1.165 +				in_double_quotes = false;
   1.166 +				token_end = cur;
   1.167 +
   1.168 +				ProcessDataToken(output_tokens,token_begin,token_end,line,column);
   1.169 +				pending_data_token = false;
   1.170 +			}
   1.171 +			continue;
   1.172 +		}
   1.173 +
   1.174 +		switch(c)
   1.175 +		{
   1.176 +		case '\"':
   1.177 +			if (token_begin) {
   1.178 +				TokenizeError("unexpected double-quote", line, column);
   1.179 +			}
   1.180 +			token_begin = cur;
   1.181 +			in_double_quotes = true;
   1.182 +			continue;
   1.183 +
   1.184 +		case ';':
   1.185 +			ProcessDataToken(output_tokens,token_begin,token_end,line,column);
   1.186 +			comment = true;
   1.187 +			continue;
   1.188 +
   1.189 +		case '{':
   1.190 +			ProcessDataToken(output_tokens,token_begin,token_end, line, column);
   1.191 +			output_tokens.push_back(new_Token(cur,cur+1,TokenType_OPEN_BRACKET,line,column));
   1.192 +			continue;
   1.193 +
   1.194 +		case '}':
   1.195 +			ProcessDataToken(output_tokens,token_begin,token_end,line,column);
   1.196 +			output_tokens.push_back(new_Token(cur,cur+1,TokenType_CLOSE_BRACKET,line,column));
   1.197 +			continue;
   1.198 +		
   1.199 +		case ',':
   1.200 +			if (pending_data_token) {
   1.201 +				ProcessDataToken(output_tokens,token_begin,token_end,line,column,TokenType_DATA,true);
   1.202 +			}
   1.203 +			output_tokens.push_back(new_Token(cur,cur+1,TokenType_COMMA,line,column));
   1.204 +			continue;
   1.205 +
   1.206 +		case ':':
   1.207 +			if (pending_data_token) {
   1.208 +				ProcessDataToken(output_tokens,token_begin,token_end,line,column,TokenType_KEY,true);
   1.209 +			}
   1.210 +			else {
   1.211 +				TokenizeError("unexpected colon", line, column);
   1.212 +			}
   1.213 +			continue;
   1.214 +		}
   1.215 +		
   1.216 +		if (IsSpaceOrNewLine(c)) {
   1.217 +
   1.218 +			if (token_begin) {
   1.219 +				// peek ahead and check if the next token is a colon in which
   1.220 +				// case this counts as KEY token.
   1.221 +				TokenType type = TokenType_DATA;
   1.222 +				for (const char* peek = cur;  *peek && IsSpaceOrNewLine(*peek); ++peek) {
   1.223 +					if (*peek == ':') {
   1.224 +						type = TokenType_KEY;
   1.225 +						cur = peek;
   1.226 +						break;
   1.227 +					}
   1.228 +				}
   1.229 +
   1.230 +				ProcessDataToken(output_tokens,token_begin,token_end,line,column,type);
   1.231 +			}
   1.232 +
   1.233 +			pending_data_token = false;
   1.234 +		}
   1.235 +		else {
   1.236 +			token_end = cur;
   1.237 +			if (!token_begin) {
   1.238 +				token_begin = cur;
   1.239 +			}
   1.240 +
   1.241 +			pending_data_token = true;
   1.242 +		}
   1.243 +	}
   1.244 +}
   1.245 +
   1.246 +} // !FBX
   1.247 +} // !Assimp
   1.248 +
   1.249 +#endif