Visualization Library 2.0.0-b3

A lightweight C++ OpenGL middleware for 2D/3D graphics

VL     Star     Watch     Fork     Issue

[Download] [Tutorials] [All Classes] [Grouped Classes]
VLTTokenizer.cpp
Go to the documentation of this file.
1 /**************************************************************************************/
2 /* */
3 /* Visualization Library */
4 /* http://visualizationlibrary.org */
5 /* */
6 /* Copyright (c) 2005-2017, Michele Bosi */
7 /* All rights reserved. */
8 /* */
9 /* Redistribution and use in source and binary forms, with or without modification, */
10 /* are permitted provided that the following conditions are met: */
11 /* */
12 /* - Redistributions of source code must retain the above copyright notice, this */
13 /* list of conditions and the following disclaimer. */
14 /* */
15 /* - Redistributions in binary form must reproduce the above copyright notice, this */
16 /* list of conditions and the following disclaimer in the documentation and/or */
17 /* other materials provided with the distribution. */
18 /* */
19 /* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND */
20 /* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED */
21 /* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */
22 /* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR */
23 /* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */
24 /* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; */
25 /* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON */
26 /* ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */
27 /* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS */
28 /* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */
29 /* */
30 /**************************************************************************************/
31 
32 #include <vlCore/VLTTokenizer.hpp>
33 #include <vlCore/Log.hpp>
34 #include <vlCore/Say.hpp>
35 #include <ctime>
36 
37 using namespace vl;
38 
40 {
42  /*token.mString.resize(0);*/
43 
44  // Must be done before StringTurbo is declared
45  if (mRawtextBlock)
46  return getRawtextBlock(token);
47 
48  struct StringTurbo
49  {
50  inline StringTurbo(std::string* str): mString(str)
51  {
52  mPosition = 0;
53  }
54 
55  inline ~StringTurbo()
56  {
57  mBuffer[mPosition] = '\0';
58  *mString = mBuffer;
59  }
60 
61  inline void push_back(const char& ch)
62  {
63  mBuffer[mPosition++] = ch;
64  }
65 
66  inline void operator=(const char& ch)
67  {
68  mBuffer[0] = ch;
69  mPosition = 1;
70  }
71 
72  inline void operator=(const char* s)
73  {
74  int len = strlen(s);
75  memcpy( mBuffer, s, len);
76  mPosition = len;
77  }
78 
79  inline bool operator==(const char* str)
80  {
81  size_t len = strlen(str);
82  if (len != mPosition)
83  return false;
84  else
85  return memcmp(mBuffer, str, len) == 0;
86  }
87 
88  char mBuffer[1024*4];
89  size_t mPosition;
90  std::string* mString;
91  } string_turbo(&token.mString);
92 
93  // read chars skipping spaces
94  char ch1=0, ch2=0;
95  do
96  {
97  if (!readTextChar(ch1))
98  {
99  token.mType = VLTToken::TOKEN_EOF;
100  return true;
101  }
102 
103  if (ch1 == '\n')
104  ++mLineNumber;
105  else
106  // eat comments
107  if (ch1 == '/')
108  {
109  if(readTextChar(ch2))
110  {
111  if (ch2 == '/') // single line comment
112  {
113  // eat everything till the end of the line
114  for(ch1 = 0; readTextChar(ch1) && ch1 != '\n'; )
115  {
116  // eat everything
117  }
118  if (ch1 == '\n')
119  ++mLineNumber;
120  }
121  else
122  if (ch2 == '*') // multi line comment
123  {
124  // eat everything till the end of the line
125  while(readTextChar(ch1))
126  {
127  if (ch1 == '*' && readTextChar(ch2) && ch2 == '/')
128  {
129  ch1 = '\n'; // pretend it's a space to stay in the loop
130  break;
131  }
132  // eat everything
133  if (ch1 == '\n')
134  ++mLineNumber;
135  }
136  }
137  else
138  {
139  Log::error( Say("Line %n : unexpected character '%c' after '/'.\n") << mLineNumber << ch2 );
140  return false;
141  }
142  }
143  else
144  {
145  Log::error( Say("Line %n : unexpected end of file in comment.\n") << mLineNumber);
146  return false;
147  }
148  continue;
149  }
150 
151  } while(ch1 == ' ' || ch1 == '\t' || ch1 == '\n');
152 
153  switch(ch1)
154  {
155  case '(':
157  string_turbo = "(";
158  return true;
159 
160  case ')':
162  string_turbo = ")";
163  return true;
164 
165  case '[':
167  string_turbo = "[";
168  return true;
169 
170  case ']':
172  string_turbo = "]";
173  return true;
174 
175  case '{':
176  if(readTextChar(ch2) && ch2 == '<')
177  {
178  // actual data starts at the next new line
179  // eat all the spaces until the end of the current line
180  while(ch2 != '\n' && readTextChar(ch2))
181  {
182  switch(ch2)
183  {
184  case '\f':
185  case '\b':
186  case '\v':
187  case '\t':
188  case ' ':
189  continue;
190 
191  case '\n':
192  ++mLineNumber;
193  break;
194 
195  default:
196  string_turbo = ch2;
197  return false;
198  }
199  }
200 
201  if (ch2 == '\n')
202  {
204  string_turbo = "{<";
205  mRawtextBlock = true;
206  return true;
207  }
208  else
209  {
210  string_turbo = ch2;
211  return false;
212  }
213  }
214  else
215  {
217  string_turbo = "{";
218  if(!isEndOfFile())
219  ungetToken(ch2);
220  }
221  return true;
222 
223  case '}':
225  string_turbo = "}";
226  return true;
227 
228  case '>':
229  if(readTextChar(ch2))
230  {
231  if(ch2 == '}')
232  {
234  string_turbo = ">}";
235  return true;
236  }
237  else
238  {
239  Log::error( Say("Line %n : expected '}' instead of '%c' after '>'.\n") << mLineNumber << ch2 );
240  return false;
241  }
242  }
243  else
244  {
245  Log::error( Say("Line %n : unexpected end of file.\n") << mLineNumber );
246  return false;
247  }
248 
249  case '=':
250  token.mType = VLTToken::Equals;
251  string_turbo = "=";
252  return true;
253 
254  case '<':
255  string_turbo = "<";
256  while(readTextChar(ch1) && ch1 != '>')
257  {
258  if ( (ch1 >= 'a' && ch1 <= 'z') || (ch1 >= 'A' && ch1 <= 'Z') || (ch1 >= '0' && ch1 <= '9') || ch1 == '_' || ch1 == ':' )
259  string_turbo.push_back(ch1);
260  else
261  {
262  Log::error( Say("Line %n : unexpected character '%c'.\n") << mLineNumber << ch1 );
263  return false;
264  }
265  }
266  string_turbo.push_back('>');
267  if (isEndOfFile())
268  {
269  Log::error( Say("Line %n : unexpected end of file while reading object header.\n") << mLineNumber );
270  return false;
271  }
272  token.mType = VLTToken::TagHeader;
273  return true;
274 
275  case '#':
276  string_turbo = "#";
277  while(readTextChar(ch1))
278  {
279  if ( (ch1 >= 'a' && ch1 <= 'z') || (ch1 >= 'A' && ch1 <= 'Z') || (ch1 >= '0' && ch1 <= '9') || ch1 == '_' )
280  string_turbo.push_back(ch1);
281  else
282  {
283  ungetToken(ch1);
284  break;
285  }
286  }
287  if (string_turbo == "#_")
288  {
289  Log::error( Say("Line %n : illegal id '#_' found.\n") << mLineNumber );
290  return false;
291  }
292  token.mType = VLTToken::ID;
293  return true;
294 
295  case '"':
296  while(readTextChar(ch1))
297  {
298  // end string
299  if (ch1 == '"')
300  break;
301  else
302  // return found before end of string
303  if (ch1 == '\n')
304  {
305  Log::error( Say("Line %n : end of line found before end of string, did you forget a \"?.\n") << mLineNumber );
306  return false;
307  }
308  else
309  // escape sequences
310  if (ch1 == '\\' && readTextChar(ch2))
311  {
312  if (ch2 == '"')
313  ch1 = '"';
314  else
315  if (ch2 == '\\')
316  ch1 = '\\';
317  else
318  if (ch2 == 'b')
319  ch1 = '\b';
320  else
321  if (ch2 == 'f')
322  ch1 = '\f';
323  else
324  if (ch2 == 'r')
325  ch1 = '\r';
326  else
327  if (ch2 == 'n')
328  ch1 = '\n';
329  else
330  if (ch2 == 't')
331  ch1 = '\t';
332  else
333  ungetToken(ch2);
334  string_turbo.push_back(ch1);
335  }
336  else
337  // accept everyhing else
338  string_turbo.push_back(ch1);
339  }
340  if (isEndOfFile())
341  {
342  Log::error( Say("Line %n : end of file found before end of string, did you forget a \"?.\n") << mLineNumber );
343  return false;
344  }
345  else
346  {
347  token.mType = VLTToken::String;
348  return true;
349  }
350 
351  default:
352  // identifier
353  if ( (ch1 >= 'a' && ch1 <= 'z') || (ch1 >= 'A' && ch1 <= 'Z') || ch1 == '_' )
354  {
355  string_turbo.push_back(ch1);
356  while(readTextChar(ch1))
357  {
358  if ( (ch1 >= 'a' && ch1 <= 'z') || (ch1 >= 'A' && ch1 <= 'Z') || (ch1 >= '0' && ch1 <= '9') || ch1 == '_' )
359  string_turbo.push_back(ch1);
360  else
361  {
362  ungetToken(ch1);
363  break;
364  }
365  }
366  if (string_turbo == "_")
367  {
368  Log::error( Say("Line %n : unexpected character '_'.\n") << mLineNumber );
369  return false;
370  }
371  else
372  {
373  // check if it's a boolean
374  if (string_turbo == "true" || string_turbo == "false")
375  token.mType = VLTToken::Boolean;
376  else
377  token.mType = VLTToken::Identifier;
378  return true;
379  }
380  }
381  else
382  // Integer / real
383  //
384  // ACCEPTED:
385  // 123
386  // +123.123E+10 -123.123e-10
387  // +123
388  // +.123
389  // 0.123
390  // 123.123
391  //
392  // REJECTED:
393  // 01234
394  // 01.234
395  // 123.
396  // 123.123E
397  // 123.123e+
398  if ( (ch1 >= '0' && ch1 <= '9') || ch1 == '.' || ch1 == '+' || ch1 == '-' )
399  {
401  string_turbo.push_back(ch1);
402 
403  enum { sZERO, sPLUS_MINUS, sINT, sFRAC, sPOINT, sE, sPLUS_MINUS_EXP, sEXP } state = sINT;
404 
405  if ( ch1 >= '1' && ch1 <= '9' )
406  state = sINT;
407  else
408  if (ch1 == '0')
409  state = sZERO;
410  else
411  if (ch1 == '.')
412  state = sPOINT;
413  else
414  if (ch1 == '+' || ch1 == '-')
415  state = sPLUS_MINUS;
416 
417  while(readTextChar(ch1))
418  {
419  switch(state)
420  {
421  // if starting with 0 must be 0.0-9
422  case sZERO:
423  if (ch1 == '.')
424  {
425  string_turbo.push_back(ch1);
426  state = sPOINT;
427  }
428  else
429  {
430  token.mType = VLTToken::Integer;
431  ungetToken(ch1);
432  return true;
433  }
434  break;
435 
436  case sPLUS_MINUS:
437  if (ch1 == '0')
438  {
439  string_turbo.push_back(ch1);
440  state = sZERO;
441  }
442  else
443  if (ch1 >= '1' && ch1 <= '9')
444  {
445  string_turbo.push_back(ch1);
446  state = sINT;
447  }
448  else
449  if (ch1 == '.')
450  {
451  string_turbo.push_back(ch1);
452  state = sPOINT;
453  }
454  else
455  {
456  Log::error( Say("Line %n :unexpected character '%c'.\n") << mLineNumber << ch1 );
457  return false;
458  }
459  break;
460 
461  case sINT:
462  if (ch1 >= '0' && ch1 <= '9')
463  string_turbo.push_back(ch1);
464  else
465  if (ch1 == '.')
466  {
467  string_turbo.push_back(ch1);
468  state = sPOINT;
469  }
470  else
471  {
472  token.mType = VLTToken::Integer;
473  ungetToken(ch1);
474  return true;
475  }
476  break;
477 
478  case sPOINT:
479  if (ch1 >= '0' && ch1 <= '9')
480  {
481  string_turbo.push_back(ch1);
482  state = sFRAC;
483  }
484  else
485  {
486  Log::error( Say("Line %n :unexpected character '%c'.\n") << mLineNumber << ch1 );
487  return false;
488  }
489  break;
490 
491  case sFRAC:
492  if (ch1 >= '0' && ch1 <= '9')
493  string_turbo.push_back(ch1);
494  else
495  if (ch1 == 'E' || ch1 == 'e')
496  {
497  string_turbo.push_back(ch1);
498  state = sE;
499  }
500  else
501  {
502  token.mType = VLTToken::real;
503  ungetToken(ch1);
504  return true;
505  }
506  break;
507 
508  case sE:
509  if (ch1 == '+' || ch1 == '-')
510  {
511  string_turbo.push_back(ch1);
512  state = sPLUS_MINUS_EXP;
513  }
514  else
515  {
516  Log::error( Say("Line %n :unexpected character '%c'.\n") << mLineNumber << ch1 );
517  return false;
518  }
519  break;
520 
521  case sPLUS_MINUS_EXP:
522  if (ch1 >= '0' && ch1 <= '9')
523  {
524  string_turbo.push_back(ch1);
525  state = sEXP;
526  }
527  else
528  {
529  Log::error( Say("Line %n :unexpected character '%c'.\n") << mLineNumber << ch1 );
530  return false;
531  }
532  break;
533 
534  case sEXP:
535  if (ch1 >= '0' && ch1 <= '9')
536  string_turbo.push_back(ch1);
537  else
538  {
539  token.mType = VLTToken::real;
540  ungetToken(ch1);
541  return true;
542  }
543  break;
544  }
545  }
546  // reached TOKEN_EOF in the middle of the parsing so we check where we were, note that it cannot be a Integer or a real
547  if (state == sINT)
548  {
549  token.mType = VLTToken::Integer;
550  return true;
551  }
552  else
553  if (state == sFRAC || state == sEXP)
554  {
555  token.mType = VLTToken::real;
556  return true;
557  }
558  else
559  return false;
560  }
561  else
562  {
563  Log::error( Say("Line %n : unexpected character '%c'.\n") << mLineNumber << ch1 );
564  return false;
565  }
566  }
567 }
568 //-----------------------------------------------------------------------------
570 {
571  mRawtextBlock = false;
572 
574  token.mString.resize(0);
575 
576  char ch =0;
577  while(readTextChar(ch))
578  {
579  if (ch == '\n')
580  ++mLineNumber;
581 
582  if (ch == '>')
583  {
584  // check for rawtext block end >}
585  char ch2 = 0;
586  if (readTextChar(ch2))
587  {
588  if(ch2 == '}')
589  {
590  // check if it was escaped
591  if (!token.mString.empty() && token.mString[ token.mString.size() - 1 ] == '\\')
592  {
593  token.mString.resize( token.mString.size() - 1 );
594  token.mString += ">}";
595  continue;
596  }
597  else
598  {
600  ungetToken('}');
601  ungetToken('>');
602  return true;
603  }
604  }
605  else
606  ungetToken(ch2);
607  }
608  }
609 
610  token.mString.push_back(ch);
611  }
612 
613  return false;
614 }
615 //-----------------------------------------------------------------------------
VLCORE_EXPORT bool getRawtextBlock(VLTToken &token)
std::string mString
A simple String formatting class.
Definition: Say.hpp:124
static void error(const String &message)
Use this function to provide information about run-time errors: file not found, out of memory...
Definition: Log.cpp:164
Visualization Library main namespace.
VLCORE_EXPORT bool getToken(VLTToken &token)
A token of information as defined by VLT specification.
bool operator==(const ref< T1 > &o1, const ref< T2 > &o2)
Definition: Object.hpp:144
Object & operator=(const Object &other)
Copy operator: copies the object&#39;s name, ref count mutex and user data.
Definition: Object.hpp:201