Logo Search packages:      
Sourcecode: uncrustify version File versions

Lexer.d

/+
      Copyright (c) 1999-2006 by Digital Mars
      All Rights Reserved
      written by Walter Bright www.digitalmars.com    
      License for redistribution is by either the Artistic License in artistic.txt, or the GNU General Public License in gnu.txt.
      See the included readme.txt for details.
      D Language conversion by: J Duncan
+/

/**
      d language lexer 
*/

module dparser.Lexer;

import dparser.Root;

import dparser.Tokens;
import dparser.Token;
import dparser.Keyword;

import dparser.Types;

import dparser.Module;
import dparser.Identifier;
import dparser.unialpha;

import dparser.OutBuffer;

//private import std.ctype;
//private import std.string;
//import dwf.core.debugapi;

int errno = 0;

//#if _WIN32 && __DMC__
// from \dm\src\include\setlocal.h
//extern "C" char * __cdecl __locale_decpoint;
char* __locale_decpoint;
//#endif
//const uint LS = 0x2028;     // UTF line separator
//const uint PS = 0x2029;     // UTF paragraph separator

//extern int isUniAlpha(unsigned u);
//extern int HtmlNamedEntity(unsigned char *p, int length);

/**
      Lexer object
*/

00051 class Lexer
{
   
      static Identifier[char[]]     stringtable;
    static OutBuffer          stringbuffer;
    static Token*             freelist;

    Token token;        // current token
    Module mod;               // current module
    Loc loc;                  // for error messages
    ubyte *base;        // pointer to start of buffer
    ubyte *end;               // past end of buffer
    ubyte *p;                 // current character
    int doDocComment;   // collect doc comment information
    int anyToken;       // !=0 means seen at least one token
    int commentToken;   // !=0 means comments are TOKcomment's


    this( Module mod, ubyte* base, uint begoffset, uint endoffset, int doDocComment, int commentToken )
      {
          if( stringbuffer is null )
                stringbuffer = new OutBuffer;
          loc = Loc(mod, 1);

          this.base                 = base;
          this.end                  = base + endoffset;
          this.p                    = base + begoffset;
          this.mod                  = mod;
          this.doDocComment   = doDocComment;
          this.commentToken   = commentToken;

            /* 
                  If first line starts with '#!', ignore the line
            */

            if( p[0] == '#' && p[1] =='!' )
            {
                  p += 2;
                  while( true )
                  {   
                        ubyte c = *p;
                        switch( c )
                        {
                              case '\n':
                                    p++;
                                    break;

                              case '\r':
                                    p++;
                                    if( *p == '\n' )
                                          p++;
                                    break;

                              case 0:
                              case 0x1A:
                                    break;

                              default:
                                    if( c & 0x80 )
                                    {   
                                          uint u = decodeUTF();
                                          if( u == PS || u == LS )
                                                break;
                                    }
                                    p++;
                                    continue;
                        }
                        break;
                  }

                  loc.linnum = 2;
            }

      }



      // generate a unique identifier for this string
    static Identifier idPool( in char[] str )
    {
//        StringValue sv;
//        uint len = s.length;
//        StringValue sv = stringtable.update(s, len);
//        Identifier* id = cast(Identifier*) sv.ptrvalue;
//        if( id is null )
          if( (str in stringtable) == null )
          {
                  stringtable[str] = new Identifier( str, TOK.TOKidentifier );
          }
          return stringtable[str];
      }

    static void initKeywords()
      {
            // build character map
          cmtable_init();
            
            // create keyword tokens & identifiers
            dparser.Keyword.initKeywords();

            // create standard lexer tokens
            dparser.Token.createLexerTokens();
      }

      // Combine two document comments into one.
    static char[] combineComments( char[] c1, char[] c2 )
      {
          char[] c = c2;
          if( c1.length )
          {
                  c = c1;
                  if( c2.length )
                  {
                        c = c1 ~ "\n" ~ c2;
                  }
          }
          return c;
      }

      // Decode UTF character. Issue error messages for invalid sequences. Return decoded character, advance p to last character in UTF sequence.
      //! fix
00172       uint decodeUTF()
      {
          ubyte* s = p;
          ubyte  c = *s;

          assert( c & 0x80 );
            if( !(c & 0x80) )
                  return c;

            return cast(uint) 'X';
      /*
          dchar u;
          uint len;



          // Check length of remaining string up to 6 UTF-8 characters
          for( len = 1; len < 6 && s[len]; len++ )
          {

          }
                  /+
          uint idx = 0;
          char* msg = utf_decodeChar( s, len, &idx, &u );
          p += idx - 1;
          if( msg )
          {
                  error(msg);
          }
                  +/
          return u;
      */
      }

      void error( ... )
      {
          if( (mod !is null) && !global.gag )
          {
                  writefln( formatLoc( loc, _arguments, _argptr ) );
                  /*
                  char[] p = loc.toChars();
                  if( p.length )
                      writef( "%s: ", p );
                writefx( stdout, _arguments, _argptr, 1 );
                  */
                  if( global.errors >= global.max_errors )  // moderate blizzard of cascading messages
                  throw new Exception( "too many errors" );
          }

          global.errors++;
      }

      void errorLoc(Loc loc, ...)
      {
          if( (mod !is null) && !global.gag )
          {
                  writefln( formatLoc( loc, _arguments, _argptr ) );
                  /*
                  char[] p = loc.toChars();
                  if( p.length )
                      writef("%s: ", p);
                writefx(stdout, _arguments, _argptr, 1);
                  */
                  if( global.errors >= 20 )     // moderate blizzard of cascading messages
                  throw new Exception( "too many errors" );
          }

          global.errors++;
      }


      TOK nextToken()
      {
          if( token.next )
          {
                  Token* t = token.next;
                  memcpy( &token, t, Token.sizeof );
//                t.next = freelist;
//                freelist = t;
          }
          else
          {
                  scan( &token );
          }
//        token.print();
          return token.value;
      }

      Token* peek( inout Token ct )
      {
            Token* t;
          if( ct.next )
                  t = ct.next;
          else
          {
                  t = new Token;
                  scan( t );
                  t.next = null;
                  ct.next = t;
          }
          return t;
      }

      // Turn next token in buffer into a token.

00277       void scan( Token* t )
      {
//          debug writefln("scan token");
          uint lastLine = loc.linnum;
          uint linnum;
          t.blockComment      = null;
          t.lineComment       = null;
          while( true )
          {
                  t.ptr = p;
//                debug writefln( "    p = %d, *p = ", cast(uint)p, cast(char)*p );
                  switch( *p )
                  {
                      case 0:
                      case 0x1a:
                              t.value = TOK.TOKeof;               // end of file
//                            debug writefln( "    EOF" );
                              return;

                      case ' ':
                      case '\t':
                      case '\v':
                      case '\f':
                              p++;
//                            debug writefln( "    whitespace" );
                              continue;                           // skip white space

                      case '\r':
//                            debug writefln( "    cr" );
                              p++;
                              if( *p != '\n' )              // if CR stands by itself
                                  loc.linnum++;
                              continue;                           // skip white space

                      case '\n':
//                            debug writefln( "    nl" );
                              p++;
                              loc.linnum++;
                              continue;                           // skip white space

                      case '0':     case '1':   case '2':   case '3':   case '4':
                      case '5':     case '6':   case '7':   case '8':   case '9':
                              t.value = number(t);
                              return;

/* 
      #if CSTRINGS
                      case '\'':
                        t.value = charConstant(t, 0);
                        return;

                      case '"':
                        t.value = stringConstant(t,0);
                        return;

                      case 'l':
                      case 'L':
                        if( p[1] == '\'')
                        {
                            p++;
                            t.value = charConstant(t, 1);
                            return;
                        }
                        else if( p[1] == '"')
                        {
                            p++;
                            t.value = stringConstant(t, 1);
                            return;
                        }
      #else 
*/
                      case '\'':
//                            debug writefln( "    char" );
                              t.value = charConstant(t,0);
                              return;

                      case 'r':
//                            debug writefln( "    wysiwyg" );
                              if( p[1] != '"')
                                  goto case_ident;
                              p++;
                            case '`':
                              t.value = wysiwygStringConstant(t, *p);
                              return;

                      case 'x':
//                            debug writefln( "    hex string" );
                              if( p[1] != '"')
                                  goto case_ident;
                              p++;
                              t.value = hexStringConstant(t);
                              return;


                      case '"':
//                            debug writefln( "    string" );
                              t.value = escapeStringConstant( t, 0 );
//                            debug writefln( t.ustring );
                              return;

                      case '\\':                // escaped string literal
//                            debug writefln( "    escaped string literal" );
                              uint c;
                              stringbuffer.offset = 0;
                              do
                              {
                                  p++;
                                  c = escapeSequence();
                                  stringbuffer.write(c);
                              } while (*p == '\\');
//                            t.len = stringbuffer.offset;
//                            stringbuffer.write(cast(byte)0);
                              t.ustring = stringbuffer.toString;
//                            memcpy( t.ustring.ptr, stringbuffer.data, stringbuffer.offset );
                              t.postfix = 0;
                              t.value = TOK.TOKstring;
                              return;

                      case 'l':
                      case 'L':
//    #endif

                      case 'a':     case 'b':   case 'c':   case 'd':   case 'e':
                      case 'f':     case 'g':   case 'h':   case 'i':   case 'j':
                      case 'k':                 case 'm':   case 'n':   case 'o':
                      case 'p':     case 'q': /*case 'r':*/ case 's':   case 't':
                      case 'u':     case 'v':   case 'w': /*case 'x':*/ case 'y':
                      case 'z':
                      case 'A':     case 'B':   case 'C':   case 'D':   case 'E':
                      case 'F':     case 'G':   case 'H':   case 'I':   case 'J':
                      case 'K':                 case 'M':   case 'N':   case 'O':
                      case 'P':     case 'Q':   case 'R':   case 'S':   case 'T':
                      case 'U':     case 'V':   case 'W':   case 'X':   case 'Y':
                      case 'Z':
                      case '_':
                      case_ident:
                      {
//                            debug writefln( "    identifier" );
                              ubyte c;
                              do
                              {
                                  c = *++p;
                              } while( isidchar(c) || (c & 0x80 && isUniAlpha( decodeUTF())) );

//                            sv = stringtable.update((char *)t.ptr, p - t.ptr);
                              char[] tmp;
                              tmp.length = p - t.ptr;
                              memcpy( tmp.ptr, t.ptr, p - t.ptr );
                              Identifier id;
                              Identifier* pid = tmp in stringtable;
                              if( pid )
                              {
                                    id = *pid;
                              }

                              if( id is null )
                              {
                                    id = new Identifier( tmp, TOK.TOKidentifier );
                                    stringtable[tmp] = id;
                              }

                              t.ident = id;
                              t.value = cast(TOK) id.value;
                              anyToken = 1;

                              // if special identifier token
                              if( *t.ptr == '_')
                              {
                                  static char date[11+1];
                                  static char time[8+1];
                                  static char timestamp[24+1];

                                  if( !date[0] )      // lazy evaluation
                                  {
                                    //!!
                                    /+
                                          time_t t;
                                          char *p;
                                          .time(&t);
                                          p = ctime(&t);
                                          assert(p);
                                          sprintf(date.ptr, "%.6s %.4s", p + 4, p + 20);
                                          sprintf(time.ptr, "%.8s", p + 11);
                                          sprintf(timestamp.ptr, "%.24s", p);
                                          +/
                                  }

                                  if( mod && id is Id.FILE )
                                  {
                                          t.value = TOK.TOKstring;
                                          if( loc.filename.length )
                                                t.ustring = loc.filename;
                                          else
                                                t.ustring = mod.ident.toChars();
                                          goto Llen;
                                  }
                                  else if( mod && id == Id.LINE )
                                  {
                                          t.value = TOK.TOKint64v;
                                          t.uns64value = loc.linnum;
                                  }
                                  else if( id == Id.DATE )
                                  {
                                          t.value = TOK.TOKstring;
                                          //! t.ustring = date;
                                          goto Llen;
                                  }
                                  else if( id == Id.TIME )
                                  {
                                          t.value = TOK.TOKstring;
                                          //! t.ustring = time;
                                          goto Llen;
                                  }
                                  else if( id == Id.TIMESTAMP )
                                  {
                                          t.value = TOK.TOKstring;
                                          //! t.ustring = timestamp;
                                   Llen:
                                          t.postfix = 0;
//                                        t.len = strlen((char *)t.ustring);
                                  }
                              }
                              //printf("t.value = %d\n",t.value);
                              return;
                      }

                        // comments
                      case '/':
                              p++;
                              switch( *p )
                              {
                                  case '=':
                                          p++;
                                          t.value = TOK.TOKdivass;
                                          return;

                                  case '*':     // '/*'
                                          p++;
                                          linnum = loc.linnum;
                                          while( true )
                                          {
                                              while( true )
                                              {
                                                      ubyte c = *p;
                                                      switch( c )
                                                      {
                                                          case '/':
                                                                  break;

                                                          case '\n':
                                                                  loc.linnum++;
                                                                  p++;
                                                                  continue;

                                                          case '\r':
                                                                  p++;
                                                                  if( *p != '\n')
                                                                      loc.linnum++;
                                                                  continue;

                                                          case 0:
                                                          case 0x1A:
                                                                  error("unterminated /* */ comment");
                                                                  p = end;
                                                                  t.value = TOK.TOKeof;
                                                                  return;

                                                          default:
                                                                  if( c & 0x80)
                                                                  {   
                                                                        uint u = decodeUTF();
                                                                      if( u == PS || u == LS )
                                                                              loc.linnum++;
                                                                  }
                                                                  p++;
                                                                  continue;
                                                      }
                                                      break;
                                              }
                                              p++;
                                              if( p[-2] == '*' && p - 3 != t.ptr )
                                                      break;
                                          }
                                          
                                          if( commentToken )
                                          {
                                              t.value = TOK.TOKcomment;
                                              return;
                                          }
                                          // if /** but not /**/
                                          else  if( doDocComment && t.ptr[2] == '*' && p - 4 != t.ptr )
                                              getDocComment( t, lastLine == linnum );     //! ?
                                          continue;

                                  case '/':           // do // style comments
                                          linnum = loc.linnum;
                                          while (1)
                                          {  
                                                ubyte c = *++p;
                                              switch (c)
                                              {
                                                      case '\n':
                                                          break;
                              
                                                      case '\r':
                                                          if( p[1] == '\n')
                                                            p++;
                                                          break;
                              
                                                      case 0:
                                                      case 0x1a:
                                                          if( commentToken )
                                                          {
                                                                  p = end;
                                                                  t.value = TOK.TOKcomment;
                                                                  return;
                                                          }
                                                          if( doDocComment && t.ptr[2] == '/' )
                                                                  getDocComment( t, lastLine == linnum );
                                                          p = end;
                                                          t.value = TOK.TOKeof;
                                                          return;
                              
                                                      default:
                                                          if( c & 0x80)
                                                          {
                                                                  uint u = decodeUTF();
                                                                  if( u == PS || u == LS)
                                                                      break;
                                                          }
                                                          continue;
                                              }
                                              break;
                                          }

                                          if( commentToken )
                                          {
                                              p++;
                                              loc.linnum++;
                                              t.value = TOK.TOKcomment;
                                              return;
                                          }
                                          if( doDocComment && t.ptr[2] == '/' )
                                              getDocComment( t, lastLine == linnum );
                        
                                          p++;
                                          loc.linnum++;
                                          continue;
      
                                  case '+':
                                        { int nest;
                                                linnum = loc.linnum;
                                                p++;
                                                nest = 1;
                                                while (1)
                                                {
                                                      ubyte c = *p;
                                                    switch (c)
                                                    {
                                                            case '/':
                                                                p++;
                                                                if( *p == '+')
                                                                {
                                                                        p++;
                                                                        nest++;
                                                                }
                                                                continue;
                                    
                                                            case '+':
                                                                p++;
                                                                if( *p == '/')
                                                                {
                                                                        p++;
                                                                        if( --nest == 0)
                                                                            break;
                                                                }
                                                                continue;
                                    
                                                            case '\r':
                                                                p++;
                                                                if( *p != '\n')
                                                                  loc.linnum++;
                                                                continue;
                                    
                                                            case '\n':
                                                                loc.linnum++;
                                                                p++;
                                                                continue;
                                    
                                                            case 0:
                                                            case 0x1A:
                                                                error("unterminated /+ +/ comment");
                                                                p = end;
                                                                t.value = TOK.TOKeof;
                                                                return;
                                    
                                                            default:
                                                                if( c & 0x80 )
                                                                {
                                                                        uint u = decodeUTF();
                                                                        if( u == PS || u == LS)
                                                                            loc.linnum++;
                                                                }
                                                                p++;
                                                                continue;
                                                    }
                                                    break;
                                                }
                                                if( commentToken )
                                                {
                                                    t.value = TOK.TOKcomment;
                                                    return;
                                                }
                                                if( doDocComment && t.ptr[2] == '+' && p - 4 != t.ptr )
                                                {  
                                                      // if /++ but not /++/
                                                    getDocComment(t, lastLine == linnum);
                                                }
                                                continue;
                                    }
                                  default:
                                        break;
                              }
                              t.value = TOK.TOKdiv;
                              return;

                      case '.':
                              p++;
                              if( isdigit(*p))
                              {
                                  p--;
                                  t.value = inreal(t);
                              }
                              else if( p[0] == '.')
                              {
                                  if( p[1] == '.')
                                  {
                                          p += 2;
                                          t.value = TOK.TOKdotdotdot;
                                  }
                                  else
                                  {
                                          p++;
                                          t.value = TOK.TOKslice;
                                  }
                              }
                              else
                                  t.value = TOK.TOKdot;
                              return;

                      case '&':
                              p++;
                              if( *p == '=')
                              {   
                                    p++;
                                  t.value = TOK.TOKandass;
                              }
                              else if( *p == '&')
                              {   
                                    p++;
                                  t.value = TOK.TOKandand;
                              }
                              else
                                  t.value = TOK.TOKand;
                              return;
      
                        // |, ||, |=
                      case '|':
                              p++;
                              if( *p == '=' )
                              {   p++;
                                  t.value = TOK.TOKorass;
                              }
                              else if( *p == '|')
                              {   p++;
                                  t.value = TOK.TOKoror;
                              }
                              else
                                  t.value = TOK.TOKor;
                              return;
                  
                            case '-':
                              p++;
                              if( *p == '=')
                              {   p++;
                                  t.value = TOK.TOKminass;
                              }
                              else if( *p == '-')
                              {   p++;
                                  t.value = TOK.TOKminusminus;
                              }
                              else
                                  t.value = TOK.TOKmin;
                              return;

                        // +, +=, ++
                      case '+':
                              p++;
                              if( *p == '=')
                              {   
                                    p++;
                                  t.value = TOK.TOKaddass;        // +=
                              }
                              else if( *p == '+')
                              {   p++;
                                  t.value = TOK.TOKplusplus;                  // ++
                              }
                              else
                                  t.value = TOK.TOKadd;                       // +
                              return;

                        // <, <=, <<=, <<, <>=, <>
                      case '<':
                              p++;
                              if( *p == '=')
                              {   p++;
                                  t.value = TOK.TOKle;                  // <=
                              }
                              else if( *p == '<')
                              {   p++;
                                  if( *p == '=')
                                  {   p++;
                                    t.value = TOK.TOKshlass;            // <<=
                                  }
                                  else
                                    t.value = TOK.TOKshl;               // <<
                              }
                              else if( *p == '>')
                              {   p++;
                                  if( *p == '=')
                                  {   p++;
                                    t.value = TOK.TOKleg;               // <>=
                                  }
                                  else
                                    t.value = TOK.TOKlg;                // <>
                              }
                              else
                                  t.value = TOK.TOKlt;                  // <
                              return;
            
                        // >, >>, >>>, >=, >>=, >>>=
                      case '>':
                              p++;
                              if( *p == '=')
                              {   p++;
                                  t.value = TOK.TOKge;                  // >=
                              }
                              else if( *p == '>')
                              {   p++;
                                  if( *p == '=')
                                  {   p++;
                                    t.value = TOK.TOKshrass;            // >>=
                                  }
                                  else if( *p == '>')
                                  { p++;
                                    if( *p == '=')
                                    {   p++;
                                        t.value = TOK.TOKushrass; // >>>=
                                    }
                                    else
                                        t.value = TOK.TOKushr;          // >>>
                                  }
                                  else
                                    t.value = TOK.TOKshr;               // >>
                              }
                              else
                                  t.value = TOK.TOKgt;                  // >
                              return;

                      case '!':
                              p++;
                              if( *p == '=')
                              {   p++;
                                  if( *p == '=')
                                  { 
                                          p++;
                                          t.value = TOK.TOKnotidentity; // !==
                                  }
                                  else
                                    t.value = TOK.TOKnotequal;          // !=
                              }
                              else if( *p == '<')
                              {   
                                    p++;
                                  if( *p == '>')
                                  { 
                                          p++;
                                          if( *p == '=')
                                          {  
                                                p++;
                                          t.value = TOK.TOKunord; // !<>=
                                          }
                                          else
                                            t.value = TOK.TOKue;  // !<>
                                  }
                                  else if( *p == '=')
                                  {
                                          p++;
                                          t.value = TOK.TOKug;    // !<=
                                  }
                                  else
                                          t.value = TOK.TOKuge;   // !<
                              }
                              else if( *p == '>')
                              {   p++;
                                  if( *p == '=')
                                  {
                                          p++;
                                          t.value = TOK.TOKul;    // !>=
                                  }
                                  else
                                          t.value = TOK.TOKule;   // !>
                              }
                              else
                                  t.value = TOK.TOKnot;           // !
                              return;

                      case '=':
                              p++;
                              if( *p == '=')
                              {     
                                    p++;
                                  if( *p == '=')
                                  {
                                          p++;
                                          t.value = TOK.TOKidentity;    // ===
                                  }
                                  else
                                          t.value = TOK.TOKequal;       // ==
                              }
                              else
                                  t.value = TOK.TOKassign;        // =
                              return;

                        case '~':
                              p++;
                              if( *p == '=' )
                              {   
                                    p++;
                                    t.value = TOK.TOKcatass;            // ~=
                              }
                              else
                                    t.value = TOK.TOKtilde;             // ~
                              return;

            // SINGLE
                      case '(': p++; t.value = TOK.TOKlparen;     return;
                      case ')': p++; t.value = TOK.TOKrparen;     return;
                      case '[': p++; t.value = TOK.TOKlbracket;   return;
                      case ']': p++; t.value = TOK.TOKrbracket;   return;
                      case '{': p++; t.value = TOK.TOKlcurly;     return;
                      case '}': p++; t.value = TOK.TOKrcurly;     return;
                      case '?': p++; t.value = TOK.TOKquestion;   return;
                      case ',': p++; t.value = TOK.TOKcomma;            return;
                      case ';': p++; t.value = TOK.TOKsemicolon;  return;
                      case ':': p++; t.value = TOK.TOKcolon;            return;
                      case '$': p++; t.value = TOK.TOKdollar;     return;
            // DOUBLE
                      case '*': p++; if( *p == '=' ) { p++; t.value = TOK.TOKmulass; } else t.value = TOK.TOKmul; return;
                      case '%': p++; if( *p == '=' ) { p++; t.value = TOK.TOKmodass; } else t.value = TOK.TOKmod; return;
                      case '^': p++; if( *p == '=' ) { p++; t.value = TOK.TOKxorass; } else t.value = TOK.TOKxor; return;
// removed 148    case '~': p++; if( *p == '=' ) { p++; t.value = TOK.TOKcatass; } else t.value = TOK.TOKtilde; return;


                      case '#':
                              p++;
                              Pragma();
                              continue;
            
                      default:
                      { 
                              debug writefln( "    default char" );
                              ubyte c = *p;
                              if( c & 0x80 )
                              {
                                    uint u = decodeUTF();
                                  // Check for start of unicode identifier
                                  if( isUniAlpha(u) )
                                          goto case_ident;

                                  if( u == PS || u == LS )
                                  {
                                          loc.linnum++;
                                          p++;
                                          continue;
                                  }
                              }
                              if( isprint(c))
                                  error("unsupported char '%s'", cast(char)c);
                              else
                                  error("unsupported char 0x%02x", cast(ubyte)c);
                              p++;
                              continue;
                      }
                  }
          }
      }



      // Parse escape sequence.
00978       uint escapeSequence()
      {
            uint c;
          int n;
          int ndigits;

          c = *p;
          switch ( c )
          {
                  case '\'':
                  case '"':
                  case '?':
                  case '\\':
                  Lconsume:
                        p++;
                        break;

                  case 'a':   c = 7;            goto Lconsume;
                  case 'b':   c = 8;            goto Lconsume;
                  case 'f':   c = 12;           goto Lconsume;
                  case 'n':   c = 10;           goto Lconsume;
                  case 'r':   c = 13;           goto Lconsume;
                  case 't':   c = 9;            goto Lconsume;
                  case 'v':   c = 11;           goto Lconsume;
      
                  case 'u':
                        ndigits = 4;
                        goto Lhex;
                  case 'U':
                        ndigits = 8;
                        goto Lhex;
                  case 'x':
                        ndigits = 2;
                  Lhex:
                        p++;
                        c = *p;
                        if( ishex(c))
                        {   
                              uint v;
                            n = 0;
                            v = 0;
                            while (1)
                            {
                                    if( isdigit(c))
                                        c -= '0';
                                    else if( islower(c))
                                        c -= 'a' - 10;
                                    else
                                        c -= 'A' - 10;
                                    v = v * 16 + c;
                                    c = *++p;
                                    if( ++n == ndigits)
                                        break;
                                    if( !ishex(c))
                                    {   error("escape hex sequence has %d hex digits instead of %d", n, ndigits);
                                        break;
                                    }
                            }
//!                         if( ndigits != 2 && !utf_isValidDchar(v))
//!                                 error("invalid UTF character \\U%08x", v);
                            c = v;
                        }
                        else
                            error("undefined escape hex sequence \\%s\n",c);
                        break;

                  case '&':               // named character entity
                        for( ubyte *idstart = ++p; 1; p++ )
                        {
                            switch( *p )
                            {
                                    case ';':
                                          //!!!
                                          /+
                                        c = HtmlNamedEntity(idstart, p - idstart);
                                        if( c == ~0 )
                                        {
                                                error("unnamed character entity &%.*s;", p - idstart, idstart);
                                                c = ' ';
                                        }

                                        p++;
                                        +/
                                        break;

                                    default:
                                        if( isalpha(*p) || (p != idstart + 1 && isdigit(*p)))
                                                continue;
                                        error("unterminated named entity");
                                        break;
                            }
                            break;
                        }
                        break;

                  case 0:
                  case 0x1a:              // end of file
                        c = '\\';
                        break;

                  default:
                        if( isoctal(c) )
                        {   
                              ubyte v;
                            n = 0;
                            do
                            {
                                    v = v * 8 + (c - '0');
                                    c = *++p;
                            } while (++n < 3 && isoctal(c));
                            c = v;
                        }
                        else
                            error("undefined escape sequence \\%s\n",c);
                        break;
          }
          return c;
      }

      /**************************************
       */

      TOK wysiwygStringConstant( Token *t, int tc )
      {
            uint c;
          Loc start = loc;

          p++;
          stringbuffer.offset = 0;
          while (1)
          {
                  c = *p++;
                  switch( c )
                  {
                      case '\n':
                              loc.linnum++;
                              break;

                      case '\r':
                              if( *p == '\n')
                                  continue;     // ignore
                              c = '\n';   // treat EndOfLine as \n character
                              loc.linnum++;
                              break;

                      case 0:
                      case 0x1a:
                              error("unterminated string constant starting at %s", start.toChars());
                              t.ustring = "";
                              t.postfix = 0;
                              return TOK.TOKstring;

                      case '"':
                      case '`':
                        if( c == tc)
                        {
//                          t.len = stringbuffer.offset;
                            stringbuffer.write(cast(byte)0);
                            t.ustring = stringbuffer.toString;
//                          t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);
//                          memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);
                            stringPostfix(t);
                            return TOK.TOKstring;
                        }
                        break;

                      default:
                        if( c & 0x80)
                        {
                              p--;
                            uint u = decodeUTF();
                            p++;
                            if( u == PS || u == LS)
                                    loc.linnum++;
                            stringbuffer.write(u);
                            continue;
                        }
                        break;
                  }
                  stringbuffer.write(c);
          }
      }

      /**************************************
       * Lex hex strings:
       *    x"0A ae 34FE BD"
       */

      TOK hexStringConstant(Token *t)
      {
            uint c;
          Loc start = loc;
          uint n = 0;
          uint v;
      
          p++;
          stringbuffer.offset = 0;
          while (1)
          {
                  c = *p++;
                  switch (c)
                  {
                      case ' ':
                      case '\t':
                      case '\v':
                      case '\f':
                              continue;               // skip white space

                      case '\r':
                              if( *p == '\n')
                                  continue;                 // ignore
                        // Treat isolated '\r' as if it were a '\n'
                      case '\n':
                              loc.linnum++;
                              continue;

                      case 0:
                      case 0x1a:
                              error("unterminated string constant starting at %s", start.toChars());
                              t.ustring = "";
                              t.postfix = 0;
                              return TOK.TOKstring;

                      case '"':
                              if( n & 1 )
                              {
                                    error("odd number (%d) of hex characters in hex string", n);
                                  stringbuffer.write(v);
                              }
//                      t.len = stringbuffer.offset;
//                      stringbuffer.write(cast(byte)0);
                              t.ustring = stringbuffer.toString;
//                      t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);
//                      memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);
                              stringPostfix(t);
                              return TOK.TOKstring;

                      default:
                              if( c >= '0' && c <= '9')
                                  c -= '0';
                              else if( c >= 'a' && c <= 'f')
                                  c -= 'a' - 10;
                              else if( c >= 'A' && c <= 'F')
                                  c -= 'A' - 10;
                              else if( c & 0x80)
                              {   
                                    p--;
                                  uint u = decodeUTF();
                                  p++;
                                  if( u == PS || u == LS)
                                          loc.linnum++;
                                  else
                                          error("non-hex character \\u%x", u);
                              }
                              else
                                  error("non-hex character '%s'", c);
                              if( n & 1)
                              {
                                    v = (v << 4) | c;
                                  stringbuffer.write(v);
                              }
                              else
                                  v = c;
                              n++;
                              break;
                  }
          }
      }

      /**************************************
       */

      TOK escapeStringConstant(Token *t, int wide)
      {
            uint c;
          Loc start = loc;
          p++;
          stringbuffer.offset = 0;
      //    debug writefln( "escape string constant: %s", std.string.toString( cast(char*)p ) );
          while( 1 )
          {
                  c = *p++;
                  switch( c )
                  {
                      case '\\':
                              switch (*p)
                              {
                                  case 'u':
                                  case 'U':
                                  case '&':
                                          c = escapeSequence();
                                          stringbuffer.write(c);
                                          continue;
      
                                  default:
                                          c = escapeSequence();
                                          break;
                              }
                              break;

                      case '\n':
                              loc.linnum++;
                              break;

                      case '\r':
                              if( *p == '\n')
                                  continue;     // ignore
                              c = '\n';   // treat EndOfLine as \n character
                              loc.linnum++;
                              break;

                      case '"':
//                      writefln( "end of string: ", stringbuffer.toString );
                              t.ustring = stringbuffer.toString().dup;
      //                      t.len = stringbuffer.offset;
      //                      stringbuffer.write(cast(byte)0);
      //                      t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);
      //                      memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);
                              stringPostfix(t);
                              
                              return TOK.TOKstring;

                      case 0:
                      case 0x1a:
                              p--;
                              error("unterminated string constant starting at %s", start.toChars());
                              t.ustring = "";
//                            t.len = 0;
                              t.postfix = 0;
                              return TOK.TOKstring;

                      default:
                              if( c & 0x80 )
                              {
                                  p--;
                                  c = decodeUTF();
                                  if( c == LS || c == PS )
                                  {
                                          c = '\n';
                                          loc.linnum++;
                                  }
                                  p++;
                                  stringbuffer.write(cast(char)c);
                                  continue;
                              }
                              break;
                  }
                  stringbuffer.write(cast(char)c);
//                writefln( stringbuffer.toString );
          }
      }

      //**************************************
      TOK charConstant(Token *t, int wide)
      {
            uint c;
          TOK tk = TOK.TOKcharv;
          //printf("Lexer.charConstant\n");
          p++;
          c = *p++;
          switch( c )
          {
                  case '\\':
                      switch (*p)
                      {
                              case 'u':
                                  t.uns64value = escapeSequence();
                                  tk = TOK.TOKwcharv;
                                  break;

                              case 'U':
                              case '&':
                                  t.uns64value = escapeSequence();
                                  tk = TOK.TOKdcharv;
                                  break;

                              default:
                                  t.uns64value = escapeSequence();
                                  break;
                      }
                      break;

                  case '\n':
                  L1:
                      loc.linnum++;
                  case '\r':
                  case 0:
                  case 0x1a:
                  case '\'':
                      error("unterminated character constant");
                      return tk;

                  default:
                      if( c & 0x80)
                      {
                              p--;
                              c = decodeUTF();
                              p++;
                              if( c == LS || c == PS )
                                  goto L1;
                              if( c < 0xd800 || (c >= 0xe000 && c < 0xfffe))
                                  tk = TOK.TOKwcharv;
                              else
                                  tk = TOK.TOKdcharv;
                      }
                      t.uns64value = c;
                      break;
          }

          if( *p != '\'' )
          {
                  error("unterminated character constant");
                  return tk;
          }
          p++;
          return tk;
      }

      // Get postfix of string literal.
      void stringPostfix(Token *t)
      {
          switch( *p )
          {
                  case 'c':
                  case 'w':
                  case 'd':
                      t.postfix = *p;
                      p++;
                      break;

                  default:
                      t.postfix = 0;
                      break;
          }
      }

      /***************************************
       * Read \u or \U unicode sequence
       * Input:
       *    u     'u' or 'U'
       */
      /*
      uint Wchar(uint u)
      {
          uint value;
          uint n;
          ubyte c;
          uint nchars;
      
          nchars = (u == 'U') ? 8 : 4;
          value = 0;
          for (n = 0; 1; n++)
          {
                  ++p;
                  if( n == nchars)
                      break;
                  c = *p;
                  if( !ishex(c))
                  {
                        error("\\%s sequence must be followed by %d hex characters", u, nchars);
                      break;
                  }
                  if( isdigit(c))
                      c -= '0';
                  else if( islower(c))
                      c -= 'a' - 10;
                  else
                      c -= 'A' - 10;
                  value <<= 4;
                  value |= c;
          }
          return value;
      }
      */

      /**************************************
       * Read in a number.
       * If it's an integer, store it in tok.TKutok.Vlong.
       *    integers can be decimal, octal or hex
       *    Handle the suffixes U, UL, LU, L, etc.
       * If it's double, store it in tok.TKutok.Vdouble.
       * Returns:
       *    TKnum
       *    TKdouble,...
       */

01464       TOK number(Token *t)
      {
          //debug writefln("Lexer.number()");
          // We use a state machine to collect numbers
          enum STATE
            {
                  STATE_initial,
                  STATE_0,
                  STATE_decimal,
                  STATE_octal,
                  STATE_octale,
                  STATE_hex,
                  STATE_binary,
                  STATE_hex0,
                  STATE_binary0,
                  STATE_hexh,
                  STATE_error
            }

          enum FLAGS
          {
                  FLAGS_decimal  = 1,           // decimal
                  FLAGS_unsigned = 2,           // u or U suffix
                  FLAGS_long     = 4,           // l or L suffix
          }
          FLAGS flags = FLAGS.FLAGS_decimal;

          int i;
          TOK result;
          int base;
          stringbuffer.offset = 0;
//          stringbuffer.data = null;
          STATE state = STATE.STATE_initial;
          ubyte* start = p;

            TOK _isreal()
            {
                  p = start;
                  return inreal(t);
            }

          while( true )
          {
                  char c = cast(char)*p;
                  switch( state )
                  {
                      case STATE.STATE_initial:       // opening state
                              if( c == '0' )
                                  state = STATE.STATE_0;
                              else
                                  state = STATE.STATE_decimal;
                              break;

                      case STATE.STATE_0:
                              flags = cast(FLAGS) (flags & ~FLAGS.FLAGS_decimal);
                              switch (c)
                              {
      //    #if ZEROH
//                                case 'H':                 // 0h
//                                case 'h':
//                                        goto hexh;
      //    #endif
                                  case 'X':
                                  case 'x':
                                          state = STATE.STATE_hex0;
                                          break;

                                  case '.':
                                          if( p[1] == '.')  // .. is a separate token
                                              goto done;
                                  case 'i':
                                  case 'f':
                                  case 'F':
                                          goto _Real;
      //    #if ZEROH
//                                case 'E':
//                                case 'e':
//                                        goto case_hex;
      //    #endif
                                  case 'B':
                                  case 'b':
                                          state = STATE.STATE_binary0;
                                          break;

                                  case '0': case '1': case '2': case '3':
                                  case '4': case '5': case '6': case '7':
                                          state = STATE.STATE_octal;
                                          break;

      //    #if ZEROH
//                                case '8': case '9': case 'A':
//                                case 'C': case 'D': case 'F':
//                                case 'a': case 'c': case 'd': case 'f':
//                                      case_hex:
//                                        state = STATE.STATE_hexh;
//                                        break;
      //    #endif
                                  case '_':
                                          state = STATE.STATE_octal;
                                          p++;
                                          continue;

                                  default:
                                          goto done;
                              }
                              break;

                      case STATE.STATE_decimal:       // reading decimal number

                              // if its not a digit - decimal complete or not a decimal
                              if( !isdigit( c ) )
                              {
//                                  debug writefln( "\tnon-digit( %s )", c );
            //    #if ZEROH
//                                if( ishex(c) || c == 'H' || c == 'h' )
//                                        goto hexh;
            //    #endif
                                    //! wtf ?
                                    // ignore embedded _
                                  if( c == '_' )
                                  {
                                          p++;
                                          continue;
                                  }

                                  // check decimal point - make real
                                  if( c == '.' && p[1] != '.' )
                                          goto _Real;

                                    // check for mantra - make real
                                  if( c == 'i' || c == 'f' || c == 'F' || c == 'e' || c == 'E' )
                                  {
                                  _Real:  // It's a real number. Back up and rescan as a real
                                          p = start;
                                          return inreal(t);
                                  }

                                  goto done;
                              }
                              break;

                      case STATE.STATE_hex0:          // reading hex number
                      case STATE.STATE_hex:
                              if( !ishex(c) )
                              {
                                  if( c == '_')       // ignore embedded _
                                  {
                                          p++;
                                          continue;
                                  }
                                  if( c == '.' && p[1] != '.' )
                                          goto _Real;
                                  if( c == 'P' || c == 'p' || c == 'i' )
                                          goto _Real;
                                  if( state == STATE.STATE_hex0 )
                                          error("Hex digit expected, not '%s'", c);
                                  goto done;
                              }
                              state = STATE.STATE_hex;
                              break;
      
            //    #if ZEROH
//                          hexh:
//                                  state = STATE.STATE_hexh;
//
//                          case STATE.STATE_hexh:          // parse numbers like 0FFh
//                                  if( !ishex(c))
//                                  {
//                                      if( c == 'H' || c == 'h')
//                                      {
//                                              p++;
//                                              base = 16;
//                                              goto done;
//                                      }
//                                      else
//                                      {
//                                              // Check for something like 1E3 or 0E24
//                                              if( memchr(stringbuffer.data.ptr, 'E', stringbuffer.offset) || memchr( stringbuffer.data.ptr, 'e', stringbuffer.offset))
//                                                  goto _Real;
//                                              error("Hex digit expected, not '%s'", c);
//                                              goto done;
//                                      }
//                                  }
//                                  break;
      //          #endif

                            case STATE.STATE_octal:               // reading octal number
                            case STATE.STATE_octale:        // reading octal number with non-octal digits
                                    if( !isoctal(c) )
                                    {
//                      #if ZEROH
//                                      if( ishex(c) || c == 'H' || c == 'h' )
//                                              goto hexh;
//                      #endif
                                        if( c == '_' )            // ignore embedded _
                                        {
                                                p++;
                                                continue;
                                        }
                                        if( c == '.' && p[1] != '.' )
                                                goto _Real;
                                        if( c == 'i' )
                                                goto _Real;
                                        if( isdigit(c) )
                                                state = STATE.STATE_octale;
                                        else
                                                goto done;
                                    }
                                    break;

                            case STATE.STATE_binary0:       // starting binary number
                            case STATE.STATE_binary:        // reading binary number
                                    if( c != '0' && c != '1')
                                    {
      //                #if ZEROH
//                                      if( ishex(c) || c == 'H' || c == 'h' )
//                                              goto hexh;
      //                #endif
                                        if( c == '_' )            // ignore embedded _
                                        {
                                                p++;
                                                continue;
                                        }
                                        if( state == STATE.STATE_binary0 )
                                        {
                                                error("binary digit expected");
                                                state = STATE.STATE_error;
                                                break;
                                        }
                                        else
                                                goto done;
                                    }
                                    state = STATE.STATE_binary;
                                    break;

                            case STATE.STATE_error:         // for error recovery
                                    if( !isdigit(c) ) // scan until non-digit
                                        goto done;
                                    break;

                            default:
                                    assert(0);
                        }
                        stringbuffer.write(cast(char)c);
                        p++;
                }
      done:
          stringbuffer.write( cast(char)0 );          // terminate string

//          debug writefln( "\tdigit complete( %s )", stringbuffer.toString );

          if( state == STATE.STATE_octale )
                  error("Octal digit expected");

            uinteger_t n;                 // unsigned >=64 bit integer type

          if( stringbuffer.offset == 2 && ( state == STATE.STATE_decimal || state == STATE.STATE_0 ) )
                  n = stringbuffer.data[0] - '0';
          else
          {
            // Convert string to integer
                  char* p = cast(char*)stringbuffer.data.ptr;
                  int r = 10;
                  int d;
                  if( *p == '0' )
                  {
                      if( p[1] == 'x' || p[1] == 'X')
                        { 
                              // "0x#"
                              p += 2; 
                              r = 16; 
                        }
                      else if( p[1] == 'b' || p[1] == 'B')
                        { 
                              // "0b#" - binary
                              p += 2; 
                              r = 2; 
                        }
                      else if( isdigit(p[1]))
                        { 
                              p += 1; 
                              r = 8; 
                        }
                  }

                  n = 0;

                  while( true )
                  {
                      if( *p >= '0' && *p <= '9' )
                              d = *p - '0';
                      else if( *p >= 'a' && *p <= 'z' )
                              d = *p - 'a' + 10;
                      else if( *p >= 'A' && *p <= 'Z' )
                              d = *p - 'A' + 10;
                      else
                              break;

                        if( d >= r )
                              break;
                      
                        if( n * r + d < n)
                      {
                              error( "integer overflow" );
                              break;
                      }

                      n = n * r + d;
                      p++;
                  }

                  // if n needs more than 64 bits
                  if( n.sizeof > 8 && n > 0xffffffffffffffffL )
                      error("integer overflow");
          }

          // Parse trailing 'u', 'U', 'l' or 'L' in any combination
          while( true )
          {
                  ubyte f;
                  switch( *p )
                  {
                        case 'U':
                      case 'u':
                              f = FLAGS.FLAGS_unsigned;
                              goto L1;
                      case 'L':
                      case 'l':
                              f = FLAGS.FLAGS_long;
                      L1:
                              p++;
                              if( flags & f )
                                  error("unrecognized token");
                              flags = cast(FLAGS) (flags | f);
                              continue;
                      default:
                              break;
                  }
                  break;
          }

          switch ( flags )
          {
                  case 0:
                        /* Octal or Hexadecimal constant.
                         * First that fits: int, uint, long, ulong
                         */
                      if( n & 0x8000000000000000L )
                            result = TOK.TOKuns64v;
                      else if( n & 0xffffffff00000000L )
                            result = TOK.TOKint64v;
                      else if( n & 0x80000000 )
                            result = TOK.TOKuns32v;
                      else
                            result = TOK.TOKint32v;
                      break;

                  case FLAGS.FLAGS_decimal:
                        /* First that fits: int, long, long long
                         */
                      if( n & 0x8000000000000000L )
                      {
                              error("signed integer overflow");
                            result = TOK.TOKuns64v;
                      }
                      else if( n & 0xffffffff80000000L )
                            result = TOK.TOKint64v;
                      else
                            result = TOK.TOKint32v;
                      break;

                  case FLAGS.FLAGS_unsigned:
                  case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned:
                        /* First that fits: uint, ulong
                         */
                      if( n & 0xffffffff00000000L )
                            result = TOK.TOKuns64v;
                      else
                            result = TOK.TOKuns32v;
                      break;

                  case FLAGS.FLAGS_decimal | FLAGS.FLAGS_long:
                      if( n & 0x8000000000000000L )
                      {
                              error("signed integer overflow");
                            result = TOK.TOKuns64v;
                      }
                      else
                            result = TOK.TOKint64v;
                      break;

                  case FLAGS.FLAGS_long:
                      if( n & 0x8000000000000000L )
                            result = TOK.TOKuns64v;
                      else
                            result = TOK.TOKint64v;
                      break;

                  case FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long:
                  case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long:
                      result = TOK.TOKuns64v;
                      break;

                  default:
                        debug writefln("%x",flags);
                      assert(0);
          }
          t.uns64value = n;
          return result;
      }
      
      /**************************************
       * Read in characters, converting them to real.
       * Bugs:
       *    Exponent overflow not detected.
       *    Too much requested precision is not detected.
       */
      
      TOK inreal(Token *t)
      {
            int dblstate;
          uint c;
          char hex;                 // is this a hexadecimal-floating-constant?
          TOK result;

          //printf("Lexer.inreal()\n");
          stringbuffer.offset = 0;
          dblstate = 0;
          hex = 0;
      Lnext:
          while (1)
          {
            // Get next char from input
            c = *p++;
            //printf("dblstate = %d, c = '%s'\n", dblstate, c);
            while (1)
            {
                switch (dblstate)
                {
                        case 0:                 // opening state
                            if( c == '0')
                                    dblstate = 9;
                            else if( c == '.')
                                    dblstate = 3;
                            else
                                    dblstate = 1;
                            break;
      
                        case 9:
                            dblstate = 1;
                            if( c == 'X' || c == 'x')
                            {
                                    hex++;
                                    break;
                            }
                        case 1:                 // digits to left of .
                        case 3:                 // digits to right of .
                        case 7:                 // continuing exponent digits
                            if( !isdigit(c) && !(hex && isxdigit(c)))
                            {
                                    if( c == '_')
                                        goto Lnext;   // ignore embedded '_'
                                    dblstate++;
                                    continue;
                            }
                            break;

                        case 2:                 // no more digits to left of .
                            if( c == '.')
                            {
                                    dblstate++;
                                    break;
                            }
                        case 4:                 // no more digits to right of .
                            if( (c == 'E' || c == 'e') || hex && (c == 'P' || c == 'p'))
                            {
                                    dblstate = 5;
                                    hex = 0;    // exponent is always decimal
                                    break;
                            }
                            if( hex)
                                    error("binary-exponent-part required");
                            goto done;

                        case 5:                 // looking immediately to right of E
                            dblstate++;
                            if( c == '-' || c == '+')
                              break;
                        case 6:                 // 1st exponent digit expected
                            if( !isdigit(c))
                              error("exponent expected");
                            dblstate++;
                            break;

                        case 8:                 // past end of exponent digits
                            goto done;
                }
                break;
            }
            stringbuffer.write(c);
          }
      done:
          p--;

          stringbuffer.write(cast(byte)0);

//    #if _WIN32 && __DMC__
          char *save = __locale_decpoint;
          __locale_decpoint = ".";
//    #endif
          t.float80value = strtold(cast(char *)stringbuffer.data.ptr, null);
          errno = 0;
          switch( *p )
          {
                  case 'F':
                  case 'f':
                      strtof(cast(char *)stringbuffer.data.ptr, null);
                      result = TOK.TOKfloat32v;
                      p++;
                      break;

                  default:
                      strtod(cast(char *)stringbuffer.data.ptr, null);
                      result = TOK.TOKfloat64v;
                      break;

                  case 'L':
                  case 'l':
                      result = TOK.TOKfloat80v;
                      p++;
                      break;
          }
          if( *p == 'i' || *p == 'I')
          {
                  p++;
                  switch( result )
                  {
                      case TOK.TOKfloat32v:
                              result = TOK.TOKimaginary32v;
                              break;
                      case TOK.TOKfloat64v:
                              result = TOK.TOKimaginary64v;
                              break;
                      case TOK.TOKfloat80v:
                              result = TOK.TOKimaginary80v;
                              break;
                  }
          }
//    #if _WIN32 && __DMC__
          __locale_decpoint = save;
//    #endif
          if( errno == ERANGE)
                  error("number is not representable");
          return result;
      }




      /*********************************************
       * Do pragma.
       * Currently, the only pragma supported is:
       *    #line linnum [filespec]
       */

02029       void Pragma()
      {
          Token tok;
          int linnum;
          char[] filespec;
          Loc loc = this.loc;

          scan(&tok);

          if( tok.value != TOK.TOKidentifier || tok.ident != Id.line )
                  goto Lerr;

          scan(&tok);
          if( tok.value == TOK.TOKint32v || tok.value == TOK.TOKint64v )
                  linnum = tok.uns64value - 1;
          else
                  goto Lerr;

          while (1)
          {
                  switch (*p)
                  {
                      case 0:
                      case 0x1a:
                      case '\n':
                            Lnewline:
                              this.loc.linnum = linnum;
                              if( filespec.length )
                                  this.loc.filename = filespec;
                              return;

                      case '\r':
                              p++;
                              if( *p != '\n')
                              {   p--;
                                  goto Lnewline;
                              }
                              continue;

                      case ' ':
                      case '\t':
                      case '\v':
                      case '\f':
                              p++;
                              continue;               // skip white space

                      case '_':
                              if( mod && memcmp(p, cast(char*)"__FILE__", 8) == 0)
                              {
                                  p += 8;
//!                               filespec = mem.strdup(loc.filename ? loc.filename : mod.ident.toChars());
                              }
                              continue;

                      case '"':
                              if( filespec )
                                  goto Lerr;
                              stringbuffer.offset = 0;
                              p++;
                              while (1)
                              {
                                    uint c;
                                  c = *p;
                                  switch (c)
                                  {
                                          case '\n':
                                          case '\r':
                                          case 0:
                                          case 0x1a:
                                              goto Lerr;
      
                                          case '"':
                                              stringbuffer.write(cast(byte)0);
      //                                      filespec = mem.strdup((char *)stringbuffer.data);
                                              filespec = stringbuffer.toString.dup;
                                              p++;
                                              break;
      
                                          default:
                                              if( c & 0x80 )
                                              {
                                                      uint u = decodeUTF();
                                                      if( u == PS || u == LS )
                                                          goto Lerr;
                                              }
                                              stringbuffer.write(c);
                                              p++;
                                              continue;
                                        }
                                        break;
                              }
                              continue;

                      default:
                              if( *p & 0x80 )
                              {   
                                    uint u = decodeUTF();
                                  if( u == PS || u == LS)
                                    goto Lnewline;
                              }
                              goto Lerr;
                  }
          }

      Lerr:
          errorLoc(loc, "#line integer [\"filespec\"]\\n expected");
      }



      /***************************************************
       * Parse doc comment embedded between t.ptr and p.
       * Remove trailing blanks and tabs from lines.
       * Replace all newlines with \n.
       * Remove leading comment character from each line.
       * Decide if it's a lineComment or a blockComment.
       * Append to previous one for this token.
       */

      void getDocComment( Token *t, uint lineComment )
      {
          auto OutBuffer buf = new OutBuffer;
          ubyte ct = t.ptr[2];
          ubyte *q = t.ptr + 3;     // start of comment text
          int linestart = 0;

          ubyte *qend = p;

          if( ct == '*' || ct == '+')
                  qend -= 2;

          // Scan over initial row of ****'s or ++++'s or ////'s
          for (; q < qend; q++)
          {
                  if( *q != ct)
                      break;
          }

          // Remove trailing row of ****'s or ++++'s
          if( ct != '/')
          {
                  for (; q < qend; qend--)
                  {
                        if( qend[-1] != ct)
                              break;
                  }
          }

          for (; q < qend; q++)
          {
                  ubyte c = *q;

                  switch (c)
                  {
                        case '*':
                        case '+':
                        if( linestart && c == ct)
                        {   linestart = 0;
                              // Trim preceding whitespace up to preceding \n
                              while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t'))
                              buf.offset--;
                              continue;
                        }
                        break;
            
                        case ' ':
                        case '\t':
                        break;

                        case '\r':
                        if( q[1] == '\n')
                              continue;         // skip the \r
                        goto Lnewline;
            
                        default:
                        if( c == 226)
                        {
                              // If LS or PS
                              if( q[1] == 128 &&
                              (q[2] == 168 || q[2] == 169))
                              {
                              q += 2;
                              goto Lnewline;
                              }
                        }
                        linestart = 0;
                        break;

                        Lnewline:
                        c = '\n';         // replace all newlines with \n
                        case '\n':
                        linestart = 1;
            
                        // Trim trailing whitespace
                        while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t'))
                              buf.offset--;
            
                        break;
                  }
                  buf.write(c);
          }
      
          // Always end with a newline
          if( !buf.offset || buf.data[buf.offset - 1] != '\n' )
                  buf.writenl();
      
          //buf.write(cast(char)0);
      
          // It's a line comment if the start of the doc comment comes
          // after other non-whitespace on the same line.
//        ubyte** dc = (lineComment && anyToken)
//                       ? &t.lineComment
//                       : &t.blockComment;
      
          char[] dc = (lineComment && anyToken) ? t.lineComment : t.blockComment;

          // Combine with previous doc comment, if any
          if( dc.length )
                  dc = combineComments( dc, buf.toString().dup );
          else
                  dc = buf.toString().dup;
            
//          writefln( dc );

          if( lineComment && anyToken )
                  t.lineComment = dc;
            else
                  t.blockComment = dc;

      }

}

// character maps
static ubyte[256] cmtable;

const int CMoctal       = 0x1;
const int CMhex   = 0x2;
const int CMidchar      = 0x4;

ubyte isoctal (ubyte c) { return cmtable[c] & CMoctal; }
ubyte ishex   (ubyte c) { return cmtable[c] & CMhex; }
ubyte isidchar(ubyte c) { return cmtable[c] & CMidchar; }

static void cmtable_init()
{
    for( uint c = 0; c < cmtable.length; c++ )
    {
            if( '0' <= c && c <= '7' )
                cmtable[c] |= CMoctal;
            if( isdigit(c) || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F') )
                cmtable[c] |= CMhex;
            if( isalnum(c) || c == '_' )
                cmtable[c] |= CMidchar;
    }
}


/+
struct StringValue
{
    union
    {
            int intvalue;
            void *ptrvalue;
            dchar *string;
    }

    char[] lstring;
}
#define CASE_BASIC_TYPES                  
      case TOKwchar: case TOKdchar:       
      case TOKbit: case TOKbool: case TOKchar:        
      case TOKint8: case TOKuns8:         
      case TOKint16: case TOKuns16:       
      case TOKint32: case TOKuns32:       
      case TOKint64: case TOKuns64:       
      case TOKfloat32: case TOKfloat64: case TOKfloat80:          
      case TOKimaginary32: case TOKimaginary64: case TOKimaginary80:    
      case TOKcomplex32: case TOKcomplex64: case TOKcomplex80:    
      case TOKvoid:

#define CASE_BASIC_TYPES_X(t)                         \
      case TOKvoid:      t = Type::tvoid;  goto LabelX;           \
      case TOKint8:      t = Type::tint8;  goto LabelX;           \
      case TOKuns8:      t = Type::tuns8;  goto LabelX;           \
      case TOKint16:     t = Type::tint16; goto LabelX;           \
      case TOKuns16:     t = Type::tuns16; goto LabelX;           \
      case TOKint32:     t = Type::tint32; goto LabelX;           \
      case TOKuns32:     t = Type::tuns32; goto LabelX;           \
      case TOKint64:     t = Type::tint64; goto LabelX;           \
      case TOKuns64:     t = Type::tuns64; goto LabelX;           \
      case TOKfloat32: t = Type::tfloat32; goto LabelX;     \
      case TOKfloat64: t = Type::tfloat64; goto LabelX;     \
      case TOKfloat80: t = Type::tfloat80; goto LabelX;     \
      case TOKimaginary32: t = Type::timaginary32; goto LabelX;   \
      case TOKimaginary64: t = Type::timaginary64; goto LabelX;   \
      case TOKimaginary80: t = Type::timaginary80; goto LabelX;   \
      case TOKcomplex32: t = Type::tcomplex32; goto LabelX; \
      case TOKcomplex64: t = Type::tcomplex64; goto LabelX; \
      case TOKcomplex80: t = Type::tcomplex80; goto LabelX; \
      case TOKbit:       t = Type::tbit;     goto LabelX;   \
      case TOKchar:      t = Type::tchar;    goto LabelX;   \
      case TOKwchar:     t = Type::twchar; goto LabelX;     \
      case TOKdchar:     t = Type::tdchar; goto LabelX;     \
      LabelX
+/

Generated by  Doxygen 1.6.0   Back to index