Logo Search packages:      
Sourcecode: uncrustify version File versions

40007-Lexer.d

/+
 *      Copyright (c) 1999-2006 by Digital Mars
 *      All Rights Reserved
 *      written by Walter Bright www.digitalmars.com
 *      License for redistribution is by either the Artistic License in artistic.txt, or the GNU General Public License in gnu.txt.
 *      See the included readme.txt for details.
 *      D Language conversion by: J Duncan
 +/

/**
 *      d language lexer
 */

module dparser.Lexer;

import dparser.Root;

import dparser.Tokens;
import dparser.Token;
import dparser.Keyword;

import dparser.Types;

import dparser.Module;
import dparser.Identifier;
import dparser.unialpha;

import dparser.OutBuffer;

//private import std.ctype;
//private import std.string;
//import dwf.core.debugapi;

int errno = 0;

//#if _WIN32 && __DMC__
// from \dm\src\include\setlocal.h
//extern "C" char * __cdecl __locale_decpoint;
char * __locale_decpoint;
//#endif
//const uint LS = 0x2028;     // UTF line separator
//const uint PS = 0x2029;     // UTF paragraph separator

//extern int isUniAlpha(unsigned u);
//extern int HtmlNamedEntity(unsigned char *p, int length);

/**
 *      Lexer object
 */

class Lexer
{
    static           Identifier[char[]]       stringtable;
    static OutBuffer stringbuffer;
    static Token     *                       freelist;

    Token            token;        // current token
    Module           mod;          // current module
    Loc              loc;          // for error messages
    ubyte            * base;       // pointer to start of buffer
    ubyte            * end;        // past end of buffer
    ubyte            * p;          // current character
    int              doDocComment; // collect doc comment information
    int              anyToken;     // !=0 means seen at least one token
    int              commentToken; // !=0 means comments are TOKcomment's


    this(Module mod, ubyte * base, uint begoffset, uint endoffset, int doDocComment, int commentToken)
    {
        if (stringbuffer is null) {
            stringbuffer = new OutBuffer;
        }
        loc = Loc(mod, 1);

        this.base         = base;
        this.end          = base + endoffset;
        this.p            = base + begoffset;
        this.mod          = mod;
        this.doDocComment = doDocComment;
        this.commentToken = commentToken;

        /*
         *      If first line starts with '#!', ignore the line
         */

        if (p[0] == '#' && p[1] == '!') {
            p += 2;
            while (true) {
                ubyte c = *p;
                switch (c) {
                case '\n':
                    p++;
                    break;

                case '\r':
                    p++;
                    if (*p == '\n') {
                        p++;
                    }
                    break;

                case 0:
                case 0x1A:
                    break;

                default:
                    if (c & 0x80) {
                        uint u = decodeUTF();
                        if (u == PS || u == LS) {
                            break;
                        }
                    }
                    p++;
                    continue;
                }
                break;
            }

            loc.linnum = 2;
        }
    }



    // generate a unique identifier for this string
    static Identifier idPool(in char[] str) {
//        StringValue sv;
//        uint len = s.length;
//        StringValue sv = stringtable.update(s, len);
//        Identifier* id = cast(Identifier*) sv.ptrvalue;
//        if( id is null )
        if ((str in stringtable) == null) {
            stringtable[str] = new Identifier(str, TOK.TOKidentifier);
        }
        return(stringtable[str]);
    }

    static void initKeywords() {
        // build character map
        cmtable_init();

        // create keyword tokens & identifiers
        dparser.Keyword.initKeywords();

        // create standard lexer tokens
        dparser.Token.createLexerTokens();
    }

    // Combine two document comments into one.
    static char[] combineComments(char[] c1, char[] c2) {
        char[] c = c2;
        if (c1.length) {
            c = c1;
            if (c2.length) {
                c = c1 ~ "\n" ~ c2;
            }
        }
        return(c);
    }

    // Decode UTF character. Issue error messages for invalid sequences. Return decoded character, advance p to last character in UTF sequence.
    //! fix
00163     uint decodeUTF() {
        ubyte * s = p;
        ubyte c   = *s;

        assert(c & 0x80);
        if (!(c & 0x80)) {
            return(c);
        }

        return(cast(uint)'X');
        /*
         *  dchar u;
         *  uint len;
         *
         *
         *
         *  // Check length of remaining string up to 6 UTF-8 characters
         *  for( len = 1; len < 6 && s[len]; len++ )
         *  {
         *
         *  }
         *              /+
         *  uint idx = 0;
         *  char* msg = utf_decodeChar( s, len, &idx, &u );
         *  p += idx - 1;
         *  if( msg )
         *  {
         *              error(msg);
         *  }
         +/
         *  return u;
         */
    }

    void error(...) {
        if ((mod !is null) && !global.gag) {
            writefln(formatLoc(loc, _arguments, _argptr));
            /*
             * char[] p = loc.toChars();
             * if( p.length )
             *  writef( "%s: ", p );
             * writefx( stdout, _arguments, _argptr, 1 );
             */
            if (global.errors >= global.max_errors) {                   // moderate blizzard of cascading messages
                throw new Exception("too many errors");
            }
        }

        global.errors++;
    }

    void errorLoc(Loc loc, ...) {
        if ((mod !is null) && !global.gag) {
            writefln(formatLoc(loc, _arguments, _argptr));
            /*
             * char[] p = loc.toChars();
             * if( p.length )
             *  writef("%s: ", p);
             * writefx(stdout, _arguments, _argptr, 1);
             */
            if (global.errors >= 20) {                  // moderate blizzard of cascading messages
                throw new Exception("too many errors");
            }
        }

        global.errors++;
    }


    TOK nextToken() {
        if (token.next) {
            Token * t = token.next;
            memcpy(&token, t, Token.sizeof);
//                t.next = freelist;
//                freelist = t;
        }
        else {
            scan(&token);
        }
//        token.print();
        return(token.value);
    }

    Token * peek(inout Token ct) {
        Token * t;

        if (ct.next) {
            t = ct.next;
        }
        else {
            t = new Token;
            scan(t);
            t.next  = null;
            ct.next = t;
        }
        return(t);
    }

    // Turn next token in buffer into a token.

00263     void scan(Token * t) {
//          debug writefln("scan token");
        uint lastLine = loc.linnum;
        uint linnum;

        t.blockComment = null;
        t.lineComment  = null;
        while (true) {
            t.ptr = p;
//                debug writefln( "    p = %d, *p = ", cast(uint)p, cast(char)*p );
            switch (*p) {
            case 0:
            case 0x1a:
                t.value = TOK.TOKeof;                                           // end of file
//                            debug writefln( "    EOF" );
                return;

            case ' ':
            case '\t':
            case '\v':
            case '\f':
                p++;
//                            debug writefln( "    whitespace" );
                continue;                                                               // skip white space

            case '\r':
//                            debug writefln( "    cr" );
                p++;
                if (*p != '\n') {                                       // if CR stands by itself
                    loc.linnum++;
                }
                continue;                                                               // skip white space

            case '\n':
//                            debug writefln( "    nl" );
                p++;
                loc.linnum++;
                continue;                                                               // skip white space

            case '0':
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                t.value = number(t);
                return;

/*
 #if CSTRINGS
 *                          case '\'':
 *                              t.value = charConstant(t, 0);
 *                              return;
 *
 *                          case '"':
 *                              t.value = stringConstant(t,0);
 *                              return;
 *
 *                          case 'l':
 *                          case 'L':
 *                              if( p[1] == '\'')
 *                              {
 *                                  p++;
 *                                  t.value = charConstant(t, 1);
 *                                  return;
 *                              }
 *                              else if( p[1] == '"')
 *                              {
 *                                  p++;
 *                                  t.value = stringConstant(t, 1);
 *                                  return;
 *                              }
 #else
 */
            case '\'':
//                            debug writefln( "    char" );
                t.value = charConstant(t, 0);
                return;

            case 'r':
//                            debug writefln( "    wysiwyg" );
                if (p[1] != '"') {
                    goto case_ident;
                }
                p++;

            case '`':
                t.value = wysiwygStringConstant(t, *p);
                return;

            case 'x':
//                            debug writefln( "    hex string" );
                if (p[1] != '"') {
                    goto case_ident;
                }
                p++;
                t.value = hexStringConstant(t);
                return;


            case '"':
//                            debug writefln( "    string" );
                t.value = escapeStringConstant(t, 0);
//                            debug writefln( t.ustring );
                return;

            case '\\':                                  // escaped string literal
//                            debug writefln( "    escaped string literal" );
                uint c;
                stringbuffer.offset = 0;
                do {
                    p++;
                    c = escapeSequence();
                    stringbuffer.write(c);
                } while (*p == '\\');
//                            t.len = stringbuffer.offset;
//                            stringbuffer.write(cast(byte)0);
                t.ustring = stringbuffer.toString;
//                            memcpy( t.ustring.ptr, stringbuffer.data, stringbuffer.offset );
                t.postfix = 0;
                t.value   = TOK.TOKstring;
                return;

            case 'l':
            case 'L':
//    #endif

            case 'a':
            case 'b':
            case 'c':
            case 'd':
            case 'e':
            case 'f':
            case 'g':
            case 'h':
            case 'i':
            case 'j':
            case 'k':
            case 'm':
            case 'n':
            case 'o':
            case 'p':
            case 'q':                            /*case 'r':*/
            case 's':
            case 't':
            case 'u':
            case 'v':
            case 'w':                                       /*case 'x':*/
            case 'y':
            case 'z':
            case 'A':
            case 'B':
            case 'C':
            case 'D':
            case 'E':
            case 'F':
            case 'G':
            case 'H':
            case 'I':
            case 'J':
            case 'K':
            case 'M':
            case 'N':
            case 'O':
            case 'P':
            case 'Q':
            case 'R':
            case 'S':
            case 'T':
            case 'U':
            case 'V':
            case 'W':
            case 'X':
            case 'Y':
            case 'Z':
            case '_':
case_ident:
                {
//                            debug writefln( "    identifier" );
                    ubyte c;
                    do {
                        c = *++p;
                    } while (isidchar(c) || (c & 0x80 && isUniAlpha(decodeUTF())));

//                            sv = stringtable.update((char *)t.ptr, p - t.ptr);
                    char[] tmp;
                    tmp.length = p - t.ptr;
                    memcpy(tmp.ptr, t.ptr, p - t.ptr);
                    Identifier id;
                    Identifier * pid = tmp in stringtable;
                    if (pid) {
                        id = *pid;
                    }

                    if (id is null) {
                        id               = new Identifier(tmp, TOK.TOKidentifier);
                        stringtable[tmp] = id;
                    }

                    t.ident  = id;
                    t.value  = cast(TOK)id.value;
                    anyToken = 1;

                    // if special identifier token
                    if (*t.ptr == '_') {
                        static char date[11 + 1];
                        static char time[8 + 1];
                        static char timestamp[24 + 1];

                        if (!date[0]) {                         // lazy evaluation
                            //!!
                            /+
                             *      time_t t;
                             *      char *p;
                             *      .time(&t);
                             *      p = ctime(&t);
                             *      assert(p);
                             *      sprintf(date.ptr, "%.6s %.4s", p + 4, p + 20);
                             *      sprintf(time.ptr, "%.8s", p + 11);
                             *      sprintf(timestamp.ptr, "%.24s", p);
                             +/
                        }

                        if (mod && id is Id.FILE) {
                            t.value = TOK.TOKstring;
                            if (loc.filename.length) {
                                t.ustring = loc.filename;
                            }
                            else {
                                t.ustring = mod.ident.toChars();
                            }
                            goto Llen;
                        }
                        else if (mod && id == Id.LINE) {
                            t.value      = TOK.TOKint64v;
                            t.uns64value = loc.linnum;
                        }
                        else if (id == Id.DATE) {
                            t.value = TOK.TOKstring;
                            //! t.ustring = date;
                            goto Llen;
                        }
                        else if (id == Id.TIME) {
                            t.value = TOK.TOKstring;
                            //! t.ustring = time;
                            goto Llen;
                        }
                        else if (id == Id.TIMESTAMP) {
                            t.value = TOK.TOKstring;
                            //! t.ustring = timestamp;
Llen:
                            t.postfix = 0;
//                                        t.len = strlen((char *)t.ustring);
                        }
                    }
                    //printf("t.value = %d\n",t.value);
                    return;
                }

                // comments
            case '/':
                p++;
                switch (*p) {
                case '=':
                    p++;
                    t.value = TOK.TOKdivass;
                    return;

                case '*':                               // '/*'
                    p++;
                    linnum = loc.linnum;
                    while (true) {
                        while (true) {
                            ubyte c = *p;
                            switch (c) {
                            case '/':
                                break;

                            case '\n':
                                loc.linnum++;
                                p++;
                                continue;

                            case '\r':
                                p++;
                                if (*p != '\n') {
                                    loc.linnum++;
                                }
                                continue;

                            case 0:
                            case 0x1A:
                                error("unterminated /* */ comment");
                                p       = end;
                                t.value = TOK.TOKeof;
                                return;

                            default:
                                if (c & 0x80) {
                                    uint u = decodeUTF();
                                    if (u == PS || u == LS) {
                                        loc.linnum++;
                                    }
                                }
                                p++;
                                continue;
                            }
                            break;
                        }
                        p++;
                        if (p[-2] == '*' && p - 3 != t.ptr) {
                            break;
                        }
                    }

                    if (commentToken) {
                        t.value = TOK.TOKcomment;
                        return;
                    }
                    // if /** but not /**/
                    else  if (doDocComment && t.ptr[2] == '*' && p - 4 != t.ptr) {
                        getDocComment(t, lastLine == linnum);
                    }                                                                                   //! ?
                    continue;

                case '/':                                       // do // style comments
                    linnum = loc.linnum;
                    while (1) {
                        ubyte c = *++p;
                        switch (c) {
                        case '\n':
                            break;

                        case '\r':
                            if (p[1] == '\n') {
                                p++;
                            }
                            break;

                        case 0:
                        case 0x1a:
                            if (commentToken) {
                                p       = end;
                                t.value = TOK.TOKcomment;
                                return;
                            }
                            if (doDocComment && t.ptr[2] == '/') {
                                getDocComment(t, lastLine == linnum);
                            }
                            p       = end;
                            t.value = TOK.TOKeof;
                            return;

                        default:
                            if (c & 0x80) {
                                uint u = decodeUTF();
                                if (u == PS || u == LS) {
                                    break;
                                }
                            }
                            continue;
                        }
                        break;
                    }

                    if (commentToken) {
                        p++;
                        loc.linnum++;
                        t.value = TOK.TOKcomment;
                        return;
                    }
                    if (doDocComment && t.ptr[2] == '/') {
                        getDocComment(t, lastLine == linnum);
                    }

                    p++;
                    loc.linnum++;
                    continue;

                case '+':
                    { int nest;
                      linnum = loc.linnum;
                      p++;
                      nest = 1;
                      while (1) {
                          ubyte c = *p;
                          switch (c) {
                          case '/':
                              p++;
                              if (*p == '+') {
                                  p++;
                                  nest++;
                              }
                              continue;

                          case '+':
                              p++;
                              if (*p == '/') {
                                  p++;
                                  if (--nest == 0) {
                                      break;
                                  }
                              }
                              continue;

                          case '\r':
                              p++;
                              if (*p != '\n') {
                                  loc.linnum++;
                              }
                              continue;

                          case '\n':
                              loc.linnum++;
                              p++;
                              continue;

                          case 0:
                          case 0x1A:
                              error("unterminated /+ +/ comment");
                              p       = end;
                              t.value = TOK.TOKeof;
                              return;

                          default:
                              if (c & 0x80) {
                                  uint u = decodeUTF();
                                  if (u == PS || u == LS) {
                                      loc.linnum++;
                                  }
                              }
                              p++;
                              continue;
                          }
                          break;
                      }
                      if (commentToken) {
                          t.value = TOK.TOKcomment;
                          return;
                      }
                      if (doDocComment && t.ptr[2] == '+' && p - 4 != t.ptr) {
                          // if /++ but not /++/
                          getDocComment(t, lastLine == linnum);
                      }
                      continue; }

                default:
                    break;
                }
                t.value = TOK.TOKdiv;
                return;

            case '.':
                p++;
                if (isdigit(*p)) {
                    p--;
                    t.value = inreal(t);
                }
                else if (p[0] == '.') {
                    if (p[1] == '.') {
                        p      += 2;
                        t.value = TOK.TOKdotdotdot;
                    }
                    else {
                        p++;
                        t.value = TOK.TOKslice;
                    }
                }
                else {
                    t.value = TOK.TOKdot;
                }
                return;

            case '&':
                p++;
                if (*p == '=') {
                    p++;
                    t.value = TOK.TOKandass;
                }
                else if (*p == '&') {
                    p++;
                    t.value = TOK.TOKandand;
                }
                else {
                    t.value = TOK.TOKand;
                }
                return;

                // |, ||, |=
            case '|':
                p++;
                if (*p == '=') {
                    p++;
                    t.value = TOK.TOKorass;
                }
                else if (*p == '|') {
                    p++;
                    t.value = TOK.TOKoror;
                }
                else {
                    t.value = TOK.TOKor;
                }
                return;

            case '-':
                p++;
                if (*p == '=') {
                    p++;
                    t.value = TOK.TOKminass;
                }
                else if (*p == '-') {
                    p++;
                    t.value = TOK.TOKminusminus;
                }
                else {
                    t.value = TOK.TOKmin;
                }
                return;

                // +, +=, ++
            case '+':
                p++;
                if (*p == '=') {
                    p++;
                    t.value = TOK.TOKaddass;                                    // +=
                }
                else if (*p == '+') {
                    p++;
                    t.value = TOK.TOKplusplus;                                          // ++
                }
                else {
                    t.value = TOK.TOKadd;
                }                                                                               // +
                return;

                // <, <=, <<=, <<, <>=, <>
            case '<':
                p++;
                if (*p == '=') {
                    p++;
                    t.value = TOK.TOKle;                                                // <=
                }
                else if (*p == '<') {
                    p++;
                    if (*p == '=') {
                        p++;
                        t.value = TOK.TOKshlass;                                        // <<=
                    }
                    else {
                        t.value = TOK.TOKshl;
                    }                                                                   // <<
                }
                else if (*p == '>') {
                    p++;
                    if (*p == '=') {
                        p++;
                        t.value = TOK.TOKleg;                                           // <>=
                    }
                    else {
                        t.value = TOK.TOKlg;
                    }                                                                   // <>
                }
                else {
                    t.value = TOK.TOKlt;
                }                                                                       // <
                return;

                // >, >>, >>>, >=, >>=, >>>=
            case '>':
                p++;
                if (*p == '=') {
                    p++;
                    t.value = TOK.TOKge;                                                // >=
                }
                else if (*p == '>') {
                    p++;
                    if (*p == '=') {
                        p++;
                        t.value = TOK.TOKshrass;                                        // >>=
                    }
                    else if (*p == '>') {
                        p++;
                        if (*p == '=') {
                            p++;
                            t.value = TOK.TOKushrass;                           // >>>=
                        }
                        else {
                            t.value = TOK.TOKushr;
                        }                                                               // >>>
                    }
                    else {
                        t.value = TOK.TOKshr;
                    }                                                                   // >>
                }
                else {
                    t.value = TOK.TOKgt;
                }                                                                       // >
                return;

            case '!':
                p++;
                if (*p == '=') {
                    p++;
                    if (*p == '=') {
                        p++;
                        t.value = TOK.TOKnotidentity;                                   // !==
                    }
                    else {
                        t.value = TOK.TOKnotequal;
                    }                                                                   // !=
                }
                else if (*p == '<') {
                    p++;
                    if (*p == '>') {
                        p++;
                        if (*p == '=') {
                            p++;
                            t.value = TOK.TOKunord;                             // !<>=
                        }
                        else {
                            t.value = TOK.TOKue;
                        }                                                       // !<>
                    }
                    else if (*p == '=') {
                        p++;
                        t.value = TOK.TOKug;                                    // !<=
                    }
                    else {
                        t.value = TOK.TOKuge;
                    }                                                           // !<
                }
                else if (*p == '>') {
                    p++;
                    if (*p == '=') {
                        p++;
                        t.value = TOK.TOKul;                                    // !>=
                    }
                    else {
                        t.value = TOK.TOKule;
                    }                                                           // !>
                }
                else {
                    t.value = TOK.TOKnot;
                }                                                               // !
                return;

            case '=':
                p++;
                if (*p == '=') {
                    p++;
                    if (*p == '=') {
                        p++;
                        t.value = TOK.TOKidentity;                                      // ===
                    }
                    else {
                        t.value = TOK.TOKequal;
                    }                                                                   // ==
                }
                else {
                    t.value = TOK.TOKassign;
                }                                                               // =
                return;

            case '~':
                p++;
                if (*p == '=') {
                    p++;
                    t.value = TOK.TOKcatass;                                            // ~=
                }
                else {
                    t.value = TOK.TOKtilde;
                }                                                                       // ~
                return;

                // SINGLE
            case '(': p++; t.value = TOK.TOKlparen;      return;

            case ')': p++; t.value = TOK.TOKrparen;      return;

            case '[': p++; t.value = TOK.TOKlbracket;    return;

            case ']': p++; t.value = TOK.TOKrbracket;    return;

            case '{': p++; t.value = TOK.TOKlcurly;      return;

            case '}': p++; t.value = TOK.TOKrcurly;      return;

            case '?': p++; t.value = TOK.TOKquestion;    return;

            case ',': p++; t.value = TOK.TOKcomma;               return;

            case ';': p++; t.value = TOK.TOKsemicolon;   return;

            case ':': p++; t.value = TOK.TOKcolon;               return;

            case '$': p++; t.value = TOK.TOKdollar;      return;

                // DOUBLE
            case '*': p++; if (*p == '=') {
                    p++; t.value = TOK.TOKmulass;
            }
                else { t.value = TOK.TOKmul; } return;

            case '%': p++; if (*p == '=') {
                    p++; t.value = TOK.TOKmodass;
            }
                else { t.value = TOK.TOKmod; } return;

            case '^': p++; if (*p == '=') {
                    p++; t.value = TOK.TOKxorass;
            }
                else { t.value = TOK.TOKxor; } return;

// removed 148    case '~': p++; if( *p == '=' ) { p++; t.value = TOK.TOKcatass; } else t.value = TOK.TOKtilde; return;


            case '#':
                p++;
                Pragma();
                continue;

            default:
                {
                    debug writefln("    default char");
                    ubyte c = *p;
                    if (c & 0x80) {
                        uint u = decodeUTF();
                        // Check for start of unicode identifier
                        if (isUniAlpha(u)) {
                            goto case_ident;
                        }

                        if (u == PS || u == LS) {
                            loc.linnum++;
                            p++;
                            continue;
                        }
                    }
                    if (isprint(c)) {
                        error("unsupported char '%s'", cast(char)c);
                    }
                    else {
                        error("unsupported char 0x%02x", cast(ubyte)c);
                    }
                    p++;
                    continue;
                }
            }
        }
    }



    // Parse escape sequence.
01021     uint escapeSequence() {
        uint c;
        int  n;
        int  ndigits;

        c = *p;
        switch (c) {
        case '\'':
        case '"':
        case '?':
        case '\\':
Lconsume:
            p++;
            break;

        case 'a':        c = 7;          goto Lconsume;

        case 'b':        c = 8;          goto Lconsume;

        case 'f':        c = 12;         goto Lconsume;

        case 'n':        c = 10;         goto Lconsume;

        case 'r':        c = 13;         goto Lconsume;

        case 't':        c = 9;          goto Lconsume;

        case 'v':        c = 11;         goto Lconsume;

        case 'u':
            ndigits = 4;
            goto Lhex;

        case 'U':
            ndigits = 8;
            goto Lhex;

        case 'x':
            ndigits = 2;
Lhex:
            p++;
            c = *p;
            if (ishex(c)) {
                uint v;
                n = 0;
                v = 0;
                while (1) {
                    if (isdigit(c)) {
                        c -= '0';
                    }
                    else if (islower(c)) {
                        c -= 'a' - 10;
                    }
                    else {
                        c -= 'A' - 10;
                    }
                    v = v * 16 + c;
                    c = *++p;
                    if (++n == ndigits) {
                        break;
                    }
                    if (!ishex(c)) {
                        error("escape hex sequence has %d hex digits instead of %d", n, ndigits);
                        break;
                    }
                }
//!                         if( ndigits != 2 && !utf_isValidDchar(v))
//!                                 error("invalid UTF character \\U%08x", v);
                c = v;
            }
            else {
                error("undefined escape hex sequence \\%s\n", c);
            }
            break;

        case '&':                                       // named character entity
            for (ubyte * idstart = ++p; 1; p++) {
                switch (*p) {
                case ';':
                    //!!!
                    /+
                     * c = HtmlNamedEntity(idstart, p - idstart);
                     * if( c == ~0 )
                     * {
                     *      error("unnamed character entity &%.*s;", p - idstart, idstart);
                     *      c = ' ';
                     * }
                     *
                     * p++;
                     +/
                    break;

                default:
                    if (isalpha(*p) || (p != idstart + 1 && isdigit(*p))) {
                        continue;
                    }
                    error("unterminated named entity");
                    break;
                }
                break;
            }
            break;

        case 0:
        case 0x1a:                                      // end of file
            c = '\\';
            break;

        default:
            if (isoctal(c)) {
                ubyte v;
                n = 0;
                do {
                    v = v * 8 + (c - '0');
                    c = *++p;
                } while (++n < 3 && isoctal(c));
                c = v;
            }
            else {
                error("undefined escape sequence \\%s\n", c);
            }
            break;
        }
        return(c);
    }

    /**************************************
     */

    TOK wysiwygStringConstant(Token * t, int tc) {
        uint c;
        Loc  start = loc;

        p++;
        stringbuffer.offset = 0;
        while (1) {
            c = *p++;
            switch (c) {
            case '\n':
                loc.linnum++;
                break;

            case '\r':
                if (*p == '\n') {
                    continue;
                }                                       // ignore
                c = '\n';                               // treat EndOfLine as \n character
                loc.linnum++;
                break;

            case 0:
            case 0x1a:
                error("unterminated string constant starting at %s", start.toChars());
                t.ustring = "";
                t.postfix = 0;
                return(TOK.TOKstring);

            case '"':
            case '`':
                if (c == tc) {
//                          t.len = stringbuffer.offset;
                    stringbuffer.write(cast(byte)0);
                    t.ustring = stringbuffer.toString;
//                          t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);
//                          memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);
                    stringPostfix(t);
                    return(TOK.TOKstring);
                }
                break;

            default:
                if (c & 0x80) {
                    p--;
                    uint u = decodeUTF();
                    p++;
                    if (u == PS || u == LS) {
                        loc.linnum++;
                    }
                    stringbuffer.write(u);
                    continue;
                }
                break;
            }
            stringbuffer.write(c);
        }
    }

    /**************************************
     * Lex hex strings:
     *      x"0A ae 34FE BD"
     */

    TOK hexStringConstant(Token * t) {
        uint c;
        Loc  start = loc;
        uint n     = 0;
        uint v;

        p++;
        stringbuffer.offset = 0;
        while (1) {
            c = *p++;
            switch (c) {
            case ' ':
            case '\t':
            case '\v':
            case '\f':
                continue;                                               // skip white space

            case '\r':
                if (*p == '\n') {
                    continue;
                }                                                       // ignore

                // Treat isolated '\r' as if it were a '\n'
            case '\n':
                loc.linnum++;
                continue;

            case 0:
            case 0x1a:
                error("unterminated string constant starting at %s", start.toChars());
                t.ustring = "";
                t.postfix = 0;
                return(TOK.TOKstring);

            case '"':
                if (n & 1) {
                    error("odd number (%d) of hex characters in hex string", n);
                    stringbuffer.write(v);
                }
//                      t.len = stringbuffer.offset;
//                      stringbuffer.write(cast(byte)0);
                t.ustring = stringbuffer.toString;
//                      t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);
//                      memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);
                stringPostfix(t);
                return(TOK.TOKstring);

            default:
                if (c >= '0' && c <= '9') {
                    c -= '0';
                }
                else if (c >= 'a' && c <= 'f') {
                    c -= 'a' - 10;
                }
                else if (c >= 'A' && c <= 'F') {
                    c -= 'A' - 10;
                }
                else if (c & 0x80) {
                    p--;
                    uint u = decodeUTF();
                    p++;
                    if (u == PS || u == LS) {
                        loc.linnum++;
                    }
                    else {
                        error("non-hex character \\u%x", u);
                    }
                }
                else {
                    error("non-hex character '%s'", c);
                }
                if (n & 1) {
                    v = (v << 4) | c;
                    stringbuffer.write(v);
                }
                else {
                    v = c;
                }
                n++;
                break;
            }
        }
    }

    /**************************************
     */

    TOK escapeStringConstant(Token * t, int wide) {
        uint c;
        Loc  start = loc;

        p++;
        stringbuffer.offset = 0;
        //    debug writefln( "escape string constant: %s", std.string.toString( cast(char*)p ) );
        while (1) {
            c = *p++;
            switch (c) {
            case '\\':
                switch (*p) {
                case 'u':
                case 'U':
                case '&':
                    c = escapeSequence();
                    stringbuffer.write(c);
                    continue;

                default:
                    c = escapeSequence();
                    break;
                }
                break;

            case '\n':
                loc.linnum++;
                break;

            case '\r':
                if (*p == '\n') {
                    continue;
                }                                       // ignore
                c = '\n';                               // treat EndOfLine as \n character
                loc.linnum++;
                break;

            case '"':
//                      writefln( "end of string: ", stringbuffer.toString );
                t.ustring = stringbuffer.toString().dup;
                //                        t.len = stringbuffer.offset;
                //                        stringbuffer.write(cast(byte)0);
                //                        t.ustring = (ubyte *)mem.malloc(stringbuffer.offset);
                //                        memcpy(t.ustring, stringbuffer.data, stringbuffer.offset);
                stringPostfix(t);

                return(TOK.TOKstring);

            case 0:
            case 0x1a:
                p--;
                error("unterminated string constant starting at %s", start.toChars());
                t.ustring = "";
//                            t.len = 0;
                t.postfix = 0;
                return(TOK.TOKstring);

            default:
                if (c & 0x80) {
                    p--;
                    c = decodeUTF();
                    if (c == LS || c == PS) {
                        c = '\n';
                        loc.linnum++;
                    }
                    p++;
                    stringbuffer.write(cast(char)c);
                    continue;
                }
                break;
            }
            stringbuffer.write(cast(char)c);
//                writefln( stringbuffer.toString );
        }
    }

    //**************************************
    TOK charConstant(Token * t, int wide) {
        uint c;
        TOK  tk = TOK.TOKcharv;

        //printf("Lexer.charConstant\n");
        p++;
        c = *p++;
        switch (c) {
        case '\\':
            switch (*p) {
            case 'u':
                t.uns64value = escapeSequence();
                tk           = TOK.TOKwcharv;
                break;

            case 'U':
            case '&':
                t.uns64value = escapeSequence();
                tk           = TOK.TOKdcharv;
                break;

            default:
                t.uns64value = escapeSequence();
                break;
            }
            break;

        case '\n':
L1:
            loc.linnum++;

        case '\r':
        case 0:
        case 0x1a:
        case '\'':
            error("unterminated character constant");
            return(tk);

        default:
            if (c & 0x80) {
                p--;
                c = decodeUTF();
                p++;
                if (c == LS || c == PS) {
                    goto L1;
                }
                if (c < 0xd800 || (c >= 0xe000 && c < 0xfffe)) {
                    tk = TOK.TOKwcharv;
                }
                else {
                    tk = TOK.TOKdcharv;
                }
            }
            t.uns64value = c;
            break;
        }

        if (*p != '\'') {
            error("unterminated character constant");
            return(tk);
        }
        p++;
        return(tk);
    }

    // Get postfix of string literal.
    void stringPostfix(Token * t) {
        switch (*p) {
        case 'c':
        case 'w':
        case 'd':
            t.postfix = *p;
            p++;
            break;

        default:
            t.postfix = 0;
            break;
        }
    }

    /***************************************
     * Read \u or \U unicode sequence
     * Input:
     *      u     'u' or 'U'
     */
    /*
     * uint Wchar(uint u)
     * {
     *  uint value;
     *  uint n;
     *  ubyte c;
     *  uint nchars;
     *
     *  nchars = (u == 'U') ? 8 : 4;
     *  value = 0;
     *  for (n = 0; 1; n++)
     *  {
     ++p;
     *              if( n == nchars)
     *                  break;
     *              c = *p;
     *              if( !ishex(c))
     *              {
     *                      error("\\%s sequence must be followed by %d hex characters", u, nchars);
     *                  break;
     *              }
     *              if( isdigit(c))
     *                  c -= '0';
     *              else if( islower(c))
     *                  c -= 'a' - 10;
     *              else
     *                  c -= 'A' - 10;
     *              value <<= 4;
     *              value |= c;
     *  }
     *  return value;
     * }
     */

    /**************************************
     * Read in a number.
     * If it's an integer, store it in tok.TKutok.Vlong.
     *      integers can be decimal, octal or hex
     *      Handle the suffixes U, UL, LU, L, etc.
     * If it's double, store it in tok.TKutok.Vdouble.
     * Returns:
     *      TKnum
     *      TKdouble,...
     */

01508     TOK number(Token * t) {
        //debug writefln("Lexer.number()");
        // We use a state machine to collect numbers
        enum STATE {
            STATE_initial,
            STATE_0,
            STATE_decimal,
            STATE_octal,
            STATE_octale,
            STATE_hex,
            STATE_binary,
            STATE_hex0,
            STATE_binary0,
            STATE_hexh,
            STATE_error
        }

        enum FLAGS {
            FLAGS_decimal = 1,                          // decimal
            FLAGS_unsigned = 2,                         // u or U suffix
            FLAGS_long     = 4,                         // l or L suffix
        }
        FLAGS flags = FLAGS.FLAGS_decimal;

        int   i;
        TOK   result;
        int   base;
        stringbuffer.offset = 0;
//          stringbuffer.data = null;
        STATE state   = STATE.STATE_initial;
        ubyte * start = p;

        TOK _isreal() {
            p = start;
            return(inreal(t));
        }

        while (true) {
            char c = cast(char)*p;
            switch (state) {
            case STATE.STATE_initial:                           // opening state
                if (c == '0') {
                    state = STATE.STATE_0;
                }
                else {
                    state = STATE.STATE_decimal;
                }
                break;

            case STATE.STATE_0:
                flags = cast(FLAGS)(flags & ~FLAGS.FLAGS_decimal);
                switch (c) {
                    //  #if ZEROH
//                                case 'H':                 // 0h
//                                case 'h':
//                                        goto hexh;
                    //  #endif
                case 'X':
                case 'x':
                    state = STATE.STATE_hex0;
                    break;

                case '.':
                    if (p[1] == '.') {                                  // .. is a separate token
                        goto done;
                    }

                case 'i':
                case 'f':
                case 'F':
                    goto _Real;

                    //  #if ZEROH
//                                case 'E':
//                                case 'e':
//                                        goto case_hex;
                    //  #endif
                case 'B':
                case 'b':
                    state = STATE.STATE_binary0;
                    break;

                case '0':
                case '1':
                case '2':
                case '3':
                case '4':
                case '5':
                case '6':
                case '7':
                    state = STATE.STATE_octal;
                    break;

                    //  #if ZEROH
//                                case '8': case '9': case 'A':
//                                case 'C': case 'D': case 'F':
//                                case 'a': case 'c': case 'd': case 'f':
//                                      case_hex:
//                                        state = STATE.STATE_hexh;
//                                        break;
                    //  #endif
                case '_':
                    state = STATE.STATE_octal;
                    p++;
                    continue;

                default:
                    goto done;
                }
                break;

            case STATE.STATE_decimal:                           // reading decimal number

                // if its not a digit - decimal complete or not a decimal
                if (!isdigit(c)) {
//                                  debug writefln( "\tnon-digit( %s )", c );
                    //  #if ZEROH
//                                if( ishex(c) || c == 'H' || c == 'h' )
//                                        goto hexh;
                    //  #endif
                    //! wtf ?
                    // ignore embedded _
                    if (c == '_') {
                        p++;
                        continue;
                    }

                    // check decimal point - make real
                    if (c == '.' && p[1] != '.') {
                        goto _Real;
                    }

                    // check for mantra - make real
                    if (c == 'i' || c == 'f' || c == 'F' || c == 'e' || c == 'E') {
_Real:                                                  // It's a real number. Back up and rescan as a real
                        p = start;
                        return(inreal(t));
                    }

                    goto done;
                }
                break;

            case STATE.STATE_hex0:                              // reading hex number
            case STATE.STATE_hex:
                if (!ishex(c)) {
                    if (c == '_') {                                     // ignore embedded _
                        p++;
                        continue;
                    }
                    if (c == '.' && p[1] != '.') {
                        goto _Real;
                    }
                    if (c == 'P' || c == 'p' || c == 'i') {
                        goto _Real;
                    }
                    if (state == STATE.STATE_hex0) {
                        error("Hex digit expected, not '%s'", c);
                    }
                    goto done;
                }
                state = STATE.STATE_hex;
                break;

                //      #if ZEROH
//                          hexh:
//                                  state = STATE.STATE_hexh;
//
//                          case STATE.STATE_hexh:          // parse numbers like 0FFh
//                                  if( !ishex(c))
//                                  {
//                                      if( c == 'H' || c == 'h')
//                                      {
//                                              p++;
//                                              base = 16;
//                                              goto done;
//                                      }
//                                      else
//                                      {
//                                              // Check for something like 1E3 or 0E24
//                                              if( memchr(stringbuffer.data.ptr, 'E', stringbuffer.offset) || memchr( stringbuffer.data.ptr, 'e', stringbuffer.offset))
//                                                  goto _Real;
//                                              error("Hex digit expected, not '%s'", c);
//                                              goto done;
//                                      }
//                                  }
//                                  break;
                //            #endif

            case STATE.STATE_octal:                                             // reading octal number
            case STATE.STATE_octale:                                            // reading octal number with non-octal digits
                if (!isoctal(c)) {
//                      #if ZEROH
//                                      if( ishex(c) || c == 'H' || c == 'h' )
//                                              goto hexh;
//                      #endif
                    if (c == '_') {                                             // ignore embedded _
                        p++;
                        continue;
                    }
                    if (c == '.' && p[1] != '.') {
                        goto _Real;
                    }
                    if (c == 'i') {
                        goto _Real;
                    }
                    if (isdigit(c)) {
                        state = STATE.STATE_octale;
                    }
                    else {
                        goto done;
                    }
                }
                break;

            case STATE.STATE_binary0:                                   // starting binary number
            case STATE.STATE_binary:                                    // reading binary number
                if (c != '0' && c != '1') {
                    //              #if ZEROH
//                                      if( ishex(c) || c == 'H' || c == 'h' )
//                                              goto hexh;
                    //              #endif
                    if (c == '_') {                                             // ignore embedded _
                        p++;
                        continue;
                    }
                    if (state == STATE.STATE_binary0) {
                        error("binary digit expected");
                        state = STATE.STATE_error;
                        break;
                    }
                    else {
                        goto done;
                    }
                }
                state = STATE.STATE_binary;
                break;

            case STATE.STATE_error:                                     // for error recovery
                if (!isdigit(c)) {                                      // scan until non-digit
                    goto done;
                }
                break;

            default:
                assert(0);
            }
            stringbuffer.write(cast(char)c);
            p++;
        }
done:
        stringbuffer.write(cast(char)0);                // terminate string

//          debug writefln( "\tdigit complete( %s )", stringbuffer.toString );

        if (state == STATE.STATE_octale) {
            error("Octal digit expected");
        }

        uinteger_t n;                           // unsigned >=64 bit integer type

        if (stringbuffer.offset == 2 && (state == STATE.STATE_decimal || state == STATE.STATE_0)) {
            n = stringbuffer.data[0] - '0';
        }
        else {
            // Convert string to integer
            char * p = cast(char *)stringbuffer.data.ptr;
            int  r   = 10;
            int  d;
            if (*p == '0') {
                if (p[1] == 'x' || p[1] == 'X') {
                    // "0x#"
                    p += 2;
                    r  = 16;
                }
                else if (p[1] == 'b' || p[1] == 'B') {
                    // "0b#" - binary
                    p += 2;
                    r  = 2;
                }
                else if (isdigit(p[1])) {
                    p += 1;
                    r  = 8;
                }
            }

            n = 0;

            while (true) {
                if (*p >= '0' && *p <= '9') {
                    d = *p - '0';
                }
                else if (*p >= 'a' && *p <= 'z') {
                    d = *p - 'a' + 10;
                }
                else if (*p >= 'A' && *p <= 'Z') {
                    d = *p - 'A' + 10;
                }
                else {
                    break;
                }

                if (d >= r) {
                    break;
                }

                if (n * r + d < n) {
                    error("integer overflow");
                    break;
                }

                n = n * r + d;
                p++;
            }

            // if n needs more than 64 bits
            if (n.sizeof > 8 && n > 0xffffffffffffffffL) {
                error("integer overflow");
            }
        }

        // Parse trailing 'u', 'U', 'l' or 'L' in any combination
        while (true) {
            ubyte f;
            switch (*p) {
            case 'U':
            case 'u':
                f = FLAGS.FLAGS_unsigned;
                goto L1;

            case 'L':
            case 'l':
                f = FLAGS.FLAGS_long;
L1:
                p++;
                if (flags & f) {
                    error("unrecognized token");
                }
                flags = cast(FLAGS)(flags | f);
                continue;

            default:
                break;
            }
            break;
        }

        switch (flags) {
        case 0:
            /* Octal or Hexadecimal constant.
             * First that fits: int, uint, long, ulong
             */
            if (n & 0x8000000000000000L) {
                result = TOK.TOKuns64v;
            }
            else if (n & 0xffffffff00000000L) {
                result = TOK.TOKint64v;
            }
            else if (n & 0x80000000) {
                result = TOK.TOKuns32v;
            }
            else {
                result = TOK.TOKint32v;
            }
            break;

        case FLAGS.FLAGS_decimal:
            /* First that fits: int, long, long long
             */
            if (n & 0x8000000000000000L) {
                error("signed integer overflow");
                result = TOK.TOKuns64v;
            }
            else if (n & 0xffffffff80000000L) {
                result = TOK.TOKint64v;
            }
            else {
                result = TOK.TOKint32v;
            }
            break;

        case FLAGS.FLAGS_unsigned:
        case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned:
            /* First that fits: uint, ulong
             */
            if (n & 0xffffffff00000000L) {
                result = TOK.TOKuns64v;
            }
            else {
                result = TOK.TOKuns32v;
            }
            break;

        case FLAGS.FLAGS_decimal | FLAGS.FLAGS_long:
            if (n & 0x8000000000000000L) {
                error("signed integer overflow");
                result = TOK.TOKuns64v;
            }
            else {
                result = TOK.TOKint64v;
            }
            break;

        case FLAGS.FLAGS_long:
            if (n & 0x8000000000000000L) {
                result = TOK.TOKuns64v;
            }
            else {
                result = TOK.TOKint64v;
            }
            break;

        case FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long:
        case FLAGS.FLAGS_decimal | FLAGS.FLAGS_unsigned | FLAGS.FLAGS_long:
            result = TOK.TOKuns64v;
            break;

        default:
            debug writefln("%x", flags);
            assert(0);
        }
        t.uns64value = n;
        return(result);
    }

    /**************************************
     * Read in characters, converting them to real.
     * Bugs:
     *      Exponent overflow not detected.
     *      Too much requested precision is not detected.
     */

    TOK inreal(Token * t) {
        int  dblstate;
        uint c;
        char hex;                       // is this a hexadecimal-floating-constant?
        TOK  result;

        //printf("Lexer.inreal()\n");
        stringbuffer.offset = 0;
        dblstate            = 0;
        hex                 = 0;
Lnext:
        while (1) {
            // Get next char from input
            c = *p++;
            //printf("dblstate = %d, c = '%s'\n", dblstate, c);
            while (1) {
                switch (dblstate) {
                case 0:                                 // opening state
                    if (c == '0') {
                        dblstate = 9;
                    }
                    else if (c == '.') {
                        dblstate = 3;
                    }
                    else {
                        dblstate = 1;
                    }
                    break;

                case 9:
                    dblstate = 1;
                    if (c == 'X' || c == 'x') {
                        hex++;
                        break;
                    }

                case 1:                                 // digits to left of .
                case 3:                                 // digits to right of .
                case 7:                                 // continuing exponent digits
                    if (!isdigit(c) && !(hex && isxdigit(c))) {
                        if (c == '_') {
                            goto Lnext;
                        }                                       // ignore embedded '_'
                        dblstate++;
                        continue;
                    }
                    break;

                case 2:                                 // no more digits to left of .
                    if (c == '.') {
                        dblstate++;
                        break;
                    }

                case 4:                                 // no more digits to right of .
                    if ((c == 'E' || c == 'e') || hex && (c == 'P' || c == 'p')) {
                        dblstate = 5;
                        hex      = 0;                           // exponent is always decimal
                        break;
                    }
                    if (hex) {
                        error("binary-exponent-part required");
                    }
                    goto done;

                case 5:                                 // looking immediately to right of E
                    dblstate++;
                    if (c == '-' || c == '+') {
                        break;
                    }

                case 6:                                 // 1st exponent digit expected
                    if (!isdigit(c)) {
                        error("exponent expected");
                    }
                    dblstate++;
                    break;

                case 8:                                 // past end of exponent digits
                    goto done;
                }
                break;
            }
            stringbuffer.write(c);
        }
done:
        p--;

        stringbuffer.write(cast(byte)0);

//    #if _WIN32 && __DMC__
        char * save = __locale_decpoint;
        __locale_decpoint = ".";
//    #endif
        t.float80value = strtold(cast(char *)stringbuffer.data.ptr, null);
        errno          = 0;
        switch (*p) {
        case 'F':
        case 'f':
            strtof(cast(char *)stringbuffer.data.ptr, null);
            result = TOK.TOKfloat32v;
            p++;
            break;

        default:
            strtod(cast(char *)stringbuffer.data.ptr, null);
            result = TOK.TOKfloat64v;
            break;

        case 'L':
        case 'l':
            result = TOK.TOKfloat80v;
            p++;
            break;
        }
        if (*p == 'i' || *p == 'I') {
            p++;
            switch (result) {
            case TOK.TOKfloat32v:
                result = TOK.TOKimaginary32v;
                break;

            case TOK.TOKfloat64v:
                result = TOK.TOKimaginary64v;
                break;

            case TOK.TOKfloat80v:
                result = TOK.TOKimaginary80v;
                break;
            }
        }
//    #if _WIN32 && __DMC__
        __locale_decpoint = save;
//    #endif
        if (errno == ERANGE) {
            error("number is not representable");
        }
        return(result);
    }




    /*********************************************
     * Do pragma.
     * Currently, the only pragma supported is:
     *      #line linnum [filespec]
     */

02089     void Pragma() {
        Token  tok;
        int    linnum;

        char[] filespec;
        Loc    loc = this.loc;

        scan(&tok);

        if (tok.value != TOK.TOKidentifier || tok.ident != Id.line) {
            goto Lerr;
        }

        scan(&tok);
        if (tok.value == TOK.TOKint32v || tok.value == TOK.TOKint64v) {
            linnum = tok.uns64value - 1;
        }
        else {
            goto Lerr;
        }

        while (1) {
            switch (*p) {
            case 0:
            case 0x1a:
            case '\n':
Lnewline:
                this.loc.linnum = linnum;
                if (filespec.length) {
                    this.loc.filename = filespec;
                }
                return;

            case '\r':
                p++;
                if (*p != '\n') {
                    p--;
                    goto Lnewline;
                }
                continue;

            case ' ':
            case '\t':
            case '\v':
            case '\f':
                p++;
                continue;                                               // skip white space

            case '_':
                if (mod && memcmp(p, cast(char *)"__FILE__", 8) == 0) {
                    p += 8;
//!                               filespec = mem.strdup(loc.filename ? loc.filename : mod.ident.toChars());
                }
                continue;

            case '"':
                if (filespec) {
                    goto Lerr;
                }
                stringbuffer.offset = 0;
                p++;
                while (1) {
                    uint c;
                    c = *p;
                    switch (c) {
                    case '\n':
                    case '\r':
                    case 0:
                    case 0x1a:
                        goto Lerr;

                    case '"':
                        stringbuffer.write(cast(byte)0);
                        //                                      filespec = mem.strdup((char *)stringbuffer.data);
                        filespec = stringbuffer.toString.dup;
                        p++;
                        break;

                    default:
                        if (c & 0x80) {
                            uint u = decodeUTF();
                            if (u == PS || u == LS) {
                                goto Lerr;
                            }
                        }
                        stringbuffer.write(c);
                        p++;
                        continue;
                    }
                    break;
                }
                continue;

            default:
                if (*p & 0x80) {
                    uint u = decodeUTF();
                    if (u == PS || u == LS) {
                        goto Lnewline;
                    }
                }
                goto Lerr;
            }
        }

Lerr:
        errorLoc(loc, "#line integer [\"filespec\"]\\n expected");
    }



    /***************************************************
     * Parse doc comment embedded between t.ptr and p.
     * Remove trailing blanks and tabs from lines.
     * Replace all newlines with \n.
     * Remove leading comment character from each line.
     * Decide if it's a lineComment or a blockComment.
     * Append to previous one for this token.
     */

    void getDocComment(Token * t, uint lineComment) {
        auto OutBuffer buf       = new OutBuffer;
        ubyte          ct        = t.ptr[2];
        ubyte          * q       = t.ptr + 3; // start of comment text
        int            linestart = 0;

        ubyte          * qend = p;

        if (ct == '*' || ct == '+') {
            qend -= 2;
        }

        // Scan over initial row of ****'s or ++++'s or ////'s
        for (; q < qend; q++) {
            if (*q != ct) {
                break;
            }
        }

        // Remove trailing row of ****'s or ++++'s
        if (ct != '/') {
            for (; q < qend; qend--) {
                if (qend[-1] != ct) {
                    break;
                }
            }
        }

        for (; q < qend; q++) {
            ubyte c = *q;

            switch (c) {
            case '*':
            case '+':
                if (linestart && c == ct) {
                    linestart = 0;
                    // Trim preceding whitespace up to preceding \n
                    while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t')) {
                        buf.offset--;
                    }
                    continue;
                }
                break;

            case ' ':
            case '\t':
                break;

            case '\r':
                if (q[1] == '\n') {
                    continue;
                }                                               // skip the \r
                goto Lnewline;

            default:
                if (c == 226) {
                    // If LS or PS
                    if (q[1] == 128 &&
                        (q[2] == 168 || q[2] == 169)) {
                        q += 2;
                        goto Lnewline;
                    }
                }
                linestart = 0;
                break;

Lnewline:
                c = '\n';                       // replace all newlines with \n

            case '\n':
                linestart = 1;

                // Trim trailing whitespace
                while (buf.offset && (buf.data[buf.offset - 1] == ' ' || buf.data[buf.offset - 1] == '\t')) {
                    buf.offset--;
                }

                break;
            }
            buf.write(c);
        }

        // Always end with a newline
        if (!buf.offset || buf.data[buf.offset - 1] != '\n') {
            buf.writenl();
        }

        //buf.write(cast(char)0);

        // It's a line comment if the start of the doc comment comes
        // after other non-whitespace on the same line.
//        ubyte** dc = (lineComment && anyToken)
//                       ? &t.lineComment
//                       : &t.blockComment;

        char[]         dc = (lineComment && anyToken) ? t.lineComment : t.blockComment;

        // Combine with previous doc comment, if any
        if (dc.length) {
            dc = combineComments(dc, buf.toString().dup);
        }
        else {
            dc = buf.toString().dup;
        }

//          writefln( dc );

        if (lineComment && anyToken) {
            t.lineComment = dc;
        }
        else {
            t.blockComment = dc;
        }
    }
}

// character maps
static ubyte[256] cmtable;

const int CMoctal  = 0x1;
const int CMhex    = 0x2;
const int CMidchar = 0x4;

ubyte isoctal(ubyte c) {
    return(cmtable[c] & CMoctal);
}
ubyte ishex(ubyte c) {
    return(cmtable[c] & CMhex);
}
ubyte isidchar(ubyte c) {
    return(cmtable[c] & CMidchar);
}

static void cmtable_init() {
    for (uint c = 0; c < cmtable.length; c++) {
        if ('0' <= c && c <= '7') {
            cmtable[c] |= CMoctal;
        }
        if (isdigit(c) || ('a' <= c && c <= 'f') || ('A' <= c && c <= 'F')) {
            cmtable[c] |= CMhex;
        }
        if (isalnum(c) || c == '_') {
            cmtable[c] |= CMidchar;
        }
    }
}


/+
 * struct StringValue
 * {
 *  union
 *  {
 *              int intvalue;
 *              void *ptrvalue;
 *              dchar *string;
 *  }
 *
 *  char[] lstring;
 * }
 #define CASE_BASIC_TYPES
 *      case TOKwchar: case TOKdchar:
 *      case TOKbit: case TOKbool: case TOKchar:
 *      case TOKint8: case TOKuns8:
 *      case TOKint16: case TOKuns16:
 *      case TOKint32: case TOKuns32:
 *      case TOKint64: case TOKuns64:
 *      case TOKfloat32: case TOKfloat64: case TOKfloat80:
 *      case TOKimaginary32: case TOKimaginary64: case TOKimaginary80:
 *      case TOKcomplex32: case TOKcomplex64: case TOKcomplex80:
 *      case TOKvoid:
 *
 #define CASE_BASIC_TYPES_X(t)                              \
 *      case TOKvoid:    t = Type::tvoid;  goto LabelX;           \
 *      case TOKint8:    t = Type::tint8;  goto LabelX;           \
 *      case TOKuns8:    t = Type::tuns8;  goto LabelX;           \
 *      case TOKint16:   t = Type::tint16; goto LabelX;           \
 *      case TOKuns16:   t = Type::tuns16; goto LabelX;           \
 *      case TOKint32:   t = Type::tint32; goto LabelX;           \
 *      case TOKuns32:   t = Type::tuns32; goto LabelX;           \
 *      case TOKint64:   t = Type::tint64; goto LabelX;           \
 *      case TOKuns64:   t = Type::tuns64; goto LabelX;           \
 *      case TOKfloat32: t = Type::tfloat32; goto LabelX;   \
 *      case TOKfloat64: t = Type::tfloat64; goto LabelX;   \
 *      case TOKfloat80: t = Type::tfloat80; goto LabelX;   \
 *      case TOKimaginary32: t = Type::timaginary32; goto LabelX; \
 *      case TOKimaginary64: t = Type::timaginary64; goto LabelX; \
 *      case TOKimaginary80: t = Type::timaginary80; goto LabelX; \
 *      case TOKcomplex32: t = Type::tcomplex32; goto LabelX;     \
 *      case TOKcomplex64: t = Type::tcomplex64; goto LabelX;     \
 *      case TOKcomplex80: t = Type::tcomplex80; goto LabelX;     \
 *      case TOKbit:     t = Type::tbit;     goto LabelX;   \
 *      case TOKchar:    t = Type::tchar;    goto LabelX;   \
 *      case TOKwchar:   t = Type::twchar; goto LabelX;     \
 *      case TOKdchar:   t = Type::tdchar; goto LabelX;     \
 *      LabelX
 +/

Generated by  Doxygen 1.6.0   Back to index