// lexer.cpp
// Lexical analyser implementation
// F.J. Alberti
// Created: 03/05/2001
// Last Modified: 28/05/2001
//
// Modification history:
// FA     28/05/2001      Support for kModeScript mode
// FA     05/06/2001      Correct getChar() when the character read has value 0xff
// FA     12/06/2001      Define MSGxxx constants for error messages and warnings
// FA     03/07/2001      Integrate simple cipher algorithm (defined in cipher.cpp)
//                        Comment #ifdef SERVER/#endif
// FA     13/07/2001      Add the utility functions a2f() and f2a()
// FA     17/07/2001      Move Token class implementation to separate file token.cpp
// FA     17/07/2001      Include codec.h, where McodecCall() has been declared
// FA     20/07/2001      a2i() extended to allow optional negative sign
// FA     26/07/2001      a2i() and a2f() extended to allow optional positive sign
// FA     27/07/2001      a2f() extended to allow optional positive sign in exponent 
//                        (E-notation). Write specialised version a2i10()
// FA     23/10/2001      Bug correction: kEOI and kNewline were not assigned to the
//                        proper token kind (either _look or _current) in Lexer::advance()
// FA     18/03/2002      Implement new discardLine() method, as a replacement of 
//                        eatUpTo(Token::kNewline)

//$ LB (03/06/2002)       Mopenpack function : ClubHouse loads only the crypted packages

//$JI - julia.interactive@wanadoo.fr
//$JI     16/11/2005      New functions: f2a5d(), f2a4d(), f2a3d(), f2a2d() & f2a1d()

//$BLG - http://www.kyrien.com
//$BLG    21/11/2005      Added BLG_MopenpackS() to fix the _loadS() Scol function
//$BLG    05/01/2006      Removed the functions added by $JI - using f2a() in fixed Scol functions


#include "cipher.h"         // => PackageLoader file
#include "codec.h"          // => PackageLoader file (declaration of McodecCall())
#include "compiler/lexer.h"
#include "scolMMemory.h"
#include <math.h>
#include <stdio.h>
#include <string.h>

//$ LB (03/06/2002) : for the ClubHouse Mopenpack
#include "common/vscol.h"
extern "C" {
#include "common/common.h"
}




//
// Utilities
//

bool isDigit(int c, uint base)
{
  if (c >= '0' && c <= '9')
    return (int)base > (c-'0');
  if (c >= 'A' && c <= 'F')
    return (int)base > (c-'A')+10;
  if (c >= 'a' && c <= 'f')
    return (int)base > (c-'a')+10;
  return false;
}


bool isBaseSpecifier(int c)
{
  return c == 'b' || c == 'B' 
      || c == 'o' || c == 'O' 
      || c == 'x' || c == 'X';
}

bool isSpecial(int c)
{
  return c == '+' || c == '-' || c == '*' || c == '/' || c == '^' || c == '='
      || c == '<' || c == '>' || c == '!' || c == '.' || c == '&' || c == '|'
      || c == '~' || c == ':' || c == '(' || c == ')' || c == '{' || c == '}'
      || c == '[' || c == ']' || c == ',' || c == '@' || c == ';';
}


bool a2f(const char* s, float* res)
// Converts a string into a float. The function returns true
// if the *whole* string codes a legal SCOL float; it returns
// false otherwise. In either case, 'res' contains the
// float value corresponding to the part of the string that
// was recongnised
{
  *res = 0.0;
  if (!s || *s == '\0')
    return false;
  int sgn = 1;
  if (*s == '-' || *s == '+') {
    sgn = (*s == '-') ? -1 : 1;
    s++;
  }
  double n = 0.0;
  while (isdigit(*s)) {
    n = n*10+(*s-'0');
    s++;
  }
#if defined(SCOL_STRICT)
  if (*s == '.') {
#else
  // The french decimal separator is supported for backward
  // compatibility
  if (*s == '.' || *s == ',') {
#endif
    s++;
    double d = 1.0;
    while (isdigit(*s)) {
      n  = n*10+(*s-'0');
      d *= 10;
      s++;
    }
    n = n/d;
    if (n != 0.0 && (*s == 'E' || *s == 'e')) {
      s++;
      bool inv = false;
      if (*s == '-' || *s == '+') {
        inv = (*s == '-');
        s++;
      }
      int e = 0;
      while (isdigit(*s)) {
        e = e*10+(*s-'0');
        s++;
      }
      n = inv ? n/pow((float)10, (int)e) : n*pow((float)10, (int)e);
    }
  }
  *res = (float)sgn*n;
  return *s == '\0';
}


bool f2a(float f, char* res)
// Converts a float into string. The function returns true if
// the conversion succeeds; false otherwise. The result is
// written in 'res', that should be sufficiently large to
// hold the result
{
  // Use %g when we shall allow exponents to have an explicit '+' sign
  return sprintf(res, "%f", (double)f) > 0;
}


//$JI - Start - v4.6a4 - New type F display functions
//$BLG - v4.6a4 - Removed these functions, finally using f2a() in fixed Scol functions
/*
bool f2a5d(float f, char* res)
{
  return sprintf(res, "%.5f", (double)f) > 0;
}
bool f2a4d(float f, char* res)
{
  return sprintf(res, "%.4f", (double)f) > 0;
}
bool f2a3d(float f, char* res)
{
  return sprintf(res, "%.3f", (double)f) > 0;
}
bool f2a2d(float f, char* res)
{
  return sprintf(res, "%.2f", (double)f) > 0;
}
bool f2a1d(float f, char* res)
{
  return sprintf(res, "%.1f", (double)f) > 0;
}
*/
//$JI - End


bool h2i(const char* s, int* res)
// Converts a string into an integer. The strings is expected
// to contain a non-empty sequence of hexadecimal digits
{
  *res = 0;
  if (!s || *s == '\0')
    return false;
  int n = 0;
  while (*s) {
    int d;
    if (isdigit(*s))
      d = *s-'0';
    else if (*s >= 'A' && *s <= 'F')
      d = 10+(*s-'A');
    else if (*s >= 'a' && *s <= 'f')
      d = 10+(*s-'a');
    else break;
    n = (n<<4)+d;
    s++;
  }
  *res = n;
  return *s == '\0';
}


bool i2h(int i, char* res)
// Converts an integer into an hex string. The function returns 
// true if the conversion succeeds; false otherwise. The result
// is written in 'res', that should be sufficiently large to
// hold the result
{
  return sprintf(res, "%x", i) > 0;
}


bool a2i(const char* s, int* res)
// Converts a string into an integer. The string is expected
// to encode a SCOL integer literal (with an optional base
// base specifier). The function returns true if the *whole*
// string could be successfully converted; false otherwise.
// In either case, 'res' contains the integer value 
// corresponding to the part of the string that was recongnised
{
  *res = 0;
  if (!s || *s == '\0')
    return false;
  int sgn = 1;
  if (*s == '-' || *s == '+') {
    sgn = (*s == '-') ? -1 : 1;
    s++;
  }
  int base = 10;
  if (*s++ == '0' && isBaseSpecifier(*s)) {
    switch (*s++) {
      case 'b':
      case 'B':
        base = 2;
        break;
        
      case 'o':
      case 'O':
        base = 8;
        break;
        
      case 'x':
      case 'X':
        base = 16;
        break;
    }
  } else s--;
  int n = 0;
  while (isDigit(*s, base)) {
    int d;
    if (isdigit(*s))
      d = *s-'0';
    else if (*s >= 'A' && *s <= 'F')
      d = (*s-'A')+10;
    else if (*s >= 'a' && *s <= 'f')
      d = (*s-'a')+10;
    n = n*base+d;
    s++;
  }
  *res = sgn*n;
  return *s == '\0';
}


bool a2i10(const char* s, int* res)
// Specialised version of a2i() for integers in base 10
{
  *res = 0;
  if (!s || *s == '\0')
    return false;
  int sgn = 1;
  if (*s == '-' || *s == '+') {
    sgn = (*s == '-') ? -1 : 1;
    s++;
  }
  int n = 0;
  while (isDigit(*s)) {
    n = n*10+(*s-'0');
    s++;
  }
  *res = sgn*n;
  return *s == '\0';
}


bool i2a(int i, char* res)
// Converts an integer into a string
{
  return sprintf(res, "%d", i) > 0;
}



//
// Lexer methods
//


// Internal errors
#define MSGIPUTBACKSTACKFULL  "Lexer: Character putback stack full!"
#define MSGIREADPASTEND       "Lexer: Cannot read past end of input"
// SCOL Transitional warnings
#define MSGWILLCHAR           "Illegal character '\\%02x' in input ignored"
// General errors
#define MSGEILLCHAR           "Illegal character '\\%02x'"
#define MSGEEOFINCOMMENT      "'*/' expected before end of input"
#define MSGEEOFINSTRING       "'\"' expected before end of input"
#define MSGESTRINGTOOLARGE    "String literal too large"
#define MSGECTLCHAR           "Non-control character expected after \"'\" in character literal"
#define MSGELITERALTOOLARGE   "Literal constant too large"
#define MSGENOEXPONENT        "Exponent expected"
#define MSGEIDENTTOOLARGE     "Identifier too large"
#define MSGEINPUTENCRYPTED    "File is encrypted!"



Lexer::Lexer(const char* source)
  : _source   (source), 
    _mode     (kModePlain),
    _pos      (TextPosition(0, 0)),
    _eoi      (false), 
    _unread   (kMaxPutback), 
    _err      (kErrOk),
    _lineOffs (0),
    _istest    (false)
{
}


Lexer::~Lexer()
{
}

void Lexer::setTestMode(bool mode)
{
  _istest = mode;
}

bool Lexer::getTestMode()
{
  return _istest;
}

void Lexer::redirect(const char* source)
{
  _source = source;
}

const Token& Lexer::next()
{
  advance();
  return _current;
}


int Lexer::getChar()
{
  int c;
  
  // Return characters that were putback with putChar()
  if (_unread.empty()) {
    if ((c = (unsigned char)_source[_pos.off]) == '\0')
      c = kEof;
  } else 
    c = _unread.pop();

  _pos.off++;
  if (c == kEof)
    _eoi = true;
  else if (c == '\n')
    _pos.line++;

  return c;
}


void Lexer::putChar(int c)
{
#if defined(SCOL_DEBUG)
  if (_unread.full()) {
    MMechostr(MSKRUNTIME, "(!) "MSGIPUTBACKSTACKFULL"\n");
    return;
  }
#endif
  if (c == kEof)
    _eoi = false;
  else {
    _pos.off--;
    if (c == '\n')
      _pos.line--;
  }
  _unread.push(c);
}


void Lexer::rewind()
{
  // Should verify any errors here
  _mode = kModePlain;
  _pos  = TextPosition(0, 0);
  _eoi  = false;
  _err  = kErrOk;
  _look.nullify();
  _current.nullify();
  _unread.dropAll();
}


void Lexer::discardLine()
{
  int c;
  while ((c = getChar()) != kEof && c != '\n')
    ;
  _err = kErrOk;
  _look.nullify();
  _current.nullify();
}


const Token& Lexer::look()
{
  if (_look.null())
    advance(_look);
  return _look;
}


void Lexer::skipWhites()
{
  int st    = 0;
  int depth = 0;
  int c;
  
  while (st >= 0 && st != 6) {
    c = getChar();
    switch (st) {
      case 0:
        if (c == '/')
          st = 1;
        // 28/05/2001: Newlines are significant inside scripts
        else if (_mode == kModeScript && c == '\n') {  
          putChar(c);
          st = 6;    // ok
        } else if (!isWhite(c)) {
          if (is3xWhite(c)) {
#if defined(SCOL_STRICT)
            MMechostr(MSKRUNTIME, "(!) "MSGEILLCHAR"\n", c);
            _err = kErrIllegalCharacter;
            return;
#else
            MMechostr(MSKWARNING, "(!) "MSGWILLCHAR"\n", c);
            break;
#endif
          }
          putChar(c);
          st = 6;    // ok
        }
        break;

      case 1:
        if (c == '*') {
          depth++;
          st = 2;
        } else if (c == '/')
          st = 5;
        else {
          putChar(c);
          putChar('/');
          st = 6;    // ok
        }
        break;
        
      case 2:
        if (c == kEof)
          st = -1;  // error
        else if (c == '/')
          st = 3;
        else if (c == '*')
          st = 4;
        break;
        
      case 3:
        if (c == kEof)
          st = -1;  // error
        else if (c != '/') {
          st = 2;
          if (c == '*')
            depth++;
        }
        break;
        
      case 4:
        if (c == kEof)
          st = -1;  // error
        else if (c != '*')
          st = (c == '/' && --depth == 0) ? 0 : 2;
        break;

      case 5:
        if (c == kEof || c == '\n') {
          putChar(c);
          st = 0;
        }
        break;
    }
  }
  
  if (st < 0) {
    _err = kErrCommentUnclosed;
    MMechostr(MSKRUNTIME, "(!) "MSGEEOFINCOMMENT"\n");
  }
}


void Lexer::advance()
{
  if (!_look.null()) {
    _current = _look;
    _look.nullify();
    return;
  }
  advance(_current);
}


void Lexer::readString(Token& tok)
// Reads a SCOL string literal
//
// <string-literal> ::= '"' {<character>}* '"'
{
  int c;
  int i;
 
  if ((c = getChar()) == '"') {
    tok.append(c);
    for (i = 1; i < kMaxStringLiteralLength && (c = getChar()) != '"'; i++) {
      if (c == '\\') {
        // Interpret backslash (escape) sequences
        if ((c = getChar()) == kEof) {
          MMechostr(MSKRUNTIME, "(!) "MSGEEOFINSTRING"\n");
          _err = kErrIllegalToken;
          return;
        } else 
        if (c != '\n') {
          tok.append('\\');
          tok.append(c);
          i++;
        }
      } else 
      if (c == kEof) {
        MMechostr(MSKRUNTIME, "(!) "MSGEEOFINSTRING"\n");
        _err = kErrIllegalToken;
        return;
      } else 
        tok.append(c);
    } // for
    
    if (i == kMaxStringLiteralLength) {
      MMechostr(MSKRUNTIME, "(!) "MSGESTRINGTOOLARGE"\n");
      _err = kErrIllegalLength;
    } else {
      tok.append(c);
      tok._kind = Token::kString;
    }
  } else
    putChar(c);
}


void Lexer::readCharacter(Token& tok)
// Read a SCOL character literal
//
// <character-literal> ::= '\'' <non-control character>
{
  int c;

  if ((c = getChar()) == '\'') {
    tok.append(c);
    if ((c = getChar()) <= ' ' || c == kEof) {  // control characters are disallowed
      MMechostr(MSKRUNTIME, "(!) "MSGECTLCHAR"\n");
      _err = kErrIllegalToken;
    } else {
      tok.append(c);      
      tok._kind = Token::kChar;
    } 
  } else
    putChar(c);
}


void Lexer::readNumber(Token& tok)
// Reads a SCOL integer or float literal
//
// <integer-literal> ::= [<base prefix>] {<digit>}+
//
// <base prefix>     ::= '0' <base specifier>
//
// <base specifier>  ::= 'b'|'B'|'o'|'O'|'x'|'X'
//
// <digit>           ::= a digit according to given base
//
// <float-literal>   ::= {<digit>}+ '.' {<digit>}+ [('E'|'e') ['-'] {<digit>+}]
{
  int  c;
  // 28/05/2001: Default base is 16 inside scripts 
  uint base = (_mode == kModeScript) ? 16 : 10;
  int  len  = kMaxTokenLength;
  int  i;
  
  if ((c = getChar()) == '0') {
    tok.append(c);
    len--;
    if (isBaseSpecifier(c = getChar())) {
      tok.append(c);
      len--;
      switch (c) {
        case 'b':
        case 'B':
          base = 2;
          break;
        
        case 'o':
        case 'O':
          base = 8;
          break;
        
        case 'x':
        case 'X':
          base = 16;
          break;
      } // switch
    } else putChar(c);
  } else putChar(c);

  for (i = 0; len-- && isDigit(c = getChar(), base); i++)
    tok.append(c);
    
  if (len <= 0) {
    _err = kErrIllegalLength;
    MMechostr(MSKRUNTIME, "(!) "MSGELITERALTOOLARGE"\n");
  } else 
  if (c == '.' && base == 10 && _mode != kModeDotAddressing) {
    // Recognise a SCOL float literal
    tok.append(c);
    for (i = 0; len-- && isDigit(c = getChar()); i++)
      tok.append(c);
    if (len <= 0) {
      _err = kErrIllegalLength;
      MMechostr(MSKRUNTIME, "(!) "MSGELITERALTOOLARGE"\n");
    } else if (c == 'E' || c == 'e') {
      tok.append(c);
      if ((c = getChar()) == '-') {
        tok.append(c);
        c = getChar();
      }
      if (isDigit(c)) {
        tok.append(c);
        for (i = 0; len-- && isDigit(c = getChar()); i++)
          tok.append(c);
        if (len <= 0) {
          _err = kErrIllegalLength;
          MMechostr(MSKRUNTIME, "(!) "MSGELITERALTOOLARGE"\n");
        } else {
          putChar(c);
          tok._kind = Token::kFloat;
        }
      } else {
        putChar(c);
        MMechostr(MSKRUNTIME, "(!) "MSGENOEXPONENT"\n");
        _err = kErrIllegalToken;
      }
    } else {
      putChar(c);
      tok._kind = Token::kFloat;
    }
  } else {
    putChar(c);
    tok._kind = Token::kInt;
  }
}


void Lexer::readIdentifier(Token& tok)
// Reads a SCOL identifier (non-qualified) and interpret keywords
//
// <identifier> ::= <letter> {<letter>|<digit>}
{
  int c;
  int i;
  
  if (isLetter(c = getChar())) {
    tok.append(c);
    for (i = 1; i < kMaxTokenLength && (isLetter(c = getChar()) || isDigit(c)); i++)
      tok.append(c);
    if (i == kMaxTokenLength) {
      _err = kErrIllegalLength;
      MMechostr(MSKRUNTIME, "(!) "MSGEIDENTTOOLARGE"\n");
    } else {
      putChar(c);
      tok._kind = Token::kIdent;
    }
  } else 
    putChar(c);
}


void Lexer::readOperator(Token& tok)
// Reads a SCOL operator or punctuation
{
  int c;
  
  if (isSpecial(c = getChar())) {
    tok.append(c);
    switch (c) {
      case '+':
        if ((c = getChar()) == '.') {
          tok.append(c);
          tok._kind = Token::kFAdd;                    // '+.'
        } else {
          putChar(c);
          tok._kind = Token::kIAdd;                    // '+'
        }
        break;

      case '-':
        if ((c = getChar()) == '.' || c == '>') {
          tok.append(c);
          if (c == '.')
            tok._kind = Token::kFSub;                  // '-.'
          else
            tok._kind = Token::kGoesTo;                // '->'
        } else {
          putChar(c);
          tok._kind = Token::kISub;                    // '-'
        }
        break;

      case '*':
        if ((c = getChar()) == '.') {
          tok.append(c);
          tok._kind = Token::kFMul;                    // '*.'
        } else {
          putChar(c);
          tok._kind = Token::kIMul;                    // '*'
        }
        break;
       
      case '/':
        if ((c = getChar()) == '.') {
          tok.append(c);
          tok._kind = Token::kFDiv;                    // '/.'
        } else {
          putChar(c); 
          tok._kind = Token::kIDiv;                    // '/'
        }
        break;
      
      case '^':
        if ((c = getChar()) == '^') {
          tok.append(c);
          tok._kind = Token::kIPow;                    // '^^'
        } else {
          putChar(c); 
          tok._kind = Token::kIXor;                    // '^'
        }
        break;
      
      case '<':
        if ((c = getChar()) == '<' || c == '.' || c == '=' || c == '-') {
          tok.append(c);
          switch (c) {
            case '<':
              tok._kind = Token::kIShl;                // '<<'
              break;
            case '.':
              tok._kind = Token::kFLt;                 // '<.'
              break;
            case '=':
              if ((c = getChar()) == '.') {
                tok.append(c);
                tok._kind = Token::kFLe;               // '<=.'
              } else {
                putChar(c);
                tok._kind = Token::kILe;               // '<='
              }
              break;
            case '-':
              tok._kind = Token::kTakes;               // '<-'
              break;
          } // switch
        } else {
          putChar(c);
          tok._kind = Token::kILt;                     // '<'
        }
        break;
      
      case '>':
        if ((c = getChar()) == '>' || c == '.' || c == '=') {
          tok.append(c);
          switch (c) {
            case '>':
              tok._kind = Token::kIShr;                // '>>'
              break;
            case '.':
              tok._kind = Token::kFGt;                 // '>.'
              break;
            case '=':
              if ((c = getChar()) == '.') {
                tok.append(c);
                tok._kind = Token::kFGe;               // '>=.'
              } else {
                putChar(c);
                tok._kind = Token::kIGe;               // '>='
              }
              break;
          }
        } else {
          putChar(c);
          tok._kind = Token::kIGt;                     // '>'
        }
        break;
      
      case '=':
        if ((c = getChar()) == '=' || c == '.') {
          tok.append(c);
          if (c == '=')
            tok._kind = Token::kEq;                    // '=='
          else
            tok._kind = Token::kFEq;                   // '=.'
        } else {
          putChar(c);
          tok._kind = Token::kLet;                     // '='
        }
        break;

      case '!':
        if ((c = getChar()) == '=') {
          tok.append(c);
          if ((c = getChar()) == '.') {
            tok.append(c);
            tok._kind = Token::kFNe;                   // '!=.'
          } else {
            putChar(c);
            tok._kind = Token::kNe;                    // '!='
          }
        } else {
          putChar(c);
          tok._kind = Token::kLNot;                    // '!'
        }
        break;
 

      case '&':
        if ((c = getChar()) == '&') {
          tok.append(c);    
          tok._kind = Token::kLAnd;                    // '&&'
        } else {
          putChar(c);
          tok._kind = Token::kIAnd;                    // '&'
        }
        break;

      case '|':
        if ((c = getChar()) == '|') {
          tok.append(c);    
          tok._kind = Token::kLOr;                     // '||'
        } else {
          putChar(c);
          tok._kind = Token::kIOr;                     // '|'
        }
        break;
      
      case '~':
        tok._kind = Token::kINot;                      // '~'
        break;

      case '.':
        tok._kind = Token::kDot;                       // '.'  
        break;
      
      case ':':
        if ((c = getChar()) == ':') {
          tok.append(c);    
          tok._kind = Token::kCons;                    // '::'
        } else {
          putChar(c);
          tok._kind = Token::kColon;                   // ':'
        }
        break;
      
      case ';':
        if ((c = getChar()) == ';') {
          tok.append(c); 
          tok._kind = Token::kEnds;                    // ';;'
        } else {
          putChar(c);
          tok._kind = Token::kSeq;                     // ';'
        }
        break;

      case ',':
        tok._kind = Token::kComma;                     // ','
        break;
      
      case '(':
        tok._kind = Token::kLParen;                    // '('
        break;

      case ')':
        tok._kind = Token::kRParen;                    // ')'
        break;

      case '{':
        tok._kind = Token::kLCurly;                    // '{'
        break;

      case '}':
        tok._kind = Token::kRCurly;                    // '}'
        break;

      case '[':
        tok._kind = Token::kLBrack;                    // '['
        break;

      case ']':
        tok._kind = Token::kRBrack;                    // ']'
        break;

      case '@':
        tok._kind = Token::kAtSign;                    // '@'
        break;
     } // switch
  } else 
    putChar(c);
}


void Lexer::advance(Token& tok)
// The lexer itself
{
#if defined(SCOL_DEBUG)
  if (_eoi) {
    _err = kErrReadPastEnd;
    MMechostr(MSKRUNTIME, "(!) "MSGIREADPASTEND"\n");
    return;
  }
#endif

  // Ignore leading whites
  skipWhites();
  if (!ok())
    return;

  tok.nullify();
  tok._span.beg.line = _pos.line;
  tok._span.beg.off  = _pos.off;  

  int c = getChar();
  if (c == kEof)
    tok._kind = Token::kEOI;
  else 
  // 28/05/2001: Newlines are tokens inside scripts
  if (c == '\n') {
    tok.append(c);  // needed, as it is only detected
    tok._kind = Token::kNewline;
  } else   
  if (isLetter(c)) {
    // Read an identifier
    putChar(c);
    readIdentifier(tok);
  } else 
  if (isDigit(c)) {
    // Read an integer or float literal
    putChar(c);
    readNumber(tok);
  } else 
  if (c == '"') {
    // Read a string literal
    putChar(c);
    readString(tok);
  } else 
  if (c == '\'') {
    // Read a character literal
    putChar(c);
    readCharacter(tok);
  } else
  if (isSpecial(c)) {
    // Read an operator or punctuation
    putChar(c);
    readOperator(tok);
  } else {
    // Don't know what it is
    _err = kErrIllegalCharacter;
    MMechostr(MSKRUNTIME, "(!) "MSGEILLCHAR"\n", c);
  }

  tok._span.end.line = _pos.line;
  tok._span.end.off  = _pos.off-1;

  //MMechostr(MSKTRACE, "token = '%s' (%d) at (%d,%d)-(%d,%d)\n", tok._lexeme, tok._kind,
  //  tok._span.beg.line, tok._span.beg.off, tok._span.end.line, tok._span.end.off);
}


void Lexer::getPosition(const Token& tok, char* dst)
{
  // Calculate line offsets (if not already cached)
  if (_lineOffs == 0) {
    const uint kGrowFactor = 500;
    if ((_lineOffs = new uint[kGrowFactor]) == 0)
      //=> throw Exception();
      return;
    // First line is always at index 0
    uint line = 0;
    _lineOffs[line++] = 0;
    uint size = kGrowFactor;
    for (uint off = 0; _source[off]; off++) {
      if (_source[off] == '\n' && _source[off+1]) {
        // Expand offsets array if capacity is exceeded
        if (line == size) {
          uint* offs = new uint[size+kGrowFactor];
          if (offs == 0)
            //=> throw Exception();
            return;
          memcpy(offs, _lineOffs, size*sizeof(uint));
          delete [] _lineOffs;
          size += kGrowFactor;
          _lineOffs = offs;
        }
        _lineOffs[line++] = off+1;
      }
    } // for
  }

  // Print line where token occurs. The lexer starts counting lines at 0.
  TextSpan span = tok.span();
  sprintf(dst, "%s(!) Line #%d:\n", dst, span.beg.line+1);
  // Underline portion of token that was recognised
  uint i = 0;
  for (uint off = _lineOffs[span.beg.line]; _source[off] && _source[off] != '\n'; off++) {
    if (off == span.beg.off)
       sprintf(dst, "%s??", dst);
    sprintf(dst, "%s%c", dst,_source[off]);
  } // for
	sprintf(dst, "%s\n", dst);
}


void Lexer::printPosition(const Token& tok)
{
  // Calculate line offsets (if not already cached)
  if (_lineOffs == 0) {
    const uint kGrowFactor = 500;
    if ((_lineOffs = new uint[kGrowFactor]) == 0)
      //=> throw Exception();
      return;
    // First line is always at index 0
    uint line = 0;
    _lineOffs[line++] = 0;
    uint size = kGrowFactor;
    for (uint off = 0; _source[off]; off++) {
      if (_source[off] == '\n' && _source[off+1]) {
        // Expand offsets array if capacity is exceeded
        if (line == size) {
          uint* offs = new uint[size+kGrowFactor];
          if (offs == 0)
            //=> throw Exception();
            return;
          memcpy(offs, _lineOffs, size*sizeof(uint));
          delete [] _lineOffs;
          size += kGrowFactor;
          _lineOffs = offs;
        }
        _lineOffs[line++] = off+1;
      }
    } // for
  }

  // Print line where token occurs. The lexer starts counting lines at 0.
  TextSpan span = tok.span();
  MMechostr(MSKRUNTIME, "(!) Line #%d:\n", span.beg.line+1);
  // Underline portion of token that was recognised
  uint i = 0;
  for (uint off = _lineOffs[span.beg.line]; _source[off] && _source[off] != '\n'; off++) {
    if (off == span.beg.off)
      MMechostr(MSKRUNTIME, "??");
    MMechostr(MSKRUNTIME, "%c", _source[off]);
  } // for
  MMechostr(MSKRUNTIME, "\n");
}


void Lexer::printSource(const TextSpan& span)
{
  // We suppose span is a correct text selection
  for (uint off = span.beg.off; _source[off] && off <= span.end.off; off++)
    MMechostr(MSKTRACE, "%c", _source[off]);
}



//
// Implementation of old functions
//
int Mopenpack(manlyz z, char *filename)
{
  FILE *fd;
  int len;


  if ((fd = fopen(filename, "rb")) == NULL)
    return MANLZNF;
    
  fseek(fd, 0, SEEK_END);
  len = ftell(fd);
  fseek(fd, 0, SEEK_SET);
  if ((z->extbuf = new char[len+1]) == 0) 
  {
    fclose(fd);
    return MANLZNF; // really, a memory full error
  }
  
  fread(z->extbuf, len, 1, fd);
  fclose(fd);
  z->extbuf[len] = z->buf[2] = 0;

//#ifdef SERVER
  // Before feeding the lexer with the package source, apply any registered
  // codecs. (buf[2] should be set to 1 if source was crypted to indicate
  // that the source should not be shown if any errors occur.)
	if (McodecCall(&(z->extbuf), &len))
    z->buf[2] = 1;
//#endif



	if (len >= 4 && getint(z->extbuf) == 0x04030201) 
	{
	  uncipher(z->extbuf, len);
		z->buf[2] = 1;
	} 
	

//$ LB (03/06/2002) : ClubHouse loads only the crypted packages
#ifdef CLUBHOUSE_SERVER


  else
  {
	z->extbuf[0] = '\0';
	len = 0;
	z->buf[2] = 1;
	EndScolMachine(-1);
  }


#else


  else
  // If first byte in source is zero, we suppose that source is garbled
  // Put this code inside future 'PackageLoader::load()' method
  if (z->extbuf && len > 0 && z->extbuf[0] == 0x00) 
  {
    int i;
    for (i = 2; i < len; i++)
      z->extbuf[i-2] = z->extbuf[i] - z->extbuf[i-1];
    z->extbuf[i-2] = '\0';
    //len -= 2;
    z->buf[2] = 1;
  }


#endif


  z->giveback = 0;
  z->lex.redirect(z->extbuf);
  return 0;
}


//$BLG: Same as Mopenpack but used with _loadS/SPloadS
int BLG_MopenpackS(manlyz z, char *pkgcode)
{
  int len;

  len = strlen(pkgcode);
  if ((z->extbuf = new char[len+1]) == 0)
    return MANLZNF; // really, a memory full error
  strncpy(z->extbuf, pkgcode, len);
  z->extbuf[len] = z->buf[2] = 0;
  
	if (McodecCall(&(z->extbuf), &len))
    z->buf[2] = 1;

	if (len >= 4 && getint(z->extbuf) == 0x04030201) 
	{
	  uncipher(z->extbuf, len);
		z->buf[2] = 1;
	} 
#ifdef CLUBHOUSE_SERVER
  else
  {
		z->extbuf[0] = '\0';
		len = 0;
		z->buf[2] = 1;
		EndScolMachine(-1);
  }
#else
  else
	  if ((z->extbuf) && (len > 0) && (z->extbuf[0] == 0x00)) 
	  {
	    int i;
	    for (i = 2; i < len; i++)
	      z->extbuf[i-2] = z->extbuf[i] - z->extbuf[i-1];
	    z->extbuf[i-2] = '\0';
	    z->buf[2] = 1;
	  }
#endif

  z->giveback = 0;
  z->lex.redirect(z->extbuf);
  return 0;
}



int Mopenstring(manlyz z, char *buf)
{
  z->extbuf = NULL;
  if (strlen(buf) >= MAXSIZEBUF-1) 
    return MANLZTOOLONG;
  strcpy(z->buf, buf);
  z->giveback = 0;
  z->lex.redirect(z->buf);
  return 0;
}


int Mclosepack(manlyz z)
{
  if (z->extbuf) 
    delete [] z->extbuf;
  z->extbuf = NULL;
  return 0;
}


int Mreadtok(manlyz z)
{
  if (z->giveback) {
    z->giveback = 0;
    return 0;
  }
  // Read next token
  z->tok[0] = '\0';
  z->lex.advance();
  if (!z->lex.ok())
    return -1;
  // Copy current token's lexeme to analyzer structure 'tok' field
  strcpy(z->tok, z->lex.current().lexeme());
  // Detect end of input
  if (z->lex.current().kind() == Token::kEOI)
    return MANLZEOF;
  return 0;
}


// Reset lexer
int Mreinitpack(manlyz z)
{
  z->lex.rewind();
  z->giveback = 0;
  return 0;
}


int Maffligne(manlyz z)
{
  if (z->buf[2]) {
    MMechostr(MSKRUNTIME,"(!) "MSGEINPUTENCRYPTED"\n");
	return 0;
  }
  if (z->extbuf != NULL)
    z->lex.printPosition();
  return 0;
}

//$BB get line
int Mgetligne(manlyz z, char* dst)
{
  if (z->buf[2]) {
    sprintf(dst,"%s(!) "MSGEINPUTENCRYPTED"\n", dst);
	return 0;
  }
  if (z->extbuf != NULL)
    z->lex.getPosition(dst);
  return 0;
}