lexer.d.ts 1.3 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758
  1. import { Token } from './ast';
  2. import { Source } from './source';
  3. import { TokenKindEnum } from './tokenKind';
  4. /**
  5. * Given a Source object, this returns a Lexer for that source.
  6. * A Lexer is a stateful stream generator in that every time
  7. * it is advanced, it returns the next token in the Source. Assuming the
  8. * source lexes, the final Token emitted by the lexer will be of kind
  9. * EOF, after which the lexer will repeatedly return the same EOF token
  10. * whenever called.
  11. */
  12. export class Lexer {
  13. source: Source;
  14. /**
  15. * The previously focused non-ignored token.
  16. */
  17. lastToken: Token;
  18. /**
  19. * The currently focused non-ignored token.
  20. */
  21. token: Token;
  22. /**
  23. * The (1-indexed) line containing the current token.
  24. */
  25. line: number;
  26. /**
  27. * The character offset at which the current line begins.
  28. */
  29. lineStart: number;
  30. constructor(source: Source);
  31. /**
  32. * Advances the token stream to the next non-ignored token.
  33. */
  34. advance(): Token;
  35. /**
  36. * Looks ahead and returns the next non-ignored token, but does not change
  37. * the state of Lexer.
  38. */
  39. lookahead(): Token;
  40. }
  41. /**
  42. * @internal
  43. */
  44. export function isPunctuatorToken(token: Token): boolean;
  45. /**
  46. * @internal
  47. */
  48. export function isPunctuatorTokenKind(kind: TokenKindEnum): boolean;