lexer.d.ts 1.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960
  1. import { syntaxError } from '../error';
  2. import { Token } from './ast';
  3. import { Source } from './source';
  4. /**
  5. * Given a Source object, this returns a Lexer for that source.
  6. * A Lexer is a stateful stream generator in that every time
  7. * it is advanced, it returns the next token in the Source. Assuming the
  8. * source lexes, the final Token emitted by the lexer will be of kind
  9. * EOF, after which the lexer will repeatedly return the same EOF token
  10. * whenever called.
  11. */
  12. export function createLexer<TOptions>(
  13. source: Source,
  14. options: TOptions,
  15. ): Lexer<TOptions>;
  16. /**
  17. * The return type of createLexer.
  18. */
  19. export interface Lexer<TOptions> {
  20. source: Source;
  21. options: TOptions;
  22. /**
  23. * The previously focused non-ignored token.
  24. */
  25. lastToken: Token;
  26. /**
  27. * The currently focused non-ignored token.
  28. */
  29. token: Token;
  30. /**
  31. * The (1-indexed) line containing the current token.
  32. */
  33. line: number;
  34. /**
  35. * The character offset at which the current line begins.
  36. */
  37. lineStart: number;
  38. /**
  39. * Advances the token stream to the next non-ignored token.
  40. */
  41. advance(): Token;
  42. /**
  43. * Looks ahead and returns the next non-ignored token, but does not change
  44. * the Lexer's state.
  45. */
  46. lookahead(): Token;
  47. }
  48. /**
  49. * @internal
  50. */
  51. export function isPunctuatorToken(token: Token): boolean;