123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690 |
- "use strict";
- Object.defineProperty(exports, "__esModule", {
- value: true
- });
- exports.isPunctuatorTokenKind = isPunctuatorTokenKind;
- exports.Lexer = void 0;
- var _syntaxError = require("../error/syntaxError.js");
- var _ast = require("./ast.js");
- var _tokenKind = require("./tokenKind.js");
- var _blockString = require("./blockString.js");
- /**
- * Given a Source object, creates a Lexer for that source.
- * A Lexer is a stateful stream generator in that every time
- * it is advanced, it returns the next token in the Source. Assuming the
- * source lexes, the final Token emitted by the lexer will be of kind
- * EOF, after which the lexer will repeatedly return the same EOF token
- * whenever called.
- */
- var Lexer = /*#__PURE__*/function () {
- /**
- * The previously focused non-ignored token.
- */
- /**
- * The currently focused non-ignored token.
- */
- /**
- * The (1-indexed) line containing the current token.
- */
- /**
- * The character offset at which the current line begins.
- */
- function Lexer(source) {
- var startOfFileToken = new _ast.Token(_tokenKind.TokenKind.SOF, 0, 0, 0, 0, null);
- this.source = source;
- this.lastToken = startOfFileToken;
- this.token = startOfFileToken;
- this.line = 1;
- this.lineStart = 0;
- }
- /**
- * Advances the token stream to the next non-ignored token.
- */
- var _proto = Lexer.prototype;
- _proto.advance = function advance() {
- this.lastToken = this.token;
- var token = this.token = this.lookahead();
- return token;
- }
- /**
- * Looks ahead and returns the next non-ignored token, but does not change
- * the state of Lexer.
- */
- ;
- _proto.lookahead = function lookahead() {
- var token = this.token;
- if (token.kind !== _tokenKind.TokenKind.EOF) {
- do {
- var _token$next;
- // Note: next is only mutable during parsing, so we cast to allow this.
- token = (_token$next = token.next) !== null && _token$next !== void 0 ? _token$next : token.next = readToken(this, token);
- } while (token.kind === _tokenKind.TokenKind.COMMENT);
- }
- return token;
- };
- return Lexer;
- }();
- /**
- * @internal
- */
- exports.Lexer = Lexer;
- function isPunctuatorTokenKind(kind) {
- return kind === _tokenKind.TokenKind.BANG || kind === _tokenKind.TokenKind.DOLLAR || kind === _tokenKind.TokenKind.AMP || kind === _tokenKind.TokenKind.PAREN_L || kind === _tokenKind.TokenKind.PAREN_R || kind === _tokenKind.TokenKind.SPREAD || kind === _tokenKind.TokenKind.COLON || kind === _tokenKind.TokenKind.EQUALS || kind === _tokenKind.TokenKind.AT || kind === _tokenKind.TokenKind.BRACKET_L || kind === _tokenKind.TokenKind.BRACKET_R || kind === _tokenKind.TokenKind.BRACE_L || kind === _tokenKind.TokenKind.PIPE || kind === _tokenKind.TokenKind.BRACE_R;
- }
- function printCharCode(code) {
- return (// NaN/undefined represents access beyond the end of the file.
- isNaN(code) ? _tokenKind.TokenKind.EOF : // Trust JSON for ASCII.
- code < 0x007f ? JSON.stringify(String.fromCharCode(code)) : // Otherwise print the escaped form.
- "\"\\u".concat(('00' + code.toString(16).toUpperCase()).slice(-4), "\"")
- );
- }
- /**
- * Gets the next token from the source starting at the given position.
- *
- * This skips over whitespace until it finds the next lexable token, then lexes
- * punctuators immediately or calls the appropriate helper function for more
- * complicated tokens.
- */
- function readToken(lexer, prev) {
- var source = lexer.source;
- var body = source.body;
- var bodyLength = body.length;
- var pos = prev.end;
- while (pos < bodyLength) {
- var code = body.charCodeAt(pos);
- var _line = lexer.line;
- var _col = 1 + pos - lexer.lineStart; // SourceCharacter
- switch (code) {
- case 0xfeff: // <BOM>
- case 9: // \t
- case 32: // <space>
- case 44:
- // ,
- ++pos;
- continue;
- case 10:
- // \n
- ++pos;
- ++lexer.line;
- lexer.lineStart = pos;
- continue;
- case 13:
- // \r
- if (body.charCodeAt(pos + 1) === 10) {
- pos += 2;
- } else {
- ++pos;
- }
- ++lexer.line;
- lexer.lineStart = pos;
- continue;
- case 33:
- // !
- return new _ast.Token(_tokenKind.TokenKind.BANG, pos, pos + 1, _line, _col, prev);
- case 35:
- // #
- return readComment(source, pos, _line, _col, prev);
- case 36:
- // $
- return new _ast.Token(_tokenKind.TokenKind.DOLLAR, pos, pos + 1, _line, _col, prev);
- case 38:
- // &
- return new _ast.Token(_tokenKind.TokenKind.AMP, pos, pos + 1, _line, _col, prev);
- case 40:
- // (
- return new _ast.Token(_tokenKind.TokenKind.PAREN_L, pos, pos + 1, _line, _col, prev);
- case 41:
- // )
- return new _ast.Token(_tokenKind.TokenKind.PAREN_R, pos, pos + 1, _line, _col, prev);
- case 46:
- // .
- if (body.charCodeAt(pos + 1) === 46 && body.charCodeAt(pos + 2) === 46) {
- return new _ast.Token(_tokenKind.TokenKind.SPREAD, pos, pos + 3, _line, _col, prev);
- }
- break;
- case 58:
- // :
- return new _ast.Token(_tokenKind.TokenKind.COLON, pos, pos + 1, _line, _col, prev);
- case 61:
- // =
- return new _ast.Token(_tokenKind.TokenKind.EQUALS, pos, pos + 1, _line, _col, prev);
- case 64:
- // @
- return new _ast.Token(_tokenKind.TokenKind.AT, pos, pos + 1, _line, _col, prev);
- case 91:
- // [
- return new _ast.Token(_tokenKind.TokenKind.BRACKET_L, pos, pos + 1, _line, _col, prev);
- case 93:
- // ]
- return new _ast.Token(_tokenKind.TokenKind.BRACKET_R, pos, pos + 1, _line, _col, prev);
- case 123:
- // {
- return new _ast.Token(_tokenKind.TokenKind.BRACE_L, pos, pos + 1, _line, _col, prev);
- case 124:
- // |
- return new _ast.Token(_tokenKind.TokenKind.PIPE, pos, pos + 1, _line, _col, prev);
- case 125:
- // }
- return new _ast.Token(_tokenKind.TokenKind.BRACE_R, pos, pos + 1, _line, _col, prev);
- case 34:
- // "
- if (body.charCodeAt(pos + 1) === 34 && body.charCodeAt(pos + 2) === 34) {
- return readBlockString(source, pos, _line, _col, prev, lexer);
- }
- return readString(source, pos, _line, _col, prev);
- case 45: // -
- case 48: // 0
- case 49: // 1
- case 50: // 2
- case 51: // 3
- case 52: // 4
- case 53: // 5
- case 54: // 6
- case 55: // 7
- case 56: // 8
- case 57:
- // 9
- return readNumber(source, pos, code, _line, _col, prev);
- case 65: // A
- case 66: // B
- case 67: // C
- case 68: // D
- case 69: // E
- case 70: // F
- case 71: // G
- case 72: // H
- case 73: // I
- case 74: // J
- case 75: // K
- case 76: // L
- case 77: // M
- case 78: // N
- case 79: // O
- case 80: // P
- case 81: // Q
- case 82: // R
- case 83: // S
- case 84: // T
- case 85: // U
- case 86: // V
- case 87: // W
- case 88: // X
- case 89: // Y
- case 90: // Z
- case 95: // _
- case 97: // a
- case 98: // b
- case 99: // c
- case 100: // d
- case 101: // e
- case 102: // f
- case 103: // g
- case 104: // h
- case 105: // i
- case 106: // j
- case 107: // k
- case 108: // l
- case 109: // m
- case 110: // n
- case 111: // o
- case 112: // p
- case 113: // q
- case 114: // r
- case 115: // s
- case 116: // t
- case 117: // u
- case 118: // v
- case 119: // w
- case 120: // x
- case 121: // y
- case 122:
- // z
- return readName(source, pos, _line, _col, prev);
- }
- throw (0, _syntaxError.syntaxError)(source, pos, unexpectedCharacterMessage(code));
- }
- var line = lexer.line;
- var col = 1 + pos - lexer.lineStart;
- return new _ast.Token(_tokenKind.TokenKind.EOF, bodyLength, bodyLength, line, col, prev);
- }
- /**
- * Report a message that an unexpected character was encountered.
- */
- function unexpectedCharacterMessage(code) {
- if (code < 0x0020 && code !== 0x0009 && code !== 0x000a && code !== 0x000d) {
- return "Cannot contain the invalid character ".concat(printCharCode(code), ".");
- }
- if (code === 39) {
- // '
- return 'Unexpected single quote character (\'), did you mean to use a double quote (")?';
- }
- return "Cannot parse the unexpected character ".concat(printCharCode(code), ".");
- }
- /**
- * Reads a comment token from the source file.
- *
- * #[\u0009\u0020-\uFFFF]*
- */
- function readComment(source, start, line, col, prev) {
- var body = source.body;
- var code;
- var position = start;
- do {
- code = body.charCodeAt(++position);
- } while (!isNaN(code) && ( // SourceCharacter but not LineTerminator
- code > 0x001f || code === 0x0009));
- return new _ast.Token(_tokenKind.TokenKind.COMMENT, start, position, line, col, prev, body.slice(start + 1, position));
- }
- /**
- * Reads a number token from the source file, either a float
- * or an int depending on whether a decimal point appears.
- *
- * Int: -?(0|[1-9][0-9]*)
- * Float: -?(0|[1-9][0-9]*)(\.[0-9]+)?((E|e)(+|-)?[0-9]+)?
- */
- function readNumber(source, start, firstCode, line, col, prev) {
- var body = source.body;
- var code = firstCode;
- var position = start;
- var isFloat = false;
- if (code === 45) {
- // -
- code = body.charCodeAt(++position);
- }
- if (code === 48) {
- // 0
- code = body.charCodeAt(++position);
- if (code >= 48 && code <= 57) {
- throw (0, _syntaxError.syntaxError)(source, position, "Invalid number, unexpected digit after 0: ".concat(printCharCode(code), "."));
- }
- } else {
- position = readDigits(source, position, code);
- code = body.charCodeAt(position);
- }
- if (code === 46) {
- // .
- isFloat = true;
- code = body.charCodeAt(++position);
- position = readDigits(source, position, code);
- code = body.charCodeAt(position);
- }
- if (code === 69 || code === 101) {
- // E e
- isFloat = true;
- code = body.charCodeAt(++position);
- if (code === 43 || code === 45) {
- // + -
- code = body.charCodeAt(++position);
- }
- position = readDigits(source, position, code);
- code = body.charCodeAt(position);
- } // Numbers cannot be followed by . or NameStart
- if (code === 46 || isNameStart(code)) {
- throw (0, _syntaxError.syntaxError)(source, position, "Invalid number, expected digit but got: ".concat(printCharCode(code), "."));
- }
- return new _ast.Token(isFloat ? _tokenKind.TokenKind.FLOAT : _tokenKind.TokenKind.INT, start, position, line, col, prev, body.slice(start, position));
- }
- /**
- * Returns the new position in the source after reading digits.
- */
- function readDigits(source, start, firstCode) {
- var body = source.body;
- var position = start;
- var code = firstCode;
- if (code >= 48 && code <= 57) {
- // 0 - 9
- do {
- code = body.charCodeAt(++position);
- } while (code >= 48 && code <= 57); // 0 - 9
- return position;
- }
- throw (0, _syntaxError.syntaxError)(source, position, "Invalid number, expected digit but got: ".concat(printCharCode(code), "."));
- }
- /**
- * Reads a string token from the source file.
- *
- * "([^"\\\u000A\u000D]|(\\(u[0-9a-fA-F]{4}|["\\/bfnrt])))*"
- */
- function readString(source, start, line, col, prev) {
- var body = source.body;
- var position = start + 1;
- var chunkStart = position;
- var code = 0;
- var value = '';
- while (position < body.length && !isNaN(code = body.charCodeAt(position)) && // not LineTerminator
- code !== 0x000a && code !== 0x000d) {
- // Closing Quote (")
- if (code === 34) {
- value += body.slice(chunkStart, position);
- return new _ast.Token(_tokenKind.TokenKind.STRING, start, position + 1, line, col, prev, value);
- } // SourceCharacter
- if (code < 0x0020 && code !== 0x0009) {
- throw (0, _syntaxError.syntaxError)(source, position, "Invalid character within String: ".concat(printCharCode(code), "."));
- }
- ++position;
- if (code === 92) {
- // \
- value += body.slice(chunkStart, position - 1);
- code = body.charCodeAt(position);
- switch (code) {
- case 34:
- value += '"';
- break;
- case 47:
- value += '/';
- break;
- case 92:
- value += '\\';
- break;
- case 98:
- value += '\b';
- break;
- case 102:
- value += '\f';
- break;
- case 110:
- value += '\n';
- break;
- case 114:
- value += '\r';
- break;
- case 116:
- value += '\t';
- break;
- case 117:
- {
- // uXXXX
- var charCode = uniCharCode(body.charCodeAt(position + 1), body.charCodeAt(position + 2), body.charCodeAt(position + 3), body.charCodeAt(position + 4));
- if (charCode < 0) {
- var invalidSequence = body.slice(position + 1, position + 5);
- throw (0, _syntaxError.syntaxError)(source, position, "Invalid character escape sequence: \\u".concat(invalidSequence, "."));
- }
- value += String.fromCharCode(charCode);
- position += 4;
- break;
- }
- default:
- throw (0, _syntaxError.syntaxError)(source, position, "Invalid character escape sequence: \\".concat(String.fromCharCode(code), "."));
- }
- ++position;
- chunkStart = position;
- }
- }
- throw (0, _syntaxError.syntaxError)(source, position, 'Unterminated string.');
- }
- /**
- * Reads a block string token from the source file.
- *
- * """("?"?(\\"""|\\(?!=""")|[^"\\]))*"""
- */
- function readBlockString(source, start, line, col, prev, lexer) {
- var body = source.body;
- var position = start + 3;
- var chunkStart = position;
- var code = 0;
- var rawValue = '';
- while (position < body.length && !isNaN(code = body.charCodeAt(position))) {
- // Closing Triple-Quote (""")
- if (code === 34 && body.charCodeAt(position + 1) === 34 && body.charCodeAt(position + 2) === 34) {
- rawValue += body.slice(chunkStart, position);
- return new _ast.Token(_tokenKind.TokenKind.BLOCK_STRING, start, position + 3, line, col, prev, (0, _blockString.dedentBlockStringValue)(rawValue));
- } // SourceCharacter
- if (code < 0x0020 && code !== 0x0009 && code !== 0x000a && code !== 0x000d) {
- throw (0, _syntaxError.syntaxError)(source, position, "Invalid character within String: ".concat(printCharCode(code), "."));
- }
- if (code === 10) {
- // new line
- ++position;
- ++lexer.line;
- lexer.lineStart = position;
- } else if (code === 13) {
- // carriage return
- if (body.charCodeAt(position + 1) === 10) {
- position += 2;
- } else {
- ++position;
- }
- ++lexer.line;
- lexer.lineStart = position;
- } else if ( // Escape Triple-Quote (\""")
- code === 92 && body.charCodeAt(position + 1) === 34 && body.charCodeAt(position + 2) === 34 && body.charCodeAt(position + 3) === 34) {
- rawValue += body.slice(chunkStart, position) + '"""';
- position += 4;
- chunkStart = position;
- } else {
- ++position;
- }
- }
- throw (0, _syntaxError.syntaxError)(source, position, 'Unterminated string.');
- }
- /**
- * Converts four hexadecimal chars to the integer that the
- * string represents. For example, uniCharCode('0','0','0','f')
- * will return 15, and uniCharCode('0','0','f','f') returns 255.
- *
- * Returns a negative number on error, if a char was invalid.
- *
- * This is implemented by noting that char2hex() returns -1 on error,
- * which means the result of ORing the char2hex() will also be negative.
- */
- function uniCharCode(a, b, c, d) {
- return char2hex(a) << 12 | char2hex(b) << 8 | char2hex(c) << 4 | char2hex(d);
- }
- /**
- * Converts a hex character to its integer value.
- * '0' becomes 0, '9' becomes 9
- * 'A' becomes 10, 'F' becomes 15
- * 'a' becomes 10, 'f' becomes 15
- *
- * Returns -1 on error.
- */
- function char2hex(a) {
- return a >= 48 && a <= 57 ? a - 48 // 0-9
- : a >= 65 && a <= 70 ? a - 55 // A-F
- : a >= 97 && a <= 102 ? a - 87 // a-f
- : -1;
- }
- /**
- * Reads an alphanumeric + underscore name from the source.
- *
- * [_A-Za-z][_0-9A-Za-z]*
- */
- function readName(source, start, line, col, prev) {
- var body = source.body;
- var bodyLength = body.length;
- var position = start + 1;
- var code = 0;
- while (position !== bodyLength && !isNaN(code = body.charCodeAt(position)) && (code === 95 || // _
- code >= 48 && code <= 57 || // 0-9
- code >= 65 && code <= 90 || // A-Z
- code >= 97 && code <= 122) // a-z
- ) {
- ++position;
- }
- return new _ast.Token(_tokenKind.TokenKind.NAME, start, position, line, col, prev, body.slice(start, position));
- } // _ A-Z a-z
- function isNameStart(code) {
- return code === 95 || code >= 65 && code <= 90 || code >= 97 && code <= 122;
- }
|