lexer.mjs 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617
  1. import { syntaxError } from "../error/syntaxError.mjs";
  2. import { Token } from "./ast.mjs";
  3. import { TokenKind } from "./tokenKind.mjs";
  4. import { dedentBlockStringValue } from "./blockString.mjs";
  5. /**
  6. * Given a Source object, creates a Lexer for that source.
  7. * A Lexer is a stateful stream generator in that every time
  8. * it is advanced, it returns the next token in the Source. Assuming the
  9. * source lexes, the final Token emitted by the lexer will be of kind
  10. * EOF, after which the lexer will repeatedly return the same EOF token
  11. * whenever called.
  12. */
  13. export var Lexer = /*#__PURE__*/function () {
  14. /**
  15. * The previously focused non-ignored token.
  16. */
  17. /**
  18. * The currently focused non-ignored token.
  19. */
  20. /**
  21. * The (1-indexed) line containing the current token.
  22. */
  23. /**
  24. * The character offset at which the current line begins.
  25. */
  26. function Lexer(source) {
  27. var startOfFileToken = new Token(TokenKind.SOF, 0, 0, 0, 0, null);
  28. this.source = source;
  29. this.lastToken = startOfFileToken;
  30. this.token = startOfFileToken;
  31. this.line = 1;
  32. this.lineStart = 0;
  33. }
  34. /**
  35. * Advances the token stream to the next non-ignored token.
  36. */
  37. var _proto = Lexer.prototype;
  38. _proto.advance = function advance() {
  39. this.lastToken = this.token;
  40. var token = this.token = this.lookahead();
  41. return token;
  42. }
  43. /**
  44. * Looks ahead and returns the next non-ignored token, but does not change
  45. * the state of Lexer.
  46. */
  47. ;
  48. _proto.lookahead = function lookahead() {
  49. var token = this.token;
  50. if (token.kind !== TokenKind.EOF) {
  51. do {
  52. var _token$next;
  53. // Note: next is only mutable during parsing, so we cast to allow this.
  54. token = (_token$next = token.next) !== null && _token$next !== void 0 ? _token$next : token.next = readToken(this, token);
  55. } while (token.kind === TokenKind.COMMENT);
  56. }
  57. return token;
  58. };
  59. return Lexer;
  60. }();
  61. /**
  62. * @internal
  63. */
  64. export function isPunctuatorTokenKind(kind) {
  65. return kind === TokenKind.BANG || kind === TokenKind.DOLLAR || kind === TokenKind.AMP || kind === TokenKind.PAREN_L || kind === TokenKind.PAREN_R || kind === TokenKind.SPREAD || kind === TokenKind.COLON || kind === TokenKind.EQUALS || kind === TokenKind.AT || kind === TokenKind.BRACKET_L || kind === TokenKind.BRACKET_R || kind === TokenKind.BRACE_L || kind === TokenKind.PIPE || kind === TokenKind.BRACE_R;
  66. }
  67. function printCharCode(code) {
  68. return (// NaN/undefined represents access beyond the end of the file.
  69. isNaN(code) ? TokenKind.EOF : // Trust JSON for ASCII.
  70. code < 0x007f ? JSON.stringify(String.fromCharCode(code)) : // Otherwise print the escaped form.
  71. "\"\\u".concat(('00' + code.toString(16).toUpperCase()).slice(-4), "\"")
  72. );
  73. }
  74. /**
  75. * Gets the next token from the source starting at the given position.
  76. *
  77. * This skips over whitespace until it finds the next lexable token, then lexes
  78. * punctuators immediately or calls the appropriate helper function for more
  79. * complicated tokens.
  80. */
  81. function readToken(lexer, prev) {
  82. var source = lexer.source;
  83. var body = source.body;
  84. var bodyLength = body.length;
  85. var pos = positionAfterWhitespace(body, prev.end, lexer);
  86. var line = lexer.line;
  87. var col = 1 + pos - lexer.lineStart;
  88. if (pos >= bodyLength) {
  89. return new Token(TokenKind.EOF, bodyLength, bodyLength, line, col, prev);
  90. }
  91. var code = body.charCodeAt(pos); // SourceCharacter
  92. switch (code) {
  93. // !
  94. case 33:
  95. return new Token(TokenKind.BANG, pos, pos + 1, line, col, prev);
  96. // #
  97. case 35:
  98. return readComment(source, pos, line, col, prev);
  99. // $
  100. case 36:
  101. return new Token(TokenKind.DOLLAR, pos, pos + 1, line, col, prev);
  102. // &
  103. case 38:
  104. return new Token(TokenKind.AMP, pos, pos + 1, line, col, prev);
  105. // (
  106. case 40:
  107. return new Token(TokenKind.PAREN_L, pos, pos + 1, line, col, prev);
  108. // )
  109. case 41:
  110. return new Token(TokenKind.PAREN_R, pos, pos + 1, line, col, prev);
  111. // .
  112. case 46:
  113. if (body.charCodeAt(pos + 1) === 46 && body.charCodeAt(pos + 2) === 46) {
  114. return new Token(TokenKind.SPREAD, pos, pos + 3, line, col, prev);
  115. }
  116. break;
  117. // :
  118. case 58:
  119. return new Token(TokenKind.COLON, pos, pos + 1, line, col, prev);
  120. // =
  121. case 61:
  122. return new Token(TokenKind.EQUALS, pos, pos + 1, line, col, prev);
  123. // @
  124. case 64:
  125. return new Token(TokenKind.AT, pos, pos + 1, line, col, prev);
  126. // [
  127. case 91:
  128. return new Token(TokenKind.BRACKET_L, pos, pos + 1, line, col, prev);
  129. // ]
  130. case 93:
  131. return new Token(TokenKind.BRACKET_R, pos, pos + 1, line, col, prev);
  132. // {
  133. case 123:
  134. return new Token(TokenKind.BRACE_L, pos, pos + 1, line, col, prev);
  135. // |
  136. case 124:
  137. return new Token(TokenKind.PIPE, pos, pos + 1, line, col, prev);
  138. // }
  139. case 125:
  140. return new Token(TokenKind.BRACE_R, pos, pos + 1, line, col, prev);
  141. // A-Z _ a-z
  142. case 65:
  143. case 66:
  144. case 67:
  145. case 68:
  146. case 69:
  147. case 70:
  148. case 71:
  149. case 72:
  150. case 73:
  151. case 74:
  152. case 75:
  153. case 76:
  154. case 77:
  155. case 78:
  156. case 79:
  157. case 80:
  158. case 81:
  159. case 82:
  160. case 83:
  161. case 84:
  162. case 85:
  163. case 86:
  164. case 87:
  165. case 88:
  166. case 89:
  167. case 90:
  168. case 95:
  169. case 97:
  170. case 98:
  171. case 99:
  172. case 100:
  173. case 101:
  174. case 102:
  175. case 103:
  176. case 104:
  177. case 105:
  178. case 106:
  179. case 107:
  180. case 108:
  181. case 109:
  182. case 110:
  183. case 111:
  184. case 112:
  185. case 113:
  186. case 114:
  187. case 115:
  188. case 116:
  189. case 117:
  190. case 118:
  191. case 119:
  192. case 120:
  193. case 121:
  194. case 122:
  195. return readName(source, pos, line, col, prev);
  196. // - 0-9
  197. case 45:
  198. case 48:
  199. case 49:
  200. case 50:
  201. case 51:
  202. case 52:
  203. case 53:
  204. case 54:
  205. case 55:
  206. case 56:
  207. case 57:
  208. return readNumber(source, pos, code, line, col, prev);
  209. // "
  210. case 34:
  211. if (body.charCodeAt(pos + 1) === 34 && body.charCodeAt(pos + 2) === 34) {
  212. return readBlockString(source, pos, line, col, prev, lexer);
  213. }
  214. return readString(source, pos, line, col, prev);
  215. }
  216. throw syntaxError(source, pos, unexpectedCharacterMessage(code));
  217. }
  218. /**
  219. * Report a message that an unexpected character was encountered.
  220. */
  221. function unexpectedCharacterMessage(code) {
  222. if (code < 0x0020 && code !== 0x0009 && code !== 0x000a && code !== 0x000d) {
  223. return "Cannot contain the invalid character ".concat(printCharCode(code), ".");
  224. }
  225. if (code === 39) {
  226. // '
  227. return 'Unexpected single quote character (\'), did you mean to use a double quote (")?';
  228. }
  229. return "Cannot parse the unexpected character ".concat(printCharCode(code), ".");
  230. }
  231. /**
  232. * Reads from body starting at startPosition until it finds a non-whitespace
  233. * character, then returns the position of that character for lexing.
  234. */
  235. function positionAfterWhitespace(body, startPosition, lexer) {
  236. var bodyLength = body.length;
  237. var position = startPosition;
  238. while (position < bodyLength) {
  239. var code = body.charCodeAt(position); // tab | space | comma | BOM
  240. if (code === 9 || code === 32 || code === 44 || code === 0xfeff) {
  241. ++position;
  242. } else if (code === 10) {
  243. // new line
  244. ++position;
  245. ++lexer.line;
  246. lexer.lineStart = position;
  247. } else if (code === 13) {
  248. // carriage return
  249. if (body.charCodeAt(position + 1) === 10) {
  250. position += 2;
  251. } else {
  252. ++position;
  253. }
  254. ++lexer.line;
  255. lexer.lineStart = position;
  256. } else {
  257. break;
  258. }
  259. }
  260. return position;
  261. }
  262. /**
  263. * Reads a comment token from the source file.
  264. *
  265. * #[\u0009\u0020-\uFFFF]*
  266. */
  267. function readComment(source, start, line, col, prev) {
  268. var body = source.body;
  269. var code;
  270. var position = start;
  271. do {
  272. code = body.charCodeAt(++position);
  273. } while (!isNaN(code) && ( // SourceCharacter but not LineTerminator
  274. code > 0x001f || code === 0x0009));
  275. return new Token(TokenKind.COMMENT, start, position, line, col, prev, body.slice(start + 1, position));
  276. }
  277. /**
  278. * Reads a number token from the source file, either a float
  279. * or an int depending on whether a decimal point appears.
  280. *
  281. * Int: -?(0|[1-9][0-9]*)
  282. * Float: -?(0|[1-9][0-9]*)(\.[0-9]+)?((E|e)(+|-)?[0-9]+)?
  283. */
  284. function readNumber(source, start, firstCode, line, col, prev) {
  285. var body = source.body;
  286. var code = firstCode;
  287. var position = start;
  288. var isFloat = false;
  289. if (code === 45) {
  290. // -
  291. code = body.charCodeAt(++position);
  292. }
  293. if (code === 48) {
  294. // 0
  295. code = body.charCodeAt(++position);
  296. if (code >= 48 && code <= 57) {
  297. throw syntaxError(source, position, "Invalid number, unexpected digit after 0: ".concat(printCharCode(code), "."));
  298. }
  299. } else {
  300. position = readDigits(source, position, code);
  301. code = body.charCodeAt(position);
  302. }
  303. if (code === 46) {
  304. // .
  305. isFloat = true;
  306. code = body.charCodeAt(++position);
  307. position = readDigits(source, position, code);
  308. code = body.charCodeAt(position);
  309. }
  310. if (code === 69 || code === 101) {
  311. // E e
  312. isFloat = true;
  313. code = body.charCodeAt(++position);
  314. if (code === 43 || code === 45) {
  315. // + -
  316. code = body.charCodeAt(++position);
  317. }
  318. position = readDigits(source, position, code);
  319. code = body.charCodeAt(position);
  320. } // Numbers cannot be followed by . or NameStart
  321. if (code === 46 || isNameStart(code)) {
  322. throw syntaxError(source, position, "Invalid number, expected digit but got: ".concat(printCharCode(code), "."));
  323. }
  324. return new Token(isFloat ? TokenKind.FLOAT : TokenKind.INT, start, position, line, col, prev, body.slice(start, position));
  325. }
  326. /**
  327. * Returns the new position in the source after reading digits.
  328. */
  329. function readDigits(source, start, firstCode) {
  330. var body = source.body;
  331. var position = start;
  332. var code = firstCode;
  333. if (code >= 48 && code <= 57) {
  334. // 0 - 9
  335. do {
  336. code = body.charCodeAt(++position);
  337. } while (code >= 48 && code <= 57); // 0 - 9
  338. return position;
  339. }
  340. throw syntaxError(source, position, "Invalid number, expected digit but got: ".concat(printCharCode(code), "."));
  341. }
  342. /**
  343. * Reads a string token from the source file.
  344. *
  345. * "([^"\\\u000A\u000D]|(\\(u[0-9a-fA-F]{4}|["\\/bfnrt])))*"
  346. */
  347. function readString(source, start, line, col, prev) {
  348. var body = source.body;
  349. var position = start + 1;
  350. var chunkStart = position;
  351. var code = 0;
  352. var value = '';
  353. while (position < body.length && !isNaN(code = body.charCodeAt(position)) && // not LineTerminator
  354. code !== 0x000a && code !== 0x000d) {
  355. // Closing Quote (")
  356. if (code === 34) {
  357. value += body.slice(chunkStart, position);
  358. return new Token(TokenKind.STRING, start, position + 1, line, col, prev, value);
  359. } // SourceCharacter
  360. if (code < 0x0020 && code !== 0x0009) {
  361. throw syntaxError(source, position, "Invalid character within String: ".concat(printCharCode(code), "."));
  362. }
  363. ++position;
  364. if (code === 92) {
  365. // \
  366. value += body.slice(chunkStart, position - 1);
  367. code = body.charCodeAt(position);
  368. switch (code) {
  369. case 34:
  370. value += '"';
  371. break;
  372. case 47:
  373. value += '/';
  374. break;
  375. case 92:
  376. value += '\\';
  377. break;
  378. case 98:
  379. value += '\b';
  380. break;
  381. case 102:
  382. value += '\f';
  383. break;
  384. case 110:
  385. value += '\n';
  386. break;
  387. case 114:
  388. value += '\r';
  389. break;
  390. case 116:
  391. value += '\t';
  392. break;
  393. case 117:
  394. {
  395. // uXXXX
  396. var charCode = uniCharCode(body.charCodeAt(position + 1), body.charCodeAt(position + 2), body.charCodeAt(position + 3), body.charCodeAt(position + 4));
  397. if (charCode < 0) {
  398. var invalidSequence = body.slice(position + 1, position + 5);
  399. throw syntaxError(source, position, "Invalid character escape sequence: \\u".concat(invalidSequence, "."));
  400. }
  401. value += String.fromCharCode(charCode);
  402. position += 4;
  403. break;
  404. }
  405. default:
  406. throw syntaxError(source, position, "Invalid character escape sequence: \\".concat(String.fromCharCode(code), "."));
  407. }
  408. ++position;
  409. chunkStart = position;
  410. }
  411. }
  412. throw syntaxError(source, position, 'Unterminated string.');
  413. }
  414. /**
  415. * Reads a block string token from the source file.
  416. *
  417. * """("?"?(\\"""|\\(?!=""")|[^"\\]))*"""
  418. */
  419. function readBlockString(source, start, line, col, prev, lexer) {
  420. var body = source.body;
  421. var position = start + 3;
  422. var chunkStart = position;
  423. var code = 0;
  424. var rawValue = '';
  425. while (position < body.length && !isNaN(code = body.charCodeAt(position))) {
  426. // Closing Triple-Quote (""")
  427. if (code === 34 && body.charCodeAt(position + 1) === 34 && body.charCodeAt(position + 2) === 34) {
  428. rawValue += body.slice(chunkStart, position);
  429. return new Token(TokenKind.BLOCK_STRING, start, position + 3, line, col, prev, dedentBlockStringValue(rawValue));
  430. } // SourceCharacter
  431. if (code < 0x0020 && code !== 0x0009 && code !== 0x000a && code !== 0x000d) {
  432. throw syntaxError(source, position, "Invalid character within String: ".concat(printCharCode(code), "."));
  433. }
  434. if (code === 10) {
  435. // new line
  436. ++position;
  437. ++lexer.line;
  438. lexer.lineStart = position;
  439. } else if (code === 13) {
  440. // carriage return
  441. if (body.charCodeAt(position + 1) === 10) {
  442. position += 2;
  443. } else {
  444. ++position;
  445. }
  446. ++lexer.line;
  447. lexer.lineStart = position;
  448. } else if ( // Escape Triple-Quote (\""")
  449. code === 92 && body.charCodeAt(position + 1) === 34 && body.charCodeAt(position + 2) === 34 && body.charCodeAt(position + 3) === 34) {
  450. rawValue += body.slice(chunkStart, position) + '"""';
  451. position += 4;
  452. chunkStart = position;
  453. } else {
  454. ++position;
  455. }
  456. }
  457. throw syntaxError(source, position, 'Unterminated string.');
  458. }
  459. /**
  460. * Converts four hexadecimal chars to the integer that the
  461. * string represents. For example, uniCharCode('0','0','0','f')
  462. * will return 15, and uniCharCode('0','0','f','f') returns 255.
  463. *
  464. * Returns a negative number on error, if a char was invalid.
  465. *
  466. * This is implemented by noting that char2hex() returns -1 on error,
  467. * which means the result of ORing the char2hex() will also be negative.
  468. */
  469. function uniCharCode(a, b, c, d) {
  470. return char2hex(a) << 12 | char2hex(b) << 8 | char2hex(c) << 4 | char2hex(d);
  471. }
  472. /**
  473. * Converts a hex character to its integer value.
  474. * '0' becomes 0, '9' becomes 9
  475. * 'A' becomes 10, 'F' becomes 15
  476. * 'a' becomes 10, 'f' becomes 15
  477. *
  478. * Returns -1 on error.
  479. */
  480. function char2hex(a) {
  481. return a >= 48 && a <= 57 ? a - 48 // 0-9
  482. : a >= 65 && a <= 70 ? a - 55 // A-F
  483. : a >= 97 && a <= 102 ? a - 87 // a-f
  484. : -1;
  485. }
  486. /**
  487. * Reads an alphanumeric + underscore name from the source.
  488. *
  489. * [_A-Za-z][_0-9A-Za-z]*
  490. */
  491. function readName(source, start, line, col, prev) {
  492. var body = source.body;
  493. var bodyLength = body.length;
  494. var position = start + 1;
  495. var code = 0;
  496. while (position !== bodyLength && !isNaN(code = body.charCodeAt(position)) && (code === 95 || // _
  497. code >= 48 && code <= 57 || // 0-9
  498. code >= 65 && code <= 90 || // A-Z
  499. code >= 97 && code <= 122) // a-z
  500. ) {
  501. ++position;
  502. }
  503. return new Token(TokenKind.NAME, start, position, line, col, prev, body.slice(start, position));
  504. } // _ A-Z a-z
  505. function isNameStart(code) {
  506. return code === 95 || code >= 65 && code <= 90 || code >= 97 && code <= 122;
  507. }