tokenize.js 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108
  1. import {tokTypes as tt, Token, isNewLine, SourceLocation, getLineInfo, lineBreakG} from "../index"
  2. import {LooseParser} from "./state"
  3. const lp = LooseParser.prototype
  4. function isSpace(ch) {
  5. return (ch < 14 && ch > 8) || ch === 32 || ch === 160 || isNewLine(ch)
  6. }
  7. lp.next = function() {
  8. this.last = this.tok
  9. if (this.ahead.length)
  10. this.tok = this.ahead.shift()
  11. else
  12. this.tok = this.readToken()
  13. if (this.tok.start >= this.nextLineStart) {
  14. while (this.tok.start >= this.nextLineStart) {
  15. this.curLineStart = this.nextLineStart
  16. this.nextLineStart = this.lineEnd(this.curLineStart) + 1
  17. }
  18. this.curIndent = this.indentationAfter(this.curLineStart)
  19. }
  20. }
  21. lp.readToken = function() {
  22. for (;;) {
  23. try {
  24. this.toks.next()
  25. if (this.toks.type === tt.dot &&
  26. this.input.substr(this.toks.end, 1) === "." &&
  27. this.options.ecmaVersion >= 6) {
  28. this.toks.end++
  29. this.toks.type = tt.ellipsis
  30. }
  31. return new Token(this.toks)
  32. } catch(e) {
  33. if (!(e instanceof SyntaxError)) throw e
  34. // Try to skip some text, based on the error message, and then continue
  35. let msg = e.message, pos = e.raisedAt, replace = true
  36. if (/unterminated/i.test(msg)) {
  37. pos = this.lineEnd(e.pos + 1)
  38. if (/string/.test(msg)) {
  39. replace = {start: e.pos, end: pos, type: tt.string, value: this.input.slice(e.pos + 1, pos)}
  40. } else if (/regular expr/i.test(msg)) {
  41. let re = this.input.slice(e.pos, pos)
  42. try { re = new RegExp(re) } catch(e) {}
  43. replace = {start: e.pos, end: pos, type: tt.regexp, value: re}
  44. } else if (/template/.test(msg)) {
  45. replace = {start: e.pos, end: pos,
  46. type: tt.template,
  47. value: this.input.slice(e.pos, pos)}
  48. } else {
  49. replace = false
  50. }
  51. } else if (/invalid (unicode|regexp|number)|expecting unicode|octal literal|is reserved|directly after number|expected number in radix/i.test(msg)) {
  52. while (pos < this.input.length && !isSpace(this.input.charCodeAt(pos))) ++pos
  53. } else if (/character escape|expected hexadecimal/i.test(msg)) {
  54. while (pos < this.input.length) {
  55. let ch = this.input.charCodeAt(pos++)
  56. if (ch === 34 || ch === 39 || isNewLine(ch)) break
  57. }
  58. } else if (/unexpected character/i.test(msg)) {
  59. pos++
  60. replace = false
  61. } else if (/regular expression/i.test(msg)) {
  62. replace = true
  63. } else {
  64. throw e
  65. }
  66. this.resetTo(pos)
  67. if (replace === true) replace = {start: pos, end: pos, type: tt.name, value: "✖"}
  68. if (replace) {
  69. if (this.options.locations)
  70. replace.loc = new SourceLocation(
  71. this.toks,
  72. getLineInfo(this.input, replace.start),
  73. getLineInfo(this.input, replace.end))
  74. return replace
  75. }
  76. }
  77. }
  78. }
  79. lp.resetTo = function(pos) {
  80. this.toks.pos = pos
  81. let ch = this.input.charAt(pos - 1)
  82. this.toks.exprAllowed = !ch || /[\[\{\(,;:?\/*=+\-~!|&%^<>]/.test(ch) ||
  83. /[enwfd]/.test(ch) &&
  84. /\b(keywords|case|else|return|throw|new|in|(instance|type)of|delete|void)$/.test(this.input.slice(pos - 10, pos))
  85. if (this.options.locations) {
  86. this.toks.curLine = 1
  87. this.toks.lineStart = lineBreakG.lastIndex = 0
  88. let match
  89. while ((match = lineBreakG.exec(this.input)) && match.index < pos) {
  90. ++this.toks.curLine
  91. this.toks.lineStart = match.index + match[0].length
  92. }
  93. }
  94. }
  95. lp.lookAhead = function(n) {
  96. while (n > this.ahead.length)
  97. this.ahead.push(this.readToken())
  98. return this.ahead[n - 1]
  99. }