tokenize.js 8.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271
  1. "use strict";
  2. exports.__esModule = true;
  3. exports["default"] = tokenize;
  4. exports.FIELDS = void 0;
  5. var t = _interopRequireWildcard(require("./tokenTypes"));
  6. var _unescapable, _wordDelimiters;
  7. function _getRequireWildcardCache() { if (typeof WeakMap !== "function") return null; var cache = new WeakMap(); _getRequireWildcardCache = function _getRequireWildcardCache() { return cache; }; return cache; }
  8. function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { "default": obj }; } var cache = _getRequireWildcardCache(); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj["default"] = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
  9. var unescapable = (_unescapable = {}, _unescapable[t.tab] = true, _unescapable[t.newline] = true, _unescapable[t.cr] = true, _unescapable[t.feed] = true, _unescapable);
  10. var wordDelimiters = (_wordDelimiters = {}, _wordDelimiters[t.space] = true, _wordDelimiters[t.tab] = true, _wordDelimiters[t.newline] = true, _wordDelimiters[t.cr] = true, _wordDelimiters[t.feed] = true, _wordDelimiters[t.ampersand] = true, _wordDelimiters[t.asterisk] = true, _wordDelimiters[t.bang] = true, _wordDelimiters[t.comma] = true, _wordDelimiters[t.colon] = true, _wordDelimiters[t.semicolon] = true, _wordDelimiters[t.openParenthesis] = true, _wordDelimiters[t.closeParenthesis] = true, _wordDelimiters[t.openSquare] = true, _wordDelimiters[t.closeSquare] = true, _wordDelimiters[t.singleQuote] = true, _wordDelimiters[t.doubleQuote] = true, _wordDelimiters[t.plus] = true, _wordDelimiters[t.pipe] = true, _wordDelimiters[t.tilde] = true, _wordDelimiters[t.greaterThan] = true, _wordDelimiters[t.equals] = true, _wordDelimiters[t.dollar] = true, _wordDelimiters[t.caret] = true, _wordDelimiters[t.slash] = true, _wordDelimiters);
  11. var hex = {};
  12. var hexChars = "0123456789abcdefABCDEF";
  13. for (var i = 0; i < hexChars.length; i++) {
  14. hex[hexChars.charCodeAt(i)] = true;
  15. }
  16. /**
  17. * Returns the last index of the bar css word
  18. * @param {string} css The string in which the word begins
  19. * @param {number} start The index into the string where word's first letter occurs
  20. */
  21. function consumeWord(css, start) {
  22. var next = start;
  23. var code;
  24. do {
  25. code = css.charCodeAt(next);
  26. if (wordDelimiters[code]) {
  27. return next - 1;
  28. } else if (code === t.backslash) {
  29. next = consumeEscape(css, next) + 1;
  30. } else {
  31. // All other characters are part of the word
  32. next++;
  33. }
  34. } while (next < css.length);
  35. return next - 1;
  36. }
  37. /**
  38. * Returns the last index of the escape sequence
  39. * @param {string} css The string in which the sequence begins
  40. * @param {number} start The index into the string where escape character (`\`) occurs.
  41. */
  42. function consumeEscape(css, start) {
  43. var next = start;
  44. var code = css.charCodeAt(next + 1);
  45. if (unescapable[code]) {// just consume the escape char
  46. } else if (hex[code]) {
  47. var hexDigits = 0; // consume up to 6 hex chars
  48. do {
  49. next++;
  50. hexDigits++;
  51. code = css.charCodeAt(next + 1);
  52. } while (hex[code] && hexDigits < 6); // if fewer than 6 hex chars, a trailing space ends the escape
  53. if (hexDigits < 6 && code === t.space) {
  54. next++;
  55. }
  56. } else {
  57. // the next char is part of the current word
  58. next++;
  59. }
  60. return next;
  61. }
  62. var FIELDS = {
  63. TYPE: 0,
  64. START_LINE: 1,
  65. START_COL: 2,
  66. END_LINE: 3,
  67. END_COL: 4,
  68. START_POS: 5,
  69. END_POS: 6
  70. };
  71. exports.FIELDS = FIELDS;
  72. function tokenize(input) {
  73. var tokens = [];
  74. var css = input.css.valueOf();
  75. var _css = css,
  76. length = _css.length;
  77. var offset = -1;
  78. var line = 1;
  79. var start = 0;
  80. var end = 0;
  81. var code, content, endColumn, endLine, escaped, escapePos, last, lines, next, nextLine, nextOffset, quote, tokenType;
  82. function unclosed(what, fix) {
  83. if (input.safe) {
  84. // fyi: this is never set to true.
  85. css += fix;
  86. next = css.length - 1;
  87. } else {
  88. throw input.error('Unclosed ' + what, line, start - offset, start);
  89. }
  90. }
  91. while (start < length) {
  92. code = css.charCodeAt(start);
  93. if (code === t.newline) {
  94. offset = start;
  95. line += 1;
  96. }
  97. switch (code) {
  98. case t.space:
  99. case t.tab:
  100. case t.newline:
  101. case t.cr:
  102. case t.feed:
  103. next = start;
  104. do {
  105. next += 1;
  106. code = css.charCodeAt(next);
  107. if (code === t.newline) {
  108. offset = next;
  109. line += 1;
  110. }
  111. } while (code === t.space || code === t.newline || code === t.tab || code === t.cr || code === t.feed);
  112. tokenType = t.space;
  113. endLine = line;
  114. endColumn = next - offset - 1;
  115. end = next;
  116. break;
  117. case t.plus:
  118. case t.greaterThan:
  119. case t.tilde:
  120. case t.pipe:
  121. next = start;
  122. do {
  123. next += 1;
  124. code = css.charCodeAt(next);
  125. } while (code === t.plus || code === t.greaterThan || code === t.tilde || code === t.pipe);
  126. tokenType = t.combinator;
  127. endLine = line;
  128. endColumn = start - offset;
  129. end = next;
  130. break;
  131. // Consume these characters as single tokens.
  132. case t.asterisk:
  133. case t.ampersand:
  134. case t.bang:
  135. case t.comma:
  136. case t.equals:
  137. case t.dollar:
  138. case t.caret:
  139. case t.openSquare:
  140. case t.closeSquare:
  141. case t.colon:
  142. case t.semicolon:
  143. case t.openParenthesis:
  144. case t.closeParenthesis:
  145. next = start;
  146. tokenType = code;
  147. endLine = line;
  148. endColumn = start - offset;
  149. end = next + 1;
  150. break;
  151. case t.singleQuote:
  152. case t.doubleQuote:
  153. quote = code === t.singleQuote ? "'" : '"';
  154. next = start;
  155. do {
  156. escaped = false;
  157. next = css.indexOf(quote, next + 1);
  158. if (next === -1) {
  159. unclosed('quote', quote);
  160. }
  161. escapePos = next;
  162. while (css.charCodeAt(escapePos - 1) === t.backslash) {
  163. escapePos -= 1;
  164. escaped = !escaped;
  165. }
  166. } while (escaped);
  167. tokenType = t.str;
  168. endLine = line;
  169. endColumn = start - offset;
  170. end = next + 1;
  171. break;
  172. default:
  173. if (code === t.slash && css.charCodeAt(start + 1) === t.asterisk) {
  174. next = css.indexOf('*/', start + 2) + 1;
  175. if (next === 0) {
  176. unclosed('comment', '*/');
  177. }
  178. content = css.slice(start, next + 1);
  179. lines = content.split('\n');
  180. last = lines.length - 1;
  181. if (last > 0) {
  182. nextLine = line + last;
  183. nextOffset = next - lines[last].length;
  184. } else {
  185. nextLine = line;
  186. nextOffset = offset;
  187. }
  188. tokenType = t.comment;
  189. line = nextLine;
  190. endLine = nextLine;
  191. endColumn = next - nextOffset;
  192. } else if (code === t.slash) {
  193. next = start;
  194. tokenType = code;
  195. endLine = line;
  196. endColumn = start - offset;
  197. end = next + 1;
  198. } else {
  199. next = consumeWord(css, start);
  200. tokenType = t.word;
  201. endLine = line;
  202. endColumn = next - offset;
  203. }
  204. end = next + 1;
  205. break;
  206. } // Ensure that the token structure remains consistent
  207. tokens.push([tokenType, // [0] Token type
  208. line, // [1] Starting line
  209. start - offset, // [2] Starting column
  210. endLine, // [3] Ending line
  211. endColumn, // [4] Ending column
  212. start, // [5] Start position / Source index
  213. end // [6] End position
  214. ]); // Reset offset for the next token
  215. if (nextOffset) {
  216. offset = nextOffset;
  217. nextOffset = null;
  218. }
  219. start = end;
  220. }
  221. return tokens;
  222. }