tokenize.js 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302
  1. 'use strict';
  2. exports.__esModule = true;
  3. exports.default = tokenize;
  4. var _input = require('./input');
  5. var _input2 = _interopRequireDefault(_input);
  6. var _tokenizeString2 = require('./tokenize-string');
  7. var _tokenizeString3 = _interopRequireDefault(_tokenizeString2);
  8. var _tokenizeComment2 = require('./tokenize-comment');
  9. var _tokenizeComment3 = _interopRequireDefault(_tokenizeComment2);
  10. var _tokenizeInterpolant2 = require('./tokenize-interpolant');
  11. var _tokenizeInterpolant3 = _interopRequireDefault(_tokenizeInterpolant2);
  12. function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
  13. var singleQuote = "'".charCodeAt(0),
  14. doubleQuote = '"'.charCodeAt(0),
  15. dollar = '$'.charCodeAt(0),
  16. hash = '#'.charCodeAt(0),
  17. backslash = '\\'.charCodeAt(0),
  18. slash = '/'.charCodeAt(0),
  19. newline = '\n'.charCodeAt(0),
  20. space = ' '.charCodeAt(0),
  21. feed = '\f'.charCodeAt(0),
  22. tab = '\t'.charCodeAt(0),
  23. cr = '\r'.charCodeAt(0),
  24. openBracket = '('.charCodeAt(0),
  25. closeBracket = ')'.charCodeAt(0),
  26. openCurly = '{'.charCodeAt(0),
  27. closeCurly = '}'.charCodeAt(0),
  28. semicolon = ';'.charCodeAt(0),
  29. asterisk = '*'.charCodeAt(0),
  30. colon = ':'.charCodeAt(0),
  31. at = '@'.charCodeAt(0),
  32. comma = ','.charCodeAt(0),
  33. plus = '+'.charCodeAt(0),
  34. minus = '-'.charCodeAt(0),
  35. decComb = '>'.charCodeAt(0),
  36. adjComb = '~'.charCodeAt(0),
  37. number = /[+-]?(\d+(\.\d+)?|\.\d+)|(e[+-]\d+)/gi,
  38. sQuoteEnd = /(.*?)[^\\](?=((#{)|'))/gm,
  39. dQuoteEnd = /(.*?)[^\\](?=((#{)|"))/gm,
  40. wordEnd = /[ \n\t\r\(\)\{\},:;@!'"\\]|\/(?=\*)|#(?={)/g,
  41. ident = /-?([a-z_]|\\[^\\])([a-z-_0-9]|\\[^\\])*/gi;
  42. function tokenize(input, l, p) {
  43. var tokens = [];
  44. var css = input.css.valueOf();
  45. var code = void 0,
  46. next = void 0,
  47. quote = void 0,
  48. lines = void 0,
  49. last = void 0,
  50. content = void 0,
  51. escape = void 0,
  52. nextLine = void 0,
  53. nextOffset = void 0,
  54. escaped = void 0,
  55. escapePos = void 0,
  56. inInterpolant = void 0,
  57. inComment = void 0,
  58. inString = void 0;
  59. var length = css.length;
  60. var offset = -1;
  61. var line = l || 1;
  62. var pos = p || 0;
  63. while (pos < length) {
  64. code = css.charCodeAt(pos);
  65. if (code === newline) {
  66. offset = pos;
  67. line += 1;
  68. }
  69. switch (code) {
  70. case space:
  71. case tab:
  72. case cr:
  73. case feed:
  74. next = pos;
  75. do {
  76. next += 1;
  77. code = css.charCodeAt(next);
  78. if (code === newline) {
  79. offset = next;
  80. line += 1;
  81. }
  82. } while (code === space || code === tab || code === cr || code === feed);
  83. tokens.push(['space', css.slice(pos, next)]);
  84. pos = next - 1;
  85. break;
  86. case newline:
  87. tokens.push(['newline', '\n', line, pos - offset]);
  88. break;
  89. case plus:
  90. tokens.push(['+', '+', line, pos - offset]);
  91. break;
  92. case minus:
  93. tokens.push(['-', '-', line, pos - offset]);
  94. break;
  95. case decComb:
  96. tokens.push(['>', '>', line, pos - offset]);
  97. break;
  98. case adjComb:
  99. tokens.push(['~', '~', line, pos - offset]);
  100. break;
  101. case openCurly:
  102. tokens.push(['{', '{', line, pos - offset]);
  103. break;
  104. case closeCurly:
  105. if (inInterpolant) {
  106. inInterpolant = false;
  107. tokens.push(['endInterpolant', '}', line, pos - offset]);
  108. } else {
  109. tokens.push(['}', '}', line, pos - offset]);
  110. }
  111. break;
  112. case comma:
  113. tokens.push([',', ',', line, pos - offset]);
  114. break;
  115. case dollar:
  116. tokens.push(['$', '$', line, pos - offset]);
  117. break;
  118. case colon:
  119. tokens.push([':', ':', line, pos - offset]);
  120. break;
  121. case semicolon:
  122. tokens.push([';', ';', line, pos - offset]);
  123. break;
  124. case openBracket:
  125. tokens.push(['(', '(', line, pos - offset]);
  126. break;
  127. case closeBracket:
  128. tokens.push([')', ')', line, pos - offset]);
  129. break;
  130. case singleQuote:
  131. case doubleQuote:
  132. quote = code === singleQuote ? "'" : '"';
  133. tokens.push([quote, quote, line, pos - offset]);
  134. next = pos + 1;
  135. var _tokenizeString = (0, _tokenizeString3.default)(input, line, next, quote),
  136. t = _tokenizeString.tokens,
  137. _p = _tokenizeString.pos;
  138. tokens = tokens.concat(t);
  139. next = _p;
  140. pos = next;
  141. break;
  142. case at:
  143. tokens.push(['@', '@', line, pos - offset]);
  144. break;
  145. case backslash:
  146. next = pos;
  147. escape = true;
  148. while (css.charCodeAt(next + 1) === backslash) {
  149. next += 1;
  150. escape = !escape;
  151. }
  152. code = css.charCodeAt(next + 1);
  153. if (escape && code !== space && code !== newline && code !== tab && code !== cr && code !== feed) {
  154. next += 1;
  155. }
  156. tokens.push(['word', css.slice(pos, next + 1), line, pos - offset, line, next - offset]);
  157. pos = next;
  158. break;
  159. default:
  160. ident.lastIndex = pos;
  161. number.lastIndex = pos;
  162. wordEnd.lastIndex = pos;
  163. if (code === slash && css.charCodeAt(pos + 1) === asterisk) {
  164. inComment = true;
  165. tokens.push(['startComment', '/*', line, pos + 1 - offset]);
  166. next = pos + 1;
  167. var _tokenizeComment = (0, _tokenizeComment3.default)(input, line, next + 1),
  168. _t = _tokenizeComment.tokens,
  169. _l = _tokenizeComment.line,
  170. _p2 = _tokenizeComment.pos,
  171. o = _tokenizeComment.offset;
  172. tokens = tokens.concat(_t);
  173. next = _p2;
  174. line = _l;
  175. offset = o;
  176. pos = next;
  177. break;
  178. }
  179. if (code === asterisk && css.charCodeAt(pos + 1) !== slash) {
  180. tokens.push(['*', '*', line, pos - offset]);
  181. break;
  182. }
  183. if (inComment && code === asterisk && css.charCodeAt(pos + 1) === slash) {
  184. inComment = false;
  185. tokens.push(['endComment', '*/', line, pos + 1 - offset]);
  186. pos += 2;
  187. break;
  188. }
  189. if (code === slash && css.charCodeAt(pos + 1) !== slash) {
  190. tokens.push(['/', '/', line, pos - offset]);
  191. break;
  192. }
  193. if (code === hash && css.charCodeAt(pos + 1) === openCurly) {
  194. inInterpolant = true;
  195. tokens.push(['startInterpolant', '#{', line, pos + 1 - offset]);
  196. next = pos + 1;
  197. var _tokenizeInterpolant = (0, _tokenizeInterpolant3.default)(input, line, next + 1),
  198. _t2 = _tokenizeInterpolant.tokens,
  199. _p3 = _tokenizeInterpolant.pos;
  200. tokens = tokens.concat(_t2);
  201. next = _p3;
  202. pos = next;
  203. break;
  204. }
  205. if (code === slash && css.charCodeAt(pos + 1) === slash) {
  206. next = css.indexOf('\n', pos + 2);
  207. next = (next > 0 ? next : css.length) - 1;
  208. tokens.push(['scssComment', css.slice(pos, next + 1), line, pos - offset, line, next - offset]);
  209. pos = next;
  210. break;
  211. }
  212. if (ident.test(css) && (ident.lastIndex = pos || 1) && ident.exec(css).index === pos) {
  213. next = ident.lastIndex - 1;
  214. tokens.push(['ident', css.slice(pos, next + 1), line, pos - offset, line, next - offset]);
  215. pos = next;
  216. break;
  217. }
  218. if (number.test(css) && (number.lastIndex = pos || 1) && number.exec(css).index === pos) {
  219. next = number.lastIndex - 1;
  220. tokens.push(['number', css.slice(pos, next + 1), line, pos - offset, line, next - offset]);
  221. pos = next;
  222. break;
  223. }
  224. wordEnd.lastIndex = pos + 1;
  225. wordEnd.test(css);
  226. if (wordEnd.lastIndex === 0) {
  227. next = css.length - 1;
  228. } else {
  229. next = wordEnd.lastIndex - 2;
  230. }
  231. tokens.push(['word', css.slice(pos, next + 1), line, pos - offset, line, next - offset]);
  232. pos = next;
  233. break;
  234. }
  235. pos++;
  236. }
  237. return tokens;
  238. }