rewrite-pattern.js 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346
  1. 'use strict';
  2. const generate = require('regjsgen').generate;
  3. const parse = require('regjsparser').parse;
  4. const regenerate = require('regenerate');
  5. const unicodeMatchProperty = require('unicode-match-property-ecmascript');
  6. const unicodeMatchPropertyValue = require('unicode-match-property-value-ecmascript');
  7. const iuMappings = require('./data/iu-mappings.js');
  8. const ESCAPE_SETS = require('./data/character-class-escape-sets.js');
  9. // Prepare a Regenerate set containing all code points, used for negative
  10. // character classes (if any).
  11. const UNICODE_SET = regenerate().addRange(0x0, 0x10FFFF);
  12. // Without the `u` flag, the range stops at 0xFFFF.
  13. // https://mths.be/es6#sec-pattern-semantics
  14. const BMP_SET = regenerate().addRange(0x0, 0xFFFF);
  15. // Prepare a Regenerate set containing all code points that are supposed to be
  16. // matched by `/./u`. https://mths.be/es6#sec-atom
  17. const DOT_SET_UNICODE = UNICODE_SET.clone() // all Unicode code points
  18. .remove(
  19. // minus `LineTerminator`s (https://mths.be/es6#sec-line-terminators):
  20. 0x000A, // Line Feed <LF>
  21. 0x000D, // Carriage Return <CR>
  22. 0x2028, // Line Separator <LS>
  23. 0x2029 // Paragraph Separator <PS>
  24. );
  25. const getCharacterClassEscapeSet = (character, unicode, ignoreCase) => {
  26. if (unicode) {
  27. if (ignoreCase) {
  28. return ESCAPE_SETS.UNICODE_IGNORE_CASE.get(character);
  29. }
  30. return ESCAPE_SETS.UNICODE.get(character);
  31. }
  32. return ESCAPE_SETS.REGULAR.get(character);
  33. };
  34. const getUnicodeDotSet = (dotAll) => {
  35. return dotAll ? UNICODE_SET : DOT_SET_UNICODE;
  36. };
  37. const getUnicodePropertyValueSet = (property, value) => {
  38. const path = value ?
  39. `${ property }/${ value }` :
  40. `Binary_Property/${ property }`;
  41. try {
  42. return require(`regenerate-unicode-properties/${ path }.js`);
  43. } catch (exception) {
  44. throw new Error(
  45. `Failed to recognize value \`${ value }\` for property ` +
  46. `\`${ property }\`.`
  47. );
  48. }
  49. };
  50. const handleLoneUnicodePropertyNameOrValue = (value) => {
  51. // It could be a `General_Category` value or a binary property.
  52. // Note: `unicodeMatchPropertyValue` throws on invalid values.
  53. try {
  54. const property = 'General_Category';
  55. const category = unicodeMatchPropertyValue(property, value);
  56. return getUnicodePropertyValueSet(property, category);
  57. } catch (exception) {}
  58. // It’s not a `General_Category` value, so check if it’s a binary
  59. // property. Note: `unicodeMatchProperty` throws on invalid properties.
  60. const property = unicodeMatchProperty(value);
  61. return getUnicodePropertyValueSet(property);
  62. };
  63. const getUnicodePropertyEscapeSet = (value, isNegative) => {
  64. const parts = value.split('=');
  65. const firstPart = parts[0];
  66. let set;
  67. if (parts.length == 1) {
  68. set = handleLoneUnicodePropertyNameOrValue(firstPart);
  69. } else {
  70. // The pattern consists of two parts, i.e. `Property=Value`.
  71. const property = unicodeMatchProperty(firstPart);
  72. const value = unicodeMatchPropertyValue(property, parts[1]);
  73. set = getUnicodePropertyValueSet(property, value);
  74. }
  75. if (isNegative) {
  76. return UNICODE_SET.clone().remove(set);
  77. }
  78. return set.clone();
  79. };
  80. // Given a range of code points, add any case-folded code points in that range
  81. // to a set.
  82. regenerate.prototype.iuAddRange = function(min, max) {
  83. const $this = this;
  84. do {
  85. const folded = caseFold(min);
  86. if (folded) {
  87. $this.add(folded);
  88. }
  89. } while (++min <= max);
  90. return $this;
  91. };
  92. const update = (item, pattern) => {
  93. let tree = parse(pattern, config.useUnicodeFlag ? 'u' : '');
  94. switch (tree.type) {
  95. case 'characterClass':
  96. case 'group':
  97. case 'value':
  98. // No wrapping needed.
  99. break;
  100. default:
  101. // Wrap the pattern in a non-capturing group.
  102. tree = wrap(tree, pattern);
  103. }
  104. Object.assign(item, tree);
  105. };
  106. const wrap = (tree, pattern) => {
  107. // Wrap the pattern in a non-capturing group.
  108. return {
  109. 'type': 'group',
  110. 'behavior': 'ignore',
  111. 'body': [tree],
  112. 'raw': `(?:${ pattern })`
  113. };
  114. };
  115. const caseFold = (codePoint) => {
  116. return iuMappings.get(codePoint) || false;
  117. };
  118. const processCharacterClass = (characterClassItem, regenerateOptions) => {
  119. const set = regenerate();
  120. for (const item of characterClassItem.body) {
  121. switch (item.type) {
  122. case 'value':
  123. set.add(item.codePoint);
  124. if (config.ignoreCase && config.unicode && !config.useUnicodeFlag) {
  125. const folded = caseFold(item.codePoint);
  126. if (folded) {
  127. set.add(folded);
  128. }
  129. }
  130. break;
  131. case 'characterClassRange':
  132. const min = item.min.codePoint;
  133. const max = item.max.codePoint;
  134. set.addRange(min, max);
  135. if (config.ignoreCase && config.unicode && !config.useUnicodeFlag) {
  136. set.iuAddRange(min, max);
  137. }
  138. break;
  139. case 'characterClassEscape':
  140. set.add(getCharacterClassEscapeSet(
  141. item.value,
  142. config.unicode,
  143. config.ignoreCase
  144. ));
  145. break;
  146. case 'unicodePropertyEscape':
  147. set.add(getUnicodePropertyEscapeSet(item.value, item.negative));
  148. break;
  149. // The `default` clause is only here as a safeguard; it should never be
  150. // reached. Code coverage tools should ignore it.
  151. /* istanbul ignore next */
  152. default:
  153. throw new Error(`Unknown term type: ${ item.type }`);
  154. }
  155. }
  156. if (characterClassItem.negative) {
  157. update(characterClassItem, `(?!${set.toString(regenerateOptions)})[\\s\\S]`)
  158. } else {
  159. update(characterClassItem, set.toString(regenerateOptions));
  160. }
  161. return characterClassItem;
  162. };
  163. const updateNamedReference = (item, index) => {
  164. delete item.name;
  165. item.matchIndex = index;
  166. };
  167. const assertNoUnmatchedReferences = (groups) => {
  168. const unmatchedReferencesNames = Object.keys(groups.unmatchedReferences);
  169. if (unmatchedReferencesNames.length > 0) {
  170. throw new Error(`Unknown group names: ${unmatchedReferencesNames}`);
  171. }
  172. };
  173. const processTerm = (item, regenerateOptions, groups) => {
  174. switch (item.type) {
  175. case 'dot':
  176. if (config.useDotAllFlag) {
  177. break;
  178. } else if (config.unicode) {
  179. update(
  180. item,
  181. getUnicodeDotSet(config.dotAll).toString(regenerateOptions)
  182. );
  183. } else if (config.dotAll) {
  184. // TODO: consider changing this at the regenerate level.
  185. update(item, '[\\s\\S]');
  186. }
  187. break;
  188. case 'characterClass':
  189. item = processCharacterClass(item, regenerateOptions);
  190. break;
  191. case 'unicodePropertyEscape':
  192. if (config.unicodePropertyEscape) {
  193. update(
  194. item,
  195. getUnicodePropertyEscapeSet(item.value, item.negative)
  196. .toString(regenerateOptions)
  197. );
  198. }
  199. break;
  200. case 'characterClassEscape':
  201. update(
  202. item,
  203. getCharacterClassEscapeSet(
  204. item.value,
  205. config.unicode,
  206. config.ignoreCase
  207. ).toString(regenerateOptions)
  208. );
  209. break;
  210. case 'group':
  211. if (item.behavior == 'normal') {
  212. groups.lastIndex++;
  213. }
  214. if (item.name && config.namedGroup) {
  215. const name = item.name.value;
  216. if (groups.names[name]) {
  217. throw new Error(
  218. `Multiple groups with the same name (${ name }) are not allowed.`
  219. );
  220. }
  221. const index = groups.lastIndex;
  222. delete item.name;
  223. groups.names[name] = index;
  224. if (groups.onNamedGroup) {
  225. groups.onNamedGroup.call(null, name, index);
  226. }
  227. if (groups.unmatchedReferences[name]) {
  228. groups.unmatchedReferences[name].forEach(reference => {
  229. updateNamedReference(reference, index);
  230. });
  231. delete groups.unmatchedReferences[name];
  232. }
  233. }
  234. /* falls through */
  235. case 'alternative':
  236. case 'disjunction':
  237. case 'quantifier':
  238. item.body = item.body.map(term => {
  239. return processTerm(term, regenerateOptions, groups);
  240. });
  241. break;
  242. case 'value':
  243. const codePoint = item.codePoint;
  244. const set = regenerate(codePoint);
  245. if (config.ignoreCase && config.unicode && !config.useUnicodeFlag) {
  246. const folded = caseFold(codePoint);
  247. if (folded) {
  248. set.add(folded);
  249. }
  250. }
  251. update(item, set.toString(regenerateOptions));
  252. break;
  253. case 'reference':
  254. if (item.name) {
  255. const name = item.name.value;
  256. const index = groups.names[name];
  257. if (index) {
  258. updateNamedReference(item, index);
  259. break;
  260. }
  261. if (!groups.unmatchedReferences[name]) {
  262. groups.unmatchedReferences[name] = [];
  263. }
  264. // Keep track of references used before the corresponding group.
  265. groups.unmatchedReferences[name].push(item);
  266. }
  267. break;
  268. case 'anchor':
  269. case 'empty':
  270. case 'group':
  271. // Nothing to do here.
  272. break;
  273. // The `default` clause is only here as a safeguard; it should never be
  274. // reached. Code coverage tools should ignore it.
  275. /* istanbul ignore next */
  276. default:
  277. throw new Error(`Unknown term type: ${ item.type }`);
  278. }
  279. return item;
  280. };
  281. const config = {
  282. 'ignoreCase': false,
  283. 'unicode': false,
  284. 'dotAll': false,
  285. 'useDotAllFlag': false,
  286. 'useUnicodeFlag': false,
  287. 'unicodePropertyEscape': false,
  288. 'namedGroup': false
  289. };
  290. const rewritePattern = (pattern, flags, options) => {
  291. config.unicode = flags && flags.includes('u');
  292. const regjsparserFeatures = {
  293. 'unicodePropertyEscape': config.unicode,
  294. 'namedGroups': true,
  295. 'lookbehind': options && options.lookbehind
  296. };
  297. config.ignoreCase = flags && flags.includes('i');
  298. const supportDotAllFlag = options && options.dotAllFlag;
  299. config.dotAll = supportDotAllFlag && flags && flags.includes('s');
  300. config.namedGroup = options && options.namedGroup;
  301. config.useDotAllFlag = options && options.useDotAllFlag;
  302. config.useUnicodeFlag = options && options.useUnicodeFlag;
  303. config.unicodePropertyEscape = options && options.unicodePropertyEscape;
  304. if (supportDotAllFlag && config.useDotAllFlag) {
  305. throw new Error('`useDotAllFlag` and `dotAllFlag` cannot both be true!');
  306. }
  307. const regenerateOptions = {
  308. 'hasUnicodeFlag': config.useUnicodeFlag,
  309. 'bmpOnly': !config.unicode
  310. };
  311. const groups = {
  312. 'onNamedGroup': options && options.onNamedGroup,
  313. 'lastIndex': 0,
  314. 'names': Object.create(null), // { [name]: index }
  315. 'unmatchedReferences': Object.create(null) // { [name]: Array<reference> }
  316. };
  317. const tree = parse(pattern, flags, regjsparserFeatures);
  318. // Note: `processTerm` mutates `tree` and `groups`.
  319. processTerm(tree, regenerateOptions, groups);
  320. assertNoUnmatchedReferences(groups);
  321. return generate(tree);
  322. };
  323. module.exports = rewritePattern;