index.js 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282
  1. 'use strict';
  2. const fs = require('fs');
  3. const { Readable } = require('stream');
  4. const sysPath = require('path');
  5. const { promisify } = require('util');
  6. const picomatch = require('picomatch');
  7. const readdir = promisify(fs.readdir);
  8. const stat = promisify(fs.stat);
  9. const lstat = promisify(fs.lstat);
  10. const realpath = promisify(fs.realpath);
  11. /**
  12. * @typedef {Object} EntryInfo
  13. * @property {String} path
  14. * @property {String} fullPath
  15. * @property {fs.Stats=} stats
  16. * @property {fs.Dirent=} dirent
  17. * @property {String} basename
  18. */
  19. const BANG = '!';
  20. const NORMAL_FLOW_ERRORS = new Set(['ENOENT', 'EPERM', 'EACCES', 'ELOOP']);
  21. const FILE_TYPE = 'files';
  22. const DIR_TYPE = 'directories';
  23. const FILE_DIR_TYPE = 'files_directories';
  24. const EVERYTHING_TYPE = 'all';
  25. const ALL_TYPES = [FILE_TYPE, DIR_TYPE, FILE_DIR_TYPE, EVERYTHING_TYPE];
  26. const isNormalFlowError = error => NORMAL_FLOW_ERRORS.has(error.code);
  27. const normalizeFilter = filter => {
  28. if (filter === undefined) return;
  29. if (typeof filter === 'function') return filter;
  30. if (typeof filter === 'string') {
  31. const glob = picomatch(filter.trim());
  32. return entry => glob(entry.basename);
  33. }
  34. if (Array.isArray(filter)) {
  35. const positive = [];
  36. const negative = [];
  37. for (const item of filter) {
  38. const trimmed = item.trim();
  39. if (trimmed.charAt(0) === BANG) {
  40. negative.push(picomatch(trimmed.slice(1)));
  41. } else {
  42. positive.push(picomatch(trimmed));
  43. }
  44. }
  45. if (negative.length > 0) {
  46. if (positive.length > 0) {
  47. return entry =>
  48. positive.some(f => f(entry.basename)) && !negative.some(f => f(entry.basename));
  49. }
  50. return entry => !negative.some(f => f(entry.basename));
  51. }
  52. return entry => positive.some(f => f(entry.basename));
  53. }
  54. };
  55. class ReaddirpStream extends Readable {
  56. static get defaultOptions() {
  57. return {
  58. root: '.',
  59. /* eslint-disable no-unused-vars */
  60. fileFilter: (path) => true,
  61. directoryFilter: (path) => true,
  62. /* eslint-enable no-unused-vars */
  63. type: FILE_TYPE,
  64. lstat: false,
  65. depth: 2147483648,
  66. alwaysStat: false
  67. };
  68. }
  69. constructor(options = {}) {
  70. super({
  71. objectMode: true,
  72. autoDestroy: true,
  73. highWaterMark: options.highWaterMark || 4096
  74. });
  75. const opts = { ...ReaddirpStream.defaultOptions, ...options };
  76. const { root, type } = opts;
  77. this._fileFilter = normalizeFilter(opts.fileFilter);
  78. this._directoryFilter = normalizeFilter(opts.directoryFilter);
  79. const statMethod = opts.lstat ? lstat : stat;
  80. // Use bigint stats if it's windows and stat() supports options (node 10+).
  81. if (process.platform === 'win32' && stat.length === 3) {
  82. this._stat = path => statMethod(path, { bigint: true });
  83. } else {
  84. this._stat = statMethod;
  85. }
  86. this._maxDepth = opts.depth;
  87. this._wantsDir = [DIR_TYPE, FILE_DIR_TYPE, EVERYTHING_TYPE].includes(type);
  88. this._wantsFile = [FILE_TYPE, FILE_DIR_TYPE, EVERYTHING_TYPE].includes(type);
  89. this._wantsEverything = type === EVERYTHING_TYPE;
  90. this._root = sysPath.resolve(root);
  91. this._isDirent = ('Dirent' in fs) && !opts.alwaysStat;
  92. this._statsProp = this._isDirent ? 'dirent' : 'stats';
  93. this._rdOptions = { encoding: 'utf8', withFileTypes: this._isDirent };
  94. // Launch stream with one parent, the root dir.
  95. this.parents = [this._exploreDir(root, 1)];
  96. this.reading = false;
  97. this.parent = undefined;
  98. }
  99. async _read(batch) {
  100. if (this.reading) return;
  101. this.reading = true;
  102. try {
  103. while (!this.destroyed && batch > 0) {
  104. const { path, depth, files = [] } = this.parent || {};
  105. if (files.length > 0) {
  106. const slice = files.splice(0, batch).map(dirent => this._formatEntry(dirent, path));
  107. for (const entry of await Promise.all(slice)) {
  108. if (this.destroyed) return;
  109. const entryType = await this._getEntryType(entry);
  110. if (entryType === 'directory' && this._directoryFilter(entry)) {
  111. if (depth <= this._maxDepth) {
  112. this.parents.push(this._exploreDir(entry.fullPath, depth + 1));
  113. }
  114. if (this._wantsDir) {
  115. this.push(entry);
  116. batch--;
  117. }
  118. } else if ((entryType === 'file' || this._includeAsFile(entry)) && this._fileFilter(entry)) {
  119. if (this._wantsFile) {
  120. this.push(entry);
  121. batch--;
  122. }
  123. }
  124. }
  125. } else {
  126. const parent = this.parents.pop();
  127. if (!parent) {
  128. this.push(null);
  129. break;
  130. }
  131. this.parent = await parent;
  132. if (this.destroyed) return;
  133. }
  134. }
  135. } catch (error) {
  136. this.destroy(error);
  137. } finally {
  138. this.reading = false;
  139. }
  140. }
  141. async _exploreDir(path, depth) {
  142. let files;
  143. try {
  144. files = await readdir(path, this._rdOptions);
  145. } catch (error) {
  146. this._onError(error);
  147. }
  148. return {files, depth, path};
  149. }
  150. async _formatEntry(dirent, path) {
  151. let entry;
  152. try {
  153. const basename = this._isDirent ? dirent.name : dirent;
  154. const fullPath = sysPath.resolve(sysPath.join(path, basename));
  155. entry = {path: sysPath.relative(this._root, fullPath), fullPath, basename};
  156. entry[this._statsProp] = this._isDirent ? dirent : await this._stat(fullPath);
  157. } catch (err) {
  158. this._onError(err);
  159. }
  160. return entry;
  161. }
  162. _onError(err) {
  163. if (isNormalFlowError(err) && !this.destroyed) {
  164. this.emit('warn', err);
  165. } else {
  166. this.destroy(err);
  167. }
  168. }
  169. async _getEntryType(entry) {
  170. // entry may be undefined, because a warning or an error were emitted
  171. // and the statsProp is undefined
  172. const stats = entry && entry[this._statsProp];
  173. if (!stats) {
  174. return;
  175. }
  176. if (stats.isFile()) {
  177. return 'file';
  178. }
  179. if (stats.isDirectory()) {
  180. return 'directory';
  181. }
  182. if (stats && stats.isSymbolicLink()) {
  183. const full = entry.fullPath;
  184. try {
  185. const entryRealPath = await realpath(full);
  186. const entryRealPathStats = await lstat(entryRealPath);
  187. if (entryRealPathStats.isFile()) {
  188. return 'file';
  189. }
  190. if (entryRealPathStats.isDirectory()) {
  191. const len = entryRealPath.length;
  192. if (full.startsWith(entryRealPath) && full.substr(len, 1) === sysPath.sep) {
  193. return this._onError(new Error(
  194. `Circular symlink detected: "${full}" points to "${entryRealPath}"`
  195. ));
  196. }
  197. return 'directory';
  198. }
  199. } catch (error) {
  200. this._onError(error);
  201. }
  202. }
  203. }
  204. _includeAsFile(entry) {
  205. const stats = entry && entry[this._statsProp];
  206. return stats && this._wantsEverything && !stats.isDirectory();
  207. }
  208. }
  209. /**
  210. * @typedef {Object} ReaddirpArguments
  211. * @property {Function=} fileFilter
  212. * @property {Function=} directoryFilter
  213. * @property {String=} type
  214. * @property {Number=} depth
  215. * @property {String=} root
  216. * @property {Boolean=} lstat
  217. * @property {Boolean=} bigint
  218. */
  219. /**
  220. * Main function which ends up calling readdirRec and reads all files and directories in given root recursively.
  221. * @param {String} root Root directory
  222. * @param {ReaddirpArguments=} options Options to specify root (start directory), filters and recursion depth
  223. */
  224. const readdirp = (root, options = {}) => {
  225. let type = options.entryType || options.type;
  226. if (type === 'both') type = FILE_DIR_TYPE; // backwards-compatibility
  227. if (type) options.type = type;
  228. if (!root) {
  229. throw new Error('readdirp: root argument is required. Usage: readdirp(root, options)');
  230. } else if (typeof root !== 'string') {
  231. throw new TypeError('readdirp: root argument must be a string. Usage: readdirp(root, options)');
  232. } else if (type && !ALL_TYPES.includes(type)) {
  233. throw new Error(`readdirp: Invalid type passed. Use one of ${ALL_TYPES.join(', ')}`);
  234. }
  235. options.root = root;
  236. return new ReaddirpStream(options);
  237. };
  238. const readdirpPromise = (root, options = {}) => {
  239. return new Promise((resolve, reject) => {
  240. const files = [];
  241. readdirp(root, options)
  242. .on('data', entry => files.push(entry))
  243. .on('end', () => resolve(files))
  244. .on('error', error => reject(error));
  245. });
  246. };
  247. readdirp.promise = readdirpPromise;
  248. readdirp.ReaddirpStream = ReaddirpStream;
  249. readdirp.default = readdirp;
  250. module.exports = readdirp;