index.js 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287
  1. 'use strict';
  2. const fs = require('fs');
  3. const { Readable } = require('stream');
  4. const sysPath = require('path');
  5. const { promisify } = require('util');
  6. const picomatch = require('picomatch');
  7. const readdir = promisify(fs.readdir);
  8. const stat = promisify(fs.stat);
  9. const lstat = promisify(fs.lstat);
  10. const realpath = promisify(fs.realpath);
  11. /**
  12. * @typedef {Object} EntryInfo
  13. * @property {String} path
  14. * @property {String} fullPath
  15. * @property {fs.Stats=} stats
  16. * @property {fs.Dirent=} dirent
  17. * @property {String} basename
  18. */
  19. const BANG = '!';
  20. const RECURSIVE_ERROR_CODE = 'READDIRP_RECURSIVE_ERROR';
  21. const NORMAL_FLOW_ERRORS = new Set(['ENOENT', 'EPERM', 'EACCES', 'ELOOP', RECURSIVE_ERROR_CODE]);
  22. const FILE_TYPE = 'files';
  23. const DIR_TYPE = 'directories';
  24. const FILE_DIR_TYPE = 'files_directories';
  25. const EVERYTHING_TYPE = 'all';
  26. const ALL_TYPES = [FILE_TYPE, DIR_TYPE, FILE_DIR_TYPE, EVERYTHING_TYPE];
  27. const isNormalFlowError = error => NORMAL_FLOW_ERRORS.has(error.code);
  28. const [maj, min] = process.versions.node.split('.').slice(0, 2).map(n => Number.parseInt(n, 10));
  29. const wantBigintFsStats = process.platform === 'win32' && (maj > 10 || (maj === 10 && min >= 5));
  30. const normalizeFilter = filter => {
  31. if (filter === undefined) return;
  32. if (typeof filter === 'function') return filter;
  33. if (typeof filter === 'string') {
  34. const glob = picomatch(filter.trim());
  35. return entry => glob(entry.basename);
  36. }
  37. if (Array.isArray(filter)) {
  38. const positive = [];
  39. const negative = [];
  40. for (const item of filter) {
  41. const trimmed = item.trim();
  42. if (trimmed.charAt(0) === BANG) {
  43. negative.push(picomatch(trimmed.slice(1)));
  44. } else {
  45. positive.push(picomatch(trimmed));
  46. }
  47. }
  48. if (negative.length > 0) {
  49. if (positive.length > 0) {
  50. return entry =>
  51. positive.some(f => f(entry.basename)) && !negative.some(f => f(entry.basename));
  52. }
  53. return entry => !negative.some(f => f(entry.basename));
  54. }
  55. return entry => positive.some(f => f(entry.basename));
  56. }
  57. };
  58. class ReaddirpStream extends Readable {
  59. static get defaultOptions() {
  60. return {
  61. root: '.',
  62. /* eslint-disable no-unused-vars */
  63. fileFilter: (path) => true,
  64. directoryFilter: (path) => true,
  65. /* eslint-enable no-unused-vars */
  66. type: FILE_TYPE,
  67. lstat: false,
  68. depth: 2147483648,
  69. alwaysStat: false
  70. };
  71. }
  72. constructor(options = {}) {
  73. super({
  74. objectMode: true,
  75. autoDestroy: true,
  76. highWaterMark: options.highWaterMark || 4096
  77. });
  78. const opts = { ...ReaddirpStream.defaultOptions, ...options };
  79. const { root, type } = opts;
  80. this._fileFilter = normalizeFilter(opts.fileFilter);
  81. this._directoryFilter = normalizeFilter(opts.directoryFilter);
  82. const statMethod = opts.lstat ? lstat : stat;
  83. // Use bigint stats if it's windows and stat() supports options (node 10+).
  84. if (wantBigintFsStats) {
  85. this._stat = path => statMethod(path, { bigint: true });
  86. } else {
  87. this._stat = statMethod;
  88. }
  89. this._maxDepth = opts.depth;
  90. this._wantsDir = [DIR_TYPE, FILE_DIR_TYPE, EVERYTHING_TYPE].includes(type);
  91. this._wantsFile = [FILE_TYPE, FILE_DIR_TYPE, EVERYTHING_TYPE].includes(type);
  92. this._wantsEverything = type === EVERYTHING_TYPE;
  93. this._root = sysPath.resolve(root);
  94. this._isDirent = ('Dirent' in fs) && !opts.alwaysStat;
  95. this._statsProp = this._isDirent ? 'dirent' : 'stats';
  96. this._rdOptions = { encoding: 'utf8', withFileTypes: this._isDirent };
  97. // Launch stream with one parent, the root dir.
  98. this.parents = [this._exploreDir(root, 1)];
  99. this.reading = false;
  100. this.parent = undefined;
  101. }
  102. async _read(batch) {
  103. if (this.reading) return;
  104. this.reading = true;
  105. try {
  106. while (!this.destroyed && batch > 0) {
  107. const { path, depth, files = [] } = this.parent || {};
  108. if (files.length > 0) {
  109. const slice = files.splice(0, batch).map(dirent => this._formatEntry(dirent, path));
  110. for (const entry of await Promise.all(slice)) {
  111. if (this.destroyed) return;
  112. const entryType = await this._getEntryType(entry);
  113. if (entryType === 'directory' && this._directoryFilter(entry)) {
  114. if (depth <= this._maxDepth) {
  115. this.parents.push(this._exploreDir(entry.fullPath, depth + 1));
  116. }
  117. if (this._wantsDir) {
  118. this.push(entry);
  119. batch--;
  120. }
  121. } else if ((entryType === 'file' || this._includeAsFile(entry)) && this._fileFilter(entry)) {
  122. if (this._wantsFile) {
  123. this.push(entry);
  124. batch--;
  125. }
  126. }
  127. }
  128. } else {
  129. const parent = this.parents.pop();
  130. if (!parent) {
  131. this.push(null);
  132. break;
  133. }
  134. this.parent = await parent;
  135. if (this.destroyed) return;
  136. }
  137. }
  138. } catch (error) {
  139. this.destroy(error);
  140. } finally {
  141. this.reading = false;
  142. }
  143. }
  144. async _exploreDir(path, depth) {
  145. let files;
  146. try {
  147. files = await readdir(path, this._rdOptions);
  148. } catch (error) {
  149. this._onError(error);
  150. }
  151. return { files, depth, path };
  152. }
  153. async _formatEntry(dirent, path) {
  154. let entry;
  155. try {
  156. const basename = this._isDirent ? dirent.name : dirent;
  157. const fullPath = sysPath.resolve(sysPath.join(path, basename));
  158. entry = { path: sysPath.relative(this._root, fullPath), fullPath, basename };
  159. entry[this._statsProp] = this._isDirent ? dirent : await this._stat(fullPath);
  160. } catch (err) {
  161. this._onError(err);
  162. }
  163. return entry;
  164. }
  165. _onError(err) {
  166. if (isNormalFlowError(err) && !this.destroyed) {
  167. this.emit('warn', err);
  168. } else {
  169. this.destroy(err);
  170. }
  171. }
  172. async _getEntryType(entry) {
  173. // entry may be undefined, because a warning or an error were emitted
  174. // and the statsProp is undefined
  175. const stats = entry && entry[this._statsProp];
  176. if (!stats) {
  177. return;
  178. }
  179. if (stats.isFile()) {
  180. return 'file';
  181. }
  182. if (stats.isDirectory()) {
  183. return 'directory';
  184. }
  185. if (stats && stats.isSymbolicLink()) {
  186. const full = entry.fullPath;
  187. try {
  188. const entryRealPath = await realpath(full);
  189. const entryRealPathStats = await lstat(entryRealPath);
  190. if (entryRealPathStats.isFile()) {
  191. return 'file';
  192. }
  193. if (entryRealPathStats.isDirectory()) {
  194. const len = entryRealPath.length;
  195. if (full.startsWith(entryRealPath) && full.substr(len, 1) === sysPath.sep) {
  196. const recursiveError = new Error(
  197. `Circular symlink detected: "${full}" points to "${entryRealPath}"`
  198. );
  199. recursiveError.code = RECURSIVE_ERROR_CODE;
  200. return this._onError(recursiveError);
  201. }
  202. return 'directory';
  203. }
  204. } catch (error) {
  205. this._onError(error);
  206. }
  207. }
  208. }
  209. _includeAsFile(entry) {
  210. const stats = entry && entry[this._statsProp];
  211. return stats && this._wantsEverything && !stats.isDirectory();
  212. }
  213. }
  214. /**
  215. * @typedef {Object} ReaddirpArguments
  216. * @property {Function=} fileFilter
  217. * @property {Function=} directoryFilter
  218. * @property {String=} type
  219. * @property {Number=} depth
  220. * @property {String=} root
  221. * @property {Boolean=} lstat
  222. * @property {Boolean=} bigint
  223. */
  224. /**
  225. * Main function which ends up calling readdirRec and reads all files and directories in given root recursively.
  226. * @param {String} root Root directory
  227. * @param {ReaddirpArguments=} options Options to specify root (start directory), filters and recursion depth
  228. */
  229. const readdirp = (root, options = {}) => {
  230. let type = options.entryType || options.type;
  231. if (type === 'both') type = FILE_DIR_TYPE; // backwards-compatibility
  232. if (type) options.type = type;
  233. if (!root) {
  234. throw new Error('readdirp: root argument is required. Usage: readdirp(root, options)');
  235. } else if (typeof root !== 'string') {
  236. throw new TypeError('readdirp: root argument must be a string. Usage: readdirp(root, options)');
  237. } else if (type && !ALL_TYPES.includes(type)) {
  238. throw new Error(`readdirp: Invalid type passed. Use one of ${ALL_TYPES.join(', ')}`);
  239. }
  240. options.root = root;
  241. return new ReaddirpStream(options);
  242. };
  243. const readdirpPromise = (root, options = {}) => {
  244. return new Promise((resolve, reject) => {
  245. const files = [];
  246. readdirp(root, options)
  247. .on('data', entry => files.push(entry))
  248. .on('end', () => resolve(files))
  249. .on('error', error => reject(error));
  250. });
  251. };
  252. readdirp.promise = readdirpPromise;
  253. readdirp.ReaddirpStream = ReaddirpStream;
  254. readdirp.default = readdirp;
  255. module.exports = readdirp;