pack.js 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397
  1. 'use strict'
  2. // A readable tar stream creator
  3. // Technically, this is a transform stream that you write paths into,
  4. // and tar format comes out of.
  5. // The `add()` method is like `write()` but returns this,
  6. // and end() return `this` as well, so you can
  7. // do `new Pack(opt).add('files').add('dir').end().pipe(output)
  8. // You could also do something like:
  9. // streamOfPaths().pipe(new Pack()).pipe(new fs.WriteStream('out.tar'))
  10. class PackJob {
  11. constructor (path, absolute) {
  12. this.path = path || './'
  13. this.absolute = absolute
  14. this.entry = null
  15. this.stat = null
  16. this.readdir = null
  17. this.pending = false
  18. this.ignore = false
  19. this.piped = false
  20. }
  21. }
  22. const MiniPass = require('minipass')
  23. const zlib = require('minizlib')
  24. const ReadEntry = require('./read-entry.js')
  25. const WriteEntry = require('./write-entry.js')
  26. const WriteEntrySync = WriteEntry.Sync
  27. const WriteEntryTar = WriteEntry.Tar
  28. const Yallist = require('yallist')
  29. const EOF = Buffer.alloc(1024)
  30. const ONSTAT = Symbol('onStat')
  31. const ENDED = Symbol('ended')
  32. const QUEUE = Symbol('queue')
  33. const CURRENT = Symbol('current')
  34. const PROCESS = Symbol('process')
  35. const PROCESSING = Symbol('processing')
  36. const PROCESSJOB = Symbol('processJob')
  37. const JOBS = Symbol('jobs')
  38. const JOBDONE = Symbol('jobDone')
  39. const ADDFSENTRY = Symbol('addFSEntry')
  40. const ADDTARENTRY = Symbol('addTarEntry')
  41. const STAT = Symbol('stat')
  42. const READDIR = Symbol('readdir')
  43. const ONREADDIR = Symbol('onreaddir')
  44. const PIPE = Symbol('pipe')
  45. const ENTRY = Symbol('entry')
  46. const ENTRYOPT = Symbol('entryOpt')
  47. const WRITEENTRYCLASS = Symbol('writeEntryClass')
  48. const WRITE = Symbol('write')
  49. const ONDRAIN = Symbol('ondrain')
  50. const fs = require('fs')
  51. const path = require('path')
  52. const warner = require('./warn-mixin.js')
  53. const normPath = require('./normalize-windows-path.js')
  54. const Pack = warner(class Pack extends MiniPass {
  55. constructor (opt) {
  56. super(opt)
  57. opt = opt || Object.create(null)
  58. this.opt = opt
  59. this.file = opt.file || ''
  60. this.cwd = opt.cwd || process.cwd()
  61. this.maxReadSize = opt.maxReadSize
  62. this.preservePaths = !!opt.preservePaths
  63. this.strict = !!opt.strict
  64. this.noPax = !!opt.noPax
  65. this.prefix = normPath(opt.prefix || '')
  66. this.linkCache = opt.linkCache || new Map()
  67. this.statCache = opt.statCache || new Map()
  68. this.readdirCache = opt.readdirCache || new Map()
  69. this[WRITEENTRYCLASS] = WriteEntry
  70. if (typeof opt.onwarn === 'function')
  71. this.on('warn', opt.onwarn)
  72. this.portable = !!opt.portable
  73. this.zip = null
  74. if (opt.gzip) {
  75. if (typeof opt.gzip !== 'object')
  76. opt.gzip = {}
  77. if (this.portable)
  78. opt.gzip.portable = true
  79. this.zip = new zlib.Gzip(opt.gzip)
  80. this.zip.on('data', chunk => super.write(chunk))
  81. this.zip.on('end', _ => super.end())
  82. this.zip.on('drain', _ => this[ONDRAIN]())
  83. this.on('resume', _ => this.zip.resume())
  84. } else
  85. this.on('drain', this[ONDRAIN])
  86. this.noDirRecurse = !!opt.noDirRecurse
  87. this.follow = !!opt.follow
  88. this.noMtime = !!opt.noMtime
  89. this.mtime = opt.mtime || null
  90. this.filter = typeof opt.filter === 'function' ? opt.filter : _ => true
  91. this[QUEUE] = new Yallist()
  92. this[JOBS] = 0
  93. this.jobs = +opt.jobs || 4
  94. this[PROCESSING] = false
  95. this[ENDED] = false
  96. }
  97. [WRITE] (chunk) {
  98. return super.write(chunk)
  99. }
  100. add (path) {
  101. this.write(path)
  102. return this
  103. }
  104. end (path) {
  105. if (path)
  106. this.write(path)
  107. this[ENDED] = true
  108. this[PROCESS]()
  109. return this
  110. }
  111. write (path) {
  112. if (this[ENDED])
  113. throw new Error('write after end')
  114. if (path instanceof ReadEntry)
  115. this[ADDTARENTRY](path)
  116. else
  117. this[ADDFSENTRY](path)
  118. return this.flowing
  119. }
  120. [ADDTARENTRY] (p) {
  121. const absolute = normPath(path.resolve(this.cwd, p.path))
  122. // in this case, we don't have to wait for the stat
  123. if (!this.filter(p.path, p))
  124. p.resume()
  125. else {
  126. const job = new PackJob(p.path, absolute, false)
  127. job.entry = new WriteEntryTar(p, this[ENTRYOPT](job))
  128. job.entry.on('end', _ => this[JOBDONE](job))
  129. this[JOBS] += 1
  130. this[QUEUE].push(job)
  131. }
  132. this[PROCESS]()
  133. }
  134. [ADDFSENTRY] (p) {
  135. const absolute = normPath(path.resolve(this.cwd, p))
  136. this[QUEUE].push(new PackJob(p, absolute))
  137. this[PROCESS]()
  138. }
  139. [STAT] (job) {
  140. job.pending = true
  141. this[JOBS] += 1
  142. const stat = this.follow ? 'stat' : 'lstat'
  143. fs[stat](job.absolute, (er, stat) => {
  144. job.pending = false
  145. this[JOBS] -= 1
  146. if (er)
  147. this.emit('error', er)
  148. else
  149. this[ONSTAT](job, stat)
  150. })
  151. }
  152. [ONSTAT] (job, stat) {
  153. this.statCache.set(job.absolute, stat)
  154. job.stat = stat
  155. // now we have the stat, we can filter it.
  156. if (!this.filter(job.path, stat))
  157. job.ignore = true
  158. this[PROCESS]()
  159. }
  160. [READDIR] (job) {
  161. job.pending = true
  162. this[JOBS] += 1
  163. fs.readdir(job.absolute, (er, entries) => {
  164. job.pending = false
  165. this[JOBS] -= 1
  166. if (er)
  167. return this.emit('error', er)
  168. this[ONREADDIR](job, entries)
  169. })
  170. }
  171. [ONREADDIR] (job, entries) {
  172. this.readdirCache.set(job.absolute, entries)
  173. job.readdir = entries
  174. this[PROCESS]()
  175. }
  176. [PROCESS] () {
  177. if (this[PROCESSING])
  178. return
  179. this[PROCESSING] = true
  180. for (let w = this[QUEUE].head;
  181. w !== null && this[JOBS] < this.jobs;
  182. w = w.next) {
  183. this[PROCESSJOB](w.value)
  184. if (w.value.ignore) {
  185. const p = w.next
  186. this[QUEUE].removeNode(w)
  187. w.next = p
  188. }
  189. }
  190. this[PROCESSING] = false
  191. if (this[ENDED] && !this[QUEUE].length && this[JOBS] === 0) {
  192. if (this.zip)
  193. this.zip.end(EOF)
  194. else {
  195. super.write(EOF)
  196. super.end()
  197. }
  198. }
  199. }
  200. get [CURRENT] () {
  201. return this[QUEUE] && this[QUEUE].head && this[QUEUE].head.value
  202. }
  203. [JOBDONE] (job) {
  204. this[QUEUE].shift()
  205. this[JOBS] -= 1
  206. this[PROCESS]()
  207. }
  208. [PROCESSJOB] (job) {
  209. if (job.pending)
  210. return
  211. if (job.entry) {
  212. if (job === this[CURRENT] && !job.piped)
  213. this[PIPE](job)
  214. return
  215. }
  216. if (!job.stat) {
  217. if (this.statCache.has(job.absolute))
  218. this[ONSTAT](job, this.statCache.get(job.absolute))
  219. else
  220. this[STAT](job)
  221. }
  222. if (!job.stat)
  223. return
  224. // filtered out!
  225. if (job.ignore)
  226. return
  227. if (!this.noDirRecurse && job.stat.isDirectory() && !job.readdir) {
  228. if (this.readdirCache.has(job.absolute))
  229. this[ONREADDIR](job, this.readdirCache.get(job.absolute))
  230. else
  231. this[READDIR](job)
  232. if (!job.readdir)
  233. return
  234. }
  235. // we know it doesn't have an entry, because that got checked above
  236. job.entry = this[ENTRY](job)
  237. if (!job.entry) {
  238. job.ignore = true
  239. return
  240. }
  241. if (job === this[CURRENT] && !job.piped)
  242. this[PIPE](job)
  243. }
  244. [ENTRYOPT] (job) {
  245. return {
  246. onwarn: (code, msg, data) => this.warn(code, msg, data),
  247. noPax: this.noPax,
  248. cwd: this.cwd,
  249. absolute: job.absolute,
  250. preservePaths: this.preservePaths,
  251. maxReadSize: this.maxReadSize,
  252. strict: this.strict,
  253. portable: this.portable,
  254. linkCache: this.linkCache,
  255. statCache: this.statCache,
  256. noMtime: this.noMtime,
  257. mtime: this.mtime,
  258. prefix: this.prefix,
  259. }
  260. }
  261. [ENTRY] (job) {
  262. this[JOBS] += 1
  263. try {
  264. return new this[WRITEENTRYCLASS](job.path, this[ENTRYOPT](job))
  265. .on('end', () => this[JOBDONE](job))
  266. .on('error', er => this.emit('error', er))
  267. } catch (er) {
  268. this.emit('error', er)
  269. }
  270. }
  271. [ONDRAIN] () {
  272. if (this[CURRENT] && this[CURRENT].entry)
  273. this[CURRENT].entry.resume()
  274. }
  275. // like .pipe() but using super, because our write() is special
  276. [PIPE] (job) {
  277. job.piped = true
  278. if (job.readdir) {
  279. job.readdir.forEach(entry => {
  280. const p = job.path
  281. const base = p === './' ? '' : p.replace(/\/*$/, '/')
  282. this[ADDFSENTRY](base + entry)
  283. })
  284. }
  285. const source = job.entry
  286. const zip = this.zip
  287. if (zip) {
  288. source.on('data', chunk => {
  289. if (!zip.write(chunk))
  290. source.pause()
  291. })
  292. } else {
  293. source.on('data', chunk => {
  294. if (!super.write(chunk))
  295. source.pause()
  296. })
  297. }
  298. }
  299. pause () {
  300. if (this.zip)
  301. this.zip.pause()
  302. return super.pause()
  303. }
  304. })
  305. class PackSync extends Pack {
  306. constructor (opt) {
  307. super(opt)
  308. this[WRITEENTRYCLASS] = WriteEntrySync
  309. }
  310. // pause/resume are no-ops in sync streams.
  311. pause () {}
  312. resume () {}
  313. [STAT] (job) {
  314. const stat = this.follow ? 'statSync' : 'lstatSync'
  315. this[ONSTAT](job, fs[stat](job.absolute))
  316. }
  317. [READDIR] (job, stat) {
  318. this[ONREADDIR](job, fs.readdirSync(job.absolute))
  319. }
  320. // gotta get it all in this tick
  321. [PIPE] (job) {
  322. const source = job.entry
  323. const zip = this.zip
  324. if (job.readdir) {
  325. job.readdir.forEach(entry => {
  326. const p = job.path
  327. const base = p === './' ? '' : p.replace(/\/*$/, '/')
  328. this[ADDFSENTRY](base + entry)
  329. })
  330. }
  331. if (zip) {
  332. source.on('data', chunk => {
  333. zip.write(chunk)
  334. })
  335. } else {
  336. source.on('data', chunk => {
  337. super[WRITE](chunk)
  338. })
  339. }
  340. }
  341. }
  342. Pack.Sync = PackSync
  343. module.exports = Pack