body.js 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334
  1. 'use strict'
  2. const Minipass = require('minipass')
  3. const MinipassSized = require('minipass-sized')
  4. const Blob = require('./blob.js')
  5. const {BUFFER} = Blob
  6. const FetchError = require('./fetch-error.js')
  7. // optional dependency on 'encoding'
  8. let convert
  9. try {
  10. convert = require('encoding').convert
  11. } catch (e) {}
  12. const INTERNALS = Symbol('Body internals')
  13. const CONSUME_BODY = Symbol('consumeBody')
  14. class Body {
  15. constructor (bodyArg, options = {}) {
  16. const { size = 0, timeout = 0 } = options
  17. const body = bodyArg === undefined || bodyArg === null ? null
  18. : isURLSearchParams(bodyArg) ? Buffer.from(bodyArg.toString())
  19. : isBlob(bodyArg) ? bodyArg
  20. : Buffer.isBuffer(bodyArg) ? bodyArg
  21. : Object.prototype.toString.call(bodyArg) === '[object ArrayBuffer]'
  22. ? Buffer.from(bodyArg)
  23. : ArrayBuffer.isView(bodyArg)
  24. ? Buffer.from(bodyArg.buffer, bodyArg.byteOffset, bodyArg.byteLength)
  25. : Minipass.isStream(bodyArg) ? bodyArg
  26. : Buffer.from(String(bodyArg))
  27. this[INTERNALS] = {
  28. body,
  29. disturbed: false,
  30. error: null,
  31. }
  32. this.size = size
  33. this.timeout = timeout
  34. if (Minipass.isStream(body)) {
  35. body.on('error', er => {
  36. const error = er.name === 'AbortError' ? er
  37. : new FetchError(`Invalid response while trying to fetch ${
  38. this.url}: ${er.message}`, 'system', er)
  39. this[INTERNALS].error = error
  40. })
  41. }
  42. }
  43. get body () {
  44. return this[INTERNALS].body
  45. }
  46. get bodyUsed () {
  47. return this[INTERNALS].disturbed
  48. }
  49. arrayBuffer () {
  50. return this[CONSUME_BODY]().then(buf =>
  51. buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength))
  52. }
  53. blob () {
  54. const ct = this.headers && this.headers.get('content-type') || ''
  55. return this[CONSUME_BODY]().then(buf => Object.assign(
  56. new Blob([], { type: ct.toLowerCase() }),
  57. { [BUFFER]: buf }
  58. ))
  59. }
  60. json () {
  61. return this[CONSUME_BODY]().then(buf => {
  62. try {
  63. return JSON.parse(buf.toString())
  64. } catch (er) {
  65. return Promise.reject(new FetchError(
  66. `invalid json response body at ${
  67. this.url} reason: ${er.message}`, 'invalid-json'))
  68. }
  69. })
  70. }
  71. text () {
  72. return this[CONSUME_BODY]().then(buf => buf.toString())
  73. }
  74. buffer () {
  75. return this[CONSUME_BODY]()
  76. }
  77. textConverted () {
  78. return this[CONSUME_BODY]().then(buf => convertBody(buf, this.headers))
  79. }
  80. [CONSUME_BODY] () {
  81. if (this[INTERNALS].disturbed)
  82. return Promise.reject(new TypeError(`body used already for: ${
  83. this.url}`))
  84. this[INTERNALS].disturbed = true
  85. if (this[INTERNALS].error)
  86. return Promise.reject(this[INTERNALS].error)
  87. // body is null
  88. if (this.body === null) {
  89. return Promise.resolve(Buffer.alloc(0))
  90. }
  91. if (Buffer.isBuffer(this.body))
  92. return Promise.resolve(this.body)
  93. const upstream = isBlob(this.body) ? this.body.stream() : this.body
  94. /* istanbul ignore if: should never happen */
  95. if (!Minipass.isStream(upstream))
  96. return Promise.resolve(Buffer.alloc(0))
  97. const stream = this.size && upstream instanceof MinipassSized ? upstream
  98. : !this.size && upstream instanceof Minipass &&
  99. !(upstream instanceof MinipassSized) ? upstream
  100. : this.size ? new MinipassSized({ size: this.size })
  101. : new Minipass()
  102. // allow timeout on slow response body
  103. const resTimeout = this.timeout ? setTimeout(() => {
  104. stream.emit('error', new FetchError(
  105. `Response timeout while trying to fetch ${
  106. this.url} (over ${this.timeout}ms)`, 'body-timeout'))
  107. }, this.timeout) : null
  108. // do not keep the process open just for this timeout, even
  109. // though we expect it'll get cleared eventually.
  110. if (resTimeout) {
  111. resTimeout.unref()
  112. }
  113. // do the pipe in the promise, because the pipe() can send too much
  114. // data through right away and upset the MP Sized object
  115. return new Promise((resolve, reject) => {
  116. // if the stream is some other kind of stream, then pipe through a MP
  117. // so we can collect it more easily.
  118. if (stream !== upstream) {
  119. upstream.on('error', er => stream.emit('error', er))
  120. upstream.pipe(stream)
  121. }
  122. resolve()
  123. }).then(() => stream.concat()).then(buf => {
  124. clearTimeout(resTimeout)
  125. return buf
  126. }).catch(er => {
  127. clearTimeout(resTimeout)
  128. // request was aborted, reject with this Error
  129. if (er.name === 'AbortError' || er.name === 'FetchError')
  130. throw er
  131. else if (er.name === 'RangeError')
  132. throw new FetchError(`Could not create Buffer from response body for ${
  133. this.url}: ${er.message}`, 'system', er)
  134. else
  135. // other errors, such as incorrect content-encoding or content-length
  136. throw new FetchError(`Invalid response body while trying to fetch ${
  137. this.url}: ${er.message}`, 'system', er)
  138. })
  139. }
  140. static clone (instance) {
  141. if (instance.bodyUsed)
  142. throw new Error('cannot clone body after it is used')
  143. const body = instance.body
  144. // check that body is a stream and not form-data object
  145. // NB: can't clone the form-data object without having it as a dependency
  146. if (Minipass.isStream(body) && typeof body.getBoundary !== 'function') {
  147. // create a dedicated tee stream so that we don't lose data
  148. // potentially sitting in the body stream's buffer by writing it
  149. // immediately to p1 and not having it for p2.
  150. const tee = new Minipass()
  151. const p1 = new Minipass()
  152. const p2 = new Minipass()
  153. tee.on('error', er => {
  154. p1.emit('error', er)
  155. p2.emit('error', er)
  156. })
  157. body.on('error', er => tee.emit('error', er))
  158. tee.pipe(p1)
  159. tee.pipe(p2)
  160. body.pipe(tee)
  161. // set instance body to one fork, return the other
  162. instance[INTERNALS].body = p1
  163. return p2
  164. } else
  165. return instance.body
  166. }
  167. static extractContentType (body) {
  168. return body === null || body === undefined ? null
  169. : typeof body === 'string' ? 'text/plain;charset=UTF-8'
  170. : isURLSearchParams(body)
  171. ? 'application/x-www-form-urlencoded;charset=UTF-8'
  172. : isBlob(body) ? body.type || null
  173. : Buffer.isBuffer(body) ? null
  174. : Object.prototype.toString.call(body) === '[object ArrayBuffer]' ? null
  175. : ArrayBuffer.isView(body) ? null
  176. : typeof body.getBoundary === 'function'
  177. ? `multipart/form-data;boundary=${body.getBoundary()}`
  178. : Minipass.isStream(body) ? null
  179. : 'text/plain;charset=UTF-8'
  180. }
  181. static getTotalBytes (instance) {
  182. const {body} = instance
  183. return (body === null || body === undefined) ? 0
  184. : isBlob(body) ? body.size
  185. : Buffer.isBuffer(body) ? body.length
  186. : body && typeof body.getLengthSync === 'function' && (
  187. // detect form data input from form-data module
  188. body._lengthRetrievers &&
  189. /* istanbul ignore next */ body._lengthRetrievers.length == 0 || // 1.x
  190. body.hasKnownLength && body.hasKnownLength()) // 2.x
  191. ? body.getLengthSync()
  192. : null
  193. }
  194. static writeToStream (dest, instance) {
  195. const {body} = instance
  196. if (body === null || body === undefined)
  197. dest.end()
  198. else if (Buffer.isBuffer(body) || typeof body === 'string')
  199. dest.end(body)
  200. else {
  201. // body is stream or blob
  202. const stream = isBlob(body) ? body.stream() : body
  203. stream.on('error', er => dest.emit('error', er)).pipe(dest)
  204. }
  205. return dest
  206. }
  207. }
  208. Object.defineProperties(Body.prototype, {
  209. body: { enumerable: true },
  210. bodyUsed: { enumerable: true },
  211. arrayBuffer: { enumerable: true },
  212. blob: { enumerable: true },
  213. json: { enumerable: true },
  214. text: { enumerable: true }
  215. })
  216. const isURLSearchParams = obj =>
  217. // Duck-typing as a necessary condition.
  218. (typeof obj !== 'object' ||
  219. typeof obj.append !== 'function' ||
  220. typeof obj.delete !== 'function' ||
  221. typeof obj.get !== 'function' ||
  222. typeof obj.getAll !== 'function' ||
  223. typeof obj.has !== 'function' ||
  224. typeof obj.set !== 'function') ? false
  225. // Brand-checking and more duck-typing as optional condition.
  226. : obj.constructor.name === 'URLSearchParams' ||
  227. Object.prototype.toString.call(obj) === '[object URLSearchParams]' ||
  228. typeof obj.sort === 'function'
  229. const isBlob = obj =>
  230. typeof obj === 'object' &&
  231. typeof obj.arrayBuffer === 'function' &&
  232. typeof obj.type === 'string' &&
  233. typeof obj.stream === 'function' &&
  234. typeof obj.constructor === 'function' &&
  235. typeof obj.constructor.name === 'string' &&
  236. /^(Blob|File)$/.test(obj.constructor.name) &&
  237. /^(Blob|File)$/.test(obj[Symbol.toStringTag])
  238. const convertBody = (buffer, headers) => {
  239. /* istanbul ignore if */
  240. if (typeof convert !== 'function')
  241. throw new Error('The package `encoding` must be installed to use the textConverted() function')
  242. const ct = headers && headers.get('content-type')
  243. let charset = 'utf-8'
  244. let res, str
  245. // header
  246. if (ct)
  247. res = /charset=([^;]*)/i.exec(ct)
  248. // no charset in content type, peek at response body for at most 1024 bytes
  249. str = buffer.slice(0, 1024).toString()
  250. // html5
  251. if (!res && str)
  252. res = /<meta.+?charset=(['"])(.+?)\1/i.exec(str)
  253. // html4
  254. if (!res && str) {
  255. res = /<meta[\s]+?http-equiv=(['"])content-type\1[\s]+?content=(['"])(.+?)\2/i.exec(str)
  256. if (!res) {
  257. res = /<meta[\s]+?content=(['"])(.+?)\1[\s]+?http-equiv=(['"])content-type\3/i.exec(str)
  258. if (res)
  259. res.pop() // drop last quote
  260. }
  261. if (res)
  262. res = /charset=(.*)/i.exec(res.pop())
  263. }
  264. // xml
  265. if (!res && str)
  266. res = /<\?xml.+?encoding=(['"])(.+?)\1/i.exec(str)
  267. // found charset
  268. if (res) {
  269. charset = res.pop()
  270. // prevent decode issues when sites use incorrect encoding
  271. // ref: https://hsivonen.fi/encoding-menu/
  272. if (charset === 'gb2312' || charset === 'gbk')
  273. charset = 'gb18030'
  274. }
  275. // turn raw buffers into a single utf-8 buffer
  276. return convert(
  277. buffer,
  278. 'UTF-8',
  279. charset
  280. ).toString()
  281. }
  282. module.exports = Body