index.js 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341
  1. 'use strict'
  2. const Url = require('url')
  3. const http = require('http')
  4. const https = require('https')
  5. const zlib = require('minizlib')
  6. const Minipass = require('minipass')
  7. const Body = require('./body.js')
  8. const { writeToStream, getTotalBytes } = Body
  9. const Response = require('./response.js')
  10. const Headers = require('./headers.js')
  11. const { createHeadersLenient } = Headers
  12. const Request = require('./request.js')
  13. const { getNodeRequestOptions } = Request
  14. const FetchError = require('./fetch-error.js')
  15. const AbortError = require('./abort-error.js')
  16. const resolveUrl = Url.resolve
  17. const fetch = (url, opts) => {
  18. if (/^data:/.test(url)) {
  19. const request = new Request(url, opts)
  20. try {
  21. const split = url.split(',')
  22. const data = Buffer.from(split[1], 'base64')
  23. const type = split[0].match(/^data:(.*);base64$/)[1]
  24. return Promise.resolve(new Response(data, {
  25. headers: {
  26. 'Content-Type': type,
  27. 'Content-Length': data.length,
  28. }
  29. }))
  30. } catch (er) {
  31. return Promise.reject(new FetchError(`[${request.method}] ${
  32. request.url} invalid URL, ${er.message}`, 'system', er))
  33. }
  34. }
  35. return new Promise((resolve, reject) => {
  36. // build request object
  37. const request = new Request(url, opts)
  38. let options
  39. try {
  40. options = getNodeRequestOptions(request)
  41. } catch (er) {
  42. return reject(er)
  43. }
  44. const send = (options.protocol === 'https:' ? https : http).request
  45. const { signal } = request
  46. let response = null
  47. const abort = () => {
  48. const error = new AbortError('The user aborted a request.')
  49. reject(error)
  50. if (Minipass.isStream(request.body) &&
  51. typeof request.body.destroy === 'function') {
  52. request.body.destroy(error)
  53. }
  54. if (response && response.body) {
  55. response.body.emit('error', error)
  56. }
  57. }
  58. if (signal && signal.aborted)
  59. return abort()
  60. const abortAndFinalize = () => {
  61. abort()
  62. finalize()
  63. }
  64. const finalize = () => {
  65. req.abort()
  66. if (signal)
  67. signal.removeEventListener('abort', abortAndFinalize)
  68. clearTimeout(reqTimeout)
  69. }
  70. // send request
  71. const req = send(options)
  72. if (signal)
  73. signal.addEventListener('abort', abortAndFinalize)
  74. let reqTimeout = null
  75. if (request.timeout) {
  76. req.once('socket', socket => {
  77. reqTimeout = setTimeout(() => {
  78. reject(new FetchError(`network timeout at: ${
  79. request.url}`, 'request-timeout'))
  80. finalize()
  81. }, request.timeout)
  82. })
  83. }
  84. req.on('error', er => {
  85. // if a 'response' event is emitted before the 'error' event, then by the
  86. // time this handler is run it's too late to reject the Promise for the
  87. // response. instead, we forward the error event to the response stream
  88. // so that the error will surface to the user when they try to consume
  89. // the body. this is done as a side effect of aborting the request except
  90. // for in windows, where we must forward the event manually, otherwise
  91. // there is no longer a ref'd socket attached to the request and the
  92. // stream never ends so the event loop runs out of work and the process
  93. // exits without warning.
  94. // coverage skipped here due to the difficulty in testing
  95. // istanbul ignore next
  96. if (req.res)
  97. req.res.emit('error', er)
  98. reject(new FetchError(`request to ${request.url} failed, reason: ${
  99. er.message}`, 'system', er))
  100. finalize()
  101. })
  102. req.on('response', res => {
  103. clearTimeout(reqTimeout)
  104. const headers = createHeadersLenient(res.headers)
  105. // HTTP fetch step 5
  106. if (fetch.isRedirect(res.statusCode)) {
  107. // HTTP fetch step 5.2
  108. const location = headers.get('Location')
  109. // HTTP fetch step 5.3
  110. const locationURL = location === null ? null
  111. : resolveUrl(request.url, location)
  112. // HTTP fetch step 5.5
  113. switch (request.redirect) {
  114. case 'error':
  115. reject(new FetchError(`uri requested responds with a redirect, redirect mode is set to error: ${
  116. request.url}`, 'no-redirect'))
  117. finalize()
  118. return
  119. case 'manual':
  120. // node-fetch-specific step: make manual redirect a bit easier to
  121. // use by setting the Location header value to the resolved URL.
  122. if (locationURL !== null) {
  123. // handle corrupted header
  124. try {
  125. headers.set('Location', locationURL)
  126. } catch (err) {
  127. /* istanbul ignore next: nodejs server prevent invalid
  128. response headers, we can't test this through normal
  129. request */
  130. reject(err)
  131. }
  132. }
  133. break
  134. case 'follow':
  135. // HTTP-redirect fetch step 2
  136. if (locationURL === null) {
  137. break
  138. }
  139. // HTTP-redirect fetch step 5
  140. if (request.counter >= request.follow) {
  141. reject(new FetchError(`maximum redirect reached at: ${
  142. request.url}`, 'max-redirect'))
  143. finalize()
  144. return
  145. }
  146. // HTTP-redirect fetch step 9
  147. if (res.statusCode !== 303 &&
  148. request.body &&
  149. getTotalBytes(request) === null) {
  150. reject(new FetchError(
  151. 'Cannot follow redirect with body being a readable stream',
  152. 'unsupported-redirect'
  153. ))
  154. finalize()
  155. return
  156. }
  157. // Update host due to redirection
  158. request.headers.set('host', Url.parse(locationURL).host)
  159. // HTTP-redirect fetch step 6 (counter increment)
  160. // Create a new Request object.
  161. const requestOpts = {
  162. headers: new Headers(request.headers),
  163. follow: request.follow,
  164. counter: request.counter + 1,
  165. agent: request.agent,
  166. compress: request.compress,
  167. method: request.method,
  168. body: request.body,
  169. signal: request.signal,
  170. timeout: request.timeout,
  171. }
  172. // HTTP-redirect fetch step 11
  173. if (res.statusCode === 303 || (
  174. (res.statusCode === 301 || res.statusCode === 302) &&
  175. request.method === 'POST'
  176. )) {
  177. requestOpts.method = 'GET'
  178. requestOpts.body = undefined
  179. requestOpts.headers.delete('content-length')
  180. }
  181. // HTTP-redirect fetch step 15
  182. resolve(fetch(new Request(locationURL, requestOpts)))
  183. finalize()
  184. return
  185. }
  186. } // end if(isRedirect)
  187. // prepare response
  188. res.once('end', () =>
  189. signal && signal.removeEventListener('abort', abortAndFinalize))
  190. const body = new Minipass()
  191. // exceedingly rare that the stream would have an error,
  192. // but just in case we proxy it to the stream in use.
  193. res.on('error', /* istanbul ignore next */ er => body.emit('error', er))
  194. res.on('data', (chunk) => body.write(chunk))
  195. res.on('end', () => body.end())
  196. const responseOptions = {
  197. url: request.url,
  198. status: res.statusCode,
  199. statusText: res.statusMessage,
  200. headers: headers,
  201. size: request.size,
  202. timeout: request.timeout,
  203. counter: request.counter,
  204. trailer: new Promise(resolve =>
  205. res.on('end', () => resolve(createHeadersLenient(res.trailers))))
  206. }
  207. // HTTP-network fetch step 12.1.1.3
  208. const codings = headers.get('Content-Encoding')
  209. // HTTP-network fetch step 12.1.1.4: handle content codings
  210. // in following scenarios we ignore compression support
  211. // 1. compression support is disabled
  212. // 2. HEAD request
  213. // 3. no Content-Encoding header
  214. // 4. no content response (204)
  215. // 5. content not modified response (304)
  216. if (!request.compress ||
  217. request.method === 'HEAD' ||
  218. codings === null ||
  219. res.statusCode === 204 ||
  220. res.statusCode === 304) {
  221. response = new Response(body, responseOptions)
  222. resolve(response)
  223. return
  224. }
  225. // Be less strict when decoding compressed responses, since sometimes
  226. // servers send slightly invalid responses that are still accepted
  227. // by common browsers.
  228. // Always using Z_SYNC_FLUSH is what cURL does.
  229. const zlibOptions = {
  230. flush: zlib.constants.Z_SYNC_FLUSH,
  231. finishFlush: zlib.constants.Z_SYNC_FLUSH,
  232. }
  233. // for gzip
  234. if (codings == 'gzip' || codings == 'x-gzip') {
  235. const unzip = new zlib.Gunzip(zlibOptions)
  236. response = new Response(
  237. // exceedingly rare that the stream would have an error,
  238. // but just in case we proxy it to the stream in use.
  239. body.on('error', /* istanbul ignore next */ er => unzip.emit('error', er)).pipe(unzip),
  240. responseOptions
  241. )
  242. resolve(response)
  243. return
  244. }
  245. // for deflate
  246. if (codings == 'deflate' || codings == 'x-deflate') {
  247. // handle the infamous raw deflate response from old servers
  248. // a hack for old IIS and Apache servers
  249. const raw = res.pipe(new Minipass())
  250. raw.once('data', chunk => {
  251. // see http://stackoverflow.com/questions/37519828
  252. const decoder = (chunk[0] & 0x0F) === 0x08
  253. ? new zlib.Inflate()
  254. : new zlib.InflateRaw()
  255. // exceedingly rare that the stream would have an error,
  256. // but just in case we proxy it to the stream in use.
  257. body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
  258. response = new Response(decoder, responseOptions)
  259. resolve(response)
  260. })
  261. return
  262. }
  263. // for br
  264. if (codings == 'br') {
  265. // ignoring coverage so tests don't have to fake support (or lack of) for brotli
  266. // istanbul ignore next
  267. try {
  268. var decoder = new zlib.BrotliDecompress()
  269. } catch (err) {
  270. reject(err)
  271. finalize()
  272. return
  273. }
  274. // exceedingly rare that the stream would have an error,
  275. // but just in case we proxy it to the stream in use.
  276. body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
  277. response = new Response(decoder, responseOptions)
  278. resolve(response)
  279. return
  280. }
  281. // otherwise, use response as-is
  282. response = new Response(body, responseOptions)
  283. resolve(response)
  284. })
  285. writeToStream(req, request)
  286. })
  287. }
  288. module.exports = fetch
  289. fetch.isRedirect = code =>
  290. code === 301 ||
  291. code === 302 ||
  292. code === 303 ||
  293. code === 307 ||
  294. code === 308
  295. fetch.Headers = Headers
  296. fetch.Request = Request
  297. fetch.Response = Response
  298. fetch.FetchError = FetchError