index.js 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531
  1. var url = require("url");
  2. var URL = url.URL;
  3. var http = require("http");
  4. var https = require("https");
  5. var Writable = require("stream").Writable;
  6. var assert = require("assert");
  7. var debug = require("./debug");
  8. // Create handlers that pass events from native requests
  9. var events = ["abort", "aborted", "connect", "error", "socket", "timeout"];
  10. var eventHandlers = Object.create(null);
  11. events.forEach(function (event) {
  12. eventHandlers[event] = function (arg1, arg2, arg3) {
  13. this._redirectable.emit(event, arg1, arg2, arg3);
  14. };
  15. });
  16. // Error types with codes
  17. var RedirectionError = createErrorType(
  18. "ERR_FR_REDIRECTION_FAILURE",
  19. ""
  20. );
  21. var TooManyRedirectsError = createErrorType(
  22. "ERR_FR_TOO_MANY_REDIRECTS",
  23. "Maximum number of redirects exceeded"
  24. );
  25. var MaxBodyLengthExceededError = createErrorType(
  26. "ERR_FR_MAX_BODY_LENGTH_EXCEEDED",
  27. "Request body larger than maxBodyLength limit"
  28. );
  29. var WriteAfterEndError = createErrorType(
  30. "ERR_STREAM_WRITE_AFTER_END",
  31. "write after end"
  32. );
  33. // An HTTP(S) request that can be redirected
  34. function RedirectableRequest(options, responseCallback) {
  35. // Initialize the request
  36. Writable.call(this);
  37. this._sanitizeOptions(options);
  38. this._options = options;
  39. this._ended = false;
  40. this._ending = false;
  41. this._redirectCount = 0;
  42. this._redirects = [];
  43. this._requestBodyLength = 0;
  44. this._requestBodyBuffers = [];
  45. // Attach a callback if passed
  46. if (responseCallback) {
  47. this.on("response", responseCallback);
  48. }
  49. // React to responses of native requests
  50. var self = this;
  51. this._onNativeResponse = function (response) {
  52. self._processResponse(response);
  53. };
  54. // Perform the first request
  55. this._performRequest();
  56. }
  57. RedirectableRequest.prototype = Object.create(Writable.prototype);
  58. RedirectableRequest.prototype.abort = function () {
  59. abortRequest(this._currentRequest);
  60. this.emit("abort");
  61. };
  62. // Writes buffered data to the current native request
  63. RedirectableRequest.prototype.write = function (data, encoding, callback) {
  64. // Writing is not allowed if end has been called
  65. if (this._ending) {
  66. throw new WriteAfterEndError();
  67. }
  68. // Validate input and shift parameters if necessary
  69. if (!(typeof data === "string" || typeof data === "object" && ("length" in data))) {
  70. throw new TypeError("data should be a string, Buffer or Uint8Array");
  71. }
  72. if (typeof encoding === "function") {
  73. callback = encoding;
  74. encoding = null;
  75. }
  76. // Ignore empty buffers, since writing them doesn't invoke the callback
  77. // https://github.com/nodejs/node/issues/22066
  78. if (data.length === 0) {
  79. if (callback) {
  80. callback();
  81. }
  82. return;
  83. }
  84. // Only write when we don't exceed the maximum body length
  85. if (this._requestBodyLength + data.length <= this._options.maxBodyLength) {
  86. this._requestBodyLength += data.length;
  87. this._requestBodyBuffers.push({ data: data, encoding: encoding });
  88. this._currentRequest.write(data, encoding, callback);
  89. }
  90. // Error when we exceed the maximum body length
  91. else {
  92. this.emit("error", new MaxBodyLengthExceededError());
  93. this.abort();
  94. }
  95. };
  96. // Ends the current native request
  97. RedirectableRequest.prototype.end = function (data, encoding, callback) {
  98. // Shift parameters if necessary
  99. if (typeof data === "function") {
  100. callback = data;
  101. data = encoding = null;
  102. }
  103. else if (typeof encoding === "function") {
  104. callback = encoding;
  105. encoding = null;
  106. }
  107. // Write data if needed and end
  108. if (!data) {
  109. this._ended = this._ending = true;
  110. this._currentRequest.end(null, null, callback);
  111. }
  112. else {
  113. var self = this;
  114. var currentRequest = this._currentRequest;
  115. this.write(data, encoding, function () {
  116. self._ended = true;
  117. currentRequest.end(null, null, callback);
  118. });
  119. this._ending = true;
  120. }
  121. };
  122. // Sets a header value on the current native request
  123. RedirectableRequest.prototype.setHeader = function (name, value) {
  124. this._options.headers[name] = value;
  125. this._currentRequest.setHeader(name, value);
  126. };
  127. // Clears a header value on the current native request
  128. RedirectableRequest.prototype.removeHeader = function (name) {
  129. delete this._options.headers[name];
  130. this._currentRequest.removeHeader(name);
  131. };
  132. // Global timeout for all underlying requests
  133. RedirectableRequest.prototype.setTimeout = function (msecs, callback) {
  134. var self = this;
  135. if (callback) {
  136. this.on("timeout", callback);
  137. }
  138. function destroyOnTimeout(socket) {
  139. socket.setTimeout(msecs);
  140. socket.removeListener("timeout", socket.destroy);
  141. socket.addListener("timeout", socket.destroy);
  142. }
  143. // Sets up a timer to trigger a timeout event
  144. function startTimer(socket) {
  145. if (self._timeout) {
  146. clearTimeout(self._timeout);
  147. }
  148. self._timeout = setTimeout(function () {
  149. self.emit("timeout");
  150. clearTimer();
  151. }, msecs);
  152. destroyOnTimeout(socket);
  153. }
  154. // Prevent a timeout from triggering
  155. function clearTimer() {
  156. clearTimeout(this._timeout);
  157. if (callback) {
  158. self.removeListener("timeout", callback);
  159. }
  160. if (!this.socket) {
  161. self._currentRequest.removeListener("socket", startTimer);
  162. }
  163. }
  164. // Start the timer when the socket is opened
  165. if (this.socket) {
  166. startTimer(this.socket);
  167. }
  168. else {
  169. this._currentRequest.once("socket", startTimer);
  170. }
  171. this.on("socket", destroyOnTimeout);
  172. this.once("response", clearTimer);
  173. this.once("error", clearTimer);
  174. return this;
  175. };
  176. // Proxy all other public ClientRequest methods
  177. [
  178. "flushHeaders", "getHeader",
  179. "setNoDelay", "setSocketKeepAlive",
  180. ].forEach(function (method) {
  181. RedirectableRequest.prototype[method] = function (a, b) {
  182. return this._currentRequest[method](a, b);
  183. };
  184. });
  185. // Proxy all public ClientRequest properties
  186. ["aborted", "connection", "socket"].forEach(function (property) {
  187. Object.defineProperty(RedirectableRequest.prototype, property, {
  188. get: function () { return this._currentRequest[property]; },
  189. });
  190. });
  191. RedirectableRequest.prototype._sanitizeOptions = function (options) {
  192. // Ensure headers are always present
  193. if (!options.headers) {
  194. options.headers = {};
  195. }
  196. // Since http.request treats host as an alias of hostname,
  197. // but the url module interprets host as hostname plus port,
  198. // eliminate the host property to avoid confusion.
  199. if (options.host) {
  200. // Use hostname if set, because it has precedence
  201. if (!options.hostname) {
  202. options.hostname = options.host;
  203. }
  204. delete options.host;
  205. }
  206. // Complete the URL object when necessary
  207. if (!options.pathname && options.path) {
  208. var searchPos = options.path.indexOf("?");
  209. if (searchPos < 0) {
  210. options.pathname = options.path;
  211. }
  212. else {
  213. options.pathname = options.path.substring(0, searchPos);
  214. options.search = options.path.substring(searchPos);
  215. }
  216. }
  217. };
  218. // Executes the next native request (initial or redirect)
  219. RedirectableRequest.prototype._performRequest = function () {
  220. // Load the native protocol
  221. var protocol = this._options.protocol;
  222. var nativeProtocol = this._options.nativeProtocols[protocol];
  223. if (!nativeProtocol) {
  224. this.emit("error", new TypeError("Unsupported protocol " + protocol));
  225. return;
  226. }
  227. // If specified, use the agent corresponding to the protocol
  228. // (HTTP and HTTPS use different types of agents)
  229. if (this._options.agents) {
  230. var scheme = protocol.substr(0, protocol.length - 1);
  231. this._options.agent = this._options.agents[scheme];
  232. }
  233. // Create the native request
  234. var request = this._currentRequest =
  235. nativeProtocol.request(this._options, this._onNativeResponse);
  236. this._currentUrl = url.format(this._options);
  237. // Set up event handlers
  238. request._redirectable = this;
  239. for (var e = 0; e < events.length; e++) {
  240. request.on(events[e], eventHandlers[events[e]]);
  241. }
  242. // End a redirected request
  243. // (The first request must be ended explicitly with RedirectableRequest#end)
  244. if (this._isRedirect) {
  245. // Write the request entity and end.
  246. var i = 0;
  247. var self = this;
  248. var buffers = this._requestBodyBuffers;
  249. (function writeNext(error) {
  250. // Only write if this request has not been redirected yet
  251. /* istanbul ignore else */
  252. if (request === self._currentRequest) {
  253. // Report any write errors
  254. /* istanbul ignore if */
  255. if (error) {
  256. self.emit("error", error);
  257. }
  258. // Write the next buffer if there are still left
  259. else if (i < buffers.length) {
  260. var buffer = buffers[i++];
  261. /* istanbul ignore else */
  262. if (!request.finished) {
  263. request.write(buffer.data, buffer.encoding, writeNext);
  264. }
  265. }
  266. // End the request if `end` has been called on us
  267. else if (self._ended) {
  268. request.end();
  269. }
  270. }
  271. }());
  272. }
  273. };
  274. // Processes a response from the current native request
  275. RedirectableRequest.prototype._processResponse = function (response) {
  276. // Store the redirected response
  277. var statusCode = response.statusCode;
  278. if (this._options.trackRedirects) {
  279. this._redirects.push({
  280. url: this._currentUrl,
  281. headers: response.headers,
  282. statusCode: statusCode,
  283. });
  284. }
  285. // RFC7231§6.4: The 3xx (Redirection) class of status code indicates
  286. // that further action needs to be taken by the user agent in order to
  287. // fulfill the request. If a Location header field is provided,
  288. // the user agent MAY automatically redirect its request to the URI
  289. // referenced by the Location field value,
  290. // even if the specific status code is not understood.
  291. var location = response.headers.location;
  292. if (location && this._options.followRedirects !== false &&
  293. statusCode >= 300 && statusCode < 400) {
  294. // Abort the current request
  295. abortRequest(this._currentRequest);
  296. // Discard the remainder of the response to avoid waiting for data
  297. response.destroy();
  298. // RFC7231§6.4: A client SHOULD detect and intervene
  299. // in cyclical redirections (i.e., "infinite" redirection loops).
  300. if (++this._redirectCount > this._options.maxRedirects) {
  301. this.emit("error", new TooManyRedirectsError());
  302. return;
  303. }
  304. // RFC7231§6.4: Automatic redirection needs to done with
  305. // care for methods not known to be safe, […]
  306. // RFC7231§6.4.2–3: For historical reasons, a user agent MAY change
  307. // the request method from POST to GET for the subsequent request.
  308. if ((statusCode === 301 || statusCode === 302) && this._options.method === "POST" ||
  309. // RFC7231§6.4.4: The 303 (See Other) status code indicates that
  310. // the server is redirecting the user agent to a different resource […]
  311. // A user agent can perform a retrieval request targeting that URI
  312. // (a GET or HEAD request if using HTTP) […]
  313. (statusCode === 303) && !/^(?:GET|HEAD)$/.test(this._options.method)) {
  314. this._options.method = "GET";
  315. // Drop a possible entity and headers related to it
  316. this._requestBodyBuffers = [];
  317. removeMatchingHeaders(/^content-/i, this._options.headers);
  318. }
  319. // Drop the Host header, as the redirect might lead to a different host
  320. var previousHostName = removeMatchingHeaders(/^host$/i, this._options.headers) ||
  321. url.parse(this._currentUrl).hostname;
  322. // Create the redirected request
  323. var redirectUrl = url.resolve(this._currentUrl, location);
  324. debug("redirecting to", redirectUrl);
  325. this._isRedirect = true;
  326. var redirectUrlParts = url.parse(redirectUrl);
  327. Object.assign(this._options, redirectUrlParts);
  328. // Drop the Authorization header if redirecting to another host
  329. if (redirectUrlParts.hostname !== previousHostName) {
  330. removeMatchingHeaders(/^authorization$/i, this._options.headers);
  331. }
  332. // Evaluate the beforeRedirect callback
  333. if (typeof this._options.beforeRedirect === "function") {
  334. var responseDetails = { headers: response.headers };
  335. try {
  336. this._options.beforeRedirect.call(null, this._options, responseDetails);
  337. }
  338. catch (err) {
  339. this.emit("error", err);
  340. return;
  341. }
  342. this._sanitizeOptions(this._options);
  343. }
  344. // Perform the redirected request
  345. try {
  346. this._performRequest();
  347. }
  348. catch (cause) {
  349. var error = new RedirectionError("Redirected request failed: " + cause.message);
  350. error.cause = cause;
  351. this.emit("error", error);
  352. }
  353. }
  354. else {
  355. // The response is not a redirect; return it as-is
  356. response.responseUrl = this._currentUrl;
  357. response.redirects = this._redirects;
  358. this.emit("response", response);
  359. // Clean up
  360. this._requestBodyBuffers = [];
  361. }
  362. };
  363. // Wraps the key/value object of protocols with redirect functionality
  364. function wrap(protocols) {
  365. // Default settings
  366. var exports = {
  367. maxRedirects: 21,
  368. maxBodyLength: 10 * 1024 * 1024,
  369. };
  370. // Wrap each protocol
  371. var nativeProtocols = {};
  372. Object.keys(protocols).forEach(function (scheme) {
  373. var protocol = scheme + ":";
  374. var nativeProtocol = nativeProtocols[protocol] = protocols[scheme];
  375. var wrappedProtocol = exports[scheme] = Object.create(nativeProtocol);
  376. // Executes a request, following redirects
  377. function request(input, options, callback) {
  378. // Parse parameters
  379. if (typeof input === "string") {
  380. var urlStr = input;
  381. try {
  382. input = urlToOptions(new URL(urlStr));
  383. }
  384. catch (err) {
  385. /* istanbul ignore next */
  386. input = url.parse(urlStr);
  387. }
  388. }
  389. else if (URL && (input instanceof URL)) {
  390. input = urlToOptions(input);
  391. }
  392. else {
  393. callback = options;
  394. options = input;
  395. input = { protocol: protocol };
  396. }
  397. if (typeof options === "function") {
  398. callback = options;
  399. options = null;
  400. }
  401. // Set defaults
  402. options = Object.assign({
  403. maxRedirects: exports.maxRedirects,
  404. maxBodyLength: exports.maxBodyLength,
  405. }, input, options);
  406. options.nativeProtocols = nativeProtocols;
  407. assert.equal(options.protocol, protocol, "protocol mismatch");
  408. debug("options", options);
  409. return new RedirectableRequest(options, callback);
  410. }
  411. // Executes a GET request, following redirects
  412. function get(input, options, callback) {
  413. var wrappedRequest = wrappedProtocol.request(input, options, callback);
  414. wrappedRequest.end();
  415. return wrappedRequest;
  416. }
  417. // Expose the properties on the wrapped protocol
  418. Object.defineProperties(wrappedProtocol, {
  419. request: { value: request, configurable: true, enumerable: true, writable: true },
  420. get: { value: get, configurable: true, enumerable: true, writable: true },
  421. });
  422. });
  423. return exports;
  424. }
  425. /* istanbul ignore next */
  426. function noop() { /* empty */ }
  427. // from https://github.com/nodejs/node/blob/master/lib/internal/url.js
  428. function urlToOptions(urlObject) {
  429. var options = {
  430. protocol: urlObject.protocol,
  431. hostname: urlObject.hostname.startsWith("[") ?
  432. /* istanbul ignore next */
  433. urlObject.hostname.slice(1, -1) :
  434. urlObject.hostname,
  435. hash: urlObject.hash,
  436. search: urlObject.search,
  437. pathname: urlObject.pathname,
  438. path: urlObject.pathname + urlObject.search,
  439. href: urlObject.href,
  440. };
  441. if (urlObject.port !== "") {
  442. options.port = Number(urlObject.port);
  443. }
  444. return options;
  445. }
  446. function removeMatchingHeaders(regex, headers) {
  447. var lastValue;
  448. for (var header in headers) {
  449. if (regex.test(header)) {
  450. lastValue = headers[header];
  451. delete headers[header];
  452. }
  453. }
  454. return lastValue;
  455. }
  456. function createErrorType(code, defaultMessage) {
  457. function CustomError(message) {
  458. Error.captureStackTrace(this, this.constructor);
  459. this.message = message || defaultMessage;
  460. }
  461. CustomError.prototype = new Error();
  462. CustomError.prototype.constructor = CustomError;
  463. CustomError.prototype.name = "Error [" + code + "]";
  464. CustomError.prototype.code = code;
  465. return CustomError;
  466. }
  467. function abortRequest(request) {
  468. for (var e = 0; e < events.length; e++) {
  469. request.removeListener(events[e], eventHandlers[events[e]]);
  470. }
  471. request.on("error", noop);
  472. request.abort();
  473. }
  474. // Exports
  475. module.exports = wrap({ http: http, https: https });
  476. module.exports.wrap = wrap;