|
# -*- coding: utf-8 -*-
requests.session ~~~~~~~~~~~~~~~~
This module provides a Session object to manage and persist settings across requests (cookies, auth, proxies). """
cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError)
requote_uri, get_environ_proxies, get_netrc_auth, should_bypass_proxies, get_auth_from_url, rewind_body )
# formerly defined here, reexposed here for backward compatibility
# Preferred clock, based on which one is more accurate on a given system. try: # Python 3.4+ preferred_clock = time.perf_counter except AttributeError: # Earlier than Python 3. preferred_clock = time.clock else:
"""Determines appropriate setting for a given request, taking into account the explicit setting on that request, and the setting in the session. If a setting is a dictionary, they will be merged together using `dict_class` """
# Bypass if not a dictionary (e.g. verify) isinstance(session_setting, Mapping) and isinstance(request_setting, Mapping) ):
# Remove keys that are set to None. Extract keys first to avoid altering # the dictionary during iteration. del merged_setting[key]
"""Properly merges both requests and session hooks.
This is necessary because when request_hooks == {'response': []}, the merge breaks Session hooks entirely. """
if request_hooks is None or request_hooks.get('response') == []: return session_hooks
return merge_setting(request_hooks, session_hooks, dict_class)
"""Receives a Response. Returns a redirect URI or ``None``""" # Due to the nature of how requests processes redirects this method will # be called at least once upon the original response and at least twice # on each subsequent redirect response (if any). # If a custom mixin is used to handle this logic, it may be advantageous # to cache the redirect location onto the response object as a private # attribute. # Currently the underlying http module on py3 decode headers # in latin1, but empirical evidence suggests that latin1 is very # rarely used with non-ASCII characters in HTTP headers. # It is more likely to get UTF8 header rather than latin1. # This causes incorrect handling of UTF8 encoded location headers. # To solve this, we re-encode the location in latin1.
"""Decide whether Authorization header should be removed when redirecting""" old_parsed = urlparse(old_url) new_parsed = urlparse(new_url) if old_parsed.hostname != new_parsed.hostname: return True # Special case: allow http -> https redirect when using the standard # ports. This isn't specified by RFC 7235, but is kept to avoid # breaking backwards compatibility with older versions of requests # that allowed any redirects on the same host. if (old_parsed.scheme == 'http' and old_parsed.port in (80, None) and new_parsed.scheme == 'https' and new_parsed.port in (443, None)): return False
# Handle default port usage corresponding to scheme. changed_port = old_parsed.port != new_parsed.port changed_scheme = old_parsed.scheme != new_parsed.scheme default_port = (DEFAULT_PORTS.get(old_parsed.scheme, None), None) if (not changed_scheme and old_parsed.port in default_port and new_parsed.port in default_port): return False
# Standard case: root URI must match return changed_port or changed_scheme
verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs): """Receives a Response. Returns a generator of Responses or Requests."""
# Update history and keep track of redirects. # resp.history must ignore the original request in this loop
except (ChunkedEncodingError, ContentDecodingError, RuntimeError): resp.raw.read(decode_content=False)
raise TooManyRedirects('Exceeded {} redirects.'.format(self.max_redirects), response=resp)
# Release the connection back into the pool.
# Handle redirection without scheme (see: RFC 1808 Section 4) parsed_rurl = urlparse(resp.url) url = ':'.join([to_native_string(parsed_rurl.scheme), url])
# Normalize url case and attach previous fragment if needed (RFC 7231 7.1.2) parsed = parsed._replace(fragment=previous_fragment) previous_fragment = parsed.fragment
# Facilitate relative 'location' headers, as allowed by RFC 7231. # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') # Compliant with RFC3986, we percent encode the url. url = urljoin(resp.url, requote_uri(url)) else:
# https://github.com/psf/requests/issues/1084 # https://github.com/psf/requests/issues/3490
# Extract any cookies sent on the response to the cookiejar # in the new request. Because we've mutated our copied prepared # request, use the old one that we haven't yet touched.
# Rebuild auth and proxy information.
# A failed tell() sets `_body_position` to `object()`. This non-None # value ensures `rewindable` will be True, allowing us to raise an # UnrewindableBodyError, instead of hanging the connection. prepared_request._body_position is not None and ('Content-Length' in headers or 'Transfer-Encoding' in headers) )
# Attempt to rewind consumed file-like object. rewind_body(prepared_request)
# Override the original request.
else:
req, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, allow_redirects=False, **adapter_kwargs )
# extract redirect url, if any, for the next loop
"""When being redirected we may want to strip authentication from the request to avoid leaking credentials. This method intelligently removes and reapplies authentication where possible to avoid credential loss. """
# If we get redirected to a new host, we should strip out any # authentication headers. del headers['Authorization']
# .netrc might have more auth for us on our new host. prepared_request.prepare_auth(new_auth)
"""This method re-evaluates the proxy configuration by considering the environment variables. If we are redirected to a URL covered by NO_PROXY, we strip the proxy configuration. Otherwise, we set missing proxy keys for this URL (in case they were stripped by a previous redirect).
This method also replaces the Proxy-Authorization header where necessary.
:rtype: dict """
new_proxies.setdefault(scheme, proxy)
del headers['Proxy-Authorization']
headers['Proxy-Authorization'] = _basic_auth_str(username, password)
"""When being redirected we may want to change the method of the request based on certain specs or browser behavior. """
# https://tools.ietf.org/html/rfc7231#section-6.4.4 method = 'GET'
# Do what the browsers do, despite standards... # First, turn 302s into GETs. method = 'GET'
# Second, if a POST is responded to with a 301, turn it into a GET. # This bizarre behaviour is explained in Issue 1704. method = 'GET'
"""A Requests session.
Provides cookie persistence, connection-pooling, and configuration.
Basic Usage::
>>> import requests >>> s = requests.Session() >>> s.get('https://httpbin.org/get') <Response [200]>
Or as a context manager::
>>> with requests.Session() as s: ... s.get('https://httpbin.org/get') <Response [200]> """
'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify', 'cert', 'adapters', 'stream', 'trust_env', 'max_redirects', ]
#: A case-insensitive dictionary of headers to be sent on each #: :class:`Request <Request>` sent from this #: :class:`Session <Session>`.
#: Default Authentication tuple or object to attach to #: :class:`Request <Request>`.
#: Dictionary mapping protocol or protocol and host to the URL of the proxy #: (e.g. {'http': 'foo.bar:3128', 'http://host.name': 'foo.bar:4012'}) to #: be used on each :class:`Request <Request>`.
#: Event-handling hooks.
#: Dictionary of querystring data to attach to each #: :class:`Request <Request>`. The dictionary values may be lists for #: representing multivalued query parameters.
#: Stream response content default.
#: SSL Verification default.
#: SSL client certificate default, if String, path to ssl client #: cert file (.pem). If Tuple, ('cert', 'key') pair.
#: Maximum number of redirects allowed. If the request exceeds this #: limit, a :class:`TooManyRedirects` exception is raised. #: This defaults to requests.models.DEFAULT_REDIRECT_LIMIT, which is #: 30.
#: Trust environment settings for proxy configuration, default #: authentication and similar.
#: A CookieJar containing all currently outstanding cookies set on this #: session. By default it is a #: :class:`RequestsCookieJar <requests.cookies.RequestsCookieJar>`, but #: may be any other ``cookielib.CookieJar`` compatible object.
# Default connection adapters.
return self
self.close()
"""Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it. The :class:`PreparedRequest` has settings merged from the :class:`Request <Request>` instance and those of the :class:`Session`.
:param request: :class:`Request` instance to prepare with this session's settings. :rtype: requests.PreparedRequest """
# Bootstrap CookieJar.
# Merge with session cookies merge_cookies(RequestsCookieJar(), self.cookies), cookies)
# Set environment's basic authentication if not explicitly set.
method=request.method.upper(), url=request.url, files=request.files, data=request.data, json=request.json, headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict), params=merge_setting(request.params, self.params), auth=merge_setting(auth, self.auth), cookies=merged_cookies, hooks=merge_hooks(request.hooks, self.hooks), )
params=None, data=None, headers=None, cookies=None, files=None, auth=None, timeout=None, allow_redirects=True, proxies=None, hooks=None, stream=None, verify=None, cert=None, json=None): """Constructs a :class:`Request <Request>`, prepares it and sends it. Returns :class:`Response <Response>` object.
:param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. :param data: (optional) Dictionary, list of tuples, bytes, or file-like object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of ``'filename': file-like-objects`` for multipart encoding upload. :param auth: (optional) Auth tuple or callable to enable Basic/Digest/Custom HTTP Auth. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple :param allow_redirects: (optional) Set to True by default. :type allow_redirects: bool :param proxies: (optional) Dictionary mapping protocol or protocol and hostname to the URL of the proxy. :param stream: (optional) whether to immediately download the response content. Defaults to ``False``. :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use. Defaults to ``True``. :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. :rtype: requests.Response """ # Create the Request. method=method.upper(), url=url, headers=headers, files=files, data=data or {}, json=json, params=params or {}, auth=auth, cookies=cookies, hooks=hooks, )
prep.url, proxies, stream, verify, cert )
# Send the request. 'timeout': timeout, 'allow_redirects': allow_redirects, }
r"""Sends a GET request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """
r"""Sends a OPTIONS request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """
kwargs.setdefault('allow_redirects', True) return self.request('OPTIONS', url, **kwargs)
r"""Sends a HEAD request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """
r"""Sends a POST request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, list of tuples, bytes, or file-like object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """
return self.request('POST', url, data=data, json=json, **kwargs)
r"""Sends a PUT request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, list of tuples, bytes, or file-like object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """
return self.request('PUT', url, data=data, **kwargs)
r"""Sends a PATCH request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, list of tuples, bytes, or file-like object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """
return self.request('PATCH', url, data=data, **kwargs)
r"""Sends a DELETE request. Returns :class:`Response` object.
:param url: URL for the new :class:`Request` object. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """
return self.request('DELETE', url, **kwargs)
"""Send a given PreparedRequest.
:rtype: requests.Response """ # Set defaults that the hooks can utilize to ensure they always have # the correct parameters to reproduce the previous request.
# It's possible that users might accidentally send a Request object. # Guard against that specific failure case. raise ValueError('You can only send PreparedRequests.')
# Set up variables needed for resolve_redirects and dispatching of hooks
# Get the appropriate adapter to use
# Start time (approximately) of the request
# Send the request
# Total elapsed time of the request (approximately)
# Response manipulation hooks
# Persist cookies
# If the hooks create history then we want those cookies too for resp in r.history: extract_cookies_to_jar(self.cookies, resp.request, resp.raw)
# Resolve redirects if allowed. # Redirect resolving generator. else:
# Shuffle things around if there's history. # Insert the first (original) request at the start # Get the last request made
# If redirects aren't being followed, store the response on the Request for Response.next().
""" Check the environment and merge it with some settings.
:rtype: dict """ # Gather clues from the surrounding environment. # Set environment's proxies. proxies.setdefault(k, v)
# Look for requests environment configuration and be compatible # with cURL. os.environ.get('CURL_CA_BUNDLE'))
# Merge all the kwargs.
'cert': cert}
""" Returns the appropriate connection adapter for the given URL.
:rtype: requests.adapters.BaseAdapter """
# Nothing matches :-/ raise InvalidSchema("No connection adapters were found for {!r}".format(url))
"""Closes all adapters and as such the session"""
"""Registers a connection adapter to a prefix.
Adapters are sorted in descending order by prefix length. """
self.adapters[key] = self.adapters.pop(key)
state = {attr: getattr(self, attr, None) for attr in self.__attrs__} return state
for attr, value in state.items(): setattr(self, attr, value)
""" Returns a :class:`Session` for context-management.
.. deprecated:: 1.0.0
This method has been deprecated since version 1.0.0 and is only kept for backwards compatibility. New code should use :class:`~requests.sessions.Session` to create a session. This may be removed at a future date.
:rtype: Session """ return Session() |