diff --git a/qwarc/__init__.py b/qwarc/__init__.py index 55b26d6..261bd14 100644 --- a/qwarc/__init__.py +++ b/qwarc/__init__.py @@ -59,7 +59,7 @@ class Item: out.append((key, value)) return out - async def fetch(self, url, responseHandler = qwarc.utils.handle_response_default, method = 'GET', data = None, headers = [], verify_ssl = True): + async def fetch(self, url, responseHandler = qwarc.utils.handle_response_default, method = 'GET', data = None, headers = [], verify_ssl = True, timeout = 60): ''' HTTP GET or POST a URL @@ -71,6 +71,7 @@ class Item: To remove one of the default headers, pass a value of None. If a header appears multiple times, only the last one is used. To send a header multiple times, pass a tuple of values. verify_ssl: bool, whether the SSL/TLS certificate should be validated + timeout: int or float, how long the fetch may take at most in total (sending request until finishing reading the response) Returns response (a ClientResponse object or None) and history (a tuple of (response, exception) tuples). response can be None and history can be an empty tuple, depending on the circumstances (e.g. timeouts). @@ -92,7 +93,7 @@ class Item: writeToWarc = True try: try: - with _aiohttp.Timeout(60): + with _aiohttp.Timeout(timeout): self.logger.info(f'Fetching {url}') response = await self.session.request(method, url, data = data, headers = headers, allow_redirects = False, verify_ssl = verify_ssl) try: