|
|
@@ -34,7 +34,7 @@ class Item: |
|
|
|
|
|
|
|
self.childItems = [] |
|
|
|
|
|
|
|
async def fetch(self, url, responseHandler = qwarc.utils.handle_response_default, method = 'GET', data = None, headers = [], verify_ssl = True): |
|
|
|
async def fetch(self, url, responseHandler = qwarc.utils.handle_response_default, method = 'GET', data = None, headers = [], verify_ssl = True, timeout = 60): |
|
|
|
''' |
|
|
|
HTTP GET or POST a URL |
|
|
|
|
|
|
@@ -65,7 +65,7 @@ class Item: |
|
|
|
writeToWarc = True |
|
|
|
try: |
|
|
|
try: |
|
|
|
with _aiohttp.Timeout(60): |
|
|
|
with _aiohttp.Timeout(timeout): |
|
|
|
self.logger.info(f'Fetching {url}') |
|
|
|
response = await self.session.request(method, url, data = data, headers = headers, allow_redirects = False, verify_ssl = verify_ssl) |
|
|
|
try: |
|
|
|