@@ -74,8 +74,7 @@ class Item:
verify_ssl: bool, whether the SSL/TLS certificate should be validated
verify_ssl: bool, whether the SSL/TLS certificate should be validated
timeout: int or float, how long the fetch may take at most in total (sending request until finishing reading the response)
timeout: int or float, how long the fetch may take at most in total (sending request until finishing reading the response)
Returns response (a ClientResponse object or None) and history (a tuple of (response, exception) tuples).
response can be None and history can be an empty tuple, depending on the circumstances (e.g. timeouts).
Returns response (a ClientResponse object or a qwarc.utils.DummyClientResponse object)
'''
'''
#TODO: Rewrite using 'async with self.session.get'
#TODO: Rewrite using 'async with self.session.get'
@@ -126,13 +125,15 @@ class Item:
if response and exc is None and writeToWarc:
if response and exc is None and writeToWarc:
self.warc.write_client_response(response)
self.warc.write_client_response(response)
history.append((response, exc))
history.append((response, exc))
retResponse = response if exc is None else None
retResponse = response if exc is None else qwarc.utils.DummyClientResponse()
if action in (ACTION_SUCCESS, ACTION_IGNORE):
if action in (ACTION_SUCCESS, ACTION_IGNORE):
return retResponse, tuple(history)
retResponse.qhistory = tuple(history)
return retResponse
elif action == ACTION_FOLLOW_OR_SUCCESS:
elif action == ACTION_FOLLOW_OR_SUCCESS:
redirectUrl = response.headers.get('Location') or response.headers.get('URI')
redirectUrl = response.headers.get('Location') or response.headers.get('URI')
if not redirectUrl:
if not redirectUrl:
return retResponse, tuple(history)
retResponse.qhistory = tuple(history)
return retResponse
url = url.join(yarl.URL(redirectUrl))
url = url.join(yarl.URL(redirectUrl))
if response.status in (301, 302, 303) and method == 'POST':
if response.status in (301, 302, 303) and method == 'POST':
method = 'GET'
method = 'GET'
@@ -140,7 +141,8 @@ class Item:
attempt = 0
attempt = 0
elif action == ACTION_RETRIES_EXCEEDED:
elif action == ACTION_RETRIES_EXCEEDED:
self.logger.error(f'Request for {url} failed {attempt} times')
self.logger.error(f'Request for {url} failed {attempt} times')
return retResponse, tuple(history)
retResponse.qhistory = tuple(history)
return retResponse
elif action == ACTION_RETRY:
elif action == ACTION_RETRY:
# Nothing to do, just go to the next cycle
# Nothing to do, just go to the next cycle
pass
pass