|
|
@@ -263,14 +263,11 @@ class QWARC: |
|
|
|
raise |
|
|
|
|
|
|
|
async def run(self, loop): |
|
|
|
headers = [('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0')] #TODO: Move elsewhere |
|
|
|
|
|
|
|
for i in range(self._concurrency): |
|
|
|
session = _aiohttp.ClientSession( |
|
|
|
connector = qwarc.aiohttp.TCPConnector(loop = loop), |
|
|
|
request_class = qwarc.aiohttp.ClientRequest, |
|
|
|
response_class = qwarc.aiohttp.ClientResponse, |
|
|
|
skip_auto_headers = ['Accept-Encoding'], |
|
|
|
loop = loop |
|
|
|
) |
|
|
|
self._sessions.append(session) |
|
|
@@ -332,7 +329,7 @@ class QWARC: |
|
|
|
raise |
|
|
|
|
|
|
|
session = self._freeSessions.popleft() |
|
|
|
item = self._make_item(itemType, itemValue, session, headers) |
|
|
|
item = self._make_item(itemType, itemValue, session, DEFAULT_HEADERS) |
|
|
|
task = asyncio.ensure_future(item.process()) |
|
|
|
#TODO: Is there a better way to add custom information to a task/coroutine object? |
|
|
|
task.taskType = 'process' |
|
|
|