From b6003af1e58e1421c68cd76da0bbc62d7f3cb3bf Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Mon, 16 Mar 2020 02:53:02 +0000 Subject: [PATCH] Work around aiohttp bug on parsing chunked transfer encoding responses when the buffer ends in an unfortunate spot https://github.com/aio-libs/aiohttp/issues/4630 --- qwarc/aiohttp.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qwarc/aiohttp.py b/qwarc/aiohttp.py index 36e99e8..48ebf49 100644 --- a/qwarc/aiohttp.py +++ b/qwarc/aiohttp.py @@ -127,11 +127,15 @@ class ClientResponse(aiohttp.client_reqrep.ClientResponse): except (KeyError, ValueError, TypeError): length = None parser = aiohttp.http_parser.HttpPayloadParser(payload, length = length, chunked = respMsg.chunked, compression = respMsg.compression, code = respMsg.code, method = self.method) + while beginning.endswith(b'0\r\n') or beginning.endswith(b'0\r\n\r'): # https://github.com/aio-libs/aiohttp/issues/4630 + beginning = beginning + self._rawData.responseData.read(4) eof, data = parser.feed_data(beginning[pos + 4:]) while True: chunk = self._rawData.responseData.read(1048576) if not chunk: break + while chunk.endswith(b'0\r\n') or chunk.endswith(b'0\r\n\r'): # https://github.com/aio-libs/aiohttp/issues/4630 + chunk = chunk + self._rawData.responseData.read(4) eof, data = parser.feed_data(chunk) if nbytes is not None and payload.data.tell() >= nbytes: if payload.exc: