Sfoglia il codice sorgente

Use f-strings instead of str.format

tags/v0.2.0
JustAnotherArchivist 4 anni fa
parent
commit
8647d6b396
4 ha cambiato i file con 13 aggiunte e 13 eliminazioni
  1. +7
    -7
      qwarc/__init__.py
  2. +2
    -2
      qwarc/cli.py
  3. +1
    -1
      qwarc/utils.py
  4. +3
    -3
      qwarc/warc.py

+ 7
- 7
qwarc/__init__.py Vedi File

@@ -62,7 +62,7 @@ class Item:
try:
try:
with _aiohttp.Timeout(60):
logging.info('Fetching {}'.format(url))
logging.info(f'Fetching {url}')
response = await self.session.request(method, url, data = data, headers = headers, allow_redirects = False)
try:
ret = await response.read()
@@ -73,12 +73,12 @@ class Item:
else:
tx = len(response.rawRequestData)
rx = len(response.rawResponseData)
logging.info('Fetched {}: {} (tx {}, rx {})'.format(url, response.status, tx, rx))
logging.info(f'Fetched {url}: {response.status} (tx {tx}, rx {rx})')
self.stats['tx'] += tx
self.stats['rx'] += rx
self.stats['requests'] += 1
except (asyncio.TimeoutError, _aiohttp.ClientError) as e:
logging.error('Request for {} failed: {!r}'.format(url, e))
logging.error(f'Request for {url} failed: {e!r}')
action, writeToWarc = await responseHandler(url, attempt, response, e)
exc = e # Pass the exception outward for the history
else:
@@ -163,7 +163,7 @@ class QWARC:
try:
itemClass = self._itemTypeMap[itemType]
except KeyError:
raise RuntimeError('No such item type: {!r}'.format(itemType))
raise RuntimeError(f'No such item type: {itemType!r}')
return itemClass(itemValue, session, headers, warc)

async def run(self, loop):
@@ -203,7 +203,7 @@ class QWARC:
# Got cancelled, nothing we can do about it, but let's log a warning if it's a process task
if isinstance(future, asyncio.Task):
if future.taskType == 'process_item':
logging.warning('Task for {}:{} cancelled: {!r}'.format(future.itemType, future.itemValue, future))
logging.warning(f'Task for {future.itemType}:{future.itemValue} cancelled: {future!r}')
elif future.taskType == 'sleep':
sleepTasks.remove(future)
continue
@@ -212,7 +212,7 @@ class QWARC:
sleepTasks.remove(future)
continue
item = future.item
logging.info('{itemType}:{itemValue} done: {requests} requests, {tx} tx, {rx} rx'.format(itemType = future.itemType, itemValue = future.itemValue, **item.stats))
logging.info(f'{future.itemType}:{future.itemValue} done: {item.stats["requests"]} requests, {item.stats["tx"]} tx, {item.stats["rx"]} rx')
cursor = await self.obtain_exclusive_db_lock(db)
try:
cursor.execute('UPDATE items SET status = ? WHERE id = ?', (STATUS_DONE, future.id))
@@ -287,7 +287,7 @@ class QWARC:
logging.info('Gracefully shutting down due to STOP file')
break
if self._memoryLimit and qwarc.utils.uses_too_much_memory(self._memoryLimit):
logging.info('Gracefully shutting down due to memory usage (current = {} > limit = {})'.format(qwarc.utils.get_rss(), self._memoryLimit))
logging.info(f'Gracefully shutting down due to memory usage (current = {qwarc.utils.get_rss()} > limit = {self._memoryLimit})')
break

for sleepTask in sleepTasks:


+ 2
- 2
qwarc/cli.py Vedi File

@@ -28,10 +28,10 @@ def setup_logging(logFilename):
def check_files(specFilename, logFilename):
success = True
if not os.path.isfile(specFilename):
print('Error: "{}" does not exist or is not a regular file', file = sys.stderr)
print(f'Error: "{specFilename}" does not exist or is not a regular file', file = sys.stderr)
success = False
if os.path.exists(logFilename):
print('Error: "{}" already exists'.format(logFilename), file = sys.stderr)
print(f'Error: "{logFilename}" already exists', file = sys.stderr)
success = False
if os.path.exists('STOP'):
print('Error: "STOP" exists', file = sys.stderr)


+ 1
- 1
qwarc/utils.py Vedi File

@@ -118,7 +118,7 @@ def generate_range_items(start, stop, step):
'''

for i in range(start, stop + 1, step):
yield '{}-{}'.format(i, min(i + step - 1, stop))
yield f'{i}-{min(i + step - 1, stop)}'


async def handle_response_default(url, attempt, response, exc):


+ 3
- 3
qwarc/warc.py Vedi File

@@ -34,17 +34,17 @@ class WARC:
#TODO: This opens a new file also at the end, which can result in empty WARCs. Should try to reorder this to only open a WARC when writing a record, and to only close the current WARC if the size is exceeded after write_client_response.
self.close()
while True:
filename = '{}-{:05d}.warc.gz'.format(self._prefix, self._counter)
filename = f'{self._prefix}-{self._counter:05d}.warc.gz'
try:
# Try to open the file for writing, requiring that it does not exist yet, and attempt to get an exclusive, non-blocking lock on it
self._file = open(filename, 'xb')
fcntl.flock(self._file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
except FileExistsError:
logging.info('{} already exists, skipping'.format(filename))
logging.info(f'{filename} already exists, skipping')
self._counter += 1
else:
break
logging.info('Opened {}'.format(filename))
logging.info(f'Opened {filename}')
self._warcWriter = warcio.warcwriter.WARCWriter(self._file, gzip = True)
self._closed = False
self._counter += 1


Caricamento…
Annulla
Salva