From b1a1c03f7ee8ee1af436ef79279c3d26caca5f49 Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Wed, 11 Dec 2019 01:18:10 +0000 Subject: [PATCH] Handle STOP file and high memory usage before full disk to allow stopping while the disk is above the limit --- qwarc/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/qwarc/__init__.py b/qwarc/__init__.py index a739320..8abaca4 100644 --- a/qwarc/__init__.py +++ b/qwarc/__init__.py @@ -287,6 +287,13 @@ class QWARC: while len(self._tasks) >= self._concurrency: await self._wait_for_free_task() + if os.path.exists('STOP'): + logging.info('Gracefully shutting down due to STOP file') + break + if self._memoryLimit and qwarc.utils.uses_too_much_memory(self._memoryLimit): + logging.info(f'Gracefully shutting down due to memory usage (current = {qwarc.utils.get_rss()} > limit = {self._memoryLimit})') + break + if self._minFreeDisk and qwarc.utils.too_little_disk_space(self._minFreeDisk): logging.info('Disk space is low, sleeping') sleepTask = asyncio.ensure_future(asyncio.sleep(random.uniform(self._concurrency / 2, self._concurrency * 1.5))) @@ -295,13 +302,6 @@ class QWARC: self._sleepTasks.add(sleepTask) continue - if os.path.exists('STOP'): - logging.info('Gracefully shutting down due to STOP file') - break - if self._memoryLimit and qwarc.utils.uses_too_much_memory(self._memoryLimit): - logging.info(f'Gracefully shutting down due to memory usage (current = {qwarc.utils.get_rss()} > limit = {self._memoryLimit})') - break - cursor = await self.obtain_exclusive_db_lock() try: cursor.execute('SELECT id, type, value, status FROM items WHERE status = ? LIMIT 1', (STATUS_TODO,))