From 9f31ba8828e00398924de3e4033da82113c23299 Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Tue, 15 Nov 2022 03:37:31 +0000 Subject: [PATCH] Add archivebot-fix-queue-counters --- archivebot-fix-queue-counters | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100755 archivebot-fix-queue-counters diff --git a/archivebot-fix-queue-counters b/archivebot-fix-queue-counters new file mode 100755 index 0000000..6d16c06 --- /dev/null +++ b/archivebot-fix-queue-counters @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +import os +import redis +import sqlite3 + + +assert os.path.exists('wpull.db'), 'no wpull.db file in this directory' + +jobid = os.getcwd().rsplit('/', 1)[-1] + +r = redis.StrictRedis.from_url(os.environ.get('REDIS_URL', 'redis://127.0.0.1:16379/0'), decode_responses = True) +curDownloaded, curQueued = r.hmget(jobid, 'items_downloaded', 'items_queued') +assert curDownloaded is not None and curQueued is not None, f'could not fetch downloaded and/or queued count for {jobid}' + +print(f'Current control node values: {curDownloaded}/{curQueued}') + +db = sqlite3.connect('wpull.db') +cur = db.cursor() +res = cur.execute('SELECT (SELECT MAX(id) FROM queued_urls) AS total, (SELECT COUNT(id) AS i FROM queued_urls WHERE status = "todo") + (SELECT COUNT(id) AS i FROM queued_urls WHERE status = "error") + (SELECT COUNT(id) AS i FROM queued_urls WHERE status = "in_progress") AS queued') +total, queued = res.fetchone() + +print(f'Setting new values: {total - queued}/{total}') + +print(r.hmset(jobid, {'items_downloaded': total - queued, 'items_queued': total})) + +print('Done')