|
|
@@ -0,0 +1,26 @@ |
|
|
|
#!/usr/bin/env python3 |
|
|
|
import os |
|
|
|
import redis |
|
|
|
import sqlite3 |
|
|
|
|
|
|
|
|
|
|
|
assert os.path.exists('wpull.db'), 'no wpull.db file in this directory' |
|
|
|
|
|
|
|
jobid = os.getcwd().rsplit('/', 1)[-1] |
|
|
|
|
|
|
|
r = redis.StrictRedis.from_url(os.environ.get('REDIS_URL', 'redis://127.0.0.1:16379/0'), decode_responses = True) |
|
|
|
curDownloaded, curQueued = r.hmget(jobid, 'items_downloaded', 'items_queued') |
|
|
|
assert curDownloaded is not None and curQueued is not None, f'could not fetch downloaded and/or queued count for {jobid}' |
|
|
|
|
|
|
|
print(f'Current control node values: {curDownloaded}/{curQueued}') |
|
|
|
|
|
|
|
db = sqlite3.connect('wpull.db') |
|
|
|
cur = db.cursor() |
|
|
|
res = cur.execute('SELECT (SELECT MAX(id) FROM queued_urls) AS total, (SELECT COUNT(id) AS i FROM queued_urls WHERE status = "todo") + (SELECT COUNT(id) AS i FROM queued_urls WHERE status = "error") + (SELECT COUNT(id) AS i FROM queued_urls WHERE status = "in_progress") AS queued') |
|
|
|
total, queued = res.fetchone() |
|
|
|
|
|
|
|
print(f'Setting new values: {total - queued}/{total}') |
|
|
|
|
|
|
|
print(r.hmset(jobid, {'items_downloaded': total - queued, 'items_queued': total})) |
|
|
|
|
|
|
|
print('Done') |