|
|
@@ -12,21 +12,21 @@ import urllib.request |
|
|
|
|
|
|
|
# Column definitions |
|
|
|
columns = { |
|
|
|
'jobid': (lambda job, pipelines: job["job_data"]["ident"], ()), |
|
|
|
'url': (lambda job, pipelines: job["job_data"]["url"], ('truncatable',)), |
|
|
|
'user': (lambda job, pipelines: job["job_data"]["started_by"], ()), |
|
|
|
'pipenick': (lambda job, pipelines: pipelines[job["job_data"]["pipeline_id"]] if job["job_data"]["pipeline_id"] in pipelines else "unknown", ()), |
|
|
|
'queued': (lambda job, pipelines: job["job_data"]["queued_at"], ('date', 'numeric')), |
|
|
|
'started': (lambda job, pipelines: job["job_data"]["started_at"], ('date', 'numeric')), |
|
|
|
'last active': (lambda job, pipelines: int(job["ts"]), ('date', 'coloured', 'numeric')), |
|
|
|
'dl urls': (lambda job, pipelines: job["job_data"]["items_downloaded"], ('numeric',)), |
|
|
|
'dl size': (lambda job, pipelines: job["job_data"]["bytes_downloaded"], ('size', 'numeric')), |
|
|
|
'queue': (lambda job, pipelines: job["job_data"]["items_queued"] - job["job_data"]["items_downloaded"], ('numeric',)), |
|
|
|
'eta': (lambda job, pipelines: int((curTime := time.time()) + (job["job_data"]["items_queued"] - job["job_data"]["items_downloaded"]) / (job["job_data"]["items_downloaded"] / (curTime - job["job_data"]["started_at"]))) if job["job_data"]["items_downloaded"] > 0 else 0, ('date', 'numeric')), |
|
|
|
'con': (lambda job, pipelines: job["job_data"]["concurrency"], ('numeric',)), |
|
|
|
'delay min': (lambda job, pipelines: int(job["job_data"]["delay_min"]), ('hidden', 'numeric')), |
|
|
|
'delay max': (lambda job, pipelines: int(job["job_data"]["delay_max"]), ('hidden', 'numeric')), |
|
|
|
'delay': (lambda job, pipelines: str(int(job["job_data"]["delay_min"])) + '-' + str(int(job["job_data"]["delay_max"])) if job["job_data"]["delay_min"] != job["job_data"]["delay_max"] else str(int(job["job_data"]["delay_min"])), ()), |
|
|
|
'jobid': (lambda job, pipelines: job['job_data']['ident'], ()), |
|
|
|
'url': (lambda job, pipelines: job['job_data']['url'], ('truncatable',)), |
|
|
|
'user': (lambda job, pipelines: job['job_data']['started_by'], ()), |
|
|
|
'pipenick': (lambda job, pipelines: pipelines[job['job_data']['pipeline_id']] if job['job_data']['pipeline_id'] in pipelines else 'unknown', ()), |
|
|
|
'queued': (lambda job, pipelines: job['job_data']['queued_at'], ('date', 'numeric')), |
|
|
|
'started': (lambda job, pipelines: job['job_data']['started_at'], ('date', 'numeric')), |
|
|
|
'last active': (lambda job, pipelines: int(job['ts']), ('date', 'coloured', 'numeric')), |
|
|
|
'dl urls': (lambda job, pipelines: job['job_data']['items_downloaded'], ('numeric',)), |
|
|
|
'dl size': (lambda job, pipelines: job['job_data']['bytes_downloaded'], ('size', 'numeric')), |
|
|
|
'queue': (lambda job, pipelines: job['job_data']['items_queued'] - job['job_data']['items_downloaded'], ('numeric',)), |
|
|
|
'eta': (lambda job, pipelines: int((curTime := time.time()) + (job['job_data']['items_queued'] - job['job_data']['items_downloaded']) / (job['job_data']['items_downloaded'] / (curTime - job['job_data']['started_at']))) if job['job_data']['items_downloaded'] > 0 else 0, ('date', 'numeric')), |
|
|
|
'con': (lambda job, pipelines: job['job_data']['concurrency'], ('numeric',)), |
|
|
|
'delay min': (lambda job, pipelines: int(job['job_data']['delay_min']), ('hidden', 'numeric')), |
|
|
|
'delay max': (lambda job, pipelines: int(job['job_data']['delay_max']), ('hidden', 'numeric')), |
|
|
|
'delay': (lambda job, pipelines: str(int(job['job_data']['delay_min'])) + '-' + str(int(job['job_data']['delay_max'])) if job['job_data']['delay_min'] != job['job_data']['delay_max'] else str(int(job['job_data']['delay_min'])), ()), |
|
|
|
} |
|
|
|
defaultSort = 'jobid' |
|
|
|
|
|
|
@@ -38,13 +38,13 @@ if any('truncatable' in colDef[1] and any(x in colDef[1] for x in ('date', 'colo |
|
|
|
# Filter function |
|
|
|
def make_field_filter(column, op, value, caseSensitive = True): |
|
|
|
compFunc = { |
|
|
|
"=": lambda a, b: a == b, |
|
|
|
"<": lambda a, b: a < b, |
|
|
|
">": lambda a, b: a > b, |
|
|
|
"^": lambda a, b: a.startswith(b), |
|
|
|
"*": lambda a, b: b in a, |
|
|
|
"$": lambda a, b: a.endswith(b), |
|
|
|
"~": lambda a, b: re.search(b, a) is not None, |
|
|
|
'=': lambda a, b: a == b, |
|
|
|
'<': lambda a, b: a < b, |
|
|
|
'>': lambda a, b: a > b, |
|
|
|
'^': lambda a, b: a.startswith(b), |
|
|
|
'*': lambda a, b: b in a, |
|
|
|
'$': lambda a, b: a.endswith(b), |
|
|
|
'~': lambda a, b: re.search(b, a) is not None, |
|
|
|
}[op] |
|
|
|
transform = { |
|
|
|
True: (lambda x: x), |
|
|
@@ -64,12 +64,12 @@ class FilterAction(argparse.Action): |
|
|
|
setattr(namespace, self.dest, lambda job: eval(func, {}, {'job': job})) |
|
|
|
return |
|
|
|
global columns |
|
|
|
match = re.match(r"^(?P<column>[A-Za-z ]+)(?P<op>[=<>^*$~])(?P<value>.*)$", values[0]) |
|
|
|
match = re.match(r'^(?P<column>[A-Za-z ]+)(?P<op>[=<>^*$~])(?P<value>.*)$', values[0]) |
|
|
|
if not match: |
|
|
|
parser.error('Invalid filter') |
|
|
|
filterDict = match.groupdict() |
|
|
|
filterDict["column"] = filterDict["column"].lower() |
|
|
|
assert filterDict["column"] in columns |
|
|
|
filterDict['column'] = filterDict['column'].lower() |
|
|
|
assert filterDict['column'] in columns |
|
|
|
if 'numeric' in columns[filterDict['column']][1]: |
|
|
|
filterDict['value'] = float(filterDict['value']) |
|
|
|
if 'date' in columns[filterDict['column']][1] and filterDict['value'] < 0: |
|
|
@@ -151,7 +151,7 @@ pipelinedata = fetch('http://dashboard.at.ninjawedding.org/pipelines') |
|
|
|
currentTime = time.time() |
|
|
|
|
|
|
|
# Process |
|
|
|
pipelines = {p["id"]: p["nickname"] for p in pipelinedata["pipelines"]} |
|
|
|
pipelines = {p['id']: p['nickname'] for p in pipelinedata['pipelines']} |
|
|
|
|
|
|
|
jobs = [] |
|
|
|
for job in jobdata: |
|
|
@@ -210,26 +210,26 @@ elif args.mode == 'atdash': |
|
|
|
def render_date(ts, coloured = False): |
|
|
|
global args, currentTime |
|
|
|
diff = currentTime - ts |
|
|
|
colourStr = f"\x1b[{0 if diff < 6 * 3600 else 7};31m" if coloured and diff >= 300 else "" |
|
|
|
colourEndStr = "\x1b[0m" if colourStr else "" |
|
|
|
colourStr = f'\x1b[{0 if diff < 6 * 3600 else 7};31m' if coloured and diff >= 300 else '' |
|
|
|
colourEndStr = '\x1b[0m' if colourStr else '' |
|
|
|
if ts == 0: |
|
|
|
return 'N/A' |
|
|
|
if args.dates: |
|
|
|
return (colourStr, datetime.datetime.fromtimestamp(ts).isoformat(sep = " "), colourEndStr) |
|
|
|
return (colourStr, datetime.datetime.fromtimestamp(ts).isoformat(sep = ' '), colourEndStr) |
|
|
|
if diff < -86400: |
|
|
|
return (colourStr, f"in {-diff // 86400:.0f}d {(-diff % 86400) // 3600:.0f}h", colourEndStr) |
|
|
|
return (colourStr, f'in {-diff // 86400:.0f}d {(-diff % 86400) // 3600:.0f}h', colourEndStr) |
|
|
|
elif diff < -60: |
|
|
|
return (colourStr, "in " + (f"{-diff // 3600:.0f}h " if diff <= -3600 else "") + f"{(-diff % 3600) // 60:.0f}mn", colourEndStr) |
|
|
|
return (colourStr, 'in ' + (f'{-diff // 3600:.0f}h ' if diff <= -3600 else '') + f'{(-diff % 3600) // 60:.0f}mn', colourEndStr) |
|
|
|
elif diff < 0: |
|
|
|
return "in <1 min" |
|
|
|
return 'in <1 min' |
|
|
|
elif diff == 0: |
|
|
|
return "now" |
|
|
|
return 'now' |
|
|
|
elif diff < 60: |
|
|
|
return "<1 min ago" |
|
|
|
return '<1 min ago' |
|
|
|
elif diff < 86400: |
|
|
|
return (colourStr, (f"{diff // 3600:.0f}h " if diff >= 3600 else "") + f"{(diff % 3600) // 60:.0f}mn ago", colourEndStr) |
|
|
|
return (colourStr, (f'{diff // 3600:.0f}h ' if diff >= 3600 else '') + f'{(diff % 3600) // 60:.0f}mn ago', colourEndStr) |
|
|
|
else: |
|
|
|
return (colourStr, f"{diff // 86400:.0f}d {(diff % 86400) // 3600:.0f}h ago", colourEndStr) |
|
|
|
return (colourStr, f'{diff // 86400:.0f}d {(diff % 86400) // 3600:.0f}h ago', colourEndStr) |
|
|
|
|
|
|
|
def render_size(size): |
|
|
|
units = ('B', 'KiB', 'MiB', 'GiB', 'TiB') |
|
|
@@ -240,12 +240,12 @@ def render_size(size): |
|
|
|
|
|
|
|
renderers = {} |
|
|
|
for column, (_, columnAttr) in columns.items(): |
|
|
|
if "date" in columnAttr: |
|
|
|
if "coloured" in columnAttr: |
|
|
|
if 'date' in columnAttr: |
|
|
|
if 'coloured' in columnAttr: |
|
|
|
renderers[column] = lambda x: render_date(x, coloured = not args.no_colours) |
|
|
|
else: |
|
|
|
renderers[column] = render_date |
|
|
|
elif "size" in columnAttr: |
|
|
|
elif 'size' in columnAttr: |
|
|
|
renderers[column] = render_size |
|
|
|
elif isinstance(jobs[0][column], (int, float)): |
|
|
|
renderers[column] = str |
|
|
@@ -288,9 +288,9 @@ if not args.no_table and not args.no_truncate: |
|
|
|
|
|
|
|
# Print |
|
|
|
output = [] |
|
|
|
output.append(tuple(column.upper() for column in columns if "hidden" not in columns[column][1])) |
|
|
|
output.append(tuple(column.upper() for column in columns if 'hidden' not in columns[column][1])) |
|
|
|
for job in jobs: |
|
|
|
output.append(tuple(job[column] for column in columns if "hidden" not in columns[column][1])) |
|
|
|
output.append(tuple(job[column] for column in columns if 'hidden' not in columns[column][1])) |
|
|
|
|
|
|
|
if not args.no_table: |
|
|
|
widths = tuple(max(len(field) if isinstance(field, str) else len(field[1]) for field in column) for column in zip(*output)) |
|
|
|