diff --git a/archivebot-jobs b/archivebot-jobs index da40b15..4253c11 100755 --- a/archivebot-jobs +++ b/archivebot-jobs @@ -1,227 +1,171 @@ -#!/bin/bash -declare -a columns columndefs columnattributes -columns+=('jobid'); columndefs+=('job["job_data"]["ident"]'); columnattributes+=(''); -columns+=('url'); columndefs+=('job["job_data"]["url"]'); columnattributes+=(''); -columns+=('user'); columndefs+=('job["job_data"]["started_by"]'); columnattributes+=(''); -columns+=('pipenick'); columndefs+=('pipelines[job["job_data"]["pipeline_id"]] if job["job_data"]["pipeline_id"] in pipelines else "unknown"'); columnattributes+=(''); -columns+=('queued'); columndefs+=('job["job_data"]["queued_at"]'); columnattributes+=('date'); -columns+=('started'); columndefs+=('job["job_data"]["started_at"]'); columnattributes+=('date'); -columns+=('last active'); columndefs+=('int(job["ts"])'); columnattributes+=('date,coloured'); - -function valid_column { - local candidate="$1" - local column - for column in "${columns[@]}" - do - [[ "${candidate}" == "${column}" ]] && return 0 - done - return 1 +#!/usr/bin/env python3 +import argparse +import datetime +import json +import re +import sys +import time +import urllib.request + +# Column definitions +columns = { + 'jobid': (lambda job, pipelines: job["job_data"]["ident"], ()), + 'url': (lambda job, pipelines: job["job_data"]["url"], ()), + 'user': (lambda job, pipelines: job["job_data"]["started_by"], ()), + 'pipenick': (lambda job, pipelines: pipelines[job["job_data"]["pipeline_id"]] if job["job_data"]["pipeline_id"] in pipelines else "unknown", ()), + 'queued': (lambda job, pipelines: job["job_data"]["queued_at"], ('date',)), + 'started': (lambda job, pipelines: job["job_data"]["started_at"], ('date',)), + 'last active': (lambda job, pipelines: int(job["ts"]), ('date', 'coloured')), } - -sortcolumns=() -filter= -filtercaseinsensitive= -nocolours= -notable= -dates= # Whether to use full dates for the time columns rather than expired time strings -while [[ $# -gt 0 ]] -do - if [[ "$1" == "--help" || "$1" == "-h" ]] - then - echo "Usage: archivebot-jobs [options]" >&2 - echo "Prints a table of current AB jobs" >&2 - echo "Options:" >&2 - echo " --help, -h Show this message and exit." >&2 - echo " --sort [-]COLUMN, -s Sort the table by a column (descending if preceded by '-'). This can be used multiple times to refine the sorting." >&2 - echo " --filter FILTER, -f Filter the table for rows where a COLUMN has a certain VALUE. If specified multiple times, only the last value is used." >&2 - echo " The FILTER has this format: COLUMN{=|<|>|^|*|$|~}VALUE" >&2 - echo " = means the value must be exactly as specified; < and > mean it must be less/greater than the specified; ^ and $ mean it must start/end with the specified; * means it must contain the specified; ~ means it must match the specified regex." >&2 - echo " --ifilter FILTER, -i Like --filter, but case-insensitive" >&2 - echo " --no-colours, --no-colors Don't colourise the last activity column if it's been a while." >&2 - echo " --no-table Raw output without feeding through column(1); columns are separated by tabs." >&2 - echo " --dates Print dates instead of elapsed times for queued/started/last active columns." >&2 - echo "The COLUMNs are the names of each column, printed in capital letters in the first line of the output." >&2 - exit 0 - elif [[ "$1" == "--sort" || "$1" == "-s" ]] - then - sortcolumns+=("$2") - shift - elif [[ "$1" == "--filter" || "$1" == "-f" ]] - then - filter="$2" - filtercaseinsensitive= - shift - elif [[ "$1" == "--ifilter" || "$1" == "-i" ]] - then - filter="$2" - filtercaseinsensitive=1 - shift - elif [[ "$1" == "--no-colours" || "$1" == "--no-colors" ]] - then - nocolours=1 - elif [[ "$1" == "--no-table" ]] - then - notable=1 - elif [[ "$1" == "--dates" ]] - then - dates=1 - else - echo "Unknown option: $1" >&2 - exit 1 - fi - shift -done - -# Validate sortcolumns and filter -if [[ "${filter}" ]] -then - if [[ "${filter}" == *$'\n'* ]] - then - echo "Invalid filter: newlines not allowed" >&2 - exit 1 - fi - if [[ ! ( "${filter}" == *'='* || "${filter}" == *'<'* || "${filter}" == *'>'* || "${filter}" == *'^'* || "${filter}" == *'*'* || "${filter}" == *'$'* || "${filter}" == *'~'* ) ]] - then - echo "Invalid filter: ${filter}" >&2 - exit 1 - fi - column="${filter%%[=<>^*$~]*}" - if ! valid_column "${column,,}" - then - echo "Invalid filter column: ${column}" >&2 - exit 1 - fi -fi -if [[ ${#sortcolumns[@]} -gt 0 ]] -then - for column in "${sortcolumns[@]}" - do - column="${column#-}" - if ! valid_column "${column,,}" - then - echo "Invalid sort column: ${column}" >&2 - exit 1 - fi - done -else - # Default sort order - sortcolumns+=("jobid") -fi - -if [[ "${notable}" ]] -then - column=("cat") -else - column=("column" "-t" $'-s\t') -fi - -jobdata="$(curl -s -H "Accept: application/json" "http://dashboard.at.ninjawedding.org/logs/recent?count=1" 2>/dev/null)" -pipelinedata="$(curl -s -H "Accept: application/json" "http://dashboard.at.ninjawedding.org/pipelines" 2>/dev/null)" - -if [[ -z "${jobdata}" || -z "${pipelinedata}" ]] -then - echo "Error retrieving job or pipeline data" >&2 - exit 1 -fi - -{ echo "${jobdata}"; echo "${pipelinedata}"; echo "${filter}"; } | python3 -c \ -' -if True: # For sensible indentation - import datetime - import json - import sys - import time - - currentTime = time.time() - def render_date(ts, coloured = False): - global currentTime - diff = currentTime - ts - colourStr = f"\x1b[{0 if diff < 6 * 3600 else 7};31m" if coloured and diff >= 300 else "" - colourEndStr = "\x1b[0m" if colourStr else "" - if "'${dates}'": - return colourStr + datetime.datetime.fromtimestamp(ts).isoformat(sep = " ") + colourEndStr - if diff <= 0: - return "now" - elif diff < 60: - return "<1 min ago" - elif diff < 86400: - return colourStr + (f"{diff // 3600:.0f}h " if diff >= 3600 else "") + f"{(diff % 3600) // 60:.0f}mn ago" + colourEndStr - else: - return colourStr + f"{diff // 86400:.0f}d {(diff % 86400) // 3600:.0f}h ago" + colourEndStr - - jobdata = json.loads(sys.stdin.readline()) - pipelinedata = json.loads(sys.stdin.readline()) - filter = sys.stdin.readline().strip() - - pipelines = {p["id"]: p["nickname"] for p in pipelinedata["pipelines"]} - - jobs = [] - for job in jobdata: - jobs.append({'"$(for i in ${!columns[@]}; do echo ' - "'"${columns[$i]}"'": '"${columndefs[$i]}"','; done)"' - }) - - columns = ('"$(for column in "${columns[@]}"; do echo '"'"${column}"'", '; done)"') - columnAttributes = {'"$(for i in ${!columns[@]}; do echo -n '"'"${columns[$i]}"'": "'"${columnattributes[$i]}"'".split(","), '; done)"'} - - # Filter - if filter: - import re - match = re.match(r"^(?P[A-Za-z ]+)(?P[=<>^*$~])(?P.*)$", filter) +defaultSort = 'jobid' + +# Parse arguments +class FilterAction(argparse.Action): + def __call__(self, parser, namespace, values, optionString = None): + global columns + match = re.match(r"^(?P[A-Za-z ]+)(?P[=<>^*$~])(?P.*)$", values[0]) + if not match: + raise argparse.ArgumentError('Invalid filter') filterDict = match.groupdict() filterDict["column"] = filterDict["column"].lower() assert filterDict["column"] in columns - compFunc = { - "=": lambda a, b: a == b, - "<": lambda a, b: a < b, - ">": lambda a, b: a > b, - "^": lambda a, b: a.startswith(b), - "*": lambda a, b: b in a, - "$": lambda a, b: a.endswith(b), - "~": lambda a, b: re.search(b, a) is not None, - }[filterDict["op"]] - if isinstance(jobs[0][filterDict["column"]], (int, float)): - filterDict["value"] = float(filterDict["value"]) - transform = lambda x: x.lower() if "'${filtercaseinsensitive}'" and isinstance(x, str) else x - jobs = [job for job in jobs if compFunc(transform(job[filterDict["column"]]), transform(filterDict["value"]))] + transform = (lambda x: x.lower() if isinstance(x, str) else x) if optionString in ('--ifilter', '-i') else (lambda x: x) + setattr(namespace, self.dest, (filterDict, transform)) + +def parse_sort(value): + global columns + sortDesc = value.startswith('-') + if sortDesc: + value = value[1:] + value = value.lower() + if value not in columns: + raise argparse.ArgumentError('Invalid column name') + return (value, sortDesc) + +class SortAction(argparse.Action): + def __call__(self, parser, namespace, values, optionString = None): + result = parse_sort(values[0]) + if getattr(namespace, self.dest, None) is None: + setattr(namespace, self.dest, []) + getattr(namespace, self.dest).append(result) + +parser = argparse.ArgumentParser(formatter_class = argparse.RawTextHelpFormatter) +parser.add_argument('--sort', '-s', nargs = 1, type = str, action = SortAction, help = "Sort the table by a COLUMN (descending if preceded by '-'). This can be used multiple times to refine the sorting.") +parser.add_argument('--filter', '-f', nargs = 1, type = str, action = FilterAction, help = '\n'.join([ + 'Filter the table for rows where a COLUMN has a certain VALUE. If specified multiple times, only the last value is used.', + 'FILTER has the format COLUMN{=|<|>|^|*|$|~}VALUE', + ' = means the value must be exactly as specified.', + ' < and > mean it must be less/greater than the specified.', + ' ^ and $ mean it must start/end with the specified.', + ' * means it must contain the specified.', + ' ~ means it must match the specified regex.', + ])) +parser.add_argument('--ifilter', '-i', nargs = 1, type = str, action = FilterAction, dest = 'filter', help = 'Like --filter but case-insensitive') +parser.add_argument('--no-colours', '--no-colors', action = 'store_true', help = "Don't colourise the last activity column if it's been a while.") +parser.add_argument('--no-table', action = 'store_true', help = 'Raw output without feeding through column(1); columns are separated by tabs.') +parser.add_argument('--dates', action = 'store_true', help = 'Print dates instead of elapsed times for queued/started/last active columns.') +args = parser.parse_args() + +if not args.sort: + args.sort = [parse_sort(defaultSort)] + +# Retrieve +def fetch(url): + req = urllib.request.Request(url) + req.add_header('Accept', 'application/json') + with urllib.request.urlopen(req) as f: + if f.getcode() != 200: + raise RuntimeError('Could not fetch job data') + return json.load(f) + +jobdata = fetch('http://dashboard.at.ninjawedding.org/logs/recent?count=1') +pipelinedata = fetch('http://dashboard.at.ninjawedding.org/pipelines') +currentTime = time.time() + +# Process +pipelines = {p["id"]: p["nickname"] for p in pipelinedata["pipelines"]} + +jobs = [] +for job in jobdata: + jobs.append({column: columnFunc(job, pipelines) for column, (columnFunc, _) in columns.items()}) + +if not jobs: + # Nothing to do + sys.exit(0) + +# Filter +if args.filter: + filterDict, transform = args.filter + compFunc = { + "=": lambda a, b: a == b, + "<": lambda a, b: a < b, + ">": lambda a, b: a > b, + "^": lambda a, b: a.startswith(b), + "*": lambda a, b: b in a, + "$": lambda a, b: a.endswith(b), + "~": lambda a, b: re.search(b, a) is not None, + }[filterDict["op"]] + if isinstance(jobs[0][filterDict["column"]], (int, float)): + filterDict["value"] = float(filterDict["value"]) + jobs = [job for job in jobs if compFunc(transform(job[filterDict["column"]]), transform(filterDict["value"]))] if not jobs: sys.exit(0) - # Sort - class reversor: # https://stackoverflow.com/a/56842689 - def __init__(self, obj): - self.obj = obj - - def __eq__(self, other): - return other.obj == self.obj - - def __lt__(self, other): - return other.obj < self.obj - - sortColumnsRaw = ('"$(printf "'%s', " "${sortcolumns[@]}")"') - sortColumns = tuple((column[1:] if column.startswith("-") else column).lower() for column in sortColumnsRaw) - sortColumnAsc = tuple(not column.startswith("-") for column in sortColumnsRaw) - assert all(column in columns for column in sortColumns) - if not "'${dates}'": - # Reverse sorting order for columns which have a date attribute since the column will have elapsed time - sortColumnAttrs = tuple(columnAttr for column, columnAttr in columnAttributes.items() if column in sortColumns) - sortColumnAsc = tuple(not columnAsc if "date" in columnAttr else columnAsc for columnAsc, column, columnAttr in zip(sortColumnAsc, sortColumns, sortColumnAttrs)) - jobs = sorted(jobs, key = lambda job: tuple(job[column] if columnAsc else reversor(job[column]) for column, columnAsc in zip(sortColumns, sortColumnAsc))) - - # Renderers - renderers = {} - for column, columnAttr in columnAttributes.items(): - if "date" in columnAttr: - if "coloured" in columnAttr: - renderers[column] = lambda x: render_date(x, coloured = not "'${nocolours}'") - else: - renderers[column] = render_date - - # Print - print("\t".join(column.upper() for column in columns)) - for job in jobs: - for column in renderers: - job[column] = renderers[column](job[column]) - print("\t".join(job[column] for column in columns)) -' | "${column[@]}" +# Sort +class reversor: # https://stackoverflow.com/a/56842689 + def __init__(self, obj): + self.obj = obj + + def __eq__(self, other): + return other.obj == self.obj + + def __lt__(self, other): + return other.obj < self.obj + +sortColumns = tuple((column, descending, columns[column]) for column, descending in args.sort) +if not args.dates: + # Reverse sorting order for columns which have a date attribute since the column will have elapsed time + sortColumns = tuple((column, not descending if 'date' in columnInfo[1] else descending, columnInfo) for column, descending, columnInfo in sortColumns) +jobs = sorted(jobs, key = lambda job: tuple(job[column] if not descending else reversor(job[column]) for column, descending, _ in sortColumns)) + +# Renderers +def render_date(ts, coloured = False): + global args, currentTime + diff = currentTime - ts + colourStr = f"\x1b[{0 if diff < 6 * 3600 else 7};31m" if coloured and diff >= 300 else "" + colourEndStr = "\x1b[0m" if colourStr else "" + if args.dates: + return (colourStr, datetime.datetime.fromtimestamp(ts).isoformat(sep = " "), colourEndStr) + if diff <= 0: + return "now" + elif diff < 60: + return "<1 min ago" + elif diff < 86400: + return (colourStr, (f"{diff // 3600:.0f}h " if diff >= 3600 else "") + f"{(diff % 3600) // 60:.0f}mn ago", colourEndStr) + else: + return (colourStr, f"{diff // 86400:.0f}d {(diff % 86400) // 3600:.0f}h ago", colourEndStr) + +renderers = {} +for column, (_, columnAttr) in columns.items(): + if "date" in columnAttr: + if "coloured" in columnAttr: + renderers[column] = lambda x: render_date(x, coloured = not args.no_colours) + else: + renderers[column] = render_date + +# Print +output = [] +output.append(tuple(column.upper() for column in columns)) +for job in jobs: + for column in renderers: + job[column] = renderers[column](job[column]) + output.append(tuple(job[column] for column in columns)) + +if not args.no_table: + widths = tuple(max(len(field) if isinstance(field, str) else len(field[1]) for field in column) for column in zip(*output)) + for row in output: + print(' '.join((value.ljust(width) if isinstance(value, str) else ''.join((value[0], value[1], value[2], ' ' * (width - len(value[1]))))) for value, width in zip(row, widths))) +else: + for row in output: + print('\t'.join(field if isinstance(field, str) else ''.join(field) for field in row))