The little things give you away... A collection of various small helper stuff
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

300 lines
14 KiB

  1. #!/usr/bin/env python3
  2. import argparse
  3. import datetime
  4. import itertools
  5. import json
  6. import math
  7. import os
  8. import re
  9. import sys
  10. import time
  11. import urllib.request
  12. # Column definitions
  13. columns = {
  14. 'jobid': (lambda job, pipelines: job["job_data"]["ident"], ()),
  15. 'url': (lambda job, pipelines: job["job_data"]["url"], ('truncatable',)),
  16. 'user': (lambda job, pipelines: job["job_data"]["started_by"], ()),
  17. 'pipenick': (lambda job, pipelines: pipelines[job["job_data"]["pipeline_id"]] if job["job_data"]["pipeline_id"] in pipelines else "unknown", ()),
  18. 'queued': (lambda job, pipelines: job["job_data"]["queued_at"], ('date', 'numeric')),
  19. 'started': (lambda job, pipelines: job["job_data"]["started_at"], ('date', 'numeric')),
  20. 'last active': (lambda job, pipelines: int(job["ts"]), ('date', 'coloured', 'numeric')),
  21. 'dl urls': (lambda job, pipelines: job["job_data"]["items_downloaded"], ('numeric',)),
  22. 'dl size': (lambda job, pipelines: job["job_data"]["bytes_downloaded"], ('size', 'numeric')),
  23. 'queue': (lambda job, pipelines: job["job_data"]["items_queued"] - job["job_data"]["items_downloaded"], ('numeric',)),
  24. 'eta': (lambda job, pipelines: int((curTime := time.time()) + (job["job_data"]["items_queued"] - job["job_data"]["items_downloaded"]) / (job["job_data"]["items_downloaded"] / (curTime - job["job_data"]["started_at"]))) if job["job_data"]["items_downloaded"] > 0 else 0, ('date', 'numeric')),
  25. 'con': (lambda job, pipelines: job["job_data"]["concurrency"], ('numeric',)),
  26. 'delay min': (lambda job, pipelines: int(job["job_data"]["delay_min"]), ('hidden', 'numeric')),
  27. 'delay max': (lambda job, pipelines: int(job["job_data"]["delay_max"]), ('hidden', 'numeric')),
  28. 'delay': (lambda job, pipelines: str(int(job["job_data"]["delay_min"])) + '-' + str(int(job["job_data"]["delay_max"])) if job["job_data"]["delay_min"] != job["job_data"]["delay_max"] else str(int(job["job_data"]["delay_min"])), ()),
  29. }
  30. defaultSort = 'jobid'
  31. # Validate
  32. if any('truncatable' in colDef[1] and any(x in colDef[1] for x in ('date', 'coloured', 'size')) for colDef in columns.values()):
  33. # Truncation code can't handle renderers
  34. raise RuntimeError('Invalid column definitions: cannot combine date/coloured/size with truncatable')
  35. # Filter function
  36. def make_field_filter(column, op, value, caseSensitive = True):
  37. compFunc = {
  38. "=": lambda a, b: a == b,
  39. "<": lambda a, b: a < b,
  40. ">": lambda a, b: a > b,
  41. "^": lambda a, b: a.startswith(b),
  42. "*": lambda a, b: b in a,
  43. "$": lambda a, b: a.endswith(b),
  44. "~": lambda a, b: re.search(b, a) is not None,
  45. }[op]
  46. transform = {
  47. True: (lambda x: x),
  48. False: (lambda x: x.lower() if isinstance(x, str) else x)
  49. }[caseSensitive]
  50. return (lambda job: compFunc(transform(job[column]), transform(value)))
  51. # Parse arguments
  52. class FilterAction(argparse.Action):
  53. def __call__(self, parser, namespace, values, optionString = None):
  54. if optionString == '--pyfilter':
  55. try:
  56. func = compile(values[0], '<pyfilter>', 'eval')
  57. except Exception as e:
  58. parser.error(f'Could not compile filter expression: {type(e).__module__}.{type(e).__name__}: {e!s}')
  59. setattr(namespace, self.dest, lambda job: eval(func, {}, {'job': job}))
  60. return
  61. global columns
  62. match = re.match(r"^(?P<column>[A-Za-z ]+)(?P<op>[=<>^*$~])(?P<value>.*)$", values[0])
  63. if not match:
  64. parser.error('Invalid filter')
  65. filterDict = match.groupdict()
  66. filterDict["column"] = filterDict["column"].lower()
  67. assert filterDict["column"] in columns
  68. if 'numeric' in columns[filterDict['column']][1]:
  69. filterDict['value'] = float(filterDict['value'])
  70. if 'date' in columns[filterDict['column']][1] and filterDict['value'] < 0:
  71. filterDict['value'] = time.time() + filterDict['value']
  72. setattr(namespace, self.dest, make_field_filter(filterDict['column'], filterDict['op'], filterDict['value'], caseSensitive = (optionString in ('--filter', '-f'))))
  73. def parse_sort(value):
  74. global columns
  75. sortDesc = value.startswith('-')
  76. if sortDesc:
  77. value = value[1:]
  78. value = value.lower()
  79. if value not in columns:
  80. parser.error('Invalid column name')
  81. return (value, sortDesc)
  82. class SortAction(argparse.Action):
  83. def __call__(self, parser, namespace, values, optionString = None):
  84. result = parse_sort(values[0])
  85. if getattr(namespace, self.dest, None) is None:
  86. setattr(namespace, self.dest, [])
  87. getattr(namespace, self.dest).append(result)
  88. parser = argparse.ArgumentParser(formatter_class = argparse.RawTextHelpFormatter)
  89. parser.add_argument('--filter', '-f', nargs = 1, type = str, action = FilterAction, help = '\n'.join([
  90. 'Filter the table for rows where a COLUMN has a certain VALUE. If specified multiple times, only the last value is used.',
  91. 'FILTER has the format COLUMN{=|<|>|^|*|$|~}VALUE',
  92. ' = means the value must be exactly as specified.',
  93. ' < and > mean it must be less/greater than the specified.',
  94. ' ^ and $ mean it must start/end with the specified.',
  95. ' * means it must contain the specified.',
  96. ' ~ means it must match the specified regex.',
  97. ]))
  98. parser.add_argument('--ifilter', '-i', nargs = 1, type = str, action = FilterAction, dest = 'filter', help = 'Like --filter but case-insensitive')
  99. parser.add_argument('--pyfilter', nargs = 1, type = str, action = FilterAction, dest = 'filter', help = 'A Python expression for filtering using the local variable `job`')
  100. parser.add_argument('--sort', '-s', nargs = 1, type = str, action = SortAction, help = "Sort the table by a COLUMN (descending if preceded by '-'). This can be used multiple times to refine the sorting.")
  101. parser.add_argument('--mode', choices = ('table', 'dashboard-regex', 'con-d-commands', 'format', 'atdash'), default = 'table', help = '\n'.join([
  102. 'Output modes:',
  103. ' table: print a table of the matched jobs',
  104. ' dashboard-regex: compose a regular expression that can be used on the dashboard to actively watch the jobs matched by the filter',
  105. ' con-d-commands: print !con and !d commands for the current settings',
  106. ' format: print some output for each job, separated by newlines; this requires the --format option',
  107. ' atdash: print the URL for displaying the matched jobs on atdash',
  108. ]))
  109. parser.add_argument('--no-colours', '--no-colors', action = 'store_true', help = "Don't colourise the last activity column if it's been a while. (Table mode only)")
  110. parser.add_argument('--no-table', action = 'store_true', help = 'Raw output without feeding through column(1); columns are separated by tabs. (Table mode only)')
  111. parser.add_argument('--no-truncate', action = 'store_true', help = 'Disable truncating long values if the terminal width would be exceeded. (Table mode without --no-table only)')
  112. parser.add_argument('--dates', action = 'store_true', help = 'Print dates instead of elapsed times for queued/started/last active columns. (Table mode only)')
  113. parser.add_argument('--replace-concurrency', nargs = 1, metavar = 'CON', type = int, help = 'Replace the delay values with the specified ones. (con-d-commands mode only)')
  114. parser.add_argument('--replace-delay', nargs = 2, metavar = ('MIN', 'MAX'), type = int, help = 'Replace the delay values with the specified ones. (con-d-commands mode only)')
  115. parser.add_argument('--format', help = 'Output format for the format mode; this must be a Python format string and can use any column name in lower-case with spaces replaced by underscores; e.g. "{url} {last_active}". (Format mode only)')
  116. args = parser.parse_args()
  117. if args.mode == 'format' and not args.format:
  118. print('Error: when using format mode, --format is required.', file = sys.stderr)
  119. sys.exit(1)
  120. if not args.sort:
  121. args.sort = [parse_sort(defaultSort)]
  122. if args.mode == 'con-d-commands':
  123. args.mode = 'format'
  124. args.format = '!con {jobid} {con}\n!d {jobid} {delay_min} {delay_max}'
  125. else:
  126. args.replace_concurrency = None
  127. args.replace_delay = None
  128. # Retrieve
  129. def fetch(url):
  130. req = urllib.request.Request(url)
  131. req.add_header('Accept', 'application/json')
  132. with urllib.request.urlopen(req) as f:
  133. if f.getcode() != 200:
  134. raise RuntimeError('Could not fetch job data')
  135. return json.load(f)
  136. jobdata = fetch('http://dashboard.at.ninjawedding.org/logs/recent?count=1')
  137. pipelinedata = fetch('http://dashboard.at.ninjawedding.org/pipelines')
  138. currentTime = time.time()
  139. # Process
  140. pipelines = {p["id"]: p["nickname"] for p in pipelinedata["pipelines"]}
  141. jobs = []
  142. for job in jobdata:
  143. jobs.append({column: columnFunc(job, pipelines) for column, (columnFunc, _) in columns.items()})
  144. if not jobs:
  145. # Nothing to do
  146. sys.exit(0)
  147. # Filter
  148. if args.filter:
  149. jobs = [job for job in jobs if args.filter(job)]
  150. if not jobs:
  151. sys.exit(0)
  152. # Sort
  153. class reversor: # https://stackoverflow.com/a/56842689
  154. def __init__(self, obj):
  155. self.obj = obj
  156. def __eq__(self, other):
  157. return other.obj == self.obj
  158. def __lt__(self, other):
  159. return other.obj < self.obj
  160. sortColumns = tuple((column, descending, columns[column]) for column, descending in args.sort)
  161. if not args.dates:
  162. # Reverse sorting order for columns which have a date attribute since the column will have elapsed time
  163. sortColumns = tuple((column, not descending if 'date' in columnInfo[1] else descending, columnInfo) for column, descending, columnInfo in sortColumns)
  164. jobs = sorted(jobs, key = lambda job: tuple(job[column] if not descending else reversor(job[column]) for column, descending, _ in sortColumns))
  165. # Concurrency and delay overrides if specified and relevant
  166. if args.replace_concurrency is not None or args.replace_delay is not None:
  167. for job in jobs:
  168. if args.replace_concurrency is not None:
  169. job['con'] = args.replace_concurrency[0]
  170. if args.replace_delay is not None:
  171. job['delay min'] = args.replace_delay[0]
  172. job['delay max'] = args.replace_delay[1]
  173. # Non-table output modes
  174. if args.mode == 'dashboard-regex':
  175. print('^(' + '|'.join(re.escape(job['url']) for job in jobs) + ')$')
  176. sys.exit(0)
  177. elif args.mode == 'format':
  178. for job in jobs:
  179. print(args.format.format(**{key.replace(' ', '_'): value for key, value in job.items()}))
  180. sys.exit(0)
  181. elif args.mode == 'atdash':
  182. print('https://atdash.meo.ws/d/nipgvEwmk/archivebot?orgId=1&' + '&'.join(f'var-ident={job["jobid"]}' for job in jobs))
  183. sys.exit(0)
  184. # Renderers
  185. def render_date(ts, coloured = False):
  186. global args, currentTime
  187. diff = currentTime - ts
  188. colourStr = f"\x1b[{0 if diff < 6 * 3600 else 7};31m" if coloured and diff >= 300 else ""
  189. colourEndStr = "\x1b[0m" if colourStr else ""
  190. if args.dates:
  191. return (colourStr, datetime.datetime.fromtimestamp(ts).isoformat(sep = " "), colourEndStr)
  192. if diff < -86400:
  193. return (colourStr, f"in {-diff // 86400:.0f}d {(-diff % 86400) // 3600:.0f}h", colourEndStr)
  194. elif diff < -60:
  195. return (colourStr, "in " + (f"{-diff // 3600:.0f}h " if diff <= -3600 else "") + f"{(-diff % 3600) // 60:.0f}mn", colourEndStr)
  196. elif diff < 0:
  197. return "in <1 min"
  198. elif diff == 0:
  199. return "now"
  200. elif diff < 60:
  201. return "<1 min ago"
  202. elif diff < 86400:
  203. return (colourStr, (f"{diff // 3600:.0f}h " if diff >= 3600 else "") + f"{(diff % 3600) // 60:.0f}mn ago", colourEndStr)
  204. else:
  205. return (colourStr, f"{diff // 86400:.0f}d {(diff % 86400) // 3600:.0f}h ago", colourEndStr)
  206. def render_size(size):
  207. units = ('B', 'KiB', 'MiB', 'GiB', 'TiB')
  208. unitIdx = min(int(math.log(size, 1024)), len(units) - 1) if size >= 1 else 0
  209. if unitIdx == 0:
  210. return f'{size} B' # No decimal places
  211. return f'{size / 1024 ** unitIdx:.1f} {units[unitIdx]}'
  212. renderers = {}
  213. for column, (_, columnAttr) in columns.items():
  214. if "date" in columnAttr:
  215. if "coloured" in columnAttr:
  216. renderers[column] = lambda x: render_date(x, coloured = not args.no_colours)
  217. else:
  218. renderers[column] = render_date
  219. elif "size" in columnAttr:
  220. renderers[column] = render_size
  221. elif isinstance(jobs[0][column], (int, float)):
  222. renderers[column] = str
  223. for job in jobs:
  224. for column in renderers:
  225. job[column] = renderers[column](job[column])
  226. # Truncate if applicable
  227. printableColumns = {column: colDef for column, colDef in columns.items() if 'hidden' not in colDef[1]}
  228. if not args.no_table and not args.no_truncate:
  229. widthsD = {column: max(itertools.chain((len(column),), (len(job[column]) if isinstance(job[column], str) else len(job[column][1]) for job in jobs))) for column in printableColumns}
  230. minWidthsD = {column: len(column) for column in printableColumns}
  231. try:
  232. termWidth = os.get_terminal_size().columns
  233. except OSError as e:
  234. if e.errno == 25:
  235. # Inappropriate ioctl for device (stdout not a terminal, happens e.g. when redirecting or piping)
  236. # Silently ignore this and don't truncate
  237. termWidth = float('Inf')
  238. else:
  239. raise
  240. overage = sum(x + 2 for x in widthsD.values()) - 2 - termWidth
  241. if overage > 0:
  242. if sum((widthsD[column] if 'truncatable' not in colDef[1] else minWidthsD[column]) + 2 for column, colDef in printableColumns.items()) - 2 > termWidth:
  243. # Even truncating all truncatable columns to the minimum width is not sufficient, i.e. can't match this terminal width. Print a warning and proceed normally
  244. print('Sorry, cannot truncate columns to terminal width', file = sys.stderr)
  245. else:
  246. # Distribute overage to truncatable columns proportionally to each column's length over the minimum
  247. truncatableColumns = {column: colDef for column, colDef in columns.items() if 'truncatable' in colDef[1]}
  248. totalOverMin = sum(widthsD[column] - minWidthsD[column] for column in truncatableColumns)
  249. trWidthsD = {column: math.floor(widthsD[column] - (widthsD[column] - minWidthsD[column]) / totalOverMin * overage) for column in truncatableColumns}
  250. if sum(widthsD[column] - trWidthsD[column] for column in truncatableColumns) - overage == 1:
  251. # Truncated one more character than necessary due to the flooring; add it again to the shortest column
  252. trWidthsD[min(trWidthsD, key = trWidthsD.get)] += 1
  253. for job in jobs:
  254. for column in truncatableColumns:
  255. if len(job[column]) > trWidthsD[column]:
  256. job[column] = job[column][:trWidthsD[column] - 1] + '…'
  257. # Print
  258. output = []
  259. output.append(tuple(column.upper() for column in columns if "hidden" not in columns[column][1]))
  260. for job in jobs:
  261. output.append(tuple(job[column] for column in columns if "hidden" not in columns[column][1]))
  262. if not args.no_table:
  263. widths = tuple(max(len(field) if isinstance(field, str) else len(field[1]) for field in column) for column in zip(*output))
  264. for row in output:
  265. print(' '.join((value.ljust(width) if isinstance(value, str) else ''.join((value[0], value[1], value[2], ' ' * (width - len(value[1]))))) for value, width in zip(row, widths)))
  266. else:
  267. for row in output:
  268. print('\t'.join(field if isinstance(field, str) else ''.join(field) for field in row))