Merge warc-peek repository into little-things

warc-peek

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+# `warc-peek.py` is a small script to help looking into gzipped WARC files without decompressing the entire file.
+# It searches a window in the file for gzip's magic bytes `1F 8B`, attempts decompression, compares the result to the expected beginning of a WARC record, and prints all valid offsets.
+# These can then be used with e.g. `tail` and `zless` to actually look at the records.
+#
+# Usage: warc-peek.py WARCFILE OFFSET LENGTH
+#  Opens `WARCFILE`, reads `LENGTH` bytes starting at `OFFSET` (zero-based), and prints valid WARC record offsets to stdout (one integer per line).
+#
+# Caveats
+# - This script only works with WARCs in which each record is compressed individually.
+#   This is what the specification recommends and what most tools should generate by default, but there definitely exist valid compressed WARCs which can't be processed in this way.
+# - When you want to use `tail -c+OFFSET WARCFILE | zless` to look at the records, keep in mind that `tail` uses one-based indices, i.e. you will have to add one to the indices returned by `warc-peek.py`.
+# - `warc-peek.py` will miss valid record offsets in the last 512 bytes of the window.
+#   This is because a certain length of the compressed data is necessary to be able to decompress it. `warc-peek.py` uses 512 bytes for this and will therefore
+#   not attempt decompression when `1F 8B` is found in the last 512 bytes of the window. You can increase `LENGTH` to compensate for this if necessary.
+
+import argparse
+import logging
+import zlib
+
+
+logger = logging.getLogger('warc-peek')
+
+
+def finditer(b, sub):
+	pos = 0
+	while True:
+		pos = b.find(sub, pos)
+		if pos < 0:
+			break
+		yield pos
+		pos += 1
+
+
+def find_offsets(warcfile, offset, length):
+	with open(warcfile, 'rb') as fp:
+		fp.seek(offset)
+		buffer = fp.read(length)
+
+	logger.debug('Buffer length: {:d}'.format(len(buffer)))
+	for pos in finditer(buffer, b'\x1f\x8b'):
+		logger.debug('Trying relative offset {:d}'.format(pos))
+		if pos > len(buffer) - 512: # 512 bytes might be a bit too much, but at least it ensures that the decompression will work.
+			break
+		try:
+			dec = zlib.decompressobj(zlib.MAX_WBITS | 32).decompress(buffer[pos:pos+512])
+		except:
+			continue
+		logger.debug('First 100 bytes of decompressed data: {!r}'.format(dec[:100]))
+		if dec.startswith(b'WARC/1.0\r\n'):
+			yield offset + pos
+
+
+if __name__ == '__main__':
+	parser = argparse.ArgumentParser()
+	parser.add_argument('--debug', action = 'store_true', help = 'Enable debug output')
+	parser.add_argument('warcfile', help = 'A .warc.gz file')
+	parser.add_argument('offset', type = int, help = 'Zero-based byte offset of the window')
+	parser.add_argument('length', type = int, help = 'Length in bytes of the window')
+	args = parser.parse_args()
+
+	if args.debug:
+		logging.basicConfig(
+			format = '{asctime}  {levelname}  {name}  {message}',
+			style = '{',
+			level = logging.DEBUG,
+		)
+	for offset in find_offsets(args.warcfile, args.offset, args.length):
+		print(offset)