little-things/moinmoin-url-list

#!/bin/bash
if [[ $# -ne 1 || "$1" != http* ]]; then
	printf 'Usage: %q FRONT_PAGE_URL\n' "$0" >&2
	exit 1
fi

url="$1"
if [[ "${url}" == *\?* ]]; then
	printf 'Error: URLs with query strings not supported\n' >&2
	exit 1
fi
url="${url%/*}"

curl "${url}/TitleIndex" | \
	grep -A 1000000 -F 'href="/TitleIndex?allpages=1"' | \
	grep -B 1000000 -F 'id="pagebottom"' | \
	grep -Po 'href="\K[^"]+' | \
	grep -Fxv '/TitleIndex?allpages=1' | \
	sed 's,\?action=AttachFile$,,' | \
	perl -ne 'print if ! $a{$_}++' | \
	awk -v url="${url}" \
		'{
			print url $0;
			print url $0 "?action=info";
			print url $0 "?action=info&general=1";
			print url $0 "?action=raw";
			print url $0 "?action=AttachFile";
		}'