|
|
@@ -0,0 +1,15 @@ |
|
|
|
#!/bin/bash |
|
|
|
set -euo pipefail |
|
|
|
|
|
|
|
if [[ $# -eq 0 || "$1" == '-h' || "$1" == '--help' ]] |
|
|
|
then |
|
|
|
echo "Usage: ia-cdx-search-subdomains [OPTIONS] DOMAIN" >&2 |
|
|
|
echo "Extracts all known subdomains from IA's CDX API, printing unique subdomains to stdout" >&2 |
|
|
|
echo "Any options are passed through to ia-cdx-search; use this to customise parallelism or resume after errors" >&2 |
|
|
|
exit 1 |
|
|
|
fi |
|
|
|
|
|
|
|
domain="${@: -1}" |
|
|
|
set -- "${@:1:$(($#-1))}" |
|
|
|
scriptpath="$(cd "$(dirname "$0")"; pwd -P)" |
|
|
|
"${scriptpath}/ia-cdx-search" "$@" "url=${domain}&collapse=urlkey&fl=original&matchType=domain" | grep -o '^[^/]*//[^/]*' | sed 's,^.*/,,; s,:[0-9]\+$,,' | "${scriptpath}/uniqify" |