diff --git a/deb-repo-urls b/deb-repo-urls new file mode 100755 index 0000000..7387cbd --- /dev/null +++ b/deb-repo-urls @@ -0,0 +1,165 @@ +#!/bin/bash +set -e + +if [[ $# -ne 2 || "$1" == '-h' || "$1" == '--help' ]] +then + echo 'Lists all relevant files in a Debian-style repository' >&2 + echo >&2 + echo 'Usage: deb-repo-ls ARCHIVEROOT DISTRIBUTION' >&2 + exit 1 +fi + +archiveRoot="$1" +distribution="$2" + +while [[ "${archiveRoot}" == */ ]]; do archiveRoot="${archiveRoot%/}"; done # Strip trailing slashes since they're added again manually below +if [[ "${distribution}" == */ || "${distribution}" == /* ]]; then echo "Invalid distribution" >&2; exit 1; fi + +declare -i fetchedSize +declare -i totalSize=0 + +pipe=$(mktemp -u); mkfifo "${pipe}"; exec 3<>"${pipe}"; rm "${pipe}"; unset pipe # For fetch size + +function maybe_decompress { + url="$1" + if [[ "${url}" == *'.gz' ]] + then + gunzip + elif [[ "${url}" == *'.bz2' ]] + then + bunzip2 + elif [[ "${url}" == *'.xz' ]] + then + unxz + else + # Peek at the data to see if it's compressed + beginning="$(head -c 10 | xxd -p)" + { + xxd -p -r <<<"${beginning}" + cat + } | { + if [[ "${beginning}" == '1f8b'* ]] + then + gunzip + elif [[ "${beginning}" == '425a68'??'314159265359'* ]] + then + bunzip2 + elif [[ "${beginning}" == 'fd377a585a00'* ]] + then + unxz + else + cat + fi + } + fi +} + +function fetch_and_print { + url="$1" + # Fetch url, print it to stdout; as side effects, the HTTP body is stored in fetchedBody and its size is added to totalSize + # If the url ends with .gz, it's gunzipped; if it ends in .bz2, it gets bunzip2'd. The totalSize reflects the compressed size. + echo "Fetching ${url}" >&2 + fetchedBody="$(curl -A 'Debian APT-HTTP/1.3 (1.8.2)' -s "${url}" | tee >(wc -c >&3) | maybe_decompress "${url}")" + echo "${url}" + #totalSize+=$(LANG=C; LC_ALL=C; echo ${#fetchedBody}) + totalSize+=$(head -1 <&3) +} + +function head_and_print { + url="$1" + # Issue a HEAD request on url, print it to stdout, and add its content length to totalSize + echo "Heading ${url}" >&2 + declare -i contentLength="$(curl -A 'Debian APT-HTTP/1.3 (1.8.2)' -s --head "${url}" | grep -i '^Content-Length: ' | awk '{print $2}' | tr -d '\r')" + echo "${url}" + totalSize+=${contentLength} +} + +# Retrieve Release and accompanying files, parse it and extract the next level +fetch_and_print "${archiveRoot}/dists/${distribution}/Release" +inHashSection= +declare -A files # filename -> size +while IFS='' read -r line +do + if [[ "${line}" == 'MD5Sum:' || "${line}" == 'SHA1:' || "${line}" == 'SHA256:' || "${line}" == 'SHA512:' ]] + then + inHashSection=1 + continue + fi + if [[ "${line}" != ' '* ]] + then + inHashSection= + continue + fi + if [[ "${inHashSection}" ]] + then + files["$(awk '{print $3}' <<<"${line}")"]="$(awk '{print $2}' <<<"${line}")" + fi +done <<<"${fetchedBody}" + +head_and_print "${archiveRoot}/dists/${distribution}/Release.gpg" +head_and_print "${archiveRoot}/dists/${distribution}/InRelease" + +# Process files +filename= +declare -i size=-1 +declare -A debfiles # filename -> size +for fn in "${!files[@]}" +do + # Legacy releases + if [[ "${fn}" =~ /Release$ ]] + then + echo "${archiveRoot}/dists/${distribution}/${fn}" + totalSize+="${files[$fn]}" + + # Package indices + elif [[ "${fn}" =~ /Packages(\.[^/]*)?$ ]] + then + echo "Processing package: ${fn}" + fetch_and_print "${archiveRoot}/dists/${distribution}/${fn}" + filename= + size=-1 + while IFS= read -r line + do + if [[ "${line}" == '' ]] + then + filename= + size=-1 + elif [[ "${line}" == 'Filename: '* ]] + then + filename="${line:10}" + elif [[ "${line}" == 'Size: '* ]] + then + size="${line:6}" + fi + if [[ "${filename}" && ${size} -ge 0 ]] + then + debfiles["${archiveRoot}/dists/${distribution}/${filename}"]=${size} + filename= + size=-1 + fi + done <<<"${fetchedBody}" + + # Contents indices + elif [[ "${fn}" =~ /Contents-[^/]*$ ]] + then + # Nothing really to do here but take note of its existence + echo "${archiveRoot}/dists/${distribution}/${fn}" + totalSize+="${files[$fn]}" + + # Anything else + else + echo "${archiveRoot}/dists/${distribution}/${fn}" + totalSize+="${files[$fn]}" + echo "Skipping unknown file: ${fn}" >&2 + fi +done + +# Print the debfiles and add them to the total size +for debfile in "${!debfiles[@]}" +do + echo "${debfile}" + totalSize+="${debfiles[$debfile]}" +done + +# Report total size to stderr +echo "Total size: ${totalSize} bytes" >&2