The little things give you away... A collection of various small helper stuff
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

166 lines
4.1 KiB

  1. #!/bin/bash
  2. set -e
  3. if [[ $# -ne 2 || "$1" == '-h' || "$1" == '--help' ]]
  4. then
  5. echo 'Lists all relevant files in a Debian-style repository' >&2
  6. echo >&2
  7. echo 'Usage: deb-repo-ls ARCHIVEROOT DISTRIBUTION' >&2
  8. exit 1
  9. fi
  10. archiveRoot="$1"
  11. distribution="$2"
  12. while [[ "${archiveRoot}" == */ ]]; do archiveRoot="${archiveRoot%/}"; done # Strip trailing slashes since they're added again manually below
  13. if [[ "${distribution}" == */ || "${distribution}" == /* ]]; then echo "Invalid distribution" >&2; exit 1; fi
  14. declare -i fetchedSize
  15. declare -i totalSize=0
  16. pipe=$(mktemp -u); mkfifo "${pipe}"; exec 3<>"${pipe}"; rm "${pipe}"; unset pipe # For fetch size
  17. function maybe_decompress {
  18. url="$1"
  19. if [[ "${url}" == *'.gz' ]]
  20. then
  21. gunzip
  22. elif [[ "${url}" == *'.bz2' ]]
  23. then
  24. bunzip2
  25. elif [[ "${url}" == *'.xz' ]]
  26. then
  27. unxz
  28. else
  29. # Peek at the data to see if it's compressed
  30. beginning="$(head -c 10 | xxd -p)"
  31. {
  32. xxd -p -r <<<"${beginning}"
  33. cat
  34. } | {
  35. if [[ "${beginning}" == '1f8b'* ]]
  36. then
  37. gunzip
  38. elif [[ "${beginning}" == '425a68'??'314159265359'* ]]
  39. then
  40. bunzip2
  41. elif [[ "${beginning}" == 'fd377a585a00'* ]]
  42. then
  43. unxz
  44. else
  45. cat
  46. fi
  47. }
  48. fi
  49. }
  50. function fetch_and_print {
  51. url="$1"
  52. # Fetch url, print it to stdout; as side effects, the HTTP body is stored in fetchedBody and its size is added to totalSize
  53. # If the url ends with .gz, it's gunzipped; if it ends in .bz2, it gets bunzip2'd. The totalSize reflects the compressed size.
  54. echo "Fetching ${url}" >&2
  55. fetchedBody="$(curl -A 'Debian APT-HTTP/1.3 (1.8.2)' -s "${url}" | tee >(wc -c >&3) | maybe_decompress "${url}")"
  56. echo "${url}"
  57. #totalSize+=$(LANG=C; LC_ALL=C; echo ${#fetchedBody})
  58. totalSize+=$(head -1 <&3)
  59. }
  60. function head_and_print {
  61. url="$1"
  62. # Issue a HEAD request on url, print it to stdout, and add its content length to totalSize
  63. echo "Heading ${url}" >&2
  64. declare -i contentLength="$(curl -A 'Debian APT-HTTP/1.3 (1.8.2)' -s --head "${url}" | grep -i '^Content-Length: ' | awk '{print $2}' | tr -d '\r')"
  65. echo "${url}"
  66. totalSize+=${contentLength}
  67. }
  68. # Retrieve Release and accompanying files, parse it and extract the next level
  69. fetch_and_print "${archiveRoot}/dists/${distribution}/Release"
  70. inHashSection=
  71. declare -A files # filename -> size
  72. while IFS='' read -r line
  73. do
  74. if [[ "${line}" == 'MD5Sum:' || "${line}" == 'SHA1:' || "${line}" == 'SHA256:' || "${line}" == 'SHA512:' ]]
  75. then
  76. inHashSection=1
  77. continue
  78. fi
  79. if [[ "${line}" != ' '* ]]
  80. then
  81. inHashSection=
  82. continue
  83. fi
  84. if [[ "${inHashSection}" ]]
  85. then
  86. files["$(awk '{print $3}' <<<"${line}")"]="$(awk '{print $2}' <<<"${line}")"
  87. fi
  88. done <<<"${fetchedBody}"
  89. head_and_print "${archiveRoot}/dists/${distribution}/Release.gpg"
  90. head_and_print "${archiveRoot}/dists/${distribution}/InRelease"
  91. # Process files
  92. filename=
  93. declare -i size=-1
  94. declare -A debfiles # filename -> size
  95. for fn in "${!files[@]}"
  96. do
  97. # Legacy releases
  98. if [[ "${fn}" =~ /Release$ ]]
  99. then
  100. echo "${archiveRoot}/dists/${distribution}/${fn}"
  101. totalSize+="${files[$fn]}"
  102. # Package indices
  103. elif [[ "${fn}" =~ /Packages(\.[^/]*)?$ ]]
  104. then
  105. echo "Processing package: ${fn}" >&2
  106. fetch_and_print "${archiveRoot}/dists/${distribution}/${fn}"
  107. filename=
  108. size=-1
  109. while IFS= read -r line
  110. do
  111. if [[ "${line}" == '' ]]
  112. then
  113. filename=
  114. size=-1
  115. elif [[ "${line}" == 'Filename: '* ]]
  116. then
  117. filename="${line:10}"
  118. elif [[ "${line}" == 'Size: '* ]]
  119. then
  120. size="${line:6}"
  121. fi
  122. if [[ "${filename}" && ${size} -ge 0 ]]
  123. then
  124. debfiles["${archiveRoot}/${filename}"]=${size}
  125. filename=
  126. size=-1
  127. fi
  128. done <<<"${fetchedBody}"
  129. # Contents indices
  130. elif [[ "${fn}" =~ /Contents-[^/]*$ ]]
  131. then
  132. # Nothing really to do here but take note of its existence
  133. echo "${archiveRoot}/dists/${distribution}/${fn}"
  134. totalSize+="${files[$fn]}"
  135. # Anything else
  136. else
  137. echo "${archiveRoot}/dists/${distribution}/${fn}"
  138. totalSize+="${files[$fn]}"
  139. echo "Skipping unknown file: ${fn}" >&2
  140. fi
  141. done
  142. # Print the debfiles and add them to the total size
  143. for debfile in "${!debfiles[@]}"
  144. do
  145. echo "${debfile}"
  146. totalSize+="${debfiles[$debfile]}"
  147. done
  148. # Report total size to stderr
  149. echo "Total size: ${totalSize} bytes" >&2