The little things give you away... A collection of various small helper stuff
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

64 lines
2.2 KiB

  1. #!/bin/bash
  2. # Fetch the SHA-1 hashes from an IA item and ensure that they match the local files (i.e. that the upload was successful)
  3. identifier="$1"
  4. escapedIdentifier="$(sed 's/[.[\*^$()+?{|]/\\&/g' <<<"${identifier}")"
  5. readarray -t iasha1sums < <(curl -sL "https://archive.org/download/${identifier}/${identifier}_files.xml" | tr -d '\n' | grep -Po '<file .*?</file>' | grep 'source="original".*<sha1>' | sed 's,^.*name=",,; s,".*<sha1>, ,; s,</sha1>.*$,,' | grep -Pv "^${escapedIdentifier}"'(\.cdx\.(gz|idx)|_meta\.(sqlite|xml)) ' | sed 's,&amp;,\&,g' | awk '{ print $NF " " substr($0, 1, length($0) - length($NF) - 1) }')
  6. localFiles=()
  7. while IFS= read -r -d $'\0' f; do localFiles+=("${f:2}"); done < <(find . -type f -print0)
  8. readarray -t iaFiles < <(printf "%s\n" "${iasha1sums[@]}" | sed 's,^.\{40\} ,,')
  9. readarray -t localFilesSorted < <(printf "%s\n" "${localFiles[@]}" | sort)
  10. readarray -t iaFilesSorted < <(printf "%s\n" "${iaFiles[@]}" | sort)
  11. readarray -t localMissing < <(comm -13 <(printf "%s\n" "${localFilesSorted[@]}") <(printf "%s\n" "${iaFilesSorted[@]}"))
  12. readarray -t iaMissing < <(comm -23 <(printf "%s\n" "${localFilesSorted[@]}") <(printf "%s\n" "${iaFilesSorted[@]}"))
  13. status=0
  14. RED=$'\x1b[1;31m'
  15. GREEN=$'\x1b[0;32m'
  16. RESET=$'\x1b[0m'
  17. if [[ ${#localMissing[@]} -eq 0 && ${#iaMissing[@]} -eq 0 ]]
  18. then
  19. echo "File list comparison: ${GREEN}OK${RESET}"
  20. else
  21. echo "File list comparison: ${RED}FAIL${RESET}"
  22. fi
  23. if [[ ${#iaMissing[@]} -gt 0 ]]
  24. then
  25. echo "Local files that are not in the IA item:"
  26. printf " %s\n" "${iaMissing[@]}"
  27. status=1
  28. fi
  29. if [[ ${#localMissing[@]} -gt 0 ]]
  30. then
  31. echo "IA item files that are not in the local directory:"
  32. printf " %s\n" "${localMissing[@]}"
  33. status=1
  34. fi
  35. echo
  36. echo "SHA-1 comparison:"
  37. sha1sum -c < <(printf "%s\n" "${iasha1sums[@]}") > >(perl -pe 's,^(.*)(?<!: OK)$,'"${RED}"'\1'"${RESET}"',; s,: OK$,: '"${GREEN}OK${RESET}"',; s,^, ,') 2>&1
  38. shast=$?
  39. wait # Wait for Perl to finish; Bash 4.4+ only
  40. if [[ ${shast} -eq 0 ]]
  41. then
  42. echo "SHA-1 comparison: ${GREEN}OK${RESET}"
  43. else
  44. echo "SHA-1 comparison: ${RED}FAIL${RESET}"
  45. status=1
  46. fi
  47. echo
  48. if [[ ${status} -eq 0 ]]
  49. then
  50. echo "${GREEN}OK${RESET}"
  51. else
  52. echo "${RED}FAIL${RESET}"
  53. fi
  54. exit ${status}