You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

offload-one 3.2 KiB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131
  1. #!/bin/bash
  2. # Offloads megawarcs from the upload queue.
  3. # (Needs a config.sh in the working directory.)
  4. #
  5. # ./offload-one
  6. #
  7. # 1. Grabs an item from UPLOAD_QUEUE_DIR
  8. # 2. Reserves the item by moving the directory to the
  9. # UPLOADER_WORKING_DIR
  10. # 3. Offloads the item to the target defined in OFFLOAD_TARGET
  11. # or in the offload_targets file
  12. # 4. Removes the source files from the working directory
  13. # If COMPLETED_DIR is set, offloaded files are moved there.
  14. #
  15. # The program exits with 1 on any nontransient error.
  16. #
  17. SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
  18. source ./config.sh || exit 1
  19. mkdir -p "${UPLOAD_QUEUE_DIR}" || exit 1
  20. mkdir -p "${UPLOADER_WORKING_DIR}" || exit 1
  21. if [ ! -z "${COMPLETED_DIR}" ]
  22. then
  23. mkdir -p "${COMPLETED_DIR}" || exit 1
  24. fi
  25. function mayicontinue {
  26. echo
  27. # echo "May I continue?"
  28. # read
  29. # echo
  30. }
  31. mayicontinue
  32. if test -z "${OFFLOAD_TARGET}" && ! cat ./offload_targets 2> /dev/null | grep -qE '^rsync://[^/]+/[^/]+'; then
  33. echo "No valid offload target specified in OFFLOAD_TARGET environment variable or ./offload_targets file, aborting offload"
  34. sleep 30
  35. exit 1
  36. fi
  37. # try to grab an item from UPLOAD_QUEUE_DIR
  38. ITEM=none
  39. while [[ "${ITEM}" = none ]]
  40. do
  41. possible_item=$( ls -1 "${UPLOAD_QUEUE_DIR}" | grep -E '[0-9]{14}_[a-f0-9]{8}$' | sort | head -n 1 )
  42. if test -n "${possible_item}"
  43. then
  44. echo "Trying to grab ${possible_item}"
  45. if mv "${UPLOAD_QUEUE_DIR}/${possible_item}" "${UPLOADER_WORKING_DIR}/"
  46. then
  47. ITEM="${possible_item}"
  48. else
  49. echo "Failed to move ${possible_item}"
  50. sleep 5
  51. fi
  52. else
  53. date
  54. echo "No current item found!"
  55. sleep 30
  56. exit 0
  57. fi
  58. done
  59. echo "$( date ): Start offloading for item ${ITEM}" >> uploader.log
  60. result=1
  61. while [[ "${result}" -ne 0 ]]
  62. do
  63. _OFFLOAD_TARGET="${OFFLOAD_TARGET}"
  64. if test -z "${_OFFLOAD_TARGET}"; then
  65. _OFFLOAD_TARGET=$(cat "./offload_targets" 2> /dev/null | grep -E '^rsync://[^/]+/[^/]+' | shuf -n 1)
  66. fi
  67. if test -z "${_OFFLOAD_TARGET}"; then
  68. echo "No valid offload target specified in OFFLOAD_TARGET environment variable or ./offload_targets file"
  69. echo "Will retry in 30 seconds"
  70. sleep 30
  71. continue
  72. fi
  73. echo "Offloading to ${_OFFLOAD_TARGET}/${ITEM}/"
  74. rsync -rltv --timeout=900 --contimeout=60 --sockopts=SO_SNDBUF=8388608,SO_RCVBUF=8388608 --progress --stats --no-owner --no-group --partial --partial-dir .rsync-tmp --delay-updates --no-compress --compress-level 0 "${UPLOADER_WORKING_DIR}/${ITEM}/" "${_OFFLOAD_TARGET}/${ITEM}/"
  75. result="${?}"
  76. if [[ "${result}" -ne 0 ]]
  77. then
  78. date
  79. echo "Error while offloading ${ITEM}, rsync said ${result}"
  80. echo "Will retry in 30 seconds"
  81. sleep 30
  82. fi
  83. done
  84. echo "Offloaded ${ITEM}"
  85. echo "$( date ): Completed offloading for item ${ITEM}" >> uploader.log
  86. mayicontinue
  87. # move or remove megawarc
  88. if [ -z "${COMPLETED_DIR}" ]
  89. then
  90. # remove
  91. rm -rf "${UPLOADER_WORKING_DIR}/${ITEM}"
  92. result="${?}"
  93. if [[ "${result}" -ne 0 ]]
  94. then
  95. date
  96. echo "rm -rf megawarc exited with ${result} for ${ITEM}"
  97. exit 1
  98. fi
  99. else
  100. # move
  101. mv "${UPLOADER_WORKING_DIR}/${ITEM}" "${COMPLETED_DIR}/"
  102. result="${?}"
  103. if [[ "${result}" -ne 0 ]]
  104. then
  105. date
  106. echo "rm -rf megawarc exited with ${result} for ${ITEM}"
  107. exit 1
  108. fi
  109. fi
  110. exit 0