Ви не можете вибрати більше 25 тем Теми мають розпочинатися з літери або цифри, можуть містити дефіси (-) і не повинні перевищувати 35 символів.

89 рядки
2.8 KiB

  1. #!/bin/bash
  2. # Create a copy of this config.sh, customise it and place it in the
  3. # working directory of the packing and upload scripts.
  4. ####################
  5. # CHUNKER SETTINGS #
  6. ####################
  7. # start a new chunk when the current chunk is at least this large
  8. MEGABYTES_PER_CHUNK=$((1024*25))
  9. ###################
  10. # UPLOAD METADATA #
  11. ###################
  12. # your Archive.org S3 keys
  13. IA_AUTH="ACCESS_KEY:SECRET"
  14. # the name of the collection to add the uploads to
  15. IA_COLLECTION="archiveteam_TODO"
  16. # the title of the items (" ${item_timestamp}" will be appended)
  17. IA_ITEM_TITLE="Archive Team TODO:"
  18. # the prefix of the item name ("${item_timestamp}" is appended)
  19. IA_ITEM_PREFIX="archiveteam_todo_"
  20. # the prefix of the megawarc filename ("${item_timestamp}" is appended)
  21. FILE_PREFIX="todo_"
  22. # the date field for the item
  23. IA_ITEM_DATE=$( date +"%Y-%m" )
  24. # offload items to another rsync storage instead of uploading to IA
  25. OFFLOAD_TARGET="rsync://somewhere-far-away:portnum/module-name/directory/"
  26. # it is also possible to create a list of targets and the offloader will pick one at random and retry others on failure
  27. # simply comment out the line above and put all rsync target urls separated by newline in a file called "offload_targets"
  28. # the API for requesting the ZSTD dictionaries
  29. ZST_DICTIONARY_API="API_URL"
  30. ###############
  31. # DIRECTORIES #
  32. ###############
  33. # Put your directories on one or two filesystems (see README).
  34. FS1_BASE_DIR="/archiveteam/ssd/project"
  35. FS2_BASE_DIR="/archiveteam/disk/project"
  36. ## THESE DIRECTORIES ON FILESYSTEM 1: for warcs
  37. # the rsync upload directory
  38. # (the chunker will package the .warc.gz files in this directory)
  39. INCOMING_UPLOADS_DIR="${FS1_BASE_DIR}/incoming-uploads"
  40. # the chunker working directory
  41. # (this directory will hold the current in-progress chunk)
  42. CHUNKER_WORKING_DIR="${FS1_BASE_DIR}/chunker-work"
  43. # the chunker output directory / the packer queue
  44. # (this directory will hold the completed chunks)
  45. PACKING_QUEUE_DIR="${FS1_BASE_DIR}/packing-queue"
  46. # the packer working directory - warc side
  47. # (this directory will hold the current chunk)
  48. PACKER_WORKING_CHUNKS_DIR="${FS1_BASE_DIR}/packer-work-in"
  49. ## THESE DIRECTORIES ON FILESYSTEM 2: for megawarcs
  50. # the packer working directory - megawarc side
  51. # (this directory will hold the current megawarc)
  52. PACKER_WORKING_MEGAWARC_DIR="${FS2_BASE_DIR}/packer-work-out"
  53. # the packer output directory / the upload queue
  54. # (this directory will hold the completed megawarcs)
  55. UPLOAD_QUEUE_DIR="${FS2_BASE_DIR}/upload-queue"
  56. # the uploader working directory
  57. # (this directory will hold the current megawarc)
  58. UPLOADER_WORKING_DIR="${FS2_BASE_DIR}/uploader-work"
  59. # the final destination for uploaded megawarcs
  60. # leave this empty to remove megawarcs after uploading
  61. COMPLETED_DIR="${FS2_BASE_DIR}/uploaded"
  62. # remove this
  63. echo "config.sh not customised."
  64. exit 1