|
|
@@ -90,6 +90,11 @@ while [[ "${prefix}" == */ ]]; do prefix="${prefix%/}"; done # Remove trailing s |
|
|
|
prefix="${prefix//\//_}" # Replace slashes with underscores |
|
|
|
|
|
|
|
# Ensure no collisions |
|
|
|
if [[ -e s3-bucket-list-qwarc ]] |
|
|
|
then |
|
|
|
echo 'Error: s3-bucket-list-qwarc exists in this directory.' >&2 |
|
|
|
exit 1 |
|
|
|
fi |
|
|
|
if [[ "$(shopt -s nullglob; echo "${prefix}"*)" ]] |
|
|
|
then |
|
|
|
echo "Error: there already exist files with the prefix '${prefix}' in this directory." >&2 |
|
|
@@ -223,7 +228,7 @@ readarray code <<EOF |
|
|
|
|
|
|
|
|
|
|
|
specDependencies = qwarc.utils.SpecDependencies( |
|
|
|
files = [os.environ['S3_MARKERS_FILENAME']], |
|
|
|
files = ['s3-bucket-list-qwarc', os.environ['S3_MARKERS_FILENAME']], |
|
|
|
extra = {'env': {k: os.environ.get(k) for k in ('S3BL_CMD', 'S3_FORMAT', 'S3_BUCKET_URL', 'S3_MARKERS_FILENAME', 'S3_WITH_LIST_URLS')}} |
|
|
|
) |
|
|
|
EOF |
|
|
@@ -232,6 +237,9 @@ printf '%s' "${code[@]# }" >"${prefix}.py" # That's a tab character after the h |
|
|
|
# Generate the markers file |
|
|
|
{ if [[ -z "${noStartMarker}" ]]; then echo; fi; printf '%s\n' "$@"; if [[ -z "${noEndMarker}" ]]; then echo; fi; } >"${prefix}-markers" |
|
|
|
|
|
|
|
# Copy this script |
|
|
|
rsync -a "$0" s3-bucket-list-qwarc |
|
|
|
|
|
|
|
# Collect environment variables |
|
|
|
envvars=() |
|
|
|
envvars+=(S3BL_CMD="${cmd}") |
|
|
|