|
|
@@ -232,10 +232,13 @@ printf '%s' "${code[@]# }" >"${prefix}.py" # That's a tab character after the h |
|
|
|
# Generate the markers file |
|
|
|
{ if [[ -z "${noStartMarker}" ]]; then echo; fi; printf '%s\n' "$@"; if [[ -z "${noEndMarker}" ]]; then echo; fi; } >"${prefix}-markers" |
|
|
|
|
|
|
|
# Collect environment variables |
|
|
|
envvars=() |
|
|
|
envvars+=(S3BL_CMD="${cmd}") |
|
|
|
envvars+=(S3_FORMAT="${format}") |
|
|
|
envvars+=(S3_BUCKET_URL="${bucketUrl}") |
|
|
|
envvars+=(S3_MARKERS_FILENAME="${prefix}-markers") |
|
|
|
if [[ "${listUrls}" ]]; then envvars+=(S3_WITH_LIST_URLS="${listUrls}"); fi |
|
|
|
|
|
|
|
# Lift-off! |
|
|
|
S3BL_CMD="${cmd}" \ |
|
|
|
S3_FORMAT="${format}" \ |
|
|
|
S3_BUCKET_URL="${bucketUrl}" \ |
|
|
|
S3_MARKERS_FILENAME="${prefix}-markers" \ |
|
|
|
S3_WITH_LIST_URLS="${listUrls}" \ |
|
|
|
qwarc --concurrency "${concurrency}" --database "${prefix}.db" --log "${prefix}.log" --warc "${prefix}" "${prefix}.py" |
|
|
|
env "${envvars[@]}" qwarc --concurrency "${concurrency}" --database "${prefix}.db" --log "${prefix}.log" --warc "${prefix}" "${prefix}.py" |