|
|
@@ -20,6 +20,13 @@ logging.basicConfig(level=logging.INFO) |
|
|
|
BACKFEED_DELIM = "\n" |
|
|
|
|
|
|
|
|
|
|
|
# TODO: Create a function that abstracts away the while True try except loops. With exponential backoff. |
|
|
|
# time.sleep(min(2**tries, 64)) |
|
|
|
# TODO: Add rsync support |
|
|
|
# TODO: Add rsync+ssh support |
|
|
|
# TODO: Add webdav support. |
|
|
|
# TODO: Fix the "ctrl-c handling" logic so it actually cleans up in the s3 bucket. |
|
|
|
|
|
|
|
@click.group() |
|
|
|
def sender(): |
|
|
|
pass |
|
|
@@ -204,14 +211,18 @@ def single_impl(item_directory: pathlib.Path, ia_collection: str, ia_item_title: |
|
|
|
logging.warning(f"Skipping backfeed! Would have submitted: {bf_item}") |
|
|
|
else: |
|
|
|
while True: |
|
|
|
u = f"https://legacy-api.arpa.li/backfeed/legacy/{backfeed_key}" |
|
|
|
logging.info(f"Attempting to submit bf item {bf_item} to {u}...") |
|
|
|
resp = requests.post(u, params={"skipbloom": "1", "delimiter": BACKFEED_DELIM}, |
|
|
|
data=f"{bf_item}{BACKFEED_DELIM}".encode("UTF-8"), timeout=60) |
|
|
|
if resp.status_code == 200: |
|
|
|
break |
|
|
|
logging.warning(f"Failed to submit to backfeed {resp.status_code}: {resp.text}") |
|
|
|
time.sleep(30) |
|
|
|
try: |
|
|
|
u = f"https://legacy-api.arpa.li/backfeed/legacy/{backfeed_key}" |
|
|
|
logging.info(f"Attempting to submit bf item {bf_item} to {u}...") |
|
|
|
resp = requests.post(u, params={"skipbloom": "1", "delimiter": BACKFEED_DELIM}, |
|
|
|
data=f"{bf_item}{BACKFEED_DELIM}".encode("UTF-8"), timeout=60) |
|
|
|
if resp.status_code == 200: |
|
|
|
break |
|
|
|
logging.warning(f"Failed to submit to backfeed {resp.status_code}: {resp.text}") |
|
|
|
time.sleep(30) |
|
|
|
except Exception: |
|
|
|
logging.exception("Failed to submit to backfeed") |
|
|
|
time.sleep(30) |
|
|
|
logging.info("Backfeed submit complete!") |
|
|
|
|
|
|
|
if delete: |
|
|
|