|
|
@@ -38,8 +38,7 @@ def watch_pass(input_directory: pathlib.Path, work_directory: pathlib.Path, ia_c |
|
|
|
except FileNotFoundError: |
|
|
|
logging.warning(f"Unable to move item {original_directory}") |
|
|
|
single_impl(new_directory, ia_collection, ia_item_title, ia_item_prefix, ia_item_date, project, |
|
|
|
dispatcher, |
|
|
|
delete, backfeed_key) |
|
|
|
dispatcher, delete, backfeed_key) |
|
|
|
return True |
|
|
|
finally: |
|
|
|
# If we exit somehow without deleting, move it back. Likely ctrl+c. |
|
|
@@ -138,7 +137,7 @@ def single_impl(item_directory: pathlib.Path, ia_collection: str, ia_item_title: |
|
|
|
meta_json["SIZE_HINT"] = str(total_size) |
|
|
|
while True: |
|
|
|
try: |
|
|
|
r = requests.get(f"{dispatcher}/offload_target", params=meta_json) |
|
|
|
r = requests.get(f"{dispatcher}/offload_target", params=meta_json, timeout=60) |
|
|
|
if r.status_code == 200: |
|
|
|
data = r.json() |
|
|
|
url = data["url"] |
|
|
@@ -150,12 +149,20 @@ def single_impl(item_directory: pathlib.Path, ia_collection: str, ia_item_title: |
|
|
|
time.sleep(30) |
|
|
|
logging.info(f"Assigned target {url}") |
|
|
|
parsed_url = urllib.parse.urlparse(url) |
|
|
|
bf_item = None |
|
|
|
if parsed_url.scheme == "minio+http" or parsed_url.scheme == "minio+https": |
|
|
|
secure = (parsed_url.scheme == "minio+https") |
|
|
|
ep = parsed_url.hostname |
|
|
|
if parsed_url.port is not None: |
|
|
|
ep = f"{ep}:{parsed_url.port}" |
|
|
|
client = minio.Minio(endpoint=ep, access_key=parsed_url.username, secret_key=parsed_url.password, secure=secure) |
|
|
|
client = None |
|
|
|
while True: |
|
|
|
try: |
|
|
|
client = minio.Minio(endpoint=ep, access_key=parsed_url.username, secret_key=parsed_url.password, secure=secure) |
|
|
|
break |
|
|
|
except Exception: |
|
|
|
logging.exception("Failed to connect to minio") |
|
|
|
time.sleep(30) |
|
|
|
bucket_name = item_directory.name.replace("_", "-") |
|
|
|
logging.info("Making bucket...") |
|
|
|
while True: |
|
|
@@ -200,7 +207,7 @@ def single_impl(item_directory: pathlib.Path, ia_collection: str, ia_item_title: |
|
|
|
u = f"https://legacy-api.arpa.li/backfeed/legacy/{backfeed_key}" |
|
|
|
logging.info(f"Attempting to submit bf item {bf_item} to {u}...") |
|
|
|
resp = requests.post(u, params={"skipbloom": "1", "delimiter": BACKFEED_DELIM}, |
|
|
|
data=f"{bf_item}{BACKFEED_DELIM}".encode("UTF-8")) |
|
|
|
data=f"{bf_item}{BACKFEED_DELIM}".encode("UTF-8"), timeout=60) |
|
|
|
if resp.status_code == 200: |
|
|
|
break |
|
|
|
logging.warning(f"Failed to submit to backfeed {resp.status_code}: {resp.text}") |
|
|
|