Add some extra checks

main.py

@@ -38,8 +38,7 @@ def watch_pass(input_directory: pathlib.Path, work_directory: pathlib.Path, ia_c
                 except FileNotFoundError:
                     logging.warning(f"Unable to move item {original_directory}")
                 single_impl(new_directory, ia_collection, ia_item_title, ia_item_prefix, ia_item_date, project,
-                            dispatcher,
-                            delete, backfeed_key)
+                            dispatcher, delete, backfeed_key)
                 return True
             finally:
                 # If we exit somehow without deleting, move it back. Likely ctrl+c.
@@ -138,7 +137,7 @@ def single_impl(item_directory: pathlib.Path, ia_collection: str, ia_item_title:
     meta_json["SIZE_HINT"] = str(total_size)
     while True:
         try:
-            r = requests.get(f"{dispatcher}/offload_target", params=meta_json)
+            r = requests.get(f"{dispatcher}/offload_target", params=meta_json, timeout=60)
             if r.status_code == 200:
                 data = r.json()
                 url = data["url"]
@@ -150,12 +149,20 @@ def single_impl(item_directory: pathlib.Path, ia_collection: str, ia_item_title:
             time.sleep(30)
     logging.info(f"Assigned target {url}")
     parsed_url = urllib.parse.urlparse(url)
+    bf_item = None
     if parsed_url.scheme == "minio+http" or parsed_url.scheme == "minio+https":
         secure = (parsed_url.scheme == "minio+https")
         ep = parsed_url.hostname
         if parsed_url.port is not None:
             ep = f"{ep}:{parsed_url.port}"
-        client = minio.Minio(endpoint=ep, access_key=parsed_url.username, secret_key=parsed_url.password, secure=secure)
+        client = None
+        while True:
+            try:
+                client = minio.Minio(endpoint=ep, access_key=parsed_url.username, secret_key=parsed_url.password, secure=secure)
+                break
+            except Exception:
+                logging.exception("Failed to connect to minio")
+                time.sleep(30)
         bucket_name = item_directory.name.replace("_", "-")
         logging.info("Making bucket...")
         while True:
@@ -200,7 +207,7 @@ def single_impl(item_directory: pathlib.Path, ia_collection: str, ia_item_title:
             u = f"https://legacy-api.arpa.li/backfeed/legacy/{backfeed_key}"
             logging.info(f"Attempting to submit bf item {bf_item} to {u}...")
             resp = requests.post(u, params={"skipbloom": "1", "delimiter": BACKFEED_DELIM},
-                                 data=f"{bf_item}{BACKFEED_DELIM}".encode("UTF-8"))
+                                 data=f"{bf_item}{BACKFEED_DELIM}".encode("UTF-8"), timeout=60)
             if resp.status_code == 200:
                 break
             logging.warning(f"Failed to submit to backfeed {resp.status_code}: {resp.text}")