From c2f6f5054c1d0bee1824aad067eb5cb737f70fa1 Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Sun, 17 Jul 2022 20:33:29 +0000 Subject: [PATCH] Handle actual 429 --- ia-cdx-search | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ia-cdx-search b/ia-cdx-search index 114c2b9..de9cbcb 100755 --- a/ia-cdx-search +++ b/ia-cdx-search @@ -26,7 +26,9 @@ def fetch(url, tries, connection): status = r.status print(f'{status} {url}', file = sys.stderr) if status == 302 and r.getheader('Location') in ('https://web.archive.org/429.html', '/429.html'): - # The CDX API is stupid and doesn't return 429s directly... + # The CDX API is (was?) stupid and doesn't return 429s directly... + status = 429 + if status == 429: print('Exceeded rate limit, waiting...', file = sys.stderr) time.sleep(30) raise RuntimeError(f'Rate-limited on {url}')