|
|
@@ -26,7 +26,9 @@ def fetch(url, tries, connection): |
|
|
|
status = r.status |
|
|
|
print(f'{status} {url}', file = sys.stderr) |
|
|
|
if status == 302 and r.getheader('Location') in ('https://web.archive.org/429.html', '/429.html'): |
|
|
|
# The CDX API is stupid and doesn't return 429s directly... |
|
|
|
# The CDX API is (was?) stupid and doesn't return 429s directly... |
|
|
|
status = 429 |
|
|
|
if status == 429: |
|
|
|
print('Exceeded rate limit, waiting...', file = sys.stderr) |
|
|
|
time.sleep(30) |
|
|
|
raise RuntimeError(f'Rate-limited on {url}') |
|
|
|