Retry on internal errors

s3-bucket-list

@@ -2,6 +2,7 @@
 import html
 import http.client
 import os
+import shlex
 import sys
 import urllib.parse
 
@@ -51,6 +52,7 @@ conn = http.client.HTTPSConnection(hostname)
 params = {}
 if startMarker is not None:
 	params['marker'] = startMarker
+attempt = 1
 while True:
 	queryString = urllib.parse.urlencode(params)
 	url = f'{baseUrl}{"?" + queryString if queryString else ""}'
@@ -59,6 +61,14 @@ while True:
 	conn.request('GET', url[url.index('/', 8):])
 	resp = conn.getresponse()
 	body = resp.read()
+	if b'<Error><Code>InternalError</Code><Message>We encountered an internal error. Please try again.</Message>' in body:
+		print(f'Got internal error on {url} on attempt {attempt}; {"retrying" if attempt < 10 else "aborting"}', file = sys.stderr)
+		if attempt >= 10:
+			if 'marker' in params:
+				print(f'To retry, use --marker {shlex.quote(params["marker"])}', file = sys.stderr)
+			break
+		attempt += 1
+		continue
 	if not body.startswith(b'<?xml version="1.0" encoding="UTF-8"?>\n<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">'):
 		raise RuntimeError(f'Invalid body: {body[:200]}...')
 
@@ -97,3 +107,4 @@ while True:
 	if 'marker' in params and params['marker'] == lastKey:
 		raise RuntimeError('Marker loop (same last key as previous marker)')
 	params['marker'] = lastKey
+	attempt = 1