From 22bd262c99504389fbc7ad55005000c004700050 Mon Sep 17 00:00:00 2001
From: Eric Bailey <git@esb.lol>
Date: Wed, 22 May 2024 13:49:47 -0500
Subject: [PATCH] Add request retry backoff handling

---
 src/lib/async/retry.ts | 54 +++++++++++++++++++++++++++++++++++++++---
 1 file changed, 51 insertions(+), 3 deletions(-)

diff --git a/src/lib/async/retry.ts b/src/lib/async/retry.ts
index 2e01be9043..f7c7592d21 100644
--- a/src/lib/async/retry.ts
+++ b/src/lib/async/retry.ts
@@ -6,6 +6,8 @@ export const RETRYABLE_NETWORK_STATUS_CODES = [
   1, 408, 425, 429, 500, 502, 503, 504, 522, 524,
 ]
 
+export const DEFAULT_BACKOFF_DELAY = 500
+
 function isRetryableError(e: unknown) {
   if (e instanceof XRPCError) {
     if (RETRYABLE_NETWORK_STATUS_CODES.includes(e.status)) {
@@ -16,37 +18,83 @@ function isRetryableError(e: unknown) {
   return isNetworkError(e)
 }
 
+function computeBackoff({
+  exception: e,
+  retryCount,
+}: {
+  exception: any
+  retryCount: number
+}) {
+  if (e instanceof XRPCError) {
+    if (e.status === 429) {
+      const retryAfter = parseInt(
+        e.headers?.['Retry-After'] || e.headers?.['retry-after'] || '',
+      )
+      if (typeof retryAfter === 'number' && !Number.isNaN(retryAfter)) {
+        return retryAfter * 1000
+      }
+    }
+  }
+
+  return DEFAULT_BACKOFF_DELAY * Math.pow(2, retryCount)
+}
+
 export async function retry<P>(
   fn: () => Promise<P>,
   {
-    retries,
+    retries: retriesRemaining,
     checkIsRetryableError = isRetryableError,
+    backoff,
   }: {
     retries: number
     checkIsRetryableError?: (err: any) => boolean
+    backoff?: boolean
   },
 ): Promise<P> {
   let lastErr
-  while (retries > 0) {
+  let retryCount = 0
+  let backoffDelay = DEFAULT_BACKOFF_DELAY
+
+  while (retriesRemaining > 0) {
     try {
+      if (retryCount > 0 && backoff) {
+        await sleep(backoffDelay)
+      }
+
       return await fn()
     } catch (e: any) {
       lastErr = e
+
       if (checkIsRetryableError(e)) {
-        retries--
+        backoffDelay = computeBackoff({
+          exception: e,
+          retryCount: retryCount,
+        })
+        retryCount++
+        retriesRemaining--
         continue
       }
+
       throw e
     }
   }
+
   throw lastErr
 }
 
 export async function networkRetry<P>(
   retries: number,
   fn: () => Promise<P>,
+  options: {
+    backoff?: boolean
+  },
 ): Promise<P> {
   return retry(fn, {
     retries,
+    backoff: options.backoff,
   })
 }
+
+export function sleep(time: number) {
+  return new Promise(y => setTimeout(y, time))
+}