Add graceful shutdown to Blink (#11523)

bskylink/src/index.ts

@@ -11,10 +11,12 @@ import {AppContext} from './context.js'
 import i18n from './i18n.js'
 import {createPrometheusRegistry} from './prometheus.js'
 import {default as routes, errorHandler} from './routes/index.js'
+import {REQUEST_DRAIN_TIMEOUT_MS} from './shutdown.js'
 
 export * from './config.js'
 export * from './db/index.js'
 export * from './logger.js'
+export * from './shutdown.js'
 
 export class LinkService {
   public server?: http.Server
@@ -22,6 +24,7 @@ export class LinkService {
   private terminator?: HttpTerminator
   private metricsTerminator?: HttpTerminator
   private metricsRegistry: Registry
+  private destroyPromise?: Promise<void>
 
   constructor(
     public app: express.Application,
@@ -46,7 +49,10 @@ export class LinkService {
     this.ctx.metrics.start()
     this.server = this.app.listen(this.ctx.cfg.service.port)
     this.server.keepAliveTimeout = 90000
-    this.terminator = createHttpTerminator({server: this.server})
+    this.terminator = createHttpTerminator({
+      server: this.server,
+      gracefulTerminationTimeout: REQUEST_DRAIN_TIMEOUT_MS,
+    })
     await events.once(this.server, 'listening')
 
     const metricsApp = express()
@@ -62,13 +68,26 @@ export class LinkService {
     await events.once(this.metricsServer, 'listening')
   }
 
-  async destroy() {
+  destroy(): Promise<void> {
+    this.destroyPromise ??= this.destroyInternal()
+
+    return this.destroyPromise
+  }
+
+  private async destroyInternal() {
     this.ctx.abortController.abort()
-    await Promise.all([
-      this.terminator?.terminate(),
-      this.metricsTerminator?.terminate(),
-    ])
-    await this.ctx.db.close()
-    this.ctx.metrics.stop()
+    try {
+      await Promise.all([
+        this.terminator?.terminate(),
+        this.metricsTerminator?.terminate(),
+        this.ctx.safelinkClient.stop(REQUEST_DRAIN_TIMEOUT_MS),
+      ])
+    } finally {
+      try {
+        await this.ctx.db.close()
+      } finally {
+        this.ctx.metrics.stop()
+      }
+    }
   }
 }