Add Blink Prometheus listener (#11509)

.oxlintrc.json

@@ -385,6 +385,7 @@
     },
     {
       "files": [
+        "bskylink/**/*.{js,jsx,ts,tsx}",
         "bskyogcard/**/*.{js,jsx,ts,tsx}",
         "dev-env/**/*.{js,jsx,ts,tsx}"
       ],

Dockerfile.bskylink

@@ -26,8 +26,9 @@ COPY --from=build /app /app
 RUN mkdir /app/data && chown node /app/data
 
 VOLUME /app/data
-EXPOSE 3000
+EXPOSE 3000 9090
 ENV LINK_PORT=3000
+ENV LINK_METRICS_PORT=9090
 ENV NODE_ENV=production
 
 # https://github.com/nodejs/docker-node/blob/master/docs/BestPractices.md#non-root-user

bskylink/package.json

@@ -23,6 +23,7 @@
     "lru-cache": "^11.1.0",
     "pg": "^8.12.0",
     "pino": "^9.2.0",
+    "prom-client": "^15.1.3",
     "uint8arrays": "^5.1.0"
   },
   "devDependencies": {

bskylink/src/bin.ts

@@ -10,6 +10,7 @@ async function main() {
     httpLogger.info(
       {
         port: cfg.service.port,
+        metricsPort: cfg.service.metricsPort,
         safelinkEnabled: cfg.service.safelinkEnabled,
         hasDbUrl: !!cfg.db.url,
         hasDbMigrationUrl: !!cfg.db.migrationUrl,
@@ -33,16 +34,18 @@ async function main() {
 
     if (link.ctx.cfg.service.safelinkEnabled) {
       httpLogger.info('Starting Safelink client')
-      link.ctx.safelinkClient.runFetchEvents()
+      void link.ctx.safelinkClient.runFetchEvents()
     }
 
     await link.start()
     httpLogger.info('Link service is running')
 
-    process.on('SIGTERM', async () => {
+    process.on('SIGTERM', () => {
+      void (async () => {
         httpLogger.info('Link service is stopping')
         await link.destroy()
         httpLogger.info('Link service is stopped')
+      })()
     })
   } catch (error) {
     httpLogger.error(

bskylink/src/config.ts

@@ -7,6 +7,7 @@ export type Config = {
 
 export type ServiceConfig = {
   port: number
+  metricsPort: number
   version?: string
   hostnames: string[]
   hostnamesSet: Set<string>
@@ -33,6 +34,7 @@ export type DbPoolConfig = {
 
 export type Environment = {
   port?: number
+  metricsPort?: number
   version?: string
   hostnames: string[]
   appHostname?: string
@@ -52,6 +54,7 @@ export type Environment = {
 export const readEnv = (): Environment => {
   return {
     port: envInt('LINK_PORT'),
+    metricsPort: envInt('LINK_METRICS_PORT'),
     version: envStr('LINK_VERSION'),
     hostnames: envList('LINK_HOSTNAMES'),
     appHostname: envStr('LINK_APP_HOSTNAME'),
@@ -74,6 +77,7 @@ export const readEnv = (): Environment => {
 export const envToCfg = (env: Environment): Config => {
   const serviceCfg: ServiceConfig = {
     port: env.port ?? 3000,
+    metricsPort: env.metricsPort ?? 9090,
     version: env.version,
     hostnames: env.hostnames,
     hostnamesSet: new Set(env.hostnames),

bskylink/src/index.test.ts

@@ -0,0 +1,49 @@
+import assert from 'node:assert'
+import {type AddressInfo} from 'node:net'
+import {test} from 'node:test'
+
+import {envToCfg} from './config.js'
+import {LinkService} from './index.js'
+
+const testConfig = () =>
+  envToCfg({
+    dbPostgresUrl: 'postgres://localhost:1/blink',
+    hostnames: ['go.bsky.app'],
+    metricsPort: 0,
+    port: 0,
+    safelinkAgentIdentifier: 'test',
+    safelinkAgentPass: 'test',
+    safelinkPdsUrl: 'https://example.com',
+  })
+
+void test('serves and terminates the Prometheus listener', async () => {
+  const service = await LinkService.create(testConfig())
+
+  try {
+    await service.start()
+    const {port} = service.metricsServer?.address() as AddressInfo
+    const res = await fetch(`http://127.0.0.1:${port}/metrics`)
+
+    assert.strictEqual(res.status, 200)
+    assert.match(res.headers.get('content-type') ?? '', /text\/plain/)
+    const metrics = await res.text()
+    assert.match(metrics, /process_cpu_user_seconds_total/)
+    assert.match(metrics, /nodejs_eventloop_lag_max_seconds/)
+    assert.match(metrics, /bskylink_db_pool_connections\{state="idle"\} 0/)
+    assert.match(metrics, /bskylink_db_pool_connections\{state="in_use"\} 0/)
+    assert.match(metrics, /bskylink_db_pool_max_connections 10/)
+    assert.match(metrics, /bskylink_db_pool_waiting_requests 0/)
+    assert.doesNotMatch(metrics, /http_request_duration_seconds/)
+  } finally {
+    await service.destroy()
+  }
+
+  assert.strictEqual(service.metricsServer?.listening, false)
+})
+
+void test('isolates the Prometheus registry per service', async () => {
+  const first = await LinkService.create(testConfig())
+  const second = await LinkService.create(testConfig())
+
+  await Promise.all([first.destroy(), second.destroy()])
+})

bskylink/src/index.ts

@@ -4,10 +4,12 @@ import type http from 'node:http'
 import cors from 'cors'
 import express from 'express'
 import {createHttpTerminator, type HttpTerminator} from 'http-terminator'
+import {type Registry} from 'prom-client'
 
 import {type Config} from './config.js'
 import {AppContext} from './context.js'
 import i18n from './i18n.js'
+import {createPrometheusRegistry} from './prometheus.js'
 import {default as routes, errorHandler} from './routes/index.js'
 
 export * from './config.js'
@@ -16,12 +18,17 @@ export * from './logger.js'
 
 export class LinkService {
   public server?: http.Server
+  public metricsServer?: http.Server
   private terminator?: HttpTerminator
+  private metricsTerminator?: HttpTerminator
+  private metricsRegistry: Registry
 
   constructor(
     public app: express.Application,
     public ctx: AppContext,
-  ) {}
+  ) {
+    this.metricsRegistry = createPrometheusRegistry(ctx)
+  }
 
   static async create(cfg: Config): Promise<LinkService> {
     let app = express()
@@ -41,11 +48,26 @@ export class LinkService {
     this.server.keepAliveTimeout = 90000
     this.terminator = createHttpTerminator({server: this.server})
     await events.once(this.server, 'listening')
+
+    const metricsApp = express()
+    metricsApp.get('/metrics', (_req, res, next) => {
+      res.set('Content-Type', this.metricsRegistry.contentType)
+      this.metricsRegistry.metrics().then(metrics => res.end(metrics), next)
+    })
+    this.metricsServer = metricsApp.listen(this.ctx.cfg.service.metricsPort)
+    this.metricsTerminator = createHttpTerminator({
+      server: this.metricsServer,
+      gracefulTerminationTimeout: 2000,
+    })
+    await events.once(this.metricsServer, 'listening')
   }
 
   async destroy() {
     this.ctx.abortController.abort()
-    await this.terminator?.terminate()
+    await Promise.all([
+      this.terminator?.terminate(),
+      this.metricsTerminator?.terminate(),
+    ])
     await this.ctx.db.close()
     this.ctx.metrics.stop()
   }

bskylink/src/prometheus.ts

@@ -0,0 +1,54 @@
+import {collectDefaultMetrics, Gauge, Registry} from 'prom-client'
+
+import {type AppContext} from './context.js'
+
+let runtimeRegistry: Registry | undefined
+
+const getRuntimeRegistry = (): Registry => {
+  if (!runtimeRegistry) {
+    runtimeRegistry = new Registry()
+
+    // Beyla already exports HTTP RED metrics and traces for Blink. These
+    // process metrics cover the runtime-only failure modes it cannot see,
+    // particularly event-loop stalls, GC pauses, and V8 heap pressure.
+    collectDefaultMetrics({register: runtimeRegistry})
+  }
+
+  return runtimeRegistry
+}
+
+export const createPrometheusRegistry = (ctx: AppContext): Registry => {
+  const poolRegistry = new Registry()
+
+  new Gauge<'state'>({
+    name: 'bskylink_db_pool_connections',
+    help: 'PostgreSQL client connections by usage state.',
+    labelNames: ['state'],
+    registers: [poolRegistry],
+    collect() {
+      const {idleCount, totalCount} = ctx.db.cfg.pool
+      this.set({state: 'idle'}, idleCount)
+      this.set({state: 'in_use'}, totalCount - idleCount)
+    },
+  })
+
+  new Gauge({
+    name: 'bskylink_db_pool_max_connections',
+    help: 'Configured maximum PostgreSQL client connections.',
+    registers: [poolRegistry],
+    collect() {
+      this.set(ctx.cfg.db.pool.size)
+    },
+  })
+
+  new Gauge({
+    name: 'bskylink_db_pool_waiting_requests',
+    help: 'Requests waiting for a PostgreSQL client connection.',
+    registers: [poolRegistry],
+    collect() {
+      this.set(ctx.db.cfg.pool.waitingCount)
+    },
+  })
+
+  return Registry.merge([getRuntimeRegistry(), poolRegistry])
+}

bskylink/yarn.lock

@@ -161,6 +161,11 @@
   dependencies:
     make-plural "^7.0.0"
 
+"@opentelemetry/api@^1.4.0":
+  version "1.9.1"
+  resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.1.tgz#c1b0346de336ba55af2d5a7970882037baedec05"
+  integrity sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==
+
 "@tsconfig/node10@^1.0.7":
   version "1.0.11"
   resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.11.tgz#6ee46400685f130e278128c7b38b7e031ff5b2f2"
@@ -338,6 +343,11 @@ base64-js@^1.3.1:
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
   integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
 
+bintrees@1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8"
+  integrity sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==
+
 body-parser@1.20.2, body-parser@^1.20.2:
   version "1.20.2"
   resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd"
@@ -1060,6 +1070,14 @@ process@^0.11.10:
   resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182"
   integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==
 
+prom-client@^15.1.3:
+  version "15.1.3"
+  resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-15.1.3.tgz#69fa8de93a88bc9783173db5f758dc1c69fa8fc2"
+  integrity sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==
+  dependencies:
+    "@opentelemetry/api" "^1.4.0"
+    tdigest "^0.1.1"
+
 proxy-addr@~2.0.7:
   version "2.0.7"
   resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025"
@@ -1232,6 +1250,13 @@ string_decoder@^1.3.0:
   dependencies:
     safe-buffer "~5.2.0"
 
+tdigest@^0.1.1:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.2.tgz#96c64bac4ff10746b910b0e23b515794e12faced"
+  integrity sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==
+  dependencies:
+    bintrees "1.0.2"
+
 thread-stream@^2.6.0:
   version "2.7.0"
   resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-2.7.0.tgz#d8a8e1b3fd538a6cca8ce69dbe5d3d097b601e11"