add prom metrics to blink

bskylink/package.json

@@ -21,6 +21,7 @@
     "lru-cache": "^11.1.0",
     "pg": "^8.12.0",
     "pino": "^9.2.0",
+    "prom-client": "^15.1.3",
     "uint8arrays": "^5.1.0"
   },
   "devDependencies": {

bskylink/src/bin.ts

@@ -24,6 +24,9 @@ async function main() {
     await link.destroy()
     httpLogger.info('link service is stopped')
   })
+
+  await link.ctx.metrics.start(cfg.service.metricsPort)
+  httpLogger.info('link metrics is running')
 }
 
 main()

bskylink/src/cache/safelinkClient.ts

@@ -12,6 +12,7 @@ import {type ServiceConfig} from '../config.js'
 import type Database from '../db/index.js'
 import {type SafelinkRule} from '../db/schema.js'
 import {redirectLogger} from '../logger.js'
+import {type Metrics} from '../metrics.js'
 
 const SAFELINK_MIN_FETCH_INTERVAL = 1_000
 const SAFELINK_MAX_FETCH_INTERVAL = 10_000
@@ -22,12 +23,21 @@ export class SafelinkClient {
   private urlCache: LRUCache<string, SafelinkRule | 'ok'>
 
   private db: Database
+  private metrics: Metrics
 
   private ozoneAgent: OzoneAgent
 
   private cursor?: string
 
-  constructor({cfg, db}: {cfg: ServiceConfig; db: Database}) {
+  constructor({
+    cfg,
+    db,
+    metrics,
+  }: {
+    cfg: ServiceConfig
+    db: Database
+    metrics: Metrics
+  }) {
     this.domainCache = new LRUCache<string, SafelinkRule | 'ok'>({
       max: 10000,
     })
@@ -37,6 +47,7 @@ export class SafelinkClient {
     })
 
     this.db = db
+    this.metrics = metrics
 
     this.ozoneAgent = new OzoneAgent(
       cfg.safelinkPdsUrl!,
@@ -46,6 +57,17 @@ export class SafelinkClient {
   }
 
   public async tryFindRule(link: string): Promise<SafelinkRule | 'ok'> {
+    const start = process.hrtime.bigint()
+    const addMetrics = (status: 'ok' | 'error', cached: boolean) => {
+      const end = process.hrtime.bigint()
+      const respTimeMs = Number(end - start) / 1_000_000 // ns to ms :3
+
+      this.metrics.safeLinkLookups.labels(status, cached ? 'yes' : 'no').inc()
+      this.metrics.safeLinkLookupDuration
+        .labels(status, cached ? 'yes' : 'no')
+        .observe(respTimeMs)
+    }
+
     let url: string
     let domain: string
     try {
@@ -56,6 +78,7 @@ export class SafelinkClient {
         {error: e, inputUrl: link},
         'failed to normalize looked up link',
       )
+      addMetrics('error', false)
       // fail open
       return 'ok'
     }
@@ -65,18 +88,23 @@ export class SafelinkClient {
     // _and_ it is not 'ok'.
     const urlRule = this.urlCache.get(url)
     if (urlRule && urlRule !== 'ok') {
+      addMetrics('ok', true)
       return urlRule
     }
 
     // If we find a domain rule of _any_ kind, including 'ok', we can now return that rule.
     const domainRule = this.domainCache.get(domain)
     if (domainRule) {
+      addMetrics('ok', true)
       return domainRule
     }
 
     try {
       const maybeUrlRule = await this.getRule(this.db, url, 'url')
       this.urlCache.set(url, maybeUrlRule)
+
+      addMetrics('ok', false)
+
       return maybeUrlRule
     } catch (e) {
       this.urlCache.set(url, 'ok')
@@ -85,11 +113,15 @@ export class SafelinkClient {
     try {
       const maybeDomainRule = await this.getRule(this.db, domain, 'domain')
       this.domainCache.set(domain, maybeDomainRule)
+
+      addMetrics('ok', false)
+
       return maybeDomainRule
     } catch (e) {
       this.domainCache.set(domain, 'ok')
     }
 
+    addMetrics('ok', false)
     return 'ok'
   }
 

bskylink/src/config.ts

@@ -7,6 +7,7 @@ export type Config = {
 
 export type ServiceConfig = {
   port: number
+  metricsPort: number
   version?: string
   hostnames: string[]
   hostnamesSet: Set<string>
@@ -32,6 +33,7 @@ export type DbPoolConfig = {
 
 export type Environment = {
   port?: number
+  metricsPort?: number
   version?: string
   hostnames: string[]
   appHostname?: string
@@ -50,6 +52,7 @@ export type Environment = {
 export const readEnv = (): Environment => {
   return {
     port: envInt('LINK_PORT'),
+    metricsPort: envInt('LINK_METRICS_PORT'),
     version: envStr('LINK_VERSION'),
     hostnames: envList('LINK_HOSTNAMES'),
     appHostname: envStr('LINK_APP_HOSTNAME'),
@@ -71,6 +74,7 @@ export const readEnv = (): Environment => {
 export const envToCfg = (env: Environment): Config => {
   const serviceCfg: ServiceConfig = {
     port: env.port ?? 3000,
+    metricsPort: env.metricsPort ?? 9090,
     version: env.version,
     hostnames: env.hostnames,
     hostnamesSet: new Set(env.hostnames),

bskylink/src/context.ts

@@ -1,6 +1,7 @@
 import {SafelinkClient} from './cache/safelinkClient.js'
 import {type Config} from './config.js'
 import Database from './db/index.js'
+import {Metrics} from './metrics.js'
 
 export type AppContextOptions = {
   cfg: Config
@@ -10,6 +11,7 @@ export type AppContextOptions = {
 export class AppContext {
   cfg: Config
   db: Database
+  metrics: Metrics = new Metrics()
   safelinkClient: SafelinkClient
   abortController = new AbortController()
 
@@ -19,6 +21,7 @@ export class AppContext {
     this.safelinkClient = new SafelinkClient({
       cfg: this.opts.cfg.service,
       db: this.opts.db,
+      metrics: this.metrics,
     })
   }
 

bskylink/src/index.ts

@@ -2,7 +2,7 @@ import events from 'node:events'
 import type http from 'node:http'
 
 import cors from 'cors'
-import express from 'express'
+import express, {type Response} from 'express'
 import {createHttpTerminator, type HttpTerminator} from 'http-terminator'
 
 import {type Config} from './config.js'
@@ -32,6 +32,28 @@ export class LinkService {
     app = routes(ctx, app)
     app.use(errorHandler)
 
+    // request duration logging
+    app.use((req, res, next) => {
+      const start = process.hrtime.bigint()
+      const originalEnd = res.end.bind(res) as Response['end']
+      res.end = function (
+        this: Response,
+        ...args: Parameters<Response['end']>
+      ): ReturnType<Response['end']> {
+        const end = process.hrtime.bigint()
+        const respTimeMs = Number(end - start) / 1_000_000 // ns to ms :3
+
+        if (req.route) {
+          ctx.metrics.requestDuration
+            .labels(req.route.path, req.method, res.statusCode.toString())
+            .observe(respTimeMs)
+        }
+
+        return originalEnd(...args)
+      } as Response['end']
+      next()
+    })
+
     return new LinkService(app, ctx)
   }
 

bskylink/src/metrics.ts

@@ -0,0 +1,79 @@
+import {once} from 'events'
+import express, {type Application} from 'express'
+import * as prometheus from 'prom-client'
+
+export class Metrics {
+  private app: Application
+
+  private registry: prometheus.Registry
+
+  requestDuration: prometheus.Histogram
+  redirects: prometheus.Counter
+  shortLinkRequests: prometheus.Counter
+  safeLinkLookups: prometheus.Counter
+  safeLinkLookupDuration: prometheus.Histogram
+
+  constructor() {
+    const app = express()
+    this.app = app
+
+    const registry = new prometheus.Registry()
+    this.registry = registry
+
+    // Add add metrics route to express
+    app.get('/metrics', async (_req, res) => {
+      res.set('content-type', registry.contentType)
+      res.end(await registry.metrics())
+    })
+
+    // Collect default service metrics
+    prometheus.collectDefaultMetrics({
+      prefix: 'blink_',
+      register: this.registry,
+      gcDurationBuckets: [0.001, 0.01, 0.1, 1, 2, 5],
+    })
+
+    // Add other metrics
+    this.requestDuration = new prometheus.Histogram({
+      name: 'blink_request_duration_millis',
+      help: 'Request duration in mmillis',
+      labelNames: ['path', 'method', 'code'],
+      buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+      registers: [registry],
+    })
+
+    this.redirects = new prometheus.Counter({
+      name: 'blink_redirects',
+      help: 'Number of link redirects handled',
+      labelNames: ['safelink_rule', 'code'],
+      registers: [registry],
+    })
+
+    this.shortLinkRequests = new prometheus.Counter({
+      name: 'blink_shortlink_requests',
+      help: 'Number of shortlink requests handled',
+      labelNames: ['method', 'code'],
+      registers: [registry],
+    })
+
+    this.safeLinkLookups = new prometheus.Counter({
+      name: 'blink_safelink_lookups',
+      help: 'Number of safelink lookups handled',
+      labelNames: ['status', 'cached'],
+      registers: [registry],
+    })
+
+    this.safeLinkLookupDuration = new prometheus.Histogram({
+      name: 'blink_safelink_lookup_duration_millis',
+      help: 'Request duration in millis',
+      labelNames: ['status', 'cached'],
+      buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+      registers: [registry],
+    })
+  }
+
+  async start(port: number) {
+    const server = this.app.listen(port)
+    await once(server, 'listening')
+  }
+}

bskylink/src/routes/createShortLink.ts

@@ -13,16 +13,24 @@ export default function (ctx: AppContext, app: Express) {
     '/link',
     bodyParser.json(),
     handler(async (req, res) => {
+      const addMetrics = (statusCode: number) => {
+        ctx.metrics.shortLinkRequests
+          .labels('POST', statusCode.toString())
+          .inc()
+      }
+
       let path: string
       if (typeof req.body?.path === 'string') {
         path = req.body.path
       } else {
+        addMetrics(400)
         return res.status(400).json({
           error: 'InvalidPath',
           message: '"path" parameter is missing or not a string',
         })
       }
       if (!path.startsWith('/')) {
+        addMetrics(400)
         return res.status(400).json({
           error: 'InvalidPath',
           message:
@@ -34,6 +42,7 @@ export default function (ctx: AppContext, app: Express) {
         // link pattern: /start/{did}/{rkey}
         if (!parts[1].startsWith('did:')) {
           // enforce strong links
+          addMetrics(400)
           return res.status(400).json({
             error: 'InvalidPath',
             message:
@@ -41,8 +50,11 @@ export default function (ctx: AppContext, app: Express) {
           })
         }
         const id = await ensureLink(ctx, LinkType.StarterPack, parts)
+        addMetrics(200)
         return res.json({url: getUrl(ctx, req, id)})
       }
+
+      addMetrics(400)
       return res.status(400).json({
         error: 'InvalidPath',
         message: '"path" parameter does not have a known format',

bskylink/src/routes/redirect.ts

@@ -19,6 +19,10 @@ export default function (ctx: AppContext, app: Express) {
   return app.get(
     '/redirect',
     handler(async (req, res) => {
+      const addMetrics = (ruleStr: string, statusCode: number) => {
+        ctx.metrics.redirects.labels(ruleStr, statusCode.toString()).inc()
+      }
+
       let link = req.query.u
       assert(
         typeof link === 'string',
@@ -39,6 +43,7 @@ export default function (ctx: AppContext, app: Express) {
       ) {
         res.setHeader('Cache-Control', 'no-store')
         res.setHeader('Location', `https://${ctx.cfg.service.appHostname}`)
+        addMetrics('bad_url', 302)
         return res.status(302).end()
       }
 
@@ -49,9 +54,12 @@ export default function (ctx: AppContext, app: Express) {
 
       let html: string | undefined
 
+      let ruleStr = 'ok'
       if (ctx.cfg.service.safelinkEnabled) {
         const rule = await ctx.safelinkClient.tryFindRule(link)
         if (rule !== 'ok') {
+          ruleStr = rule.action
+
           switch (rule.action) {
             case 'whitelist':
               redirectLogger.info({rule}, 'Whitelist rule matched')
@@ -89,6 +97,8 @@ export default function (ctx: AppContext, app: Express) {
         html = linkRedirectContents(url.href)
       }
 
+      addMetrics(ruleStr, 302)
+
       return res.end(html)
     }),
   )

bskylink/src/routes/shortLink.ts

@@ -1,15 +1,19 @@
 import assert from 'node:assert'
 
 import {DAY, SECOND} from '@atproto/common'
-import {Express} from 'express'
+import {type Express} from 'express'
 
-import {AppContext} from '../context.js'
+import {type AppContext} from '../context.js'
 import {handler} from './util.js'
 
 export default function (ctx: AppContext, app: Express) {
   return app.get(
     '/:linkId',
     handler(async (req, res) => {
+      const addMetrics = (statusCode: number) => {
+        ctx.metrics.shortLinkRequests.labels('GET', statusCode.toString()).inc()
+      }
+
       const linkId = req.params.linkId
       const contentType = req.accepts(['html', 'json'])
       assert(
@@ -35,6 +39,7 @@ export default function (ctx: AppContext, app: Express) {
         }
         // send the user to the app
         res.setHeader('Location', `https://${ctx.cfg.service.appHostname}`)
+        addMetrics(302)
         return res.status(302).end()
       }
       // build url from original url in order to preserve query params
@@ -45,9 +50,11 @@ export default function (ctx: AppContext, app: Express) {
       url.pathname = found.path
       res.setHeader('Cache-Control', `max-age=${(7 * DAY) / SECOND}`)
       if (contentType === 'json') {
+        addMetrics(302)
         return res.json({url: url.href}).end()
       }
       res.setHeader('Location', url.href)
+      addMetrics(302)
       return res.status(301).end()
     }),
   )

bskylink/yarn.lock

@@ -161,6 +161,11 @@
   dependencies:
     make-plural "^7.0.0"
 
+"@opentelemetry/api@^1.4.0":
+  version "1.9.0"
+  resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe"
+  integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==
+
 "@tsconfig/node10@^1.0.7":
   version "1.0.11"
   resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.11.tgz#6ee46400685f130e278128c7b38b7e031ff5b2f2"
@@ -338,6 +343,11 @@ base64-js@^1.3.1:
   resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
   integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
 
+bintrees@1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8"
+  integrity sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==
+
 body-parser@1.20.2, body-parser@^1.20.2:
   version "1.20.2"
   resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd"
@@ -1060,6 +1070,14 @@ process@^0.11.10:
   resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182"
   integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==
 
+prom-client@^15.1.3:
+  version "15.1.3"
+  resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-15.1.3.tgz#69fa8de93a88bc9783173db5f758dc1c69fa8fc2"
+  integrity sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==
+  dependencies:
+    "@opentelemetry/api" "^1.4.0"
+    tdigest "^0.1.1"
+
 proxy-addr@~2.0.7:
   version "2.0.7"
   resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025"
@@ -1232,6 +1250,13 @@ string_decoder@^1.3.0:
   dependencies:
     safe-buffer "~5.2.0"
 
+tdigest@^0.1.1:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.2.tgz#96c64bac4ff10746b910b0e23b515794e12faced"
+  integrity sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==
+  dependencies:
+    bintrees "1.0.2"
+
 thread-stream@^2.6.0:
   version "2.7.0"
   resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-2.7.0.tgz#d8a8e1b3fd538a6cca8ce69dbe5d3d097b601e11"