Merge branch 'temp-pr-branch' into migrate-blink

bskylink/package.json

@@ -10,6 +10,7 @@
   "dependencies": {
     "@atproto/api": "^0.16.7",
     "@atproto/common": "^0.4.11",
+    "@haileyok/ts-util": "^1.3.4",
     "@types/escape-html": "^1.0.4",
     "body-parser": "^1.20.2",
     "cors": "^2.8.5",

bskylink/src/cache/safelinkClient.ts

@@ -8,7 +8,7 @@ import {ExpiredTokenError} from '@atproto/api/dist/client/types/com/atproto/serv
 import {MINUTE} from '@atproto/common'
 import {LRUCache} from 'lru-cache'
 
-import {type ServiceConfig} from '../config.js'
+import {type AppContext} from '../context.js'
 import type Database from '../db/index.js'
 import {type SafelinkRule} from '../db/schema.js'
 import {redirectLogger} from '../logger.js'
@@ -21,13 +21,12 @@ export class SafelinkClient {
   private domainCache: LRUCache<string, SafelinkRule | 'ok'>
   private urlCache: LRUCache<string, SafelinkRule | 'ok'>
 
-  private db: Database
+  private ctx: AppContext
 
   private ozoneAgent: OzoneAgent
-
   private cursor?: string
 
-  constructor({cfg, db}: {cfg: ServiceConfig; db: Database}) {
+  constructor(ctx: AppContext) {
     this.domainCache = new LRUCache<string, SafelinkRule | 'ok'>({
       max: 10000,
     })
@@ -36,16 +35,31 @@ export class SafelinkClient {
       max: 25000,
     })
 
-    this.db = db
+    this.ctx = ctx
 
     this.ozoneAgent = new OzoneAgent(
-      cfg.safelinkPdsUrl!,
+      this.ctx.cfg.service.safelinkPdsUrl!,
-      cfg.safelinkAgentIdentifier!,
+      this.ctx.cfg.service.safelinkAgentIdentifier!,
-      cfg.safelinkAgentPass!,
+      this.ctx.cfg.service.safelinkAgentPass!,
     )
   }
 
   public async tryFindRule(link: string): Promise<SafelinkRule | 'ok'> {
+    const start = process.hrtime.bigint()
+    const addMetrics = (status: 'ok' | 'error', cached: boolean) => {
+      const end = process.hrtime.bigint()
+      const respTimeMs = Number(end - start) / 1_000_000 // ns to ms :3
+
+      this.ctx.metrics
+        .getCounter('safeLinkLookups')
+        .labels(status, cached ? 'yes' : 'no')
+        .inc()
+      this.ctx.metrics
+        .getHistogram('safeLinkLookupDuration')
+        .labels(status, cached ? 'yes' : 'no')
+        .observe(respTimeMs)
+    }
+
     let url: string
     let domain: string
     try {
@@ -56,6 +70,7 @@ export class SafelinkClient {
         {error: e, inputUrl: link},
         'failed to normalize looked up link',
       )
+      addMetrics('error', false)
       // fail open
       return 'ok'
     }
@@ -65,31 +80,38 @@ export class SafelinkClient {
     // _and_ it is not 'ok'.
     const urlRule = this.urlCache.get(url)
     if (urlRule && urlRule !== 'ok') {
+      addMetrics('ok', true)
       return urlRule
     }
 
     // If we find a domain rule of _any_ kind, including 'ok', we can now return that rule.
     const domainRule = this.domainCache.get(domain)
     if (domainRule) {
+      addMetrics('ok', true)
       return domainRule
     }
 
     try {
-      const maybeUrlRule = await this.getRule(this.db, url, 'url')
+      const maybeUrlRule = await this.getRule(this.ctx.db, url, 'url')
       this.urlCache.set(url, maybeUrlRule)
+
+      addMetrics('ok', false)
       return maybeUrlRule
     } catch (e) {
       this.urlCache.set(url, 'ok')
     }
 
     try {
-      const maybeDomainRule = await this.getRule(this.db, domain, 'domain')
+      const maybeDomainRule = await this.getRule(this.ctx.db, domain, 'domain')
       this.domainCache.set(domain, maybeDomainRule)
+
+      addMetrics('ok', false)
       return maybeDomainRule
     } catch (e) {
       this.domainCache.set(domain, 'ok')
     }
 
+    addMetrics('ok', false)
     return 'ok'
   }
 
@@ -219,7 +241,7 @@ export class SafelinkClient {
       redirectLogger.info('received no new safelink events from ozone')
       setTimeout(() => this.runFetchEvents(), SAFELINK_MAX_FETCH_INTERVAL)
     } else {
-      await this.db.transaction(async db => {
+      await this.ctx.db.transaction(async db => {
         for (const rule of res.data.events) {
           switch (rule.eventType) {
             case 'removeRule':
@@ -247,7 +269,7 @@ export class SafelinkClient {
 
   private async getCursor() {
     if (this.cursor === '') {
-      const res = await this.db.db
+      const res = await this.ctx.db.db
         .selectFrom('safelink_cursor')
         .selectAll()
         .where('id', '=', 1)
@@ -263,7 +285,7 @@ export class SafelinkClient {
   private async setCursor(cursor: string) {
     const updatedAt = new Date()
     try {
-      await this.db.db
+      await this.ctx.db.db
         .insertInto('safelink_cursor')
         .values({
           id: 1,

bskylink/src/config.ts

@@ -7,6 +7,7 @@ export type Config = {
 
 export type ServiceConfig = {
   port: number
+  metricsPort: number
   version?: string
   hostnames: string[]
   hostnamesSet: Set<string>
@@ -32,6 +33,7 @@ export type DbPoolConfig = {
 
 export type Environment = {
   port?: number
+  metricsPort?: number
   version?: string
   hostnames: string[]
   appHostname?: string
@@ -50,6 +52,7 @@ export type Environment = {
 export const readEnv = (): Environment => {
   return {
     port: envInt('LINK_PORT'),
+    metricsPort: envInt('LINK_METRICS_PORT'),
     version: envStr('LINK_VERSION'),
     hostnames: envList('LINK_HOSTNAMES'),
     appHostname: envStr('LINK_APP_HOSTNAME'),
@@ -71,6 +74,7 @@ export const readEnv = (): Environment => {
 export const envToCfg = (env: Environment): Config => {
   const serviceCfg: ServiceConfig = {
     port: env.port ?? 3000,
+    metricsPort: env.metricsPort ?? 9090,
     version: env.version,
     hostnames: env.hostnames,
     hostnamesSet: new Set(env.hostnames),

bskylink/src/context.ts

@@ -1,3 +1,5 @@
+import {type MetricConfig, Metrics} from '@haileyok/ts-util'
+
 import {SafelinkClient} from './cache/safelinkClient.js'
 import {type Config} from './config.js'
 import Database from './db/index.js'
@@ -7,19 +9,66 @@ export type AppContextOptions = {
   db: Database
 }
 
+type BlinkMetricNames =
+  | 'requestDuration'
+  | 'redirects'
+  | 'shortLinkRequests'
+  | 'safeLinkLookups'
+  | 'safeLinkLookupDuration'
+
+type BlinkMetricConfig = Record<BlinkMetricNames, MetricConfig>
+
 export class AppContext {
   cfg: Config
   db: Database
+  metrics: Metrics<BlinkMetricConfig>
   safelinkClient: SafelinkClient
   abortController = new AbortController()
 
   constructor(private opts: AppContextOptions) {
+    const metricsConfig: BlinkMetricConfig = {
+      requestDuration: {
+        type: 'histogram',
+        name: 'request_duration_millis',
+        help: 'Request duration in millis',
+        labelNames: ['path', 'method', 'code'],
+        buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+      },
+      redirects: {
+        type: 'counter',
+        name: 'redirects',
+        help: 'Number of link redirects handled',
+        labelNames: ['safelink_rule', 'code'],
+      },
+      shortLinkRequests: {
+        type: 'counter',
+        name: 'shortlink_requests',
+        help: 'Number of shortlink requests handled',
+        labelNames: ['method', 'code'],
+      },
+      safeLinkLookups: {
+        type: 'counter',
+        name: 'safelink_lookups',
+        help: 'Number of safelink lookups handled',
+        labelNames: ['status', 'cached'],
+      },
+      safeLinkLookupDuration: {
+        type: 'histogram',
+        name: 'safelink_lookup_duration_millis',
+        help: 'Request duration in millis',
+        labelNames: ['status', 'cached'],
+        buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+      },
+    }
+
+    this.metrics = new Metrics(metricsConfig, {
+      prefix: 'blink_',
+      collectDefaultMetrics: true,
+    })
+
     this.cfg = this.opts.cfg
     this.db = this.opts.db
-    this.safelinkClient = new SafelinkClient({
+    this.safelinkClient = new SafelinkClient(this)
-      cfg: this.opts.cfg.service,
-      db: this.opts.db,
-    })
   }
 
   static async fromConfig(cfg: Config, overrides?: Partial<AppContextOptions>) {

bskylink/src/index.ts

@@ -32,6 +32,8 @@ export class LinkService {
     app = routes(ctx, app)
     app.use(errorHandler)
 
+    ctx.metrics.registerExpressMetrics(app)
+
     return new LinkService(app, ctx)
   }
 

bskylink/src/routes/createShortLink.ts

@@ -13,16 +13,25 @@ export default function (ctx: AppContext, app: Express) {
     '/link',
     bodyParser.json(),
     handler(async (req, res) => {
+      const addMetrics = (statusCode: number) => {
+        ctx.metrics
+          .getCounter('shortLinkRequests')
+          .labels('POST', statusCode.toString())
+          .inc()
+      }
+
       let path: string
       if (typeof req.body?.path === 'string') {
         path = req.body.path
       } else {
+        addMetrics(400)
         return res.status(400).json({
           error: 'InvalidPath',
           message: '"path" parameter is missing or not a string',
         })
       }
       if (!path.startsWith('/')) {
+        addMetrics(400)
         return res.status(400).json({
           error: 'InvalidPath',
           message:
@@ -34,6 +43,7 @@ export default function (ctx: AppContext, app: Express) {
         // link pattern: /start/{did}/{rkey}
         if (!parts[1].startsWith('did:')) {
           // enforce strong links
+          addMetrics(400)
           return res.status(400).json({
             error: 'InvalidPath',
             message:
@@ -41,8 +51,11 @@ export default function (ctx: AppContext, app: Express) {
           })
         }
         const id = await ensureLink(ctx, LinkType.StarterPack, parts)
+        addMetrics(200)
         return res.json({url: getUrl(ctx, req, id)})
       }
+
+      addMetrics(400)
       return res.status(400).json({
         error: 'InvalidPath',
         message: '"path" parameter does not have a known format',

bskylink/src/routes/redirect.ts

@@ -19,6 +19,13 @@ export default function (ctx: AppContext, app: Express) {
   return app.get(
     '/redirect',
     handler(async (req, res) => {
+      const addMetrics = (ruleStr: string, statusCode: number) => {
+        ctx.metrics
+          .getCounter('redirects')
+          .labels(ruleStr, statusCode.toString())
+          .inc()
+      }
+
       let link = req.query.u
       assert(
         typeof link === 'string',
@@ -39,6 +46,7 @@ export default function (ctx: AppContext, app: Express) {
       ) {
         res.setHeader('Cache-Control', 'no-store')
         res.setHeader('Location', `https://${ctx.cfg.service.appHostname}`)
+        addMetrics('bad_url', 302)
         return res.status(302).end()
       }
 
@@ -49,9 +57,12 @@ export default function (ctx: AppContext, app: Express) {
 
       let html: string | undefined
 
+      let ruleStr = 'ok'
       if (ctx.cfg.service.safelinkEnabled) {
         const rule = await ctx.safelinkClient.tryFindRule(link)
         if (rule !== 'ok') {
+          ruleStr = rule.action
+
           switch (rule.action) {
             case 'whitelist':
               redirectLogger.info({rule}, 'Whitelist rule matched')
@@ -89,6 +100,7 @@ export default function (ctx: AppContext, app: Express) {
         html = linkRedirectContents(url.href)
       }
 
+      addMetrics(ruleStr, 302)
       return res.end(html)
     }),
   )

bskylink/src/routes/shortLink.ts

@@ -1,15 +1,22 @@
 import assert from 'node:assert'
 
 import {DAY, SECOND} from '@atproto/common'
-import {Express} from 'express'
+import {type Express} from 'express'
 
-import {AppContext} from '../context.js'
+import {type AppContext} from '../context.js'
 import {handler} from './util.js'
 
 export default function (ctx: AppContext, app: Express) {
   return app.get(
     '/:linkId',
     handler(async (req, res) => {
+      const addMetrics = (statusCode: number) => {
+        ctx.metrics
+          .getCounter('shortLinkRequests')
+          .labels('GET', statusCode.toString())
+          .inc()
+      }
+
       const linkId = req.params.linkId
       const contentType = req.accepts(['html', 'json'])
       assert(
@@ -35,6 +42,7 @@ export default function (ctx: AppContext, app: Express) {
         }
         // send the user to the app
         res.setHeader('Location', `https://${ctx.cfg.service.appHostname}`)
+        addMetrics(302)
         return res.status(302).end()
       }
       // build url from original url in order to preserve query params
@@ -45,9 +53,11 @@ export default function (ctx: AppContext, app: Express) {
       url.pathname = found.path
       res.setHeader('Cache-Control', `max-age=${(7 * DAY) / SECOND}`)
       if (contentType === 'json') {
+        addMetrics(302)
         return res.json({url: url.href}).end()
       }
       res.setHeader('Location', url.href)
+      addMetrics(302)
       return res.status(301).end()
     }),
   )