diff --git a/bskylink/package.json b/bskylink/package.json index c57cb3cd89..0d9b8444d3 100644 --- a/bskylink/package.json +++ b/bskylink/package.json @@ -21,6 +21,7 @@ "lru-cache": "^11.1.0", "pg": "^8.12.0", "pino": "^9.2.0", + "prom-client": "^15.1.3", "uint8arrays": "^5.1.0" }, "devDependencies": { diff --git a/bskylink/src/bin.ts b/bskylink/src/bin.ts index 3e0746a989..f5deaa40b7 100644 --- a/bskylink/src/bin.ts +++ b/bskylink/src/bin.ts @@ -24,6 +24,9 @@ async function main() { await link.destroy() httpLogger.info('link service is stopped') }) + + await link.ctx.metrics.start(cfg.service.metricsPort) + httpLogger.info('link metrics is running') } main() diff --git a/bskylink/src/cache/safelinkClient.ts b/bskylink/src/cache/safelinkClient.ts index 547a9c8438..720642db32 100644 --- a/bskylink/src/cache/safelinkClient.ts +++ b/bskylink/src/cache/safelinkClient.ts @@ -12,6 +12,7 @@ import {type ServiceConfig} from '../config.js' import type Database from '../db/index.js' import {type SafelinkRule} from '../db/schema.js' import {redirectLogger} from '../logger.js' +import {type Metrics} from '../metrics.js' const SAFELINK_MIN_FETCH_INTERVAL = 1_000 const SAFELINK_MAX_FETCH_INTERVAL = 10_000 @@ -22,12 +23,21 @@ export class SafelinkClient { private urlCache: LRUCache private db: Database + private metrics: Metrics private ozoneAgent: OzoneAgent private cursor?: string - constructor({cfg, db}: {cfg: ServiceConfig; db: Database}) { + constructor({ + cfg, + db, + metrics, + }: { + cfg: ServiceConfig + db: Database + metrics: Metrics + }) { this.domainCache = new LRUCache({ max: 10000, }) @@ -37,6 +47,7 @@ export class SafelinkClient { }) this.db = db + this.metrics = metrics this.ozoneAgent = new OzoneAgent( cfg.safelinkPdsUrl!, @@ -46,6 +57,17 @@ export class SafelinkClient { } public async tryFindRule(link: string): Promise { + const start = process.hrtime.bigint() + const addMetrics = (status: 'ok' | 'error', cached: boolean) => { + const end = process.hrtime.bigint() + const respTimeMs = Number(end - start) / 1_000_000 // ns to ms :3 + + this.metrics.safeLinkLookups.labels(status, cached ? 'yes' : 'no').inc() + this.metrics.safeLinkLookupDuration + .labels(status, cached ? 'yes' : 'no') + .observe(respTimeMs) + } + let url: string let domain: string try { @@ -56,6 +78,7 @@ export class SafelinkClient { {error: e, inputUrl: link}, 'failed to normalize looked up link', ) + addMetrics('error', false) // fail open return 'ok' } @@ -65,18 +88,23 @@ export class SafelinkClient { // _and_ it is not 'ok'. const urlRule = this.urlCache.get(url) if (urlRule && urlRule !== 'ok') { + addMetrics('ok', true) return urlRule } // If we find a domain rule of _any_ kind, including 'ok', we can now return that rule. const domainRule = this.domainCache.get(domain) if (domainRule) { + addMetrics('ok', true) return domainRule } try { const maybeUrlRule = await this.getRule(this.db, url, 'url') this.urlCache.set(url, maybeUrlRule) + + addMetrics('ok', false) + return maybeUrlRule } catch (e) { this.urlCache.set(url, 'ok') @@ -85,11 +113,15 @@ export class SafelinkClient { try { const maybeDomainRule = await this.getRule(this.db, domain, 'domain') this.domainCache.set(domain, maybeDomainRule) + + addMetrics('ok', false) + return maybeDomainRule } catch (e) { this.domainCache.set(domain, 'ok') } + addMetrics('ok', false) return 'ok' } diff --git a/bskylink/src/config.ts b/bskylink/src/config.ts index 795a7210ff..b45fd38356 100644 --- a/bskylink/src/config.ts +++ b/bskylink/src/config.ts @@ -7,6 +7,7 @@ export type Config = { export type ServiceConfig = { port: number + metricsPort: number version?: string hostnames: string[] hostnamesSet: Set @@ -32,6 +33,7 @@ export type DbPoolConfig = { export type Environment = { port?: number + metricsPort?: number version?: string hostnames: string[] appHostname?: string @@ -50,6 +52,7 @@ export type Environment = { export const readEnv = (): Environment => { return { port: envInt('LINK_PORT'), + metricsPort: envInt('LINK_METRICS_PORT'), version: envStr('LINK_VERSION'), hostnames: envList('LINK_HOSTNAMES'), appHostname: envStr('LINK_APP_HOSTNAME'), @@ -71,6 +74,7 @@ export const readEnv = (): Environment => { export const envToCfg = (env: Environment): Config => { const serviceCfg: ServiceConfig = { port: env.port ?? 3000, + metricsPort: env.metricsPort ?? 9090, version: env.version, hostnames: env.hostnames, hostnamesSet: new Set(env.hostnames), diff --git a/bskylink/src/context.ts b/bskylink/src/context.ts index 1520513ceb..adc7fb1e10 100644 --- a/bskylink/src/context.ts +++ b/bskylink/src/context.ts @@ -1,6 +1,7 @@ import {SafelinkClient} from './cache/safelinkClient.js' import {type Config} from './config.js' import Database from './db/index.js' +import {Metrics} from './metrics.js' export type AppContextOptions = { cfg: Config @@ -10,6 +11,7 @@ export type AppContextOptions = { export class AppContext { cfg: Config db: Database + metrics: Metrics = new Metrics() safelinkClient: SafelinkClient abortController = new AbortController() @@ -19,6 +21,7 @@ export class AppContext { this.safelinkClient = new SafelinkClient({ cfg: this.opts.cfg.service, db: this.opts.db, + metrics: this.metrics, }) } diff --git a/bskylink/src/index.ts b/bskylink/src/index.ts index c7d52681df..51462fec9e 100644 --- a/bskylink/src/index.ts +++ b/bskylink/src/index.ts @@ -2,7 +2,7 @@ import events from 'node:events' import type http from 'node:http' import cors from 'cors' -import express from 'express' +import express, {type Response} from 'express' import {createHttpTerminator, type HttpTerminator} from 'http-terminator' import {type Config} from './config.js' @@ -32,6 +32,28 @@ export class LinkService { app = routes(ctx, app) app.use(errorHandler) + // request duration logging + app.use((req, res, next) => { + const start = process.hrtime.bigint() + const originalEnd = res.end.bind(res) as Response['end'] + res.end = function ( + this: Response, + ...args: Parameters + ): ReturnType { + const end = process.hrtime.bigint() + const respTimeMs = Number(end - start) / 1_000_000 // ns to ms :3 + + if (req.route) { + ctx.metrics.requestDuration + .labels(req.route.path, req.method, res.statusCode.toString()) + .observe(respTimeMs) + } + + return originalEnd(...args) + } as Response['end'] + next() + }) + return new LinkService(app, ctx) } diff --git a/bskylink/src/metrics.ts b/bskylink/src/metrics.ts new file mode 100644 index 0000000000..7ffc58364d --- /dev/null +++ b/bskylink/src/metrics.ts @@ -0,0 +1,79 @@ +import {once} from 'events' +import express, {type Application} from 'express' +import * as prometheus from 'prom-client' + +export class Metrics { + private app: Application + + private registry: prometheus.Registry + + requestDuration: prometheus.Histogram + redirects: prometheus.Counter + shortLinkRequests: prometheus.Counter + safeLinkLookups: prometheus.Counter + safeLinkLookupDuration: prometheus.Histogram + + constructor() { + const app = express() + this.app = app + + const registry = new prometheus.Registry() + this.registry = registry + + // Add add metrics route to express + app.get('/metrics', async (_req, res) => { + res.set('content-type', registry.contentType) + res.end(await registry.metrics()) + }) + + // Collect default service metrics + prometheus.collectDefaultMetrics({ + prefix: 'blink_', + register: this.registry, + gcDurationBuckets: [0.001, 0.01, 0.1, 1, 2, 5], + }) + + // Add other metrics + this.requestDuration = new prometheus.Histogram({ + name: 'blink_request_duration_millis', + help: 'Request duration in mmillis', + labelNames: ['path', 'method', 'code'], + buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10], + registers: [registry], + }) + + this.redirects = new prometheus.Counter({ + name: 'blink_redirects', + help: 'Number of link redirects handled', + labelNames: ['safelink_rule', 'code'], + registers: [registry], + }) + + this.shortLinkRequests = new prometheus.Counter({ + name: 'blink_shortlink_requests', + help: 'Number of shortlink requests handled', + labelNames: ['method', 'code'], + registers: [registry], + }) + + this.safeLinkLookups = new prometheus.Counter({ + name: 'blink_safelink_lookups', + help: 'Number of safelink lookups handled', + labelNames: ['status', 'cached'], + registers: [registry], + }) + + this.safeLinkLookupDuration = new prometheus.Histogram({ + name: 'blink_safelink_lookup_duration_millis', + help: 'Request duration in millis', + labelNames: ['status', 'cached'], + buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10], + registers: [registry], + }) + } + + async start(port: number) { + const server = this.app.listen(port) + await once(server, 'listening') + } +} diff --git a/bskylink/src/routes/createShortLink.ts b/bskylink/src/routes/createShortLink.ts index 629119059f..2eeb5960e5 100644 --- a/bskylink/src/routes/createShortLink.ts +++ b/bskylink/src/routes/createShortLink.ts @@ -13,16 +13,24 @@ export default function (ctx: AppContext, app: Express) { '/link', bodyParser.json(), handler(async (req, res) => { + const addMetrics = (statusCode: number) => { + ctx.metrics.shortLinkRequests + .labels('POST', statusCode.toString()) + .inc() + } + let path: string if (typeof req.body?.path === 'string') { path = req.body.path } else { + addMetrics(400) return res.status(400).json({ error: 'InvalidPath', message: '"path" parameter is missing or not a string', }) } if (!path.startsWith('/')) { + addMetrics(400) return res.status(400).json({ error: 'InvalidPath', message: @@ -34,6 +42,7 @@ export default function (ctx: AppContext, app: Express) { // link pattern: /start/{did}/{rkey} if (!parts[1].startsWith('did:')) { // enforce strong links + addMetrics(400) return res.status(400).json({ error: 'InvalidPath', message: @@ -41,8 +50,11 @@ export default function (ctx: AppContext, app: Express) { }) } const id = await ensureLink(ctx, LinkType.StarterPack, parts) + addMetrics(200) return res.json({url: getUrl(ctx, req, id)}) } + + addMetrics(400) return res.status(400).json({ error: 'InvalidPath', message: '"path" parameter does not have a known format', diff --git a/bskylink/src/routes/redirect.ts b/bskylink/src/routes/redirect.ts index 681dc0bb90..d4ce26ff70 100644 --- a/bskylink/src/routes/redirect.ts +++ b/bskylink/src/routes/redirect.ts @@ -19,6 +19,10 @@ export default function (ctx: AppContext, app: Express) { return app.get( '/redirect', handler(async (req, res) => { + const addMetrics = (ruleStr: string, statusCode: number) => { + ctx.metrics.redirects.labels(ruleStr, statusCode.toString()).inc() + } + let link = req.query.u assert( typeof link === 'string', @@ -39,6 +43,7 @@ export default function (ctx: AppContext, app: Express) { ) { res.setHeader('Cache-Control', 'no-store') res.setHeader('Location', `https://${ctx.cfg.service.appHostname}`) + addMetrics('bad_url', 302) return res.status(302).end() } @@ -49,9 +54,12 @@ export default function (ctx: AppContext, app: Express) { let html: string | undefined + let ruleStr = 'ok' if (ctx.cfg.service.safelinkEnabled) { const rule = await ctx.safelinkClient.tryFindRule(link) if (rule !== 'ok') { + ruleStr = rule.action + switch (rule.action) { case 'whitelist': redirectLogger.info({rule}, 'Whitelist rule matched') @@ -89,6 +97,8 @@ export default function (ctx: AppContext, app: Express) { html = linkRedirectContents(url.href) } + addMetrics(ruleStr, 302) + return res.end(html) }), ) diff --git a/bskylink/src/routes/shortLink.ts b/bskylink/src/routes/shortLink.ts index 276aae1ca1..c46d1e2c67 100644 --- a/bskylink/src/routes/shortLink.ts +++ b/bskylink/src/routes/shortLink.ts @@ -1,15 +1,19 @@ import assert from 'node:assert' import {DAY, SECOND} from '@atproto/common' -import {Express} from 'express' +import {type Express} from 'express' -import {AppContext} from '../context.js' +import {type AppContext} from '../context.js' import {handler} from './util.js' export default function (ctx: AppContext, app: Express) { return app.get( '/:linkId', handler(async (req, res) => { + const addMetrics = (statusCode: number) => { + ctx.metrics.shortLinkRequests.labels('GET', statusCode.toString()).inc() + } + const linkId = req.params.linkId const contentType = req.accepts(['html', 'json']) assert( @@ -35,6 +39,7 @@ export default function (ctx: AppContext, app: Express) { } // send the user to the app res.setHeader('Location', `https://${ctx.cfg.service.appHostname}`) + addMetrics(302) return res.status(302).end() } // build url from original url in order to preserve query params @@ -45,9 +50,11 @@ export default function (ctx: AppContext, app: Express) { url.pathname = found.path res.setHeader('Cache-Control', `max-age=${(7 * DAY) / SECOND}`) if (contentType === 'json') { + addMetrics(302) return res.json({url: url.href}).end() } res.setHeader('Location', url.href) + addMetrics(302) return res.status(301).end() }), ) diff --git a/bskylink/yarn.lock b/bskylink/yarn.lock index f30f67cb1c..9676e82b2d 100644 --- a/bskylink/yarn.lock +++ b/bskylink/yarn.lock @@ -161,6 +161,11 @@ dependencies: make-plural "^7.0.0" +"@opentelemetry/api@^1.4.0": + version "1.9.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.0.tgz#d03eba68273dc0f7509e2a3d5cba21eae10379fe" + integrity sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg== + "@tsconfig/node10@^1.0.7": version "1.0.11" resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.11.tgz#6ee46400685f130e278128c7b38b7e031ff5b2f2" @@ -338,6 +343,11 @@ base64-js@^1.3.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== +bintrees@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8" + integrity sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw== + body-parser@1.20.2, body-parser@^1.20.2: version "1.20.2" resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd" @@ -1060,6 +1070,14 @@ process@^0.11.10: resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A== +prom-client@^15.1.3: + version "15.1.3" + resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-15.1.3.tgz#69fa8de93a88bc9783173db5f758dc1c69fa8fc2" + integrity sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g== + dependencies: + "@opentelemetry/api" "^1.4.0" + tdigest "^0.1.1" + proxy-addr@~2.0.7: version "2.0.7" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" @@ -1232,6 +1250,13 @@ string_decoder@^1.3.0: dependencies: safe-buffer "~5.2.0" +tdigest@^0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.2.tgz#96c64bac4ff10746b910b0e23b515794e12faced" + integrity sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA== + dependencies: + bintrees "1.0.2" + thread-stream@^2.6.0: version "2.7.0" resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-2.7.0.tgz#d8a8e1b3fd538a6cca8ce69dbe5d3d097b601e11"