diff --git a/.oxlintrc.json b/.oxlintrc.json index 907b47af39..319b110386 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -385,6 +385,7 @@ }, { "files": [ + "bskylink/**/*.{js,jsx,ts,tsx}", "bskyogcard/**/*.{js,jsx,ts,tsx}", "dev-env/**/*.{js,jsx,ts,tsx}" ], diff --git a/Dockerfile.bskylink b/Dockerfile.bskylink index 7c4f94ead8..6fe98b8e70 100644 --- a/Dockerfile.bskylink +++ b/Dockerfile.bskylink @@ -26,8 +26,9 @@ COPY --from=build /app /app RUN mkdir /app/data && chown node /app/data VOLUME /app/data -EXPOSE 3000 +EXPOSE 3000 9090 ENV LINK_PORT=3000 +ENV LINK_METRICS_PORT=9090 ENV NODE_ENV=production # https://github.com/nodejs/docker-node/blob/master/docs/BestPractices.md#non-root-user diff --git a/bskylink/package.json b/bskylink/package.json index 23fb08aac2..32271b3403 100644 --- a/bskylink/package.json +++ b/bskylink/package.json @@ -23,6 +23,7 @@ "lru-cache": "^11.1.0", "pg": "^8.12.0", "pino": "^9.2.0", + "prom-client": "^15.1.3", "uint8arrays": "^5.1.0" }, "devDependencies": { diff --git a/bskylink/src/bin.ts b/bskylink/src/bin.ts index 3dea1a7527..0194471dc2 100644 --- a/bskylink/src/bin.ts +++ b/bskylink/src/bin.ts @@ -10,6 +10,7 @@ async function main() { httpLogger.info( { port: cfg.service.port, + metricsPort: cfg.service.metricsPort, safelinkEnabled: cfg.service.safelinkEnabled, hasDbUrl: !!cfg.db.url, hasDbMigrationUrl: !!cfg.db.migrationUrl, @@ -33,16 +34,18 @@ async function main() { if (link.ctx.cfg.service.safelinkEnabled) { httpLogger.info('Starting Safelink client') - link.ctx.safelinkClient.runFetchEvents() + void link.ctx.safelinkClient.runFetchEvents() } await link.start() httpLogger.info('Link service is running') - process.on('SIGTERM', async () => { - httpLogger.info('Link service is stopping') - await link.destroy() - httpLogger.info('Link service is stopped') + process.on('SIGTERM', () => { + void (async () => { + httpLogger.info('Link service is stopping') + await link.destroy() + httpLogger.info('Link service is stopped') + })() }) } catch (error) { httpLogger.error( diff --git a/bskylink/src/config.ts b/bskylink/src/config.ts index e583a1f259..1a7f3d113d 100644 --- a/bskylink/src/config.ts +++ b/bskylink/src/config.ts @@ -7,6 +7,7 @@ export type Config = { export type ServiceConfig = { port: number + metricsPort: number version?: string hostnames: string[] hostnamesSet: Set @@ -33,6 +34,7 @@ export type DbPoolConfig = { export type Environment = { port?: number + metricsPort?: number version?: string hostnames: string[] appHostname?: string @@ -52,6 +54,7 @@ export type Environment = { export const readEnv = (): Environment => { return { port: envInt('LINK_PORT'), + metricsPort: envInt('LINK_METRICS_PORT'), version: envStr('LINK_VERSION'), hostnames: envList('LINK_HOSTNAMES'), appHostname: envStr('LINK_APP_HOSTNAME'), @@ -74,6 +77,7 @@ export const readEnv = (): Environment => { export const envToCfg = (env: Environment): Config => { const serviceCfg: ServiceConfig = { port: env.port ?? 3000, + metricsPort: env.metricsPort ?? 9090, version: env.version, hostnames: env.hostnames, hostnamesSet: new Set(env.hostnames), diff --git a/bskylink/src/index.test.ts b/bskylink/src/index.test.ts new file mode 100644 index 0000000000..b869448a25 --- /dev/null +++ b/bskylink/src/index.test.ts @@ -0,0 +1,49 @@ +import assert from 'node:assert' +import {type AddressInfo} from 'node:net' +import {test} from 'node:test' + +import {envToCfg} from './config.js' +import {LinkService} from './index.js' + +const testConfig = () => + envToCfg({ + dbPostgresUrl: 'postgres://localhost:1/blink', + hostnames: ['go.bsky.app'], + metricsPort: 0, + port: 0, + safelinkAgentIdentifier: 'test', + safelinkAgentPass: 'test', + safelinkPdsUrl: 'https://example.com', + }) + +void test('serves and terminates the Prometheus listener', async () => { + const service = await LinkService.create(testConfig()) + + try { + await service.start() + const {port} = service.metricsServer?.address() as AddressInfo + const res = await fetch(`http://127.0.0.1:${port}/metrics`) + + assert.strictEqual(res.status, 200) + assert.match(res.headers.get('content-type') ?? '', /text\/plain/) + const metrics = await res.text() + assert.match(metrics, /process_cpu_user_seconds_total/) + assert.match(metrics, /nodejs_eventloop_lag_max_seconds/) + assert.match(metrics, /bskylink_db_pool_connections\{state="idle"\} 0/) + assert.match(metrics, /bskylink_db_pool_connections\{state="in_use"\} 0/) + assert.match(metrics, /bskylink_db_pool_max_connections 10/) + assert.match(metrics, /bskylink_db_pool_waiting_requests 0/) + assert.doesNotMatch(metrics, /http_request_duration_seconds/) + } finally { + await service.destroy() + } + + assert.strictEqual(service.metricsServer?.listening, false) +}) + +void test('isolates the Prometheus registry per service', async () => { + const first = await LinkService.create(testConfig()) + const second = await LinkService.create(testConfig()) + + await Promise.all([first.destroy(), second.destroy()]) +}) diff --git a/bskylink/src/index.ts b/bskylink/src/index.ts index c7111c78a5..24e1a798a0 100644 --- a/bskylink/src/index.ts +++ b/bskylink/src/index.ts @@ -4,10 +4,12 @@ import type http from 'node:http' import cors from 'cors' import express from 'express' import {createHttpTerminator, type HttpTerminator} from 'http-terminator' +import {type Registry} from 'prom-client' import {type Config} from './config.js' import {AppContext} from './context.js' import i18n from './i18n.js' +import {createPrometheusRegistry} from './prometheus.js' import {default as routes, errorHandler} from './routes/index.js' export * from './config.js' @@ -16,12 +18,17 @@ export * from './logger.js' export class LinkService { public server?: http.Server + public metricsServer?: http.Server private terminator?: HttpTerminator + private metricsTerminator?: HttpTerminator + private metricsRegistry: Registry constructor( public app: express.Application, public ctx: AppContext, - ) {} + ) { + this.metricsRegistry = createPrometheusRegistry(ctx) + } static async create(cfg: Config): Promise { let app = express() @@ -41,11 +48,26 @@ export class LinkService { this.server.keepAliveTimeout = 90000 this.terminator = createHttpTerminator({server: this.server}) await events.once(this.server, 'listening') + + const metricsApp = express() + metricsApp.get('/metrics', (_req, res, next) => { + res.set('Content-Type', this.metricsRegistry.contentType) + this.metricsRegistry.metrics().then(metrics => res.end(metrics), next) + }) + this.metricsServer = metricsApp.listen(this.ctx.cfg.service.metricsPort) + this.metricsTerminator = createHttpTerminator({ + server: this.metricsServer, + gracefulTerminationTimeout: 2000, + }) + await events.once(this.metricsServer, 'listening') } async destroy() { this.ctx.abortController.abort() - await this.terminator?.terminate() + await Promise.all([ + this.terminator?.terminate(), + this.metricsTerminator?.terminate(), + ]) await this.ctx.db.close() this.ctx.metrics.stop() } diff --git a/bskylink/src/prometheus.ts b/bskylink/src/prometheus.ts new file mode 100644 index 0000000000..f2f8a31408 --- /dev/null +++ b/bskylink/src/prometheus.ts @@ -0,0 +1,54 @@ +import {collectDefaultMetrics, Gauge, Registry} from 'prom-client' + +import {type AppContext} from './context.js' + +let runtimeRegistry: Registry | undefined + +const getRuntimeRegistry = (): Registry => { + if (!runtimeRegistry) { + runtimeRegistry = new Registry() + + // Beyla already exports HTTP RED metrics and traces for Blink. These + // process metrics cover the runtime-only failure modes it cannot see, + // particularly event-loop stalls, GC pauses, and V8 heap pressure. + collectDefaultMetrics({register: runtimeRegistry}) + } + + return runtimeRegistry +} + +export const createPrometheusRegistry = (ctx: AppContext): Registry => { + const poolRegistry = new Registry() + + new Gauge<'state'>({ + name: 'bskylink_db_pool_connections', + help: 'PostgreSQL client connections by usage state.', + labelNames: ['state'], + registers: [poolRegistry], + collect() { + const {idleCount, totalCount} = ctx.db.cfg.pool + this.set({state: 'idle'}, idleCount) + this.set({state: 'in_use'}, totalCount - idleCount) + }, + }) + + new Gauge({ + name: 'bskylink_db_pool_max_connections', + help: 'Configured maximum PostgreSQL client connections.', + registers: [poolRegistry], + collect() { + this.set(ctx.cfg.db.pool.size) + }, + }) + + new Gauge({ + name: 'bskylink_db_pool_waiting_requests', + help: 'Requests waiting for a PostgreSQL client connection.', + registers: [poolRegistry], + collect() { + this.set(ctx.db.cfg.pool.waitingCount) + }, + }) + + return Registry.merge([getRuntimeRegistry(), poolRegistry]) +} diff --git a/bskylink/yarn.lock b/bskylink/yarn.lock index f30f67cb1c..017ef29846 100644 --- a/bskylink/yarn.lock +++ b/bskylink/yarn.lock @@ -161,6 +161,11 @@ dependencies: make-plural "^7.0.0" +"@opentelemetry/api@^1.4.0": + version "1.9.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.9.1.tgz#c1b0346de336ba55af2d5a7970882037baedec05" + integrity sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q== + "@tsconfig/node10@^1.0.7": version "1.0.11" resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.11.tgz#6ee46400685f130e278128c7b38b7e031ff5b2f2" @@ -338,6 +343,11 @@ base64-js@^1.3.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== +bintrees@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8" + integrity sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw== + body-parser@1.20.2, body-parser@^1.20.2: version "1.20.2" resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.2.tgz#6feb0e21c4724d06de7ff38da36dad4f57a747fd" @@ -1060,6 +1070,14 @@ process@^0.11.10: resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A== +prom-client@^15.1.3: + version "15.1.3" + resolved "https://registry.yarnpkg.com/prom-client/-/prom-client-15.1.3.tgz#69fa8de93a88bc9783173db5f758dc1c69fa8fc2" + integrity sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g== + dependencies: + "@opentelemetry/api" "^1.4.0" + tdigest "^0.1.1" + proxy-addr@~2.0.7: version "2.0.7" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" @@ -1232,6 +1250,13 @@ string_decoder@^1.3.0: dependencies: safe-buffer "~5.2.0" +tdigest@^0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/tdigest/-/tdigest-0.1.2.tgz#96c64bac4ff10746b910b0e23b515794e12faced" + integrity sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA== + dependencies: + bintrees "1.0.2" + thread-stream@^2.6.0: version "2.7.0" resolved "https://registry.yarnpkg.com/thread-stream/-/thread-stream-2.7.0.tgz#d8a8e1b3fd538a6cca8ce69dbe5d3d097b601e11"