preliminary review ready impl of safelink

This commit is contained in:
will berry
2025-06-10 15:37:31 -05:00
committed by BlueSkiesAndGreenPastures
parent c6fc298d1e
commit 5e8ee142b1
12 changed files with 633 additions and 29 deletions
BIN
View File
Binary file not shown.
+9 -1
View File
@@ -1,5 +1,5 @@
import {adaptiveFetchAndUpdate, eventCache} from './cache/cache.js'
import {Database, envToCfg, httpLogger, LinkService, readEnv} from './index.js'
async function main() {
const env = readEnv()
const cfg = envToCfg(env)
@@ -11,9 +11,17 @@ async function main() {
await migrateDb.migrateToLatestOrThrow()
await migrateDb.close()
}
const link = await LinkService.create(cfg)
if (cfg.service.safelink === 1) {
eventCache.init(cfg.service)
adaptiveFetchAndUpdate()
}
await link.start()
httpLogger.info('link service is running')
console.log('running service')
process.on('SIGTERM', async () => {
httpLogger.info('link service is stopping')
await link.destroy()
+256
View File
@@ -0,0 +1,256 @@
import {ToolsOzoneSafelinkDefs} from '@atproto/api'
import {type ServiceConfig} from '../config.js'
import {redirectLogger} from '../logger.js'
// import {ozoneAgent} from './agent.js'
import {OzoneAgent} from './ozoneAgent.js'
let cacheCursor: string | undefined
export class EventCache {
private rules = new Map<string, ToolsOzoneSafelinkDefs.Event>()
private cfg: ServiceConfig | undefined = undefined
constructor() {
this.cfg = undefined
}
async init(cfg: ServiceConfig) {
this.cfg = cfg
}
async getConfig(): Promise<ServiceConfig | undefined> {
return this.cfg
}
insert(key: string, evt: ToolsOzoneSafelinkDefs.Event) {
const existing = this.rules.get(key)
if (!existing || new Date(evt.createdAt) > new Date(existing.createdAt)) {
this.rules.set(key, evt)
}
}
smartUpdateDomain(event: ToolsOzoneSafelinkDefs.Event) {
try {
const domain = new URL(event.url).hostname
event.url = domain
redirectLogger.info(
`[EventCache] smartUpdateDomain called for domain: ${domain}, action: ${event.action}`,
)
if (event.action === ToolsOzoneSafelinkDefs.REMOVERULE) {
// If the action is to remove the rule, delete it from the cache
this.insert(domain, event)
redirectLogger.info(
`[EventCache] Removed rule for domain, adjusted audit log: ${domain}`,
)
return
}
// if update should happen
if (event.action === ToolsOzoneSafelinkDefs.WHITELIST) {
this.insert(domain, event)
redirectLogger.info(`[EventCache] Whitelisted domain: ${domain}`)
return
}
if (event.action === ToolsOzoneSafelinkDefs.BLOCK) {
this.insert(domain, event)
redirectLogger.info(`[EventCache] Blocked domain: ${domain}`)
return
}
if (event.action === ToolsOzoneSafelinkDefs.WARN) {
this.insert(domain, event)
redirectLogger.info(`[EventCache] Warned domain: ${domain}`)
return
}
} catch (error) {
redirectLogger.error(
`[EventCache:smartUpdateDomain] Error in smartUpdateDomain: ${error}`,
)
throw new Error(
`[EventCache:smartUpdateDomain] Error processing domain event: ${error}`,
)
}
}
smartUpdateUrl(event: ToolsOzoneSafelinkDefs.Event) {
redirectLogger.info(
`[EventCache] smartUpdateUrl called for url: ${event.url}, action: ${event.action}`,
)
if (event.action === ToolsOzoneSafelinkDefs.REMOVERULE) {
// If the action is to remove the rule, delete it from the cache
this.insert(event.url, event)
redirectLogger.info(`[EventCache] Removed rule for url: ${event.url}`)
return
}
// if update should happen
if (event.action === ToolsOzoneSafelinkDefs.WHITELIST) {
this.insert(event.url, event)
redirectLogger.info(`[EventCache] Whitelisted url: ${event.url}`)
return
}
if (event.action === ToolsOzoneSafelinkDefs.BLOCK) {
this.insert(event.url, event)
redirectLogger.info(`[EventCache] Blocked url: ${event.url}`)
return
}
if (event.action === ToolsOzoneSafelinkDefs.WARN) {
this.insert(event.url, event)
redirectLogger.info(`[EventCache] Warned url: ${event.url}`)
return
}
}
// Insert or update an event
smartUpdate(event: ToolsOzoneSafelinkDefs.Event) {
if (event.pattern === ToolsOzoneSafelinkDefs.DOMAIN) {
redirectLogger.info(
`[EventCache] smartUpdate called for domain event: ${event.url}, performing$ ${event.action}`,
)
return this.smartUpdateDomain(event)
}
if (event.pattern === ToolsOzoneSafelinkDefs.URL) {
redirectLogger.info(
`[EventCache] smartUpdate called for domain event: ${event.url}`,
)
return this.smartUpdateUrl(event)
}
throw new Error('[EventCache] Unknown event pattern')
}
/**
* Attempts to retrieve an event for the given URL.
* Checks in order: domain, domain+path, then full URL.
*/
smartGet(url: string): ToolsOzoneSafelinkDefs.Event | undefined {
const parsedUrl = new URL(url)
const domain = parsedUrl.hostname
const domainAndPath = domain + parsedUrl.pathname
// 1. Check for a rule by domain only
const byDomain = this.rules.get(domain)
if (byDomain) {
return byDomain
}
// 2. Check for a rule by domain + path
const byDomainAndPath = this.rules.get(domainAndPath)
if (byDomainAndPath) {
return byDomainAndPath
}
// 3. Check for a rule by full URL
return this.rules.get(url)
}
delete(event: ToolsOzoneSafelinkDefs.Event) {
this.rules.delete(event.url)
}
// Get an event by full URL
get(url: string): ToolsOzoneSafelinkDefs.Event | undefined {
const event = this.rules.get(url)
return event
}
// List all events
list(): ToolsOzoneSafelinkDefs.Event[] {
return Array.from(this.rules.values())
}
}
// Adaptive polling: slow down if no new events, speed up if updates found
let pollInterval = 1 * 1000 // start at 1 seconds
export async function adaptiveFetchAndUpdate() {
const prevCursor = cacheCursor
const eventConfig = await eventCache.getConfig()
if (eventConfig === undefined) {
redirectLogger.info(
`[adaptiveFetchAndUpdate] No Configuration found, skipping fetch.`,
)
} else {
await fetchAndUpdateEvents(eventConfig)
}
// If no new events, increase interval (up to 10 minutes), else reset to 5s
if (cacheCursor === prevCursor) {
pollInterval = Math.min(pollInterval * 2, 10 * 60 * 1000)
redirectLogger.info(
`[adaptiveFetchAndUpdate] No new events, backing off. Next poll in ${
pollInterval / 1000
}s`,
)
} else {
pollInterval = 5 * 1000
redirectLogger.info(
`[adaptiveFetchAndUpdate] New events found, resetting poll interval to ${
pollInterval / 1000
}s`,
)
}
setTimeout(adaptiveFetchAndUpdate, pollInterval)
}
// Export a singleton instance
export const eventCache = new EventCache()
// Start adaptive polling
// adaptiveFetchAndUpdate()
// Function to fetch and update events from the server
export async function fetchAndUpdateEvents(cfg: ServiceConfig) {
if (!cfg || !cfg.ozoneUrl || !cfg.ozoneAgentHandle || !cfg.ozoneAgentPass) {
console.error(
'[eventCache:fetchAndUpdateEvents] No active config, skipping actions',
)
return
}
redirectLogger.info(
`[eventCache] Fetching events with cursor: ${cacheCursor}`,
)
// Use current session DID instead of env variable
const ozoneAgent = await new OzoneAgent(cfg)
const ozoneSession = await ozoneAgent.getSession?.()
if (!ozoneSession) {
console.error('[eventCache:fetchAndUpdateEvents] No active session found.')
return
}
const ozoneDid = ozoneSession.did
if (!ozoneDid || ozoneDid === 'did:plc:invalid') {
console.error(
'[eventCache:fetchAndUpdateEvents] Invalid or missing session DID.',
)
return
}
ozoneAgent.agent.setHeader?.('atproto-proxy', `${ozoneDid}#atproto_labeler`)
const res = await ozoneAgent.agent.tools?.ozone?.safelink?.queryEvents?.({
cursor: cacheCursor,
limit: 100, // Adjust as needed
})
if (res?.data.cursor === cacheCursor) {
redirectLogger.info(
'[eventCache:fetchAndUpdateEvents] No new events to update.',
)
return
}
redirectLogger.info(`[eventCache:fetchAndUpdateEvents] Received response:`, {
...res,
data: {
...res?.data,
rules: Array.isArray(res?.data?.events)
? res.data.events.map(event => JSON.stringify(event))
: res?.data?.events,
},
})
for (const event of res.data.events) {
eventCache.smartUpdate(event)
}
cacheCursor = res.data?.cursor
redirectLogger.info(
'[eventCache:fetchAndUpdateEvents] Current cache contents:',
)
redirectLogger.info(eventCache.list())
}
+49
View File
@@ -0,0 +1,49 @@
import {AtpAgent, CredentialSession} from '@atproto/api'
import {type ServiceConfig} from '../config'
export class OzoneAgent {
public session: CredentialSession
public agent: AtpAgent
private cfg?: ServiceConfig
constructor(cfg: ServiceConfig) {
this.cfg = cfg
this.session = new CredentialSession(
new URL(cfg?.ozoneUrl || 'http://localhost:2583'),
)
this.agent = new AtpAgent(this.session)
}
public async getSession(): Promise<CredentialSession> {
if (!this.session.hasSession) {
await this.getAgent()
}
return this.session
}
public async getAgent(): Promise<AtpAgent> {
if (!this.cfg?.ozoneAgentHandle && !this.cfg?.ozoneAgentPass) {
throw new Error(
'OZONE_AGENT_HANDLE and OZONE_AGENT_PASS environment variables must be set',
)
}
const identifier = this.cfg?.ozoneAgentHandle || 'did:plc:invalid'
const password = this.cfg?.ozoneAgentPass || 'invalid'
if (!this.session.hasSession) {
await this.session.login({identifier, password})
}
try {
await this.agent.com.atproto.server.getSession()
} catch (err) {
if ((err as any).status === 401) {
await this.session.login({identifier, password})
}
}
return this.agent
}
}
+29
View File
@@ -0,0 +1,29 @@
export type RuleEventType = '#addRule' | '#updateRule' | '#removeRule'
export type RulePatternType = '#domain' | '#url'
export type RuleActionType = '#block' | '#warn' | '#whitelist'
export type RuleReasonType = '#csam' | '#spam' | '#phishing' | '#none'
export interface Rule {
id: number
eventType: RuleEventType
url: string
pattern: RulePatternType
action: RuleActionType
reason: RuleReasonType
createdBy: string // DID format
createdAt: string // ISO datetime string
comment?: string
}
// Example Rule object
export const exampleRule: Rule = {
id: 1,
eventType: '#addRule',
url: 'https://malicious.example.com/phishing',
pattern: '#domain',
action: '#block',
reason: '#phishing',
createdBy: 'did:plc:adminozonetools',
createdAt: '2024-06-01T12:00:00Z',
comment: 'Known phishing domain detected by automated scan.',
}
+22
View File
@@ -1,7 +1,11 @@
import {envInt, envList, envStr} from '@atproto/common'
// import { type EventCache, eventCache } from '../cache/cache.js'
import {type EventCache, eventCache} from './cache/cache.js'
export type Config = {
service: ServiceConfig
eventCache: EventCache
db: DbConfig
}
@@ -10,6 +14,10 @@ export type ServiceConfig = {
version?: string
hostnames: string[]
appHostname: string
safelink: number
ozoneUrl?: string
ozoneAgentHandle?: string
ozoneAgentPass?: string
}
export type DbConfig = {
@@ -36,6 +44,10 @@ export type Environment = {
dbPostgresPoolSize?: number
dbPostgresPoolMaxUses?: number
dbPostgresPoolIdleTimeoutMs?: number
safelink?: number
ozoneUrl?: string
ozoneAgentHandle?: string
ozoneAgentPass?: string
}
export const readEnv = (): Environment => {
@@ -52,6 +64,10 @@ export const readEnv = (): Environment => {
dbPostgresPoolIdleTimeoutMs: envInt(
'LINK_DB_POSTGRES_POOL_IDLE_TIMEOUT_MS',
),
safelink: envInt('SAFELINK'),
ozoneUrl: envStr('OZONE_URL'),
ozoneAgentHandle: envStr('OZONE_AGENT_HANDLE'),
ozoneAgentPass: envStr('OZONE_AGENT_PASS'),
}
}
@@ -61,6 +77,10 @@ export const envToCfg = (env: Environment): Config => {
version: env.version,
hostnames: env.hostnames,
appHostname: env.appHostname || 'bsky.app',
safelink: env.safelink || 0,
ozoneUrl: env.ozoneUrl || undefined,
ozoneAgentHandle: env.ozoneAgentHandle || undefined,
ozoneAgentPass: env.ozoneAgentPass || undefined,
}
if (!env.dbPostgresUrl) {
throw new Error('Must configure postgres url (LINK_DB_POSTGRES_URL)')
@@ -75,8 +95,10 @@ export const envToCfg = (env: Environment): Config => {
size: env.dbPostgresPoolSize ?? 10,
},
}
return {
service: serviceCfg,
eventCache: eventCache,
db: dbCfg,
}
}
+3 -3
View File
@@ -1,11 +1,11 @@
import events from 'node:events'
import http from 'node:http'
import type http from 'node:http'
import cors from 'cors'
import express from 'express'
import {createHttpTerminator, HttpTerminator} from 'http-terminator'
import {createHttpTerminator, type HttpTerminator} from 'http-terminator'
import {Config} from './config.js'
import {type Config} from './config.js'
import {AppContext} from './context.js'
import {default as routes, errorHandler} from './routes/index.js'
+16 -2
View File
@@ -1,4 +1,18 @@
import {subsystemLogger} from '@atproto/common'
import {type Logger} from 'pino'
export const httpLogger = subsystemLogger('bskylink')
export const dbLogger = subsystemLogger('bskylink:db')
export const httpLogger: Logger = subsystemLogger('bskylink')
export const dbLogger: Logger = subsystemLogger('bskylink:db')
export const redirectLogger: Logger = subsystemLogger('bskylink:redirect')
// Also log to stdout
redirectLogger.info = (
orig =>
(...args: any[]) => {
const [msg, ...rest] = args
orig.apply(redirectLogger, [String(msg), ...rest])
// Print to stdout as well
// You can format this as needed
console.log('[bskylink:redirect]', ...args)
}
)(redirectLogger.info) as typeof redirectLogger.info
+7 -6
View File
@@ -1,9 +1,9 @@
import assert from 'node:assert'
import bodyParser from 'body-parser'
import {Express, Request} from 'express'
import {type Express, type Request} from 'express'
import {AppContext} from '../context.js'
import {type AppContext} from '../context.js'
import {LinkType} from '../db/schema.js'
import {randomId} from '../util.js'
import {handler} from './util.js'
@@ -78,13 +78,14 @@ const getUrl = (ctx: AppContext, req: Request, id: string) => {
if (!ctx.cfg.service.hostnames.length) {
assert(req.headers.host, 'request must be made with host header')
const baseUrl =
req.protocol === 'http' && req.headers.host.startsWith('localhost:')
req.protocol === 'http' && req.headers.host!.startsWith('localhost:')
? `http://${req.headers.host}`
: `https://${req.headers.host}`
return `${baseUrl}/${id}`
}
const baseUrl = ctx.cfg.service.hostnames.includes(req.headers.host)
? `https://${req.headers.host}`
const host = req.headers.host ?? ''
const baseUrl = ctx.cfg.service.hostnames.includes(host)
? `https://${host}`
: `https://${ctx.cfg.service.hostnames[0]}`
return `${baseUrl}/${id}`
}
@@ -94,7 +95,7 @@ const normalizedPathFromParts = (parts: string[]): string => {
'/' +
parts
.map(encodeURIComponent)
.map(part => part.replaceAll('%3A', ':')) // preserve colons
.map(part => part.replace(/%3A/g, ':')) // preserve colons
.join('/')
)
}
+2 -2
View File
@@ -1,6 +1,6 @@
import {Express} from 'express'
import {type Express} from 'express'
import {AppContext} from '../context.js'
import {type AppContext} from '../context.js'
import {default as createShortLink} from './createShortLink.js'
import {default as health} from './health.js'
import {default as redirect} from './redirect.js'
+222 -6
View File
@@ -1,10 +1,14 @@
import assert from 'node:assert'
import {ToolsOzoneSafelinkDefs} from '@atproto/api'
import {DAY, SECOND} from '@atproto/common'
import escapeHTML from 'escape-html'
import {Express} from 'express'
import {type Express} from 'express'
import {AppContext} from '../context.js'
// IMPORTANT: Ensure this import path matches exactly (including casing and extension) everywhere it's used
// Ensure this import path matches exactly everywhere in your project to avoid module duplication
import {type AppContext} from '../context.js'
import {redirectLogger} from '../logger.js'
import {handler} from './util.js'
const INTERNAL_IP_REGEX = new RegExp(
@@ -41,12 +45,224 @@ export default function (ctx: AppContext, app: Express) {
res.setHeader('Cache-Control', `max-age=${(7 * DAY) / SECOND}`)
res.type('html')
res.status(200)
res.status(302)
if (ctx.cfg.service.safelink) {
const rulePresent: ToolsOzoneSafelinkDefs.Event | undefined =
ctx.cfg.eventCache.smartGet(link)
// begin link safety checks
if (
rulePresent &&
rulePresent.eventType === ToolsOzoneSafelinkDefs.REMOVERULE
) {
redirectLogger.info(
`No rule or remove rule matched for ${rulePresent.url}`,
)
const escaped = escapeHTML(url.href)
return safe_redirect(escaped)
}
if (
rulePresent &&
rulePresent.action === ToolsOzoneSafelinkDefs.WHITELIST
) {
redirectLogger.info(`Whitelist rule matched for ${rulePresent.url}`)
const escaped = escapeHTML(url.href)
return safe_redirect(escaped)
}
if (
rulePresent &&
rulePresent.action === ToolsOzoneSafelinkDefs.BLOCK
) {
redirectLogger.info(`Block rule matched for ${rulePresent.url}`)
res.setHeader('Cache-Control', 'no-store')
res.status(403)
return res.send(
warnRedirect(
'Blocked Link',
'This link has been identified as malicious, it has been blocked to protect your account and data',
'Go Back To BlueSky',
'DANGER',
escapeHTML(url.toString()),
`https://${ctx.cfg.service.appHostname}`,
),
)
}
if (rulePresent && rulePresent.action === ToolsOzoneSafelinkDefs.WARN) {
redirectLogger.info(`Warn rule matched for ${rulePresent.url}`)
res.setHeader('Cache-Control', 'no-store')
res.status(403)
const escaped = escapeHTML(url.href)
return res.send(
warnRedirect(
'Warning: Potentially Malicious Link',
'This link could be malicious, proceed at your own risk',
'Proceed at Your Own Risk',
'WARNING',
escapeHTML(url.toString()),
escaped, // Pass the actual URL as returnUrl so the button proceeds to the link
),
)
}
}
const escaped = escapeHTML(url.href)
return res.send(
`<html><head><meta http-equiv="refresh" content="0; URL='${escaped}'" /><style>:root { color-scheme: light dark; }</style></head></html>`,
)
return safe_redirect(escaped)
}),
)
}
const safe_redirect = (escaped: string) =>
`<html><head>
<meta http-equiv="refresh" content="0; URL='${escaped}'" />
<meta http-equiv="Cache-Control" content="no-store, no-cache, must-revalidate, max-age=0" />
<meta http-equiv="Pragma" content="no-cache" />
<meta http-equiv="Expires" content="0" />
<style>:root { color-scheme: light dark; }</style>
</head></html>`
const warnRedirect = (
mainText: string,
warningText: string,
buttonText: string,
reason: string,
siteUrl: string,
returnUrl = 'https://bsky.app',
) => {
return `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="Cache-Control" content="no-store, no-cache, must-revalidate, max-age=0" />
<meta http-equiv="Pragma" content="no-cache" />
<meta http-equiv="Expires" content="0" />
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>${mainText}</title>
<style>
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Arial, sans-serif;
background-color: #ffffff;
min-height: 100vh;
display: flex;
align-items: center;
justify-content: center;
padding: 20px;
}
.container {
width: 100%;
max-width: 400px;
text-align: center;
}
.warning-icon {
font-size: 48px;
margin-bottom: 16px;
}
h1 {
font-size: 20px;
font-weight: 600;
margin-bottom: 12px;
color: #000000;
}
.warning-text {
font-size: 15px;
color: #536471;
line-height: 1.4;
margin-bottom: 24px;
padding: 0 20px;
}
.blocked-site {
background-color: #f7f9fa;
border-radius: 12px;
padding: 16px;
margin-bottom: 24px;
text-align: left;
word-break: break-all;
}
.site-name {
font-size: 16px;
font-weight: 500;
color: #000000;
margin-bottom: 4px;
word-break: break-word;
display: block;
text-align: center;
}
.site-url {
font-size: 14px;
color: #536471;
word-break: break-all;
display: block;
text-align: center;
}
.back-button {
background-color: #1d9bf0;
color: white;
border: none;
border-radius: 24px;
padding: 12px 32px;
font-size: 16px;
font-weight: 600;
cursor: pointer;
width: 100%;
max-width: 280px;
transition: background-color 0.2s;
}
.back-button:hover {
background-color: #1a8cd8;
}
.back-button:active {
background-color: #1681c4;
}
@media (max-width: 480px) {
.warning-text {
padding: 0 10px;
}
.blocked-site {
padding: 8px;
}
}
</style>
</head>
<body>
<div class="container">
<div class="warning-icon">⚠️</div>
<h1>${mainText}</h1>
<p class="warning-text">${escapeHTML(warningText)}</p>
<div class="blocked-site">
<span class="site-name">${escapeHTML(reason)}</span>
<span class="site-url">${escapeHTML(siteUrl)}</span>
</div>
<button class="back-button" id="redirect-button">${escapeHTML(
buttonText,
)}</button>
</div>
<script>
document.getElementById('redirect-button').addEventListener('click', function() {
window.location.href = ${JSON.stringify(returnUrl)};
});
</script>
</body>
</html>`
}
+18 -9
View File
@@ -1,10 +1,19 @@
{
"compilerOptions": {
"module": "NodeNext",
"esModuleInterop": true,
"moduleResolution": "NodeNext",
"outDir": "dist",
"lib": ["ES2021.String"]
},
"include": ["./src/index.ts", "./src/bin.ts"]
}
"compilerOptions": {
"target": "ES2020",
"module": "ESNext",
"moduleResolution": "bundler",
"allowSyntheticDefaultImports": true,
"esModuleInterop": true,
"skipLibCheck": true,
"strict": true,
"outDir": "./dist",
"rootDir": "./src",
"declaration": true,
"declarationMap": true,
"sourceMap": true
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]
}