diff --git a/package.json b/package.json index ff2223b489..ca96efd0c8 100644 --- a/package.json +++ b/package.json @@ -114,6 +114,7 @@ "base64-js": "^1.5.1", "bcp-47": "^2.1.0", "bcp-47-match": "^2.0.3", + "bloomfilter": "^0.0.18", "date-fns": "^2.30.0", "deprecated-react-native-prop-types": "^5.0.0", "email-validator": "^2.0.4", diff --git a/src/lib/api/feed/custom.ts b/src/lib/api/feed/custom.ts index dbb02467fe..3e11ddc06c 100644 --- a/src/lib/api/feed/custom.ts +++ b/src/lib/api/feed/custom.ts @@ -5,6 +5,8 @@ import { jsonStringToLex, } from '@atproto/api' +import {DISCOVER_FEED_URI} from '#/lib/constants' +import {isLikelyGloballySeenPost} from '#/state/feed-feedback' import { getAppLanguageAsContentLanguage, getContentLanguages, @@ -72,6 +74,35 @@ export class CustomFeedAPI implements FeedAPI { ) : await loggedOutFetch({...this.params, cursor, limit}) if (res.success) { + if (this.params.feed === DISCOVER_FEED_URI) { + // Try not to show a post in Discover if you've seen it elsewhere in the app. + // A proper way to do this might be to let a feed declare it wants client-side filtering. + // I'm hacking this in here because it needs to be consistent for re-renders so + // the decision on what got included or not for this page needs to happen once. + const candidateFeed = res.data.feed.filter(post => { + if (isLikelyGloballySeenPost(post.post.uri)) { + return false + } + if (post.reply) { + if ( + AppBskyFeedDefs.isPostView(post.reply.parent) && + isLikelyGloballySeenPost(post.reply.parent.uri) + ) { + return false + } + if ( + AppBskyFeedDefs.isPostView(post.reply.root) && + isLikelyGloballySeenPost(post.reply.root.uri) + ) { + return false + } + } + return true + }) + if (candidateFeed.length >= res.data.feed.length / 2) { + res.data.feed = candidateFeed + } + } // NOTE // some custom feeds fail to enforce the pagination limit // so we manually truncate here diff --git a/src/state/feed-feedback.tsx b/src/state/feed-feedback.tsx index 02f98ad145..bb4292d9c6 100644 --- a/src/state/feed-feedback.tsx +++ b/src/state/feed-feedback.tsx @@ -1,6 +1,8 @@ import React from 'react' import {AppState, AppStateStatus} from 'react-native' import {AppBskyFeedDefs} from '@atproto/api' +// @ts-ignore +import {BloomFilter} from 'bloomfilter' import throttle from 'lodash.throttle' import {PROD_DEFAULT_FEED} from '#/lib/constants' @@ -93,24 +95,25 @@ export function useFeedFeedback(feed: FeedDescriptor, hasSession: boolean) { const onItemSeen = React.useCallback( (feedItem: any) => { - if (!enabled) { - return - } const slice = getFeedPostSlice(feedItem) if (slice === null) { return } for (const postItem of slice.items) { - if (!history.current.has(postItem)) { - history.current.add(postItem) - queue.current.add( - toString({ - item: postItem.uri, - event: 'app.bsky.feed.defs#interactionSeen', - feedContext: slice.feedContext, - }), - ) - sendToFeed() + markGloballySeenPost(postItem.uri) + + if (enabled) { + if (!history.current.has(postItem)) { + history.current.add(postItem) + queue.current.add( + toString({ + item: postItem.uri, + event: 'app.bsky.feed.defs#interactionSeen', + feedContext: slice.feedContext, + }), + ) + sendToFeed() + } } } }, @@ -254,3 +257,22 @@ function flushToStatsig(stats: AggregatedStats | null) { stats.seenCount = 0 } } + +// https://hur.st/bloomfilter/?n=50k&p=0.001&m=&k= +const p = 0.001 // 0.1% probability of collisions... +const n = 50_000 // ...while we stay below 50k expected items +const m = Math.ceil((n * Math.log(p)) / Math.log(1 / Math.pow(2, Math.log(2)))) +const k = Math.round((m / n) * Math.log(2)) +let globalBloomFilter = new BloomFilter(m, k) + +function markGloballySeenPost(uri: string) { + if (globalBloomFilter.size >= n) { + // If we ever get here, just restart to avoid saturation. + globalBloomFilter = new BloomFilter(m, k) + } + globalBloomFilter.add(uri) +} + +export function isLikelyGloballySeenPost(uri: string) { + return globalBloomFilter.test(uri) +} diff --git a/yarn.lock b/yarn.lock index 61ed0f66c7..59f4d1ae78 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8200,6 +8200,11 @@ bl@^4.0.3, bl@^4.1.0: inherits "^2.0.4" readable-stream "^3.4.0" +bloomfilter@^0.0.18: + version "0.0.18" + resolved "https://registry.yarnpkg.com/bloomfilter/-/bloomfilter-0.0.18.tgz#6d55d34f0a214b235287b4eac9203ac623413dab" + integrity sha512-CbnyHE78gY1tpXS/Ap+B0RJxKdRWCDzjBnX97UJSG8rdLv1PK8GiTWc/CCQyWu6PWVD4lUceeFrqC6Mf3nMgOA== + bn.js@^4.0.0, bn.js@^4.11.8, bn.js@^4.11.9: version "4.12.0" resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.12.0.tgz#775b3f278efbb9718eec7361f483fb36fbbfea88"