[Discover] Track seen items on the client side

package.json

@@ -114,6 +114,7 @@
     "base64-js": "^1.5.1",
     "bcp-47": "^2.1.0",
     "bcp-47-match": "^2.0.3",
+    "bloomfilter": "^0.0.18",
     "date-fns": "^2.30.0",
     "deprecated-react-native-prop-types": "^5.0.0",
     "email-validator": "^2.0.4",

src/lib/api/feed/custom.ts

@@ -5,6 +5,8 @@ import {
   jsonStringToLex,
 } from '@atproto/api'
 
+import {DISCOVER_FEED_URI} from '#/lib/constants'
+import {isLikelyGloballySeenPost} from '#/state/feed-feedback'
 import {
   getAppLanguageAsContentLanguage,
   getContentLanguages,
@@ -72,6 +74,35 @@ export class CustomFeedAPI implements FeedAPI {
         )
       : await loggedOutFetch({...this.params, cursor, limit})
     if (res.success) {
+      if (this.params.feed === DISCOVER_FEED_URI) {
+        // Try not to show a post in Discover if you've seen it elsewhere in the app.
+        // A proper way to do this might be to let a feed declare it wants client-side filtering.
+        // I'm hacking this in here because it needs to be consistent for re-renders so
+        // the decision on what got included or not for this page needs to happen once.
+        const candidateFeed = res.data.feed.filter(post => {
+          if (isLikelyGloballySeenPost(post.post.uri)) {
+            return false
+          }
+          if (post.reply) {
+            if (
+              AppBskyFeedDefs.isPostView(post.reply.parent) &&
+              isLikelyGloballySeenPost(post.reply.parent.uri)
+            ) {
+              return false
+            }
+            if (
+              AppBskyFeedDefs.isPostView(post.reply.root) &&
+              isLikelyGloballySeenPost(post.reply.root.uri)
+            ) {
+              return false
+            }
+          }
+          return true
+        })
+        if (candidateFeed.length >= res.data.feed.length / 2) {
+          res.data.feed = candidateFeed
+        }
+      }
       // NOTE
       // some custom feeds fail to enforce the pagination limit
       // so we manually truncate here

src/state/feed-feedback.tsx

@@ -1,6 +1,8 @@
 import React from 'react'
 import {AppState, AppStateStatus} from 'react-native'
 import {AppBskyFeedDefs} from '@atproto/api'
+// @ts-ignore
+import {BloomFilter} from 'bloomfilter'
 import throttle from 'lodash.throttle'
 
 import {PROD_DEFAULT_FEED} from '#/lib/constants'
@@ -93,24 +95,25 @@ export function useFeedFeedback(feed: FeedDescriptor, hasSession: boolean) {
 
   const onItemSeen = React.useCallback(
     (feedItem: any) => {
-      if (!enabled) {
-        return
-      }
       const slice = getFeedPostSlice(feedItem)
       if (slice === null) {
         return
       }
       for (const postItem of slice.items) {
-        if (!history.current.has(postItem)) {
+        markGloballySeenPost(postItem.uri)
-          history.current.add(postItem)
+
-          queue.current.add(
+        if (enabled) {
-            toString({
+          if (!history.current.has(postItem)) {
-              item: postItem.uri,
+            history.current.add(postItem)
-              event: 'app.bsky.feed.defs#interactionSeen',
+            queue.current.add(
-              feedContext: slice.feedContext,
+              toString({
-            }),
+                item: postItem.uri,
-          )
+                event: 'app.bsky.feed.defs#interactionSeen',
-          sendToFeed()
+                feedContext: slice.feedContext,
+              }),
+            )
+            sendToFeed()
+          }
         }
       }
     },
@@ -254,3 +257,22 @@ function flushToStatsig(stats: AggregatedStats | null) {
     stats.seenCount = 0
   }
 }
+
+// https://hur.st/bloomfilter/?n=50k&p=0.001&m=&k=
+const p = 0.001 // 0.1% probability of collisions...
+const n = 50_000 // ...while we stay below 50k expected items
+const m = Math.ceil((n * Math.log(p)) / Math.log(1 / Math.pow(2, Math.log(2))))
+const k = Math.round((m / n) * Math.log(2))
+let globalBloomFilter = new BloomFilter(m, k)
+
+function markGloballySeenPost(uri: string) {
+  if (globalBloomFilter.size >= n) {
+    // If we ever get here, just restart to avoid saturation.
+    globalBloomFilter = new BloomFilter(m, k)
+  }
+  globalBloomFilter.add(uri)
+}
+
+export function isLikelyGloballySeenPost(uri: string) {
+  return globalBloomFilter.test(uri)
+}

yarn.lock

@@ -8200,6 +8200,11 @@ bl@^4.0.3, bl@^4.1.0:
     inherits "^2.0.4"
     readable-stream "^3.4.0"
 
+bloomfilter@^0.0.18:
+  version "0.0.18"
+  resolved "https://registry.yarnpkg.com/bloomfilter/-/bloomfilter-0.0.18.tgz#6d55d34f0a214b235287b4eac9203ac623413dab"
+  integrity sha512-CbnyHE78gY1tpXS/Ap+B0RJxKdRWCDzjBnX97UJSG8rdLv1PK8GiTWc/CCQyWu6PWVD4lUceeFrqC6Mf3nMgOA==
+
 bn.js@^4.0.0, bn.js@^4.11.8, bn.js@^4.11.9:
   version "4.12.0"
   resolved "https://registry.yarnpkg.com/bn.js/-/bn.js-4.12.0.tgz#775b3f278efbb9718eec7361f483fb36fbbfea88"