[Explore] Add shadow cache (#8119)

* enable shadowcache for feed previews

* extract keyextractor

* dedupe feeds

src/screens/Search/Explore.tsx

@@ -15,6 +15,10 @@ import {logger} from '#/logger'
 import {type MetricEvents} from '#/logger/metrics'
 import {useModerationOpts} from '#/state/preferences/moderation-opts'
 import {useActorSearchPaginated} from '#/state/queries/actor-search'
+import {
+  type FeedPreviewItem,
+  useFeedPreviews,
+} from '#/state/queries/explore-feed-previews'
 import {useGetPopularFeedsQuery} from '#/state/queries/feed'
 import {usePreferencesQuery} from '#/state/queries/preferences'
 import {useSuggestedFollowsQuery} from '#/state/queries/suggested-follows'
@@ -47,10 +51,6 @@ import {Loader} from '#/components/Loader'
 import * as ProfileCard from '#/components/ProfileCard'
 import {Text} from '#/components/Typography'
 import * as ModuleHeader from './components/ModuleHeader'
-import {
-  type FeedPreviewItem,
-  useFeedPreviews,
-} from './modules/ExploreFeedPreviews'
 import {
   SuggestedAccountsTabBar,
   SuggestedProfileCard,
@@ -900,7 +900,7 @@ export function Explore({
     <List
       data={items}
       renderItem={renderItem}
-      keyExtractor={item => item.key}
+      keyExtractor={keyExtractor}
       desktopFixedHeight
       contentContainerStyle={{paddingBottom: 100}}
       keyboardShouldPersistTaps="handled"
@@ -914,6 +914,10 @@ export function Explore({
   )
 }
 
+function keyExtractor(item: FeedPreviewItem) {
+  return item.key
+}
+
 const viewabilityConfig: ViewabilityConfig = {
   itemVisiblePercentThreshold: 100,
 }

src/state/cache/post-shadow.ts

@@ -2,18 +2,19 @@ import {useEffect, useMemo, useState} from 'react'
 import {
   AppBskyEmbedRecord,
   AppBskyEmbedRecordWithMedia,
-  AppBskyFeedDefs,
+  type AppBskyFeedDefs,
 } from '@atproto/api'
-import {QueryClient} from '@tanstack/react-query'
+import {type QueryClient} from '@tanstack/react-query'
 import EventEmitter from 'eventemitter3'
 
 import {batchedUpdates} from '#/lib/batchedUpdates'
-import {findAllPostsInQueryData as findAllPostsInNotifsQueryData} from '../queries/notifications/feed'
+import {findAllPostsInQueryData as findAllPostsInExploreFeedPreviewsQueryData} from '#/state/queries/explore-feed-previews'
-import {findAllPostsInQueryData as findAllPostsInFeedQueryData} from '../queries/post-feed'
+import {findAllPostsInQueryData as findAllPostsInNotifsQueryData} from '#/state/queries/notifications/feed'
-import {findAllPostsInQueryData as findAllPostsInQuoteQueryData} from '../queries/post-quotes'
+import {findAllPostsInQueryData as findAllPostsInFeedQueryData} from '#/state/queries/post-feed'
-import {findAllPostsInQueryData as findAllPostsInThreadQueryData} from '../queries/post-thread'
+import {findAllPostsInQueryData as findAllPostsInQuoteQueryData} from '#/state/queries/post-quotes'
-import {findAllPostsInQueryData as findAllPostsInSearchQueryData} from '../queries/search-posts'
+import {findAllPostsInQueryData as findAllPostsInThreadQueryData} from '#/state/queries/post-thread'
-import {castAsShadow, Shadow} from './types'
+import {findAllPostsInQueryData as findAllPostsInSearchQueryData} from '#/state/queries/search-posts'
+import {castAsShadow, type Shadow} from './types'
 export type {Shadow} from './types'
 
 export interface PostShadow {
@@ -154,4 +155,10 @@ function* findPostsInCache(
   for (let post of findAllPostsInQuoteQueryData(queryClient, uri)) {
     yield post
   }
+  for (let post of findAllPostsInExploreFeedPreviewsQueryData(
+    queryClient,
+    uri,
+  )) {
+    yield post
+  }
 }

src/state/cache/profile-shadow.ts

@@ -1,25 +1,26 @@
 import {useEffect, useMemo, useState} from 'react'
-import {QueryClient} from '@tanstack/react-query'
+import {type QueryClient} from '@tanstack/react-query'
 import EventEmitter from 'eventemitter3'
 
 import {batchedUpdates} from '#/lib/batchedUpdates'
-import * as bsky from '#/types/bsky'
+import {findAllProfilesInQueryData as findAllProfilesInActorSearchQueryData} from '#/state/queries/actor-search'
-import {findAllProfilesInQueryData as findAllProfilesInActorSearchQueryData} from '../queries/actor-search'
+import {findAllProfilesInQueryData as findAllProfilesInExploreFeedPreviewsQueryData} from '#/state/queries/explore-feed-previews'
-import {findAllProfilesInQueryData as findAllProfilesInKnownFollowersQueryData} from '../queries/known-followers'
+import {findAllProfilesInQueryData as findAllProfilesInKnownFollowersQueryData} from '#/state/queries/known-followers'
-import {findAllProfilesInQueryData as findAllProfilesInListMembersQueryData} from '../queries/list-members'
+import {findAllProfilesInQueryData as findAllProfilesInListMembersQueryData} from '#/state/queries/list-members'
-import {findAllProfilesInQueryData as findAllProfilesInListConvosQueryData} from '../queries/messages/list-conversations'
+import {findAllProfilesInQueryData as findAllProfilesInListConvosQueryData} from '#/state/queries/messages/list-conversations'
-import {findAllProfilesInQueryData as findAllProfilesInMyBlockedAccountsQueryData} from '../queries/my-blocked-accounts'
+import {findAllProfilesInQueryData as findAllProfilesInMyBlockedAccountsQueryData} from '#/state/queries/my-blocked-accounts'
-import {findAllProfilesInQueryData as findAllProfilesInMyMutedAccountsQueryData} from '../queries/my-muted-accounts'
+import {findAllProfilesInQueryData as findAllProfilesInMyMutedAccountsQueryData} from '#/state/queries/my-muted-accounts'
-import {findAllProfilesInQueryData as findAllProfilesInFeedsQueryData} from '../queries/post-feed'
+import {findAllProfilesInQueryData as findAllProfilesInFeedsQueryData} from '#/state/queries/post-feed'
-import {findAllProfilesInQueryData as findAllProfilesInPostLikedByQueryData} from '../queries/post-liked-by'
+import {findAllProfilesInQueryData as findAllProfilesInPostLikedByQueryData} from '#/state/queries/post-liked-by'
-import {findAllProfilesInQueryData as findAllProfilesInPostQuotesQueryData} from '../queries/post-quotes'
+import {findAllProfilesInQueryData as findAllProfilesInPostQuotesQueryData} from '#/state/queries/post-quotes'
-import {findAllProfilesInQueryData as findAllProfilesInPostRepostedByQueryData} from '../queries/post-reposted-by'
+import {findAllProfilesInQueryData as findAllProfilesInPostRepostedByQueryData} from '#/state/queries/post-reposted-by'
-import {findAllProfilesInQueryData as findAllProfilesInPostThreadQueryData} from '../queries/post-thread'
+import {findAllProfilesInQueryData as findAllProfilesInPostThreadQueryData} from '#/state/queries/post-thread'
-import {findAllProfilesInQueryData as findAllProfilesInProfileQueryData} from '../queries/profile'
+import {findAllProfilesInQueryData as findAllProfilesInProfileQueryData} from '#/state/queries/profile'
-import {findAllProfilesInQueryData as findAllProfilesInProfileFollowersQueryData} from '../queries/profile-followers'
+import {findAllProfilesInQueryData as findAllProfilesInProfileFollowersQueryData} from '#/state/queries/profile-followers'
-import {findAllProfilesInQueryData as findAllProfilesInProfileFollowsQueryData} from '../queries/profile-follows'
+import {findAllProfilesInQueryData as findAllProfilesInProfileFollowsQueryData} from '#/state/queries/profile-follows'
-import {findAllProfilesInQueryData as findAllProfilesInSuggestedFollowsQueryData} from '../queries/suggested-follows'
+import {findAllProfilesInQueryData as findAllProfilesInSuggestedFollowsQueryData} from '#/state/queries/suggested-follows'
-import {castAsShadow, Shadow} from './types'
+import type * as bsky from '#/types/bsky'
+import {castAsShadow, type Shadow} from './types'
 
 export type {Shadow} from './types'
 
@@ -154,4 +155,5 @@ function* findProfilesInCache(
   yield* findAllProfilesInFeedsQueryData(queryClient, did)
   yield* findAllProfilesInPostThreadQueryData(queryClient, did)
   yield* findAllProfilesInKnownFollowersQueryData(queryClient, did)
+  yield* findAllProfilesInExploreFeedPreviewsQueryData(queryClient, did)
 }

src/state/queries/explore-feed-previews.tsx

@@ -1,8 +1,17 @@
 import {useMemo} from 'react'
-import {type AppBskyFeedDefs, moderatePost} from '@atproto/api'
+import {
+  type AppBskyActorDefs,
+  AppBskyFeedDefs,
+  AtUri,
+  moderatePost,
+} from '@atproto/api'
 import {msg} from '@lingui/macro'
 import {useLingui} from '@lingui/react'
-import {useInfiniteQuery} from '@tanstack/react-query'
+import {
+  type InfiniteData,
+  type QueryClient,
+  useInfiniteQuery,
+} from '@tanstack/react-query'
 
 import {CustomFeedAPI} from '#/lib/api/feed/custom'
 import {aggregateUserInterests} from '#/lib/api/feed/utils'
@@ -14,6 +23,11 @@ import {
   type FeedPostSliceItem,
 } from '#/state/queries/post-feed'
 import {usePreferencesQuery} from '#/state/queries/preferences'
+import {
+  didOrHandleUriMatches,
+  embedViewRecordToPostView,
+  getEmbeddedPost,
+} from '#/state/queries/util'
 import {useAgent} from '#/state/session'
 
 const RQKEY_ROOT = 'feed-previews'
@@ -68,7 +82,17 @@ export type FeedPreviewItem =
       uri: string
     }
 
-export function useFeedPreviews(feeds: AppBskyFeedDefs.GeneratorView[]) {
+export function useFeedPreviews(
+  feedsMaybeWithDuplicates: AppBskyFeedDefs.GeneratorView[],
+) {
+  const feeds = useMemo(
+    () =>
+      feedsMaybeWithDuplicates.filter(
+        (f, i, a) => i === a.findIndex(f2 => f.uri === f2.uri),
+      ),
+    [feedsMaybeWithDuplicates],
+  )
+
   const uris = feeds.map(feed => feed.uri)
   const {_} = useLingui()
   const agent = useAgent()
@@ -262,3 +286,103 @@ export function useFeedPreviews(feeds: AppBskyFeedDefs.GeneratorView[]) {
     ]),
   }
 }
+
+export function* findAllPostsInQueryData(
+  queryClient: QueryClient,
+  uri: string,
+): Generator<AppBskyFeedDefs.PostView, undefined> {
+  const atUri = new AtUri(uri)
+
+  const queryDatas = queryClient.getQueriesData<
+    InfiniteData<{
+      feed: AppBskyFeedDefs.GeneratorView
+      posts: AppBskyFeedDefs.FeedViewPost[]
+    }>
+  >({
+    queryKey: [RQKEY_ROOT],
+  })
+  for (const [_queryKey, queryData] of queryDatas) {
+    if (!queryData?.pages) {
+      continue
+    }
+    for (const page of queryData?.pages) {
+      for (const item of page.posts) {
+        if (didOrHandleUriMatches(atUri, item.post)) {
+          yield item.post
+        }
+
+        const quotedPost = getEmbeddedPost(item.post.embed)
+        if (quotedPost && didOrHandleUriMatches(atUri, quotedPost)) {
+          yield embedViewRecordToPostView(quotedPost)
+        }
+
+        if (AppBskyFeedDefs.isPostView(item.reply?.parent)) {
+          if (didOrHandleUriMatches(atUri, item.reply.parent)) {
+            yield item.reply.parent
+          }
+
+          const parentQuotedPost = getEmbeddedPost(item.reply.parent.embed)
+          if (
+            parentQuotedPost &&
+            didOrHandleUriMatches(atUri, parentQuotedPost)
+          ) {
+            yield embedViewRecordToPostView(parentQuotedPost)
+          }
+        }
+
+        if (AppBskyFeedDefs.isPostView(item.reply?.root)) {
+          if (didOrHandleUriMatches(atUri, item.reply.root)) {
+            yield item.reply.root
+          }
+
+          const rootQuotedPost = getEmbeddedPost(item.reply.root.embed)
+          if (rootQuotedPost && didOrHandleUriMatches(atUri, rootQuotedPost)) {
+            yield embedViewRecordToPostView(rootQuotedPost)
+          }
+        }
+      }
+    }
+  }
+}
+
+export function* findAllProfilesInQueryData(
+  queryClient: QueryClient,
+  did: string,
+): Generator<AppBskyActorDefs.ProfileViewBasic, undefined> {
+  const queryDatas = queryClient.getQueriesData<
+    InfiniteData<{
+      feed: AppBskyFeedDefs.GeneratorView
+      posts: AppBskyFeedDefs.FeedViewPost[]
+    }>
+  >({
+    queryKey: [RQKEY_ROOT],
+  })
+  for (const [_queryKey, queryData] of queryDatas) {
+    if (!queryData?.pages) {
+      continue
+    }
+    for (const page of queryData?.pages) {
+      for (const item of page.posts) {
+        if (item.post.author.did === did) {
+          yield item.post.author
+        }
+        const quotedPost = getEmbeddedPost(item.post.embed)
+        if (quotedPost?.author.did === did) {
+          yield quotedPost.author
+        }
+        if (
+          AppBskyFeedDefs.isPostView(item.reply?.parent) &&
+          item.reply?.parent?.author.did === did
+        ) {
+          yield item.reply.parent.author
+        }
+        if (
+          AppBskyFeedDefs.isPostView(item.reply?.root) &&
+          item.reply?.root?.author.did === did
+        ) {
+          yield item.reply.root.author
+        }
+      }
+    }
+  }
+}

src/state/queries/post-thread.ts

@@ -1,18 +1,22 @@
 import {
-  AppBskyActorDefs,
+  type AppBskyActorDefs,
-  AppBskyEmbedRecord,
+  type AppBskyEmbedRecord,
   AppBskyFeedDefs,
-  AppBskyFeedGetPostThread,
+  type AppBskyFeedGetPostThread,
   AppBskyFeedPost,
   AtUri,
   moderatePost,
-  ModerationDecision,
+  type ModerationDecision,
-  ModerationOpts,
+  type ModerationOpts,
 } from '@atproto/api'
-import {QueryClient, useQuery, useQueryClient} from '@tanstack/react-query'
+import {type QueryClient, useQuery, useQueryClient} from '@tanstack/react-query'
 
+import {
+  findAllPostsInQueryData as findAllPostsInExploreFeedPreviewsQueryData,
+  findAllProfilesInQueryData as findAllProfilesInExploreFeedPreviewsQueryData,
+} from '#/state/queries/explore-feed-previews'
 import {findAllPostsInQueryData as findAllPostsInQuoteQueryData} from '#/state/queries/post-quotes'
-import {UsePreferencesQueryResponse} from '#/state/queries/preferences/types'
+import {type UsePreferencesQueryResponse} from '#/state/queries/preferences/types'
 import {
   findAllPostsInQueryData as findAllPostsInSearchQueryData,
   findAllProfilesInQueryData as findAllProfilesInSearchQueryData,
@@ -495,6 +499,12 @@ export function* findAllPostsInQueryData(
   for (let post of findAllPostsInSearchQueryData(queryClient, uri)) {
     yield postViewToPlaceholderThread(post)
   }
+  for (let post of findAllPostsInExploreFeedPreviewsQueryData(
+    queryClient,
+    uri,
+  )) {
+    yield postViewToPlaceholderThread(post)
+  }
 }
 
 export function* findAllProfilesInQueryData(
@@ -529,6 +539,12 @@ export function* findAllProfilesInQueryData(
   for (let profile of findAllProfilesInSearchQueryData(queryClient, did)) {
     yield profile
   }
+  for (let profile of findAllProfilesInExploreFeedPreviewsQueryData(
+    queryClient,
+    did,
+  )) {
+    yield profile
+  }
 }
 
 function* traverseThread(node: ThreadNode): Generator<ThreadNode, void> {