Fix pagination (#11448)

Co-authored-by: Eric Bailey <git@esb.lol>

src/lib/api/feed/custom.test.ts

@@ -0,0 +1,40 @@
+import {type Client} from '@atproto/lex'
+
+import {CustomFeedAPI} from './custom'
+
+jest.mock('#/state/preferences/languages', () => ({
+  getAppLanguageAsContentLanguage: () => '',
+  getContentLanguages: () => [],
+}))
+
+jest.mock('./utils', () => ({
+  createBskyTopicsHeader: () => ({}),
+  isBlueskyOwnedFeed: () => false,
+}))
+
+describe('CustomFeedAPI', () => {
+  it('preserves the cursor from an empty logged-out fallback page', async () => {
+    const originalFetch = global.fetch
+    const fetchMock: jest.MockedFunction<typeof fetch> = jest
+      .fn()
+      .mockResolvedValueOnce(Response.json({feed: []}))
+      .mockResolvedValueOnce(Response.json({feed: [], cursor: 'next'}))
+    global.fetch = fetchMock
+    const api = new CustomFeedAPI({
+      client: {did: undefined} as unknown as Client,
+      feedParams: {
+        feed: 'at://did:example:feed/app.bsky.feed.generator/test',
+      },
+    })
+
+    try {
+      await expect(api.fetch({cursor: undefined, limit: 10})).resolves.toEqual({
+        cursor: 'next',
+        feed: [],
+      })
+      expect(fetchMock).toHaveBeenCalledTimes(2)
+    } finally {
+      global.fetch = originalFetch
+    }
+  })
+})

src/lib/api/feed/custom.ts

@@ -96,7 +96,7 @@ export class CustomFeedAPI implements FeedAPI {
     const feed =
       data.feed.length > limit ? data.feed.slice(0, limit) : data.feed
     return {
-      cursor: feed.length ? data.cursor : undefined,
+      cursor: data.cursor,
       feed,
     }
   }
@@ -154,11 +154,7 @@ async function loggedOutFetch(
 
   // no data, try again with language headers removed
   data = await getFeedOrNull(params, '')
-  if (data?.feed?.length) {
-    return data
-  }
-
-  return null
+  return data
 }
 
 /**

src/lib/api/feed/likes.ts

@@ -48,10 +48,8 @@ export class LikesFeedAPI implements FeedAPI {
       cursor,
       limit,
     })
-    // HACKFIX: the API incorrectly returns a cursor when there are no items -sfn
-    const isEmptyPage = data.feed.length === 0
     return {
-      cursor: isEmptyPage ? undefined : data.cursor,
+      cursor: data.cursor,
       feed: data.feed,
     }
   }

src/lib/api/feed/merge.test.ts

@@ -0,0 +1,106 @@
+import {type Client} from '@atproto/lex'
+
+import {app} from '#/lexicons'
+import {MergeFeedAPI} from './merge'
+
+const post = {} as app.bsky.feed.defs.FeedViewPost
+
+describe('MergeFeedAPI', () => {
+  it('drains a terminal following queue without restarting the source', async () => {
+    const api = new MergeFeedAPI({
+      client: {} as Client,
+      feedParams: {},
+      feedTuners: [],
+    })
+    api.reset()
+    api.following.queue = [post, post, post]
+    api.following.hasMore = false
+
+    const first = await api.fetch({cursor: 'started', limit: 2})
+    const second = await api.fetch({cursor: first.cursor, limit: 2})
+
+    expect(first.feed).toHaveLength(2)
+    expect(first.cursor).toBeDefined()
+    expect(second.feed).toHaveLength(1)
+    expect(second.cursor).toBeUndefined()
+  })
+
+  it('stops when the following source repeats its cursor', async () => {
+    const call = jest
+      .fn()
+      .mockResolvedValueOnce({feed: [], cursor: 'a'})
+      .mockResolvedValueOnce({feed: [], cursor: 'a'})
+    const api = new MergeFeedAPI({
+      client: {call} as unknown as Client,
+      feedParams: {},
+      feedTuners: [],
+    })
+
+    const first = await api.fetch({cursor: undefined, limit: 1})
+    const second = await api.fetch({cursor: first.cursor, limit: 1})
+
+    expect(first.cursor).toBeDefined()
+    expect(second.cursor).toBeUndefined()
+    expect(call).toHaveBeenCalledTimes(2)
+  })
+
+  it('stops when the following source returns an empty cursor', async () => {
+    const call = jest.fn().mockResolvedValue({feed: [], cursor: ''})
+    const api = new MergeFeedAPI({
+      client: {call} as unknown as Client,
+      feedParams: {},
+      feedTuners: [],
+    })
+
+    const result = await api.fetch({cursor: undefined, limit: 1})
+
+    expect(result.cursor).toBeUndefined()
+    expect(call).toHaveBeenCalledTimes(1)
+  })
+
+  it('stops when the following source cycles to an earlier cursor', async () => {
+    const call = jest
+      .fn()
+      .mockResolvedValueOnce({feed: [], cursor: 'a'})
+      .mockResolvedValueOnce({feed: [], cursor: 'b'})
+      .mockResolvedValueOnce({feed: [], cursor: 'a'})
+    const api = new MergeFeedAPI({
+      client: {call} as unknown as Client,
+      feedParams: {},
+      feedTuners: [],
+    })
+
+    const first = await api.fetch({cursor: undefined, limit: 1})
+    const second = await api.fetch({cursor: first.cursor, limit: 1})
+    const third = await api.fetch({cursor: second.cursor, limit: 1})
+
+    expect(third.cursor).toBeUndefined()
+    expect(call).toHaveBeenCalledTimes(3)
+  })
+
+  it('drains a terminal custom-feed queue without restarting the source', async () => {
+    const call = jest.fn()
+    const api = new MergeFeedAPI({
+      client: {call} as unknown as Client,
+      feedParams: {
+        mergeFeedEnabled: true,
+        mergeFeedSources: [
+          'at://did:example:feed/app.bsky.feed.generator/test',
+        ],
+      },
+      feedTuners: [],
+    })
+    api.reset()
+    api.following.hasMore = false
+    api.customFeeds[0].queue = [post, post, post]
+    api.customFeeds[0].hasMore = false
+
+    const first = await api.fetch({cursor: 'started', limit: 2})
+    const second = await api.fetch({cursor: first.cursor, limit: 2})
+
+    expect(first.feed).toHaveLength(2)
+    expect(second.feed).toHaveLength(1)
+    expect(second.cursor).toBeUndefined()
+    expect(call).not.toHaveBeenCalled()
+  })
+})

src/lib/api/feed/merge.ts

@@ -108,8 +108,8 @@ export class MergeFeedAPI implements FeedAPI {
 
     const promises = []
 
-    // always keep following topped up
-    if (this.following.numReady < limit) {
+    // always keep following topped up while the source has another page
+    if (this.following.hasMore && this.following.numReady < limit) {
       await this.following.fetchNext(60)
     }
 
@@ -125,7 +125,7 @@ export class MergeFeedAPI implements FeedAPI {
       !this.following.hasMore && this.following.numReady < limit
     if (this.params.mergeFeedEnabled || outOfFollows) {
       for (const feed of feeds) {
-        if (feed.numReady < 5) {
+        if (feed.hasMore && feed.numReady < 5) {
           promises.push(feed.fetchNext(10))
         }
       }
@@ -145,8 +145,12 @@ export class MergeFeedAPI implements FeedAPI {
       }
     }
 
+    const hasMore =
+      this.following.hasMore ||
+      this.following.numReady > 0 ||
+      this.customFeeds.some(feed => feed.hasMore || feed.numReady > 0)
     return {
-      cursor: String(this.itemCursor),
+      cursor: hasMore ? String(this.itemCursor) : undefined,
       feed: posts,
     }
   }
@@ -155,8 +159,8 @@ export class MergeFeedAPI implements FeedAPI {
     const i = this.itemCursor++
     const candidateFeeds = this.customFeeds.filter(f => f.numReady > 0)
     const canSample = candidateFeeds.length > 0
-    const hasFollows = this.following.hasMore
     const hasFollowsReady = this.following.numReady > 0
+    const hasFollows = this.following.hasMore || hasFollowsReady
 
     // this condition establishes the frequency that custom feeds are woven into follows
     const shouldSample =
@@ -187,6 +191,7 @@ class MergeFeedSource {
   feedTuners: FeedTunerFn[]
   sourceInfo: ReasonFeedSource | undefined
   cursor: string | undefined = undefined
+  seenCursors = new Set<string>()
   queue: app.bsky.feed.defs.FeedViewPost[] = []
   hasMore = true
 
@@ -221,11 +226,16 @@ class MergeFeedSource {
     const page = await this._getFeed(this.cursor, n)
     if (page) {
       this.cursor = page.cursor
-      if (page.feed.length) {
-        this.queue = this.queue.concat(page.feed)
+      const cursor = this.cursor
+      if (cursor) {
+        this.hasMore = !this.seenCursors.has(cursor)
+        this.seenCursors.add(cursor)
       } else {
         this.hasMore = false
       }
+      if (page.feed.length) {
+        this.queue = this.queue.concat(page.feed)
+      }
     } else {
       this.hasMore = false
     }