Use composer state as source of truth for embeds/links on publish (#5606)

Co-authored-by: Mary <git@mary.my.id>
Co-authored-by: Hailey <me@haileyok.com>

src/lib/api/index.ts

@@ -6,8 +6,10 @@ import {
   AppBskyEmbedVideo,
   AppBskyFeedPostgate,
   AtUri,
+  BlobRef,
   BskyAgent,
   ComAtprotoLabelDefs,
+  ComAtprotoRepoStrongRef,
   RichText,
 } from '@atproto/api'
 
@@ -22,8 +24,10 @@ import {
   threadgateAllowUISettingToAllowRecordValue,
   writeThreadgateRecord,
 } from '#/state/queries/threadgate'
-import {ComposerState} from '#/view/com/composer/state/composer'
+import {ComposerState, EmbedDraft} from '#/view/com/composer/state/composer'
+import {createGIFDescription} from '../gif-alt-text'
 import {LinkMeta} from '../link-meta/link-meta'
+import {resolveGif, resolveLink} from './resolve'
 import {uploadBlob} from './upload-blob'
 
 export {uploadBlob}
@@ -40,11 +44,6 @@ interface PostOpts {
   composerState: ComposerState // TODO: Not used yet.
   rawText: string
   replyTo?: string
-  quote?: {
-    uri: string
-    cid: string
-  }
-  extLink?: ExternalEmbedDraft
   labels?: string[]
   threadgate: ThreadgateAllowUISetting[]
   postgate: AppBskyFeedPostgate.Record
@@ -63,7 +62,11 @@ export async function post(agent: BskyAgent, opts: PostOpts) {
   rt = shortenLinks(rt)
   rt = stripInvalidMentions(rt)
 
-  const embed = await resolveEmbed(agent, opts)
+  const embed = await resolveEmbed(
+    agent,
+    opts.composerState,
+    opts.onStateChange,
+  )
 
   // add replyTo if post is a reply to another post
   if (opts.replyTo) {
@@ -175,7 +178,8 @@ export async function post(agent: BskyAgent, opts: PostOpts) {
 
 async function resolveEmbed(
   agent: BskyAgent,
-  opts: PostOpts,
+  draft: ComposerState,
+  onStateChange: ((state: string) => void) | undefined,
 ): Promise<
   | AppBskyEmbedImages.Main
   | AppBskyEmbedVideo.Main
@@ -184,52 +188,60 @@ async function resolveEmbed(
   | AppBskyEmbedRecordWithMedia.Main
   | undefined
 > {
-  const media = await resolveMedia(agent, opts)
+  if (draft.embed.quote) {
-  if (opts.quote) {
+    const [resolvedMedia, resolvedQuote] = await Promise.all([
-    const quoteRecord = {
+      resolveMedia(agent, draft.embed, onStateChange),
-      $type: 'app.bsky.embed.record',
+      resolveRecord(agent, draft.embed.quote.uri),
-      record: {
+    ])
-        uri: opts.quote.uri,
+    if (resolvedMedia) {
-        cid: opts.quote.cid,
-      },
-    }
-    if (media) {
       return {
         $type: 'app.bsky.embed.recordWithMedia',
-        record: quoteRecord,
+        record: {
-        media,
+          $type: 'app.bsky.embed.record',
+          record: resolvedQuote,
+        },
+        media: resolvedMedia,
       }
-    } else {
+    }
-      return quoteRecord
+    return {
+      $type: 'app.bsky.embed.record',
+      record: resolvedQuote,
     }
   }
-  if (media) {
+  const resolvedMedia = await resolveMedia(agent, draft.embed, onStateChange)
-    return media
+  if (resolvedMedia) {
+    return resolvedMedia
   }
-  if (opts.extLink?.embed) {
+  if (draft.embed.link) {
-    return opts.extLink.embed
+    const resolvedLink = await resolveLink(agent, draft.embed.link.uri)
+    if (resolvedLink.type === 'record') {
+      return {
+        $type: 'app.bsky.embed.record',
+        record: resolvedLink.record,
+      }
+    }
   }
   return undefined
 }
 
 async function resolveMedia(
   agent: BskyAgent,
-  opts: PostOpts,
+  embedDraft: EmbedDraft,
+  onStateChange: ((state: string) => void) | undefined,
 ): Promise<
   | AppBskyEmbedExternal.Main
   | AppBskyEmbedImages.Main
   | AppBskyEmbedVideo.Main
   | undefined
 > {
-  const state = opts.composerState
+  if (embedDraft.media?.type === 'images') {
-  const media = state.embed.media
+    const imagesDraft = embedDraft.media.images
-  if (media?.type === 'images') {
     logger.debug(`Uploading images`, {
-      count: media.images.length,
+      count: imagesDraft.length,
     })
-    opts.onStateChange?.(`Uploading images...`)
+    onStateChange?.(`Uploading images...`)
     const images: AppBskyEmbedImages.Image[] = await Promise.all(
-      media.images.map(async (image, i) => {
+      imagesDraft.map(async (image, i) => {
         logger.debug(`Compressing image #${i}`)
         const {path, width, height, mime} = await compressImage(image)
         logger.debug(`Uploading image #${i}`)
@@ -246,10 +258,13 @@ async function resolveMedia(
       images,
     }
   }
-  if (media?.type === 'video' && media.video.status === 'done') {
+  if (
-    const video = media.video
+    embedDraft.media?.type === 'video' &&
+    embedDraft.media.video.status === 'done'
+  ) {
+    const videoDraft = embedDraft.media.video
     const captions = await Promise.all(
-      video.captions
+      videoDraft.captions
         .filter(caption => caption.lang !== '')
         .map(async caption => {
           const {data} = await agent.uploadBlob(caption.file, {
@@ -260,36 +275,66 @@ async function resolveMedia(
     )
     return {
       $type: 'app.bsky.embed.video',
-      video: video.pendingPublish.blobRef,
+      video: videoDraft.pendingPublish.blobRef,
-      alt: video.altText || undefined,
+      alt: videoDraft.altText || undefined,
       captions: captions.length === 0 ? undefined : captions,
       aspectRatio: {
-        width: video.asset.width,
+        width: videoDraft.asset.width,
-        height: video.asset.height,
+        height: videoDraft.asset.height,
       },
     }
   }
-  if (opts.extLink) {
+  if (embedDraft.media?.type === 'gif') {
-    // TODO: Read this from composer state as well.
+    const gifDraft = embedDraft.media
-    if (opts.extLink.embed) {
+    const resolvedGif = await resolveGif(agent, gifDraft.gif)
-      return undefined
+    let blob: BlobRef | undefined
-    }
+    if (resolvedGif.thumb) {
-    let thumb
+      onStateChange?.('Uploading link thumbnail...')
-    if (opts.extLink.localThumb) {
+      const {path, mime} = resolvedGif.thumb.source
-      opts.onStateChange?.('Uploading link thumbnail...')
+      const response = await uploadBlob(agent, path, mime)
-      const {path, mime} = opts.extLink.localThumb.source
+      blob = response.data.blob
-      const res = await uploadBlob(agent, path, mime)
-      thumb = res.data.blob
     }
     return {
       $type: 'app.bsky.embed.external',
       external: {
-        uri: opts.extLink.uri,
+        uri: resolvedGif.uri,
-        title: opts.extLink.meta?.title || '',
+        title: resolvedGif.title,
-        description: opts.extLink.meta?.description || '',
+        description: createGIFDescription(resolvedGif.title, gifDraft.alt),
-        thumb,
+        thumb: blob,
       },
     }
   }
+  if (embedDraft.link) {
+    const resolvedLink = await resolveLink(agent, embedDraft.link.uri)
+    if (resolvedLink.type === 'external') {
+      let blob: BlobRef | undefined
+      if (resolvedLink.thumb) {
+        onStateChange?.('Uploading link thumbnail...')
+        const {path, mime} = resolvedLink.thumb.source
+        const response = await uploadBlob(agent, path, mime)
+        blob = response.data.blob
+      }
+      return {
+        $type: 'app.bsky.embed.external',
+        external: {
+          uri: resolvedLink.uri,
+          title: resolvedLink.title,
+          description: resolvedLink.description,
+          thumb: blob,
+        },
+      }
+    }
+  }
   return undefined
 }
+
+async function resolveRecord(
+  agent: BskyAgent,
+  uri: string,
+): Promise<ComAtprotoRepoStrongRef.Main> {
+  const resolvedLink = await resolveLink(agent, uri)
+  if (resolvedLink.type !== 'record') {
+    throw Error('Expected uri to resolve to a record')
+  }
+  return resolvedLink.record
+}

src/lib/api/resolve.ts

@@ -0,0 +1,161 @@
+import {ComAtprotoRepoStrongRef} from '@atproto/api'
+import {AtUri} from '@atproto/api'
+import {BskyAgent} from '@atproto/api'
+
+import {POST_IMG_MAX} from '#/lib/constants'
+import {
+  getFeedAsEmbed,
+  getListAsEmbed,
+  getPostAsQuote,
+  getStarterPackAsEmbed,
+} from '#/lib/link-meta/bsky'
+import {getLinkMeta} from '#/lib/link-meta/link-meta'
+import {resolveShortLink} from '#/lib/link-meta/resolve-short-link'
+import {downloadAndResize} from '#/lib/media/manip'
+import {
+  isBskyCustomFeedUrl,
+  isBskyListUrl,
+  isBskyPostUrl,
+  isBskyStarterPackUrl,
+  isBskyStartUrl,
+  isShortLink,
+} from '#/lib/strings/url-helpers'
+import {ComposerImage} from '#/state/gallery'
+import {createComposerImage} from '#/state/gallery'
+import {Gif} from '#/state/queries/tenor'
+import {createGIFDescription} from '../gif-alt-text'
+
+type ResolvedExternalLink = {
+  type: 'external'
+  uri: string
+  title: string
+  description: string
+  thumb: ComposerImage | undefined
+}
+
+type ResolvedRecord = {
+  type: 'record'
+  record: ComAtprotoRepoStrongRef.Main
+}
+
+type ResolvedLink = ResolvedExternalLink | ResolvedRecord
+
+export async function resolveLink(
+  agent: BskyAgent,
+  uri: string,
+): Promise<ResolvedLink> {
+  if (isShortLink(uri)) {
+    uri = await resolveShortLink(uri)
+  }
+  if (isBskyPostUrl(uri)) {
+    // TODO: Remove this abstraction.
+    // TODO: Nice error messages (e.g. EmbeddingDisabledError).
+    const result = await getPostAsQuote(getPost, uri)
+    return {
+      type: 'record',
+      record: {
+        cid: result.cid,
+        uri: result.uri,
+      },
+    }
+  }
+  if (isBskyCustomFeedUrl(uri)) {
+    // TODO: Remove this abstraction.
+    const result = await getFeedAsEmbed(agent, fetchDid, uri)
+    return {
+      type: 'record',
+      record: result.embed!.record, // TODO: Fix types.
+    }
+  }
+  if (isBskyListUrl(uri)) {
+    // TODO: Remove this abstraction.
+    const result = await getListAsEmbed(agent, fetchDid, uri)
+    return {
+      type: 'record',
+      record: result.embed!.record, // TODO: Fix types.
+    }
+  }
+  if (isBskyStartUrl(uri) || isBskyStarterPackUrl(uri)) {
+    // TODO: Remove this abstraction.
+    const result = await getStarterPackAsEmbed(agent, fetchDid, uri)
+    return {
+      type: 'record',
+      record: result.embed!.record, // TODO: Fix types.
+    }
+  }
+  return resolveExternal(agent, uri)
+
+  // Forked from useGetPost. TODO: move into RQ.
+  async function getPost({uri}: {uri: string}) {
+    const urip = new AtUri(uri)
+    if (!urip.host.startsWith('did:')) {
+      const res = await agent.resolveHandle({
+        handle: urip.host,
+      })
+      urip.host = res.data.did
+    }
+    const res = await agent.getPosts({
+      uris: [urip.toString()],
+    })
+    if (res.success && res.data.posts[0]) {
+      return res.data.posts[0]
+    }
+    throw new Error('getPost: post not found')
+  }
+
+  // Forked from useFetchDid. TODO: move into RQ.
+  async function fetchDid(handleOrDid: string) {
+    let identifier = handleOrDid
+    if (!identifier.startsWith('did:')) {
+      const res = await agent.resolveHandle({handle: identifier})
+      identifier = res.data.did
+    }
+    return identifier
+  }
+}
+
+export async function resolveGif(
+  agent: BskyAgent,
+  gif: Gif,
+): Promise<ResolvedExternalLink> {
+  const uri = `${gif.media_formats.gif.url}?hh=${gif.media_formats.gif.dims[1]}&ww=${gif.media_formats.gif.dims[0]}`
+  return {
+    type: 'external',
+    uri,
+    title: gif.content_description,
+    description: createGIFDescription(gif.content_description),
+    thumb: await imageToThumb(gif.media_formats.preview.url),
+  }
+}
+
+async function resolveExternal(
+  agent: BskyAgent,
+  uri: string,
+): Promise<ResolvedExternalLink> {
+  const result = await getLinkMeta(agent, uri)
+  return {
+    type: 'external',
+    uri: result.url,
+    title: result.title ?? '',
+    description: result.description ?? '',
+    thumb: result.image ? await imageToThumb(result.image) : undefined,
+  }
+}
+
+async function imageToThumb(
+  imageUri: string,
+): Promise<ComposerImage | undefined> {
+  try {
+    const img = await downloadAndResize({
+      uri: imageUri,
+      width: POST_IMG_MAX.width,
+      height: POST_IMG_MAX.height,
+      mode: 'contain',
+      maxSize: POST_IMG_MAX.size,
+      timeout: 15e3,
+    })
+    if (img) {
+      return await createComposerImage(img)
+    }
+  } catch {}
+}

src/view/com/composer/Composer.tsx

@@ -425,8 +425,6 @@ export const ComposePost = ({
             composerState, // TODO: move more state here.
             rawText: richtext.text,
             replyTo: replyTo?.uri,
-            quote,
-            extLink,
             labels,
             threadgate: threadgateAllowUISettings,
             postgate,

src/view/com/composer/state/composer.ts

@@ -1,6 +1,10 @@
 import {ImagePickerAsset} from 'expo-image-picker'
 
-import {isBskyPostUrl} from '#/lib/strings/url-helpers'
+import {
+  isBskyPostUrl,
+  postUriToRelativePath,
+  toBskyAppUrl,
+} from '#/lib/strings/url-helpers'
 import {ComposerImage, createInitialImages} from '#/state/gallery'
 import {Gif} from '#/state/queries/tenor'
 import {ComposerOpts} from '#/state/shell/composer'
@@ -29,7 +33,7 @@ type Link = {
 
 // This structure doesn't exactly correspond to the data model.
 // Instead, it maps to how the UI is organized, and how we present a post.
-type EmbedDraft = {
+export type EmbedDraft = {
   // We'll always submit quote and actual media (images, video, gifs) chosen by the user.
   quote: Link | undefined
   media: ImagesMedia | VideoMedia | GifMedia | undefined
@@ -304,9 +308,13 @@ export function createComposerState({
   }
   let quote: Link | undefined
   if (initQuoteUri) {
-    quote = {
+    // TODO: Consider passing the app url directly.
-      type: 'link',
+    const path = postUriToRelativePath(initQuoteUri)
-      uri: initQuoteUri,
+    if (path) {
+      quote = {
+        type: 'link',
+        uri: toBskyAppUrl(path),
+      }
     }
   }
   // TODO: Other initial content.