diff --git a/package.json b/package.json index 510955abbf..5d92bc833f 100644 --- a/package.json +++ b/package.json @@ -333,11 +333,14 @@ "^multiformats/cid$": "/node_modules/multiformats/dist/src/cid.js", "^multiformats/bases/base32$": "/node_modules/multiformats/dist/src/bases/base32.js", "^multiformats/hashes/digest$": "/node_modules/multiformats/dist/src/hashes/digest.js", + "^multiformats/hashes/hasher$": "/node_modules/multiformats/dist/src/hashes/hasher.js", "^multiformats/hashes/sha2$": "/node_modules/multiformats/dist/src/hashes/sha2.js", "^uint8arrays/from-string$": "/node_modules/uint8arrays/dist/src/from-string.js", "^uint8arrays/to-string$": "/node_modules/uint8arrays/dist/src/to-string.js", "^unicode-segmenter/grapheme$": "/node_modules/unicode-segmenter/grapheme.cjs", - "^await-lock$": "/node_modules/await-lock/build/AwaitLock.js" + "^await-lock$": "/node_modules/await-lock/build/AwaitLock.js", + "^@ipld/dag-cbor$": "/node_modules/@ipld/dag-cbor/src/index.js", + "^cborg$": "/node_modules/cborg/cborg.js" }, "transformIgnorePatterns": [ "node_modules/(?!((jest-)?react-native|@react-native(-community)?)|expo(nent)?|@expo(nent)?/.*|@expo-google-fonts/.*|react-navigation|nanoid|@react-navigation/.*|@unimodules/.*|unimodules|sentry-expo|native-base|normalize-url|react-native-svg|@sentry/.*|sentry-expo|bcp-47-match|@atproto/.*|@bsky.app/sdk|tlds|multiformats|uint8arrays|@ipld/.*|cborg|await-lock)" diff --git a/src/lib/api/__tests__/computeCid.test.ts b/src/lib/api/__tests__/computeCid.test.ts new file mode 100644 index 0000000000..ce0adc4a85 --- /dev/null +++ b/src/lib/api/__tests__/computeCid.test.ts @@ -0,0 +1,121 @@ +/* + * The jest suite ships global manual mocks for `multiformats/cid` and + * `multiformats/hashes/hasher` (in root `__mocks__/`) so unrelated tests don't + * pull in real crypto. This test is precisely about the real CID hashing, so we + * opt back into the actual implementations here. + */ +jest.unmock('multiformats/cid') +jest.unmock('multiformats/hashes/hasher') + +import {BlobRef} from '@atproto/api' +import {CID} from 'multiformats/cid' + +import {computeCid} from '#/lib/api/computeCid' +import {type app, type com} from '#/lexicons' + +/* + * Golden-CID regression test for the composer post pipeline. + * + * `computeCid` hashes a post record in the client so a thread's later posts can + * reference earlier posts by CID before the server assigns them. The hash is + * byte-sensitive: any drift in how records (especially blobs) are serialized to + * DAG-CBOR silently produces the wrong CID and breaks reply chains with NO type + * error. These golden values MUST remain byte-identical - they gate the + * structural lex-blob shape check. + * + * The blob CID below is a fixed, deterministic CIDv1/raw/sha256 used purely as a + * stable fixture - it is not derived from any real upload. + */ +const BLOB_CID = 'bafkreieq5jui4j25lacwomsqgjeswwl3y5zcdrresptwgmfylxo2depppq' + +/** + * Build a post record with an image embed whose blob is the given value. + */ +function postWithImageBlob(blob: unknown): app.bsky.feed.post.Main { + return { + $type: 'app.bsky.feed.post', + createdAt: '2024-01-01T00:00:00.001Z', + text: 'post with image', + embed: { + $type: 'app.bsky.embed.images', + images: [ + { + image: blob, + alt: 'alt text', + aspectRatio: {width: 100, height: 200}, + }, + ], + }, + } as app.bsky.feed.post.Main +} + +describe('computeCid', () => { + it('case 1: plain post record with no blob', async () => { + const record: app.bsky.feed.post.Main = { + $type: 'app.bsky.feed.post', + createdAt: '2024-01-01T00:00:00.000Z', + text: 'hello world', + } + expect(await computeCid(record)).toBe( + 'bafyreieawtmh7hwfrqpamqkodza5r62bbfhsepe2iyustgxhgbhi6b2lfi', + ) + }) + + it('case 2: record whose embed carries a plain-JSON lex blob', async () => { + /* + * The blob shape lex `uploadBlob` returns: a plain object + * `{$type: 'blob', ref, mimeType, size}` with `ref` a parsed CID. The + * structural lex-blob guard passes it through `prepareForHashing` + * untouched and DAG-CBOR encodes its CID `ref` as a CID link. The golden + * CID below is the byte-identical value the pre-migration `BlobRef` class + * instance produced via `.ipld()`. + */ + const blob = { + $type: 'blob' as const, + ref: CID.parse(BLOB_CID), + mimeType: 'image/jpeg', + size: 12345, + } + expect(await computeCid(postWithImageBlob(blob))).toBe( + 'bafyreiem7g6vja66nebr7he4fshfnlyndyldbvle2n265oixscmepjcbii', + ) + }) + + it('case 2b: a legacy BlobRef instance hashes to the same CID', async () => { + /* + * The video pipeline still yields legacy `BlobRef` class instances, so + * `prepareForHashing` keeps its `instanceof` guard. Both branches must + * agree: this asserts the SAME golden CID as case 2. + */ + const blob = new BlobRef(CID.parse(BLOB_CID), 'image/jpeg', 12345) + expect(await computeCid(postWithImageBlob(blob))).toBe( + 'bafyreiem7g6vja66nebr7he4fshfnlyndyldbvle2n265oixscmepjcbii', + ) + }) + + it('case 3: three-post thread chains reply StrongRef CIDs', async () => { + const did = 'did:plc:abc123' + const base = new Date('2024-01-01T00:00:00.000Z') + const golden = [ + 'bafyreig62rxs34h5rvznfrracwkjlfgad5b25qxglp2hcziqdfas2nw2ee', + 'bafyreicxcj2tq5jrh5jcaczg3eli5cvxitgzu7kpu3fm5v3njq2byjxirq', + 'bafyreigvaswuhlpd7dllja2xrqswhqbruyv2kar7mvbn7gdm5ldzu6vkti', + ] + + let reply: app.bsky.feed.post.Main['reply'] | undefined + for (let i = 0; i < 3; i++) { + const now = new Date(base.getTime() + i) + const uri = `at://${did}/app.bsky.feed.post/rkey${i}` + const record = { + $type: 'app.bsky.feed.post', + createdAt: now.toISOString(), + text: `post ${i}`, + reply, + } as app.bsky.feed.post.Main + const cid = await computeCid(record) + expect(cid).toBe(golden[i]) + const ref = {cid, uri} as com.atproto.repo.strongRef.Main + reply = {root: reply?.root ?? ref, parent: ref} + } + }) +}) diff --git a/src/lib/api/computeCid.ts b/src/lib/api/computeCid.ts new file mode 100644 index 0000000000..18821793ae --- /dev/null +++ b/src/lib/api/computeCid.ts @@ -0,0 +1,148 @@ +import {BlobRef} from '@atproto/api' +import {sha256} from 'js-sha256' +import {CID} from 'multiformats/cid' +import * as Hasher from 'multiformats/hashes/hasher' + +import {type app} from '#/lexicons' + +/* + * Client-side CID computation for post records, extracted from the post + * pipeline so it can be unit-tested in isolation (importing the pipeline pulls + * in the native gallery/media chain). See `computeCid.test.ts` for the + * golden-CID regression fixtures that gate any change to this serialization. + */ + +// The built-in hashing functions from multiformats (`multiformats/hashes/sha2`) +// are meant for Node.js, this is the cross-platform equivalent. +const mf_sha256 = Hasher.from({ + name: 'sha2-256', + code: 0x12, + encode: input => { + const digest = sha256.arrayBuffer(input) + return new Uint8Array(digest) + }, +}) + +export async function computeCid( + record: app.bsky.feed.post.Main, +): Promise { + /* + * Lazily loaded since it's only needed when posting a thread, and its + * `cborg` dependency is ~190KB that would otherwise be in the initial + * web bundle. + */ + const dcbor = await importDagCbor() + /* + * IMPORTANT: `prepareForHashing` prepares the record to be hashed by + * removing fields with undefined value, and converting blobs to the right + * IPLD representation. + */ + const prepared = prepareForHashing(record) + // 1. Encode the record into DAG-CBOR format + const encoded = dcbor.encode(prepared) + // 2. Hash the record in SHA-256 (code 0x12) + const digest = await mf_sha256.digest(encoded) + // 3. Create a CIDv1, specifying DAG-CBOR as content (code 0x71) + const cid = CID.createV1(0x71, digest) + // 4. Get the Base32 representation of the CID (`b` prefix) + return cid.toString() +} + +/** + * True for a plain-JSON lexicon blob, the shape lex `uploadBlob` returns + * (`{$type: 'blob', ref, mimeType, size}` with `ref` a parsed CID). Lex blobs + * are plain objects, not class instances. + */ +function isLexBlob(v: unknown): boolean { + if (v == null || typeof v !== 'object') return false + const o = v as Record + return o.$type === 'blob' && 'ref' in o && 'mimeType' in o +} + +/* + * Returns a transformed version of the object for use in DAG-CBOR. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function prepareForHashing(v: any): any { + /* + * A plain-JSON lex blob is already in the right IPLD shape (its `ref` is a + * parsed CID that DAG-CBOR encodes as a CID link), so pass it through + * untouched. + */ + if (isLexBlob(v)) { + return v + } + + /* + * The video pipeline still reads its blob off the legacy agent + * (`app.bsky.video.getJobStatus` in composer `state/video`), which yields a + * `BlobRef` class instance. `ipld()` gives the same IPLD shape a lex blob + * already has, so both branches hash identically. Drop this guard once the + * video client is migrated. + */ + if (v instanceof BlobRef) { + return v.ipld() + } + + // Walk through arrays + if (Array.isArray(v)) { + let pure = true + const mapped = v.map(value => { + if (value !== (value = prepareForHashing(value))) { + pure = false + } + return value + }) + return pure ? v : mapped + } + + // Walk through plain objects + if (isPlainObject(v)) { + const rec = v as Record + const obj: Record = {} + let pure = true + for (const key in rec) { + let value = rec[key] + // `value` is undefined + if (value === undefined) { + pure = false + continue + } + /* + * `prepareForHashing` returned a value that's different from what we had + * before + */ + if (value !== (value = prepareForHashing(value))) { + pure = false + } + obj[key] = value + } + // Return as is if we haven't needed to tamper with anything + return pure ? v : obj + } + return v +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function isPlainObject(v: any): boolean { + if (typeof v !== 'object' || v === null) { + return false + } + const proto = Object.getPrototypeOf(v) + return proto === Object.prototype || proto === null +} + +/** + * Load `@ipld/dag-cbor` on demand. The dynamic `import()` lets web bundlers + * emit it (and its ~190KB `cborg` dependency) as a separate chunk that only + * loads when posting a thread. Under jest (which runs without + * `--experimental-vm-modules`) dynamic import throws, so we fall back to a + * lazy `require`, which resolves through the test moduleNameMapper. + */ +function importDagCbor(): Promise { + if (process.env.NODE_ENV === 'test') { + // eslint-disable-next-line @typescript-eslint/no-require-imports + return Promise.resolve(require('@ipld/dag-cbor')) + } + return import('@ipld/dag-cbor') +} diff --git a/src/lib/api/index.ts b/src/lib/api/index.ts index aee481669e..20a4418f75 100644 --- a/src/lib/api/index.ts +++ b/src/lib/api/index.ts @@ -8,7 +8,7 @@ import { type AppBskyEmbedVideo, AppBskyFeedPost, type AtpAgent, - BlobRef, + type BlobRef, ChatBskyGroupDefs, type ComAtprotoLabelDefs, type ComAtprotoRepoApplyWrites, @@ -20,9 +20,6 @@ import {toDatetimeString} from '@atproto/syntax' import {RichText} from '@bsky.app/sdk/richtext' import {t} from '@lingui/core/macro' import {type QueryClient} from '@tanstack/react-query' -import {sha256} from 'js-sha256' -import {CID} from 'multiformats/cid' -import * as Hasher from 'multiformats/hashes/hasher' import {IMAGE_SIZE_CONFIG_POSTS} from '#/lib/constants' import {isNetworkError} from '#/lib/strings/errors' @@ -42,8 +39,10 @@ import { type PostDraft, type ThreadDraft, } from '#/view/com/composer/state/composer' +import {type app} from '#/lexicons' import * as bsky from '#/types/bsky' import {createGIFDescription} from '../gif-alt-text' +import {computeCid} from './computeCid' import {uploadBlob} from './upload-blob' export {uploadBlob} @@ -172,7 +171,13 @@ export async function post( // Prepare a ref to the current post for the next post in the thread. const ref = { - cid: await computeCid(record), + /* + * `computeCid` is typed against the lex record. The pipeline still builds + * the legacy `AppBskyFeedPost.Record`; the two shapes are structurally + * equivalent for hashing, so bridge them until the pipeline itself moves + * to the lex record type. + */ + cid: await computeCid(record as unknown as app.bsky.feed.post.Main), uri, } replyPromise = { @@ -499,90 +504,3 @@ async function resolveRecord( } return resolvedLink.record } - -// The built-in hashing functions from multiformats (`multiformats/hashes/sha2`) -// are meant for Node.js, this is the cross-platform equivalent. -const mf_sha256 = Hasher.from({ - name: 'sha2-256', - code: 0x12, - encode: input => { - const digest = sha256.arrayBuffer(input) - return new Uint8Array(digest) - }, -}) - -async function computeCid(record: AppBskyFeedPost.Record): Promise { - /* - * Lazily loaded since it's only needed when posting a thread, and its - * `cborg` dependency is ~190KB that would otherwise be in the initial - * web bundle. - */ - const dcbor = await import('@ipld/dag-cbor') - // IMPORTANT: `prepareObject` prepares the record to be hashed by removing - // fields with undefined value, and converting BlobRef instances to the - // right IPLD representation. - const prepared = prepareForHashing(record) - // 1. Encode the record into DAG-CBOR format - const encoded = dcbor.encode(prepared) - // 2. Hash the record in SHA-256 (code 0x12) - const digest = await mf_sha256.digest(encoded) - // 3. Create a CIDv1, specifying DAG-CBOR as content (code 0x71) - const cid = CID.createV1(0x71, digest) - // 4. Get the Base32 representation of the CID (`b` prefix) - return cid.toString() -} - -// Returns a transformed version of the object for use in DAG-CBOR. -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function prepareForHashing(v: any): any { - // IMPORTANT: BlobRef#ipld() returns the correct object we need for hashing, - // the API client will convert this for you but we're hashing in the client, - // so we need it *now*. - if (v instanceof BlobRef) { - return v.ipld() - } - - // Walk through arrays - if (Array.isArray(v)) { - let pure = true - const mapped = v.map(value => { - if (value !== (value = prepareForHashing(value))) { - pure = false - } - return value - }) - return pure ? v : mapped - } - - // Walk through plain objects - if (isPlainObject(v)) { - const obj: Record = {} - let pure = true - for (const key in v) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - let value = v[key] - // `value` is undefined - if (value === undefined) { - pure = false - continue - } - // `prepareObject` returned a value that's different from what we had before - if (value !== (value = prepareForHashing(value))) { - pure = false - } - obj[key] = value - } - // Return as is if we haven't needed to tamper with anything - return pure ? v : obj - } - return v -} - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function isPlainObject(v: any): boolean { - if (typeof v !== 'object' || v === null) { - return false - } - const proto = Object.getPrototypeOf(v) - return proto === Object.prototype || proto === null -}