Report the percentage compiled and a lost/regained diff against the PR base

The sticky comment now leads with the share of components compiled and a
comparison with the PR's base commit: components that lost optimization
(with the compiler's reasons) and ones that regained it. Functions are
diffed by file + name rather than line so unrelated edits that shift a
component down a file do not read as changes.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

.github/workflows/react-compiler-report.yml

@@ -32,9 +32,24 @@ jobs:
           cache: pnpm
       - name: 📦 pnpm install
         run: pnpm install --frozen-lockfile
+      - name: ⬇️ Check out base commit for comparison
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          git fetch --depth=1 origin "$BASE_SHA"
+          git worktree add --detach "$RUNNER_TEMP/base" "$BASE_SHA"
+      - name: ⚛️ Snapshot React Compiler results on base
+        env:
+          REACT_COMPILER_SRC_ROOT: ${{ runner.temp }}/base
+          REACT_COMPILER_SNAPSHOT_PATH: ${{ runner.temp }}/base-snapshot.json
+          # The base run is only for the snapshot - keep its report out of the
+          # job summary, which the PR run below writes.
+          GITHUB_STEP_SUMMARY: ''
+        run: pnpm react-compiler:report > /dev/null
       - name: ⚛️ Generate React Compiler report
         env:
           REACT_COMPILER_REPORT_PATH: ${{ runner.temp }}/react-compiler-report.md
+          REACT_COMPILER_BASE_SNAPSHOT_PATH: ${{ runner.temp }}/base-snapshot.json
         run: pnpm react-compiler:report
       - name: 💬 Drop a comment
         uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5

scripts/README.md

@@ -5,7 +5,8 @@
 Reports which components and hooks React Compiler skipped optimizing, grouped by
 the compiler's own diagnostic category. Run with `pnpm react-compiler:report`;
 the React Compiler report workflow also runs it on every pull request and posts
-the report as a sticky PR comment.
+the report as a sticky PR comment, including a diff against the base commit of
+components that lost or regained optimization.
 
 ## updateExtensions.sh
 

scripts/react-compiler-report.ts

@@ -25,10 +25,17 @@
  * summary under GitHub Actions and to REACT_COMPILER_REPORT_PATH when set
  * (the React Compiler report workflow posts that file as a sticky PR
  * comment). Always exits 0 - this is a metric, not a gate.
+ *
+ * The workflow also diffs each PR against its base commit:
+ * REACT_COMPILER_SRC_ROOT points the script at another checkout,
+ * REACT_COMPILER_SNAPSHOT_PATH writes a JSON snapshot of every function's
+ * status, and REACT_COMPILER_BASE_SNAPSHOT_PATH reads such a snapshot back to
+ * report which components lost or regained optimization.
  */
 
 import {
   appendFileSync,
+  existsSync,
   readdirSync,
   readFileSync,
   statSync,
@@ -44,12 +51,29 @@ import {
   type Logger,
 } from 'babel-plugin-react-compiler'
 
-const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..')
+/**
+ * Root of the tree to analyze. Defaults to this repo; the report workflow sets
+ * REACT_COMPILER_SRC_ROOT to a checkout of the PR's base commit so this
+ * script (and this repo's node_modules) can snapshot the base too.
+ */
+const ROOT = process.env.REACT_COMPILER_SRC_ROOT
+  ? resolve(process.env.REACT_COMPILER_SRC_ROOT)
+  : resolve(dirname(fileURLToPath(import.meta.url)), '..')
 const OPT_OUT_DIRECTIVE = "'use no memo'"
 
 /** A component or hook, as `path/to/File.tsx:12`. */
 type FunctionKey = string
 
+/**
+ * A component or hook as `path/to/File.tsx::Name` (`#n` disambiguates
+ * same-named functions within one file). Snapshots are diffed on these keys
+ * so a component merely pushed down by unrelated edits does not read as
+ * having lost and regained optimization.
+ */
+type StableKey = string
+
+type FunctionStatus = 'compiled' | 'skipped' | 'optedOut'
+
 type Diagnostic = {
   category: ErrorCategory
   severity: ErrorSeverity
@@ -75,16 +99,52 @@ const optedOut = new Set<FunctionKey>()
 /** Files the plugin could not process at all, e.g. syntax it cannot parse. */
 const unreadable: [file: string, message: string][] = []
 let currentFile: string | null = null
+let currentSource: string | null = null
+
+const status = new Map<StableKey, FunctionStatus>()
+const locByStable = new Map<StableKey, FunctionKey>()
+const stableByLoc = new Map<FunctionKey, StableKey>()
+const stableNameCounts = new Map<string, number>()
+
+/**
+ * Best-effort function name from the declaration line - enough to tell
+ * `function Foo`, `const useBar = ...`, and `name: () => ...` apart.
+ */
+function functionName(source: string, line: number): string {
+  const text = source.split('\n')[line - 1] ?? ''
+  const match =
+    text.match(/function\s+([A-Za-z0-9_$]+)/) ??
+    text.match(/(?:const|let|var)\s+([A-Za-z0-9_$]+)/) ??
+    text.match(/^\s*(?:export\s+)?([A-Za-z0-9_$]+)\s*[:=(]/)
+  return match?.[1] ?? 'anonymous'
+}
 
 const logger: Logger = {
   logEvent(_filename, event) {
     if (!('fnLoc' in event) || event.fnLoc == null) return
-    const key = `${relative(ROOT, currentFile!)}:${event.fnLoc.start.line}`
+    const file = relative(ROOT, currentFile!)
+    const key = `${file}:${event.fnLoc.start.line}`
+    /*
+     * One function can emit several events (one per diagnostic), so mint its
+     * stable key once per location.
+     */
+    let stable = stableByLoc.get(key)
+    if (!stable) {
+      const name = `${file}::${functionName(currentSource!, event.fnLoc.start.line)}`
+      const n = (stableNameCounts.get(name) ?? 0) + 1
+      stableNameCounts.set(name, n)
+      stable = n > 1 ? `${name}#${n}` : name
+      stableByLoc.set(key, stable)
+      locByStable.set(stable, key)
+    }
     if (event.kind === 'CompileSuccess') {
       compiled.add(key)
+      status.set(stable, 'compiled')
     } else if (event.kind === 'CompileSkip') {
       optedOut.add(key)
+      status.set(stable, 'optedOut')
     } else if (event.kind === 'CompileError') {
+      status.set(stable, 'skipped')
       /*
        * One function emits one event per diagnostic, so collect them per
        * location - counting raw events overstates the number of skipped
@@ -102,8 +162,9 @@ const logger: Logger = {
 
 for (const file of files) {
   currentFile = file
+  currentSource = readFileSync(file, 'utf8')
   try {
-    transformSync(readFileSync(file, 'utf8'), {
+    transformSync(currentSource, {
       filename: file,
       babelrc: false,
       configFile: false,
@@ -164,11 +225,96 @@ for (const [, diagnostics] of rows) {
 }
 const ranked = [...byCategory].sort((a, b) => b[1].count - a[1].count)
 
-const headline = `Successfully compiled ${compiled.size} out of ${selected} components and hooks.`
+/** Share of selected components and hooks compiled, as e.g. `95.3%`. */
+function percent(n: number, of: number): string {
+  return `${of ? ((n / of) * 100).toFixed(1) : '0.0'}%`
+}
+
+const headline = `Successfully compiled ${compiled.size} out of ${selected} components and hooks (${percent(compiled.size, selected)}).`
 const subhead =
   `${rows.length} skipped, ${optedOut.size} opted out of compilation ` +
   `(${OPT_OUT_DIRECTIVE}), across ${affectedFiles.size} files.`
 
+type Snapshot = {
+  compiled: number
+  selected: number
+  functions: Record<StableKey, FunctionStatus>
+}
+
+/*
+ * Snapshot/diff wiring for the React Compiler report workflow: the run over
+ * the PR's base commit writes a snapshot, then the PR run diffs itself
+ * against it so the sticky comment can call out components that lost or
+ * regained optimization.
+ */
+const snapshotPath = process.env.REACT_COMPILER_SNAPSHOT_PATH
+if (snapshotPath) {
+  const snapshot: Snapshot = {
+    compiled: compiled.size,
+    selected,
+    functions: Object.fromEntries(
+      [...status].sort(([a], [b]) => a.localeCompare(b)),
+    ),
+  }
+  writeFileSync(snapshotPath, JSON.stringify(snapshot))
+}
+
+const baseSnapshotPath = process.env.REACT_COMPILER_BASE_SNAPSHOT_PATH
+const base: Snapshot | null =
+  baseSnapshotPath && existsSync(baseSnapshotPath)
+    ? JSON.parse(readFileSync(baseSnapshotPath, 'utf8'))
+    : null
+
+const lost: StableKey[] = []
+const regained: StableKey[] = []
+if (base) {
+  for (const [stable, s] of status) {
+    const was = base.functions[stable]
+    if (s === 'compiled' && was && was !== 'compiled') regained.push(stable)
+    if (s !== 'compiled' && was === 'compiled') lost.push(stable)
+  }
+  lost.sort()
+  regained.sort()
+}
+
+/** A diff list entry: location, name, and why it is not compiled. */
+function describeLost(stable: StableKey): string {
+  const loc = locByStable.get(stable)!
+  const diagnostics = skipped.get(loc)
+  const reasons = diagnostics
+    ? diagnostics.map(d => `${d.category}: ${d.reason}`).join('; ')
+    : `CompileSkip: opted out via ${OPT_OUT_DIRECTIVE}`
+  return `- \`${loc}\` \`${stable.split('::')[1]}\` - ${reasons}`
+}
+
+const diffSection = base
+  ? [
+      `**Compared to base** (${base.compiled} out of ${base.selected}, ${percent(base.compiled, base.selected)}):` +
+        (lost.length || regained.length
+          ? ''
+          : ' no components changed optimization status.'),
+      ``,
+      ...(lost.length
+        ? [
+            `⚠️ **${lost.length} lost optimization:**`,
+            ``,
+            ...lost.map(describeLost),
+            ``,
+          ]
+        : []),
+      ...(regained.length
+        ? [
+            `✅ **${regained.length} regained optimization:**`,
+            ``,
+            ...regained.map(
+              key => `- \`${locByStable.get(key)!}\` \`${key.split('::')[1]}\``,
+            ),
+            ``,
+          ]
+        : []),
+    ]
+  : []
+
 for (const [key, diagnostics] of rows) {
   console.log(key)
   for (const d of diagnostics) console.log(`    ${d.category}: ${d.reason}`)
@@ -181,6 +327,11 @@ for (const [category, {severity, count}] of ranked) {
   console.log(`${String(count).padStart(4)}  ${category} (${severity})`)
 }
 console.log(`\n${headline}\n${subhead}`)
+if (base) {
+  console.log(
+    `\nCompared to base: ${lost.length} lost optimization, ${regained.length} regained.`,
+  )
+}
 for (const [file, message] of unreadable) {
   console.log(
     `::warning file=${file}::React Compiler could not run: ${message}`,
@@ -200,6 +351,7 @@ if (summaryPath || reportPath) {
     ``,
     `**${headline}** ${subhead}`,
     ``,
+    ...diffSection,
     `| category | severity | components and hooks |`,
     `| --- | --- | --- |`,
     ...ranked.map(