attach e2e screenshots to slack alerts

.github/scripts/summarize-maestro.mjs

@@ -101,6 +101,20 @@ function readPhase(root) {
   return phaseFile ? fs.readFileSync(phaseFile, 'utf8').trim() : ''
 }
 
+function maestroScreenshot(file) {
+  const match = path
+    .basename(file)
+    .match(/^screenshot-.*?-(\d+)-\((.+)\)\.(jpe?g|png)$/i)
+  return match
+    ? {
+        file,
+        timestamp: Number(match[1]),
+        flow: match[2],
+        extension: match[3].toLowerCase(),
+      }
+    : null
+}
+
 function platformResult({name, status, root, artifactUrl}) {
   const files = walk(root)
   const reports = files.filter(file => /(?:report|junit).*\.xml$/i.test(file))
@@ -116,6 +130,24 @@ function platformResult({name, status, root, artifactUrl}) {
   // A cancelled or timed-out Maestro run may never flush JUnit. Its CLI log is
   // streamed continuously, so use those failure lines when JUnit has no detail.
   const failures = junitFailures.length > 0 ? junitFailures : cliFailures
+  const screenshots = files
+    .map(maestroScreenshot)
+    .filter(Boolean)
+    .sort((a, b) => b.timestamp - a.timestamp)
+  const failureScreenshots = failures
+    .flatMap(failure => {
+      const screenshot = screenshots.find(item => item.flow === failure.name)
+      return screenshot
+        ? [
+            {
+              flow: failure.name,
+              file: screenshot.file,
+              extension: screenshot.extension,
+            },
+          ]
+        : []
+    })
+    .slice(0, 5)
   // A skipped platform (e.g. iOS while temporarily disabled) is not a failure
   // as long as it produced no flow failures.
   const failed =
@@ -125,6 +157,7 @@ function platformResult({name, status, root, artifactUrl}) {
     status,
     failed,
     failures,
+    screenshots: failureScreenshots,
     phase: readPhase(root),
     hasJUnit: reports.length > 0,
     artifactUrl,
@@ -308,6 +341,14 @@ export function buildSummary({
       ...(index < platforms.length - 1 ? [{type: 'divider'}] : []),
     ]),
   ]
+  const screenshotUploads = platforms.flatMap(platform =>
+    platform.screenshots.map(screenshot => ({
+      file: path.relative(process.cwd(), screenshot.file),
+      filename: `${platform.name.toLowerCase()}-${screenshot.flow}.${screenshot.extension}`,
+      title: `${platform.name} — ${screenshot.flow}`,
+      alt_text: `${platform.name} failure screenshot for ${screenshot.flow}`,
+    })),
+  )
   return {
     notify,
     platforms,
@@ -319,6 +360,7 @@ export function buildSummary({
       commitUrl,
     }),
     payload: {text, blocks},
+    screenshotUploads,
   }
 }
 

.github/scripts/summarize-maestro.test.mjs

@@ -0,0 +1,67 @@
+import assert from 'node:assert/strict'
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import test from 'node:test'
+
+import {buildSummary} from './summarize-maestro.mjs'
+
+test('includes up to five matching failure screenshots and prefers the latest', t => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'maestro-summary-'))
+  t.after(() => fs.rmSync(root, {recursive: true}))
+
+  const failures = Array.from({length: 7}, (_, index) => `flow-${index + 1}`)
+  fs.writeFileSync(
+    path.join(root, 'report.xml'),
+    `<testsuite>${failures
+      .map(
+        name =>
+          `<testcase name="${name}"><failure>Failed ${name}</failure></testcase>`,
+      )
+      .join('')}</testsuite>`,
+  )
+  for (const [index, name] of failures.entries()) {
+    if (name === 'flow-2') continue
+    fs.writeFileSync(
+      path.join(root, `screenshot-❌-${index + 1}-(${name}).png`),
+      '',
+    )
+  }
+  fs.writeFileSync(path.join(root, 'screenshot-❌-7-(unrelated).png'), '')
+  fs.writeFileSync(path.join(root, 'screenshot-❌-8-(flow-1).png'), '')
+
+  const summary = buildSummary({
+    iosStatus: 'failure',
+    androidStatus: 'skipped',
+    iosRoot: root,
+    sha: '0123456789abcdef',
+    runUrl: 'https://example.com/run',
+    commitUrl: 'https://example.com/commit',
+  })
+
+  assert.deepEqual(
+    summary.screenshotUploads.map(upload => upload.filename),
+    ['flow-1', 'flow-3', 'flow-4', 'flow-5', 'flow-6'].map(
+      name => `ios-${name}.png`,
+    ),
+  )
+  assert.equal(
+    summary.screenshotUploads[0].file,
+    path.relative(
+      process.cwd(),
+      path.join(root, 'screenshot-❌-8-(flow-1).png'),
+    ),
+  )
+})
+
+test('does not create screenshot uploads for setup failures', () => {
+  const summary = buildSummary({
+    iosStatus: 'failure',
+    androidStatus: 'skipped',
+    sha: '0123456789abcdef',
+    runUrl: 'https://example.com/run',
+    commitUrl: 'https://example.com/commit',
+  })
+
+  assert.deepEqual(summary.screenshotUploads, [])
+})

.github/workflows/nightly-e2e.yml

@@ -412,13 +412,51 @@ jobs:
             --commit-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}" \
             > e2e-summary.json
           echo "notify=$(jq -r .notify e2e-summary.json)" >> "$GITHUB_OUTPUT"
-          echo "payload=$(jq -c .payload e2e-summary.json)" >> "$GITHUB_OUTPUT"
           jq -r .githubSummary e2e-summary.json >> "$GITHUB_STEP_SUMMARY"
 
+      - name: Build Slack message payload
+        if: steps.summary.outputs.notify == 'true'
+        env:
+          SLACK_CHANNEL_ID: ${{ secrets.E2E_FAILURES_SLACK_CHANNEL_ID }}
+        run: |
+          jq --arg channel "$SLACK_CHANNEL_ID" \
+            '.payload + {channel: $channel}' \
+            e2e-summary.json > slack-message.json
+
       - name: Notify Slack of E2E failures
+        id: slack_message
         if: steps.summary.outputs.notify == 'true'
         uses: slackapi/slack-github-action@0d95c9a7becc1e6e297d76df9bc735c44f4cbcbc # v3.0.5
         with:
-          webhook: ${{ secrets.E2E_FAILURES_SLACK_WEBHOOK }}
-          webhook-type: incoming-webhook
-          payload: ${{ steps.summary.outputs.payload }}
+          method: chat.postMessage
+          errors: true
+          token: ${{ secrets.E2E_FAILURES_SLACK_BOT_TOKEN }}
+          payload-file-path: slack-message.json
+
+      - name: Build Slack screenshot payload
+        id: screenshots
+        if: steps.summary.outputs.notify == 'true'
+        env:
+          SLACK_CHANNEL_ID: ${{ secrets.E2E_FAILURES_SLACK_CHANNEL_ID }}
+          SLACK_THREAD_TS: ${{ steps.slack_message.outputs.ts }}
+        run: |
+          count=$(jq '.screenshotUploads | length' e2e-summary.json)
+          echo "count=$count" >> "$GITHUB_OUTPUT"
+          jq \
+            --arg channel_id "$SLACK_CHANNEL_ID" \
+            --arg thread_ts "$SLACK_THREAD_TS" \
+            '{
+              channel_id: $channel_id,
+              thread_ts: $thread_ts,
+              initial_comment: "Failure screenshots (up to five per platform)",
+              file_uploads: .screenshotUploads
+            }' e2e-summary.json > slack-screenshots.json
+
+      - name: Upload E2E failure screenshots to Slack
+        if: steps.summary.outputs.notify == 'true' && steps.screenshots.outputs.count != '0'
+        uses: slackapi/slack-github-action@0d95c9a7becc1e6e297d76df9bc735c44f4cbcbc # v3.0.5
+        with:
+          method: files.uploadV2
+          errors: true
+          token: ${{ secrets.E2E_FAILURES_SLACK_BOT_TOKEN }}
+          payload-file-path: slack-screenshots.json