diff --git a/.github/scripts/summarize-maestro.mjs b/.github/scripts/summarize-maestro.mjs index 7154f5b0ee..4d49afcbbd 100644 --- a/.github/scripts/summarize-maestro.mjs +++ b/.github/scripts/summarize-maestro.mjs @@ -101,6 +101,20 @@ function readPhase(root) { return phaseFile ? fs.readFileSync(phaseFile, 'utf8').trim() : '' } +function maestroScreenshot(file) { + const match = path + .basename(file) + .match(/^screenshot-.*?-(\d+)-\((.+)\)\.(jpe?g|png)$/i) + return match + ? { + file, + timestamp: Number(match[1]), + flow: match[2], + extension: match[3].toLowerCase(), + } + : null +} + function platformResult({name, status, root, artifactUrl}) { const files = walk(root) const reports = files.filter(file => /(?:report|junit).*\.xml$/i.test(file)) @@ -116,6 +130,24 @@ function platformResult({name, status, root, artifactUrl}) { // A cancelled or timed-out Maestro run may never flush JUnit. Its CLI log is // streamed continuously, so use those failure lines when JUnit has no detail. const failures = junitFailures.length > 0 ? junitFailures : cliFailures + const screenshots = files + .map(maestroScreenshot) + .filter(Boolean) + .sort((a, b) => b.timestamp - a.timestamp) + const failureScreenshots = failures + .flatMap(failure => { + const screenshot = screenshots.find(item => item.flow === failure.name) + return screenshot + ? [ + { + flow: failure.name, + file: screenshot.file, + extension: screenshot.extension, + }, + ] + : [] + }) + .slice(0, 5) // A skipped platform (e.g. iOS while temporarily disabled) is not a failure // as long as it produced no flow failures. const failed = @@ -125,6 +157,7 @@ function platformResult({name, status, root, artifactUrl}) { status, failed, failures, + screenshots: failureScreenshots, phase: readPhase(root), hasJUnit: reports.length > 0, artifactUrl, @@ -308,6 +341,14 @@ export function buildSummary({ ...(index < platforms.length - 1 ? [{type: 'divider'}] : []), ]), ] + const screenshotUploads = platforms.flatMap(platform => + platform.screenshots.map(screenshot => ({ + file: path.relative(process.cwd(), screenshot.file), + filename: `${platform.name.toLowerCase()}-${screenshot.flow}.${screenshot.extension}`, + title: `${platform.name} — ${screenshot.flow}`, + alt_text: `${platform.name} failure screenshot for ${screenshot.flow}`, + })), + ) return { notify, platforms, @@ -319,6 +360,7 @@ export function buildSummary({ commitUrl, }), payload: {text, blocks}, + screenshotUploads, } } diff --git a/.github/scripts/summarize-maestro.test.mjs b/.github/scripts/summarize-maestro.test.mjs new file mode 100644 index 0000000000..8c0a4e1225 --- /dev/null +++ b/.github/scripts/summarize-maestro.test.mjs @@ -0,0 +1,67 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' + +import {buildSummary} from './summarize-maestro.mjs' + +test('includes up to five matching failure screenshots and prefers the latest', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'maestro-summary-')) + t.after(() => fs.rmSync(root, {recursive: true})) + + const failures = Array.from({length: 7}, (_, index) => `flow-${index + 1}`) + fs.writeFileSync( + path.join(root, 'report.xml'), + `${failures + .map( + name => + `Failed ${name}`, + ) + .join('')}`, + ) + for (const [index, name] of failures.entries()) { + if (name === 'flow-2') continue + fs.writeFileSync( + path.join(root, `screenshot-❌-${index + 1}-(${name}).png`), + '', + ) + } + fs.writeFileSync(path.join(root, 'screenshot-❌-7-(unrelated).png'), '') + fs.writeFileSync(path.join(root, 'screenshot-❌-8-(flow-1).png'), '') + + const summary = buildSummary({ + iosStatus: 'failure', + androidStatus: 'skipped', + iosRoot: root, + sha: '0123456789abcdef', + runUrl: 'https://example.com/run', + commitUrl: 'https://example.com/commit', + }) + + assert.deepEqual( + summary.screenshotUploads.map(upload => upload.filename), + ['flow-1', 'flow-3', 'flow-4', 'flow-5', 'flow-6'].map( + name => `ios-${name}.png`, + ), + ) + assert.equal( + summary.screenshotUploads[0].file, + path.relative( + process.cwd(), + path.join(root, 'screenshot-❌-8-(flow-1).png'), + ), + ) +}) + +test('does not create screenshot uploads for setup failures', () => { + const summary = buildSummary({ + iosStatus: 'failure', + androidStatus: 'skipped', + sha: '0123456789abcdef', + runUrl: 'https://example.com/run', + commitUrl: 'https://example.com/commit', + }) + + assert.deepEqual(summary.screenshotUploads, []) +}) diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index c6ed99e8d8..5704c4dad5 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -412,13 +412,51 @@ jobs: --commit-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}" \ > e2e-summary.json echo "notify=$(jq -r .notify e2e-summary.json)" >> "$GITHUB_OUTPUT" - echo "payload=$(jq -c .payload e2e-summary.json)" >> "$GITHUB_OUTPUT" jq -r .githubSummary e2e-summary.json >> "$GITHUB_STEP_SUMMARY" + - name: Build Slack message payload + if: steps.summary.outputs.notify == 'true' + env: + SLACK_CHANNEL_ID: ${{ secrets.E2E_FAILURES_SLACK_CHANNEL_ID }} + run: | + jq --arg channel "$SLACK_CHANNEL_ID" \ + '.payload + {channel: $channel}' \ + e2e-summary.json > slack-message.json + - name: Notify Slack of E2E failures + id: slack_message if: steps.summary.outputs.notify == 'true' uses: slackapi/slack-github-action@0d95c9a7becc1e6e297d76df9bc735c44f4cbcbc # v3.0.5 with: - webhook: ${{ secrets.E2E_FAILURES_SLACK_WEBHOOK }} - webhook-type: incoming-webhook - payload: ${{ steps.summary.outputs.payload }} + method: chat.postMessage + errors: true + token: ${{ secrets.E2E_FAILURES_SLACK_BOT_TOKEN }} + payload-file-path: slack-message.json + + - name: Build Slack screenshot payload + id: screenshots + if: steps.summary.outputs.notify == 'true' + env: + SLACK_CHANNEL_ID: ${{ secrets.E2E_FAILURES_SLACK_CHANNEL_ID }} + SLACK_THREAD_TS: ${{ steps.slack_message.outputs.ts }} + run: | + count=$(jq '.screenshotUploads | length' e2e-summary.json) + echo "count=$count" >> "$GITHUB_OUTPUT" + jq \ + --arg channel_id "$SLACK_CHANNEL_ID" \ + --arg thread_ts "$SLACK_THREAD_TS" \ + '{ + channel_id: $channel_id, + thread_ts: $thread_ts, + initial_comment: "Failure screenshots (up to five per platform)", + file_uploads: .screenshotUploads + }' e2e-summary.json > slack-screenshots.json + + - name: Upload E2E failure screenshots to Slack + if: steps.summary.outputs.notify == 'true' && steps.screenshots.outputs.count != '0' + uses: slackapi/slack-github-action@0d95c9a7becc1e6e297d76df9bc735c44f4cbcbc # v3.0.5 + with: + method: files.uploadV2 + errors: true + token: ${{ secrets.E2E_FAILURES_SLACK_BOT_TOKEN }} + payload-file-path: slack-screenshots.json