Add carousel Slack alerts for nightly Maestro failures (#11645)
This commit is contained in:
Samuel Newman
@@ -0,0 +1,55 @@
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import process from 'node:process'
|
||||
|
||||
import sharp from 'sharp'
|
||||
|
||||
export async function frameSlackScreenshots({
|
||||
inputPath,
|
||||
outputPath,
|
||||
outputDir,
|
||||
}) {
|
||||
const payload = JSON.parse(fs.readFileSync(inputPath, 'utf8'))
|
||||
if (!Array.isArray(payload.file_uploads)) {
|
||||
throw new Error('Slack upload payload must contain a file_uploads array')
|
||||
}
|
||||
|
||||
fs.mkdirSync(outputDir, {recursive: true})
|
||||
const framedUploads = []
|
||||
for (const [index, upload] of payload.file_uploads.entries()) {
|
||||
if (
|
||||
typeof upload.file !== 'string' ||
|
||||
typeof upload.filename !== 'string'
|
||||
) {
|
||||
throw new Error(`Invalid Slack file upload at index ${index}`)
|
||||
}
|
||||
const filename = `${path.parse(path.basename(upload.filename)).name}.png`
|
||||
const framedFile = path.join(outputDir, filename)
|
||||
await sharp(upload.file)
|
||||
.resize(1600, 1200, {fit: 'contain', background: '#f8f8f8'})
|
||||
.png()
|
||||
.toFile(framedFile)
|
||||
framedUploads.push({
|
||||
...upload,
|
||||
file: framedFile,
|
||||
filename,
|
||||
highlight_type: 'png',
|
||||
})
|
||||
}
|
||||
|
||||
payload.file_uploads = framedUploads
|
||||
fs.writeFileSync(outputPath, `${JSON.stringify(payload)}\n`)
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
path.resolve(process.argv[1]) === path.resolve(import.meta.filename)
|
||||
) {
|
||||
const [inputPath, outputPath, outputDir] = process.argv.slice(2)
|
||||
if (!inputPath || !outputPath || !outputDir) {
|
||||
throw new Error(
|
||||
'Usage: frame-slack-screenshots.mjs <input.json> <output.json> <output-dir>',
|
||||
)
|
||||
}
|
||||
await frameSlackScreenshots({inputPath, outputPath, outputDir})
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import fs from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import test from 'node:test'
|
||||
|
||||
import sharp from 'sharp'
|
||||
|
||||
import {frameSlackScreenshots} from './frame-slack-screenshots.mjs'
|
||||
|
||||
test('frames Slack screenshots as 4:3 PNGs', async t => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'maestro-slack-frame-'))
|
||||
t.after(() => fs.rmSync(root, {recursive: true, force: true}))
|
||||
const source = path.join(root, 'source.png')
|
||||
const input = path.join(root, 'input.json')
|
||||
const output = path.join(root, 'output.json')
|
||||
const outputDir = path.join(root, 'images')
|
||||
await sharp({
|
||||
create: {
|
||||
width: 2,
|
||||
height: 4,
|
||||
channels: 3,
|
||||
background: '#ffffff',
|
||||
},
|
||||
})
|
||||
.png()
|
||||
.toFile(source)
|
||||
fs.writeFileSync(
|
||||
input,
|
||||
JSON.stringify({
|
||||
file_uploads: [
|
||||
{
|
||||
file: source,
|
||||
filename: '1-android-login.png',
|
||||
alt_text: 'Android failure screenshot for login',
|
||||
},
|
||||
],
|
||||
}),
|
||||
)
|
||||
|
||||
await frameSlackScreenshots({inputPath: input, outputPath: output, outputDir})
|
||||
|
||||
const payload = JSON.parse(fs.readFileSync(output, 'utf8'))
|
||||
const {data, info} = await sharp(payload.file_uploads[0].file)
|
||||
.raw()
|
||||
.toBuffer({resolveWithObject: true})
|
||||
const pixelAt = (x, y) => {
|
||||
const offset = (y * info.width + x) * info.channels
|
||||
return Array.from(data.subarray(offset, offset + 3))
|
||||
}
|
||||
|
||||
assert.equal(info.width, 1600)
|
||||
assert.equal(info.height, 1200)
|
||||
assert.deepEqual(pixelAt(0, 0), [248, 248, 248])
|
||||
assert.deepEqual(pixelAt(800, 600), [255, 255, 255])
|
||||
assert.equal(payload.file_uploads[0].filename, '1-android-login.png')
|
||||
assert.equal(payload.file_uploads[0].highlight_type, 'png')
|
||||
})
|
||||
@@ -0,0 +1,330 @@
|
||||
import path from 'node:path'
|
||||
|
||||
const CAROUSEL_LIMIT = 10
|
||||
|
||||
function concise(value, limit) {
|
||||
return value.length > limit ? `${value.slice(0, limit - 1)}…` : value
|
||||
}
|
||||
|
||||
function slackEscape(value) {
|
||||
return value
|
||||
.replaceAll('&', '&')
|
||||
.replaceAll('<', '<')
|
||||
.replaceAll('>', '>')
|
||||
}
|
||||
|
||||
function mrkdwnText(value, limit) {
|
||||
return concise(slackEscape(value), limit)
|
||||
}
|
||||
|
||||
function pluralize(count, singular) {
|
||||
return `${count} ${singular}${count === 1 ? '' : 's'}`
|
||||
}
|
||||
|
||||
function stateFor(platforms) {
|
||||
if (platforms.some(platform => platform.status === 'cancelled')) {
|
||||
return 'cancelled'
|
||||
}
|
||||
if (platforms.some(platform => platform.failures.length > 0)) {
|
||||
return 'failed'
|
||||
}
|
||||
if (platforms.some(platform => platform.failed)) {
|
||||
return 'setup_failed'
|
||||
}
|
||||
return 'passed'
|
||||
}
|
||||
|
||||
function statePresentation(state, failureCount) {
|
||||
if (state === 'cancelled') {
|
||||
return {
|
||||
header: '⏹️ Nightly Maestro E2E cancelled',
|
||||
summary: 'results may be incomplete',
|
||||
fallback: 'Nightly Maestro E2E was cancelled. Results may be incomplete.',
|
||||
}
|
||||
}
|
||||
if (state === 'setup_failed') {
|
||||
return {
|
||||
header: '⚠️ Nightly Maestro setup failed',
|
||||
summary: 'no complete test results',
|
||||
fallback: 'Nightly Maestro E2E setup failed before tests could complete.',
|
||||
}
|
||||
}
|
||||
if (state === 'failed') {
|
||||
return {
|
||||
header: '🚨 Nightly Maestro E2E failed',
|
||||
summary: `${pluralize(failureCount, 'failed flow')}`,
|
||||
fallback: `Nightly Maestro E2E failed with ${pluralize(failureCount, 'failed flow')}.`,
|
||||
}
|
||||
}
|
||||
return {
|
||||
header: '✅ Nightly Maestro E2E passed',
|
||||
summary: 'all platforms passed',
|
||||
fallback: 'Nightly Maestro E2E passed on all platforms.',
|
||||
}
|
||||
}
|
||||
|
||||
function platformStatus(platform) {
|
||||
if (platform.status === 'cancelled') return '⏹️ Cancelled'
|
||||
if (platform.status === 'skipped') return '⏭️ Skipped'
|
||||
if (platform.failures.length > 0) {
|
||||
return `❌ Failed · ${pluralize(platform.failures.length, 'flow')}`
|
||||
}
|
||||
if (platform.failed && !platform.hasJUnit) return '⚠️ Setup failed'
|
||||
if (platform.failed) return '❌ Failed'
|
||||
return '✅ Passed'
|
||||
}
|
||||
|
||||
function selectFailures(platforms, limit = CAROUSEL_LIMIT) {
|
||||
const queues = platforms.map(platform =>
|
||||
platform.failures.map(failure => ({platform, failure})),
|
||||
)
|
||||
const selected = []
|
||||
|
||||
while (selected.length < limit && queues.some(queue => queue.length > 0)) {
|
||||
for (const queue of queues) {
|
||||
const next = queue.shift()
|
||||
if (next) selected.push(next)
|
||||
if (selected.length === limit) break
|
||||
}
|
||||
}
|
||||
|
||||
return selected
|
||||
}
|
||||
|
||||
function uploadFilename({platform, failure}, index) {
|
||||
const extension = path.extname(failure.screenshot) || '.png'
|
||||
const slug = failure.name
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-|-$/g, '')
|
||||
return `${index + 1}-${platform.name.toLowerCase()}-${slug || 'failed-flow'}${extension}`
|
||||
}
|
||||
|
||||
function buildUploadPayload(selectedFailures) {
|
||||
return {
|
||||
file_uploads: selectedFailures
|
||||
.filter(({failure}) => failure.screenshot)
|
||||
.map((entry, index) => ({
|
||||
file: entry.failure.screenshot,
|
||||
filename: uploadFilename(entry, index),
|
||||
highlight_type: 'png',
|
||||
alt_text: `${entry.platform.name} failure screenshot for ${entry.failure.name}`,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
function buildThreadPayload(platforms) {
|
||||
const lines = ['*All Maestro failure details*']
|
||||
for (const platform of platforms) {
|
||||
if (platform.failures.length === 0) continue
|
||||
lines.push('', `*${slackEscape(platform.name)}*`)
|
||||
for (const failure of platform.failures) {
|
||||
lines.push(
|
||||
`• *${mrkdwnText(failure.name, 140)}*\n ${mrkdwnText(failure.message, 300)}`,
|
||||
)
|
||||
}
|
||||
if (platform.artifactUrl) {
|
||||
lines.push(`<${platform.artifactUrl}|Open ${platform.name} artifacts>`)
|
||||
}
|
||||
}
|
||||
return {text: lines.join('\n')}
|
||||
}
|
||||
|
||||
function buildCarousel(selectedFailures, slackFileIds) {
|
||||
let screenshotIndex = 0
|
||||
const elements = selectedFailures.map(({platform, failure}, index) => {
|
||||
const slackFileId = failure.screenshot
|
||||
? slackFileIds[screenshotIndex++]
|
||||
: undefined
|
||||
return {
|
||||
type: 'card',
|
||||
block_id: `maestro_failure_${index + 1}`,
|
||||
title: {
|
||||
type: 'mrkdwn',
|
||||
text: `*${mrkdwnText(failure.name, 140)}*`,
|
||||
verbatim: true,
|
||||
},
|
||||
subtitle: {
|
||||
type: 'mrkdwn',
|
||||
text: `${platform.name} · failed flow`,
|
||||
verbatim: true,
|
||||
},
|
||||
...(slackFileId
|
||||
? {
|
||||
hero_image: {
|
||||
type: 'image',
|
||||
slack_file: {id: slackFileId},
|
||||
alt_text: `${platform.name} failure screenshot for ${failure.name}`,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
body: {
|
||||
type: 'mrkdwn',
|
||||
text: mrkdwnText(failure.message, 190),
|
||||
verbatim: true,
|
||||
},
|
||||
...(platform.artifactUrl
|
||||
? {
|
||||
subtext: {
|
||||
type: 'mrkdwn',
|
||||
text: `<${platform.artifactUrl}|Open logs and artifacts>`,
|
||||
verbatim: true,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
}
|
||||
})
|
||||
|
||||
return {type: 'carousel', block_id: 'maestro_failures', elements}
|
||||
}
|
||||
|
||||
function diagnosticBlock(platform) {
|
||||
const phase = mrkdwnText(
|
||||
platform.phase || 'No phase metadata was captured',
|
||||
220,
|
||||
)
|
||||
if (platform.status === 'cancelled') {
|
||||
return {
|
||||
type: 'section',
|
||||
text: {
|
||||
type: 'mrkdwn',
|
||||
text: `*${platform.name} cancelled*\nLatest phase: ${phase}\nResults may be incomplete.`,
|
||||
},
|
||||
}
|
||||
}
|
||||
if (!platform.hasJUnit) {
|
||||
return {
|
||||
type: 'section',
|
||||
text: {
|
||||
type: 'mrkdwn',
|
||||
text: `*${platform.name} setup failed*\nLatest phase: ${phase}\nNo JUnit results were produced.`,
|
||||
},
|
||||
}
|
||||
}
|
||||
return {
|
||||
type: 'section',
|
||||
text: {
|
||||
type: 'mrkdwn',
|
||||
text: `*${platform.name} job failed*\nLatest phase: ${phase}`,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function footerBlock(platforms, runUrl) {
|
||||
const links = [`<${runUrl}|Open workflow>`]
|
||||
for (const platform of platforms) {
|
||||
if (platform.artifactUrl) {
|
||||
links.push(`<${platform.artifactUrl}|${platform.name} artifacts>`)
|
||||
}
|
||||
}
|
||||
return {
|
||||
type: 'section',
|
||||
text: {type: 'mrkdwn', text: links.join(' • ')},
|
||||
}
|
||||
}
|
||||
|
||||
export function buildSlackMessage({
|
||||
platforms,
|
||||
sha,
|
||||
runUrl,
|
||||
commitUrl,
|
||||
slackFileIds = [],
|
||||
}) {
|
||||
const state = stateFor(platforms)
|
||||
const failureCount = platforms.reduce(
|
||||
(total, platform) => total + platform.failures.length,
|
||||
0,
|
||||
)
|
||||
const presentation = statePresentation(state, failureCount)
|
||||
const allFailures = selectFailures(platforms, failureCount)
|
||||
const selectedFailures = allFailures.slice(0, CAROUSEL_LIMIT)
|
||||
const uploadPayload = buildUploadPayload(allFailures)
|
||||
const detailBlocks = platforms
|
||||
.filter(
|
||||
platform =>
|
||||
platform.status === 'cancelled' ||
|
||||
(platform.failed && platform.failures.length === 0),
|
||||
)
|
||||
.map(diagnosticBlock)
|
||||
|
||||
if (selectedFailures.length > 0) {
|
||||
detailBlocks.push(buildCarousel(selectedFailures, slackFileIds), {
|
||||
type: 'context',
|
||||
elements: [
|
||||
{
|
||||
type: 'mrkdwn',
|
||||
text: `Showing ${selectedFailures.length} of ${pluralize(failureCount, 'failed flow')} • full details and screenshots are in the thread`,
|
||||
},
|
||||
],
|
||||
})
|
||||
}
|
||||
|
||||
const shortSha = sha.slice(0, 12)
|
||||
const blocks = [
|
||||
{
|
||||
type: 'header',
|
||||
text: {type: 'plain_text', text: presentation.header},
|
||||
},
|
||||
{
|
||||
type: 'context',
|
||||
elements: [
|
||||
{
|
||||
type: 'mrkdwn',
|
||||
text: `Commit <${commitUrl}|\`${shortSha}\`> • ${presentation.summary}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
fields: platforms.map(platform => ({
|
||||
type: 'mrkdwn',
|
||||
text: `*${platform.name}*\n${platformStatus(platform)}`,
|
||||
})),
|
||||
},
|
||||
...(detailBlocks.length > 0 ? [{type: 'divider'}, ...detailBlocks] : []),
|
||||
footerBlock(platforms, runUrl),
|
||||
]
|
||||
|
||||
return {
|
||||
state,
|
||||
failureCount,
|
||||
screenshotCount: uploadPayload.file_uploads.length,
|
||||
uploadPayload,
|
||||
threadPayload: buildThreadPayload(platforms),
|
||||
payload: {text: presentation.fallback, blocks},
|
||||
}
|
||||
}
|
||||
|
||||
export function extractSlackFileIds(response) {
|
||||
if (!response) return []
|
||||
|
||||
let parsed = response
|
||||
if (typeof response === 'string') {
|
||||
try {
|
||||
parsed = JSON.parse(response)
|
||||
} catch {
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
const ids = []
|
||||
const seen = new Set()
|
||||
function visit(value) {
|
||||
if (Array.isArray(value)) {
|
||||
for (const item of value) visit(item)
|
||||
return
|
||||
}
|
||||
if (!value || typeof value !== 'object') return
|
||||
if (
|
||||
typeof value.id === 'string' &&
|
||||
/^F[A-Z0-9]+$/.test(value.id) &&
|
||||
!seen.has(value.id)
|
||||
) {
|
||||
seen.add(value.id)
|
||||
ids.push(value.id)
|
||||
}
|
||||
for (const child of Object.values(value)) visit(child)
|
||||
}
|
||||
visit(parsed)
|
||||
return ids
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import test from 'node:test'
|
||||
|
||||
import {buildSlackMessage, extractSlackFileIds} from './maestro-slack.mjs'
|
||||
import {screenshotsByFlow} from './summarize-maestro.mjs'
|
||||
|
||||
function platform({
|
||||
name,
|
||||
status = 'success',
|
||||
failed = false,
|
||||
failures = [],
|
||||
phase = 'Completed',
|
||||
hasJUnit = true,
|
||||
}) {
|
||||
return {
|
||||
name,
|
||||
status,
|
||||
failed,
|
||||
failures,
|
||||
phase,
|
||||
hasJUnit,
|
||||
artifactUrl: `https://example.com/${name.toLowerCase()}`,
|
||||
}
|
||||
}
|
||||
|
||||
function build(platforms, slackFileIds = []) {
|
||||
return buildSlackMessage({
|
||||
platforms,
|
||||
sha: '1234567890abcdef',
|
||||
runUrl: 'https://example.com/run',
|
||||
commitUrl: 'https://example.com/commit',
|
||||
slackFileIds,
|
||||
})
|
||||
}
|
||||
|
||||
test('builds a screenshot carousel for failed flows', () => {
|
||||
const platforms = [
|
||||
platform({
|
||||
name: 'iOS',
|
||||
status: 'failure',
|
||||
failed: true,
|
||||
failures: [
|
||||
{
|
||||
name: 'composer',
|
||||
message: 'Element not found',
|
||||
screenshot: '/tmp/screenshot-(composer).png',
|
||||
},
|
||||
],
|
||||
}),
|
||||
platform({name: 'Android'}),
|
||||
]
|
||||
const summary = build(platforms, ['F123ABC'])
|
||||
const carousel = summary.payload.blocks.find(
|
||||
block => block.type === 'carousel',
|
||||
)
|
||||
|
||||
assert.equal(summary.state, 'failed')
|
||||
assert.equal(summary.screenshotCount, 1)
|
||||
assert.equal(
|
||||
summary.uploadPayload.file_uploads[0].file,
|
||||
platforms[0].failures[0].screenshot,
|
||||
)
|
||||
assert.equal(summary.uploadPayload.file_uploads[0].highlight_type, 'png')
|
||||
assert.deepEqual(carousel.elements[0].hero_image.slack_file, {id: 'F123ABC'})
|
||||
})
|
||||
|
||||
test('selects failures across both platforms for the carousel', () => {
|
||||
const failures = prefix =>
|
||||
Array.from({length: 7}, (_, index) => ({
|
||||
name: `${prefix}-${index}`,
|
||||
message: 'Failed',
|
||||
screenshot: `/tmp/${prefix}-${index}.png`,
|
||||
}))
|
||||
const summary = build([
|
||||
platform({
|
||||
name: 'iOS',
|
||||
status: 'failure',
|
||||
failed: true,
|
||||
failures: failures('ios'),
|
||||
}),
|
||||
platform({
|
||||
name: 'Android',
|
||||
status: 'failure',
|
||||
failed: true,
|
||||
failures: failures('android'),
|
||||
}),
|
||||
])
|
||||
const carousel = summary.payload.blocks.find(
|
||||
block => block.type === 'carousel',
|
||||
)
|
||||
|
||||
assert.equal(carousel.elements.length, 10)
|
||||
assert.equal(summary.failureCount, 14)
|
||||
assert.equal(summary.screenshotCount, 14)
|
||||
assert.equal(summary.uploadPayload.file_uploads.length, 14)
|
||||
assert.match(summary.threadPayload.text, /ios-6/)
|
||||
assert.match(summary.threadPayload.text, /android-6/)
|
||||
assert.equal(carousel.elements[0].subtitle.text, 'iOS · failed flow')
|
||||
assert.equal(carousel.elements[1].subtitle.text, 'Android · failed flow')
|
||||
})
|
||||
|
||||
test('uses a cancellation presentation for partial results', () => {
|
||||
const summary = build([
|
||||
platform({
|
||||
name: 'iOS',
|
||||
status: 'cancelled',
|
||||
failed: true,
|
||||
failures: [],
|
||||
phase: 'Building iOS development client',
|
||||
hasJUnit: false,
|
||||
}),
|
||||
platform({
|
||||
name: 'Android',
|
||||
status: 'cancelled',
|
||||
failed: true,
|
||||
failures: [],
|
||||
phase: 'Building Android development client',
|
||||
hasJUnit: false,
|
||||
}),
|
||||
])
|
||||
|
||||
assert.equal(summary.state, 'cancelled')
|
||||
assert.equal(
|
||||
summary.payload.blocks[0].text.text,
|
||||
'⏹️ Nightly Maestro E2E cancelled',
|
||||
)
|
||||
assert.match(summary.payload.text, /cancelled/)
|
||||
assert.equal(summary.screenshotCount, 0)
|
||||
})
|
||||
|
||||
test('distinguishes setup failures from failed Maestro flows', () => {
|
||||
const summary = build([
|
||||
platform({
|
||||
name: 'iOS',
|
||||
status: 'failure',
|
||||
failed: true,
|
||||
failures: [],
|
||||
phase: 'Starting Metro',
|
||||
hasJUnit: false,
|
||||
}),
|
||||
platform({name: 'Android', status: 'skipped'}),
|
||||
])
|
||||
|
||||
assert.equal(summary.state, 'setup_failed')
|
||||
assert.equal(
|
||||
summary.payload.blocks[0].text.text,
|
||||
'⚠️ Nightly Maestro setup failed',
|
||||
)
|
||||
assert.equal(
|
||||
summary.payload.blocks.some(block => block.type === 'carousel'),
|
||||
false,
|
||||
)
|
||||
})
|
||||
|
||||
test('extracts file ids from single and multi-file upload responses', () => {
|
||||
const response = {
|
||||
ok: true,
|
||||
files: [
|
||||
{
|
||||
ok: true,
|
||||
files: [{id: 'FONE'}, {id: 'FTWO'}],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
assert.deepEqual(extractSlackFileIds(response), ['FONE', 'FTWO'])
|
||||
assert.deepEqual(
|
||||
extractSlackFileIds(JSON.stringify({ok: true, files: [{id: 'FTHREE'}]})),
|
||||
['FTHREE'],
|
||||
)
|
||||
assert.deepEqual(extractSlackFileIds('not json'), [])
|
||||
})
|
||||
|
||||
test('selects the newest Maestro screenshot for each flow', () => {
|
||||
const screenshots = screenshotsByFlow([
|
||||
'/tmp/screenshot-❌-300-(composer).png',
|
||||
'/tmp/screenshot-❌-100-(composer).png',
|
||||
'/tmp/screenshot-❌-200-(login).png',
|
||||
'/tmp/artifacts/maestro/composer-self-label/screenshots/step-020-tapOnElement-openMediaBtn.png',
|
||||
'/tmp/artifacts/maestro/composer-self-label/screenshots/step-010-launchApp.png',
|
||||
'/tmp/not-a-maestro-screenshot.png',
|
||||
])
|
||||
|
||||
assert.equal(
|
||||
screenshots.get('composer'),
|
||||
'/tmp/screenshot-❌-300-(composer).png',
|
||||
)
|
||||
assert.equal(screenshots.get('login'), '/tmp/screenshot-❌-200-(login).png')
|
||||
assert.equal(
|
||||
screenshots.get('composer-self-label'),
|
||||
'/tmp/artifacts/maestro/composer-self-label/screenshots/step-020-tapOnElement-openMediaBtn.png',
|
||||
)
|
||||
assert.equal(screenshots.size, 3)
|
||||
})
|
||||
@@ -2,6 +2,8 @@ import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import process from 'node:process'
|
||||
|
||||
import {buildSlackMessage, extractSlackFileIds} from './maestro-slack.mjs'
|
||||
|
||||
const ENTITY_REPLACEMENTS = {
|
||||
'&': '&',
|
||||
''': "'",
|
||||
@@ -101,6 +103,54 @@ function readPhase(root) {
|
||||
return phaseFile ? fs.readFileSync(phaseFile, 'utf8').trim() : ''
|
||||
}
|
||||
|
||||
function screenshotMetadata(file) {
|
||||
const legacyMatch = path
|
||||
.basename(file)
|
||||
.match(/^screenshot-.*?-(\d+)-\((.+)\)\.(?:gif|jpe?g|png)$/i)
|
||||
if (legacyMatch) {
|
||||
return {
|
||||
file,
|
||||
order: Number(legacyMatch[1]),
|
||||
flowName: legacyMatch[2],
|
||||
}
|
||||
}
|
||||
|
||||
const parts = file.split(/[\\/]/)
|
||||
const screenshotsIndex = parts.lastIndexOf('screenshots')
|
||||
if (
|
||||
screenshotsIndex < 2 ||
|
||||
!parts.slice(0, screenshotsIndex - 1).includes('maestro')
|
||||
) {
|
||||
return undefined
|
||||
}
|
||||
const step = parts.at(-1)?.match(/^step-(\d+)-.*\.(?:gif|jpe?g|png)$/i)
|
||||
return step
|
||||
? {
|
||||
file,
|
||||
order: Number(step[1]),
|
||||
flowName: parts[screenshotsIndex - 1],
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
|
||||
export function screenshotsByFlow(files) {
|
||||
const screenshots = new Map()
|
||||
for (const file of files) {
|
||||
const screenshot = screenshotMetadata(file)
|
||||
if (!screenshot) continue
|
||||
const current = screenshots.get(screenshot.flowName)
|
||||
if (!current || screenshot.order > current.order) {
|
||||
screenshots.set(screenshot.flowName, screenshot)
|
||||
}
|
||||
}
|
||||
return new Map(
|
||||
[...screenshots].map(([flowName, screenshot]) => [
|
||||
flowName,
|
||||
screenshot.file,
|
||||
]),
|
||||
)
|
||||
}
|
||||
|
||||
function platformResult({name, status, root, artifactUrl}) {
|
||||
const files = walk(root)
|
||||
const reports = files.filter(file => /(?:report|junit).*\.xml$/i.test(file))
|
||||
@@ -115,7 +165,12 @@ function platformResult({name, status, root, artifactUrl}) {
|
||||
)
|
||||
// A cancelled or timed-out Maestro run may never flush JUnit. Its CLI log is
|
||||
// streamed continuously, so use those failure lines when JUnit has no detail.
|
||||
const failures = junitFailures.length > 0 ? junitFailures : cliFailures
|
||||
const rawFailures = junitFailures.length > 0 ? junitFailures : cliFailures
|
||||
const screenshots = screenshotsByFlow(files)
|
||||
const failures = rawFailures.map(failure => ({
|
||||
...failure,
|
||||
screenshot: screenshots.get(failure.name),
|
||||
}))
|
||||
// A skipped platform (e.g. iOS while temporarily disabled) is not a failure
|
||||
// as long as it produced no flow failures.
|
||||
const failed =
|
||||
@@ -131,52 +186,15 @@ function platformResult({name, status, root, artifactUrl}) {
|
||||
}
|
||||
}
|
||||
|
||||
function statusEmoji(status) {
|
||||
if (status === 'success') return ':white_check_mark:'
|
||||
if (status === 'skipped') return ':fast_forward:'
|
||||
return ':x:'
|
||||
}
|
||||
|
||||
function slackEscape(value) {
|
||||
return value
|
||||
.replaceAll('&', '&')
|
||||
.replaceAll('<', '<')
|
||||
.replaceAll('>', '>')
|
||||
}
|
||||
|
||||
function platformBlock(platform) {
|
||||
function githubSummary({state, platforms, shortSha, runUrl, commitUrl}) {
|
||||
const outcome =
|
||||
state === 'cancelled'
|
||||
? 'cancelled'
|
||||
: state === 'passed'
|
||||
? 'passed'
|
||||
: 'failed'
|
||||
const lines = [
|
||||
`${statusEmoji(platform.status)} *${platform.name}* — job status: \`${platform.status}\``,
|
||||
]
|
||||
if (platform.failures.length > 0) {
|
||||
for (const failure of platform.failures.slice(0, 8)) {
|
||||
lines.push(
|
||||
`• *${slackEscape(failure.name)}:* ${slackEscape(failure.message)}`,
|
||||
)
|
||||
}
|
||||
if (platform.failures.length > 8) {
|
||||
lines.push(`• …and ${platform.failures.length - 8} more failed flows`)
|
||||
}
|
||||
} else if (platform.failed && !platform.hasJUnit) {
|
||||
lines.push(
|
||||
`• *Setup phase:* ${slackEscape(platform.phase || 'No phase metadata was captured')}`,
|
||||
)
|
||||
} else if (platform.failed) {
|
||||
lines.push(
|
||||
`• Job failed after JUnit was written; latest phase: ${slackEscape(platform.phase || 'unknown')}`,
|
||||
)
|
||||
}
|
||||
if (platform.artifactUrl) {
|
||||
lines.push(
|
||||
`• <${platform.artifactUrl}|Open ${platform.name} logs and artifacts>`,
|
||||
)
|
||||
}
|
||||
return lines.join('\n').slice(0, 3000)
|
||||
}
|
||||
|
||||
function githubSummary({notify, platforms, shortSha, runUrl, commitUrl}) {
|
||||
const lines = [
|
||||
`# Nightly Maestro E2E ${notify ? 'failed' : 'passed'}`,
|
||||
`# Nightly Maestro E2E ${outcome}`,
|
||||
'',
|
||||
`- Commit: [\`${shortSha}\`](${commitUrl})`,
|
||||
`- Workflow run: [open run](${runUrl})`,
|
||||
@@ -235,6 +253,7 @@ export function buildSummary({
|
||||
sha,
|
||||
runUrl,
|
||||
commitUrl,
|
||||
slackFileIds = [],
|
||||
}) {
|
||||
const platforms = [
|
||||
platformResult({
|
||||
@@ -252,73 +271,29 @@ export function buildSummary({
|
||||
]
|
||||
const notify = platforms.some(platform => platform.failed)
|
||||
const shortSha = sha.slice(0, 12)
|
||||
const lines = [
|
||||
':rotating_light: *Nightly Maestro E2E failed*',
|
||||
`*Commit:* <${commitUrl}|\`${shortSha}\`>`,
|
||||
`*Workflow run:* <${runUrl}|open run>`,
|
||||
'',
|
||||
]
|
||||
|
||||
for (const platform of platforms) {
|
||||
lines.push(
|
||||
`${statusEmoji(platform.status)} *${platform.name}* — job status: \`${platform.status}\``,
|
||||
)
|
||||
if (platform.failures.length > 0) {
|
||||
for (const failure of platform.failures.slice(0, 10)) {
|
||||
lines.push(
|
||||
`• *${slackEscape(failure.name)}:* ${slackEscape(failure.message)}`,
|
||||
)
|
||||
}
|
||||
if (platform.failures.length > 10) {
|
||||
lines.push(`• …and ${platform.failures.length - 10} more failed flows`)
|
||||
}
|
||||
} else if (platform.failed && !platform.hasJUnit) {
|
||||
lines.push(
|
||||
`• Setup phase: ${platform.phase || 'No phase metadata was captured'}`,
|
||||
)
|
||||
} else if (platform.failed) {
|
||||
lines.push(
|
||||
`• The job failed after JUnit was written (latest phase: ${platform.phase || 'unknown'})`,
|
||||
)
|
||||
}
|
||||
if (platform.artifactUrl) {
|
||||
lines.push(
|
||||
`• <${platform.artifactUrl}|${platform.name} logs and artifacts>`,
|
||||
)
|
||||
}
|
||||
lines.push('')
|
||||
}
|
||||
|
||||
const text = lines.join('\n').trim()
|
||||
const blocks = [
|
||||
{
|
||||
type: 'header',
|
||||
text: {type: 'plain_text', text: 'Nightly Maestro E2E failed'},
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: {
|
||||
type: 'mrkdwn',
|
||||
text: `*Commit:* <${commitUrl}|\`${shortSha}\`>\n*Workflow run:* <${runUrl}|open run>`,
|
||||
},
|
||||
},
|
||||
{type: 'divider'},
|
||||
...platforms.flatMap((platform, index) => [
|
||||
{type: 'section', text: {type: 'mrkdwn', text: platformBlock(platform)}},
|
||||
...(index < platforms.length - 1 ? [{type: 'divider'}] : []),
|
||||
]),
|
||||
]
|
||||
const slack = buildSlackMessage({
|
||||
platforms,
|
||||
sha,
|
||||
runUrl,
|
||||
commitUrl,
|
||||
slackFileIds,
|
||||
})
|
||||
return {
|
||||
notify,
|
||||
state: slack.state,
|
||||
platforms,
|
||||
githubSummary: githubSummary({
|
||||
notify,
|
||||
state: slack.state,
|
||||
platforms,
|
||||
shortSha,
|
||||
runUrl,
|
||||
commitUrl,
|
||||
}),
|
||||
payload: {text, blocks},
|
||||
failureCount: slack.failureCount,
|
||||
screenshotCount: slack.screenshotCount,
|
||||
uploadPayload: slack.uploadPayload,
|
||||
threadPayload: slack.threadPayload,
|
||||
payload: slack.payload,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -351,6 +326,7 @@ if (
|
||||
sha: args.sha,
|
||||
runUrl: args['run-url'],
|
||||
commitUrl: args['commit-url'],
|
||||
slackFileIds: extractSlackFileIds(args['slack-upload-response']),
|
||||
})
|
||||
process.stdout.write(`${JSON.stringify(summary)}\n`)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user