Add nightly Maestro E2E workflow (#11181)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Samuel Newman
2026-07-20 17:42:53 +03:00
committed by GitHub
parent c4e80b4456
commit 0fb7251c64
25 changed files with 1508 additions and 119 deletions
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env bash
set +e
platform="${1:?usage: cleanup-nightly-e2e.sh <ios|android> <device-id>}"
device_id="${2:-}"
artifact_dir="${GITHUB_WORKSPACE:-$PWD}/artifacts/$platform"
mkdir -p "$artifact_dir"
if [[ -f i18n.log ]]; then
cp i18n.log "$artifact_dir/i18n.log"
fi
stop_process_tree() {
local pid="$1"
local child
while read -r child; do
[[ -n "$child" ]] && stop_process_tree "$child"
done < <(pgrep -P "$pid" 2>/dev/null || true)
kill -TERM "$pid" >/dev/null 2>&1 || true
}
stop_pid_file() {
[[ -f "$1" ]] || return 0
local pid
pid="$(cat "$1")"
stop_process_tree "$pid"
}
stop_pid_file "$artifact_dir/logcat.pid"
stop_pid_file "$artifact_dir/metro.pid"
stop_pid_file "$artifact_dir/mock-server.pid"
stop_pid_file "$artifact_dir/emulator.pid"
if [[ "$platform" == "ios" ]]; then
if [[ -f "$artifact_dir/redis-bin.txt" ]]; then
"$(cat "$artifact_dir/redis-bin.txt")/redis-cli" \
-h 127.0.0.1 -p 6380 shutdown nosave >/dev/null 2>&1 || true
fi
if [[ -f "$artifact_dir/postgres-bin.txt" ]]; then
"$(cat "$artifact_dir/postgres-bin.txt")/pg_ctl" \
-D "${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres" -m fast stop >/dev/null 2>&1 || true
fi
[[ -n "$device_id" ]] && xcrun simctl shutdown "$device_id" >/dev/null 2>&1 || true
else
docker compose -f dev-env/dev-infra/docker-compose.yaml logs --no-color \
>>"$artifact_dir/docker-services.log" 2>&1 || true
docker compose -f dev-env/dev-infra/docker-compose.yaml down --volumes --remove-orphans >/dev/null 2>&1 || true
[[ -n "$device_id" ]] && adb -s "$device_id" emu kill >/dev/null 2>&1 || true
fi
+179
View File
@@ -0,0 +1,179 @@
#!/usr/bin/env bash
set -Eeuo pipefail
platform="${1:?usage: run-nightly-e2e.sh <ios|android> <device-id>}"
device_id="${2:?usage: run-nightly-e2e.sh <ios|android> <device-id>}"
if [[ "$platform" != "ios" && "$platform" != "android" ]]; then
echo "Unsupported platform: $platform" >&2
exit 2
fi
artifact_dir="${GITHUB_WORKSPACE:-$PWD}/artifacts/$platform"
maestro_dir="$artifact_dir/maestro"
mkdir -p "$maestro_dir"
phase() {
printf '%s\n' "$1" >"$artifact_dir/phase.txt"
}
wait_for_port() {
local port="$1"
local label="$2"
local attempts="${3:-120}"
for ((i = 1; i <= attempts; i++)); do
if nc -z 127.0.0.1 "$port" >/dev/null 2>&1; then
return 0
fi
sleep 1
done
echo "Timed out waiting for $label on port $port" >&2
return 1
}
# shellcheck disable=SC2329 # Invoked through the cleanup trap call chain.
stop_process_tree() {
local pid="$1"
local child
while read -r child; do
[[ -n "$child" ]] && stop_process_tree "$child"
done < <(pgrep -P "$pid" 2>/dev/null || true)
kill -TERM "$pid" >/dev/null 2>&1 || true
}
# shellcheck disable=SC2329 # Invoked by cleanup, which is registered as a trap.
stop_pid_file() {
local pid_file="$1"
[[ -f "$pid_file" ]] || return 0
local pid
pid="$(cat "$pid_file")"
[[ -n "$pid" ]] || return 0
# pnpm and Expo both spawn multiple generations of children.
stop_process_tree "$pid"
}
# shellcheck disable=SC2329 # Invoked by the EXIT/INT/TERM trap below.
cleanup() {
set +e
stop_pid_file "$artifact_dir/logcat.pid"
stop_pid_file "$artifact_dir/metro.pid"
stop_pid_file "$artifact_dir/mock-server.pid"
if [[ "$platform" == "ios" ]]; then
if [[ -f "$artifact_dir/redis.pid" ]]; then
redis_bin="$(cat "$artifact_dir/redis-bin.txt")"
"$redis_bin/redis-cli" -h 127.0.0.1 -p 6380 shutdown nosave >/dev/null 2>&1 || true
fi
if [[ -f "$artifact_dir/postgres-bin.txt" && -d "${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres" ]]; then
postgres_bin="$(cat "$artifact_dir/postgres-bin.txt")"
"$postgres_bin/pg_ctl" -D "${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres" -m fast stop >/dev/null 2>&1 || true
fi
else
docker compose -f dev-env/dev-infra/docker-compose.yaml logs --no-color \
>>"$artifact_dir/docker-services.log" 2>&1 || true
docker compose -f dev-env/dev-infra/docker-compose.yaml down --volumes --remove-orphans >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT INT TERM
if [[ "$platform" == "android" ]]; then
adb -s "$device_id" logcat -c
adb -s "$device_id" logcat -v threadtime >"$artifact_dir/logcat.log" 2>&1 &
printf '%s\n' "$!" >"$artifact_dir/logcat.pid"
fi
phase "Starting PostgreSQL, Redis, and mock server"
if [[ "$platform" == "ios" ]]; then
brew install postgresql@14 2>&1 | tee "$artifact_dir/native-dependencies.log"
postgres_bin="$(brew --prefix postgresql@14)/bin"
redis_version="7.4.7"
redis_archive="${RUNNER_TEMP:-/tmp}/redis-${redis_version}.tar.gz"
redis_source="${RUNNER_TEMP:-/tmp}/redis-${redis_version}"
curl -fsSL -o "$redis_archive" \
"https://download.redis.io/releases/redis-${redis_version}.tar.gz"
echo "c97e57b0df330a9e091cacff012bebe763c275398cf36ff44cdba876814b595b $redis_archive" \
| shasum -a 256 --check | tee -a "$artifact_dir/native-dependencies.log"
rm -rf "$redis_source"
tar -xzf "$redis_archive" -C "${RUNNER_TEMP:-/tmp}"
make -C "$redis_source" -j "$(sysctl -n hw.ncpu)" \
2>&1 | tee -a "$artifact_dir/native-dependencies.log"
redis_bin="$redis_source/src"
"$redis_bin/redis-server" --version | tee -a "$artifact_dir/native-dependencies.log"
printf '%s\n' "$redis_bin" >"$artifact_dir/redis-bin.txt"
postgres_data="${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres"
rm -rf "$postgres_data"
"$postgres_bin/initdb" -D "$postgres_data" --auth=trust --username=pg --no-locale \
>"$artifact_dir/postgres-init.log" 2>&1
"$postgres_bin/pg_ctl" -D "$postgres_data" \
-o "-p 5433 -h 127.0.0.1" -l "$artifact_dir/postgres.log" start
printf '%s\n' "$postgres_bin" >"$artifact_dir/postgres-bin.txt"
"$redis_bin/redis-server" \
--bind 127.0.0.1 \
--port 6380 \
--save "" \
--appendonly no \
--daemonize yes \
--pidfile "$artifact_dir/redis.pid" \
--logfile "$artifact_dir/redis.log"
wait_for_port 5433 "PostgreSQL"
wait_for_port 6380 "Redis"
pnpm --dir dev-env start:external >"$artifact_dir/mock-server.log" 2>&1 &
else
pnpm --dir dev-env start >"$artifact_dir/mock-server.log" 2>&1 &
fi
printf '%s\n' "$!" >"$artifact_dir/mock-server.pid"
wait_for_port 1986 "the E2E mock-server manager"
phase "Starting Metro"
EXPO_PUBLIC_ENV=e2e \
NODE_ENV=test \
RN_SRC_EXT=e2e.ts,e2e.tsx \
pnpm exec expo start --dev-client --clear --port 8081 \
>"$artifact_dir/metro.log" 2>&1 &
printf '%s\n' "$!" >"$artifact_dir/metro.pid"
wait_for_port 8081 "Metro"
# Pre-warm Metro bundle so the first Maestro flow doesn't hit a cold-start delay
phase "Pre-warming Metro bundle"
bundle_platform="$platform"
curl -s -o /dev/null "http://localhost:8081/index.bundle?platform=${bundle_platform}&dev=true&minify=false"
echo "Metro bundle pre-warmed for $bundle_platform"
if [[ "$platform" == "android" ]]; then
phase "Configuring Android localhost routing"
adb -s "$device_id" reverse tcp:3000 tcp:3000
adb -s "$device_id" reverse tcp:8081 tcp:8081
fi
phase "Running Maestro flows"
set +e
maestro test \
--udid "$device_id" \
--format JUNIT \
--output "$artifact_dir/report.xml" \
--config __e2e__/config.yml \
--debug-output "$maestro_dir" \
--test-output-dir "$maestro_dir" \
--flatten-debug-output \
__e2e__ \
2>&1 | tee "$artifact_dir/maestro-cli.log"
maestro_status=${PIPESTATUS[0]}
set -e
if [[ "$maestro_status" -eq 0 ]]; then
phase "Completed"
else
phase "Maestro flow failure"
fi
exit "$maestro_status"
+356
View File
@@ -0,0 +1,356 @@
import fs from 'node:fs'
import path from 'node:path'
import process from 'node:process'
const ENTITY_REPLACEMENTS = {
'&amp;': '&',
'&apos;': "'",
'&gt;': '>',
'&lt;': '<',
'&quot;': '"',
}
function decodeXml(value = '') {
return value
.replace(/&(amp|apos|gt|lt|quot);/g, entity => ENTITY_REPLACEMENTS[entity])
.replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
.replace(/&#x([\da-f]+);/gi, (_, code) =>
String.fromCodePoint(Number.parseInt(code, 16)),
)
}
function attributes(source = '') {
const result = {}
for (const match of source.matchAll(/([\w:.-]+)\s*=\s*(["'])(.*?)\2/gs)) {
result[match[1]] = decodeXml(match[3])
}
return result
}
function concise(value, limit = 300) {
const normalized = decodeXml(value)
.replace(/<[^>]+>/g, ' ')
.replace(/\s+/g, ' ')
.trim()
return normalized.length > limit
? `${normalized.slice(0, limit - 1)}`
: normalized
}
export function parseJUnit(xml) {
const failures = []
const testcasePattern = /<testcase\b([^>]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/gi
for (const match of xml.matchAll(testcasePattern)) {
const testcase = attributes(match[1])
const body = match[2] || ''
const failure = body.match(/<(failure|error)\b([^>]*)>([\s\S]*?)<\/\1>/i)
const selfClosingFailure = body.match(/<(failure|error)\b([^>]*)\/>/i)
const failureMatch = failure || selfClosingFailure
if (!failureMatch) continue
const failureAttributes = attributes(failureMatch[2])
const name = testcase.name || testcase.classname || 'Unnamed Maestro flow'
const message = concise(
failureAttributes.message || (failure ? failureMatch[3] : '') || 'Failed',
)
failures.push({name, message})
}
if (failures.length === 0) {
const suite = xml.match(/<testsuite\b([^>]*)>/i)
const suiteAttributes = attributes(suite?.[1])
if (
Number(suiteAttributes.failures || 0) +
Number(suiteAttributes.errors || 0) >
0
) {
failures.push({
name: suiteAttributes.name || 'Maestro test suite',
message: 'JUnit reported a failure without testcase details',
})
}
}
return failures
}
export function parseMaestroCli(log) {
const failures = []
const failurePattern = /^\[Failed\]\s+(.+?)\s+\([^)]*\)\s+\((.+)\)\s*$/gm
for (const match of log.matchAll(failurePattern)) {
failures.push({
name: concise(match[1], 120),
message: concise(match[2]),
})
}
return failures
}
function walk(root) {
if (!root || !fs.existsSync(root)) return []
const entries = fs.readdirSync(root, {withFileTypes: true})
return entries.flatMap(entry => {
const candidate = path.join(root, entry.name)
return entry.isDirectory() ? walk(candidate) : [candidate]
})
}
function readPhase(root) {
const phaseFile = walk(root).find(file => path.basename(file) === 'phase.txt')
return phaseFile ? fs.readFileSync(phaseFile, 'utf8').trim() : ''
}
function platformResult({name, status, root, artifactUrl}) {
const files = walk(root)
const reports = files.filter(file => /(?:report|junit).*\.xml$/i.test(file))
const junitFailures = reports.flatMap(report =>
parseJUnit(fs.readFileSync(report, 'utf8')),
)
const maestroLogs = files.filter(
file => path.basename(file) === 'maestro-cli.log',
)
const cliFailures = maestroLogs.flatMap(log =>
parseMaestroCli(fs.readFileSync(log, 'utf8')),
)
// A cancelled or timed-out Maestro run may never flush JUnit. Its CLI log is
// streamed continuously, so use those failure lines when JUnit has no detail.
const failures = junitFailures.length > 0 ? junitFailures : cliFailures
// A skipped platform (e.g. iOS while temporarily disabled) is not a failure
// as long as it produced no flow failures.
const failed =
(status !== 'success' && status !== 'skipped') || failures.length > 0
return {
name,
status,
failed,
failures,
phase: readPhase(root),
hasJUnit: reports.length > 0,
artifactUrl,
}
}
function statusEmoji(status) {
if (status === 'success') return ':white_check_mark:'
if (status === 'skipped') return ':fast_forward:'
return ':x:'
}
function slackEscape(value) {
return value
.replaceAll('&', '&amp;')
.replaceAll('<', '&lt;')
.replaceAll('>', '&gt;')
}
function platformBlock(platform) {
const lines = [
`${statusEmoji(platform.status)} *${platform.name}* — job status: \`${platform.status}\``,
]
if (platform.failures.length > 0) {
for (const failure of platform.failures.slice(0, 8)) {
lines.push(
`• *${slackEscape(failure.name)}:* ${slackEscape(failure.message)}`,
)
}
if (platform.failures.length > 8) {
lines.push(`• …and ${platform.failures.length - 8} more failed flows`)
}
} else if (platform.failed && !platform.hasJUnit) {
lines.push(
`• *Setup phase:* ${slackEscape(platform.phase || 'No phase metadata was captured')}`,
)
} else if (platform.failed) {
lines.push(
`• Job failed after JUnit was written; latest phase: ${slackEscape(platform.phase || 'unknown')}`,
)
}
if (platform.artifactUrl) {
lines.push(
`• <${platform.artifactUrl}|Open ${platform.name} logs and artifacts>`,
)
}
return lines.join('\n').slice(0, 3000)
}
function githubSummary({notify, platforms, shortSha, runUrl, commitUrl}) {
const lines = [
`# Nightly Maestro E2E ${notify ? 'failed' : 'passed'}`,
'',
`- Commit: [\`${shortSha}\`](${commitUrl})`,
`- Workflow run: [open run](${runUrl})`,
'',
]
for (const platform of platforms) {
const headerEmoji =
platform.status === 'success'
? '✅'
: platform.status === 'skipped'
? '⏭️'
: '❌'
lines.push(
`## ${headerEmoji} ${platform.name}`,
'',
`Job status: \`${platform.status}\``,
'',
)
if (platform.failures.length > 0) {
for (const failure of platform.failures.slice(0, 10)) {
lines.push(`- **${failure.name}:** ${failure.message}`)
}
if (platform.failures.length > 10) {
lines.push(`- …and ${platform.failures.length - 10} more failed flows`)
}
lines.push('')
} else if (platform.failed && !platform.hasJUnit) {
lines.push(
`- Setup phase: ${platform.phase || 'No phase metadata was captured'}`,
'',
)
} else if (platform.failed) {
lines.push(
`- The job failed after JUnit was written (latest phase: ${platform.phase || 'unknown'})`,
'',
)
}
if (platform.artifactUrl) {
lines.push(
`- [${platform.name} logs and artifacts](${platform.artifactUrl})`,
'',
)
}
}
return lines.join('\n').trim()
}
export function buildSummary({
iosStatus,
androidStatus,
iosRoot,
androidRoot,
artifactUrls = {},
sha,
runUrl,
commitUrl,
}) {
const platforms = [
platformResult({
name: 'iOS',
status: iosStatus,
root: iosRoot,
artifactUrl: artifactUrls.ios,
}),
platformResult({
name: 'Android',
status: androidStatus,
root: androidRoot,
artifactUrl: artifactUrls.android,
}),
]
const notify = platforms.some(platform => platform.failed)
const shortSha = sha.slice(0, 12)
const lines = [
':rotating_light: *Nightly Maestro E2E failed*',
`*Commit:* <${commitUrl}|\`${shortSha}\`>`,
`*Workflow run:* <${runUrl}|open run>`,
'',
]
for (const platform of platforms) {
lines.push(
`${statusEmoji(platform.status)} *${platform.name}* — job status: \`${platform.status}\``,
)
if (platform.failures.length > 0) {
for (const failure of platform.failures.slice(0, 10)) {
lines.push(
`• *${slackEscape(failure.name)}:* ${slackEscape(failure.message)}`,
)
}
if (platform.failures.length > 10) {
lines.push(`• …and ${platform.failures.length - 10} more failed flows`)
}
} else if (platform.failed && !platform.hasJUnit) {
lines.push(
`• Setup phase: ${platform.phase || 'No phase metadata was captured'}`,
)
} else if (platform.failed) {
lines.push(
`• The job failed after JUnit was written (latest phase: ${platform.phase || 'unknown'})`,
)
}
if (platform.artifactUrl) {
lines.push(
`• <${platform.artifactUrl}|${platform.name} logs and artifacts>`,
)
}
lines.push('')
}
const text = lines.join('\n').trim()
const blocks = [
{
type: 'header',
text: {type: 'plain_text', text: 'Nightly Maestro E2E failed'},
},
{
type: 'section',
text: {
type: 'mrkdwn',
text: `*Commit:* <${commitUrl}|\`${shortSha}\`>\n*Workflow run:* <${runUrl}|open run>`,
},
},
{type: 'divider'},
...platforms.flatMap((platform, index) => [
{type: 'section', text: {type: 'mrkdwn', text: platformBlock(platform)}},
...(index < platforms.length - 1 ? [{type: 'divider'}] : []),
]),
]
return {
notify,
platforms,
githubSummary: githubSummary({
notify,
platforms,
shortSha,
runUrl,
commitUrl,
}),
payload: {text, blocks},
}
}
function parseArgs(argv) {
const result = {}
for (let i = 0; i < argv.length; i += 2) {
const key = argv[i]
if (!key?.startsWith('--') || argv[i + 1] === undefined) {
throw new Error(`Invalid argument: ${key || '<missing>'}`)
}
result[key.slice(2)] = argv[i + 1]
}
return result
}
if (
process.argv[1] &&
path.resolve(process.argv[1]) === path.resolve(import.meta.filename)
) {
const args = parseArgs(process.argv.slice(2))
const artifactUrls = args['artifact-urls']
? JSON.parse(fs.readFileSync(args['artifact-urls'], 'utf8'))
: {}
const summary = buildSummary({
iosStatus: args['ios-status'],
androidStatus: args['android-status'],
iosRoot: args['ios-root'],
androidRoot: args['android-root'],
artifactUrls,
sha: args.sha,
runUrl: args['run-url'],
commitUrl: args['commit-url'],
})
process.stdout.write(`${JSON.stringify(summary)}\n`)
}