Add nightly Maestro E2E workflow

This commit is contained in:
Samuel Newman
2026-07-17 13:04:12 +03:00
parent 21d53ce0f8
commit 90267eb655
11 changed files with 841 additions and 1 deletions
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env bash
set +e
platform="${1:?usage: cleanup-nightly-e2e.sh <ios|android> <device-id>}"
device_id="${2:-}"
artifact_dir="${GITHUB_WORKSPACE:-$PWD}/artifacts/$platform"
stop_process_tree() {
local pid="$1"
local child
while read -r child; do
[[ -n "$child" ]] && stop_process_tree "$child"
done < <(pgrep -P "$pid" 2>/dev/null || true)
kill -TERM "$pid" >/dev/null 2>&1 || true
}
stop_pid_file() {
[[ -f "$1" ]] || return 0
local pid
pid="$(cat "$1")"
stop_process_tree "$pid"
}
stop_pid_file "$artifact_dir/metro.pid"
stop_pid_file "$artifact_dir/mock-server.pid"
if [[ "$platform" == "ios" ]]; then
if [[ -f "$artifact_dir/redis-bin.txt" ]]; then
"$(cat "$artifact_dir/redis-bin.txt")/redis-cli" \
-h 127.0.0.1 -p 6380 shutdown nosave >/dev/null 2>&1 || true
fi
if [[ -f "$artifact_dir/postgres-bin.txt" ]]; then
"$(cat "$artifact_dir/postgres-bin.txt")/pg_ctl" \
-D "${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres" -m fast stop >/dev/null 2>&1 || true
fi
[[ -n "$device_id" ]] && xcrun simctl shutdown "$device_id" >/dev/null 2>&1 || true
else
docker compose -f dev-env/dev-infra/docker-compose.yaml logs --no-color \
>>"$artifact_dir/docker-services.log" 2>&1 || true
docker compose -f dev-env/dev-infra/docker-compose.yaml down --volumes --remove-orphans >/dev/null 2>&1 || true
[[ -n "$device_id" ]] && adb -s "$device_id" emu kill >/dev/null 2>&1 || true
fi
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="Maestro Android" tests="2" failures="1" errors="0">
<testcase name="login.yml" classname="Maestro" time="11.8" />
<testcase name="create-account.yml" classname="Maestro" time="8.1">
<failure message="Element not found: Create account">The expected button was not visible.</failure>
</testcase>
</testsuite>
@@ -0,0 +1 @@
Building and installing the development client
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="Maestro Android" tests="1" failures="0" errors="0">
<testcase name="login.yml" classname="Maestro" time="11.8" />
</testsuite>
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="Maestro iOS" tests="1" failures="0" errors="0">
<testcase name="login.yml" classname="Maestro" time="12.4" />
</testsuite>
+181
View File
@@ -0,0 +1,181 @@
#!/usr/bin/env bash
set -Eeuo pipefail
platform="${1:?usage: run-nightly-e2e.sh <ios|android> <device-id>}"
device_id="${2:?usage: run-nightly-e2e.sh <ios|android> <device-id>}"
if [[ "$platform" != "ios" && "$platform" != "android" ]]; then
echo "Unsupported platform: $platform" >&2
exit 2
fi
artifact_dir="${GITHUB_WORKSPACE:-$PWD}/artifacts/$platform"
maestro_dir="$artifact_dir/maestro"
mkdir -p "$maestro_dir"
phase() {
printf '%s\n' "$1" >"$artifact_dir/phase.txt"
}
wait_for_port() {
local port="$1"
local label="$2"
local attempts="${3:-120}"
for ((i = 1; i <= attempts; i++)); do
if nc -z 127.0.0.1 "$port" >/dev/null 2>&1; then
return 0
fi
sleep 1
done
echo "Timed out waiting for $label on port $port" >&2
return 1
}
# shellcheck disable=SC2329 # Invoked through the cleanup trap call chain.
stop_process_tree() {
local pid="$1"
local child
while read -r child; do
[[ -n "$child" ]] && stop_process_tree "$child"
done < <(pgrep -P "$pid" 2>/dev/null || true)
kill -TERM "$pid" >/dev/null 2>&1 || true
}
# shellcheck disable=SC2329 # Invoked by cleanup, which is registered as a trap.
stop_pid_file() {
local pid_file="$1"
[[ -f "$pid_file" ]] || return 0
local pid
pid="$(cat "$pid_file")"
[[ -n "$pid" ]] || return 0
# pnpm and Expo both spawn multiple generations of children.
stop_process_tree "$pid"
}
# shellcheck disable=SC2329 # Invoked by the EXIT/INT/TERM trap below.
cleanup() {
set +e
stop_pid_file "$artifact_dir/metro.pid"
stop_pid_file "$artifact_dir/mock-server.pid"
if [[ "$platform" == "ios" ]]; then
if [[ -f "$artifact_dir/redis.pid" ]]; then
redis_bin="$(cat "$artifact_dir/redis-bin.txt")"
"$redis_bin/redis-cli" -h 127.0.0.1 -p 6380 shutdown nosave >/dev/null 2>&1 || true
fi
if [[ -f "$artifact_dir/postgres-bin.txt" && -d "${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres" ]]; then
postgres_bin="$(cat "$artifact_dir/postgres-bin.txt")"
"$postgres_bin/pg_ctl" -D "${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres" -m fast stop >/dev/null 2>&1 || true
fi
else
docker compose -f dev-env/dev-infra/docker-compose.yaml logs --no-color \
>>"$artifact_dir/docker-services.log" 2>&1 || true
docker compose -f dev-env/dev-infra/docker-compose.yaml down --volumes --remove-orphans >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT INT TERM
phase "Starting PostgreSQL, Redis, and mock server"
if [[ "$platform" == "ios" ]]; then
brew install postgresql@14 2>&1 | tee "$artifact_dir/native-dependencies.log"
postgres_bin="$(brew --prefix postgresql@14)/bin"
redis_version="7.4.7"
redis_archive="${RUNNER_TEMP:-/tmp}/redis-${redis_version}.tar.gz"
redis_source="${RUNNER_TEMP:-/tmp}/redis-${redis_version}"
curl -fsSL -o "$redis_archive" \
"https://download.redis.io/releases/redis-${redis_version}.tar.gz"
echo "c97e57b0df330a9e091cacff012bebe763c275398cf36ff44cdba876814b595b $redis_archive" \
| shasum -a 256 --check | tee -a "$artifact_dir/native-dependencies.log"
rm -rf "$redis_source"
tar -xzf "$redis_archive" -C "${RUNNER_TEMP:-/tmp}"
make -C "$redis_source" -j "$(sysctl -n hw.ncpu)" \
2>&1 | tee -a "$artifact_dir/native-dependencies.log"
redis_bin="$redis_source/src"
"$redis_bin/redis-server" --version | tee -a "$artifact_dir/native-dependencies.log"
printf '%s\n' "$redis_bin" >"$artifact_dir/redis-bin.txt"
postgres_data="${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres"
rm -rf "$postgres_data"
"$postgres_bin/initdb" -D "$postgres_data" --auth=trust --username=pg --no-locale \
>"$artifact_dir/postgres-init.log" 2>&1
"$postgres_bin/pg_ctl" -D "$postgres_data" \
-o "-p 5433 -h 127.0.0.1" -l "$artifact_dir/postgres.log" start
printf '%s\n' "$postgres_bin" >"$artifact_dir/postgres-bin.txt"
"$redis_bin/redis-server" \
--bind 127.0.0.1 \
--port 6380 \
--save "" \
--appendonly no \
--daemonize yes \
--pidfile "$artifact_dir/redis.pid" \
--logfile "$artifact_dir/redis.log"
wait_for_port 5433 "PostgreSQL"
wait_for_port 6380 "Redis"
pnpm --dir dev-env start:external >"$artifact_dir/mock-server.log" 2>&1 &
else
pnpm --dir dev-env start >"$artifact_dir/mock-server.log" 2>&1 &
fi
printf '%s\n' "$!" >"$artifact_dir/mock-server.pid"
wait_for_port 1986 "the E2E mock-server manager"
phase "Starting Metro"
EXPO_PUBLIC_ENV=e2e \
NODE_ENV=test \
RN_SRC_EXT=e2e.ts,e2e.tsx \
pnpm exec expo start --dev-client --clear --port 8081 \
>"$artifact_dir/metro.log" 2>&1 &
printf '%s\n' "$!" >"$artifact_dir/metro.pid"
wait_for_port 8081 "Metro"
if [[ "$platform" == "android" ]]; then
phase "Configuring Android localhost routing"
adb -s "$device_id" reverse tcp:3000 tcp:3000
adb -s "$device_id" reverse tcp:8081 tcp:8081
fi
phase "Building and installing the development client"
if [[ "$platform" == "ios" ]]; then
EXPO_PUBLIC_ENV=e2e \
NODE_ENV=test \
RN_SRC_EXT=e2e.ts,e2e.tsx \
pnpm exec expo run:ios --device "$device_id" --no-bundler \
2>&1 | tee "$artifact_dir/build.log"
else
EXPO_PUBLIC_ENV=e2e \
NODE_ENV=test \
RN_SRC_EXT=e2e.ts,e2e.tsx \
pnpm exec expo run:android --device "$device_id" --no-bundler \
2>&1 | tee "$artifact_dir/build.log"
fi
phase "Running Maestro flows"
set +e
maestro test \
--udid "$device_id" \
--format JUNIT \
--output "$artifact_dir/report.xml" \
--config __e2e__/config.yml \
--debug-output "$maestro_dir" \
--test-output-dir "$maestro_dir" \
--flatten-debug-output \
__e2e__ \
2>&1 | tee "$artifact_dir/maestro-cli.log"
maestro_status=${PIPESTATUS[0]}
set -e
if [[ "$maestro_status" -eq 0 ]]; then
phase "Completed"
else
phase "Maestro flow failure"
fi
exit "$maestro_status"
+224
View File
@@ -0,0 +1,224 @@
import fs from 'node:fs'
import path from 'node:path'
import process from 'node:process'
const ENTITY_REPLACEMENTS = {
'&amp;': '&',
'&apos;': "'",
'&gt;': '>',
'&lt;': '<',
'&quot;': '"',
}
function decodeXml(value = '') {
return value
.replace(/&(amp|apos|gt|lt|quot);/g, entity => ENTITY_REPLACEMENTS[entity])
.replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
.replace(/&#x([\da-f]+);/gi, (_, code) =>
String.fromCodePoint(Number.parseInt(code, 16)),
)
}
function attributes(source = '') {
const result = {}
for (const match of source.matchAll(/([\w:.-]+)\s*=\s*(["'])(.*?)\2/gs)) {
result[match[1]] = decodeXml(match[3])
}
return result
}
function concise(value, limit = 300) {
const normalized = decodeXml(value)
.replace(/<[^>]+>/g, ' ')
.replace(/\s+/g, ' ')
.trim()
return normalized.length > limit
? `${normalized.slice(0, limit - 1)}`
: normalized
}
export function parseJUnit(xml) {
const failures = []
const testcasePattern = /<testcase\b([^>]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/gi
for (const match of xml.matchAll(testcasePattern)) {
const testcase = attributes(match[1])
const body = match[2] || ''
const failure = body.match(/<(failure|error)\b([^>]*)>([\s\S]*?)<\/\1>/i)
const selfClosingFailure = body.match(/<(failure|error)\b([^>]*)\/>/i)
const failureMatch = failure || selfClosingFailure
if (!failureMatch) continue
const failureAttributes = attributes(failureMatch[2])
const name = testcase.name || testcase.classname || 'Unnamed Maestro flow'
const message = concise(
failureAttributes.message || (failure ? failureMatch[3] : '') || 'Failed',
)
failures.push({name, message})
}
if (failures.length === 0) {
const suite = xml.match(/<testsuite\b([^>]*)>/i)
const suiteAttributes = attributes(suite?.[1])
if (
Number(suiteAttributes.failures || 0) +
Number(suiteAttributes.errors || 0) >
0
) {
failures.push({
name: suiteAttributes.name || 'Maestro test suite',
message: 'JUnit reported a failure without testcase details',
})
}
}
return failures
}
function walk(root) {
if (!root || !fs.existsSync(root)) return []
const entries = fs.readdirSync(root, {withFileTypes: true})
return entries.flatMap(entry => {
const candidate = path.join(root, entry.name)
return entry.isDirectory() ? walk(candidate) : [candidate]
})
}
function readPhase(root) {
const phaseFile = walk(root).find(file => path.basename(file) === 'phase.txt')
return phaseFile ? fs.readFileSync(phaseFile, 'utf8').trim() : ''
}
function platformResult({name, status, root, artifactUrl}) {
const reports = walk(root).filter(file =>
/(?:report|junit).*\.xml$/i.test(file),
)
const failures = reports.flatMap(report =>
parseJUnit(fs.readFileSync(report, 'utf8')),
)
const failed = status !== 'success' || failures.length > 0
return {
name,
status,
failed,
failures,
phase: readPhase(root),
hasJUnit: reports.length > 0,
artifactUrl,
}
}
function statusEmoji(status) {
return status === 'success' ? ':white_check_mark:' : ':x:'
}
function slackEscape(value) {
return value
.replaceAll('&', '&amp;')
.replaceAll('<', '&lt;')
.replaceAll('>', '&gt;')
}
export function buildSummary({
iosStatus,
androidStatus,
iosRoot,
androidRoot,
artifactUrls = {},
sha,
runUrl,
commitUrl,
}) {
const platforms = [
platformResult({
name: 'iOS',
status: iosStatus,
root: iosRoot,
artifactUrl: artifactUrls.ios,
}),
platformResult({
name: 'Android',
status: androidStatus,
root: androidRoot,
artifactUrl: artifactUrls.android,
}),
]
const notify = platforms.some(platform => platform.failed)
const shortSha = sha.slice(0, 12)
const lines = [
':rotating_light: *Nightly Maestro E2E failed*',
`*Commit:* <${commitUrl}|\`${shortSha}\`>`,
`*Workflow run:* <${runUrl}|open run>`,
'',
]
for (const platform of platforms) {
lines.push(
`${statusEmoji(platform.status)} *${platform.name}* — job status: \`${platform.status}\``,
)
if (platform.failures.length > 0) {
for (const failure of platform.failures.slice(0, 10)) {
lines.push(
`• *${slackEscape(failure.name)}:* ${slackEscape(failure.message)}`,
)
}
if (platform.failures.length > 10) {
lines.push(`• …and ${platform.failures.length - 10} more failed flows`)
}
} else if (platform.failed && !platform.hasJUnit) {
lines.push(
`• Setup phase: ${platform.phase || 'No phase metadata was captured'}`,
)
} else if (platform.failed) {
lines.push(
`• The job failed after JUnit was written (latest phase: ${platform.phase || 'unknown'})`,
)
}
if (platform.artifactUrl) {
lines.push(
`• <${platform.artifactUrl}|${platform.name} logs and artifacts>`,
)
}
lines.push('')
}
const text = lines.join('\n').trim()
return {
notify,
platforms,
payload: {text},
}
}
function parseArgs(argv) {
const result = {}
for (let i = 0; i < argv.length; i += 2) {
const key = argv[i]
if (!key?.startsWith('--') || argv[i + 1] === undefined) {
throw new Error(`Invalid argument: ${key || '<missing>'}`)
}
result[key.slice(2)] = argv[i + 1]
}
return result
}
if (
process.argv[1] &&
path.resolve(process.argv[1]) === path.resolve(import.meta.filename)
) {
const args = parseArgs(process.argv.slice(2))
const artifactUrls = args['artifact-urls']
? JSON.parse(fs.readFileSync(args['artifact-urls'], 'utf8'))
: {}
const summary = buildSummary({
iosStatus: args['ios-status'],
androidStatus: args['android-status'],
iosRoot: args['ios-root'],
androidRoot: args['android-root'],
artifactUrls,
sha: args.sha,
runUrl: args['run-url'],
commitUrl: args['commit-url'],
})
process.stdout.write(`${JSON.stringify(summary)}\n`)
}
@@ -0,0 +1,75 @@
import assert from 'node:assert/strict'
import path from 'node:path'
import test from 'node:test'
import {fileURLToPath} from 'node:url'
import {buildSummary, parseJUnit} from './summarize-maestro.mjs'
const here = path.dirname(fileURLToPath(import.meta.url))
const fixtures = path.join(here, 'fixtures')
const common = {
artifactUrls: {
ios: 'https://github.test/run/artifacts/1',
android: 'https://github.test/run/artifacts/2',
},
sha: '0123456789abcdef',
runUrl: 'https://github.test/run',
commitUrl: 'https://github.test/commit/0123456789abcdef',
}
test('does not notify for successful jobs and successful JUnit', () => {
const root = path.join(fixtures, 'success')
const summary = buildSummary({
...common,
iosStatus: 'success',
androidStatus: 'success',
iosRoot: path.join(root, 'ios'),
androidRoot: path.join(root, 'android'),
})
assert.equal(summary.notify, false)
assert.deepEqual(
summary.platforms.map(platform => platform.failures),
[[], []],
)
})
test('reports failed flow names and concise failure messages', () => {
const root = path.join(fixtures, 'failed-flow')
const summary = buildSummary({
...common,
iosStatus: 'success',
androidStatus: 'failure',
iosRoot: path.join(fixtures, 'success', 'ios'),
androidRoot: path.join(root, 'android'),
})
assert.equal(summary.notify, true)
assert.match(summary.payload.text, /create-account\.yml/)
assert.match(summary.payload.text, /Element not found: Create account/)
assert.match(summary.payload.text, /Android logs and artifacts/)
})
test('reports the latest setup phase when infrastructure fails before JUnit', () => {
const root = path.join(fixtures, 'infrastructure-failure')
const summary = buildSummary({
...common,
iosStatus: 'failure',
androidStatus: 'success',
iosRoot: path.join(root, 'ios'),
androidRoot: path.join(fixtures, 'success', 'android'),
})
assert.equal(summary.notify, true)
assert.match(
summary.payload.text,
/Setup phase: Building and installing the development client/,
)
})
test('parses self-closing JUnit failures', () => {
const failures = parseJUnit(
'<testsuite failures="1"><testcase name="flow.yml"><failure message="boom"/></testcase></testsuite>',
)
assert.deepEqual(failures, [{name: 'flow.yml', message: 'boom'}])
})