acetheking987/bsky-social-app
1
0
Fork 0
Code Issues Actions Packages Projects Releases 1 Wiki Activity

Add nightly Maestro E2E workflow

Browse Source
This commit is contained in:
Samuel Newman
2026-07-17 13:04:12 +03:00
parent 21d53ce0f8
commit 90267eb655
11 changed files with 841 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/cleanup-nightly-e2e.sh
Executable
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env bash
set +e
platform="${1:?usage: cleanup-nightly-e2e.sh <ios|android> <device-id>}"
device_id="${2:-}"
artifact_dir="${GITHUB_WORKSPACE:-$PWD}/artifacts/$platform"
stop_process_tree() {
local pid="$1"
local child
while read -r child; do
[[ -n "$child" ]] && stop_process_tree "$child"
done < <(pgrep -P "$pid" 2>/dev/null || true)
kill -TERM "$pid" >/dev/null 2>&1 || true
}
stop_pid_file() {
[[ -f "$1" ]] || return 0
local pid
pid="$(cat "$1")"
stop_process_tree "$pid"
}
stop_pid_file "$artifact_dir/metro.pid"
stop_pid_file "$artifact_dir/mock-server.pid"
if [[ "$platform" == "ios" ]]; then
if [[ -f "$artifact_dir/redis-bin.txt" ]]; then
"$(cat "$artifact_dir/redis-bin.txt")/redis-cli" \
-h 127.0.0.1 -p 6380 shutdown nosave >/dev/null 2>&1 || true
fi
if [[ -f "$artifact_dir/postgres-bin.txt" ]]; then
"$(cat "$artifact_dir/postgres-bin.txt")/pg_ctl" \
-D "${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres" -m fast stop >/dev/null 2>&1 || true
fi
[[ -n "$device_id" ]] && xcrun simctl shutdown "$device_id" >/dev/null 2>&1 || true
else
docker compose -f dev-env/dev-infra/docker-compose.yaml logs --no-color \
>>"$artifact_dir/docker-services.log" 2>&1 || true
docker compose -f dev-env/dev-infra/docker-compose.yaml down --volumes --remove-orphans >/dev/null 2>&1 || true
[[ -n "$device_id" ]] && adb -s "$device_id" emu kill >/dev/null 2>&1 || true
fi
.github/scripts/fixtures/failed-flow/android/report.xml
+7
View File
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="Maestro Android" tests="2" failures="1" errors="0">
<testcase name="login.yml" classname="Maestro" time="11.8" />
<testcase name="create-account.yml" classname="Maestro" time="8.1">
<failure message="Element not found: Create account">The expected button was not visible.</failure>
</testcase>
</testsuite>
.github/scripts/fixtures/infrastructure-failure/ios/phase.txt
+1
View File
@@ -0,0 +1 @@
Building and installing the development client
.github/scripts/fixtures/success/android/report.xml
+4
View File
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="Maestro Android" tests="1" failures="0" errors="0">
<testcase name="login.yml" classname="Maestro" time="11.8" />
</testsuite>
.github/scripts/fixtures/success/ios/report.xml
+4
View File
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuite name="Maestro iOS" tests="1" failures="0" errors="0">
<testcase name="login.yml" classname="Maestro" time="12.4" />
</testsuite>
.github/scripts/run-nightly-e2e.sh
Executable
+181
View File
@@ -0,0 +1,181 @@
#!/usr/bin/env bash
set -Eeuo pipefail
platform="${1:?usage: run-nightly-e2e.sh <ios|android> <device-id>}"
device_id="${2:?usage: run-nightly-e2e.sh <ios|android> <device-id>}"
if [[ "$platform" != "ios" && "$platform" != "android" ]]; then
echo "Unsupported platform: $platform" >&2
exit 2
fi
artifact_dir="${GITHUB_WORKSPACE:-$PWD}/artifacts/$platform"
maestro_dir="$artifact_dir/maestro"
mkdir -p "$maestro_dir"
phase() {
printf '%s\n' "$1" >"$artifact_dir/phase.txt"
}
wait_for_port() {
local port="$1"
local label="$2"
local attempts="${3:-120}"
for ((i = 1; i <= attempts; i++)); do
if nc -z 127.0.0.1 "$port" >/dev/null 2>&1; then
return 0
fi
sleep 1
done
echo "Timed out waiting for $label on port $port" >&2
return 1
}
# shellcheck disable=SC2329 # Invoked through the cleanup trap call chain.
stop_process_tree() {
local pid="$1"
local child
while read -r child; do
[[ -n "$child" ]] && stop_process_tree "$child"
done < <(pgrep -P "$pid" 2>/dev/null || true)
kill -TERM "$pid" >/dev/null 2>&1 || true
}
# shellcheck disable=SC2329 # Invoked by cleanup, which is registered as a trap.
stop_pid_file() {
local pid_file="$1"
[[ -f "$pid_file" ]] || return 0
local pid
pid="$(cat "$pid_file")"
[[ -n "$pid" ]] || return 0
# pnpm and Expo both spawn multiple generations of children.
stop_process_tree "$pid"
}
# shellcheck disable=SC2329 # Invoked by the EXIT/INT/TERM trap below.
cleanup() {
set +e
stop_pid_file "$artifact_dir/metro.pid"
stop_pid_file "$artifact_dir/mock-server.pid"
if [[ "$platform" == "ios" ]]; then
if [[ -f "$artifact_dir/redis.pid" ]]; then
redis_bin="$(cat "$artifact_dir/redis-bin.txt")"
"$redis_bin/redis-cli" -h 127.0.0.1 -p 6380 shutdown nosave >/dev/null 2>&1 || true
fi
if [[ -f "$artifact_dir/postgres-bin.txt" && -d "${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres" ]]; then
postgres_bin="$(cat "$artifact_dir/postgres-bin.txt")"
"$postgres_bin/pg_ctl" -D "${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres" -m fast stop >/dev/null 2>&1 || true
fi
else
docker compose -f dev-env/dev-infra/docker-compose.yaml logs --no-color \
>>"$artifact_dir/docker-services.log" 2>&1 || true
docker compose -f dev-env/dev-infra/docker-compose.yaml down --volumes --remove-orphans >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT INT TERM
phase "Starting PostgreSQL, Redis, and mock server"
if [[ "$platform" == "ios" ]]; then
brew install postgresql@14 2>&1 | tee "$artifact_dir/native-dependencies.log"
postgres_bin="$(brew --prefix postgresql@14)/bin"
redis_version="7.4.7"
redis_archive="${RUNNER_TEMP:-/tmp}/redis-${redis_version}.tar.gz"
redis_source="${RUNNER_TEMP:-/tmp}/redis-${redis_version}"
curl -fsSL -o "$redis_archive" \
"https://download.redis.io/releases/redis-${redis_version}.tar.gz"
echo "c97e57b0df330a9e091cacff012bebe763c275398cf36ff44cdba876814b595b $redis_archive" \
| shasum -a 256 --check | tee -a "$artifact_dir/native-dependencies.log"
rm -rf "$redis_source"
tar -xzf "$redis_archive" -C "${RUNNER_TEMP:-/tmp}"
make -C "$redis_source" -j "$(sysctl -n hw.ncpu)" \
2>&1 | tee -a "$artifact_dir/native-dependencies.log"
redis_bin="$redis_source/src"
"$redis_bin/redis-server" --version | tee -a "$artifact_dir/native-dependencies.log"
printf '%s\n' "$redis_bin" >"$artifact_dir/redis-bin.txt"
postgres_data="${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres"
rm -rf "$postgres_data"
"$postgres_bin/initdb" -D "$postgres_data" --auth=trust --username=pg --no-locale \
>"$artifact_dir/postgres-init.log" 2>&1
"$postgres_bin/pg_ctl" -D "$postgres_data" \
-o "-p 5433 -h 127.0.0.1" -l "$artifact_dir/postgres.log" start
printf '%s\n' "$postgres_bin" >"$artifact_dir/postgres-bin.txt"
"$redis_bin/redis-server" \
--bind 127.0.0.1 \
--port 6380 \
--save "" \
--appendonly no \
--daemonize yes \
--pidfile "$artifact_dir/redis.pid" \
--logfile "$artifact_dir/redis.log"
wait_for_port 5433 "PostgreSQL"
wait_for_port 6380 "Redis"
pnpm --dir dev-env start:external >"$artifact_dir/mock-server.log" 2>&1 &
else
pnpm --dir dev-env start >"$artifact_dir/mock-server.log" 2>&1 &
fi
printf '%s\n' "$!" >"$artifact_dir/mock-server.pid"
wait_for_port 1986 "the E2E mock-server manager"
phase "Starting Metro"
EXPO_PUBLIC_ENV=e2e \
NODE_ENV=test \
RN_SRC_EXT=e2e.ts,e2e.tsx \
pnpm exec expo start --dev-client --clear --port 8081 \
>"$artifact_dir/metro.log" 2>&1 &
printf '%s\n' "$!" >"$artifact_dir/metro.pid"
wait_for_port 8081 "Metro"
if [[ "$platform" == "android" ]]; then
phase "Configuring Android localhost routing"
adb -s "$device_id" reverse tcp:3000 tcp:3000
adb -s "$device_id" reverse tcp:8081 tcp:8081
fi
phase "Building and installing the development client"
if [[ "$platform" == "ios" ]]; then
EXPO_PUBLIC_ENV=e2e \
NODE_ENV=test \
RN_SRC_EXT=e2e.ts,e2e.tsx \
pnpm exec expo run:ios --device "$device_id" --no-bundler \
2>&1 | tee "$artifact_dir/build.log"
else
EXPO_PUBLIC_ENV=e2e \
NODE_ENV=test \
RN_SRC_EXT=e2e.ts,e2e.tsx \
pnpm exec expo run:android --device "$device_id" --no-bundler \
2>&1 | tee "$artifact_dir/build.log"
fi
phase "Running Maestro flows"
set +e
maestro test \
--udid "$device_id" \
--format JUNIT \
--output "$artifact_dir/report.xml" \
--config __e2e__/config.yml \
--debug-output "$maestro_dir" \
--test-output-dir "$maestro_dir" \
--flatten-debug-output \
__e2e__ \
2>&1 | tee "$artifact_dir/maestro-cli.log"
maestro_status=${PIPESTATUS[0]}
set -e
if [[ "$maestro_status" -eq 0 ]]; then
phase "Completed"
else
phase "Maestro flow failure"
fi
exit "$maestro_status"
.github/scripts/summarize-maestro.mjs
+224
View File
@@ -0,0 +1,224 @@
import fs from 'node:fs'
import path from 'node:path'
import process from 'node:process'
const ENTITY_REPLACEMENTS = {
'&amp;': '&',
'&apos;': "'",
'&gt;': '>',
'&lt;': '<',
'&quot;': '"',
}
function decodeXml(value = '') {
return value
.replace(/&(amp|apos|gt|lt|quot);/g, entity => ENTITY_REPLACEMENTS[entity])
.replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
.replace(/&#x([\da-f]+);/gi, (_, code) =>
String.fromCodePoint(Number.parseInt(code, 16)),
)
}
function attributes(source = '') {
const result = {}
for (const match of source.matchAll(/([\w:.-]+)\s*=\s*(["'])(.*?)\2/gs)) {
result[match[1]] = decodeXml(match[3])
}
return result
}
function concise(value, limit = 300) {
const normalized = decodeXml(value)
.replace(/<[^>]+>/g, ' ')
.replace(/\s+/g, ' ')
.trim()
return normalized.length > limit
? `${normalized.slice(0, limit - 1)}…`
: normalized
}
export function parseJUnit(xml) {
const failures = []
const testcasePattern = /<testcase\b([^>]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/gi
for (const match of xml.matchAll(testcasePattern)) {
const testcase = attributes(match[1])
const body = match[2] || ''
const failure = body.match(/<(failure|error)\b([^>]*)>([\s\S]*?)<\/\1>/i)
const selfClosingFailure = body.match(/<(failure|error)\b([^>]*)\/>/i)
const failureMatch = failure || selfClosingFailure
if (!failureMatch) continue
const failureAttributes = attributes(failureMatch[2])
const name = testcase.name || testcase.classname || 'Unnamed Maestro flow'
const message = concise(
failureAttributes.message || (failure ? failureMatch[3] : '') || 'Failed',
)
failures.push({name, message})
}
if (failures.length === 0) {
const suite = xml.match(/<testsuite\b([^>]*)>/i)
const suiteAttributes = attributes(suite?.[1])
if (
Number(suiteAttributes.failures || 0) +
Number(suiteAttributes.errors || 0) >
0
) {
failures.push({
name: suiteAttributes.name || 'Maestro test suite',
message: 'JUnit reported a failure without testcase details',
})
}
}
return failures
}
function walk(root) {
if (!root || !fs.existsSync(root)) return []
const entries = fs.readdirSync(root, {withFileTypes: true})
return entries.flatMap(entry => {
const candidate = path.join(root, entry.name)
return entry.isDirectory() ? walk(candidate) : [candidate]
})
}
function readPhase(root) {
const phaseFile = walk(root).find(file => path.basename(file) === 'phase.txt')
return phaseFile ? fs.readFileSync(phaseFile, 'utf8').trim() : ''
}
function platformResult({name, status, root, artifactUrl}) {
const reports = walk(root).filter(file =>
/(?:report|junit).*\.xml$/i.test(file),
)
const failures = reports.flatMap(report =>
parseJUnit(fs.readFileSync(report, 'utf8')),
)
const failed = status !== 'success' || failures.length > 0
return {
name,
status,
failed,
failures,
phase: readPhase(root),
hasJUnit: reports.length > 0,
artifactUrl,
}
}
function statusEmoji(status) {
return status === 'success' ? ':white_check_mark:' : ':x:'
}
function slackEscape(value) {
return value
.replaceAll('&', '&amp;')
.replaceAll('<', '&lt;')
.replaceAll('>', '&gt;')
}
export function buildSummary({
iosStatus,
androidStatus,
iosRoot,
androidRoot,
artifactUrls = {},
sha,
runUrl,
commitUrl,
}) {
const platforms = [
platformResult({
name: 'iOS',
status: iosStatus,
root: iosRoot,
artifactUrl: artifactUrls.ios,
}),
platformResult({
name: 'Android',
status: androidStatus,
root: androidRoot,
artifactUrl: artifactUrls.android,
}),
]
const notify = platforms.some(platform => platform.failed)
const shortSha = sha.slice(0, 12)
const lines = [
':rotating_light: *Nightly Maestro E2E failed*',
`*Commit:* <${commitUrl}|\`${shortSha}\`>`,
`*Workflow run:* <${runUrl}|open run>`,
'',
]
for (const platform of platforms) {
lines.push(
`${statusEmoji(platform.status)} *${platform.name}* — job status: \`${platform.status}\``,
)
if (platform.failures.length > 0) {
for (const failure of platform.failures.slice(0, 10)) {
lines.push(
`• *${slackEscape(failure.name)}:* ${slackEscape(failure.message)}`,
)
}
if (platform.failures.length > 10) {
lines.push(`• …and ${platform.failures.length - 10} more failed flows`)
}
} else if (platform.failed && !platform.hasJUnit) {
lines.push(
`• Setup phase: ${platform.phase || 'No phase metadata was captured'}`,
)
} else if (platform.failed) {
lines.push(
`• The job failed after JUnit was written (latest phase: ${platform.phase || 'unknown'})`,
)
}
if (platform.artifactUrl) {
lines.push(
`• <${platform.artifactUrl}|${platform.name} logs and artifacts>`,
)
}
lines.push('')
}
const text = lines.join('\n').trim()
return {
notify,
platforms,
payload: {text},
}
}
function parseArgs(argv) {
const result = {}
for (let i = 0; i < argv.length; i += 2) {
const key = argv[i]
if (!key?.startsWith('--') || argv[i + 1] === undefined) {
throw new Error(`Invalid argument: ${key || '<missing>'}`)
}
result[key.slice(2)] = argv[i + 1]
}
return result
}
if (
process.argv[1] &&
path.resolve(process.argv[1]) === path.resolve(import.meta.filename)
) {
const args = parseArgs(process.argv.slice(2))
const artifactUrls = args['artifact-urls']
? JSON.parse(fs.readFileSync(args['artifact-urls'], 'utf8'))
: {}
const summary = buildSummary({
iosStatus: args['ios-status'],
androidStatus: args['android-status'],
iosRoot: args['ios-root'],
androidRoot: args['android-root'],
artifactUrls,
sha: args.sha,
runUrl: args['run-url'],
commitUrl: args['commit-url'],
})
process.stdout.write(`${JSON.stringify(summary)}\n`)
}
.github/scripts/summarize-maestro.test.mjs
+75
View File
@@ -0,0 +1,75 @@
import assert from 'node:assert/strict'
import path from 'node:path'
import test from 'node:test'
import {fileURLToPath} from 'node:url'
import {buildSummary, parseJUnit} from './summarize-maestro.mjs'
const here = path.dirname(fileURLToPath(import.meta.url))
const fixtures = path.join(here, 'fixtures')
const common = {
artifactUrls: {
ios: 'https://github.test/run/artifacts/1',
android: 'https://github.test/run/artifacts/2',
},
sha: '0123456789abcdef',
runUrl: 'https://github.test/run',
commitUrl: 'https://github.test/commit/0123456789abcdef',
}
test('does not notify for successful jobs and successful JUnit', () => {
const root = path.join(fixtures, 'success')
const summary = buildSummary({
...common,
iosStatus: 'success',
androidStatus: 'success',
iosRoot: path.join(root, 'ios'),
androidRoot: path.join(root, 'android'),
})
assert.equal(summary.notify, false)
assert.deepEqual(
summary.platforms.map(platform => platform.failures),
[[], []],
)
})
test('reports failed flow names and concise failure messages', () => {
const root = path.join(fixtures, 'failed-flow')
const summary = buildSummary({
...common,
iosStatus: 'success',
androidStatus: 'failure',
iosRoot: path.join(fixtures, 'success', 'ios'),
androidRoot: path.join(root, 'android'),
})
assert.equal(summary.notify, true)
assert.match(summary.payload.text, /create-account\.yml/)
assert.match(summary.payload.text, /Element not found: Create account/)
assert.match(summary.payload.text, /Android logs and artifacts/)
})
test('reports the latest setup phase when infrastructure fails before JUnit', () => {
const root = path.join(fixtures, 'infrastructure-failure')
const summary = buildSummary({
...common,
iosStatus: 'failure',
androidStatus: 'success',
iosRoot: path.join(root, 'ios'),
androidRoot: path.join(fixtures, 'success', 'android'),
})
assert.equal(summary.notify, true)
assert.match(
summary.payload.text,
/Setup phase: Building and installing the development client/,
)
})
test('parses self-closing JUnit failures', () => {
const failures = parseJUnit(
'<testsuite failures="1"><testcase name="flow.yml"><failure message="boom"/></testcase></testsuite>',
)
assert.deepEqual(failures, [{name: 'flow.yml', message: 'boom'}])
})
.github/workflows/nightly-e2e.yml
+262
View File
@@ -0,0 +1,262 @@
---
name: Nightly Maestro E2E
on:
schedule:
- cron: "0 4 * * *"
workflow_dispatch:
permissions:
contents: read
concurrency:
group: nightly-maestro-e2e-${{ github.ref }}
cancel-in-progress: false
env:
CI: "1"
MAESTRO_VERSION: "2.6.1"
MAESTRO_DRIVER_STARTUP_TIMEOUT: "180000"
MAESTRO_CLI_NO_ANALYTICS: "1"
MAESTRO_CLI_ANALYSIS_NOTIFICATION_DISABLED: "true"
MAESTRO_DISABLE_UPDATE_CHECK: "1"
jobs:
ios:
name: iOS Maestro E2E
if: github.repository == 'bluesky-social/social-app'
runs-on: macos-26-xlarge
timeout-minutes: 120
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Select Xcode 26.4
uses: maxim-lobanov/setup-xcode@ed7a3b1fda3918c0306d1b724322adc0b8cc0a90 # v1.7.0
with:
xcode-version: "26.4"
- name: Set up pnpm
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
- name: Set up Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version-file: package.json
cache: pnpm
- name: Set up Java 17
uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0
with:
distribution: temurin
java-version: "17"
- name: Prepare E2E configuration
run: |
mkdir -p artifacts/ios
echo "Installing dependencies" > artifacts/ios/phase.txt
cp .env.example .env.test
cp google-services.json.example google-services.json
- name: Install dependencies
run: |
set -o pipefail
pnpm install --frozen-lockfile 2>&1 | tee artifacts/ios/dependencies.log
pnpm --dir dev-env install --frozen-lockfile 2>&1 | tee -a artifacts/ios/dependencies.log
pnpm intl:build 2>&1 | tee artifacts/ios/i18n.log
- name: Install Maestro 2.6.1
run: |
echo "Installing Maestro" > artifacts/ios/phase.txt
curl -fsSL -o "$RUNNER_TEMP/maestro.zip" \
"https://github.com/mobile-dev-inc/Maestro/releases/download/cli-${MAESTRO_VERSION}/maestro.zip"
echo "3440825f514f537c6a96bcf5de995780c2a4a7f83a43208fdc95d4f1fecfad3b $RUNNER_TEMP/maestro.zip" \
| shasum -a 256 --check
unzip -q "$RUNNER_TEMP/maestro.zip" -d "$RUNNER_TEMP"
echo "$RUNNER_TEMP/maestro/bin" >> "$GITHUB_PATH"
"$RUNNER_TEMP/maestro/bin/maestro" --version | tee artifacts/ios/maestro-version.log
test "$("$RUNNER_TEMP/maestro/bin/maestro" --version)" = "$MAESTRO_VERSION"
- name: Boot one iOS simulator
run: |
echo "Booting iOS simulator" > artifacts/ios/phase.txt
udid=$(xcrun simctl list devices available --json | jq -r \
'[.devices[][] | select(.name | startswith("iPhone"))] | first | .udid')
if [ -z "$udid" ] || [ "$udid" = "null" ]; then
echo "No available iPhone simulator was found" >&2
exit 1
fi
echo "IOS_UDID=$udid" >> "$GITHUB_ENV"
xcrun simctl shutdown all || true
xcrun simctl boot "$udid"
xcrun simctl bootstatus "$udid" -b
echo "Using iOS simulator $udid"
- name: Run iOS Maestro suite
run: .github/scripts/run-nightly-e2e.sh ios "$IOS_UDID"
- name: Clean up iOS services and simulator
if: always()
run: .github/scripts/cleanup-nightly-e2e.sh ios "${IOS_UDID:-}"
- name: Upload iOS E2E artifacts
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: nightly-e2e-ios-${{ github.run_id }}
path: artifacts/ios
if-no-files-found: warn
retention-days: 14
android:
name: Android Maestro E2E
if: github.repository == 'bluesky-social/social-app'
# Linux-x64-32core is a repository-managed runner label.
runs-on: Linux-x64-32core
timeout-minutes: 120
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Set up pnpm
uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
- name: Set up Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version-file: package.json
cache: pnpm
- name: Set up Java 17
uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0
with:
distribution: temurin
java-version: "17"
- name: Prepare E2E configuration
run: |
mkdir -p artifacts/android
echo "Installing dependencies" > artifacts/android/phase.txt
cp .env.example .env.test
cp google-services.json.example google-services.json
- name: Install dependencies
run: |
set -o pipefail
pnpm install --frozen-lockfile 2>&1 | tee artifacts/android/dependencies.log
pnpm --dir dev-env install --frozen-lockfile 2>&1 | tee -a artifacts/android/dependencies.log
pnpm intl:build 2>&1 | tee artifacts/android/i18n.log
- name: Install Maestro 2.6.1
run: |
echo "Installing Maestro" > artifacts/android/phase.txt
curl -fsSL -o "$RUNNER_TEMP/maestro.zip" \
"https://github.com/mobile-dev-inc/Maestro/releases/download/cli-${MAESTRO_VERSION}/maestro.zip"
echo "3440825f514f537c6a96bcf5de995780c2a4a7f83a43208fdc95d4f1fecfad3b $RUNNER_TEMP/maestro.zip" \
| shasum -a 256 --check
unzip -q "$RUNNER_TEMP/maestro.zip" -d "$RUNNER_TEMP"
echo "$RUNNER_TEMP/maestro/bin" >> "$GITHUB_PATH"
"$RUNNER_TEMP/maestro/bin/maestro" --version | tee artifacts/android/maestro-version.log
test "$("$RUNNER_TEMP/maestro/bin/maestro" --version)" = "$MAESTRO_VERSION"
- name: Boot one Android emulator and run Maestro suite
uses: reactivecircus/android-emulator-runner@1dcd0090116d15e7c562f8db72807de5e036a4ed # v2.34.0
with:
api-level: 35
target: google_apis
arch: x86_64
profile: pixel_6
avd-name: nightly-e2e
emulator-port: 5554
disable-animations: true
emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -camera-back none
script: .github/scripts/run-nightly-e2e.sh android emulator-5554
- name: Clean up Android services and emulator
if: always()
run: .github/scripts/cleanup-nightly-e2e.sh android emulator-5554
- name: Upload Android E2E artifacts
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: nightly-e2e-android-${{ github.run_id }}
path: artifacts/android
if-no-files-found: warn
retention-days: 14
report:
name: Report E2E failures
needs: [ios, android]
if: ${{ always() && github.repository == 'bluesky-social/social-app' }}
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
steps:
- name: Checkout
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Download iOS artifacts
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: nightly-e2e-ios-${{ github.run_id }}
path: downloaded-artifacts/ios
- name: Download Android artifacts
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: nightly-e2e-android-${{ github.run_id }}
path: downloaded-artifacts/android
- name: Resolve artifact links
env:
GH_TOKEN: ${{ github.token }}
run: |
run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}#artifacts"
jq -n --arg run "$run_url" '{ios: $run, android: $run}' > artifact-links.json
if gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \
> artifact-response.json; then
jq --arg base "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts" \
--arg run "$run_url" \
'{
ios: (.artifacts | map(select(.name | startswith("nightly-e2e-ios-"))) | first | if . then ($base + "/" + (.id | tostring)) else $run end),
android: (.artifacts | map(select(.name | startswith("nightly-e2e-android-"))) | first | if . then ($base + "/" + (.id | tostring)) else $run end)
}' artifact-response.json > artifact-links.json
fi
- name: Summarize platform results
id: summary
env:
IOS_STATUS: ${{ needs.ios.result }}
ANDROID_STATUS: ${{ needs.android.result }}
run: |
node .github/scripts/summarize-maestro.mjs \
--ios-status "$IOS_STATUS" \
--android-status "$ANDROID_STATUS" \
--ios-root downloaded-artifacts/ios \
--android-root downloaded-artifacts/android \
--artifact-urls artifact-links.json \
--sha "$GITHUB_SHA" \
--run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
--commit-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}" \
> e2e-summary.json
echo "notify=$(jq -r .notify e2e-summary.json)" >> "$GITHUB_OUTPUT"
echo "payload=$(jq -c .payload e2e-summary.json)" >> "$GITHUB_OUTPUT"
jq -r .payload.text e2e-summary.json >> "$GITHUB_STEP_SUMMARY"
- name: Notify Slack of E2E failures
if: steps.summary.outputs.notify == 'true'
uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3
with:
webhook: ${{ secrets.E2E_FAILURES_SLACK_WEBHOOK }}
webhook-type: incoming-webhook
payload: ${{ steps.summary.outputs.payload }}
dev-env/package.json
+2 -1
View File
@@ -3,7 +3,8 @@
"version": "0.0.0",
"type": "module",
"scripts": {
"start": "NODE_ENV=development ./dev-infra/with-test-redis-and-db.sh node ./mock-server.ts"
"start": "NODE_ENV=development ./dev-infra/with-test-redis-and-db.sh node ./mock-server.ts",
"start:external": "NODE_ENV=development PGPORT=5433 PGHOST=localhost PGUSER=pg PGPASSWORD=password PGDATABASE=postgres DB_POSTGRES_URL=postgresql://pg:password@127.0.0.1:5433/postgres REDIS_HOST=127.0.0.1:6380 node ./mock-server.ts"
},
"dependencies": {
"@atproto/api": "^0.20.22",
docs/testing.md
+38
View File
@@ -29,6 +29,44 @@ adb reverse tcp:3000 tcp:3000
- In a second tab, run `pnpm e2e:build`
- In a third tab, run `pnpm e2e:run __e2e__`
## Nightly Maestro CI
The `Nightly Maestro E2E` GitHub Actions workflow runs every day at 04:00 UTC
and can also be started from the Actions tab with **Run workflow**. It runs iOS
and Android concurrently, but each platform runs all of `__e2e__/config.yml`
sequentially on one explicitly selected simulator or emulator. The flows share a
stateful mock-server manager, so the suite must not be sharded.
The jobs run Maestro CLI 2.6.1 locally on GitHub Actions; Maestro Cloud is not
used. iOS runs on `macos-26-xlarge` with Xcode 26.4. Android runs on
`Linux-x64-32core`. Both use Java 17 and the Node and pnpm versions declared in
`package.json`.
The mock-server manager listens on host port 1986 and creates test services on
port 3000. Metro listens on 8081. Android reverses ports 3000 and 8081 into the
emulator; port 1986 remains host-side because Maestro JavaScript calls it from
the runner. Android uses the existing Docker Compose PostgreSQL 14 and Redis 7
services on ports 5433 and 6380. GitHub-hosted macOS cannot run nested Docker
virtualization, so iOS provisions ephemeral native PostgreSQL 14.x and Redis
7.4.7 on those same ports and starts `pnpm --dir dev-env start:external`.
Each platform uploads a `nightly-e2e-<platform>-<run-id>` artifact for 14 days.
It contains JUnit at `report.xml`, Maestro screenshots, videos, command metadata
and `maestro.log` under `maestro/`, plus Metro, native build, mock-server, service,
dependency, and translation logs. The workflow always uploads what was captured,
including when setup or the native build fails before Maestro starts.
Add the repository secret `E2E_FAILURES_SLACK_WEBHOOK` before enabling the
schedule. The aggregation job runs even when either platform fails and posts one
detailed Slack notification containing both job statuses, failed flow details or
the failed setup phase, the commit and workflow links, and links to both artifact
sets. Successful runs do not post to Slack.
Before relying on the schedule, manually dispatch the workflow and verify both
platforms against live Metro and `dev-env`, Android localhost routing, artifact
uploads on success and failure, one Slack message for a forced failure, and no
Slack message for an all-green run.
## Using Flashlight for Performance Testing
1. Make sure Maestro is installed (optional: only for automated testing) by following the instructions above
2. Install Flashlight by following [these instructions](https://docs.flashlight.dev/)