diff --git a/.github/scripts/cleanup-nightly-e2e.sh b/.github/scripts/cleanup-nightly-e2e.sh new file mode 100755 index 0000000000..b6f569253c --- /dev/null +++ b/.github/scripts/cleanup-nightly-e2e.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set +e + +platform="${1:?usage: cleanup-nightly-e2e.sh }" +device_id="${2:-}" +artifact_dir="${GITHUB_WORKSPACE:-$PWD}/artifacts/$platform" + +stop_process_tree() { + local pid="$1" + local child + while read -r child; do + [[ -n "$child" ]] && stop_process_tree "$child" + done < <(pgrep -P "$pid" 2>/dev/null || true) + kill -TERM "$pid" >/dev/null 2>&1 || true +} + +stop_pid_file() { + [[ -f "$1" ]] || return 0 + local pid + pid="$(cat "$1")" + stop_process_tree "$pid" +} + +stop_pid_file "$artifact_dir/metro.pid" +stop_pid_file "$artifact_dir/mock-server.pid" + +if [[ "$platform" == "ios" ]]; then + if [[ -f "$artifact_dir/redis-bin.txt" ]]; then + "$(cat "$artifact_dir/redis-bin.txt")/redis-cli" \ + -h 127.0.0.1 -p 6380 shutdown nosave >/dev/null 2>&1 || true + fi + if [[ -f "$artifact_dir/postgres-bin.txt" ]]; then + "$(cat "$artifact_dir/postgres-bin.txt")/pg_ctl" \ + -D "${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres" -m fast stop >/dev/null 2>&1 || true + fi + [[ -n "$device_id" ]] && xcrun simctl shutdown "$device_id" >/dev/null 2>&1 || true +else + docker compose -f dev-env/dev-infra/docker-compose.yaml logs --no-color \ + >>"$artifact_dir/docker-services.log" 2>&1 || true + docker compose -f dev-env/dev-infra/docker-compose.yaml down --volumes --remove-orphans >/dev/null 2>&1 || true + [[ -n "$device_id" ]] && adb -s "$device_id" emu kill >/dev/null 2>&1 || true +fi diff --git a/.github/scripts/fixtures/failed-flow/android/report.xml b/.github/scripts/fixtures/failed-flow/android/report.xml new file mode 100644 index 0000000000..2a45978e43 --- /dev/null +++ b/.github/scripts/fixtures/failed-flow/android/report.xml @@ -0,0 +1,7 @@ + + + + + The expected button was not visible. + + diff --git a/.github/scripts/fixtures/infrastructure-failure/ios/phase.txt b/.github/scripts/fixtures/infrastructure-failure/ios/phase.txt new file mode 100644 index 0000000000..253983fa9e --- /dev/null +++ b/.github/scripts/fixtures/infrastructure-failure/ios/phase.txt @@ -0,0 +1 @@ +Building and installing the development client diff --git a/.github/scripts/fixtures/success/android/report.xml b/.github/scripts/fixtures/success/android/report.xml new file mode 100644 index 0000000000..86664c89b8 --- /dev/null +++ b/.github/scripts/fixtures/success/android/report.xml @@ -0,0 +1,4 @@ + + + + diff --git a/.github/scripts/fixtures/success/ios/report.xml b/.github/scripts/fixtures/success/ios/report.xml new file mode 100644 index 0000000000..890d6b7a94 --- /dev/null +++ b/.github/scripts/fixtures/success/ios/report.xml @@ -0,0 +1,4 @@ + + + + diff --git a/.github/scripts/run-nightly-e2e.sh b/.github/scripts/run-nightly-e2e.sh new file mode 100755 index 0000000000..dbd49d39d0 --- /dev/null +++ b/.github/scripts/run-nightly-e2e.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +platform="${1:?usage: run-nightly-e2e.sh }" +device_id="${2:?usage: run-nightly-e2e.sh }" + +if [[ "$platform" != "ios" && "$platform" != "android" ]]; then + echo "Unsupported platform: $platform" >&2 + exit 2 +fi + +artifact_dir="${GITHUB_WORKSPACE:-$PWD}/artifacts/$platform" +maestro_dir="$artifact_dir/maestro" +mkdir -p "$maestro_dir" + +phase() { + printf '%s\n' "$1" >"$artifact_dir/phase.txt" +} + +wait_for_port() { + local port="$1" + local label="$2" + local attempts="${3:-120}" + + for ((i = 1; i <= attempts; i++)); do + if nc -z 127.0.0.1 "$port" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + + echo "Timed out waiting for $label on port $port" >&2 + return 1 +} + +# shellcheck disable=SC2329 # Invoked through the cleanup trap call chain. +stop_process_tree() { + local pid="$1" + local child + while read -r child; do + [[ -n "$child" ]] && stop_process_tree "$child" + done < <(pgrep -P "$pid" 2>/dev/null || true) + kill -TERM "$pid" >/dev/null 2>&1 || true +} + +# shellcheck disable=SC2329 # Invoked by cleanup, which is registered as a trap. +stop_pid_file() { + local pid_file="$1" + [[ -f "$pid_file" ]] || return 0 + + local pid + pid="$(cat "$pid_file")" + [[ -n "$pid" ]] || return 0 + + # pnpm and Expo both spawn multiple generations of children. + stop_process_tree "$pid" +} + +# shellcheck disable=SC2329 # Invoked by the EXIT/INT/TERM trap below. +cleanup() { + set +e + stop_pid_file "$artifact_dir/metro.pid" + stop_pid_file "$artifact_dir/mock-server.pid" + + if [[ "$platform" == "ios" ]]; then + if [[ -f "$artifact_dir/redis.pid" ]]; then + redis_bin="$(cat "$artifact_dir/redis-bin.txt")" + "$redis_bin/redis-cli" -h 127.0.0.1 -p 6380 shutdown nosave >/dev/null 2>&1 || true + fi + if [[ -f "$artifact_dir/postgres-bin.txt" && -d "${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres" ]]; then + postgres_bin="$(cat "$artifact_dir/postgres-bin.txt")" + "$postgres_bin/pg_ctl" -D "${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres" -m fast stop >/dev/null 2>&1 || true + fi + else + docker compose -f dev-env/dev-infra/docker-compose.yaml logs --no-color \ + >>"$artifact_dir/docker-services.log" 2>&1 || true + docker compose -f dev-env/dev-infra/docker-compose.yaml down --volumes --remove-orphans >/dev/null 2>&1 || true + fi +} + +trap cleanup EXIT INT TERM + +phase "Starting PostgreSQL, Redis, and mock server" +if [[ "$platform" == "ios" ]]; then + brew install postgresql@14 2>&1 | tee "$artifact_dir/native-dependencies.log" + + postgres_bin="$(brew --prefix postgresql@14)/bin" + redis_version="7.4.7" + redis_archive="${RUNNER_TEMP:-/tmp}/redis-${redis_version}.tar.gz" + redis_source="${RUNNER_TEMP:-/tmp}/redis-${redis_version}" + curl -fsSL -o "$redis_archive" \ + "https://download.redis.io/releases/redis-${redis_version}.tar.gz" + echo "c97e57b0df330a9e091cacff012bebe763c275398cf36ff44cdba876814b595b $redis_archive" \ + | shasum -a 256 --check | tee -a "$artifact_dir/native-dependencies.log" + rm -rf "$redis_source" + tar -xzf "$redis_archive" -C "${RUNNER_TEMP:-/tmp}" + make -C "$redis_source" -j "$(sysctl -n hw.ncpu)" \ + 2>&1 | tee -a "$artifact_dir/native-dependencies.log" + redis_bin="$redis_source/src" + "$redis_bin/redis-server" --version | tee -a "$artifact_dir/native-dependencies.log" + printf '%s\n' "$redis_bin" >"$artifact_dir/redis-bin.txt" + + postgres_data="${RUNNER_TEMP:-/tmp}/nightly-e2e-postgres" + rm -rf "$postgres_data" + "$postgres_bin/initdb" -D "$postgres_data" --auth=trust --username=pg --no-locale \ + >"$artifact_dir/postgres-init.log" 2>&1 + "$postgres_bin/pg_ctl" -D "$postgres_data" \ + -o "-p 5433 -h 127.0.0.1" -l "$artifact_dir/postgres.log" start + printf '%s\n' "$postgres_bin" >"$artifact_dir/postgres-bin.txt" + + "$redis_bin/redis-server" \ + --bind 127.0.0.1 \ + --port 6380 \ + --save "" \ + --appendonly no \ + --daemonize yes \ + --pidfile "$artifact_dir/redis.pid" \ + --logfile "$artifact_dir/redis.log" + + wait_for_port 5433 "PostgreSQL" + wait_for_port 6380 "Redis" + pnpm --dir dev-env start:external >"$artifact_dir/mock-server.log" 2>&1 & +else + pnpm --dir dev-env start >"$artifact_dir/mock-server.log" 2>&1 & +fi +printf '%s\n' "$!" >"$artifact_dir/mock-server.pid" +wait_for_port 1986 "the E2E mock-server manager" + +phase "Starting Metro" +EXPO_PUBLIC_ENV=e2e \ + NODE_ENV=test \ + RN_SRC_EXT=e2e.ts,e2e.tsx \ + pnpm exec expo start --dev-client --clear --port 8081 \ + >"$artifact_dir/metro.log" 2>&1 & +printf '%s\n' "$!" >"$artifact_dir/metro.pid" +wait_for_port 8081 "Metro" + +if [[ "$platform" == "android" ]]; then + phase "Configuring Android localhost routing" + adb -s "$device_id" reverse tcp:3000 tcp:3000 + adb -s "$device_id" reverse tcp:8081 tcp:8081 +fi + +phase "Building and installing the development client" +if [[ "$platform" == "ios" ]]; then + EXPO_PUBLIC_ENV=e2e \ + NODE_ENV=test \ + RN_SRC_EXT=e2e.ts,e2e.tsx \ + pnpm exec expo run:ios --device "$device_id" --no-bundler \ + 2>&1 | tee "$artifact_dir/build.log" +else + EXPO_PUBLIC_ENV=e2e \ + NODE_ENV=test \ + RN_SRC_EXT=e2e.ts,e2e.tsx \ + pnpm exec expo run:android --device "$device_id" --no-bundler \ + 2>&1 | tee "$artifact_dir/build.log" +fi + +phase "Running Maestro flows" +set +e +maestro test \ + --udid "$device_id" \ + --format JUNIT \ + --output "$artifact_dir/report.xml" \ + --config __e2e__/config.yml \ + --debug-output "$maestro_dir" \ + --test-output-dir "$maestro_dir" \ + --flatten-debug-output \ + __e2e__ \ + 2>&1 | tee "$artifact_dir/maestro-cli.log" +maestro_status=${PIPESTATUS[0]} +set -e + +if [[ "$maestro_status" -eq 0 ]]; then + phase "Completed" +else + phase "Maestro flow failure" +fi + +exit "$maestro_status" diff --git a/.github/scripts/summarize-maestro.mjs b/.github/scripts/summarize-maestro.mjs new file mode 100644 index 0000000000..b7ccb4aa19 --- /dev/null +++ b/.github/scripts/summarize-maestro.mjs @@ -0,0 +1,224 @@ +import fs from 'node:fs' +import path from 'node:path' +import process from 'node:process' + +const ENTITY_REPLACEMENTS = { + '&': '&', + ''': "'", + '>': '>', + '<': '<', + '"': '"', +} + +function decodeXml(value = '') { + return value + .replace(/&(amp|apos|gt|lt|quot);/g, entity => ENTITY_REPLACEMENTS[entity]) + .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code))) + .replace(/&#x([\da-f]+);/gi, (_, code) => + String.fromCodePoint(Number.parseInt(code, 16)), + ) +} + +function attributes(source = '') { + const result = {} + for (const match of source.matchAll(/([\w:.-]+)\s*=\s*(["'])(.*?)\2/gs)) { + result[match[1]] = decodeXml(match[3]) + } + return result +} + +function concise(value, limit = 300) { + const normalized = decodeXml(value) + .replace(/<[^>]+>/g, ' ') + .replace(/\s+/g, ' ') + .trim() + return normalized.length > limit + ? `${normalized.slice(0, limit - 1)}…` + : normalized +} + +export function parseJUnit(xml) { + const failures = [] + const testcasePattern = /]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/gi + + for (const match of xml.matchAll(testcasePattern)) { + const testcase = attributes(match[1]) + const body = match[2] || '' + const failure = body.match(/<(failure|error)\b([^>]*)>([\s\S]*?)<\/\1>/i) + const selfClosingFailure = body.match(/<(failure|error)\b([^>]*)\/>/i) + const failureMatch = failure || selfClosingFailure + if (!failureMatch) continue + + const failureAttributes = attributes(failureMatch[2]) + const name = testcase.name || testcase.classname || 'Unnamed Maestro flow' + const message = concise( + failureAttributes.message || (failure ? failureMatch[3] : '') || 'Failed', + ) + failures.push({name, message}) + } + + if (failures.length === 0) { + const suite = xml.match(/]*)>/i) + const suiteAttributes = attributes(suite?.[1]) + if ( + Number(suiteAttributes.failures || 0) + + Number(suiteAttributes.errors || 0) > + 0 + ) { + failures.push({ + name: suiteAttributes.name || 'Maestro test suite', + message: 'JUnit reported a failure without testcase details', + }) + } + } + + return failures +} + +function walk(root) { + if (!root || !fs.existsSync(root)) return [] + const entries = fs.readdirSync(root, {withFileTypes: true}) + return entries.flatMap(entry => { + const candidate = path.join(root, entry.name) + return entry.isDirectory() ? walk(candidate) : [candidate] + }) +} + +function readPhase(root) { + const phaseFile = walk(root).find(file => path.basename(file) === 'phase.txt') + return phaseFile ? fs.readFileSync(phaseFile, 'utf8').trim() : '' +} + +function platformResult({name, status, root, artifactUrl}) { + const reports = walk(root).filter(file => + /(?:report|junit).*\.xml$/i.test(file), + ) + const failures = reports.flatMap(report => + parseJUnit(fs.readFileSync(report, 'utf8')), + ) + const failed = status !== 'success' || failures.length > 0 + return { + name, + status, + failed, + failures, + phase: readPhase(root), + hasJUnit: reports.length > 0, + artifactUrl, + } +} + +function statusEmoji(status) { + return status === 'success' ? ':white_check_mark:' : ':x:' +} + +function slackEscape(value) { + return value + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') +} + +export function buildSummary({ + iosStatus, + androidStatus, + iosRoot, + androidRoot, + artifactUrls = {}, + sha, + runUrl, + commitUrl, +}) { + const platforms = [ + platformResult({ + name: 'iOS', + status: iosStatus, + root: iosRoot, + artifactUrl: artifactUrls.ios, + }), + platformResult({ + name: 'Android', + status: androidStatus, + root: androidRoot, + artifactUrl: artifactUrls.android, + }), + ] + const notify = platforms.some(platform => platform.failed) + const shortSha = sha.slice(0, 12) + const lines = [ + ':rotating_light: *Nightly Maestro E2E failed*', + `*Commit:* <${commitUrl}|\`${shortSha}\`>`, + `*Workflow run:* <${runUrl}|open run>`, + '', + ] + + for (const platform of platforms) { + lines.push( + `${statusEmoji(platform.status)} *${platform.name}* — job status: \`${platform.status}\``, + ) + if (platform.failures.length > 0) { + for (const failure of platform.failures.slice(0, 10)) { + lines.push( + `• *${slackEscape(failure.name)}:* ${slackEscape(failure.message)}`, + ) + } + if (platform.failures.length > 10) { + lines.push(`• …and ${platform.failures.length - 10} more failed flows`) + } + } else if (platform.failed && !platform.hasJUnit) { + lines.push( + `• Setup phase: ${platform.phase || 'No phase metadata was captured'}`, + ) + } else if (platform.failed) { + lines.push( + `• The job failed after JUnit was written (latest phase: ${platform.phase || 'unknown'})`, + ) + } + if (platform.artifactUrl) { + lines.push( + `• <${platform.artifactUrl}|${platform.name} logs and artifacts>`, + ) + } + lines.push('') + } + + const text = lines.join('\n').trim() + return { + notify, + platforms, + payload: {text}, + } +} + +function parseArgs(argv) { + const result = {} + for (let i = 0; i < argv.length; i += 2) { + const key = argv[i] + if (!key?.startsWith('--') || argv[i + 1] === undefined) { + throw new Error(`Invalid argument: ${key || ''}`) + } + result[key.slice(2)] = argv[i + 1] + } + return result +} + +if ( + process.argv[1] && + path.resolve(process.argv[1]) === path.resolve(import.meta.filename) +) { + const args = parseArgs(process.argv.slice(2)) + const artifactUrls = args['artifact-urls'] + ? JSON.parse(fs.readFileSync(args['artifact-urls'], 'utf8')) + : {} + const summary = buildSummary({ + iosStatus: args['ios-status'], + androidStatus: args['android-status'], + iosRoot: args['ios-root'], + androidRoot: args['android-root'], + artifactUrls, + sha: args.sha, + runUrl: args['run-url'], + commitUrl: args['commit-url'], + }) + process.stdout.write(`${JSON.stringify(summary)}\n`) +} diff --git a/.github/scripts/summarize-maestro.test.mjs b/.github/scripts/summarize-maestro.test.mjs new file mode 100644 index 0000000000..9c724a8f7d --- /dev/null +++ b/.github/scripts/summarize-maestro.test.mjs @@ -0,0 +1,75 @@ +import assert from 'node:assert/strict' +import path from 'node:path' +import test from 'node:test' +import {fileURLToPath} from 'node:url' + +import {buildSummary, parseJUnit} from './summarize-maestro.mjs' + +const here = path.dirname(fileURLToPath(import.meta.url)) +const fixtures = path.join(here, 'fixtures') +const common = { + artifactUrls: { + ios: 'https://github.test/run/artifacts/1', + android: 'https://github.test/run/artifacts/2', + }, + sha: '0123456789abcdef', + runUrl: 'https://github.test/run', + commitUrl: 'https://github.test/commit/0123456789abcdef', +} + +test('does not notify for successful jobs and successful JUnit', () => { + const root = path.join(fixtures, 'success') + const summary = buildSummary({ + ...common, + iosStatus: 'success', + androidStatus: 'success', + iosRoot: path.join(root, 'ios'), + androidRoot: path.join(root, 'android'), + }) + + assert.equal(summary.notify, false) + assert.deepEqual( + summary.platforms.map(platform => platform.failures), + [[], []], + ) +}) + +test('reports failed flow names and concise failure messages', () => { + const root = path.join(fixtures, 'failed-flow') + const summary = buildSummary({ + ...common, + iosStatus: 'success', + androidStatus: 'failure', + iosRoot: path.join(fixtures, 'success', 'ios'), + androidRoot: path.join(root, 'android'), + }) + + assert.equal(summary.notify, true) + assert.match(summary.payload.text, /create-account\.yml/) + assert.match(summary.payload.text, /Element not found: Create account/) + assert.match(summary.payload.text, /Android logs and artifacts/) +}) + +test('reports the latest setup phase when infrastructure fails before JUnit', () => { + const root = path.join(fixtures, 'infrastructure-failure') + const summary = buildSummary({ + ...common, + iosStatus: 'failure', + androidStatus: 'success', + iosRoot: path.join(root, 'ios'), + androidRoot: path.join(fixtures, 'success', 'android'), + }) + + assert.equal(summary.notify, true) + assert.match( + summary.payload.text, + /Setup phase: Building and installing the development client/, + ) +}) + +test('parses self-closing JUnit failures', () => { + const failures = parseJUnit( + '', + ) + assert.deepEqual(failures, [{name: 'flow.yml', message: 'boom'}]) +}) diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml new file mode 100644 index 0000000000..e6c0d1d2b7 --- /dev/null +++ b/.github/workflows/nightly-e2e.yml @@ -0,0 +1,262 @@ +--- +name: Nightly Maestro E2E + +on: + schedule: + - cron: "0 4 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: nightly-maestro-e2e-${{ github.ref }} + cancel-in-progress: false + +env: + CI: "1" + MAESTRO_VERSION: "2.6.1" + MAESTRO_DRIVER_STARTUP_TIMEOUT: "180000" + MAESTRO_CLI_NO_ANALYTICS: "1" + MAESTRO_CLI_ANALYSIS_NOTIFICATION_DISABLED: "true" + MAESTRO_DISABLE_UPDATE_CHECK: "1" + +jobs: + ios: + name: iOS Maestro E2E + if: github.repository == 'bluesky-social/social-app' + runs-on: macos-26-xlarge + timeout-minutes: 120 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Select Xcode 26.4 + uses: maxim-lobanov/setup-xcode@ed7a3b1fda3918c0306d1b724322adc0b8cc0a90 # v1.7.0 + with: + xcode-version: "26.4" + + - name: Set up pnpm + uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: package.json + cache: pnpm + + - name: Set up Java 17 + uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 + with: + distribution: temurin + java-version: "17" + + - name: Prepare E2E configuration + run: | + mkdir -p artifacts/ios + echo "Installing dependencies" > artifacts/ios/phase.txt + cp .env.example .env.test + cp google-services.json.example google-services.json + + - name: Install dependencies + run: | + set -o pipefail + pnpm install --frozen-lockfile 2>&1 | tee artifacts/ios/dependencies.log + pnpm --dir dev-env install --frozen-lockfile 2>&1 | tee -a artifacts/ios/dependencies.log + pnpm intl:build 2>&1 | tee artifacts/ios/i18n.log + + - name: Install Maestro 2.6.1 + run: | + echo "Installing Maestro" > artifacts/ios/phase.txt + curl -fsSL -o "$RUNNER_TEMP/maestro.zip" \ + "https://github.com/mobile-dev-inc/Maestro/releases/download/cli-${MAESTRO_VERSION}/maestro.zip" + echo "3440825f514f537c6a96bcf5de995780c2a4a7f83a43208fdc95d4f1fecfad3b $RUNNER_TEMP/maestro.zip" \ + | shasum -a 256 --check + unzip -q "$RUNNER_TEMP/maestro.zip" -d "$RUNNER_TEMP" + echo "$RUNNER_TEMP/maestro/bin" >> "$GITHUB_PATH" + "$RUNNER_TEMP/maestro/bin/maestro" --version | tee artifacts/ios/maestro-version.log + test "$("$RUNNER_TEMP/maestro/bin/maestro" --version)" = "$MAESTRO_VERSION" + + - name: Boot one iOS simulator + run: | + echo "Booting iOS simulator" > artifacts/ios/phase.txt + udid=$(xcrun simctl list devices available --json | jq -r \ + '[.devices[][] | select(.name | startswith("iPhone"))] | first | .udid') + if [ -z "$udid" ] || [ "$udid" = "null" ]; then + echo "No available iPhone simulator was found" >&2 + exit 1 + fi + echo "IOS_UDID=$udid" >> "$GITHUB_ENV" + xcrun simctl shutdown all || true + xcrun simctl boot "$udid" + xcrun simctl bootstatus "$udid" -b + echo "Using iOS simulator $udid" + + - name: Run iOS Maestro suite + run: .github/scripts/run-nightly-e2e.sh ios "$IOS_UDID" + + - name: Clean up iOS services and simulator + if: always() + run: .github/scripts/cleanup-nightly-e2e.sh ios "${IOS_UDID:-}" + + - name: Upload iOS E2E artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: nightly-e2e-ios-${{ github.run_id }} + path: artifacts/ios + if-no-files-found: warn + retention-days: 14 + + android: + name: Android Maestro E2E + if: github.repository == 'bluesky-social/social-app' + # Linux-x64-32core is a repository-managed runner label. + runs-on: Linux-x64-32core + timeout-minutes: 120 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Set up pnpm + uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version-file: package.json + cache: pnpm + + - name: Set up Java 17 + uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 + with: + distribution: temurin + java-version: "17" + + - name: Prepare E2E configuration + run: | + mkdir -p artifacts/android + echo "Installing dependencies" > artifacts/android/phase.txt + cp .env.example .env.test + cp google-services.json.example google-services.json + + - name: Install dependencies + run: | + set -o pipefail + pnpm install --frozen-lockfile 2>&1 | tee artifacts/android/dependencies.log + pnpm --dir dev-env install --frozen-lockfile 2>&1 | tee -a artifacts/android/dependencies.log + pnpm intl:build 2>&1 | tee artifacts/android/i18n.log + + - name: Install Maestro 2.6.1 + run: | + echo "Installing Maestro" > artifacts/android/phase.txt + curl -fsSL -o "$RUNNER_TEMP/maestro.zip" \ + "https://github.com/mobile-dev-inc/Maestro/releases/download/cli-${MAESTRO_VERSION}/maestro.zip" + echo "3440825f514f537c6a96bcf5de995780c2a4a7f83a43208fdc95d4f1fecfad3b $RUNNER_TEMP/maestro.zip" \ + | shasum -a 256 --check + unzip -q "$RUNNER_TEMP/maestro.zip" -d "$RUNNER_TEMP" + echo "$RUNNER_TEMP/maestro/bin" >> "$GITHUB_PATH" + "$RUNNER_TEMP/maestro/bin/maestro" --version | tee artifacts/android/maestro-version.log + test "$("$RUNNER_TEMP/maestro/bin/maestro" --version)" = "$MAESTRO_VERSION" + + - name: Boot one Android emulator and run Maestro suite + uses: reactivecircus/android-emulator-runner@1dcd0090116d15e7c562f8db72807de5e036a4ed # v2.34.0 + with: + api-level: 35 + target: google_apis + arch: x86_64 + profile: pixel_6 + avd-name: nightly-e2e + emulator-port: 5554 + disable-animations: true + emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -camera-back none + script: .github/scripts/run-nightly-e2e.sh android emulator-5554 + + - name: Clean up Android services and emulator + if: always() + run: .github/scripts/cleanup-nightly-e2e.sh android emulator-5554 + + - name: Upload Android E2E artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: nightly-e2e-android-${{ github.run_id }} + path: artifacts/android + if-no-files-found: warn + retention-days: 14 + + report: + name: Report E2E failures + needs: [ios, android] + if: ${{ always() && github.repository == 'bluesky-social/social-app' }} + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Download iOS artifacts + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: nightly-e2e-ios-${{ github.run_id }} + path: downloaded-artifacts/ios + + - name: Download Android artifacts + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: nightly-e2e-android-${{ github.run_id }} + path: downloaded-artifacts/android + + - name: Resolve artifact links + env: + GH_TOKEN: ${{ github.token }} + run: | + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}#artifacts" + jq -n --arg run "$run_url" '{ios: $run, android: $run}' > artifact-links.json + if gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts?per_page=100" \ + > artifact-response.json; then + jq --arg base "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts" \ + --arg run "$run_url" \ + '{ + ios: (.artifacts | map(select(.name | startswith("nightly-e2e-ios-"))) | first | if . then ($base + "/" + (.id | tostring)) else $run end), + android: (.artifacts | map(select(.name | startswith("nightly-e2e-android-"))) | first | if . then ($base + "/" + (.id | tostring)) else $run end) + }' artifact-response.json > artifact-links.json + fi + + - name: Summarize platform results + id: summary + env: + IOS_STATUS: ${{ needs.ios.result }} + ANDROID_STATUS: ${{ needs.android.result }} + run: | + node .github/scripts/summarize-maestro.mjs \ + --ios-status "$IOS_STATUS" \ + --android-status "$ANDROID_STATUS" \ + --ios-root downloaded-artifacts/ios \ + --android-root downloaded-artifacts/android \ + --artifact-urls artifact-links.json \ + --sha "$GITHUB_SHA" \ + --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + --commit-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}" \ + > e2e-summary.json + echo "notify=$(jq -r .notify e2e-summary.json)" >> "$GITHUB_OUTPUT" + echo "payload=$(jq -c .payload e2e-summary.json)" >> "$GITHUB_OUTPUT" + jq -r .payload.text e2e-summary.json >> "$GITHUB_STEP_SUMMARY" + + - name: Notify Slack of E2E failures + if: steps.summary.outputs.notify == 'true' + uses: slackapi/slack-github-action@45a88b9581bfab2566dc881e2cd66d334e621e2c # v3.0.3 + with: + webhook: ${{ secrets.E2E_FAILURES_SLACK_WEBHOOK }} + webhook-type: incoming-webhook + payload: ${{ steps.summary.outputs.payload }} diff --git a/dev-env/package.json b/dev-env/package.json index e77ad1dfab..c65ca9cd64 100644 --- a/dev-env/package.json +++ b/dev-env/package.json @@ -3,7 +3,8 @@ "version": "0.0.0", "type": "module", "scripts": { - "start": "NODE_ENV=development ./dev-infra/with-test-redis-and-db.sh node ./mock-server.ts" + "start": "NODE_ENV=development ./dev-infra/with-test-redis-and-db.sh node ./mock-server.ts", + "start:external": "NODE_ENV=development PGPORT=5433 PGHOST=localhost PGUSER=pg PGPASSWORD=password PGDATABASE=postgres DB_POSTGRES_URL=postgresql://pg:password@127.0.0.1:5433/postgres REDIS_HOST=127.0.0.1:6380 node ./mock-server.ts" }, "dependencies": { "@atproto/api": "^0.20.22", diff --git a/docs/testing.md b/docs/testing.md index b84d966112..9bde1e18e0 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -29,6 +29,44 @@ adb reverse tcp:3000 tcp:3000 - In a second tab, run `pnpm e2e:build` - In a third tab, run `pnpm e2e:run __e2e__` +## Nightly Maestro CI + +The `Nightly Maestro E2E` GitHub Actions workflow runs every day at 04:00 UTC +and can also be started from the Actions tab with **Run workflow**. It runs iOS +and Android concurrently, but each platform runs all of `__e2e__/config.yml` +sequentially on one explicitly selected simulator or emulator. The flows share a +stateful mock-server manager, so the suite must not be sharded. + +The jobs run Maestro CLI 2.6.1 locally on GitHub Actions; Maestro Cloud is not +used. iOS runs on `macos-26-xlarge` with Xcode 26.4. Android runs on +`Linux-x64-32core`. Both use Java 17 and the Node and pnpm versions declared in +`package.json`. + +The mock-server manager listens on host port 1986 and creates test services on +port 3000. Metro listens on 8081. Android reverses ports 3000 and 8081 into the +emulator; port 1986 remains host-side because Maestro JavaScript calls it from +the runner. Android uses the existing Docker Compose PostgreSQL 14 and Redis 7 +services on ports 5433 and 6380. GitHub-hosted macOS cannot run nested Docker +virtualization, so iOS provisions ephemeral native PostgreSQL 14.x and Redis +7.4.7 on those same ports and starts `pnpm --dir dev-env start:external`. + +Each platform uploads a `nightly-e2e--` artifact for 14 days. +It contains JUnit at `report.xml`, Maestro screenshots, videos, command metadata +and `maestro.log` under `maestro/`, plus Metro, native build, mock-server, service, +dependency, and translation logs. The workflow always uploads what was captured, +including when setup or the native build fails before Maestro starts. + +Add the repository secret `E2E_FAILURES_SLACK_WEBHOOK` before enabling the +schedule. The aggregation job runs even when either platform fails and posts one +detailed Slack notification containing both job statuses, failed flow details or +the failed setup phase, the commit and workflow links, and links to both artifact +sets. Successful runs do not post to Slack. + +Before relying on the schedule, manually dispatch the workflow and verify both +platforms against live Metro and `dev-env`, Android localhost routing, artifact +uploads on success and failure, one Slack message for a forced failure, and no +Slack message for an all-green run. + ## Using Flashlight for Performance Testing 1. Make sure Maestro is installed (optional: only for automated testing) by following the instructions above 2. Install Flashlight by following [these instructions](https://docs.flashlight.dev/)