mattermost-mobile-upstream/.github/workflows/e2e-android-template.yml
yasser khan e83c44959d
feat(e2e): Run e2e against different server version when rc branch is pushed (#9800)
* ci(e2e): scheduled CMT trigger, fix push double-fire, drop cleanup call

Aligns mobile E2E workflows with matterwick's webhook-driven model:

- cmt-provisioner.yml: replace the old stub (which relied on the dead
  workflow_run-inputs path) with a lightweight monthly scheduled trigger
  (no inputs). matterwick provisions servers for its hardcoded version set
  and dispatches compatibility-matrix-testing.yml.

- compatibility-matrix-testing.yml: remove the /cleanup_e2e curl step;
  matterwick destroys servers on the workflow_run completed event (SHA match).
2026-07-07 18:02:13 +05:30

493 lines
20 KiB
YAML

name: Detox Android E2E Tests Template
on:
workflow_call:
secrets:
MM_MOBILE_E2E_ADMIN_EMAIL:
required: false
MM_MOBILE_E2E_ADMIN_USERNAME:
required: false
MM_MOBILE_E2E_ADMIN_PASSWORD:
required: false
MM_MOBILE_E2E_ZEPHYR_API_KEY:
required: false
MM_MOBILE_E2E_TEST_CYCLE_LINK_PREFIX:
required: false
MM_MOBILE_E2E_WEBHOOK_URL:
required: false
MM_MOBILE_DETOX_AWS_ACCESS_KEY_ID:
required: false
MM_MOBILE_DETOX_AWS_SECRET_ACCESS_KEY:
required: false
inputs:
MM_TEST_SERVER_URL:
description: "The test server URL (SITE_1)"
required: false
type: string
MM_TEST_SERVER_URL_2:
description: "The second test server URL (SITE_2); optional, no automatic fallback"
required: false
type: string
MM_TEST_SERVER_URL_3:
description: "The third test server URL (SITE_3); optional, no automatic fallback"
required: false
type: string
MM_TEST_USER_NAME:
description: "The admin username of the test instance"
required: false
type: string
MM_TEST_PASSWORD:
description: "The admin password of the test instance"
required: false
type: string
MOBILE_VERSION:
description: "The mobile version to test"
required: false
default: ""
type: string
run-android-tests:
description: "Run Android tests"
required: true
type: boolean
run-type:
type: string
required: false
default: "PR"
testcase_failure_fatal:
description: "Should failures be considered fatal"
required: false
type: boolean
default: false
record_tests_in_zephyr:
description: "Record test results in Zephyr, typically for nightly and release runs"
required: false
type: boolean
default: false
low_bandwidth_mode:
description: "Enable low bandwidth mode"
required: false
type: boolean
default: false
search_path:
description: "Path to search for test specs (use for smoke test subset)"
required: false
type: string
default: "detox/e2e/test"
parallelism:
description: "Number of parallel test shards"
required: false
type: string
default: "20"
android_avd_name:
description: "Android Emulator name"
required: false
type: string
default: "detox_pixel_8"
android_api_level:
description: "Android API level"
required: false
type: string
default: "35"
artifact_run_id:
description: "Run ID of the workflow that uploaded android-build-files-* (defaults to caller run)"
required: false
type: string
default: ""
outputs:
STATUS:
value: ${{ jobs.generate-report.outputs.STATUS }}
TARGET_URL:
value: ${{ jobs.generate-report.outputs.TARGET_URL }}
FAILURES:
value: ${{ jobs.generate-report.outputs.FAILURES }}
PASSES:
value: ${{ jobs.generate-report.outputs.PASSES }}
TOTAL:
value: ${{ jobs.generate-report.outputs.TOTAL }}
SKIPPED:
value: ${{ jobs.generate-report.outputs.SKIPPED }}
ERRORS:
value: ${{ jobs.generate-report.outputs.ERRORS }}
PERCENTAGE:
value: ${{ jobs.generate-report.outputs.PERCENTAGE }}
env:
AWS_REGION: "us-east-1"
ADMIN_EMAIL: ${{ secrets.MM_MOBILE_E2E_ADMIN_EMAIL }}
ADMIN_USERNAME: ${{ inputs.MM_TEST_USER_NAME || secrets.MM_MOBILE_E2E_ADMIN_USERNAME }}
ADMIN_PASSWORD: ${{ inputs.MM_TEST_PASSWORD || secrets.MM_MOBILE_E2E_ADMIN_PASSWORD }}
BRANCH: ${{ github.event_name == 'pull_request' && github.head_ref || github.ref_name }}
COMMIT_HASH: ${{ github.sha }}
DEVICE_NAME: ${{ inputs.android_avd_name }} # This is needed to split tests as same code is used in iOS job
DEVICE_OS_VERSION: ${{ inputs.android_api_level }}
DETOX_AWS_S3_BUCKET: "mattermost-detox-report"
HEADLESS: "true"
TYPE: ${{ inputs.run-type }}
PULL_REQUEST: "https://github.com/mattermost/mattermost-mobile/pull/${{ github.event.number }}"
SITE_1_URL: ${{ inputs.MM_TEST_SERVER_URL }}
SITE_2_URL: ${{ inputs.MM_TEST_SERVER_URL_2 }}
SITE_3_URL: ${{ inputs.MM_TEST_SERVER_URL_3 }}
ZEPHYR_ENABLE: ${{ inputs.record_tests_in_zephyr }}
JIRA_PROJECT_KEY: "MM"
ZEPHYR_API_KEY: ${{ secrets.MM_MOBILE_E2E_ZEPHYR_API_KEY }}
ZEPHYR_FOLDER_ID: "3233873"
TEST_CYCLE_LINK_PREFIX: ${{ secrets.MM_MOBILE_E2E_TEST_CYCLE_LINK_PREFIX }}
WEBHOOK_URL: ${{ secrets.MM_MOBILE_E2E_WEBHOOK_URL }}
FAILURE_MESSAGE: "Something has failed"
IOS: "false"
RUNNING_E2E: "true"
AVD_NAME: ${{ inputs.android_avd_name }}
SDK_VERSION: ${{ inputs.android_api_level }}
ARTIFACT_RUN_ID: ${{ inputs.artifact_run_id || github.run_id }}
jobs:
generate-specs:
runs-on: ubuntu-22.04
outputs:
specs: ${{ steps.generate-specs.outputs.specs }}
build_id: ${{ steps.resolve-device.outputs.BUILD_ID }}
mobile_sha: ${{ steps.resolve-device.outputs.MOBILE_SHA }}
mobile_ref: ${{ steps.resolve-device.outputs.MOBILE_REF }}
workflow_hash: ${{ steps.resolve-device.outputs.WORKFLOW_HASH }}
steps:
- name: Checkout Repository
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ inputs.MOBILE_VERSION }}
- name: Set Build ID
id: resolve-device
run: |
BUILD_ID="${{ github.run_id }}-${{ env.AVD_NAME }}-${{ env.SDK_VERSION}}"
WORKFLOW_HASH=$(tr -dc a-z0-9 </dev/urandom | head -c 10)
## We need that hash to separate the artifacts
echo "WORKFLOW_HASH=${WORKFLOW_HASH}" >> ${GITHUB_OUTPUT}
echo "BUILD_ID=$(echo ${BUILD_ID} | sed 's/ /_/g')" >> ${GITHUB_OUTPUT}
echo "MOBILE_SHA=$(git rev-parse HEAD)" >> ${GITHUB_OUTPUT}
echo "MOBILE_REF=$(git rev-parse --abbrev-ref HEAD)" >> ${GITHUB_OUTPUT}
- name: Generate Test Specs
id: generate-specs
uses: ./.github/actions/generate-specs
with:
parallelism: ${{ inputs.parallelism }}
search_path: ${{ inputs.search_path }}
device_name: ${{ env.AVD_NAME }}
device_os_version: ${{ env.SDK_VERSION }}
e2e-android:
name: machine-${{ matrix.runId }}-api-${{ matrix.deviceOsVersion }}
runs-on: ubuntu-latest-8-cores
continue-on-error: true
timeout-minutes: 240
env:
ANDROID_HOME: /usr/local/lib/android/sdk
ANDROID_SDK_ROOT: /usr/local/lib/android/sdk
needs:
- generate-specs
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.generate-specs.outputs.specs) }}
steps:
- name: Checkout Repository
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ inputs.MOBILE_VERSION }}
- name: Install Dependencies
run: |
sudo apt-get update
sudo apt-get install -y libpulse0
sudo apt-get install -y scrot ffmpeg xvfb
- name: Enable KVM
run: |
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
sudo udevadm control --reload-rules
sudo udevadm trigger --name-match=kvm
- name: ci/prepare-node-deps
uses: ./.github/actions/prepare-node-deps
- name: Cache detox node_modules
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: detox/node_modules
key: ${{ runner.os }}-detox-deps-${{ hashFiles('detox/package-lock.json') }}
restore-keys: ${{ runner.os }}-detox-deps-
- name: Install Detox Dependencies
run: |
cd detox
npm install
- name: Set .env with RUNNING_E2E=true
run: |
cat > .env <<EOF
RUNNING_E2E=true
EOF
- name: Download APK artifact
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: android-build-files-${{ env.ARTIFACT_RUN_ID }}
run-id: ${{ env.ARTIFACT_RUN_ID }}
github-token: ${{ github.token }}
path: android/app/build
- name: Set up Android SDK
run: |
export ANDROID_HOME=/usr/local/lib/android/sdk
export PATH=$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/emulator:$ANDROID_HOME/tools/bin:$ANDROID_HOME/platform-tools:$PATH
echo "ANDROID_HOME=$ANDROID_HOME" >> $GITHUB_ENV
echo "PATH=$PATH" >> $GITHUB_ENV
- name: Start Xvfb
run: |
Xvfb :99 -screen 0 1920x1080x24 &
export DISPLAY=:99
echo "DISPLAY=:99" >> $GITHUB_ENV
- name: Accept Android licenses
run: |
mkdir -p ~/.android
yes | sdkmanager --licenses > sdk_licenses_output.txt || true
# Check if "All SDK package licenses accepted" appears in output
if grep -q "All SDK package licenses accepted" sdk_licenses_output.txt; then
echo "✅ All licenses accepted successfully."
else
echo "❌ Licenses not fully accepted."
cat sdk_licenses_output.txt
exit 1
fi
# Cache SDK under the workspace (not /usr/local) — tar restore into system
# SDK dirs often fails with exit code 2 when paths already exist on the image.
- name: Cache Android SDK components
id: android-sdk-cache
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: .android-sdk-cache
key: android-sdk-${{ inputs.android_api_level }}-google-apis-x86_64-v2
- name: Apply cached Android SDK components
if: steps.android-sdk-cache.outputs.cache-hit == 'true'
env:
API_LEVEL: ${{ inputs.android_api_level }}
run: |
for component in platform-tools emulator; do
if [ -d ".android-sdk-cache/${component}" ]; then
rm -rf "${ANDROID_HOME:?}/${component}"
cp -a ".android-sdk-cache/${component}" "${ANDROID_HOME}/"
fi
done
if [ -d ".android-sdk-cache/platforms/android-${API_LEVEL}" ]; then
rm -rf "${ANDROID_HOME}/platforms/android-${API_LEVEL}"
mkdir -p "${ANDROID_HOME}/platforms"
cp -a ".android-sdk-cache/platforms/android-${API_LEVEL}" "${ANDROID_HOME}/platforms/"
fi
if [ -d ".android-sdk-cache/system-images/android-${API_LEVEL}" ]; then
rm -rf "${ANDROID_HOME}/system-images/android-${API_LEVEL}"
mkdir -p "${ANDROID_HOME}/system-images"
cp -a ".android-sdk-cache/system-images/android-${API_LEVEL}" "${ANDROID_HOME}/system-images/"
fi
- name: Install Android SDK components
if: steps.android-sdk-cache.outputs.cache-hit != 'true'
env:
API_LEVEL: ${{ inputs.android_api_level }}
JAVA_HOME: ${{ env.JAVA_HOME_17_X64 }}
run: |
if ! [[ "$API_LEVEL" =~ ^[0-9]+$ ]]; then
echo "Invalid android_api_level: $API_LEVEL"
exit 1
fi
for attempt in 1 2 3; do
echo "SDK install attempt $attempt..."
if yes | sdkmanager --install "platform-tools" "emulator" "platforms;android-${API_LEVEL}" "system-images;android-${API_LEVEL};google_apis;x86_64"; then
echo "SDK install succeeded on attempt $attempt"
break
fi
echo "Attempt $attempt failed, retrying..."
if [ "$attempt" = "3" ]; then
echo "SDK install failed after 3 attempts"
exit 1
fi
sleep 10
done
mkdir -p .android-sdk-cache/platforms .android-sdk-cache/system-images
cp -a "${ANDROID_HOME}/platform-tools" .android-sdk-cache/
cp -a "${ANDROID_HOME}/emulator" .android-sdk-cache/
cp -a "${ANDROID_HOME}/platforms/android-${API_LEVEL}" .android-sdk-cache/platforms/
cp -a "${ANDROID_HOME}/system-images/android-${API_LEVEL}" .android-sdk-cache/system-images/
# CI AVDs live under detox/.android/avd (see create_android_emulator.sh), not ~/.android/avd.
- name: Cache Android Emulator & AVD
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: |
detox/.android
detox/${{ env.AVD_NAME }}_api_${{ inputs.android_api_level }}
key: android-emulator-${{ runner.os }}-${{ runner.arch }}-${{ env.AVD_NAME }}-api-${{ inputs.android_api_level }}-v3
restore-keys: |
android-emulator-${{ runner.os }}-${{ runner.arch }}-${{ env.AVD_NAME }}-api-${{ inputs.android_api_level }}-
android-emulator-${{ runner.os }}-${{ runner.arch }}-
# Retry on emulator crash (all-shards-fail signature); cleans up wedged AVD between attempts.
- name: Create and run Android Emulator
env:
MATRIX_SPECS: ${{ matrix.specs }}
SITE_1_URL: ${{ env.SITE_1_URL }}
SITE_2_URL: ${{ env.SITE_2_URL }}
SITE_3_URL: ${{ env.SITE_3_URL }}
MAX_ATTEMPTS: "2"
ATTEMPT_TIMEOUT_MIN: "60"
shell: bash
run: |
set +e
attempt=1
while (( attempt <= MAX_ATTEMPTS )); do
echo "==> Attempt ${attempt}/${MAX_ATTEMPTS} (timeout ${ATTEMPT_TIMEOUT_MIN}m)"
timeout "${ATTEMPT_TIMEOUT_MIN}m" bash -c '
cd detox
chmod +x ./create_android_emulator.sh
CI=true ./create_android_emulator.sh "$SDK_VERSION" "$AVD_NAME" $MATRIX_SPECS
'
rc=$?
if (( rc == 0 )); then
echo "==> Attempt ${attempt} succeeded"
exit 0
fi
echo "==> Attempt ${attempt} failed with exit ${rc}"
if (( attempt < MAX_ATTEMPTS )); then
echo "==> Cleaning up before retry"
adb -s emulator-5554 emu kill 2>/dev/null || true
pkill -9 -f qemu-system 2>/dev/null || true
pkill -9 -f emulator 2>/dev/null || true
rm -rf detox/artifacts/* 2>/dev/null || true
sleep 5
fi
attempt=$(( attempt + 1 ))
done
echo "==> All ${MAX_ATTEMPTS} attempts failed"
exit 1
continue-on-error: true # We want to run all the tests
- name: Upload Android Test Report
if: always() && hashFiles('detox/artifacts/**') != ''
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: android-results-${{ needs.generate-specs.outputs.workflow_hash }}-${{ matrix.runId }}
path: detox/artifacts/
generate-report:
if: always()
runs-on: ubuntu-22.04
needs:
- generate-specs
- e2e-android
outputs:
TARGET_URL: ${{ steps.set-url.outputs.TARGET_URL }}
STATUS: ${{ steps.determine-status.outputs.STATUS }}
FAILURES: ${{ steps.summary.outputs.FAILURES }}
PASSES: ${{ steps.summary.outputs.PASSES }}
TOTAL: ${{ steps.summary.outputs.TOTAL }}
SKIPPED: ${{ steps.summary.outputs.SKIPPED }}
ERRORS: ${{ steps.summary.outputs.ERRORS }}
PERCENTAGE: ${{ steps.summary.outputs.PERCENTAGE }}
steps:
- name: Checkout Repository
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ inputs.MOBILE_VERSION }}
- name: ci/prepare-node-deps
uses: ./.github/actions/prepare-node-deps
- name: Download Android Artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: detox/artifacts/
pattern: android-results-${{ needs.generate-specs.outputs.workflow_hash }}-*
continue-on-error: true
- name: Generate Report Path
id: s3
run: |
path="${{ needs.generate-specs.outputs.build_id }}-${{ needs.generate-specs.outputs.mobile_sha }}-${{ needs.generate-specs.outputs.mobile_ref }}"
echo "path=$(echo "${path}" | sed 's/\./-/g')" >> ${GITHUB_OUTPUT}
- name: Save report Detox Dependencies
id: report-link
run: |
cd detox
npm ci
npm run e2e:save-report
env:
DETOX_AWS_ACCESS_KEY_ID: ${{ secrets.MM_MOBILE_DETOX_AWS_ACCESS_KEY_ID }}
DETOX_AWS_SECRET_ACCESS_KEY: ${{ secrets.MM_MOBILE_DETOX_AWS_SECRET_ACCESS_KEY }}
BUILD_ID: ${{ needs.generate-specs.outputs.build_id }}
REPORT_PATH: ${{ steps.s3.outputs.path }}
## These are needed for the MM Webhook report
COMMIT_HASH: ${{ needs.generate-specs.outputs.mobile_sha }}
BRANCH: ${{ needs.generate-specs.outputs.mobile_ref }}
- name: Calculate failures
id: summary
run: |
if [ -f detox/artifacts/summary.json ]; then
echo "FAILURES=$(cat detox/artifacts/summary.json | jq .stats.failures)" >> ${GITHUB_OUTPUT}
echo "PASSES=$(cat detox/artifacts/summary.json | jq .stats.passes)" >> ${GITHUB_OUTPUT}
echo "SKIPPED=$(cat detox/artifacts/summary.json | jq .stats.skipped)" >> ${GITHUB_OUTPUT}
echo "TOTAL=$(cat detox/artifacts/summary.json | jq .stats.tests)" >> ${GITHUB_OUTPUT}
echo "ERRORS=$(cat detox/artifacts/summary.json | jq .stats.errors)" >> ${GITHUB_OUTPUT}
echo "PERCENTAGE=$(cat detox/artifacts/summary.json | jq .stats.passPercent)" >> ${GITHUB_OUTPUT}
else
echo "⚠️ No summary.json found — all test machines likely failed before producing results"
echo "FAILURES=-1" >> ${GITHUB_OUTPUT}
echo "PASSES=0" >> ${GITHUB_OUTPUT}
echo "SKIPPED=0" >> ${GITHUB_OUTPUT}
echo "TOTAL=0" >> ${GITHUB_OUTPUT}
echo "ERRORS=0" >> ${GITHUB_OUTPUT}
echo "PERCENTAGE=0" >> ${GITHUB_OUTPUT}
fi
- name: Set Target URL
id: set-url
run: |
echo "TARGET_URL=https://${{ env.DETOX_AWS_S3_BUCKET }}.s3.amazonaws.com/${{ steps.s3.outputs.path }}/jest-stare/android-report.html" >> ${GITHUB_OUTPUT}
- name: Determine Status
id: determine-status
env:
FAILURES: ${{ steps.summary.outputs.FAILURES }}
TESTCASE_FAILURE_FATAL: ${{ inputs.testcase_failure_fatal }}
run: |
if [[ "$FAILURES" == "-1" ]]; then
echo "STATUS=failure" >> ${GITHUB_OUTPUT}
elif [[ "$FAILURES" -gt 0 && "$TESTCASE_FAILURE_FATAL" == "true" ]]; then
echo "STATUS=failure" >> ${GITHUB_OUTPUT}
else
echo "STATUS=success" >> ${GITHUB_OUTPUT}
fi
- name: Generate Summary
env:
TOTAL: ${{ steps.summary.outputs.TOTAL }}
PASSES: ${{ steps.summary.outputs.PASSES }}
FAILURES: ${{ steps.summary.outputs.FAILURES }}
SKIPPED: ${{ steps.summary.outputs.SKIPPED }}
ERRORS: ${{ steps.summary.outputs.ERRORS }}
PERCENTAGE: ${{ steps.summary.outputs.PERCENTAGE }}
TARGET_URL: ${{ steps.set-url.outputs.TARGET_URL }}
run: |
echo "| Tests | Passed :white_check_mark: | Failed :x: | Skipped :fast_forward: | Errors :warning: | " >> ${GITHUB_STEP_SUMMARY}
echo "|:---:|:---:|:---:|:---:|:---:|" >> ${GITHUB_STEP_SUMMARY}
echo "| ${TOTAL} | ${PASSES} | ${FAILURES} | ${SKIPPED} | ${ERRORS} |" >> ${GITHUB_STEP_SUMMARY}
echo "" >> ${GITHUB_STEP_SUMMARY}
echo "You can check the full report [here](${TARGET_URL})" >> ${GITHUB_STEP_SUMMARY}
echo "There was **${PERCENTAGE}%** success rate." >> ${GITHUB_STEP_SUMMARY}