mattermost-mobile-upstream/detox/utils/tsio-report-status.js
yasser khan bcac248ec0
feat(e2e): Integrate TSIO reporting (#9925)
Retire legacy maestro per-platform contexts from cancel-e2e-runs and
document the canonical PR, master, and CMT status context mapping.
Wire Detox + Maestro (PR/main/CMT) into Test System IO with a single
commit status (e2e/mobile; CMT uses e2e/compatibility-matrix-testing)
and groups mobile-pr / mobile-main / cmt-mobile.

Upload shard results with correct framework labels, merge Jest shards
into TSIO Detox shape, stub missing/dead shards as failures, and post
channel summaries after status settles. Matterwick MASTER is treated as
MAIN for mobile-main routing.

Also harden report/fail paths, stage Maestro screenshots per flow for
TSIO galleries, fix a few Detox setup/testID issues, and skip known
platform flakes so the pipeline stays actionable while TSIO lands.
2026-07-23 06:43:28 +00:00

481 lines
19 KiB
JavaScript

// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
/* eslint-disable no-console, no-process-env, no-await-in-loop, curly, brace-style, max-statements-per-line, no-empty, no-negated-condition, multiline-ternary -- CI utility script */
// Posts one GitHub commit status from a TSIO report group (poll + display URL).
const PRODUCTION_URL = 'https://test-io.test.mattermost.com';
const STAGING_URL = 'https://staging-test-io.test.mattermost.com';
const TERMINAL_STATUSES = ['completed', 'incomplete'];
const DEFAULT_POLL_ATTEMPTS = 12;
const POLL_DELAY_MS = 5000;
const FETCH_TIMEOUT_MS = 30000;
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
async function fetchWithTimeout(url, options = {}, timeoutMs = FETCH_TIMEOUT_MS) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
return await fetch(url, {...options, signal: controller.signal});
} finally {
clearTimeout(timer);
}
}
function repoTail(repository) {
return (repository || '').split('/').pop() || repository;
}
function buildDisplayReportUrl(baseUrl, identity) {
const repoTrailing = repoTail(identity.repository);
const rawBranch = identity.branch || 'main';
const branch = rawBranch.replace(/^refs\/heads\//, '').replace(/^refs\/tags\//, '').replace(/\//g, '~');
const shortSha = (identity.commit_sha || '').slice(0, 7);
// Prefer producer-declared run_group (TSIO consolidates by this), else name.
const name = identity.run_group || identity.name || 'mobile-pr';
return `${baseUrl}/reports/${encodeURIComponent(repoTrailing)}/${encodeURIComponent(branch)}/${shortSha}/${encodeURIComponent(name)}`;
}
function buildWorkflowRunUrl(identity) {
const server = process.env.GITHUB_SERVER_URL || 'https://github.com';
const repository = process.env.GITHUB_REPOSITORY || identity.repository;
const runId = process.env.GITHUB_RUN_ID || identity.gh_run_id || '';
return `${server}/${repository}/actions/runs/${runId}`;
}
// Commit-status target_url policy:
// - Terminal TSIO group with a usable report → TSIO (success, test failures, or incomplete).
// - Clean completed report but CI failed outside tests (upstreamSucceeded=false) → workflow run.
// - Non-terminal / missing report (OIDC/begin/poll callers use runUrl separately) → workflow run.
function decideTargetUrl(state, bothTerminal, displayReportUrl, reportId, runUrl, opts = {}) {
const {
failed = 0,
reportStatus,
upstreamSucceeded = true,
} = opts;
if (!bothTerminal || !reportId || !displayReportUrl) {
return runUrl;
}
const hasTestFailures = (failed || 0) > 0;
const unrelatedCiFailure =
state === 'failure' &&
!hasTestFailures &&
reportStatus === 'completed' &&
upstreamSucceeded === false;
if (unrelatedCiFailure) {
return runUrl;
}
return `${displayReportUrl}?gid=${reportId}`;
}
function decideStatus(detail, upstreamSucceeded) {
const stats = detail.test_stats || {};
const bothTerminal = TERMINAL_STATUSES.includes(detail.status);
if (bothTerminal) {
const clean = (stats.failed || 0) === 0 && detail.status === 'completed' && upstreamSucceeded;
const passed = stats.passed || 0;
const failed = stats.failed || 0;
const skipped = stats.skipped || 0;
const state = clean ? 'success' : 'failure';
let description;
if (failed > 0) {
description = `${passed} passed, ${failed} failed, ${skipped} skipped`;
} else if (detail.status !== 'completed') {
description = `Report ${detail.status} — ${passed} passed, ${skipped} skipped`;
} else {
description = `${passed} passed, ${skipped} skipped`;
}
return {state, description: description.slice(0, 140), both_terminal: true, timed_out: false};
}
return {
state: upstreamSucceeded ? 'success' : 'failure',
description: `TSIO poll timed out (status=${detail.status}) — using CI job status`.slice(0, 140),
both_terminal: false,
timed_out: true,
};
}
const {parseArgs} = require('./cli-args');
async function mintOidcToken(audience) {
const url = process.env.ACTIONS_ID_TOKEN_REQUEST_URL;
const bearer = process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN;
if (!url || !bearer) {
throw new Error('OIDC request env vars not set — job needs permissions.id-token: write');
}
const sep = url.includes('?') ? '&' : '?';
const res = await fetchWithTimeout(`${url}${sep}audience=${encodeURIComponent(audience)}`, {
headers: {Authorization: `bearer ${bearer}`, Accept: 'application/json; api-version=2.0'},
});
if (!res.ok) throw new Error(`OIDC mint failed: ${res.status} ${await res.text()}`);
const body = await res.json();
if (!body.value) throw new Error('OIDC mint returned no token');
return body.value;
}
async function beginGroup(baseUrl, idToken, identity, totalReportsExpected) {
const body = {
repository: identity.repository,
commit: identity.commit_sha,
gh_run_id: identity.gh_run_id,
gh_run_attempt: String(identity.gh_run_attempt),
framework: identity.framework || 'detox',
name: identity.name,
branch: identity.branch,
total_reports_expected: totalReportsExpected,
...(identity.run_group ? {run_group: identity.run_group} : {}),
...(identity.gh_pr_number != null ? {gh_pr_number: parseInt(identity.gh_pr_number, 10)} : {}),
};
const res = await fetchWithTimeout(`${baseUrl}/api/v1/reports/begin`, {
method: 'POST',
headers: {'Content-Type': 'application/json', Authorization: `Bearer ${idToken}`},
body: JSON.stringify(body),
});
if (!res.ok) throw new Error(`reports/begin (${identity.name}) failed: ${res.status} ${await res.text()}`);
const {report_id} = await res.json();
if (!report_id) throw new Error(`reports/begin (${identity.name}) returned no report_id`);
return report_id;
}
async function pollGroup(baseUrl, reportId, pollAttempts) {
const attempts = Number.isFinite(pollAttempts) && pollAttempts >= 1 ?
pollAttempts :
DEFAULT_POLL_ATTEMPTS;
let detail = {status: 'unknown', test_stats: {}};
for (let attempt = 0; attempt < attempts; attempt++) {
try {
const res = await fetchWithTimeout(`${baseUrl}/api/v1/reports/${reportId}`);
if (!res.ok) throw new Error(`reports/${reportId} failed: ${res.status} ${await res.text()}`);
detail = await res.json();
if (TERMINAL_STATUSES.includes(detail.status)) return detail;
} catch (err) {
console.error(`tsio-report-status (poll attempt ${attempt + 1}/${attempts}):`, err.message);
if (attempt >= attempts - 1) throw err;
}
if (attempt < attempts - 1) await sleep(POLL_DELAY_MS);
}
return detail;
}
async function createCommitStatus(token, repository, sha, payload) {
const [owner, repo] = repository.split('/');
const res = await fetchWithTimeout(`https://api.github.com/repos/${owner}/${repo}/statuses/${sha}`, {
method: 'POST',
headers: {
Authorization: `Bearer ${token}`,
Accept: 'application/vnd.github+json',
'X-GitHub-Api-Version': '2022-11-28',
'Content-Type': 'application/json',
},
body: JSON.stringify(payload),
});
if (!res.ok) throw new Error(`createCommitStatus failed: ${res.status} ${await res.text()}`);
}
async function reportTsioStatus(options) {
const {
compositeIdentity,
totalReportsExpected,
commitStatusContext,
upstreamJobsSucceeded = true,
githubToken,
failOnTestFailures = true,
pollAttempts = DEFAULT_POLL_ATTEMPTS,
useStaging = false,
audience = 'mattermost-test-system-io',
baseUrl: baseUrlOverride,
} = options;
const baseUrl = baseUrlOverride || (useStaging ? STAGING_URL : PRODUCTION_URL);
const runUrl = buildWorkflowRunUrl(compositeIdentity);
const result = {
context: commitStatusContext,
tsio_url: baseUrl,
upstream_succeeded: upstreamJobsSucceeded,
report_id: null,
group_url: null,
display_report_url: buildDisplayReportUrl(baseUrl, compositeIdentity),
status: null,
test_stats: {},
state: 'failure',
timed_out: false,
};
const token = githubToken || process.env.GITHUB_TOKEN;
if (!token) throw new Error('githubToken (or GITHUB_TOKEN env) is required');
let idToken;
try {
idToken = await mintOidcToken(audience);
} catch (err) {
console.error('tsio-report-status (OIDC):', err.message);
await createCommitStatus(token, compositeIdentity.repository, compositeIdentity.commit_sha, {
state: 'failure',
context: commitStatusContext,
description: `TSIO error: ${err.message}`.slice(0, 140),
target_url: runUrl,
});
result.error = err.message;
return result;
}
let reportId;
try {
reportId = await beginGroup(baseUrl, idToken, compositeIdentity, totalReportsExpected);
} catch (err) {
console.error('tsio-report-status (begin):', err.message);
await createCommitStatus(token, compositeIdentity.repository, compositeIdentity.commit_sha, {
state: 'failure',
context: commitStatusContext,
description: `TSIO begin failed: ${err.message}`.slice(0, 140),
target_url: runUrl,
});
result.error = err.message;
return result;
}
result.report_id = reportId;
result.group_url = `${baseUrl}/reports/g/${reportId}`;
let detail;
try {
detail = await pollGroup(baseUrl, reportId, pollAttempts);
} catch (err) {
console.error('tsio-report-status (poll):', err.message);
await createCommitStatus(token, compositeIdentity.repository, compositeIdentity.commit_sha, {
state: 'failure',
context: commitStatusContext,
description: `TSIO poll failed: ${err.message}`.slice(0, 140),
target_url: runUrl,
});
result.error = err.message;
return result;
}
result.status = detail.status;
result.test_stats = detail.test_stats || {};
const {state, description, both_terminal: bothTerminal, timed_out: timedOut} =
decideStatus(detail, upstreamJobsSucceeded);
result.timed_out = timedOut;
result.state = state;
const targetUrl = decideTargetUrl(state, bothTerminal, result.display_report_url, reportId, runUrl, {
failed: result.test_stats.failed || 0,
reportStatus: detail.status,
upstreamSucceeded: upstreamJobsSucceeded,
});
await createCommitStatus(token, compositeIdentity.repository, compositeIdentity.commit_sha, {
state,
context: commitStatusContext,
description,
target_url: targetUrl,
});
if (process.env.GITHUB_STEP_SUMMARY) {
const stats = result.test_stats;
const lines = [
`### TSIO status — ${commitStatusContext}`,
'',
`**State:** ${state}`,
`**Display report:** [${compositeIdentity.name}](${result.display_report_url})`,
`**Group:** [${reportId}](${result.group_url}) — ${result.status}`,
`**Stats:** ${stats.passed || 0} passed, ${stats.failed || 0} failed, ${stats.skipped || 0} skipped (of ${stats.total || 0})`,
...(timedOut ? ['', ':warning: TSIO group did not reach terminal status in time — failing open to CI job status.'] : []),
];
try {
const fs = require('fs');
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join('\n') + '\n');
} catch {}
}
// Channel notify (best-effort). Routing (see resolveWebhookUrl):
// mobile-release → MM_E2E_RELEASE_WEBHOOK_URL
// mobile-main → MM_E2E_MASTER_HEALTH_WEBHOOK_URL
// mobile-pr → MM_MOBILE_E2E_WEBHOOK_URL
// Failures here must not undo a successfully written commit status.
try {
const notifyNames = new Set(['mobile-release', 'mobile-pr', 'mobile-main']);
const reportName = compositeIdentity.run_group || compositeIdentity.name;
if (notifyNames.has(reportName) || notifyNames.has(compositeIdentity.name)) {
const {notifyCmtChannel, resolveWebhookUrl} = require('./cmt-channel-notify.js');
const webhookUrl = resolveWebhookUrl(compositeIdentity.name);
if (webhookUrl) {
const failedShards = (detail.reports || []).
filter((r) => r.status && r.status !== 'complete' && r.status !== 'completed').
map((r) => r.gh_job_name || r.display_name || r.id).
filter(Boolean);
const hasFailures = (result.test_stats.failed || 0) > 0 || failedShards.length > 0;
const channelReportUrl = result.display_report_url || result.group_url || targetUrl;
const core = {
info: (msg) => console.log(msg),
warning: (msg) => console.warn(msg),
};
await notifyCmtChannel({
core,
baseUrl,
compositeIdentity,
detail,
reportUrl: channelReportUrl,
upstreamJobsSucceeded,
hasFailures,
webhookUrl,
});
}
}
} catch (error) {
console.warn(`E2E Mattermost notify setup failed: ${error.message}`);
}
if (failOnTestFailures && bothTerminal && (result.test_stats.failed || 0) > 0) {
const err = new Error('TSIO reported test failures');
err.exitCode = 1;
throw err;
}
return result;
}
function selfTest() {
const assert = (cond, msg) => {
if (!cond) { console.error('SELF-TEST FAIL:', msg); process.exit(1); }
};
const url = buildDisplayReportUrl(PRODUCTION_URL, {
repository: 'mattermost/mattermost-mobile',
commit_sha: 'abc1234deadbeef',
branch: 'feature-branch',
name: 'mobile-pr',
run_group: 'mobile-pr',
});
assert(url === 'https://test-io.test.mattermost.com/reports/mattermost-mobile/feature-branch/abc1234/mobile-pr', `display URL: ${url}`);
const urlSlashBranch = buildDisplayReportUrl(PRODUCTION_URL, {
repository: 'mattermost/mattermost-mobile',
commit_sha: 'abc1234deadbeef',
branch: 'feat/tsio-mobile-reporting',
name: 'mobile-pr',
run_group: 'mobile-pr',
});
assert(urlSlashBranch === 'https://test-io.test.mattermost.com/reports/mattermost-mobile/feat~tsio-mobile-reporting/abc1234/mobile-pr', `slash branch URL: ${urlSlashBranch}`);
const urlMain = buildDisplayReportUrl(PRODUCTION_URL, {
repository: 'mattermost/mattermost-mobile',
commit_sha: 'def5678',
branch: 'refs/heads/main',
name: 'mobile-main',
run_group: 'mobile-main',
});
assert(urlMain === 'https://test-io.test.mattermost.com/reports/mattermost-mobile/main/def5678/mobile-main', `main URL: ${urlMain}`);
const cmtUrl = buildDisplayReportUrl(PRODUCTION_URL, {
repository: 'mattermost/mattermost-mobile',
commit_sha: '111aaaa',
branch: 'release-2.40',
name: 'mobile-release',
run_group: 'mobile-release',
});
assert(cmtUrl === 'https://test-io.test.mattermost.com/reports/mattermost-mobile/release-2.40/111aaaa/mobile-release', `cmt URL: ${cmtUrl}`);
let r = decideStatus({status: 'completed', test_stats: {passed: 10, failed: 0, skipped: 1}}, true);
assert(r.state === 'success', 'clean run -> success');
r = decideStatus({status: 'completed', test_stats: {passed: 8, failed: 2, skipped: 0}}, true);
assert(r.state === 'failure', 'failures -> failure');
r = decideStatus({status: 'in_progress', test_stats: {passed: 3, failed: 0}}, true);
assert(r.timed_out === true && r.state === 'success', 'timeout fail-open upstream ok');
const display = 'https://test-io.test.mattermost.com/reports/mattermost-mobile/main/abc1234/mobile-pr';
const actions = 'https://github.com/o/r/actions/runs/99';
assert(
decideTargetUrl('success', true, display, 'gid-1', actions, {failed: 0, reportStatus: 'completed'}) ===
`${display}?gid=gid-1`,
'success -> TSIO report URL',
);
assert(
decideTargetUrl('failure', true, display, 'gid-1', actions, {failed: 2, reportStatus: 'completed'}) ===
`${display}?gid=gid-1`,
'test failures -> TSIO report URL',
);
assert(
decideTargetUrl('failure', true, display, 'gid-1', actions, {
failed: 0,
reportStatus: 'completed',
upstreamSucceeded: false,
}) === actions,
'unrelated CI failure with clean report -> workflow run URL',
);
assert(
decideTargetUrl('failure', true, display, 'gid-1', actions, {failed: 0, reportStatus: 'incomplete'}) ===
`${display}?gid=gid-1`,
'incomplete report -> TSIO report URL',
);
assert(
decideTargetUrl('success', false, display, 'gid-1', actions) === actions,
'non-terminal -> workflow run URL',
);
console.log('SELF-TEST OK');
}
async function main() {
const args = parseArgs(process.argv);
if (args['self-test'] !== undefined) { selfTest(); return; }
let identity;
try {
identity = JSON.parse(args['composite-identity']);
} catch (err) {
console.error('tsio-report-status: invalid --composite-identity JSON:', err.message);
process.exit(1);
}
const totalReportsExpected = parseInt(args['total-reports-expected'] || '1', 10);
const context = args.context || 'e2e/mobile';
const upstreamSucceeded = args['upstream-succeeded'] !== 'false';
const failOnTestFailures = args['fail-on-test-failures'] !== 'false';
const pollAttempts = parseInt(args['poll-attempts'] || String(DEFAULT_POLL_ATTEMPTS), 10);
try {
const result = await reportTsioStatus({
compositeIdentity: identity,
totalReportsExpected,
commitStatusContext: context,
upstreamJobsSucceeded: upstreamSucceeded,
githubToken: args['github-token'] || process.env.GITHUB_TOKEN,
failOnTestFailures,
pollAttempts,
useStaging: args['use-staging'] === 'true',
audience: args.audience || 'mattermost-test-system-io',
baseUrl: args['base-url'],
});
console.log(JSON.stringify(result, null, 2));
process.exit(0);
} catch (err) {
if (err.exitCode === 1) {
console.error('tsio-report-status:', err.message);
process.exit(1);
}
console.error('tsio-report-status fatal:', err.message);
process.exit(1);
}
}
module.exports = {buildDisplayReportUrl, buildWorkflowRunUrl, decideStatus, decideTargetUrl, reportTsioStatus, repoTail};
if (require.main === module) {
main().catch((err) => {
console.error('tsio-report-status fatal:', err.message);
process.exit(1);
});
}