mirror of
https://github.com/mattermost/mattermost-mobile.git
synced 2026-11-19 01:48:16 +00:00
Retire legacy maestro per-platform contexts from cancel-e2e-runs and document the canonical PR, master, and CMT status context mapping. Wire Detox + Maestro (PR/main/CMT) into Test System IO with a single commit status (e2e/mobile; CMT uses e2e/compatibility-matrix-testing) and groups mobile-pr / mobile-main / cmt-mobile. Upload shard results with correct framework labels, merge Jest shards into TSIO Detox shape, stub missing/dead shards as failures, and post channel summaries after status settles. Matterwick MASTER is treated as MAIN for mobile-main routing. Also harden report/fail paths, stage Maestro screenshots per flow for TSIO galleries, fix a few Detox setup/testID issues, and skip known platform flakes so the pipeline stays actionable while TSIO lands.
481 lines
19 KiB
JavaScript
481 lines
19 KiB
JavaScript
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
// See LICENSE.txt for license information.
|
|
/* eslint-disable no-console, no-process-env, no-await-in-loop, curly, brace-style, max-statements-per-line, no-empty, no-negated-condition, multiline-ternary -- CI utility script */
|
|
|
|
// Posts one GitHub commit status from a TSIO report group (poll + display URL).
|
|
|
|
const PRODUCTION_URL = 'https://test-io.test.mattermost.com';
|
|
const STAGING_URL = 'https://staging-test-io.test.mattermost.com';
|
|
const TERMINAL_STATUSES = ['completed', 'incomplete'];
|
|
const DEFAULT_POLL_ATTEMPTS = 12;
|
|
const POLL_DELAY_MS = 5000;
|
|
const FETCH_TIMEOUT_MS = 30000;
|
|
|
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
|
async function fetchWithTimeout(url, options = {}, timeoutMs = FETCH_TIMEOUT_MS) {
|
|
const controller = new AbortController();
|
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
try {
|
|
return await fetch(url, {...options, signal: controller.signal});
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
function repoTail(repository) {
|
|
return (repository || '').split('/').pop() || repository;
|
|
}
|
|
|
|
function buildDisplayReportUrl(baseUrl, identity) {
|
|
const repoTrailing = repoTail(identity.repository);
|
|
const rawBranch = identity.branch || 'main';
|
|
const branch = rawBranch.replace(/^refs\/heads\//, '').replace(/^refs\/tags\//, '').replace(/\//g, '~');
|
|
const shortSha = (identity.commit_sha || '').slice(0, 7);
|
|
|
|
// Prefer producer-declared run_group (TSIO consolidates by this), else name.
|
|
const name = identity.run_group || identity.name || 'mobile-pr';
|
|
return `${baseUrl}/reports/${encodeURIComponent(repoTrailing)}/${encodeURIComponent(branch)}/${shortSha}/${encodeURIComponent(name)}`;
|
|
}
|
|
|
|
function buildWorkflowRunUrl(identity) {
|
|
const server = process.env.GITHUB_SERVER_URL || 'https://github.com';
|
|
const repository = process.env.GITHUB_REPOSITORY || identity.repository;
|
|
const runId = process.env.GITHUB_RUN_ID || identity.gh_run_id || '';
|
|
return `${server}/${repository}/actions/runs/${runId}`;
|
|
}
|
|
|
|
// Commit-status target_url policy:
|
|
// - Terminal TSIO group with a usable report → TSIO (success, test failures, or incomplete).
|
|
// - Clean completed report but CI failed outside tests (upstreamSucceeded=false) → workflow run.
|
|
// - Non-terminal / missing report (OIDC/begin/poll callers use runUrl separately) → workflow run.
|
|
function decideTargetUrl(state, bothTerminal, displayReportUrl, reportId, runUrl, opts = {}) {
|
|
const {
|
|
failed = 0,
|
|
reportStatus,
|
|
upstreamSucceeded = true,
|
|
} = opts;
|
|
|
|
if (!bothTerminal || !reportId || !displayReportUrl) {
|
|
return runUrl;
|
|
}
|
|
|
|
const hasTestFailures = (failed || 0) > 0;
|
|
const unrelatedCiFailure =
|
|
state === 'failure' &&
|
|
!hasTestFailures &&
|
|
reportStatus === 'completed' &&
|
|
upstreamSucceeded === false;
|
|
|
|
if (unrelatedCiFailure) {
|
|
return runUrl;
|
|
}
|
|
|
|
return `${displayReportUrl}?gid=${reportId}`;
|
|
}
|
|
|
|
function decideStatus(detail, upstreamSucceeded) {
|
|
const stats = detail.test_stats || {};
|
|
const bothTerminal = TERMINAL_STATUSES.includes(detail.status);
|
|
if (bothTerminal) {
|
|
const clean = (stats.failed || 0) === 0 && detail.status === 'completed' && upstreamSucceeded;
|
|
const passed = stats.passed || 0;
|
|
const failed = stats.failed || 0;
|
|
const skipped = stats.skipped || 0;
|
|
const state = clean ? 'success' : 'failure';
|
|
let description;
|
|
if (failed > 0) {
|
|
description = `${passed} passed, ${failed} failed, ${skipped} skipped`;
|
|
} else if (detail.status !== 'completed') {
|
|
description = `Report ${detail.status} — ${passed} passed, ${skipped} skipped`;
|
|
} else {
|
|
description = `${passed} passed, ${skipped} skipped`;
|
|
}
|
|
return {state, description: description.slice(0, 140), both_terminal: true, timed_out: false};
|
|
}
|
|
return {
|
|
state: upstreamSucceeded ? 'success' : 'failure',
|
|
description: `TSIO poll timed out (status=${detail.status}) — using CI job status`.slice(0, 140),
|
|
both_terminal: false,
|
|
timed_out: true,
|
|
};
|
|
}
|
|
|
|
const {parseArgs} = require('./cli-args');
|
|
|
|
async function mintOidcToken(audience) {
|
|
const url = process.env.ACTIONS_ID_TOKEN_REQUEST_URL;
|
|
const bearer = process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN;
|
|
if (!url || !bearer) {
|
|
throw new Error('OIDC request env vars not set — job needs permissions.id-token: write');
|
|
}
|
|
const sep = url.includes('?') ? '&' : '?';
|
|
const res = await fetchWithTimeout(`${url}${sep}audience=${encodeURIComponent(audience)}`, {
|
|
headers: {Authorization: `bearer ${bearer}`, Accept: 'application/json; api-version=2.0'},
|
|
});
|
|
if (!res.ok) throw new Error(`OIDC mint failed: ${res.status} ${await res.text()}`);
|
|
const body = await res.json();
|
|
if (!body.value) throw new Error('OIDC mint returned no token');
|
|
return body.value;
|
|
}
|
|
|
|
async function beginGroup(baseUrl, idToken, identity, totalReportsExpected) {
|
|
const body = {
|
|
repository: identity.repository,
|
|
commit: identity.commit_sha,
|
|
gh_run_id: identity.gh_run_id,
|
|
gh_run_attempt: String(identity.gh_run_attempt),
|
|
framework: identity.framework || 'detox',
|
|
name: identity.name,
|
|
branch: identity.branch,
|
|
total_reports_expected: totalReportsExpected,
|
|
...(identity.run_group ? {run_group: identity.run_group} : {}),
|
|
...(identity.gh_pr_number != null ? {gh_pr_number: parseInt(identity.gh_pr_number, 10)} : {}),
|
|
};
|
|
const res = await fetchWithTimeout(`${baseUrl}/api/v1/reports/begin`, {
|
|
method: 'POST',
|
|
headers: {'Content-Type': 'application/json', Authorization: `Bearer ${idToken}`},
|
|
body: JSON.stringify(body),
|
|
});
|
|
if (!res.ok) throw new Error(`reports/begin (${identity.name}) failed: ${res.status} ${await res.text()}`);
|
|
const {report_id} = await res.json();
|
|
if (!report_id) throw new Error(`reports/begin (${identity.name}) returned no report_id`);
|
|
return report_id;
|
|
}
|
|
|
|
async function pollGroup(baseUrl, reportId, pollAttempts) {
|
|
const attempts = Number.isFinite(pollAttempts) && pollAttempts >= 1 ?
|
|
pollAttempts :
|
|
DEFAULT_POLL_ATTEMPTS;
|
|
let detail = {status: 'unknown', test_stats: {}};
|
|
for (let attempt = 0; attempt < attempts; attempt++) {
|
|
try {
|
|
const res = await fetchWithTimeout(`${baseUrl}/api/v1/reports/${reportId}`);
|
|
if (!res.ok) throw new Error(`reports/${reportId} failed: ${res.status} ${await res.text()}`);
|
|
detail = await res.json();
|
|
if (TERMINAL_STATUSES.includes(detail.status)) return detail;
|
|
} catch (err) {
|
|
console.error(`tsio-report-status (poll attempt ${attempt + 1}/${attempts}):`, err.message);
|
|
if (attempt >= attempts - 1) throw err;
|
|
}
|
|
if (attempt < attempts - 1) await sleep(POLL_DELAY_MS);
|
|
}
|
|
return detail;
|
|
}
|
|
|
|
async function createCommitStatus(token, repository, sha, payload) {
|
|
const [owner, repo] = repository.split('/');
|
|
const res = await fetchWithTimeout(`https://api.github.com/repos/${owner}/${repo}/statuses/${sha}`, {
|
|
method: 'POST',
|
|
headers: {
|
|
Authorization: `Bearer ${token}`,
|
|
Accept: 'application/vnd.github+json',
|
|
'X-GitHub-Api-Version': '2022-11-28',
|
|
'Content-Type': 'application/json',
|
|
},
|
|
body: JSON.stringify(payload),
|
|
});
|
|
if (!res.ok) throw new Error(`createCommitStatus failed: ${res.status} ${await res.text()}`);
|
|
}
|
|
|
|
async function reportTsioStatus(options) {
|
|
const {
|
|
compositeIdentity,
|
|
totalReportsExpected,
|
|
commitStatusContext,
|
|
upstreamJobsSucceeded = true,
|
|
githubToken,
|
|
failOnTestFailures = true,
|
|
pollAttempts = DEFAULT_POLL_ATTEMPTS,
|
|
useStaging = false,
|
|
audience = 'mattermost-test-system-io',
|
|
baseUrl: baseUrlOverride,
|
|
} = options;
|
|
|
|
const baseUrl = baseUrlOverride || (useStaging ? STAGING_URL : PRODUCTION_URL);
|
|
const runUrl = buildWorkflowRunUrl(compositeIdentity);
|
|
|
|
const result = {
|
|
context: commitStatusContext,
|
|
tsio_url: baseUrl,
|
|
upstream_succeeded: upstreamJobsSucceeded,
|
|
report_id: null,
|
|
group_url: null,
|
|
display_report_url: buildDisplayReportUrl(baseUrl, compositeIdentity),
|
|
status: null,
|
|
test_stats: {},
|
|
state: 'failure',
|
|
timed_out: false,
|
|
};
|
|
|
|
const token = githubToken || process.env.GITHUB_TOKEN;
|
|
if (!token) throw new Error('githubToken (or GITHUB_TOKEN env) is required');
|
|
|
|
let idToken;
|
|
try {
|
|
idToken = await mintOidcToken(audience);
|
|
} catch (err) {
|
|
console.error('tsio-report-status (OIDC):', err.message);
|
|
await createCommitStatus(token, compositeIdentity.repository, compositeIdentity.commit_sha, {
|
|
state: 'failure',
|
|
context: commitStatusContext,
|
|
description: `TSIO error: ${err.message}`.slice(0, 140),
|
|
target_url: runUrl,
|
|
});
|
|
result.error = err.message;
|
|
return result;
|
|
}
|
|
|
|
let reportId;
|
|
try {
|
|
reportId = await beginGroup(baseUrl, idToken, compositeIdentity, totalReportsExpected);
|
|
} catch (err) {
|
|
console.error('tsio-report-status (begin):', err.message);
|
|
await createCommitStatus(token, compositeIdentity.repository, compositeIdentity.commit_sha, {
|
|
state: 'failure',
|
|
context: commitStatusContext,
|
|
description: `TSIO begin failed: ${err.message}`.slice(0, 140),
|
|
target_url: runUrl,
|
|
});
|
|
result.error = err.message;
|
|
return result;
|
|
}
|
|
|
|
result.report_id = reportId;
|
|
result.group_url = `${baseUrl}/reports/g/${reportId}`;
|
|
|
|
let detail;
|
|
try {
|
|
detail = await pollGroup(baseUrl, reportId, pollAttempts);
|
|
} catch (err) {
|
|
console.error('tsio-report-status (poll):', err.message);
|
|
await createCommitStatus(token, compositeIdentity.repository, compositeIdentity.commit_sha, {
|
|
state: 'failure',
|
|
context: commitStatusContext,
|
|
description: `TSIO poll failed: ${err.message}`.slice(0, 140),
|
|
target_url: runUrl,
|
|
});
|
|
result.error = err.message;
|
|
return result;
|
|
}
|
|
|
|
result.status = detail.status;
|
|
result.test_stats = detail.test_stats || {};
|
|
|
|
const {state, description, both_terminal: bothTerminal, timed_out: timedOut} =
|
|
decideStatus(detail, upstreamJobsSucceeded);
|
|
result.timed_out = timedOut;
|
|
result.state = state;
|
|
|
|
const targetUrl = decideTargetUrl(state, bothTerminal, result.display_report_url, reportId, runUrl, {
|
|
failed: result.test_stats.failed || 0,
|
|
reportStatus: detail.status,
|
|
upstreamSucceeded: upstreamJobsSucceeded,
|
|
});
|
|
|
|
await createCommitStatus(token, compositeIdentity.repository, compositeIdentity.commit_sha, {
|
|
state,
|
|
context: commitStatusContext,
|
|
description,
|
|
target_url: targetUrl,
|
|
});
|
|
|
|
if (process.env.GITHUB_STEP_SUMMARY) {
|
|
const stats = result.test_stats;
|
|
const lines = [
|
|
`### TSIO status — ${commitStatusContext}`,
|
|
'',
|
|
`**State:** ${state}`,
|
|
`**Display report:** [${compositeIdentity.name}](${result.display_report_url})`,
|
|
`**Group:** [${reportId}](${result.group_url}) — ${result.status}`,
|
|
`**Stats:** ${stats.passed || 0} passed, ${stats.failed || 0} failed, ${stats.skipped || 0} skipped (of ${stats.total || 0})`,
|
|
...(timedOut ? ['', ':warning: TSIO group did not reach terminal status in time — failing open to CI job status.'] : []),
|
|
];
|
|
try {
|
|
const fs = require('fs');
|
|
fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join('\n') + '\n');
|
|
} catch {}
|
|
}
|
|
|
|
// Channel notify (best-effort). Routing (see resolveWebhookUrl):
|
|
// mobile-release → MM_E2E_RELEASE_WEBHOOK_URL
|
|
// mobile-main → MM_E2E_MASTER_HEALTH_WEBHOOK_URL
|
|
// mobile-pr → MM_MOBILE_E2E_WEBHOOK_URL
|
|
// Failures here must not undo a successfully written commit status.
|
|
try {
|
|
const notifyNames = new Set(['mobile-release', 'mobile-pr', 'mobile-main']);
|
|
const reportName = compositeIdentity.run_group || compositeIdentity.name;
|
|
if (notifyNames.has(reportName) || notifyNames.has(compositeIdentity.name)) {
|
|
const {notifyCmtChannel, resolveWebhookUrl} = require('./cmt-channel-notify.js');
|
|
const webhookUrl = resolveWebhookUrl(compositeIdentity.name);
|
|
if (webhookUrl) {
|
|
const failedShards = (detail.reports || []).
|
|
filter((r) => r.status && r.status !== 'complete' && r.status !== 'completed').
|
|
map((r) => r.gh_job_name || r.display_name || r.id).
|
|
filter(Boolean);
|
|
const hasFailures = (result.test_stats.failed || 0) > 0 || failedShards.length > 0;
|
|
const channelReportUrl = result.display_report_url || result.group_url || targetUrl;
|
|
const core = {
|
|
info: (msg) => console.log(msg),
|
|
warning: (msg) => console.warn(msg),
|
|
};
|
|
await notifyCmtChannel({
|
|
core,
|
|
baseUrl,
|
|
compositeIdentity,
|
|
detail,
|
|
reportUrl: channelReportUrl,
|
|
upstreamJobsSucceeded,
|
|
hasFailures,
|
|
webhookUrl,
|
|
});
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.warn(`E2E Mattermost notify setup failed: ${error.message}`);
|
|
}
|
|
|
|
if (failOnTestFailures && bothTerminal && (result.test_stats.failed || 0) > 0) {
|
|
const err = new Error('TSIO reported test failures');
|
|
err.exitCode = 1;
|
|
throw err;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
function selfTest() {
|
|
const assert = (cond, msg) => {
|
|
if (!cond) { console.error('SELF-TEST FAIL:', msg); process.exit(1); }
|
|
};
|
|
|
|
const url = buildDisplayReportUrl(PRODUCTION_URL, {
|
|
repository: 'mattermost/mattermost-mobile',
|
|
commit_sha: 'abc1234deadbeef',
|
|
branch: 'feature-branch',
|
|
name: 'mobile-pr',
|
|
run_group: 'mobile-pr',
|
|
});
|
|
assert(url === 'https://test-io.test.mattermost.com/reports/mattermost-mobile/feature-branch/abc1234/mobile-pr', `display URL: ${url}`);
|
|
|
|
const urlSlashBranch = buildDisplayReportUrl(PRODUCTION_URL, {
|
|
repository: 'mattermost/mattermost-mobile',
|
|
commit_sha: 'abc1234deadbeef',
|
|
branch: 'feat/tsio-mobile-reporting',
|
|
name: 'mobile-pr',
|
|
run_group: 'mobile-pr',
|
|
});
|
|
assert(urlSlashBranch === 'https://test-io.test.mattermost.com/reports/mattermost-mobile/feat~tsio-mobile-reporting/abc1234/mobile-pr', `slash branch URL: ${urlSlashBranch}`);
|
|
|
|
const urlMain = buildDisplayReportUrl(PRODUCTION_URL, {
|
|
repository: 'mattermost/mattermost-mobile',
|
|
commit_sha: 'def5678',
|
|
branch: 'refs/heads/main',
|
|
name: 'mobile-main',
|
|
run_group: 'mobile-main',
|
|
});
|
|
assert(urlMain === 'https://test-io.test.mattermost.com/reports/mattermost-mobile/main/def5678/mobile-main', `main URL: ${urlMain}`);
|
|
|
|
const cmtUrl = buildDisplayReportUrl(PRODUCTION_URL, {
|
|
repository: 'mattermost/mattermost-mobile',
|
|
commit_sha: '111aaaa',
|
|
branch: 'release-2.40',
|
|
name: 'mobile-release',
|
|
run_group: 'mobile-release',
|
|
});
|
|
assert(cmtUrl === 'https://test-io.test.mattermost.com/reports/mattermost-mobile/release-2.40/111aaaa/mobile-release', `cmt URL: ${cmtUrl}`);
|
|
|
|
let r = decideStatus({status: 'completed', test_stats: {passed: 10, failed: 0, skipped: 1}}, true);
|
|
assert(r.state === 'success', 'clean run -> success');
|
|
|
|
r = decideStatus({status: 'completed', test_stats: {passed: 8, failed: 2, skipped: 0}}, true);
|
|
assert(r.state === 'failure', 'failures -> failure');
|
|
|
|
r = decideStatus({status: 'in_progress', test_stats: {passed: 3, failed: 0}}, true);
|
|
assert(r.timed_out === true && r.state === 'success', 'timeout fail-open upstream ok');
|
|
|
|
const display = 'https://test-io.test.mattermost.com/reports/mattermost-mobile/main/abc1234/mobile-pr';
|
|
const actions = 'https://github.com/o/r/actions/runs/99';
|
|
assert(
|
|
decideTargetUrl('success', true, display, 'gid-1', actions, {failed: 0, reportStatus: 'completed'}) ===
|
|
`${display}?gid=gid-1`,
|
|
'success -> TSIO report URL',
|
|
);
|
|
assert(
|
|
decideTargetUrl('failure', true, display, 'gid-1', actions, {failed: 2, reportStatus: 'completed'}) ===
|
|
`${display}?gid=gid-1`,
|
|
'test failures -> TSIO report URL',
|
|
);
|
|
assert(
|
|
decideTargetUrl('failure', true, display, 'gid-1', actions, {
|
|
failed: 0,
|
|
reportStatus: 'completed',
|
|
upstreamSucceeded: false,
|
|
}) === actions,
|
|
'unrelated CI failure with clean report -> workflow run URL',
|
|
);
|
|
assert(
|
|
decideTargetUrl('failure', true, display, 'gid-1', actions, {failed: 0, reportStatus: 'incomplete'}) ===
|
|
`${display}?gid=gid-1`,
|
|
'incomplete report -> TSIO report URL',
|
|
);
|
|
assert(
|
|
decideTargetUrl('success', false, display, 'gid-1', actions) === actions,
|
|
'non-terminal -> workflow run URL',
|
|
);
|
|
|
|
console.log('SELF-TEST OK');
|
|
}
|
|
|
|
async function main() {
|
|
const args = parseArgs(process.argv);
|
|
if (args['self-test'] !== undefined) { selfTest(); return; }
|
|
|
|
let identity;
|
|
try {
|
|
identity = JSON.parse(args['composite-identity']);
|
|
} catch (err) {
|
|
console.error('tsio-report-status: invalid --composite-identity JSON:', err.message);
|
|
process.exit(1);
|
|
}
|
|
|
|
const totalReportsExpected = parseInt(args['total-reports-expected'] || '1', 10);
|
|
const context = args.context || 'e2e/mobile';
|
|
const upstreamSucceeded = args['upstream-succeeded'] !== 'false';
|
|
const failOnTestFailures = args['fail-on-test-failures'] !== 'false';
|
|
const pollAttempts = parseInt(args['poll-attempts'] || String(DEFAULT_POLL_ATTEMPTS), 10);
|
|
|
|
try {
|
|
const result = await reportTsioStatus({
|
|
compositeIdentity: identity,
|
|
totalReportsExpected,
|
|
commitStatusContext: context,
|
|
upstreamJobsSucceeded: upstreamSucceeded,
|
|
githubToken: args['github-token'] || process.env.GITHUB_TOKEN,
|
|
failOnTestFailures,
|
|
pollAttempts,
|
|
useStaging: args['use-staging'] === 'true',
|
|
audience: args.audience || 'mattermost-test-system-io',
|
|
baseUrl: args['base-url'],
|
|
});
|
|
console.log(JSON.stringify(result, null, 2));
|
|
process.exit(0);
|
|
} catch (err) {
|
|
if (err.exitCode === 1) {
|
|
console.error('tsio-report-status:', err.message);
|
|
process.exit(1);
|
|
}
|
|
console.error('tsio-report-status fatal:', err.message);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
module.exports = {buildDisplayReportUrl, buildWorkflowRunUrl, decideStatus, decideTargetUrl, reportTsioStatus, repoTail};
|
|
|
|
if (require.main === module) {
|
|
main().catch((err) => {
|
|
console.error('tsio-report-status fatal:', err.message);
|
|
process.exit(1);
|
|
});
|
|
}
|