Tests: Check frontend test coverage CI workflow (#116633)

* chore(git): ignore coverage summary from CI workflow

* feat(script): check if files changed for a codeowner

* feat(ci): clean up PR comments when break glass label applied

* feat(ci): add change detection to coverage workflow

* feat(ci): conditionally run coverage checks only for teams changed files

* feat(ci): compare code coverage reports by codeowner

* feat(ci): improve PR comment message

* feat(ci): add skip warning with opted-in teams list

* fix(scripts): avoid rounding errors in test coverage summaries

* fix(script): fix GHA linting errors

* fix(script): multiple space delimited filenames in change detection

* fix(scripts): round to two decimal points of precision

* feat(script): collector job to fan-in coverage comparison matrix

* fix(script): display correct git SHA

* fix(script): serial execution in each test suite for deterministism

* fix(script): use base branch SHA, not ref for main

* fix(script): ignore CI scripts written in Node.js and Jest config file

* fix(script): post failure message when coverage drops

* fix(script): use correct SHAs in PR comment message

* fix(script): fail when any one of the coverage comparisons fail

* fix(script): use the same PR comment bot for all messages

* fix(script): use the same token so comments are cleared when re-run etc.

* feat(script): make PR message more concise
This commit is contained in:
Jesse David Peterson 2026-01-22 08:46:04 -05:00 committed by GitHub
parent ba1743ef20
commit 4623a00050
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 463 additions and 35 deletions

1
.github/CODEOWNERS vendored
View file

@ -1056,6 +1056,7 @@ playwright.storybook.config.ts @grafana/grafana-frontend-platform
/scripts/clean-git-or-error.sh @grafana/grafana-as-code
/scripts/grafana-server/ @grafana/grafana-frontend-platform
/scripts/check-frontend-dev.sh @grafana/grafana-frontend-platform
/scripts/check-codeowner-affected.js @grafana/dataviz-squad
/scripts/compare-coverage-by-codeowner.js @grafana/dataviz-squad
/scripts/helpers/ @grafana/grafana-developer-enablement-squad
/scripts/import_many_dashboards.sh @torkelo

View file

@ -1,8 +1,8 @@
name: Check Fronted Test Coverage
name: Check Frontend Test Coverage
on:
pull_request:
types: [opened, synchronize, reopened]
types: [opened, synchronize, reopened, labeled, unlabeled]
paths:
- '**/*.js'
- '**/*.jsx'
@ -14,20 +14,255 @@ on:
permissions: {}
jobs:
generate-slugs:
name: Generate slug for ${{ matrix.codeowner }}
setup:
name: Setup opted-in teams
runs-on: ubuntu-x64-small
outputs:
# NOTE: Only checks test coverage for opted-in codeowners.
# Please add the GitHub user/team or email used in
# CODEOWNERS to the list below to opt-in.
opted-in-teams: >-
[
"@grafana/datapro",
"@grafana/dataviz-squad"
]
steps:
- name: Define opted-in teams
run: echo "Opted-in teams defined in job outputs"
detect-changes:
name: Detect changed files
runs-on: ubuntu-x64-small
permissions:
contents: read
outputs:
changed-files: ${{ steps.changed-files.outputs.all_changed_files }}
steps:
- uses: actions/checkout@v5
with:
persist-credentials: true
fetch-depth: 2
- name: Detect changed files
id: changed-files
uses: tj-actions/changed-files@ed68ef82c095e0d48ec87eccea555d944a631a4c # v46
with:
separator: ' '
- name: Display changed files
run: |
echo "=== Changed Files Detected ==="
echo "${{ steps.changed-files.outputs.all_changed_files }}"
echo "=============================="
cleanup-on-skip:
name: Cleanup coverage comments
needs: setup
if: contains(github.event.pull_request.labels.*.name, 'no-check-frontend-test-coverage') && github.event.pull_request.head.repo.fork == false
runs-on: ubuntu-x64-small
permissions:
pull-requests: write
id-token: write
strategy:
fail-fast: false
matrix:
codeowner: ${{ fromJSON(needs.setup.outputs.opted-in-teams) }}
steps:
- name: Get Vault secrets
id: get-secrets
uses: grafana/shared-workflows/actions/get-vault-secrets@main
with:
repo_secrets: |
GITHUB_APP_ID=grafana_pr_automation_app:app_id
GITHUB_APP_PRIVATE_KEY=grafana_pr_automation_app:app_pem
- name: Generate GitHub App token
id: generate_token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ env.GITHUB_APP_ID }}
private-key: ${{ env.GITHUB_APP_PRIVATE_KEY }}
- name: Delete coverage comment for ${{ matrix.codeowner }}
uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
with:
header: coverage-report-${{ matrix.codeowner }}
delete: true
GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}
post-skip-warning:
name: Post skip warning
needs: [setup, cleanup-on-skip]
if: contains(github.event.pull_request.labels.*.name, 'no-check-frontend-test-coverage') && github.event.pull_request.head.repo.fork == false
runs-on: ubuntu-x64-small
permissions:
pull-requests: write
id-token: write
steps:
- name: Get Vault secrets
id: get-secrets
uses: grafana/shared-workflows/actions/get-vault-secrets@main
with:
repo_secrets: |
GITHUB_APP_ID=grafana_pr_automation_app:app_id
GITHUB_APP_PRIVATE_KEY=grafana_pr_automation_app:app_pem
- name: Generate GitHub App token
id: generate_token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ env.GITHUB_APP_ID }}
private-key: ${{ env.GITHUB_APP_PRIVATE_KEY }}
- name: Generate skip message
id: skip-message
run: |
TEAMS='${{ needs.setup.outputs.opted-in-teams }}'
TEAM_LIST=$(echo "$TEAMS" | jq -r '.[] | "- `" + . + "`"' | tr '\n' '\n')
{
echo "message<<EOF"
echo "## ⚠️ Frontend Test Coverage Checks Skipped"
echo
echo "Coverage checks have been skipped for the following codeowners:"
echo
echo "$TEAM_LIST"
echo
echo "Remove the \`no-check-frontend-test-coverage\` label to re-enable coverage checks."
echo "EOF"
} >> "$GITHUB_OUTPUT"
- name: Post skip warning comment
uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
with:
header: coverage-skip-warning
message: ${{ steps.skip-message.outputs.message }}
GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}
clear-skip-warning:
name: Clear skip warning
if: "!contains(github.event.pull_request.labels.*.name, 'no-check-frontend-test-coverage') && github.event.pull_request.head.repo.fork == false"
runs-on: ubuntu-x64-small
permissions:
pull-requests: write
id-token: write
steps:
- name: Get Vault secrets
id: get-secrets
uses: grafana/shared-workflows/actions/get-vault-secrets@main
with:
repo_secrets: |
GITHUB_APP_ID=grafana_pr_automation_app:app_id
GITHUB_APP_PRIVATE_KEY=grafana_pr_automation_app:app_pem
- name: Generate GitHub App token
id: generate_token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ env.GITHUB_APP_ID }}
private-key: ${{ env.GITHUB_APP_PRIVATE_KEY }}
- name: Delete skip warning comment
uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
with:
header: coverage-skip-warning
delete: true
GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}
coverage:
name: Coverage ${{ matrix.branch }} - ${{ matrix.codeowner }}
needs: [setup, detect-changes]
if: "!contains(github.event.pull_request.labels.*.name, 'no-check-frontend-test-coverage')"
runs-on: ubuntu-x64-large
permissions:
contents: read
strategy:
fail-fast: false
matrix:
# NOTE: Only checks test coverage for opted-in codeowners.
# Please add the GitHub user/team or email used in
# CODEOWNERS to the list below to opt-in.
codeowner: &codeowners
- '@grafana/datapro'
- '@grafana/dataviz-squad'
codeowner: ${{ fromJSON(needs.setup.outputs.opted-in-teams) }}
branch: [pr, main]
steps:
- name: Checkout PR branch
uses: actions/checkout@v5
with:
persist-credentials: false
- name: Setup Node.js for manifest generation
uses: ./.github/actions/setup-node
- name: Install dependencies for manifest generation
run: yarn install --immutable
env:
PUPPETEER_SKIP_DOWNLOAD: true
CYPRESS_INSTALL_BINARY: 0
- name: Generate codeowners manifest
run: yarn codeowners-manifest
- name: Check if codeowner affected by changes
id: check-affected
run: |
AFFECTED=$(node scripts/check-codeowner-affected.js "${{ matrix.codeowner }}" "${{ needs.detect-changes.outputs.changed-files }}")
echo "affected=$AFFECTED" >> "$GITHUB_OUTPUT"
echo "Codeowner ${{ matrix.codeowner }} affected: $AFFECTED"
- name: Checkout target branch
if: steps.check-affected.outputs.affected == 'true'
uses: actions/checkout@v5
with:
ref: ${{ matrix.branch == 'main' && github.event.pull_request.base.sha || github.event.pull_request.head.sha }}
persist-credentials: false
- name: Setup Node.js
if: steps.check-affected.outputs.affected == 'true'
uses: ./.github/actions/setup-node
- name: Install dependencies
if: steps.check-affected.outputs.affected == 'true'
run: yarn install --immutable
env:
PUPPETEER_SKIP_DOWNLOAD: true
CYPRESS_INSTALL_BINARY: 0
- name: Get codeowner slug
if: steps.check-affected.outputs.affected == 'true'
id: codeowner-slug
run: |
SLUG=$(node -e "
const { createCodeownerSlug } = require('./scripts/codeowners-manifest/utils.js');
console.log(createCodeownerSlug('${{ matrix.codeowner }}'));
")
echo "slug=$SLUG" >> "$GITHUB_OUTPUT"
- name: Run coverage for ${{ matrix.codeowner }}
if: steps.check-affected.outputs.affected == 'true'
run: yarn test:coverage:by-codeowner "${{ matrix.codeowner }}" --maxWorkers=1
env:
SHOULD_OPEN_COVERAGE_REPORT: 'false'
CI: 'true'
GITHUB_SHA: ${{ matrix.branch == 'main' && github.event.pull_request.base.sha || github.event.pull_request.head.sha }}
- name: Upload coverage summary
if: steps.check-affected.outputs.affected == 'true'
uses: actions/upload-artifact@v5
with:
name: coverage-${{ matrix.branch }}-${{ steps.codeowner-slug.outputs.slug }}
path: coverage-summary.json
retention-days: 1
compare-and-report:
name: Compare & Report - ${{ matrix.codeowner }}
needs: [setup, detect-changes, coverage]
if: "!contains(github.event.pull_request.labels.*.name, 'no-check-frontend-test-coverage')"
runs-on: ubuntu-x64-small
permissions:
contents: read
pull-requests: write
id-token: write
strategy:
fail-fast: false
matrix:
codeowner: ${{ fromJSON(needs.setup.outputs.opted-in-teams) }}
steps:
- uses: actions/checkout@v5
with:
@ -36,14 +271,125 @@ jobs:
- name: Setup Node.js
uses: ./.github/actions/setup-node
- name: Generate slug for ${{ matrix.codeowner }}
- name: Install dependencies for manifest and comparison
run: yarn install --immutable
env:
PUPPETEER_SKIP_DOWNLOAD: true
CYPRESS_INSTALL_BINARY: 0
- name: Generate codeowners manifest
run: yarn codeowners-manifest
- name: Get Vault secrets
if: github.event.pull_request.head.repo.fork == false
id: get-secrets
uses: grafana/shared-workflows/actions/get-vault-secrets@main
with:
repo_secrets: |
GITHUB_APP_ID=grafana_pr_automation_app:app_id
GITHUB_APP_PRIVATE_KEY=grafana_pr_automation_app:app_pem
- name: Generate GitHub App token
if: github.event.pull_request.head.repo.fork == false
id: generate_token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ env.GITHUB_APP_ID }}
private-key: ${{ env.GITHUB_APP_PRIVATE_KEY }}
- name: Check if codeowner affected by changes
id: check-affected
run: |
AFFECTED=$(node scripts/check-codeowner-affected.js "${{ matrix.codeowner }}" "${{ needs.detect-changes.outputs.changed-files }}")
echo "affected=$AFFECTED" >> "$GITHUB_OUTPUT"
echo "Codeowner ${{ matrix.codeowner }} affected: $AFFECTED"
- name: Delete old coverage comment if not affected
if: steps.check-affected.outputs.affected == 'false' && github.event.pull_request.head.repo.fork == false
uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
with:
header: coverage-report-${{ matrix.codeowner }}
delete: true
GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}
- name: Get codeowner slug
if: steps.check-affected.outputs.affected == 'true'
id: codeowner-slug
run: |
SLUG=$(node -e "
const { createCodeownerSlug } = require('./scripts/codeowners-manifest/utils.js');
console.log(createCodeownerSlug('${{ matrix.codeowner }}'));
")
echo "slug=$SLUG" >> "$GITHUB_OUTPUT"
echo "=== Codeowner Slug Generated ==="
echo "Codeowner: ${{ matrix.codeowner }}"
echo "Slug: $SLUG"
echo "================================"
- name: Download PR coverage
if: steps.check-affected.outputs.affected == 'true'
uses: actions/download-artifact@v6
with:
name: coverage-pr-${{ steps.codeowner-slug.outputs.slug }}
path: ./coverage-pr
- name: Download main coverage
if: steps.check-affected.outputs.affected == 'true'
uses: actions/download-artifact@v6
with:
name: coverage-main-${{ steps.codeowner-slug.outputs.slug }}
path: ./coverage-main
- name: Compare coverage
if: steps.check-affected.outputs.affected == 'true'
id: compare
run: |
# Run comparison and capture exit code
if node scripts/compare-coverage-by-codeowner.js; then
echo "passed=true" >> "$GITHUB_OUTPUT"
else
echo "passed=false" >> "$GITHUB_OUTPUT"
fi
# Always capture the markdown output
{
echo "markdown<<EOF"
cat ./coverage-comparison.md
echo "EOF"
} >> "$GITHUB_OUTPUT"
- name: Post PR comment
if: steps.check-affected.outputs.affected == 'true' && github.event.pull_request.head.repo.fork == false
uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4
with:
header: coverage-report-${{ matrix.codeowner }}
message: ${{ steps.compare.outputs.markdown }}
GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}
- name: Fail if coverage decreased
if: steps.check-affected.outputs.affected == 'true' && steps.compare.outputs.passed == 'false'
run: |
echo "❌ Coverage check failed for ${{ matrix.codeowner }}"
exit 1
all-coverage-checks-pass:
name: All coverage checks pass
needs: [setup, compare-and-report]
if: always()
runs-on: ubuntu-x64-small
steps:
- name: Check if skipped
id: check-skip
run: |
if [[ "${{ contains(github.event.pull_request.labels.*.name, 'no-check-frontend-test-coverage') }}" == "true" ]]; then
echo "skipped=true" >> "$GITHUB_OUTPUT"
echo "✅ Coverage checks skipped due to 'no-check-frontend-test-coverage' label"
else
echo "skipped=false" >> "$GITHUB_OUTPUT"
echo "All opted-in teams' coverage checks have completed."
fi
- name: Verify all checks passed
if: steps.check-skip.outputs.skipped == 'false'
run: |
if [[ "${{ needs.compare-and-report.result }}" != "success" ]]; then
echo "❌ Coverage checks failed for one or more teams"
exit 1
fi
echo "✅ All coverage metrics were maintained or improved."

2
.gitignore vendored
View file

@ -262,3 +262,5 @@ public/mockServiceWorker.js
# Ignore grafana/hippocampus local cache folder
.hippo
coverage-summary.json

View file

@ -50,7 +50,11 @@ const sourceFiles = teamFiles.filter((file) => {
!file.includes('.d.ts') &&
!file.endsWith('/types.ts') &&
// and anything in graveyard
!path.matchesGlob(file, '**/graveyard/**/*')
!path.matchesGlob(file, '**/graveyard/**/*') &&
// and scripts directory
!file.startsWith('scripts/') &&
// and jest config files
!path.matchesGlob(file, '**/jest.config*.js')
);
});

View file

@ -0,0 +1,47 @@
#!/usr/bin/env node
const CODEOWNERS_MANIFEST_PATH = '../codeowners-manifest/filenames-by-team.json';
/**
* Checks if any files owned by a codeowner are in the list of changed files
* @param {string} codeowner - Codeowner name (e.g., '@grafana/dataviz-squad')
* @param {string[]} changedFiles - Array of changed file paths
* @param {string} manifestPath - Path to codeowners manifest JSON file
* @returns {boolean} True if any team files are in the changed files list
*/
function isCodeownerAffected(codeowner, changedFiles, manifestPath = CODEOWNERS_MANIFEST_PATH) {
const manifest = require(manifestPath);
const teamFiles = manifest[codeowner] || [];
if (teamFiles.length === 0) {
console.warn(`Warning: No files found for codeowner "${codeowner}"`);
return false;
}
return teamFiles.some((file) => changedFiles.includes(file));
}
/**
* Runs the codeowner affected check from command line
* @param {string} codeowner - Codeowner name from CLI args
* @param {string|string[]} changedFiles - Changed file paths (space-separated string or array)
*/
function checkCodeownerAffected(codeowner, changedFiles) {
if (!codeowner) {
console.error('Usage: node check-codeowner-affected.js <codeowner> <space-separated-files>');
console.error(' or: node check-codeowner-affected.js <codeowner> <file1> <file2> ...');
process.exit(1);
}
const filesArray = typeof changedFiles === 'string' ? changedFiles.split(/\s+/).filter(Boolean) : changedFiles;
const isAffected = isCodeownerAffected(codeowner, filesArray);
console.log(isAffected ? 'true' : 'false');
}
if (require.main === module) {
const [codeowner, ...rest] = process.argv.slice(2);
const changedFiles = rest.length === 1 ? rest[0] : rest;
checkCodeownerAffected(codeowner, changedFiles);
}
module.exports = { isCodeownerAffected, checkCodeownerAffected };

View file

@ -24,10 +24,10 @@ function readCoverageFile(filePath) {
/**
* Formats a number as a percentage string
* @param {number} value - Percentage value
* @returns {string} Formatted percentage (e.g., "85.3%")
* @returns {string} Formatted percentage (e.g., "85.34%")
*/
function formatPercentage(value) {
return `${value.toFixed(1)}%`;
return `${value.toFixed(2)}%`;
}
/**
@ -37,7 +37,11 @@ function formatPercentage(value) {
* @returns {string} Status icon and text
*/
function getStatusIcon(mainValue, prValue) {
if (prValue >= mainValue) {
// Round to 2 decimal places for comparison to match display precision
const prPct = Math.round(prValue * 100) / 100;
const mainPct = Math.round(mainValue * 100) / 100;
if (prPct >= mainPct) {
return '✅ Pass';
}
return '❌ Fail';
@ -51,10 +55,31 @@ function getStatusIcon(mainValue, prValue) {
*/
function getOverallStatus(mainSummary, prSummary) {
const metrics = ['lines', 'statements', 'functions', 'branches'];
const allPass = metrics.every((metric) => prSummary[metric].pct >= mainSummary[metric].pct);
const allPass = metrics.every((metric) => {
// Round to 2 decimal places for comparison to match display precision
const prPct = Math.round(prSummary[metric].pct * 100) / 100;
const mainPct = Math.round(mainSummary[metric].pct * 100) / 100;
return prPct >= mainPct;
});
return allPass;
}
/**
* Calculates the difference between PR and main coverage
* @param {number} prValue - PR coverage percentage
* @param {number} mainValue - Main coverage percentage
* @returns {string} Formatted delta (e.g., "+1.2%" or "-0.5%")
*/
function formatDelta(prValue, mainValue) {
const delta = prValue - mainValue;
if (delta > 0) {
return `+${delta.toFixed(2)}%`;
} else if (delta < 0) {
return `${delta.toFixed(2)}%`;
}
return '—';
}
/**
* Generates markdown report comparing main and PR coverage
* @param {Object} mainCoverage - Main branch coverage data
@ -94,28 +119,22 @@ function generateMarkdown(mainCoverage, prCoverage) {
const tableRows = rows
.map((row) => {
const status = getStatusIcon(row.main, row.pr);
return `| ${row.metric} | ${formatPercentage(row.main)} | ${formatPercentage(row.pr)} | ${status} |`;
const delta = formatDelta(row.pr, row.main);
return `| ${row.metric} | ${formatPercentage(row.main)} | ${formatPercentage(row.pr)} | ${delta} | ${status} |`;
})
.join('\n');
const overallStatus = overallPass ? '✅ Pass' : '❌ Fail';
const overallMessage = overallPass ? 'Coverage maintained or improved' : 'Coverage decreased in one or more metrics';
const overallStatus = overallPass ? '✅ Passed' : '❌ Failed';
return `## Test Coverage Report - ${teamName}
return `## Test Coverage Checks ${overallStatus} for ${teamName}
| Metric | Main Branch | PR Branch | Status |
|--------|-------------|-----------|--------|
| Metric | Main | PR | Change | Status |
|--------|------|----|----|--------|
${tableRows}
**Overall: ${overallStatus}** - ${overallMessage}
**Run locally:** 💻 \`yarn test:coverage:by-codeowner ${teamName}\`
<details>
<summary>Coverage Details</summary>
- **PR Branch**: \`${prCoverage.commit.substring(0, 7)}\` (${prCoverage.timestamp})
- **Main Branch**: \`${mainCoverage.commit.substring(0, 7)}\` (${mainCoverage.timestamp})
</details>
**Break glass:** 🚨 In case of emergency, adding the \`no-check-frontend-test-coverage\` label to this PR will skip checks.
`;
}
@ -124,6 +143,7 @@ ${tableRows}
* @param {string} mainPath - Path to main branch coverage summary JSON
* @param {string} prPath - Path to PR branch coverage summary JSON
* @param {string} outputPath - Path to write comparison markdown
* @returns {boolean} True if coverage check passed
*/
function compareCoverageByCodeowner(
mainPath = COVERAGE_MAIN_PATH,
@ -139,6 +159,7 @@ function compareCoverageByCodeowner(
}
const markdown = generateMarkdown(mainCoverage, prCoverage);
const overallPass = getOverallStatus(mainCoverage.summary, prCoverage.summary);
try {
fs.writeFileSync(outputPath, markdown, 'utf8');
@ -147,10 +168,17 @@ function compareCoverageByCodeowner(
console.error(`Error writing output file: ${err.message}`);
process.exit(1);
}
return overallPass;
}
if (require.main === module) {
compareCoverageByCodeowner();
const passed = compareCoverageByCodeowner();
if (!passed) {
console.error('❌ Coverage check failed: One or more metrics decreased');
process.exit(1);
}
console.log('✅ Coverage check passed: All metrics maintained or improved');
}
module.exports = { compareCoverageByCodeowner, generateMarkdown, getOverallStatus };