mirror of
https://github.com/mattermost/mattermost.git
synced 2026-02-03 20:40:00 -05:00
2116a6d94a
* initial implementation of test documentation in spec file with AI-assisted prompt from Claude and linter script * update snapshots --------- Co-authored-by: Mattermost Build <build@mattermost.com>
386 lines
15 KiB
JavaScript
Executable file
386 lines
15 KiB
JavaScript
Executable file
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
// See LICENSE.txt for license information.
|
|
|
|
/**
|
|
* Test Documentation Format Linter
|
|
*
|
|
* This script verifies that all spec files follow the required documentation format:
|
|
* - JSDoc with @objective and @precondition
|
|
* - Proper test title with MM-T ID
|
|
* - Tag for feature categorization
|
|
* - Action/Verification comments
|
|
*/
|
|
|
|
/* eslint-disable @typescript-eslint/no-require-imports */
|
|
/* eslint-disable import/order */
|
|
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const glob = require('glob');
|
|
|
|
/* eslint-disable no-console */
|
|
|
|
// Colors for terminal output
|
|
const colors = {
|
|
red: '\x1b[31m',
|
|
green: '\x1b[32m',
|
|
yellow: '\x1b[33m',
|
|
reset: '\x1b[0m',
|
|
cyan: '\x1b[36m',
|
|
magenta: '\x1b[35m',
|
|
white: '\x1b[37m',
|
|
};
|
|
|
|
// Regex patterns
|
|
const patterns = {
|
|
// Support both single-line and multi-line test declarations
|
|
testDeclaration:
|
|
/test\(\s*['"]([^'"]+)['"]\s*,(?:\s*|\n\s*){.*?tag:\s*['"]@.*?['"].*?}(?:\s*|\n\s*),(?:\s*|\n\s*)async/g,
|
|
jsdocBlock: /\/\*\*\s*\n(?:.*\n)*?\s*\*\//g,
|
|
objective: /@objective\s+.*?(?=\n|\*\/)/g,
|
|
precondition: /@precondition\s+.*?(?=\n|\*\/)/g,
|
|
tagDeclaration: /{tag:\s*['"]@[\w_]+['"]}|{\s*tag:\s*['"]@[\w_]+['"]\s*}/g,
|
|
// Match action comments - note the difference between # as a comment marker vs # in strings
|
|
actionComment: /(?:^|\n)\s*\/\/\s*#\s*.+/g,
|
|
// Match verification comments - note the difference between * as a comment marker vs * in strings
|
|
verificationComment: /(?:^|\n)\s*\/\/\s*\*\s*.+/g,
|
|
};
|
|
|
|
// Get all spec files
|
|
const specFiles = glob.sync(path.join(process.cwd(), 'specs/**/*.spec.ts'));
|
|
|
|
// Results
|
|
const results = {
|
|
passed: 0,
|
|
failed: 0,
|
|
warnings: 0,
|
|
errors: [],
|
|
};
|
|
|
|
// Process each file
|
|
specFiles.forEach((filePath) => {
|
|
const relativeFilePath = path.relative(process.cwd(), filePath);
|
|
const fileContent = fs.readFileSync(filePath, 'utf-8');
|
|
const fileErrors = [];
|
|
|
|
// Extract all test declarations
|
|
const testDeclarations = Array.from(fileContent.matchAll(patterns.testDeclaration));
|
|
const jsdocBlocks = Array.from(fileContent.matchAll(patterns.jsdocBlock));
|
|
|
|
// If no test declarations found, warn but don't fail
|
|
if (testDeclarations.length === 0) {
|
|
results.warnings++;
|
|
console.log(
|
|
`${colors.yellow}WARNING${colors.reset}: No test declarations found in ${colors.cyan}${relativeFilePath}${colors.reset}`,
|
|
);
|
|
return;
|
|
}
|
|
|
|
// Check each test for proper format
|
|
testDeclarations.forEach((testMatch) => {
|
|
const testDeclaration = testMatch[0];
|
|
const testName = testMatch[1];
|
|
|
|
// Get test function position to find corresponding JSDoc
|
|
const testPosition = testMatch.index;
|
|
|
|
// Get the JSDoc block that immediately precedes this test
|
|
const precedingJSDoc = jsdocBlocks.find((jsdoc) => {
|
|
const jsdocEnd = jsdoc.index + jsdoc[0].length;
|
|
// JSDoc should end right before the test with only whitespace in between
|
|
const textBetween = fileContent.substring(jsdocEnd, testPosition).trim();
|
|
return textBetween === '';
|
|
});
|
|
|
|
// Check if test has a test key (MM-T####)
|
|
if (!testName.match(/^MM-T\d+/)) {
|
|
// This is a new test without a test key
|
|
console.log(
|
|
`${colors.cyan}NEW TEST FOUND: "${testName}"${colors.reset} - Will be registered in the test management system after merge`,
|
|
);
|
|
results.newTests = results.newTests || [];
|
|
results.newTests.push({file: relativeFilePath, testName});
|
|
} else {
|
|
// This is an existing test with a test key
|
|
const baseTestKey = testName.match(/^(MM-T\d+)/)[1];
|
|
|
|
// Check if this is a step of a multi-step test case
|
|
const stepMatch = testName.match(/^MM-T\d+(_\d+)/);
|
|
const isStep = stepMatch !== null;
|
|
const stepSuffix = isStep ? stepMatch[1] : '';
|
|
const testKey = baseTestKey + (stepSuffix || '');
|
|
|
|
// Log different message for test steps
|
|
if (isStep) {
|
|
// Extract the step number without the underscore
|
|
const stepNumber = stepSuffix.substring(1);
|
|
console.log(
|
|
`${colors.magenta}DOCUMENTATION UPDATE: "${testKey}"${colors.reset} - step ${stepNumber} of ${baseTestKey} - Changes will be mapped to test management system after merge`,
|
|
);
|
|
} else {
|
|
console.log(
|
|
`${colors.magenta}DOCUMENTATION UPDATE: "${testKey}"${colors.reset} - Changes will be saved to test management system after merge`,
|
|
);
|
|
}
|
|
|
|
// Check if documentation is present and mark for update
|
|
if (precedingJSDoc) {
|
|
results.updatedTests = results.updatedTests || [];
|
|
results.updatedTests.push({
|
|
file: relativeFilePath,
|
|
testKey,
|
|
baseTestKey,
|
|
isStep,
|
|
stepSuffix,
|
|
testName,
|
|
});
|
|
}
|
|
}
|
|
|
|
// JSDoc was already retrieved above, no need to get it again
|
|
|
|
if (!precedingJSDoc) {
|
|
fileErrors.push(`Missing JSDoc documentation at "${testName}"`);
|
|
} else {
|
|
const jsdocContent = precedingJSDoc[0];
|
|
|
|
// Check for @objective
|
|
if (!jsdocContent.match(patterns.objective)) {
|
|
fileErrors.push(`Missing @objective in JSDoc at "${testName}"`);
|
|
}
|
|
|
|
// Note: @precondition is optional and should only be included when there are
|
|
// non-default requirements for the test
|
|
}
|
|
|
|
// Check for tag declaration
|
|
if (!testDeclaration.match(patterns.tagDeclaration)) {
|
|
fileErrors.push(`Missing feature tag at "${testName}"`);
|
|
}
|
|
|
|
// Simpler approach to extract test body by using string search and tracking braces to find function boundaries
|
|
const testNameEscaped = testName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
const testDeclarationPattern = new RegExp(`test\\s*\\([\\s\\n]*['"]${testNameEscaped}['"]`);
|
|
|
|
// Find the position of the test declaration
|
|
const testDeclarationMatch = testDeclarationPattern.exec(fileContent);
|
|
let testFnMatch = null;
|
|
|
|
if (testDeclarationMatch) {
|
|
// Find the position of "async" after the test declaration
|
|
const startPos = testDeclarationMatch.index;
|
|
const asyncPattern = /async\s*\([^)]*\)\s*=>\s*{/g;
|
|
asyncPattern.lastIndex = startPos;
|
|
|
|
const asyncMatch = asyncPattern.exec(fileContent);
|
|
if (asyncMatch) {
|
|
// Find the position of the opening brace of the function body
|
|
const openBracePos = asyncMatch.index + asyncMatch[0].length - 1;
|
|
|
|
// Find the matching closing brace
|
|
let braceCount = 1;
|
|
let currentPos = openBracePos + 1;
|
|
|
|
while (braceCount > 0 && currentPos < fileContent.length) {
|
|
const char = fileContent[currentPos];
|
|
|
|
if (char === '{') {
|
|
braceCount++;
|
|
} else if (char === '}') {
|
|
braceCount--;
|
|
}
|
|
|
|
currentPos++;
|
|
}
|
|
|
|
// If we found the matching closing brace, extract everything between them
|
|
if (braceCount === 0) {
|
|
const testBody = fileContent.substring(openBracePos + 1, currentPos - 1);
|
|
testFnMatch = [null, testBody];
|
|
}
|
|
}
|
|
|
|
// If we couldn't extract using the above method, try a more direct string search approach
|
|
if (!testFnMatch) {
|
|
// Find the test function by searching for the test name and extracting all content until the end of the function
|
|
const testIndex =
|
|
fileContent.indexOf(`test('${testName}'`) ||
|
|
fileContent.indexOf(`test("${testName}"`) ||
|
|
fileContent.indexOf(`test(\n '${testName}'`) ||
|
|
fileContent.indexOf(`test(\n "${testName}"`);
|
|
|
|
if (testIndex !== -1) {
|
|
// Find the position of "async" which indicates the start of the function body
|
|
const asyncIndex = fileContent.indexOf('async', testIndex);
|
|
if (asyncIndex !== -1) {
|
|
// Find the opening brace after "async"
|
|
const openBraceIndex = fileContent.indexOf('{', asyncIndex);
|
|
if (openBraceIndex !== -1) {
|
|
// Count braces to find the matching closing brace
|
|
let braceCount = 1;
|
|
let closePos = openBraceIndex + 1;
|
|
|
|
while (braceCount > 0 && closePos < fileContent.length) {
|
|
const char = fileContent[closePos];
|
|
|
|
if (char === '{') {
|
|
braceCount++;
|
|
} else if (char === '}') {
|
|
braceCount--;
|
|
}
|
|
|
|
closePos++;
|
|
}
|
|
|
|
if (braceCount === 0) {
|
|
const testBody = fileContent.substring(openBraceIndex + 1, closePos - 1);
|
|
testFnMatch = [null, testBody];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (testFnMatch) {
|
|
const testBody = testFnMatch[1];
|
|
|
|
// Check for action comments (// #) - we need to be more flexible in our detection
|
|
let hasActionComments = false;
|
|
const actionPattern = /\/\/\s*#/;
|
|
|
|
if (actionPattern.test(testBody)) {
|
|
hasActionComments = true;
|
|
}
|
|
|
|
if (!hasActionComments) {
|
|
fileErrors.push(`Missing action comments at "${testName}" (format: "// # Some descriptive action")`);
|
|
}
|
|
|
|
// Check for verification comments (// *) - we need to be more flexible in our detection
|
|
let hasVerificationComments = false;
|
|
const verificationPattern = /\/\/\s*\*/;
|
|
|
|
if (verificationPattern.test(testBody)) {
|
|
hasVerificationComments = true;
|
|
}
|
|
|
|
if (!hasVerificationComments) {
|
|
fileErrors.push(
|
|
`Missing verification comments at "${testName}" (format: "// * Some descriptive verification")`,
|
|
);
|
|
}
|
|
} else {
|
|
fileErrors.push(`Could not extract test body for "${testName}"`);
|
|
}
|
|
});
|
|
|
|
if (fileErrors.length > 0) {
|
|
results.failed++;
|
|
results.errors.push({
|
|
file: relativeFilePath,
|
|
errors: fileErrors,
|
|
});
|
|
} else {
|
|
results.passed++;
|
|
}
|
|
});
|
|
|
|
// Print results
|
|
console.log('\n' + '-'.repeat(80));
|
|
console.log(`${colors.cyan}Test Documentation Format Linter Results${colors.reset}`);
|
|
console.log(`Files checked: ${specFiles.length}`);
|
|
console.log(`${colors.green}Passed: ${results.passed}${colors.reset}`);
|
|
console.log(`${colors.red}Failed: ${results.failed}${colors.reset}`);
|
|
console.log(`${colors.yellow}Warnings: ${results.warnings}${colors.reset}`);
|
|
console.log(`${colors.cyan}New tests: ${results.newTests ? results.newTests.length : 0}${colors.reset}`);
|
|
console.log(`${colors.magenta}Updated tests: ${results.updatedTests ? results.updatedTests.length : 0}${colors.reset}`);
|
|
|
|
if (results.errors.length > 0) {
|
|
console.log('\n' + '-'.repeat(80));
|
|
console.log(`${colors.red}Errors:${colors.reset}`);
|
|
|
|
results.errors.forEach((fileResult) => {
|
|
console.log(`\n${colors.cyan}${fileResult.file}${colors.reset}:`);
|
|
fileResult.errors.forEach((error) => {
|
|
console.log(` ${colors.red}•${colors.reset} ${error}`);
|
|
});
|
|
});
|
|
}
|
|
|
|
// Display new tests summary if any were found
|
|
if (results.newTests && results.newTests.length > 0) {
|
|
console.log('\n' + '-'.repeat(80));
|
|
console.log(`${colors.cyan}New Tests to be Registered:${colors.reset}`);
|
|
|
|
results.newTests.forEach((newTest) => {
|
|
console.log(` ${colors.cyan}•${colors.reset} ${newTest.file}: "${newTest.testName}"`);
|
|
});
|
|
}
|
|
|
|
// Display updated tests summary if any were found
|
|
if (results.updatedTests && results.updatedTests.length > 0) {
|
|
console.log('\n' + '-'.repeat(80));
|
|
console.log(`${colors.magenta}Tests with Documentation Updates:${colors.reset}`);
|
|
|
|
// Group the updated tests by base test key
|
|
const groupedTests = {};
|
|
results.updatedTests.forEach((test) => {
|
|
if (!groupedTests[test.baseTestKey]) {
|
|
groupedTests[test.baseTestKey] = [];
|
|
}
|
|
groupedTests[test.baseTestKey].push(test);
|
|
});
|
|
|
|
// Display the tests grouped by base test key
|
|
Object.keys(groupedTests).forEach((baseTestKey) => {
|
|
const tests = groupedTests[baseTestKey];
|
|
|
|
// If there's only one test with this base key and it's not a step
|
|
if (tests.length === 1 && !tests[0].isStep) {
|
|
const test = tests[0];
|
|
console.log(
|
|
` ${colors.magenta}•${colors.reset} ${test.file}: ${test.testKey} "${test.testName.substring(test.testKey.length).trim()}"`,
|
|
);
|
|
}
|
|
// If there are multiple steps of the same test
|
|
else {
|
|
console.log(
|
|
` ${colors.magenta}•${colors.reset} ${tests[0].file}: ${colors.cyan}${baseTestKey}${colors.reset} (with ${tests.length} steps):`,
|
|
);
|
|
|
|
// Sort steps by their step number
|
|
tests.sort((a, b) => {
|
|
if (!a.isStep) return -1;
|
|
if (!b.isStep) return 1;
|
|
|
|
const aNum = parseInt(a.stepSuffix.substring(1), 10);
|
|
const bNum = parseInt(b.stepSuffix.substring(1), 10);
|
|
return aNum - bNum;
|
|
});
|
|
|
|
// Display each step
|
|
tests.forEach((test) => {
|
|
const namePart = test.testName.substring(test.baseTestKey.length).trim();
|
|
if (test.isStep) {
|
|
// Extract the step number without the underscore
|
|
const stepNumber = test.stepSuffix.substring(1);
|
|
console.log(
|
|
` ${colors.magenta}◦${colors.reset} step ${stepNumber}: "${namePart.substring(test.stepSuffix.length).trim()}"`,
|
|
);
|
|
} else {
|
|
console.log(` ${colors.magenta}◦${colors.reset} Base test: "${namePart}"`);
|
|
}
|
|
});
|
|
}
|
|
});
|
|
}
|
|
|
|
console.log('\n' + '-'.repeat(80));
|
|
if (results.failed > 0) {
|
|
console.log(`${colors.red}Linter failed!${colors.reset} Please fix the test documentation issues.`);
|
|
process.exit(1);
|
|
} else {
|
|
console.log(`${colors.green}Linter passed!${colors.reset} All spec files follow the required format.`);
|
|
process.exit(0);
|
|
}
|