mirror of
https://github.com/mattermost/mattermost.git
synced 2026-02-03 20:40:00 -05:00
Support E2E stress tests in run_tests.js (#29682)
* [skip ci] Adapt run_tests.js for stress testing * [skip ci] Make channel_read_after_permalink_spec.js retry-able * [skip ci] Fix ui_commands.ts fix * [skip ci] Improve postMessageAndWait * [skip ci] Fix unread_on_public_channel_spec.js * [skip ci] Demote unread_on_public_channel_spec.js * Update e2e-tests/README.md Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com> * [skip ci] Fix mesage_reply_bot_post_spec.js --------- Co-authored-by: Mattermost Build <build@mattermost.com> Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com>
This commit is contained in:
Mario Vitale
GitHub
parent
7fb6901ad1
commit
d3bd91843f
8 changed files with 51 additions and 15 deletions
|
|
@ -100,6 +100,7 @@ case "${TEST:-$TEST_DEFAULT}" in
|
|||
playwright )
|
||||
export TEST_FILTER_DEFAULT='tests/functional/system_console/system_users/actions.spec.ts --project=chrome' ;;
|
||||
* )
|
||||
export TEST_FILTER_DEFAULT='' ;;
|
||||
esac
|
||||
# OS specific defaults overrides
|
||||
case $MME2E_OSTYPE in
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ services:
|
|||
MM_SERVICEENVIRONMENT: "test"
|
||||
MM_FEATUREFLAGS_MOVETHREADSENABLED: "true"
|
||||
MM_LOGSETTINGS_ENABLEDIAGNOSTICS: "false"
|
||||
MM_LOGSETTINGS_CONSOLELEVEL: "DEBUG"
|
||||
network_mode: host
|
||||
depends_on:
|
||||
$(for service in $ENABLED_DOCKER_SERVICES; do
|
||||
|
|
|
|||
|
|
@ -61,3 +61,14 @@ Notes:
|
|||
|
||||
##### For code changes:
|
||||
* `make fmt-ci` to format and check yaml files and shell scripts.
|
||||
|
||||
##### For test stressing an E2E testcase
|
||||
|
||||
For Cypress:
|
||||
1. Enter the `cypress/` subdirectory
|
||||
2. Identify which test files you want to run, and how many times each. For instance: suppose you want to run `create_a_team_spec.js` and `demoted_user_spec.js` (which you can locate with the `find` command, under `cypress/tests/`), each run 3 times
|
||||
3. Run the chosen testcases the desired amount of times: `node run_tests.js --include-file=create_a_team_spec.js,demoted_user_spec.js --invert --stress-test-count=3`
|
||||
* Your system needs to be setup for Cypress usage, to be able to run this command. Refer to the [E2E testing developer documentation](https://developers.mattermost.com/contribute/more-info/webapp/e2e-testing/) for this.
|
||||
4. The `cypress/results/testPasses.json` file will count, for each of the testfiles, how many times it was run, and how many times each of the testcases contained in it passed. If the attempts and passes numbers do not match, that specific testcase may be flaky.
|
||||
|
||||
For Playwright: WIP
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@
|
|||
* Exclude spec files with matching directory or filename pattern. Uses `find` command under the hood. It can be of multiple values separated by comma.
|
||||
* E.g. "--exclude-file='channel'" will exclude files recursively under `channel` directory/s.
|
||||
* E.g. "--exclude-file='*channel*'" will exclude files and files under directory/s recursively that matches the name with `*channel*`.
|
||||
* --stress-test-count=[number of repeated executions]
|
||||
* The amount of times to run each spec file for. If greater than 1, report how the percentage of successes over total runs for each spec.
|
||||
|
||||
*
|
||||
* Environment:
|
||||
|
|
@ -83,8 +85,11 @@ async function runTests() {
|
|||
const browser = BROWSER || 'chrome';
|
||||
const headless = typeof HEADLESS === 'undefined' ? true : HEADLESS === 'true';
|
||||
const platform = os.platform();
|
||||
const stressTestCount = argv.stressTestCount > 1 ? argv.stressTestCount : 1;
|
||||
const testPasses = {};
|
||||
|
||||
const {sortedFiles} = getSortedTestFiles(platform, browser, headless);
|
||||
const sortedFilesObject = getSortedTestFiles(platform, browser, headless);
|
||||
const sortedFiles = sortedFilesObject.sortedFiles.flatMap((el) => Array(stressTestCount).fill(el));
|
||||
const numberOfTestFiles = sortedFiles.length;
|
||||
|
||||
if (!numberOfTestFiles) {
|
||||
|
|
@ -102,6 +107,13 @@ async function runTests() {
|
|||
printMessage(sortedFiles, i, j + 1, count);
|
||||
|
||||
const testFile = sortedFiles[i];
|
||||
var testFileAttempt = 1;
|
||||
if (testFile in testPasses === false) {
|
||||
testPasses[testFile] = {attempt: 1};
|
||||
} else {
|
||||
testPasses[testFile].attempt += 1;
|
||||
testFileAttempt = testPasses[testFile].attempt;
|
||||
}
|
||||
|
||||
const result = await cypress.run({
|
||||
browser,
|
||||
|
|
@ -111,6 +123,7 @@ async function runTests() {
|
|||
screenshotsFolder: `${MOCHAWESOME_REPORT_DIR}/screenshots`,
|
||||
videosFolder: `${MOCHAWESOME_REPORT_DIR}/videos`,
|
||||
trashAssetsBeforeRuns: false,
|
||||
retries: stressTestCount > 1 ? 0 : 2,
|
||||
},
|
||||
env: {
|
||||
firstTest: j === 0,
|
||||
|
|
@ -135,11 +148,22 @@ async function runTests() {
|
|||
headless,
|
||||
branch: BRANCH,
|
||||
buildId: BUILD_ID,
|
||||
testFileAttempt,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
for (var testCase of result.runs[0].tests) {
|
||||
var testCaseTitle = testCase.title.join(' - ');
|
||||
if (testCaseTitle in testPasses[testFile] === false) {
|
||||
testPasses[testFile][testCaseTitle] = 0;
|
||||
}
|
||||
if (testCase.state === 'passed') {
|
||||
testPasses[testFile][testCaseTitle] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Write test environment details once only
|
||||
if (i === 0) {
|
||||
const environment = {
|
||||
|
|
@ -155,6 +179,7 @@ async function runTests() {
|
|||
writeJsonToFile(environment, 'environment.json', RESULTS_DIR);
|
||||
}
|
||||
}
|
||||
writeJsonToFile(testPasses, 'testPasses.json', RESULTS_DIR);
|
||||
}
|
||||
|
||||
function printMessage(testFiles, overallIndex, currentItem, lastItem) {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ describe('Messaging', () => {
|
|||
let testUser;
|
||||
let otherUser;
|
||||
|
||||
before(() => {
|
||||
beforeEach(() => {
|
||||
cy.apiInitSetup().then(({team, channel, user}) => {
|
||||
testTeam = team;
|
||||
testChannel = channel;
|
||||
|
|
|
|||
|
|
@ -144,7 +144,7 @@ describe('Messaging', () => {
|
|||
cy.uiCloseRHS();
|
||||
|
||||
// * Verify that the reply is in the channel view with matching text
|
||||
cy.get(`#post_${replyId}`).within(() => {
|
||||
cy.get(`#post_${replyId}`).scrollIntoView().within(() => {
|
||||
cy.findByTestId('post-link').should('be.visible').and('have.text', 'Commented on ' + bot.username + 'BOT\'s message: Hello message from ' + bot.username);
|
||||
cy.get(`#postMessageText_${replyId}`).should('be.visible').and('have.text', replyMessage);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@
|
|||
// - Use element ID when selecting an element. Create one if none.
|
||||
// ***************************************************************
|
||||
|
||||
// Stage: @prod
|
||||
// Group: @channels @notifications
|
||||
|
||||
import * as TIMEOUTS from '../../../fixtures/timeouts';
|
||||
|
|
@ -18,7 +17,7 @@ describe('Notifications', () => {
|
|||
let testTeam;
|
||||
|
||||
beforeEach(() => {
|
||||
cy.apiInitSetup().then(({team, user}) => {
|
||||
cy.apiAdminLogin().apiInitSetup().then(({team, user}) => {
|
||||
testUser = user;
|
||||
testTeam = team;
|
||||
|
||||
|
|
@ -48,11 +47,6 @@ describe('Notifications', () => {
|
|||
// # Go to test channel
|
||||
cy.visit(`/${team.name}/channels/${channel.name}`);
|
||||
|
||||
// # Scroll above the last few messages
|
||||
cy.get('div.post-list__dynamic', {timeout: TIMEOUTS.ONE_MIN}).should('be.visible').
|
||||
scrollTo(0, '70%', {duration: TIMEOUTS.ONE_SEC}).
|
||||
wait(TIMEOUTS.ONE_SEC);
|
||||
|
||||
// # scroll to the last channel
|
||||
cy.get('#sidebar-left li').last().scrollIntoView();
|
||||
});
|
||||
|
|
@ -66,7 +60,11 @@ describe('Notifications', () => {
|
|||
|
||||
// # Get last post and reply a message
|
||||
cy.getLastPostId().then((postId) => {
|
||||
// # Open the RHS reply form, then scroll above the last few messages
|
||||
cy.clickPostCommentIcon(postId);
|
||||
cy.get('div.post-list__dynamic', {timeout: TIMEOUTS.ONE_MIN}).should('be.visible').
|
||||
scrollTo(0, '70%', {duration: TIMEOUTS.ONE_SEC}).
|
||||
wait(TIMEOUTS.ONE_SEC);
|
||||
const replyMessage = 'A reply to an older post';
|
||||
cy.postMessageReplyInRHS(replyMessage);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ function postMessageAndWait(textboxSelector: string, message: string, isComment
|
|||
waitForCommentDraft(message);
|
||||
}
|
||||
|
||||
cy.get(textboxSelector).should('have.value', message).type('{enter}').wait(TIMEOUTS.HALF_SEC);
|
||||
cy.get(textboxSelector).should('have.value', message).focus().type('{enter}').wait(TIMEOUTS.HALF_SEC);
|
||||
|
||||
cy.get(textboxSelector).invoke('val').then((value: string) => {
|
||||
if (value.length > 0 && value === message) {
|
||||
|
|
@ -325,12 +325,12 @@ function clickPostHeaderItem(postId: string, location: string, item: string) {
|
|||
}
|
||||
|
||||
if (postId) {
|
||||
cy.get(`#${idPrefix}_${postId}`).trigger('mouseover', {force: true});
|
||||
cy.wait(TIMEOUTS.HALF_SEC).get(`#${location}_${item}_${postId}`).click({force: true});
|
||||
cy.get(`#${idPrefix}_${postId}`).trigger('mouseover', {force: true}).
|
||||
get(`#${location}_${item}_${postId}`).scrollIntoView().trigger('mouseover', {force: true}).click({force: true});
|
||||
} else {
|
||||
cy.getLastPostId().then((lastPostId) => {
|
||||
cy.get(`#${idPrefix}_${lastPostId}`).trigger('mouseover', {force: true});
|
||||
cy.wait(TIMEOUTS.HALF_SEC).get(`#${location}_${item}_${lastPostId}`).click({force: true});
|
||||
cy.get(`#${idPrefix}_${lastPostId}`).trigger('mouseover', {force: true}).
|
||||
get(`#${location}_${item}_${lastPostId}`).scrollIntoView().trigger('mouseover', {force: true}).click({force: true});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue