From d3bd91843fe823787a76ed79fd9c1d089e321bcd Mon Sep 17 00:00:00 2001 From: Mario Vitale Date: Thu, 23 Jan 2025 14:07:38 +0100 Subject: [PATCH] Support E2E stress tests in run_tests.js (#29682) * [skip ci] Adapt run_tests.js for stress testing * [skip ci] Make channel_read_after_permalink_spec.js retry-able * [skip ci] Fix ui_commands.ts fix * [skip ci] Improve postMessageAndWait * [skip ci] Fix unread_on_public_channel_spec.js * [skip ci] Demote unread_on_public_channel_spec.js * Update e2e-tests/README.md Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com> * [skip ci] Fix mesage_reply_bot_post_spec.js --------- Co-authored-by: Mattermost Build Co-authored-by: Saturnino Abril <5334504+saturninoabril@users.noreply.github.com> --- e2e-tests/.ci/.e2erc | 1 + e2e-tests/.ci/server.generate.sh | 1 + e2e-tests/README.md | 11 ++++++++ e2e-tests/cypress/run_tests.js | 27 ++++++++++++++++++- .../channel_read_after_permalink_spec.js | 2 +- .../messaging/message_reply_bot_post_spec.js | 2 +- .../unread_on_public_channel_spec.js | 12 ++++----- .../cypress/tests/support/ui_commands.ts | 10 +++---- 8 files changed, 51 insertions(+), 15 deletions(-) diff --git a/e2e-tests/.ci/.e2erc b/e2e-tests/.ci/.e2erc index 959b4903371..e9e263eb30d 100644 --- a/e2e-tests/.ci/.e2erc +++ b/e2e-tests/.ci/.e2erc @@ -100,6 +100,7 @@ case "${TEST:-$TEST_DEFAULT}" in playwright ) export TEST_FILTER_DEFAULT='tests/functional/system_console/system_users/actions.spec.ts --project=chrome' ;; * ) + export TEST_FILTER_DEFAULT='' ;; esac # OS specific defaults overrides case $MME2E_OSTYPE in diff --git a/e2e-tests/.ci/server.generate.sh b/e2e-tests/.ci/server.generate.sh index fc1ad82dab6..ff9796d35ab 100755 --- a/e2e-tests/.ci/server.generate.sh +++ b/e2e-tests/.ci/server.generate.sh @@ -62,6 +62,7 @@ services: MM_SERVICEENVIRONMENT: "test" MM_FEATUREFLAGS_MOVETHREADSENABLED: "true" MM_LOGSETTINGS_ENABLEDIAGNOSTICS: "false" + MM_LOGSETTINGS_CONSOLELEVEL: "DEBUG" network_mode: host depends_on: $(for service in $ENABLED_DOCKER_SERVICES; do diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 87152040210..86e403d4664 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -61,3 +61,14 @@ Notes: ##### For code changes: * `make fmt-ci` to format and check yaml files and shell scripts. + +##### For test stressing an E2E testcase + +For Cypress: +1. Enter the `cypress/` subdirectory +2. Identify which test files you want to run, and how many times each. For instance: suppose you want to run `create_a_team_spec.js` and `demoted_user_spec.js` (which you can locate with the `find` command, under `cypress/tests/`), each run 3 times +3. Run the chosen testcases the desired amount of times: `node run_tests.js --include-file=create_a_team_spec.js,demoted_user_spec.js --invert --stress-test-count=3` + * Your system needs to be setup for Cypress usage, to be able to run this command. Refer to the [E2E testing developer documentation](https://developers.mattermost.com/contribute/more-info/webapp/e2e-testing/) for this. +4. The `cypress/results/testPasses.json` file will count, for each of the testfiles, how many times it was run, and how many times each of the testcases contained in it passed. If the attempts and passes numbers do not match, that specific testcase may be flaky. + +For Playwright: WIP diff --git a/e2e-tests/cypress/run_tests.js b/e2e-tests/cypress/run_tests.js index e6f4f4bb4e5..552ac28698c 100644 --- a/e2e-tests/cypress/run_tests.js +++ b/e2e-tests/cypress/run_tests.js @@ -32,6 +32,8 @@ * Exclude spec files with matching directory or filename pattern. Uses `find` command under the hood. It can be of multiple values separated by comma. * E.g. "--exclude-file='channel'" will exclude files recursively under `channel` directory/s. * E.g. "--exclude-file='*channel*'" will exclude files and files under directory/s recursively that matches the name with `*channel*`. + * --stress-test-count=[number of repeated executions] + * The amount of times to run each spec file for. If greater than 1, report how the percentage of successes over total runs for each spec. * * Environment: @@ -83,8 +85,11 @@ async function runTests() { const browser = BROWSER || 'chrome'; const headless = typeof HEADLESS === 'undefined' ? true : HEADLESS === 'true'; const platform = os.platform(); + const stressTestCount = argv.stressTestCount > 1 ? argv.stressTestCount : 1; + const testPasses = {}; - const {sortedFiles} = getSortedTestFiles(platform, browser, headless); + const sortedFilesObject = getSortedTestFiles(platform, browser, headless); + const sortedFiles = sortedFilesObject.sortedFiles.flatMap((el) => Array(stressTestCount).fill(el)); const numberOfTestFiles = sortedFiles.length; if (!numberOfTestFiles) { @@ -102,6 +107,13 @@ async function runTests() { printMessage(sortedFiles, i, j + 1, count); const testFile = sortedFiles[i]; + var testFileAttempt = 1; + if (testFile in testPasses === false) { + testPasses[testFile] = {attempt: 1}; + } else { + testPasses[testFile].attempt += 1; + testFileAttempt = testPasses[testFile].attempt; + } const result = await cypress.run({ browser, @@ -111,6 +123,7 @@ async function runTests() { screenshotsFolder: `${MOCHAWESOME_REPORT_DIR}/screenshots`, videosFolder: `${MOCHAWESOME_REPORT_DIR}/videos`, trashAssetsBeforeRuns: false, + retries: stressTestCount > 1 ? 0 : 2, }, env: { firstTest: j === 0, @@ -135,11 +148,22 @@ async function runTests() { headless, branch: BRANCH, buildId: BUILD_ID, + testFileAttempt, }, }, }, }); + for (var testCase of result.runs[0].tests) { + var testCaseTitle = testCase.title.join(' - '); + if (testCaseTitle in testPasses[testFile] === false) { + testPasses[testFile][testCaseTitle] = 0; + } + if (testCase.state === 'passed') { + testPasses[testFile][testCaseTitle] += 1; + } + } + // Write test environment details once only if (i === 0) { const environment = { @@ -155,6 +179,7 @@ async function runTests() { writeJsonToFile(environment, 'environment.json', RESULTS_DIR); } } + writeJsonToFile(testPasses, 'testPasses.json', RESULTS_DIR); } function printMessage(testFiles, overallIndex, currentItem, lastItem) { diff --git a/e2e-tests/cypress/tests/integration/channels/messaging/channel_read_after_permalink_spec.js b/e2e-tests/cypress/tests/integration/channels/messaging/channel_read_after_permalink_spec.js index a8d6826ead8..74a532bb79e 100644 --- a/e2e-tests/cypress/tests/integration/channels/messaging/channel_read_after_permalink_spec.js +++ b/e2e-tests/cypress/tests/integration/channels/messaging/channel_read_after_permalink_spec.js @@ -18,7 +18,7 @@ describe('Messaging', () => { let testUser; let otherUser; - before(() => { + beforeEach(() => { cy.apiInitSetup().then(({team, channel, user}) => { testTeam = team; testChannel = channel; diff --git a/e2e-tests/cypress/tests/integration/channels/messaging/message_reply_bot_post_spec.js b/e2e-tests/cypress/tests/integration/channels/messaging/message_reply_bot_post_spec.js index 7311cbba3db..103b1ec6ebe 100644 --- a/e2e-tests/cypress/tests/integration/channels/messaging/message_reply_bot_post_spec.js +++ b/e2e-tests/cypress/tests/integration/channels/messaging/message_reply_bot_post_spec.js @@ -144,7 +144,7 @@ describe('Messaging', () => { cy.uiCloseRHS(); // * Verify that the reply is in the channel view with matching text - cy.get(`#post_${replyId}`).within(() => { + cy.get(`#post_${replyId}`).scrollIntoView().within(() => { cy.findByTestId('post-link').should('be.visible').and('have.text', 'Commented on ' + bot.username + 'BOT\'s message: Hello message from ' + bot.username); cy.get(`#postMessageText_${replyId}`).should('be.visible').and('have.text', replyMessage); }); diff --git a/e2e-tests/cypress/tests/integration/channels/notifications/unread_on_public_channel_spec.js b/e2e-tests/cypress/tests/integration/channels/notifications/unread_on_public_channel_spec.js index c505460ac93..151f7b4c136 100644 --- a/e2e-tests/cypress/tests/integration/channels/notifications/unread_on_public_channel_spec.js +++ b/e2e-tests/cypress/tests/integration/channels/notifications/unread_on_public_channel_spec.js @@ -7,7 +7,6 @@ // - Use element ID when selecting an element. Create one if none. // *************************************************************** -// Stage: @prod // Group: @channels @notifications import * as TIMEOUTS from '../../../fixtures/timeouts'; @@ -18,7 +17,7 @@ describe('Notifications', () => { let testTeam; beforeEach(() => { - cy.apiInitSetup().then(({team, user}) => { + cy.apiAdminLogin().apiInitSetup().then(({team, user}) => { testUser = user; testTeam = team; @@ -48,11 +47,6 @@ describe('Notifications', () => { // # Go to test channel cy.visit(`/${team.name}/channels/${channel.name}`); - // # Scroll above the last few messages - cy.get('div.post-list__dynamic', {timeout: TIMEOUTS.ONE_MIN}).should('be.visible'). - scrollTo(0, '70%', {duration: TIMEOUTS.ONE_SEC}). - wait(TIMEOUTS.ONE_SEC); - // # scroll to the last channel cy.get('#sidebar-left li').last().scrollIntoView(); }); @@ -66,7 +60,11 @@ describe('Notifications', () => { // # Get last post and reply a message cy.getLastPostId().then((postId) => { + // # Open the RHS reply form, then scroll above the last few messages cy.clickPostCommentIcon(postId); + cy.get('div.post-list__dynamic', {timeout: TIMEOUTS.ONE_MIN}).should('be.visible'). + scrollTo(0, '70%', {duration: TIMEOUTS.ONE_SEC}). + wait(TIMEOUTS.ONE_SEC); const replyMessage = 'A reply to an older post'; cy.postMessageReplyInRHS(replyMessage); }); diff --git a/e2e-tests/cypress/tests/support/ui_commands.ts b/e2e-tests/cypress/tests/support/ui_commands.ts index e9cf9286c6e..d6cf35744ae 100644 --- a/e2e-tests/cypress/tests/support/ui_commands.ts +++ b/e2e-tests/cypress/tests/support/ui_commands.ts @@ -98,7 +98,7 @@ function postMessageAndWait(textboxSelector: string, message: string, isComment waitForCommentDraft(message); } - cy.get(textboxSelector).should('have.value', message).type('{enter}').wait(TIMEOUTS.HALF_SEC); + cy.get(textboxSelector).should('have.value', message).focus().type('{enter}').wait(TIMEOUTS.HALF_SEC); cy.get(textboxSelector).invoke('val').then((value: string) => { if (value.length > 0 && value === message) { @@ -325,12 +325,12 @@ function clickPostHeaderItem(postId: string, location: string, item: string) { } if (postId) { - cy.get(`#${idPrefix}_${postId}`).trigger('mouseover', {force: true}); - cy.wait(TIMEOUTS.HALF_SEC).get(`#${location}_${item}_${postId}`).click({force: true}); + cy.get(`#${idPrefix}_${postId}`).trigger('mouseover', {force: true}). + get(`#${location}_${item}_${postId}`).scrollIntoView().trigger('mouseover', {force: true}).click({force: true}); } else { cy.getLastPostId().then((lastPostId) => { - cy.get(`#${idPrefix}_${lastPostId}`).trigger('mouseover', {force: true}); - cy.wait(TIMEOUTS.HALF_SEC).get(`#${location}_${item}_${lastPostId}`).click({force: true}); + cy.get(`#${idPrefix}_${lastPostId}`).trigger('mouseover', {force: true}). + get(`#${location}_${item}_${lastPostId}`).scrollIntoView().trigger('mouseover', {force: true}).click({force: true}); }); } }