e2e: improves flakiness and speed (#103533)

* e2e: improves flakiness * Chore: refactor flaky test * e2e: more refactor * e2e: do not keep successfull test run videos * e2e: do not log selectors * chore: updates after pr feedback * chore: updates after pr feedback * chore: adds retries to flaky e2e test * e2e: skip flaky tests * e2e: skip flaky tests * e2e: revert back to timeout * chore: removes all the skips
2025-12-18 22:16:21 -05:00 · 2025-04-16 06:21:06 +02:00 · 2025-04-16 06:21:06 +02:00 · 74487726fc
commit 74487726fc
parent 5d65063987
6 changed files with 145 additions and 82 deletions
--- a/cypress.config.js
+++ b/cypress.config.js
@ -13,6 +13,9 @@ module.exports = defineConfig({
  viewportWidth: 1920,
  viewportHeight: 1080,

+  env: {
+    LOG_SELECTORS_INFO: false,
+  },
  e2e: {
    supportFile: './e2e/cypress/support/e2e.js',
    setupNodeEvents(on, config) {
@ -82,6 +85,21 @@ module.exports = defineConfig({
        // IMPORTANT: return the updated browser launch options
        return launchOptions;
      });
+
+      on('after:spec', (_, results) => {
+        if (!results || !results.video || !results.tests) {
+          return;
+        }
+
+        // Do we have failures for any retry attempts?
+        const failures = results.tests.some((test) => test.attempts.some((attempt) => attempt.state === 'failed'));
+        if (failures) {
+          return;
+        }
+
+        // delete the video if the spec passed and no tests retried
+        fs.unlinkSync(results.video);
+      });
    },
  },
 });
--- a/e2e/old-arch/panels-suite/geomap-map-controls.spec.ts
+++ b/e2e/old-arch/panels-suite/geomap-map-controls.spec.ts
@ -1,5 +1,6 @@
 import { e2e } from '../utils';
 const DASHBOARD_ID = 'P2jR04WVk';
+const TIMEOUT = 45000;

 describe('Geomap layer controls options', () => {
  beforeEach(() => {
@ -8,51 +9,62 @@ describe('Geomap layer controls options', () => {

  it('Tests map controls options', () => {
    e2e.flows.openDashboard({ uid: DASHBOARD_ID, queryParams: { editPanel: 1 } });
-    e2e.components.OptionsGroup.group('Map controls').scrollIntoView().should('exist');
+    // Wait until the query editor has been loaded by ensuring that the QueryEditor select contains the text 'flight_info_by_state.csv'
+    e2e.components.Select.singleValue().contains('flight_info_by_state.csv').should('be.visible');
+    e2e.components.OptionsGroup.group('Map controls').scrollIntoView().should('be.visible');

-    // Show zoom
+    // Show zoom field
    e2e.components.PanelEditor.showZoomField()
-      .should('exist')
+      .should('be.visible')
      .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
      });

-    cy.contains('+');
-    cy.get('.ol-zoom').should('exist');
-
    // Show attribution
    e2e.components.PanelEditor.showAttributionField()
-      .should('exist')
+      .should('be.visible')
      .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
      });

-    cy.get('.ol-attribution').should('exist');
-
    // Show scale
    e2e.components.PanelEditor.showScaleField()
-      .should('exist')
+      .should('be.visible')
      .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
      });

-    cy.get('.ol-scale-line').should('exist');
-
    // Show measure tool
    e2e.components.PanelEditor.showMeasureField()
-      .should('exist')
+      .should('be.visible')
      .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
      });

-    e2e.components.PanelEditor.measureButton().should('exist');
-
    // Show debug
    e2e.components.PanelEditor.showDebugField()
-      .should('exist')
+      .should('be.visible')
      .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
+      });
+
+    e2e.components.Panels.Panel.content({ timeout: TIMEOUT })
+      .should('be.visible')
+      .within(() => {
+        // Verify zoom
+        cy.get('.ol-zoom', { timeout: TIMEOUT }).should('be.visible');
+
+        // Verify attribution
+        cy.get('.ol-attribution', { timeout: TIMEOUT }).should('be.visible');
+
+        // Verify scale
+        cy.get('.ol-scale-line', { timeout: TIMEOUT }).should('be.visible');
+
+        // Verify measure tool
+        e2e.components.PanelEditor.measureButton({ timeout: TIMEOUT }).should('be.visible');
+
+        // Verify debug tool
+        e2e.components.DebugOverlay.wrapper({ timeout: TIMEOUT }).should('be.visible');
      });
-    e2e.components.DebugOverlay.wrapper().should('exist');
  });
 });
--- a/e2e/old-arch/utils/support/types.ts
+++ b/e2e/old-arch/utils/support/types.ts
@ -32,7 +32,9 @@ export type E2EFactoryArgs<S extends Selectors> = { selectors: S };
 export type CypressOptions = Partial<Cypress.Loggable & Cypress.Timeoutable & Cypress.Withinable & Cypress.Shadow>;

 const processSelectors = <S extends Selectors>(e2eObjects: E2EFunctions<S>, selectors: S): E2EFunctions<S> => {
-  const logOutput = (data: unknown) => cy.logToConsole('Retrieving Selector:', data);
+  const logSelectorsInfo = Boolean(Cypress.env('LOG_SELECTORS_INFO'));
+  const logOutput = logSelectorsInfo ? (data: unknown) => cy.logToConsole('Retrieving Selector:', data) : () => {};
+
  const keys = Object.keys(selectors);
  for (let index = 0; index < keys.length; index++) {
    const key = keys[index];
--- a/e2e/panels-suite/geomap-map-controls.spec.ts
+++ b/e2e/panels-suite/geomap-map-controls.spec.ts
@ -1,5 +1,6 @@
 import { e2e } from '../utils';
 const DASHBOARD_ID = 'P2jR04WVk';
+const TIMEOUT = 45000;

 describe('Geomap layer controls options', () => {
  beforeEach(() => {
@ -8,51 +9,62 @@ describe('Geomap layer controls options', () => {

  it('Tests map controls options', () => {
    e2e.flows.openDashboard({ uid: DASHBOARD_ID, queryParams: { editPanel: 1 } });
-    e2e.components.OptionsGroup.group('Map controls').scrollIntoView().should('exist');
+    // Wait until the query editor has been loaded by ensuring that the QueryEditor select contains the text 'flight_info_by_state.csv'
+    e2e.components.Select.singleValue().contains('flight_info_by_state.csv').should('be.visible');
+    e2e.components.OptionsGroup.group('Map controls').scrollIntoView().should('be.visible');

-    // Show zoom
+    // Show zoom field
    e2e.components.PanelEditor.showZoomField()
-      .should('exist')
+      .should('be.visible')
      .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
      });

-    cy.contains('+');
-    cy.get('.ol-zoom').should('exist');
-
    // Show attribution
    e2e.components.PanelEditor.showAttributionField()
-      .should('exist')
+      .should('be.visible')
      .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
      });

-    cy.get('.ol-attribution').should('exist');
-
    // Show scale
    e2e.components.PanelEditor.showScaleField()
-      .should('exist')
+      .should('be.visible')
      .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
      });

-    cy.get('.ol-scale-line').should('exist');
-
    // Show measure tool
    e2e.components.PanelEditor.showMeasureField()
-      .should('exist')
+      .should('be.visible')
      .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
      });

-    e2e.components.PanelEditor.measureButton().should('exist');
-
    // Show debug
    e2e.components.PanelEditor.showDebugField()
-      .should('exist')
+      .should('be.visible')
      .within(() => {
-        cy.get('input[type="checkbox"]').check({ force: true });
+        cy.get('input[type="checkbox"]').check({ force: true }).should('be.checked');
+      });
+
+    e2e.components.Panels.Panel.content({ timeout: TIMEOUT })
+      .should('be.visible')
+      .within(() => {
+        // Verify zoom
+        cy.get('.ol-zoom', { timeout: TIMEOUT }).should('be.visible');
+
+        // Verify attribution
+        cy.get('.ol-attribution', { timeout: TIMEOUT }).should('be.visible');
+
+        // Verify scale
+        cy.get('.ol-scale-line', { timeout: TIMEOUT }).should('be.visible');
+
+        // Verify measure tool
+        e2e.components.PanelEditor.measureButton({ timeout: TIMEOUT }).should('be.visible');
+
+        // Verify debug tool
+        e2e.components.DebugOverlay.wrapper({ timeout: TIMEOUT }).should('be.visible');
      });
-    e2e.components.DebugOverlay.wrapper().should('exist');
  });
 });
--- a/e2e/utils/support/types.ts
+++ b/e2e/utils/support/types.ts
@ -32,7 +32,9 @@ export type E2EFactoryArgs<S extends Selectors> = { selectors: S };
 export type CypressOptions = Partial<Cypress.Loggable & Cypress.Timeoutable & Cypress.Withinable & Cypress.Shadow>;

 const processSelectors = <S extends Selectors>(e2eObjects: E2EFunctions<S>, selectors: S): E2EFunctions<S> => {
-  const logOutput = (data: unknown) => cy.logToConsole('Retrieving Selector:', data);
+  const logSelectorsInfo = Boolean(Cypress.env('LOG_SELECTORS_INFO'));
+  const logOutput = logSelectorsInfo ? (data: unknown) => cy.logToConsole('Retrieving Selector:', data) : () => {};
+
  const keys = Object.keys(selectors);
  for (let index = 0; index < keys.length; index++) {
    const key = keys[index];
--- a/e2e/various-suite/bookmarks.spec.ts
+++ b/e2e/various-suite/bookmarks.spec.ts
@ -3,56 +3,73 @@ import { fromBaseUrl } from '../utils/support/url';

 describe('Pin nav items', () => {
  beforeEach(() => {
-    cy.viewport(1280, 800);
    e2e.flows.login(Cypress.env('USERNAME'), Cypress.env('PASSWORD'));
-    cy.visit(fromBaseUrl('/'));
  });
+
  afterEach(() => {
    e2e.flows.setDefaultUserPreferences();
  });

  it('should pin the selected menu item and add it as a Bookmarks menu item child', () => {
-    // Check if the mega menu is visible
-    e2e.components.NavMenu.Menu().should('be.visible');
-
-    // Check if the Bookmark section is visible
-    const bookmarkSection = cy.get('[href="/bookmarks"]');
-    bookmarkSection.should('be.visible');
-
-    // Click on the pin icon to add Administration to the Bookmarks section
-    const adminItem = cy.contains('a', 'Administration');
-    const bookmarkPinIcon = adminItem.siblings('button').should('have.attr', 'aria-label', 'Add to Bookmarks');
-    bookmarkPinIcon.click({ force: true });
-
-    // Check if the Administration menu item is visible in the Bookmarks section
-    cy.get('[aria-label="Expand section: Bookmarks"]').click();
-    const bookmarks = cy.get('[href="/bookmarks"]').parentsUntil('li').siblings('ul');
-    bookmarks.within(() => {
-      cy.get('a').should('contain.text', 'Administration');
+    cy.visit(fromBaseUrl('/'), {
+      onBeforeLoad: (win) => {
+        win.localStorage.setItem('grafana.navigation.docked', 'true'); // Make sure the menu is docked
+      },
    });
+
+    e2e.components.NavMenu.Menu()
+      .should('be.visible')
+      .within(() => {
+        cy.get('ul[aria-label="Navigation"]').should('be.visible').as('navList');
+
+        // Check if the Bookmark section is visible
+        cy.get('@navList').children().eq(1).should('be.visible').as('bookmarksItem');
+        cy.get('@bookmarksItem').should('contain.text', 'Bookmarks');
+
+        // Check if the Adminstration section is visible
+        cy.get('@navList').children().last().should('be.visible').as('adminItem');
+        cy.get('@adminItem').should('contain.text', 'Administration');
+        cy.get('@adminItem').within(() => {
+          cy.get('button[aria-label="Add to Bookmarks"]').should('exist').click({ force: true });
+        });
+
+        // Check if the Administration menu item is visible in the Bookmarks section
+        cy.get('@bookmarksItem').within(() => {
+          // Expand the Bookmarks section
+          cy.get('button[aria-label="Expand section: Bookmarks"]').should('exist').click({ force: true });
+          cy.get('a').should('contain.text', 'Administration').should('be.visible');
+        });
+      });
  });

  it('should unpin the item and remove it from the Bookmarks section', () => {
    // Set Administration as a pinned item and reload the page
    e2e.flows.setUserPreferences({ navbar: { bookmarkUrls: ['/admin'] } });
-    cy.reload();

-    // Check if the mega menu is visible
-    e2e.components.NavMenu.Menu().should('be.visible');
-
-    // Check if the Bookmark section is visible and open it
-    cy.get('[href="/bookmarks"]').should('be.visible');
-    cy.get('[aria-label="Expand section: Bookmarks"]').click();
-
-    // Check if the Administration menu item is visible in the Bookmarks section
-    const bookmarks = cy.get('[href="/bookmarks"]').parentsUntil('li').siblings('ul').children();
-    const administrationIsPinned = bookmarks.filter('li').children().should('contain.text', 'Administration');
-
-    // Click on the pin icon to remove Administration from the Bookmarks section and check if it is removed
-    administrationIsPinned.within(() => {
-      cy.get('[aria-label="Remove from Bookmarks"]').click({ force: true });
+    cy.visit(fromBaseUrl('/'), {
+      onBeforeLoad: (win) => {
+        win.localStorage.setItem('grafana.navigation.docked', 'true'); // Make sure the menu is docked
+      },
    });
-    cy.wait(500);
-    administrationIsPinned.should('not.exist');
+
+    e2e.components.NavMenu.Menu()
+      .should('be.visible')
+      .within(() => {
+        cy.get('ul[aria-label="Navigation"]').should('be.visible').as('navList');
+
+        // Check if the Bookmark section is visible
+        cy.get('@navList').children().eq(1).should('be.visible').as('bookmarksItem');
+        cy.get('@bookmarksItem').should('contain.text', 'Bookmarks');
+        cy.get('@bookmarksItem').within(() => {
+          // Expand the Bookmarks section
+          cy.get('button[aria-label="Expand section: Bookmarks"]').should('exist').click({ force: true });
+          cy.get('a').should('contain.text', 'Administration').should('be.visible');
+          cy.get('button[aria-label="Remove from Bookmarks"]').should('exist').click({ force: true });
+        });
+
+        cy.get('@bookmarksItem', { timeout: 60000 }).within(() => {
+          cy.get('a').should('have.length', 1).should('not.contain.text', 'Administration');
+        });
+      });
  });
 });