fix: ci: Ensure that junit.xml is present and non-empty after each system/unit test job

Previously, JUnit files were not generated or were generated empty for various reasons for some system/unit test runs.

Now, the number of tests collected for a MR is up from about 4k to 5.8k in the "Tests" tab of a pipeline.

Additionally, there is a check that ensures that [a somewhat sane](c5a271eb8b) `junit.xml` file is generated after every system/unit test job and fails the job otherwise.

Closes #5316

Merge branch '5316-ensure-junit-xml' into 'main'

See merge request isc-projects/bind9!10556

.gitlab-ci.yml

@@ -311,6 +311,21 @@ stages:
 .fips-feature-test: &fips_feature_test
     - if build/feature-test --have-fips-mode; then fips-mode-setup --check; fips-mode-setup --is-enabled; fi
 
+.check_for_junit_xml: &check_for_junit_xml
+    # test if junit.xml file exists and is longer 40 bytes
+    # (i.e., contains more than `<testsuites><testsuite /></testsuites>`)
+    - if [ -f "$CI_PROJECT_DIR"/junit.xml ]; then
+        if [ $(wc -c < "$CI_PROJECT_DIR"/junit.xml) -gt 40 ]; then
+          echo "junit.xml file exists and is longer than 40 bytes.";
+        else
+          echo "junit.xml file exists but is too short.";
+          exit 1;
+        fi
+      else
+        echo "junit.xml file does not exist.";
+        exit 1;
+      fi
+
 .build: &build_job
   <<: *default_triggering_rules
   stage: build
@@ -389,26 +404,32 @@ stages:
       artifacts: true
   timeout: 2h
 
-.system_test_common: &system_test_common
+.system_test_common: &system_test_job
   <<: *default_triggering_rules
   stage: system
   before_script:
     - *setup_interfaces
+  # This script needs to: 1) fail if the system tests fail, 2) fail if
+  # the junit.xml file is broken, 3) produce the junit.xml file even if
+  # the system tests fail.  Therefore, $RET is used to "cache" the
+  # result of running pytest as interrupting the script immediately when
+  # system tests fail would make checking the contents of the junit.xml
+  # file impossible (GitLab Runner uses "set -o pipefail").
   script:
     - *fips_feature_test
     - *find_pytest
     - *find_python
     - ( if [ "${CI_DISPOSABLE_ENVIRONMENT}" = "true" ]; then sleep 3000; "$PYTHON" "${CI_PROJECT_DIR}/util/get-running-system-tests.py"; fi ) &
     - cd bin/tests/system
+    - RET=0
     - >
-      "$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt
+      ("$PYTEST" --junit-xml="$CI_PROJECT_DIR"/junit.xml -n "$TEST_PARALLEL_JOBS" | tee pytest.out.txt) || RET=1
+    - *check_for_junit_xml
+    - (exit $RET)
     - '( ! grep -F "grep: warning:" pytest.out.txt )'
     - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ( cd ../../.. && ninja -C build clean >/dev/null 2>&1 )
   after_script:
     - *display_pytest_failures
-
-.system_test: &system_test_job
-  <<: *system_test_common
   artifacts:
     untracked: true
     exclude:
@@ -417,61 +438,40 @@ stages:
     reports:
       junit: junit.xml
 
-.system_test_gcov: &system_test_gcov_job
-  <<: *system_test_common
-  artifacts:
-    untracked: true
-    exclude:
-      - "**/__pycache__/**/*"
-    when: always
-
 .system_test_tsan: &system_test_tsan_job
-  <<: *system_test_common
+  <<: *system_test_job
   after_script:
     - *display_pytest_failures
     - *parse_tsan
-  artifacts:
-    untracked: true
-    exclude:
-      - "**/__pycache__/**/*"
-    when: always
-    reports:
-       junit: junit.xml
 
-.unit_test_common: &unit_test_common
+.unit_test_common: &unit_test_job
   <<: *default_triggering_rules
   stage: unit
+  # This script needs to: 1) fail if the unit tests fail, 2) fail if the
+  # junit.xml file is broken, 3) produce the junit.xml file even if the
+  # unit tests fail.  Therefore, $RET is used to "cache" the result of
+  # running "meson test" as interrupting the script immediately when
+  # unit tests fail would make checking the contents of the junit.xml
+  # file impossible (GitLab Runner uses "set -o pipefail").
   script:
     - *fips_feature_test
-    - meson test -C build --no-rebuild
-  after_script:
+    - RET=0
+    - meson test -C build --no-rebuild || RET=1
     - cp build/meson-logs/testlog.junit.xml $CI_PROJECT_DIR/junit.xml
+    - *check_for_junit_xml
+    - (exit $RET)
     - test "$CLEAN_BUILD_ARTIFACTS_ON_SUCCESS" -eq 0 || ninja -C build clean >/dev/null 2>&1
-
-.unit_test: &unit_test_job
-  <<: *unit_test_common
   artifacts:
     untracked: true
     when: always
     reports:
       junit: junit.xml
 
-.unit_test_gcov: &unit_test_gcov_job
-  <<: *unit_test_common
-  artifacts:
-    untracked: true
-    when: always
-
 .unit_test_tsan: &unit_test_tsan_job
-  <<: *unit_test_common
+  <<: *unit_test_job
   after_script:
     - *find_python
     - *parse_tsan
-  artifacts:
-    untracked: true
-    when: always
-    reports:
-      junit: junit.xml
 
 .docs: &docs_job
   stage: docs
@@ -912,7 +912,7 @@ gcc:bookworm:amd64:
 
 system:gcc:bookworm:amd64:
   <<: *debian_bookworm_amd64_image
-  <<: *system_test_gcov_job
+  <<: *system_test_job
   variables:
     CI_ENABLE_ALL_TESTS: 1
     CLEAN_BUILD_ARTIFACTS_ON_SUCCESS: 0
@@ -923,7 +923,7 @@ system:gcc:bookworm:amd64:
 
 unit:gcc:bookworm:amd64:
   <<: *debian_bookworm_amd64_image
-  <<: *unit_test_gcov_job
+  <<: *unit_test_job
   variables:
     CI_ENABLE_ALL_TESTS: 1
     CLEAN_BUILD_ARTIFACTS_ON_SUCCESS: 0