feat(regression github): send failed tests names to chat (#1459)

* feat(regression) : send failed tests names

Signed-off-by: adi_holden <adi@dragonflydb.io>

.github/workflows/regression-tests.yml

@@ -31,7 +31,7 @@ jobs:
 
       - name: Configure & Build
         run: |
-          apt update && apt install -y pip
+          apt update && apt install -y pip jq
           cmake -B ${GITHUB_WORKSPACE}/build -DCMAKE_BUILD_TYPE=${{matrix.build-type}} -GNinja \
                 -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
 
@@ -49,21 +49,44 @@ jobs:
           export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/build/dragonfly" # used by PyTests
           export UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1 # to crash on errors
 
-          pytest -sxvr dragonfly --ignore=dragonfly/replication_test.py
+          pytest --json-report --json-report-file=report.json -svr dragonfly --ignore=dragonfly/replication_test.py
+
+
+
       - name: Run PyTests replication test
         if: ${{ matrix.runner  == 'ubuntu-latest' }}
         timeout-minutes: 20
         run: |
           cd ${GITHUB_WORKSPACE}/tests
           export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/build/dragonfly" # used by PyTests
-          pytest -xvs dragonfly/replication_test.py --df alsologtostderr --df enable_multi_shard_sync=true
-          pytest -xvs dragonfly/replication_test.py --df alsologtostderr --df enable_multi_shard_sync=false
+
+          pytest --json-report --json-report-file=rep1_report.json -sv dragonfly/replication_test.py --df alsologtostderr --df enable_multi_shard_sync=true
+          pytest --json-report --json-report-file=rep2_report.json -sv dragonfly/replication_test.py --df alsologtostderr --df enable_multi_shard_sync=false
+
+
 
       - name: Send notification on failure
         if: failure() || cancelled()
         run: |
+
+          get_failed_tests() {
+            local report_file=$1
+            echo $(jq -r '.tests[] | select(.outcome == "failed") | .nodeid' "$report_file")
+          }
+          cd ${GITHUB_WORKSPACE}/tests
+          failed_tests=""
+          # The order in of if is important, and expected to be the oposite order of the pytest run.
+          # As github runner will not run the next step if the pytest failed, we start from the last
+          # report file and if exist we get the failed test from the pytest run, if there are any.
+          if [ -f rep2_report.json ]; then
+            failed_tests=$(get_failed_tests rep2_report.json)
+          elif [ -f rep1_report.json ]; then
+            failed_tests=$(get_failed_tests rep1_report.json)
+          elif [ -f pytest_report.json ]; then
+            failed_tests=$(get_failed_tests pytest_report.json)
+          fi
           curl -s \
             -X POST \
             -H 'Content-Type: application/json' \
             '${{ secrets.GSPACES_BOT_DF_BUILD }}' \
-            -d '{"text": "Unforunately the tests of dragonfly build are not passed :( \n The commit is ${{github.sha}}	"}'
+            -d "{\"text\": \"Regression tests failed.\\n The commit is ${{github.sha}}.\\n $failed_tests\"}"

tests/dragonfly/replication_test.py

@@ -967,7 +967,8 @@ async def test_flushall_in_full_sync(df_local_factory, df_seeder_factory):
     c_master = aioredis.Redis(port=master.port)
 
     # Fill master with test data
-    seeder = df_seeder_factory.create(port=master.port, keys=100_000, dbcount=1)
+    seeder = df_seeder_factory.create(
+        port=master.port, keys=100_000, dbcount=1)
     await seeder.run(target_deviation=0.1)
 
     # Start replica
@@ -992,7 +993,8 @@ async def test_flushall_in_full_sync(df_local_factory, df_seeder_factory):
     await c_master.execute_command("FLUSHALL")
 
     if not await is_full_sync_mode(c_replica):
-        logging.error("!!! Full sync finished too fast. Adjust test parameters !!!")
+        logging.error(
+            "!!! Full sync finished too fast. Adjust test parameters !!!")
         return
 
     post_seeder = df_seeder_factory.create(

tests/dragonfly/requirements.txt

@@ -14,3 +14,4 @@ pytest-asyncio==0.20.1
 pytest-repeat==0.9.1
 pymemcache==4.0.0
 numpy==1.24.3
+pytest-json-report==1.5.0