fix(regression-test): Sentinel test stabilization (#826)

* Ditch docker whcih is complex on CI in favour of local redis binary

Signed-off-by: ashotland <ari@dragonflydb.io>

* Fix typo

Signed-off-by: ashotland <ari@dragonflydb.io>

* Wait for sentinel termination

Signed-off-by: ashotland <ari@dragonflydb.io>

* fix(regression-tests): sentinel test increase timeout waiting for key to
exist in replica

* debug sentinel test

Signed-off-by: ashotland <ari@dragonflydb.io>

* add pytest repeat - tmp comment out failure notficicaiton

Signed-off-by: ashotland <ari@dragonflydb.io>

* fix typo

Signed-off-by: ashotland <ari@dragonflydb.io>

* repeat 100

Signed-off-by: ashotland <ari@dragonflydb.io>

* Increase timeout for debuging

Signed-off-by: ashotland <ari@dragonflydb.io>

* Debug prints

Signed-off-by: ashotland <ari@dragonflydb.io>

* fix

Signed-off-by: ashotland <ari@dragonflydb.io>

* increase overall timeout

Signed-off-by: ashotland <ari@dragonflydb.io>

* Debug

Signed-off-by: ashotland <ari@dragonflydb.io>

* cleanup

Signed-off-by: ashotland <ari@dragonflydb.io>

---------

Signed-off-by: ashotland <ari@dragonflydb.io>

.github/workflows/regression-tests.yml

@@ -53,7 +53,6 @@ jobs:
           export DRAGONFLY_PATH="${GITHUB_WORKSPACE}/build/dragonfly" # used by PyTests
 
           pytest -sxvr dragonfly --ignore=dragonfly/replication_test.py
-
       - name: Run PyTests replication test
         timeout-minutes: 15
         if:  ${{ inputs.run_replication }}

tests/dragonfly/requirements.txt

@@ -12,4 +12,4 @@ tomli==2.0.1
 wrapt==1.14.1
 aioredis==2.0.1
 pytest-asyncio==0.20.1
-
+pytest-repeat==0.9.1

tests/dragonfly/sentinel_test.py

@@ -127,6 +127,7 @@ async def test_failover(df_local_factory, sentinel):
 
     master_client = aioredis.Redis(port=master.port)
     replica_client = aioredis.Redis(port=replica.port)
+    print("master: " + str(master.port) + " replica: " + str(replica.port), flush=True)
 
     await replica_client.execute_command("REPLICAOF localhost " + str(master.port))
 
@@ -149,12 +150,27 @@ async def test_failover(df_local_factory, sentinel):
     assert sentinel.slaves()[0]["port"] == str(master.port)
 
     # Verify we can now write to replica and read replicated value from master.
-    await replica_client.set("key", "value")
-    await await_for(
-        lambda: master_client.get("key"),
-        lambda val: val == b"value",
-        10, "Timeout waiting for key to exist in replica."
-    )
+    assert await replica_client.set("key", "value"), "Failed to set key promoted replica."
+    try:
+        await await_for(
+            lambda: master_client.get("key"),
+            lambda val: val == b"value",
+            10, "Timeout waiting for key to exist in replica."
+        )
+    except AssertionError:
+        syncid, r_offset = await master_client.execute_command("DEBUG REPLICA OFFSET")
+        replicaoffset_cmd = "DFLY REPLICAOFFSET " + syncid.decode()
+        m_offset = await replica_client.execute_command(replicaoffset_cmd)
+        print(syncid.decode(),  r_offset, m_offset)
+        print("replica client role:")
+        print(await replica_client.execute_command("role"))
+        print("master client role:")
+        print(await master_client.execute_command("role"))
+        print("replica client info:")
+        print(await replica_client.info())
+        print("master client info:")
+        print(await master_client.info())
+        raise
 
 
 @pytest.mark.asyncio