From afdfd12bdff919ab230b394ba4e2e486f1652e37 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Nov 2015 16:57:06 +0000 Subject: [PATCH 1/7] Implement required method 'resumeProducing' --- synapse/http/matrixfederationclient.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index ca9591556d..5d80e0c4da 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -491,6 +491,9 @@ class _JsonProducer(object): def stopProducing(self): pass + def resumeProducing(self): + pass + def _flatten_response_never_received(e): if hasattr(e, "reasons"): From cf4ef5f3c7ae268a2a28c0e59eb738d6695ca865 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Nov 2015 18:26:50 +0000 Subject: [PATCH 2/7] Only retry federation requests for a long time for background requests --- synapse/federation/transport/client.py | 1 + synapse/http/matrixfederationclient.py | 26 +++++++++++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 3d59e1c650..0e0cb7ebc6 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -136,6 +136,7 @@ class TransportLayerClient(object): path=PREFIX + "/send/%s/" % transaction.transaction_id, data=json_data, json_data_callback=json_data_callback, + long_retries=True, ) logger.debug( diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 5d80e0c4da..e09a0bbe18 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -56,7 +56,8 @@ incoming_responses_counter = metrics.register_counter( ) -MAX_RETRIES = 10 +MAX_LONG_RETRIES = 10 +MAX_SHORT_RETRIES = 3 class MatrixFederationEndpointFactory(object): @@ -103,7 +104,7 @@ class MatrixFederationHttpClient(object): def _create_request(self, destination, method, path_bytes, body_callback, headers_dict={}, param_bytes=b"", query_bytes=b"", retry_on_dns_fail=True, - timeout=None): + timeout=None, long_retries=False): """ Creates and sends a request to the given url """ headers_dict[b"User-Agent"] = [self.version_string] @@ -123,7 +124,10 @@ class MatrixFederationHttpClient(object): # XXX: Would be much nicer to retry only at the transaction-layer # (once we have reliable transactions in place) - retries_left = MAX_RETRIES + if long_retries: + retries_left = MAX_LONG_RETRIES + else: + retries_left = MAX_SHORT_RETRIES http_url_bytes = urlparse.urlunparse( ("", "", path_bytes, param_bytes, query_bytes, "") @@ -184,9 +188,15 @@ class MatrixFederationHttpClient(object): ) if retries_left and not timeout: - delay = 4 ** (MAX_RETRIES + 1 - retries_left) - delay = max(delay, 60) - delay *= random.uniform(0.8, 1.4) + if long_retries: + delay = 4 ** (MAX_LONG_RETRIES + 1 - retries_left) + delay = max(delay, 60) + delay *= random.uniform(0.8, 1.4) + else: + delay = 0.5 * 2 ** (MAX_SHORT_RETRIES - retries_left) + delay = max(delay, 2) + delay *= random.uniform(0.8, 1.4) + yield sleep(delay) retries_left -= 1 else: @@ -237,7 +247,8 @@ class MatrixFederationHttpClient(object): headers_dict[b"Authorization"] = auth_headers @defer.inlineCallbacks - def put_json(self, destination, path, data={}, json_data_callback=None): + def put_json(self, destination, path, data={}, json_data_callback=None, + long_retries=False): """ Sends the specifed json data using PUT Args: @@ -273,6 +284,7 @@ class MatrixFederationHttpClient(object): path.encode("ascii"), body_callback=body_callback, headers_dict={"Content-Type": ["application/json"]}, + long_retries=long_retries, ) if 200 <= response.code < 300: From cbf3cd61515d101d57124388d629162f4983783a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Nov 2015 18:29:29 +0000 Subject: [PATCH 3/7] Add comment --- synapse/http/matrixfederationclient.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index e09a0bbe18..614c06a6d7 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -259,6 +259,8 @@ class MatrixFederationHttpClient(object): the request body. This will be encoded as JSON. json_data_callback (callable): A callable returning the dict to use as the request body. + long_retries (bool): A boolean that indicates whether we should + retry for a short or long time. Returns: Deferred: Succeeds when we get a 2xx HTTP response. The result From bd3de8f39aa5d5095273b6ece1257d66ae4b5445 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Nov 2015 18:38:48 +0000 Subject: [PATCH 4/7] Update tests --- tests/federation/test_federation.py | 2 ++ tests/handlers/test_presence.py | 13 +++++++++++++ tests/handlers/test_typing.py | 2 ++ 3 files changed, 17 insertions(+) diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py index a4ef60b911..96570f9072 100644 --- a/tests/federation/test_federation.py +++ b/tests/federation/test_federation.py @@ -197,6 +197,7 @@ class FederationTestCase(unittest.TestCase): 'pdu_failures': [], }, json_data_callback=ANY, + long_retries=True, ) @defer.inlineCallbacks @@ -228,6 +229,7 @@ class FederationTestCase(unittest.TestCase): 'pdu_failures': [], }, json_data_callback=ANY, + long_retries=True, ) @defer.inlineCallbacks diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index 10d4482cce..1172ceae8b 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -409,6 +409,7 @@ class PresenceInvitesTestCase(PresenceTestCase): } ), json_data_callback=ANY, + long_retries=True, ), defer.succeed((200, "OK")) ) @@ -443,6 +444,7 @@ class PresenceInvitesTestCase(PresenceTestCase): } ), json_data_callback=ANY, + long_retries=True, ), defer.succeed((200, "OK")) ) @@ -483,6 +485,7 @@ class PresenceInvitesTestCase(PresenceTestCase): } ), json_data_callback=ANY, + long_retries=True, ), defer.succeed((200, "OK")) ) @@ -827,6 +830,7 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase): } ), json_data_callback=ANY, + long_retries=True, ), defer.succeed((200, "OK")) ) @@ -843,6 +847,7 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase): } ), json_data_callback=ANY, + long_retries=True, ), defer.succeed((200, "OK")) ) @@ -1033,6 +1038,7 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase): } ), json_data_callback=ANY, + long_retries=True, ), defer.succeed((200, "OK")) ) @@ -1048,6 +1054,7 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase): } ), json_data_callback=ANY, + long_retries=True, ), defer.succeed((200, "OK")) ) @@ -1078,6 +1085,7 @@ class PresencePushTestCase(MockedDatastorePresenceTestCase): } ), json_data_callback=ANY, + long_retries=True, ), defer.succeed((200, "OK")) ) @@ -1184,6 +1192,7 @@ class PresencePollingTestCase(MockedDatastorePresenceTestCase): }, ), json_data_callback=ANY, + long_retries=True, ), defer.succeed((200, "OK")) ) @@ -1200,6 +1209,7 @@ class PresencePollingTestCase(MockedDatastorePresenceTestCase): }, ), json_data_callback=ANY, + long_retries=True, ), defer.succeed((200, "OK")) ) @@ -1232,6 +1242,7 @@ class PresencePollingTestCase(MockedDatastorePresenceTestCase): }, ), json_data_callback=ANY, + long_retries=True, ), defer.succeed((200, "OK")) ) @@ -1265,6 +1276,7 @@ class PresencePollingTestCase(MockedDatastorePresenceTestCase): }, ), json_data_callback=ANY, + long_retries=True, ), defer.succeed((200, "OK")) ) @@ -1297,6 +1309,7 @@ class PresencePollingTestCase(MockedDatastorePresenceTestCase): }, ), json_data_callback=ANY, + long_retries=True, ), defer.succeed((200, "OK")) ) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 2d7ba43561..5b1feeb45b 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -218,6 +218,7 @@ class TypingNotificationsTestCase(unittest.TestCase): } ), json_data_callback=ANY, + long_retries=True, ), defer.succeed((200, "OK")) ) @@ -284,6 +285,7 @@ class TypingNotificationsTestCase(unittest.TestCase): } ), json_data_callback=ANY, + long_retries=True, ), defer.succeed((200, "OK")) ) From cf844e2ad64a7b71d45c467f33cef7b3c1a76d70 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 17 Nov 2015 18:48:32 +0000 Subject: [PATCH 5/7] Bump changelog and version --- CHANGES.rst | 6 ++++++ synapse/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 068ed9f2ff..b07d47c42d 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +Changes in synapse v0.11.0-r1 (2015-11-17) +========================================== + +* Retry more aggressively, but fewer times for federation requests that aren't + sending transactions. + Changes in synapse v0.11.0 (2015-11-17) ======================================= diff --git a/synapse/__init__.py b/synapse/__init__.py index f0eac97bab..5a83f5f3ec 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -16,4 +16,4 @@ """ This is a reference implementation of a Matrix home server. """ -__version__ = "0.11.0" +__version__ = "0.11.0-r1" From fe51b3628e46afbcbb05a774027465a48dcec9b8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Nov 2015 11:31:05 +0000 Subject: [PATCH 6/7] Better change log --- CHANGES.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b07d47c42d..749654c000 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,8 +1,8 @@ Changes in synapse v0.11.0-r1 (2015-11-17) ========================================== -* Retry more aggressively, but fewer times for federation requests that aren't - sending transactions. +* Retry and fail federation requests more aggressively for requests that block + client side requests (PR #384) Changes in synapse v0.11.0 (2015-11-17) ======================================= From fbb76a4d5d2c523465f44662c689173b90d38f3b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 18 Nov 2015 14:15:03 +0000 Subject: [PATCH 7/7] Change date --- CHANGES.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 749654c000..5bea8e82d5 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,4 +1,4 @@ -Changes in synapse v0.11.0-r1 (2015-11-17) +Changes in synapse v0.11.0-r1 (2015-11-18) ========================================== * Retry and fail federation requests more aggressively for requests that block