From 0c02d10c4d4f0bbe10997d4af2c18d2a89c926ca Mon Sep 17 00:00:00 2001 From: squahtx Date: Wed, 18 May 2022 16:56:55 +0000 Subject: [PATCH] deploy: 3d8839c30c96b49588196c60e2bbf056ed6465eb --- develop/404.html | 2 +- develop/CAPTCHA_SETUP.html | 2 +- develop/admin_api/account_validity.html | 2 +- develop/admin_api/delete_group.html | 2 +- develop/admin_api/event_reports.html | 2 +- develop/admin_api/media_admin_api.html | 2 +- develop/admin_api/purge_history_api.html | 2 +- develop/admin_api/register_api.html | 2 +- develop/admin_api/room_membership.html | 2 +- develop/admin_api/rooms.html | 2 +- develop/admin_api/server_notices.html | 2 +- develop/admin_api/statistics.html | 2 +- develop/admin_api/user_admin_api.html | 2 +- develop/admin_api/version_api.html | 2 +- develop/application_services.html | 2 +- develop/auth_chain_difference_algorithm.html | 2 +- develop/code_style.html | 2 +- develop/consent_tracking.html | 2 +- develop/delegate.html | 2 +- develop/deprecation_policy.html | 2 +- develop/development/cas.html | 2 +- develop/development/contributing_guide.html | 2 +- develop/development/database_schema.html | 2 +- develop/development/demo.html | 2 +- .../development/experimental_features.html | 6 +- develop/development/git.html | 2 +- .../internal_documentation/index.html | 2 +- develop/development/releases.html | 2 +- develop/development/room-dag-concepts.html | 2 +- develop/development/saml.html | 2 +- .../synapse_architecture/cancellation.html | 536 ++++++++++++++++++ develop/development/url_previews.html | 2 +- develop/federate.html | 2 +- develop/index.html | 2 +- develop/jwt.html | 2 +- develop/log_contexts.html | 6 +- develop/manhole.html | 2 +- develop/media_repository.html | 2 +- develop/message_retention_policies.html | 2 +- develop/metrics-howto.html | 2 +- develop/modules/account_data_callbacks.html | 2 +- .../modules/account_validity_callbacks.html | 2 +- ...ackground_update_controller_callbacks.html | 2 +- develop/modules/index.html | 2 +- .../password_auth_provider_callbacks.html | 2 +- develop/modules/porting_legacy_module.html | 2 +- .../modules/presence_router_callbacks.html | 2 +- develop/modules/spam_checker_callbacks.html | 2 +- .../modules/third_party_rules_callbacks.html | 2 +- develop/modules/writing_a_module.html | 2 +- develop/openid.html | 2 +- develop/opentracing.html | 2 +- ...ing_synapse_on_single_board_computers.html | 2 +- develop/password_auth_providers.html | 2 +- develop/postgres.html | 2 +- develop/print.html | 349 +++++++++++- develop/replication.html | 2 +- develop/reverse_proxy.html | 2 +- develop/room_and_user_statistics.html | 2 +- develop/searchindex.js | 2 +- develop/searchindex.json | 2 +- develop/server_notices.html | 2 +- develop/setup/forward_proxy.html | 2 +- develop/setup/installation.html | 2 +- develop/sso_mapping_providers.html | 2 +- develop/structured_logging.html | 2 +- develop/synctl_workers.html | 2 +- develop/systemd-with-workers/index.html | 2 +- develop/tcp_replication.html | 2 +- develop/templates.html | 2 +- develop/turn-howto.html | 2 +- develop/upgrade.html | 2 +- .../admin_api/background_updates.html | 2 +- .../administration/admin_api/federation.html | 2 +- .../usage/administration/admin_api/index.html | 2 +- .../admin_api/registration_tokens.html | 2 +- develop/usage/administration/admin_faq.html | 2 +- .../database_maintenance_tools.html | 2 +- develop/usage/administration/index.html | 2 +- develop/usage/administration/request_log.html | 2 +- .../usage/administration/state_groups.html | 2 +- ...anding_synapse_through_grafana_graphs.html | 2 +- .../administration/useful_sql_for_admins.html | 2 +- .../configuration/config_documentation.html | 2 +- .../homeserver_sample_config.html | 2 +- develop/usage/configuration/index.html | 2 +- .../configuration/logging_sample_config.html | 2 +- .../user_authentication/index.html | 2 +- .../user_authentication/refresh_tokens.html | 2 +- .../single_sign_on/cas.html | 2 +- .../single_sign_on/index.html | 2 +- .../single_sign_on/saml.html | 2 +- develop/user_directory.html | 2 +- develop/welcome_and_overview.html | 2 +- develop/workers.html | 2 +- 95 files changed, 981 insertions(+), 98 deletions(-) create mode 100644 develop/development/synapse_architecture/cancellation.html diff --git a/develop/404.html b/develop/404.html index 58f0d0a8fa..8cd53682c5 100644 --- a/develop/404.html +++ b/develop/404.html @@ -77,7 +77,7 @@ diff --git a/develop/CAPTCHA_SETUP.html b/develop/CAPTCHA_SETUP.html index ea9aec6772..534cbfd1aa 100644 --- a/develop/CAPTCHA_SETUP.html +++ b/develop/CAPTCHA_SETUP.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/account_validity.html b/develop/admin_api/account_validity.html index eb4542f392..ef27129291 100644 --- a/develop/admin_api/account_validity.html +++ b/develop/admin_api/account_validity.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/delete_group.html b/develop/admin_api/delete_group.html index affc8beab4..9eb9f026cb 100644 --- a/develop/admin_api/delete_group.html +++ b/develop/admin_api/delete_group.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/event_reports.html b/develop/admin_api/event_reports.html index 488a3fca01..9dca2b0070 100644 --- a/develop/admin_api/event_reports.html +++ b/develop/admin_api/event_reports.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/media_admin_api.html b/develop/admin_api/media_admin_api.html index a0ff574d4c..9777d9df30 100644 --- a/develop/admin_api/media_admin_api.html +++ b/develop/admin_api/media_admin_api.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/purge_history_api.html b/develop/admin_api/purge_history_api.html index 649a096538..e8e2b1c02e 100644 --- a/develop/admin_api/purge_history_api.html +++ b/develop/admin_api/purge_history_api.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/register_api.html b/develop/admin_api/register_api.html index a94744fe1f..07f37850d5 100644 --- a/develop/admin_api/register_api.html +++ b/develop/admin_api/register_api.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/room_membership.html b/develop/admin_api/room_membership.html index 1dfa13cf75..7357e21013 100644 --- a/develop/admin_api/room_membership.html +++ b/develop/admin_api/room_membership.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/rooms.html b/develop/admin_api/rooms.html index a4342a59b4..e8a7612977 100644 --- a/develop/admin_api/rooms.html +++ b/develop/admin_api/rooms.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/server_notices.html b/develop/admin_api/server_notices.html index 477993b730..99d70d55e7 100644 --- a/develop/admin_api/server_notices.html +++ b/develop/admin_api/server_notices.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/statistics.html b/develop/admin_api/statistics.html index a89bb8a42a..d7d3cb46fc 100644 --- a/develop/admin_api/statistics.html +++ b/develop/admin_api/statistics.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/user_admin_api.html b/develop/admin_api/user_admin_api.html index 99ea9e03de..79d08410d1 100644 --- a/develop/admin_api/user_admin_api.html +++ b/develop/admin_api/user_admin_api.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/version_api.html b/develop/admin_api/version_api.html index 65a2cc1307..d9109169b9 100644 --- a/develop/admin_api/version_api.html +++ b/develop/admin_api/version_api.html @@ -76,7 +76,7 @@ diff --git a/develop/application_services.html b/develop/application_services.html index 4f42552c78..ac417c9373 100644 --- a/develop/application_services.html +++ b/develop/application_services.html @@ -76,7 +76,7 @@ diff --git a/develop/auth_chain_difference_algorithm.html b/develop/auth_chain_difference_algorithm.html index c75bb06287..172a2c5ea6 100644 --- a/develop/auth_chain_difference_algorithm.html +++ b/develop/auth_chain_difference_algorithm.html @@ -76,7 +76,7 @@ diff --git a/develop/code_style.html b/develop/code_style.html index 9f965d2dc1..0249777830 100644 --- a/develop/code_style.html +++ b/develop/code_style.html @@ -76,7 +76,7 @@ diff --git a/develop/consent_tracking.html b/develop/consent_tracking.html index 62f29aa5db..4c82257c4a 100644 --- a/develop/consent_tracking.html +++ b/develop/consent_tracking.html @@ -76,7 +76,7 @@ diff --git a/develop/delegate.html b/develop/delegate.html index cefbee3616..f7c8e31be2 100644 --- a/develop/delegate.html +++ b/develop/delegate.html @@ -76,7 +76,7 @@ diff --git a/develop/deprecation_policy.html b/develop/deprecation_policy.html index 0323b32958..699e2cfcc8 100644 --- a/develop/deprecation_policy.html +++ b/develop/deprecation_policy.html @@ -76,7 +76,7 @@ diff --git a/develop/development/cas.html b/develop/development/cas.html index 086a352996..5adc9578da 100644 --- a/develop/development/cas.html +++ b/develop/development/cas.html @@ -76,7 +76,7 @@ diff --git a/develop/development/contributing_guide.html b/develop/development/contributing_guide.html index 4794124f9a..f8c0da451b 100644 --- a/develop/development/contributing_guide.html +++ b/develop/development/contributing_guide.html @@ -76,7 +76,7 @@ diff --git a/develop/development/database_schema.html b/develop/development/database_schema.html index d2e647f7c7..7c0c9a6eb2 100644 --- a/develop/development/database_schema.html +++ b/develop/development/database_schema.html @@ -76,7 +76,7 @@ diff --git a/develop/development/demo.html b/develop/development/demo.html index 36cae37ec9..49590e0579 100644 --- a/develop/development/demo.html +++ b/develop/development/demo.html @@ -76,7 +76,7 @@ diff --git a/develop/development/experimental_features.html b/develop/development/experimental_features.html index d26901cf13..cd6104999e 100644 --- a/develop/development/experimental_features.html +++ b/develop/development/experimental_features.html @@ -76,7 +76,7 @@ @@ -187,7 +187,7 @@ configuration key (see the synapse.config.experimental file) and ei -
@@ -199,7 +199,7 @@ configuration key (see the synapse.config.experimental file) and ei - diff --git a/develop/development/git.html b/develop/development/git.html index 7207b899dc..9a1c2bdd79 100644 --- a/develop/development/git.html +++ b/develop/development/git.html @@ -76,7 +76,7 @@ diff --git a/develop/development/internal_documentation/index.html b/develop/development/internal_documentation/index.html index 842dadd755..fe8ab238f2 100644 --- a/develop/development/internal_documentation/index.html +++ b/develop/development/internal_documentation/index.html @@ -76,7 +76,7 @@ diff --git a/develop/development/releases.html b/develop/development/releases.html index 0a46b2456d..7a26758cb1 100644 --- a/develop/development/releases.html +++ b/develop/development/releases.html @@ -76,7 +76,7 @@ diff --git a/develop/development/room-dag-concepts.html b/develop/development/room-dag-concepts.html index e2918f7ab3..4b34c6d62f 100644 --- a/develop/development/room-dag-concepts.html +++ b/develop/development/room-dag-concepts.html @@ -76,7 +76,7 @@ diff --git a/develop/development/saml.html b/develop/development/saml.html index 4fc686f5ec..860d69cf8a 100644 --- a/develop/development/saml.html +++ b/develop/development/saml.html @@ -76,7 +76,7 @@ diff --git a/develop/development/synapse_architecture/cancellation.html b/develop/development/synapse_architecture/cancellation.html new file mode 100644 index 0000000000..d67f9eb56c --- /dev/null +++ b/develop/development/synapse_architecture/cancellation.html @@ -0,0 +1,536 @@ + + + + + + Cancellation - Synapse + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + +
+
+ +
+ +
+ +

Cancellation

+

Sometimes, requests take a long time to service and clients disconnect +before Synapse produces a response. To avoid wasting resources, Synapse +can cancel request processing for select endpoints marked with the +@cancellable decorator.

+

Synapse makes use of Twisted's Deferred.cancel() feature to make +cancellation work. The @cancellable decorator does nothing by itself +and merely acts as a flag, signalling to developers and other code alike +that a method can be cancelled.

+

Enabling cancellation for an endpoint

+
    +
  1. Check that the endpoint method, and any async functions in its call +tree handle cancellation correctly. See +Handling cancellation correctly +for a list of things to look out for.
    2. +
  2. Add the @cancellable decorator to the on_GET/POST/PUT/DELETE +method. It's not recommended to make non-GET methods cancellable, +since cancellation midway through some database updates is less +likely to be handled correctly.
    3. +
+

Mechanics

+

There are two stages to cancellation: downward propagation of a +cancel() call, followed by upwards propagation of a CancelledError +out of a blocked await. +Both Twisted and asyncio have a cancellation mechanism.

+ + + +
MethodExceptionException inherits from
TwistedDeferred.cancel()twisted.internet.defer.CancelledErrorException (!)
asyncioTask.cancel()asyncio.CancelledErrorBaseException
+

Deferred.cancel()

+

When Synapse starts handling a request, it runs the async method +responsible for handling it using defer.ensureDeferred, which returns +a Deferred. For example:

+
def do_something() -> Deferred[None]:
+    ...
+
+@cancellable
+async def on_GET() -> Tuple[int, JsonDict]:
+    d = make_deferred_yieldable(do_something())
+    await d
+    return 200, {}
+
+request = defer.ensureDeferred(on_GET())
+
+

When a client disconnects early, Synapse checks for the presence of the +@cancellable decorator on on_GET. Since on_GET is cancellable, +Deferred.cancel() is called on the Deferred from +defer.ensureDeferred, ie. request. Twisted knows which Deferred +request is waiting on and passes the cancel() call on to d.

+

The Deferred being waited on, d, may have its own handling for +cancel() and pass the call on to other Deferreds.

+

Eventually, a Deferred handles the cancel() call by resolving itself +with a CancelledError.

+

CancelledError

+

The CancelledError gets raised out of the await and bubbles up, as +per normal Python exception handling.

+

Handling cancellation correctly

+

In general, when writing code that might be subject to cancellation, two +things must be considered:

+
    +
  • The effect of CancelledErrors raised out of awaits.
    • +
  • The effect of Deferreds being cancel()ed.
    • +
+

Examples of code that handles cancellation incorrectly include:

+
    +
  • try-except blocks which swallow CancelledErrors.
    • +
  • Code that shares the same Deferred, which may be cancelled, between +multiple requests.
    • +
  • Code that starts some processing that's exempt from cancellation, but +uses a logging context from cancellable code. The logging context +will be finished upon cancellation, while the uncancelled processing +is still using it.
    • +
+

Some common patterns are listed below in more detail.

+

async function calls

+

Most functions in Synapse are relatively straightforward from a +cancellation standpoint: they don't do anything with Deferreds and +purely call and await other async functions.

+

An async function handles cancellation correctly if its own code +handles cancellation correctly and all the async function it calls +handle cancellation correctly. For example:

+
async def do_two_things() -> None:
+    check_something()
+    await do_something()
+    await do_something_else()
+
+

do_two_things handles cancellation correctly if do_something and +do_something_else handle cancellation correctly.

+

That is, when checking whether a function handles cancellation +correctly, its implementation and all its async function calls need to +be checked, recursively.

+

As check_something is not async, it does not need to be checked.

+

CancelledErrors

+

Because Twisted's CancelledErrors are Exceptions, it's easy to +accidentally catch and suppress them. Care must be taken to ensure that +CancelledErrors are allowed to propagate upwards.

+ + + + + + + + + +
+

Bad:

+
try:
+    await do_something()
+except Exception:
+    # `CancelledError` gets swallowed here.
+    logger.info(...)
+
+		 +

Good:

+
try:
+    await do_something()
+except CancelledError:
+    raise
+except Exception:
+    logger.info(...)
+
+
+

OK:

+
try:
+    check_something()
+    # A `CancelledError` won't ever be raised here.
+except Exception:
+    logger.info(...)
+
+		 +

Good:

+
try:
+    await do_something()
+except ValueError:
+    logger.info(...)
+
+
+

defer.gatherResults

+

defer.gatherResults produces a Deferred which:

+
    +
  • broadcasts cancel() calls to every Deferred being waited on.
    • +
  • wraps the first exception it sees in a FirstError.
    • +
+

Together, this means that CancelledErrors will be wrapped in +a FirstError unless unwrapped. Such FirstErrors are liable to be +swallowed, so they must be unwrapped.

+ + + + + +
+

Bad:

+
async def do_something() -> None:
+    await make_deferred_yieldable(
+        defer.gatherResults([...], consumeErrors=True)
+    )
+
+try:
+    await do_something()
+except CancelledError:
+    raise
+except Exception:
+    # `FirstError(CancelledError)` gets swallowed here.
+    logger.info(...)
+
+		 +

Good:

+
async def do_something() -> None:
+    await make_deferred_yieldable(
+        defer.gatherResults([...], consumeErrors=True)
+    ).addErrback(unwrapFirstError)
+
+try:
+    await do_something()
+except CancelledError:
+    raise
+except Exception:
+    logger.info(...)
+
+
+

Creation of Deferreds

+

If a function creates a Deferred, the effect of cancelling it must be considered. Deferreds that get shared are likely to have unintended behaviour when cancelled.

+ + + + + + + + +
+

Bad:

+
cache: Dict[str, Deferred[None]] = {}
+
+def wait_for_room(room_id: str) -> Deferred[None]:
+    deferred = cache.get(room_id)
+    if deferred is None:
+        deferred = Deferred()
+        cache[room_id] = deferred
+    # `deferred` can have multiple waiters.
+    # All of them will observe a `CancelledError`
+    # if any one of them is cancelled.
+    return make_deferred_yieldable(deferred)
+
+# Request 1
+await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org")
+# Request 2
+await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org")
+
+		 +

Good:

+
cache: Dict[str, Deferred[None]] = {}
+
+def wait_for_room(room_id: str) -> Deferred[None]:
+    deferred = cache.get(room_id)
+    if deferred is None:
+        deferred = Deferred()
+        cache[room_id] = deferred
+    # `deferred` will never be cancelled now.
+    # A `CancelledError` will still come out of
+    # the `await`.
+    # `delay_cancellation` may also be used.
+    return make_deferred_yieldable(stop_cancellation(deferred))
+
+# Request 1
+await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org")
+# Request 2
+await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org")
+
+
+ +

Good:

+
cache: Dict[str, List[Deferred[None]]] = {}
+
+def wait_for_room(room_id: str) -> Deferred[None]:
+    if room_id not in cache:
+        cache[room_id] = []
+    # Each request gets its own `Deferred` to wait on.
+    deferred = Deferred()
+    cache[room_id]].append(deferred)
+    return make_deferred_yieldable(deferred)
+
+# Request 1
+await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org")
+# Request 2
+await wait_for_room("!aAAaaAaaaAAAaAaAA:matrix.org")
+
+
+

Uncancelled processing

+

Some async functions may kick off some async processing which is +intentionally protected from cancellation, by stop_cancellation or +other means. If the async processing inherits the logcontext of the +request which initiated it, care must be taken to ensure that the +logcontext is not finished before the async processing completes.

+ + + + + + + + + +
+

Bad:

+
cache: Optional[ObservableDeferred[None]] = None
+
+async def do_something_else(
+    to_resolve: Deferred[None]
+) -> None:
+    await ...
+    logger.info("done!")
+    to_resolve.callback(None)
+
+async def do_something() -> None:
+    if not cache:
+        to_resolve = Deferred()
+        cache = ObservableDeferred(to_resolve)
+        # `do_something_else` will never be cancelled and
+        # can outlive the `request-1` logging context.
+        run_in_background(do_something_else, to_resolve)
+
+    await make_deferred_yieldable(cache.observe())
+
+with LoggingContext("request-1"):
+    await do_something()
+
+		 +

Good:

+
cache: Optional[ObservableDeferred[None]] = None
+
+async def do_something_else(
+    to_resolve: Deferred[None]
+) -> None:
+    await ...
+    logger.info("done!")
+    to_resolve.callback(None)
+
+async def do_something() -> None:
+    if not cache:
+        to_resolve = Deferred()
+        cache = ObservableDeferred(to_resolve)
+        run_in_background(do_something_else, to_resolve)
+        # We'll wait until `do_something_else` is
+        # done before raising a `CancelledError`.
+        await make_deferred_yieldable(
+            delay_cancellation(cache.observe())
+        )
+    else:
+        await make_deferred_yieldable(cache.observe())
+
+with LoggingContext("request-1"):
+    await do_something()
+
+
+

OK:

+
cache: Optional[ObservableDeferred[None]] = None
+
+async def do_something_else(
+    to_resolve: Deferred[None]
+) -> None:
+    await ...
+    logger.info("done!")
+    to_resolve.callback(None)
+
+async def do_something() -> None:
+    if not cache:
+        to_resolve = Deferred()
+        cache = ObservableDeferred(to_resolve)
+        # `do_something_else` will get its own independent
+        # logging context. `request-1` will not count any
+        # metrics from `do_something_else`.
+        run_as_background_process(
+            "do_something_else",
+            do_something_else,
+            to_resolve,
+        )
+
+    await make_deferred_yieldable(cache.observe())
+
+with LoggingContext("request-1"):
+    await do_something()
+
+		 +
+ +
+ + +
+
+ + + +
+ + + + + + + + + + + + + \ No newline at end of file diff --git a/develop/development/url_previews.html b/develop/development/url_previews.html index 42ad082399..d9b3678bda 100644 --- a/develop/development/url_previews.html +++ b/develop/development/url_previews.html @@ -76,7 +76,7 @@ diff --git a/develop/federate.html b/develop/federate.html index 17bff3ad7f..697bc4a2f7 100644 --- a/develop/federate.html +++ b/develop/federate.html @@ -76,7 +76,7 @@ diff --git a/develop/index.html b/develop/index.html index 5b93699339..d75d29f837 100644 --- a/develop/index.html +++ b/develop/index.html @@ -76,7 +76,7 @@ diff --git a/develop/jwt.html b/develop/jwt.html index 9c41f27e9a..f249c039d6 100644 --- a/develop/jwt.html +++ b/develop/jwt.html @@ -76,7 +76,7 @@ diff --git a/develop/log_contexts.html b/develop/log_contexts.html index 2d71200b85..50cb20260c 100644 --- a/develop/log_contexts.html +++ b/develop/log_contexts.html @@ -76,7 +76,7 @@ @@ -447,7 +447,7 @@ lead to leaked logcontexts which are incredibly hard to track down.