diff --git a/develop/404.html b/develop/404.html index 8cd53682c5..24ab6ee527 100644 --- a/develop/404.html +++ b/develop/404.html @@ -77,7 +77,7 @@ diff --git a/develop/CAPTCHA_SETUP.html b/develop/CAPTCHA_SETUP.html index 534cbfd1aa..43c4b64a58 100644 --- a/develop/CAPTCHA_SETUP.html +++ b/develop/CAPTCHA_SETUP.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/account_validity.html b/develop/admin_api/account_validity.html index ef27129291..fc09d8bcf5 100644 --- a/develop/admin_api/account_validity.html +++ b/develop/admin_api/account_validity.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/delete_group.html b/develop/admin_api/delete_group.html index 9eb9f026cb..564d0606f8 100644 --- a/develop/admin_api/delete_group.html +++ b/develop/admin_api/delete_group.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/event_reports.html b/develop/admin_api/event_reports.html index 9dca2b0070..0190f49184 100644 --- a/develop/admin_api/event_reports.html +++ b/develop/admin_api/event_reports.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/media_admin_api.html b/develop/admin_api/media_admin_api.html index 9777d9df30..1dfdf392b1 100644 --- a/develop/admin_api/media_admin_api.html +++ b/develop/admin_api/media_admin_api.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/purge_history_api.html b/develop/admin_api/purge_history_api.html index e8e2b1c02e..8600f99f25 100644 --- a/develop/admin_api/purge_history_api.html +++ b/develop/admin_api/purge_history_api.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/register_api.html b/develop/admin_api/register_api.html index 07f37850d5..5ab08fdcda 100644 --- a/develop/admin_api/register_api.html +++ b/develop/admin_api/register_api.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/room_membership.html b/develop/admin_api/room_membership.html index 7357e21013..12ce502057 100644 --- a/develop/admin_api/room_membership.html +++ b/develop/admin_api/room_membership.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/rooms.html b/develop/admin_api/rooms.html index e8a7612977..7608905e4d 100644 --- a/develop/admin_api/rooms.html +++ b/develop/admin_api/rooms.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/server_notices.html b/develop/admin_api/server_notices.html index 99d70d55e7..e0dbf7a76a 100644 --- a/develop/admin_api/server_notices.html +++ b/develop/admin_api/server_notices.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/statistics.html b/develop/admin_api/statistics.html index d7d3cb46fc..027715646c 100644 --- a/develop/admin_api/statistics.html +++ b/develop/admin_api/statistics.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/user_admin_api.html b/develop/admin_api/user_admin_api.html index b09d5996bc..1827ac6fa4 100644 --- a/develop/admin_api/user_admin_api.html +++ b/develop/admin_api/user_admin_api.html @@ -76,7 +76,7 @@ diff --git a/develop/admin_api/version_api.html b/develop/admin_api/version_api.html index d9109169b9..a7d14da7eb 100644 --- a/develop/admin_api/version_api.html +++ b/develop/admin_api/version_api.html @@ -76,7 +76,7 @@ diff --git a/develop/application_services.html b/develop/application_services.html index ac417c9373..8b358ad3a4 100644 --- a/develop/application_services.html +++ b/develop/application_services.html @@ -76,7 +76,7 @@ diff --git a/develop/auth_chain_difference_algorithm.html b/develop/auth_chain_difference_algorithm.html index 172a2c5ea6..b844a46954 100644 --- a/develop/auth_chain_difference_algorithm.html +++ b/develop/auth_chain_difference_algorithm.html @@ -76,7 +76,7 @@ diff --git a/develop/code_style.html b/develop/code_style.html index 0249777830..8e314a1933 100644 --- a/develop/code_style.html +++ b/develop/code_style.html @@ -76,7 +76,7 @@ diff --git a/develop/consent_tracking.html b/develop/consent_tracking.html index 4c82257c4a..ffa48224c9 100644 --- a/develop/consent_tracking.html +++ b/develop/consent_tracking.html @@ -76,7 +76,7 @@ diff --git a/develop/delegate.html b/develop/delegate.html index f7c8e31be2..66a0881af0 100644 --- a/develop/delegate.html +++ b/develop/delegate.html @@ -76,7 +76,7 @@ diff --git a/develop/deprecation_policy.html b/develop/deprecation_policy.html index 699e2cfcc8..f3c286d4c1 100644 --- a/develop/deprecation_policy.html +++ b/develop/deprecation_policy.html @@ -76,7 +76,7 @@ diff --git a/develop/development/cas.html b/develop/development/cas.html index 5adc9578da..d83c1ff4e5 100644 --- a/develop/development/cas.html +++ b/develop/development/cas.html @@ -76,7 +76,7 @@ diff --git a/develop/development/contributing_guide.html b/develop/development/contributing_guide.html index 1d4b261bb6..88f4bb7a5c 100644 --- a/develop/development/contributing_guide.html +++ b/develop/development/contributing_guide.html @@ -76,7 +76,7 @@ diff --git a/develop/development/database_schema.html b/develop/development/database_schema.html index 7c0c9a6eb2..fc76a6f99b 100644 --- a/develop/development/database_schema.html +++ b/develop/development/database_schema.html @@ -76,7 +76,7 @@ diff --git a/develop/development/demo.html b/develop/development/demo.html index b8e499587a..c4250f27a8 100644 --- a/develop/development/demo.html +++ b/develop/development/demo.html @@ -76,7 +76,7 @@ diff --git a/develop/development/dependencies.html b/develop/development/dependencies.html new file mode 100644 index 0000000000..a663ae5670 --- /dev/null +++ b/develop/development/dependencies.html @@ -0,0 +1,364 @@ + + + + + + Dependency management - Synapse + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + +
+
+ +
+ +
+ +

Managing dependencies with Poetry

+

This is a quick cheat sheet for developers on how to use poetry.

+

Background

+

Synapse uses a variety of third-party Python packages to function as a homeserver. +Some of these are direct dependencies, listed in pyproject.toml under the +[tool.poetry.dependencies] section. The rest are transitive dependencies (the +things that our direct dependencies themselves depend on, and so on recursively.)

+

We maintain a locked list of all our dependencies (transitive included) so that +we can track exactly which version of each dependency appears in a given release. +See here +for discussion of why we wanted this for Synapse. We chose to use +poetry to manage this locked list; see +this comment +for the reasoning.

+

The locked dependencies get included in our "self-contained" releases: namely, +our docker images and our debian packages. We also use the locked dependencies +in development and our continuous integration.

+

Separately, our "broad" dependencies—the version ranges specified in +pyproject.toml—are included as metadata in our "sdists" and "wheels" uploaded +to PyPI. Installing from PyPI or from +the Synapse source tree directly will not use the locked dependencies; instead, +they'll pull in the latest version of each package available at install time.

+

Example dependency

+

An example may help. We have a broad dependency on +phonenumbers, as declared in +this snippet from pyproject.toml as of Synapse 1.57:

+
[tool.poetry.dependencies]
+# ...
+phonenumbers = ">=8.2.0"
+
+

In our lockfile this is +pinned +to version 8.12.44, even though +newer versions are available.

+
[[package]]
+name = "phonenumbers"
+version = "8.12.44"
+description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers."
+category = "main"
+optional = false
+python-versions = "*"
+
+

The lockfile also includes a +cryptographic checksum +of the sdists and wheels provided for this version of phonenumbers.

+
[metadata.files]
+# ...
+phonenumbers = [
+    {file = "phonenumbers-8.12.44-py2.py3-none-any.whl", hash = "sha256:cc1299cf37b309ecab6214297663ab86cb3d64ae37fd5b88e904fe7983a874a6"},
+    {file = "phonenumbers-8.12.44.tar.gz", hash = "sha256:26cfd0257d1704fe2f88caff2caabb70d16a877b1e65b6aae51f9fbbe10aa8ce"},
+]
+
+

We can see this pinned version inside the docker image for that release:

+
$ docker pull matrixdotorg/synapse:v1.57.0
+...
+$ docker run --entrypoint pip matrixdotorg/synapse:v1.57.0 show phonenumbers
+Name: phonenumbers
+Version: 8.12.44
+Summary: Python version of Google's common library for parsing, formatting, storing and validating international phone numbers.
+Home-page: https://github.com/daviddrysdale/python-phonenumbers
+Author: David Drysdale
+Author-email: dmd@lurklurk.org
+License: Apache License 2.0
+Location: /usr/local/lib/python3.9/site-packages
+Requires:
+Required-by: matrix-synapse
+
+

Whereas the wheel metadata just contains the broad dependencies:

+
$ cd /tmp
+$ wget https://files.pythonhosted.org/packages/ca/5e/d722d572cc5b3092402b783d6b7185901b444427633bd8a6b00ea0dd41b7/matrix_synapse-1.57.0rc1-py3-none-any.whl
+...
+$ unzip -c matrix_synapse-1.57.0rc1-py3-none-any.whl matrix_synapse-1.57.0rc1.dist-info/METADATA | grep phonenumbers
+Requires-Dist: phonenumbers (>=8.2.0)
+
+

Tooling recommendation: direnv

+

direnv is a tool for activating environments in your +shell inside a given directory. Its support for poetry is unofficial (a +community wiki recipe only), but works solidly in our experience. We thoroughly +recommend it for daily use. To use it:

+
    +
  1. Install direnv - it's likely +packaged for your system already.
    2. +
  2. Teach direnv about poetry. The shell config here +needs to be added to ~/.config/direnv/direnvrc (or more generally $XDG_CONFIG_HOME/direnv/direnvrc).
    3. +
  3. Mark the synapse checkout as a poetry project: echo layout poetry > .envrc.
    4. +
  4. Convince yourself that you trust this .envrc configuration and project. +Then formally confirm this to direnv by running direnv allow.
    5. +
+

Then whenever you navigate to the synapse checkout, you should be able to run +e.g. mypy instead of poetry run mypy; python instead of +poetry run python; and your shell commands will automatically run in the +context of poetry's venv, without having to run poetry shell beforehand.

+

How do I...

+

...reset my venv to the locked environment?

+
poetry install --extras all --remove-untracked
+
+

...run a command in the poetry virtualenv?

+

Use poetry run cmd args when you need the python virtualenv context. +To avoid typing poetry run all the time, you can run poetry shell +to start a new shell in the poetry virtualenv context. Within poetry shell, +python, pip, mypy, trial, etc. are all run inside the project virtualenv +and isolated from the rest o the system.

+

Roughly speaking, the translation from a traditional virtualenv is:

+
    +
  • env/bin/activate -> poetry shell, and
    • +
  • deactivate -> close the terminal (Ctrl-D, exit, etc.)
    • +
+

See also the direnv recommendation above, which makes poetry run and +poetry shell unnecessary.

+

...inspect the poetry virtualenv?

+

Some suggestions:

+
# Current env only
+poetry env info
+# All envs: this allows you to have e.g. a poetry managed venv for Python 3.7,
+# and another for Python 3.10.
+poetry env list --full-path
+poetry run pip list
+
+

Note that poetry show describes the abstract lock file rather than your +on-disk environment. With that said, poetry show --tree can sometimes be +useful.

+

...add a new dependency?

+

Either:

+
    +
  • manually update pyproject.toml; then poetry lock --no-update; or else
    • +
  • poetry add packagename. See poetry add --help; note the --dev, +--extras and --optional flags in particular. +
      +
    • NB: this specifies the new package with a version given by a "caret bound". This won't get forced to its lowest version in the old deps CI job: see this TODO.
      • +
    +
    • +
+

Include the updated pyproject.toml and poetry.lock files in your commit.

+

...remove a dependency?

+

This is not done often and is untested, but

+
poetry remove packagename
+
+

ought to do the trick. Alternatively, manually update pyproject.toml and +poetry lock --no-update. Include the updated pyproject.toml and poetry.lock` +files in your commit.

+

...update the version range for an existing dependency?

+

Best done by manually editing pyproject.toml, then poetry lock --no-update. +Include the updated pyproject.toml and poetry.lock in your commit.

+

...update a dependency in the locked environment?

+

Use

+
poetry update packagename
+
+

to use the latest version of packagename in the locked environment, without +affecting the broad dependencies listed in the wheel.

+

There doesn't seem to be a way to do this whilst locking a specific version of +packagename. We can workaround this (crudely) as follows:

+
poetry add packagename==1.2.3
+# This should update pyproject.lock.
+
+# Now undo the changes to pyproject.toml. For example
+# git restore pyproject.toml
+
+# Get poetry to recompute the content-hash of pyproject.toml without changing
+# the locked package versions.
+poetry lock --no-update
+
+

Either way, include the updated poetry.lock file in your commit.

+

...export a requirements.txt file?

+
poetry export --extras all
+
+

Be wary of bugs in poetry export and pip install -r requirements.txt.

+

Note: poetry export will be made a plugin in Poetry 1.2. Additional config may +be required.

+

...build a test wheel?

+

I usually use

+
poetry run pip install build && poetry run python -m build
+
+

because build is a standardish tool which +doesn't require poetry. (It's what we use in CI too). However, you could try +poetry build too.

+ +
+ + +
+
+ + + +
+ + + + + + + + + + + + + \ No newline at end of file diff --git a/develop/development/experimental_features.html b/develop/development/experimental_features.html index cd6104999e..723d989e20 100644 --- a/develop/development/experimental_features.html +++ b/develop/development/experimental_features.html @@ -76,7 +76,7 @@ @@ -187,7 +187,7 @@ configuration key (see the synapse.config.experimental file) and ei -
@@ -199,7 +199,7 @@ configuration key (see the synapse.config.experimental file) and ei - diff --git a/develop/development/git.html b/develop/development/git.html index 9a1c2bdd79..3011625077 100644 --- a/develop/development/git.html +++ b/develop/development/git.html @@ -76,7 +76,7 @@ diff --git a/develop/development/internal_documentation/index.html b/develop/development/internal_documentation/index.html index fe8ab238f2..23f81c8722 100644 --- a/develop/development/internal_documentation/index.html +++ b/develop/development/internal_documentation/index.html @@ -76,7 +76,7 @@ diff --git a/develop/development/releases.html b/develop/development/releases.html index 7a26758cb1..6453515b88 100644 --- a/develop/development/releases.html +++ b/develop/development/releases.html @@ -76,7 +76,7 @@ diff --git a/develop/development/room-dag-concepts.html b/develop/development/room-dag-concepts.html index 4b34c6d62f..bcc8460602 100644 --- a/develop/development/room-dag-concepts.html +++ b/develop/development/room-dag-concepts.html @@ -76,7 +76,7 @@ diff --git a/develop/development/saml.html b/develop/development/saml.html index 860d69cf8a..59f7779b02 100644 --- a/develop/development/saml.html +++ b/develop/development/saml.html @@ -76,7 +76,7 @@ diff --git a/develop/development/synapse_architecture/cancellation.html b/develop/development/synapse_architecture/cancellation.html index d67f9eb56c..e0ede3125c 100644 --- a/develop/development/synapse_architecture/cancellation.html +++ b/develop/development/synapse_architecture/cancellation.html @@ -76,7 +76,7 @@ @@ -498,7 +498,7 @@ with LoggingContext("request-1"):