From 0eade2958c4a108ccfec904a68174a5b1c256237 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 23 Apr 2021 18:38:45 +0100 Subject: [PATCH 1/9] Remove conditionals for python <3.6 in tox.ini --- tox.ini | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 998b04b22442..ecd609271d1d 100644 --- a/tox.ini +++ b/tox.ini @@ -21,13 +21,11 @@ deps = # installed on that). # # anyway, make sure that we have a recent enough setuptools. - setuptools>=18.5 ; python_version >= '3.6' - setuptools>=18.5,<51.0.0 ; python_version < '3.6' + setuptools>=18.5 # we also need a semi-recent version of pip, because old ones fail to # install the "enum34" dependency of cryptography. - pip>=10 ; python_version >= '3.6' - pip>=10,<21.0 ; python_version < '3.6' + pip>=10 # directories/files we run the linters on. # if you update this list, make sure to do the same in scripts-dev/lint.sh @@ -168,8 +166,7 @@ skip_install = true usedevelop = false deps = coverage - pip>=10 ; python_version >= '3.6' - pip>=10,<21.0 ; python_version < '3.6' + pip>=10 commands= coverage combine coverage report From 4a8cd3ac06a1cc646041c88e0a748cefd79f68ad Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 23 Apr 2021 18:39:31 +0100 Subject: [PATCH 2/9] Remove references to Python <3.6 in python_dependencies --- synapse/python_dependencies.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 2a1c925ee8b9..2de946f46404 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -85,7 +85,7 @@ "typing-extensions>=3.7.4", # We enforce that we have a `cryptography` version that bundles an `openssl` # with the latest security patches. - "cryptography>=3.4.7;python_version>='3.6'", + "cryptography>=3.4.7", ] CONDITIONAL_REQUIREMENTS = { @@ -100,14 +100,9 @@ # that use the protocol, such as Let's Encrypt. "acme": [ "txacme>=0.9.2", - # txacme depends on eliot. Eliot 1.8.0 is incompatible with - # python 3.5.2, as per https://github.com/itamarst/eliot/issues/418 - "eliot<1.8.0;python_version<'3.5.3'", ], "saml2": [ - # pysaml2 6.4.0 is incompatible with Python 3.5 (see https://github.com/IdentityPython/pysaml2/issues/749) - "pysaml2>=4.5.0,<6.4.0;python_version<'3.6'", - "pysaml2>=4.5.0;python_version>='3.6'", + "pysaml2>=4.5.0", ], "oidc": ["authlib>=0.14.0"], # systemd-python is necessary for logging to the systemd journal via From 6fd2538f64445cfcd95bcf7c3588bacee8c1156c Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 23 Apr 2021 18:41:08 +0100 Subject: [PATCH 3/9] Remove conditional import for Python <3.6 in synapse.secrets --- synapse/secrets.py | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/synapse/secrets.py b/synapse/secrets.py index bf829251fd0a..432db8da1f23 100644 --- a/synapse/secrets.py +++ b/synapse/secrets.py @@ -15,30 +15,14 @@ """ Injectable secrets module for Synapse. -See https://docs.python.org/3/library/secrets.html#module-secrets for the API -used in Python 3.6, and the API emulated in Python 2.7. +See https://docs.python.org/3/library/secrets.html#module-secrets for the API. """ -import sys +import secrets -# secrets is available since python 3.6 -if sys.version_info[0:2] >= (3, 6): - import secrets - class Secrets: - def token_bytes(self, nbytes: int = 32) -> bytes: - return secrets.token_bytes(nbytes) +class Secrets: + def token_bytes(self, nbytes: int = 32) -> bytes: + return secrets.token_bytes(nbytes) - def token_hex(self, nbytes: int = 32) -> str: - return secrets.token_hex(nbytes) - - -else: - import binascii - import os - - class Secrets: - def token_bytes(self, nbytes: int = 32) -> bytes: - return os.urandom(nbytes) - - def token_hex(self, nbytes: int = 32) -> str: - return binascii.hexlify(self.token_bytes(nbytes)).decode("ascii") + def token_hex(self, nbytes: int = 32) -> str: + return secrets.token_hex(nbytes) From 0506cc928c190b559b0e8477890f0707d235bc75 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 23 Apr 2021 18:42:22 +0100 Subject: [PATCH 4/9] Fix comment that says cast is necessary for Python 3.5 The comment was correct when we were using json.loads. But since switching to our own JSONDecoder instead, the decode() function only supports str-type arguments. Switch happened in #8106. --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index d472676acfe1..6b68d8720c92 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -114,7 +114,7 @@ def db_to_json(db_content: Union[memoryview, bytes, bytearray, str]) -> Any: db_content = db_content.tobytes() # Decode it to a Unicode string before feeding it to the JSON decoder, since - # Python 3.5 does not support deserializing bytes. + # it only supports handling strings if isinstance(db_content, (bytes, bytearray)): db_content = db_content.decode("utf8") From 7aa710c7ec1518addd626d1f37c6588a9ddde13d Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 23 Apr 2021 18:44:28 +0100 Subject: [PATCH 5/9] Reintroduce a style of type hint that was incompatible with Python 3.5 --- synapse/rest/media/v1/filepath.py | 2 +- synapse/storage/database.py | 15 ++++++--------- synapse/util/caches/response_cache.py | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/synapse/rest/media/v1/filepath.py b/synapse/rest/media/v1/filepath.py index 4088e7a059ad..09531ebf548d 100644 --- a/synapse/rest/media/v1/filepath.py +++ b/synapse/rest/media/v1/filepath.py @@ -21,7 +21,7 @@ NEW_FORMAT_ID_RE = re.compile(r"^\d\d\d\d-\d\d-\d\d") -def _wrap_in_base_path(func: "Callable[..., str]") -> "Callable[..., str]": +def _wrap_in_base_path(func: Callable[..., str]) -> Callable[..., str]: """Takes a function that returns a relative path and turns it into an absolute path based on the location of the primary media store """ diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 9452368bf08a..bd39c095af6b 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -171,10 +171,7 @@ def __getattr__(self, name): # The type of entry which goes on our after_callbacks and exception_callbacks lists. -# -# Python 3.5.2 doesn't support Callable with an ellipsis, so we wrap it in quotes so -# that mypy sees the type but the runtime python doesn't. -_CallbackListEntry = Tuple["Callable[..., None]", Iterable[Any], Dict[str, Any]] +_CallbackListEntry = Tuple[Callable[..., None], Iterable[Any], Dict[str, Any]] R = TypeVar("R") @@ -221,7 +218,7 @@ def __init__( self.after_callbacks = after_callbacks self.exception_callbacks = exception_callbacks - def call_after(self, callback: "Callable[..., None]", *args: Any, **kwargs: Any): + def call_after(self, callback: Callable[..., None], *args: Any, **kwargs: Any): """Call the given callback on the main twisted thread after the transaction has finished. Used to invalidate the caches on the correct thread. @@ -233,7 +230,7 @@ def call_after(self, callback: "Callable[..., None]", *args: Any, **kwargs: Any) self.after_callbacks.append((callback, args, kwargs)) def call_on_exception( - self, callback: "Callable[..., None]", *args: Any, **kwargs: Any + self, callback: Callable[..., None], *args: Any, **kwargs: Any ): # if self.exception_callbacks is None, that means that whatever constructed the # LoggingTransaction isn't expecting there to be any callbacks; assert that @@ -485,7 +482,7 @@ def new_transaction( desc: str, after_callbacks: List[_CallbackListEntry], exception_callbacks: List[_CallbackListEntry], - func: "Callable[..., R]", + func: Callable[..., R], *args: Any, **kwargs: Any, ) -> R: @@ -618,7 +615,7 @@ def new_transaction( async def runInteraction( self, desc: str, - func: "Callable[..., R]", + func: Callable[..., R], *args: Any, db_autocommit: bool = False, **kwargs: Any, @@ -678,7 +675,7 @@ async def runInteraction( async def runWithConnection( self, - func: "Callable[..., R]", + func: Callable[..., R], *args: Any, db_autocommit: bool = False, **kwargs: Any, diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index 2529845c9ef0..25ea1bcc915e 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -110,7 +110,7 @@ def remove(r): return result.observe() def wrap( - self, key: T, callback: "Callable[..., Any]", *args: Any, **kwargs: Any + self, key: T, callback: Callable[..., Any], *args: Any, **kwargs: Any ) -> defer.Deferred: """Wrap together a *get* and *set* call, taking care of logcontexts From 6f13d192dee3fc14e462ff9b7b48784fa9ee25f6 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 23 Apr 2021 18:42:47 +0100 Subject: [PATCH 6/9] Actually require Python 3.6+ --- synapse/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 837e938f569c..fbd49a93e137 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -21,8 +21,8 @@ import sys # Check that we're not running on an unsupported Python version. -if sys.version_info < (3, 5): - print("Synapse requires Python 3.5 or above.") +if sys.version_info < (3, 6): + print("Synapse requires Python 3.6 or above.") sys.exit(1) # Twisted and canonicaljson will fail to import when this file is executed to From 491a3a0ce62800808f211a8766664993a3344279 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 23 Apr 2021 19:10:42 +0100 Subject: [PATCH 7/9] Remove Python 2.7 compatibility for digests --- synapse/rest/consent/consent_resource.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index c4550d3cf06b..b19cd8afc53e 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -32,14 +32,6 @@ logger = logging.getLogger(__name__) -# use hmac.compare_digest if we have it (python 2.7.7), else just use equality -if hasattr(hmac, "compare_digest"): - compare_digest = hmac.compare_digest -else: - - def compare_digest(a, b): - return a == b - class ConsentResource(DirectServeHtmlResource): """A twisted Resource to display a privacy policy and gather consent to it @@ -209,5 +201,5 @@ def _check_hash(self, userid, userhmac): .encode("ascii") ) - if not compare_digest(want_mac, userhmac): + if not hmac.compare_digest(want_mac, userhmac): raise SynapseError(HTTPStatus.FORBIDDEN, "HMAC incorrect") From bc076e53641de5bb81d67d9b58f8511c9691c61c Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 23 Apr 2021 19:13:02 +0100 Subject: [PATCH 8/9] Changelog --- changelog.d/9879.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/9879.misc diff --git a/changelog.d/9879.misc b/changelog.d/9879.misc new file mode 100644 index 000000000000..c9ca37cf4835 --- /dev/null +++ b/changelog.d/9879.misc @@ -0,0 +1 @@ +Remove backwards-compatibility code for Python versions < 3.6. \ No newline at end of file From 320291d8df3c6ea313772c8912c0d870e6dfcd1e Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Mon, 26 Apr 2021 11:59:55 +0100 Subject: [PATCH 9/9] Remove hs secrets module, fix tests --- mypy.ini | 1 - synapse/rest/admin/users.py | 3 ++- synapse/secrets.py | 28 ---------------------------- synapse/server.py | 5 ----- tests/rest/admin/test_user.py | 15 ++++++--------- tests/storage/test__base.py | 3 ++- tests/unittest.py | 2 +- 7 files changed, 11 insertions(+), 46 deletions(-) delete mode 100644 synapse/secrets.py diff --git a/mypy.ini b/mypy.ini index 32e619740980..a40f705b76f4 100644 --- a/mypy.ini +++ b/mypy.ini @@ -41,7 +41,6 @@ files = synapse/push, synapse/replication, synapse/rest, - synapse/secrets.py, synapse/server.py, synapse/server_notices, synapse/spam_checker_api, diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index edda7861fae5..8c9d21d3ea1b 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -14,6 +14,7 @@ import hashlib import hmac import logging +import secrets from http import HTTPStatus from typing import TYPE_CHECKING, Dict, List, Optional, Tuple @@ -375,7 +376,7 @@ def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: """ self._clear_old_nonces() - nonce = self.hs.get_secrets().token_hex(64) + nonce = secrets.token_hex(64) self.nonces[nonce] = int(self.reactor.seconds()) return 200, {"nonce": nonce} diff --git a/synapse/secrets.py b/synapse/secrets.py deleted file mode 100644 index 432db8da1f23..000000000000 --- a/synapse/secrets.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2018 New Vector Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Injectable secrets module for Synapse. - -See https://docs.python.org/3/library/secrets.html#module-secrets for the API. -""" -import secrets - - -class Secrets: - def token_bytes(self, nbytes: int = 32) -> bytes: - return secrets.token_bytes(nbytes) - - def token_hex(self, nbytes: int = 32) -> str: - return secrets.token_hex(nbytes) diff --git a/synapse/server.py b/synapse/server.py index 8c147be2b3d0..5da9ab24d78e 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -126,7 +126,6 @@ MediaRepository, MediaRepositoryResource, ) -from synapse.secrets import Secrets from synapse.server_notices.server_notices_manager import ServerNoticesManager from synapse.server_notices.server_notices_sender import ServerNoticesSender from synapse.server_notices.worker_server_notices_sender import ( @@ -633,10 +632,6 @@ def get_groups_attestation_signing(self) -> GroupAttestationSigning: def get_groups_attestation_renewer(self) -> GroupAttestionRenewer: return GroupAttestionRenewer(self) - @cache_in_self - def get_secrets(self) -> Secrets: - return Secrets() - @cache_in_self def get_stats_handler(self) -> StatsHandler: return StatsHandler(self) diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index b3afd51522d6..d599a4c984d9 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -18,7 +18,7 @@ import urllib.parse from binascii import unhexlify from typing import List, Optional -from unittest.mock import Mock +from unittest.mock import Mock, patch import synapse.rest.admin from synapse.api.constants import UserTypes @@ -54,8 +54,6 @@ def make_homeserver(self, reactor, clock): self.datastore = Mock(return_value=Mock()) self.datastore.get_current_state_deltas = Mock(return_value=(0, [])) - self.secrets = Mock() - self.hs = self.setup_test_homeserver() self.hs.config.registration_shared_secret = "shared" @@ -84,14 +82,13 @@ def test_get_nonce(self): Calling GET on the endpoint will return a randomised nonce, using the homeserver's secrets provider. """ - secrets = Mock() - secrets.token_hex = Mock(return_value="abcd") - - self.hs.get_secrets = Mock(return_value=secrets) + with patch("secrets.token_hex") as token_hex: + # Patch secrets.token_hex for the duration of this context + token_hex.return_value = "abcd" - channel = self.make_request("GET", self.url) + channel = self.make_request("GET", self.url) - self.assertEqual(channel.json_body, {"nonce": "abcd"}) + self.assertEqual(channel.json_body, {"nonce": "abcd"}) def test_expired_nonce(self): """ diff --git a/tests/storage/test__base.py b/tests/storage/test__base.py index 6339a43f0cb3..200b9198f910 100644 --- a/tests/storage/test__base.py +++ b/tests/storage/test__base.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import secrets from tests import unittest @@ -21,7 +22,7 @@ class UpsertManyTests(unittest.HomeserverTestCase): def prepare(self, reactor, clock, hs): self.storage = hs.get_datastore() - self.table_name = "table_" + hs.get_secrets().token_hex(6) + self.table_name = "table_" + secrets.token_hex(6) self.get_success( self.storage.db_pool.runInteraction( "create", diff --git a/tests/unittest.py b/tests/unittest.py index 5353e75c7c85..81a91db9e820 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -18,6 +18,7 @@ import hmac import inspect import logging +import secrets import time from typing import Callable, Dict, Iterable, Optional, Tuple, Type, TypeVar, Union from unittest.mock import Mock, patch @@ -625,7 +626,6 @@ def create_and_send_event( str: The new event's ID. """ event_creator = self.hs.get_event_creation_handler() - secrets = self.hs.get_secrets() requester = create_requester(user) event, context = self.get_success(