From 008aaca0b6f12a6ff5609ba081a4ec8e854486f9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 19 Feb 2020 13:56:15 +0000 Subject: [PATCH 1/5] Minor perf fixes to `get_auth_chain_ids` --- synapse/storage/data_stores/main/event_federation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/data_stores/main/event_federation.py b/synapse/storage/data_stores/main/event_federation.py index 60c67457b4d1..6edebd7d1835 100644 --- a/synapse/storage/data_stores/main/event_federation.py +++ b/synapse/storage/data_stores/main/event_federation.py @@ -72,13 +72,13 @@ def _get_auth_chain_ids_txn(self, txn, event_ids, include_given): while front: new_front = set() front_list = list(front) - chunks = [front_list[x : x + 100] for x in range(0, len(front), 100)] + chunks = (front_list[x : x + 100] for x in range(0, len(front), 100)) for chunk in chunks: clause, args = make_in_list_sql_clause( txn.database_engine, "event_id", chunk ) - txn.execute(base_sql + clause, list(args)) - new_front.update([r[0] for r in txn]) + txn.execute(base_sql + clause, args) + new_front.update(r[0] for r in txn) new_front -= results From 7cadb476a0ca700892c94a8b9ccefaebe75a5686 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 19 Feb 2020 13:57:30 +0000 Subject: [PATCH 2/5] Newsfile --- changelog.d/6954.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/6954.misc diff --git a/changelog.d/6954.misc b/changelog.d/6954.misc new file mode 100644 index 000000000000..8b84ce2f1925 --- /dev/null +++ b/changelog.d/6954.misc @@ -0,0 +1 @@ +Minor perf fixes to `get_auth_chain_ids`. From cb8fdfdf1c2da73d593c32cf0b2df132a1bc9b6b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 19 Feb 2020 15:13:51 +0000 Subject: [PATCH 3/5] Mark make_in_list_sql_clause as returning a list --- synapse/storage/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 3eeb2f7c04fc..6dcb5c04dacf 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -1504,7 +1504,7 @@ def simple_search_list_txn(cls, txn, table, term, col, retcols): def make_in_list_sql_clause( database_engine, column: str, iterable: Iterable -) -> Tuple[str, Iterable]: +) -> Tuple[str, list]: """Returns an SQL clause that checks the given column is in the iterable. On SQLite this expands to `column IN (?, ?, ...)`, whereas on Postgres From 59f1458958a91f7328422a25c03ebb687978f6b6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 19 Feb 2020 15:15:51 +0000 Subject: [PATCH 4/5] Use batch_iter and drop conversion to list --- synapse/storage/data_stores/main/event_federation.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/storage/data_stores/main/event_federation.py b/synapse/storage/data_stores/main/event_federation.py index 6edebd7d1835..9b89e56ae889 100644 --- a/synapse/storage/data_stores/main/event_federation.py +++ b/synapse/storage/data_stores/main/event_federation.py @@ -22,6 +22,7 @@ from synapse.api.errors import StoreError from synapse.metrics.background_process_metrics import run_as_background_process +from synapse.util.iterutils import batch_iter from synapse.storage._base import SQLBaseStore, make_in_list_sql_clause from synapse.storage.data_stores.main.events_worker import EventsWorkerStore from synapse.storage.data_stores.main.signatures import SignatureWorkerStore @@ -71,9 +72,7 @@ def _get_auth_chain_ids_txn(self, txn, event_ids, include_given): front = set(event_ids) while front: new_front = set() - front_list = list(front) - chunks = (front_list[x : x + 100] for x in range(0, len(front), 100)) - for chunk in chunks: + for chunk in batch_iter(front, 100): clause, args = make_in_list_sql_clause( txn.database_engine, "event_id", chunk ) From 1da0457bba1bb0ac65a6829aae76a2003e330e40 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 19 Feb 2020 15:21:53 +0000 Subject: [PATCH 5/5] pep --- synapse/storage/data_stores/main/event_federation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/storage/data_stores/main/event_federation.py b/synapse/storage/data_stores/main/event_federation.py index 9b89e56ae889..d75ab0767580 100644 --- a/synapse/storage/data_stores/main/event_federation.py +++ b/synapse/storage/data_stores/main/event_federation.py @@ -15,19 +15,18 @@ import itertools import logging -from six.moves import range from six.moves.queue import Empty, PriorityQueue from twisted.internet import defer from synapse.api.errors import StoreError from synapse.metrics.background_process_metrics import run_as_background_process -from synapse.util.iterutils import batch_iter from synapse.storage._base import SQLBaseStore, make_in_list_sql_clause from synapse.storage.data_stores.main.events_worker import EventsWorkerStore from synapse.storage.data_stores.main.signatures import SignatureWorkerStore from synapse.storage.database import Database from synapse.util.caches.descriptors import cached +from synapse.util.iterutils import batch_iter logger = logging.getLogger(__name__)