From 964a72b022bea565a10a73d6cce0a40542304a90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dion=20H=C3=A4fner?= Date: Fri, 28 Oct 2022 23:43:17 +0200 Subject: [PATCH] use globals instead of threading.local (fixes #281) (#282) * use globals instead of threading.local (fixes #281) * fix broken tesT * satisfy mypy --- terracotta/drivers/geotiff_raster_store.py | 15 ++++++------ terracotta/drivers/relational_meta_store.py | 2 +- tests/drivers/test_raster_drivers.py | 27 +++++++++++---------- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/terracotta/drivers/geotiff_raster_store.py b/terracotta/drivers/geotiff_raster_store.py index d187383b..a546a744 100644 --- a/terracotta/drivers/geotiff_raster_store.py +++ b/terracotta/drivers/geotiff_raster_store.py @@ -23,8 +23,7 @@ logger = logging.getLogger(__name__) -context = threading.local() -context.executor = None +_executor = None def create_executor() -> Executor: @@ -50,16 +49,18 @@ def create_executor() -> Executor: def submit_to_executor(task: Callable[..., Any]) -> Future: - if context.executor is None: - context.executor = create_executor() + global _executor + + if _executor is None: + _executor = create_executor() try: - future = context.executor.submit(task) + future = _executor.submit(task) except BrokenProcessPool: # re-create executor and try again logger.warn('Re-creating broken process pool') - context.executor = create_executor() - future = context.executor.submit(task) + _executor = create_executor() + future = _executor.submit(task) return future diff --git a/terracotta/drivers/relational_meta_store.py b/terracotta/drivers/relational_meta_store.py index 794809d3..be3f0b29 100644 --- a/terracotta/drivers/relational_meta_store.py +++ b/terracotta/drivers/relational_meta_store.py @@ -171,7 +171,7 @@ def version_tuple(version_string: str) -> Sequence[str]: ) self.db_version_verified = True - @property # type: ignore + @property @requires_connection @convert_exceptions(_ERROR_ON_CONNECT) def db_version(self) -> str: diff --git a/tests/drivers/test_raster_drivers.py b/tests/drivers/test_raster_drivers.py index c351eb4d..39690a89 100644 --- a/tests/drivers/test_raster_drivers.py +++ b/tests/drivers/test_raster_drivers.py @@ -504,31 +504,32 @@ def test_nodata_consistency(driver_path, provider, big_raster_file_mask, big_ras @pytest.mark.parametrize('provider', DRIVERS) -def test_broken_process_pool(driver_path, provider, raster_file): +def test_broken_process_pool(monkeypatch, driver_path, provider, raster_file): import concurrent.futures + import terracotta.drivers.geotiff_raster_store from terracotta import drivers - from terracotta.drivers.geotiff_raster_store import context class BrokenPool: def submit(self, *args, **kwargs): raise concurrent.futures.process.BrokenProcessPool('monkeypatched') - context.executor = BrokenPool() + with monkeypatch.context() as m: + m.setattr(terracotta.drivers.geotiff_raster_store, '_executor', BrokenPool()) - db = drivers.get_driver(driver_path, provider=provider) - keys = ('some', 'keynames') + db = drivers.get_driver(driver_path, provider=provider) + keys = ('some', 'keynames') - db.create(keys) - db.insert(['some', 'value'], str(raster_file)) - db.insert(['some', 'other_value'], str(raster_file)) + db.create(keys) + db.insert(['some', 'value'], str(raster_file)) + db.insert(['some', 'other_value'], str(raster_file)) - data1 = db.get_raster_tile(['some', 'value'], tile_size=(256, 256)) - assert data1.shape == (256, 256) + data1 = db.get_raster_tile(['some', 'value'], tile_size=(256, 256)) + assert data1.shape == (256, 256) - data2 = db.get_raster_tile(['some', 'other_value'], tile_size=(256, 256)) - assert data2.shape == (256, 256) + data2 = db.get_raster_tile(['some', 'other_value'], tile_size=(256, 256)) + assert data2.shape == (256, 256) - np.testing.assert_array_equal(data1, data2) + np.testing.assert_array_equal(data1, data2) def test_no_multiprocessing():