From 1b13767f8e0b3e8d011b07b6856cc0ae0673410c Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Tue, 18 Oct 2022 12:40:54 +1100 Subject: [PATCH 1/8] Make FlagBands aware of band aliases on the main product(s). --- datacube_ows/config_utils.py | 9 ++++++++- datacube_ows/ows_configuration.py | 12 +++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/datacube_ows/config_utils.py b/datacube_ows/config_utils.py index 25179eedc..4109c19b1 100644 --- a/datacube_ows/config_utils.py +++ b/datacube_ows/config_utils.py @@ -649,7 +649,8 @@ def __init__(self, cfg: CFG_DICT, product_cfg: "datacube_ows.ows_configuration.O pq_names = self.product.parse_pq_names(cfg) self.pq_names = pq_names["pq_names"] self.pq_low_res_names = pq_names["pq_low_res_names"] - self.pq_band = cfg["band"] + self.main_products = pq_names["main_products"] + self.pq_band = cfg["band"] # N.B. May be an alias if on main_products if "fuse_func" in cfg: self.pq_fuse_func: Optional[FunctionWrapper] = FunctionWrapper(self.product, cast(Mapping[str, Any], cfg["fuse_func"])) else: @@ -685,6 +686,12 @@ def make_ready(self, dc: "datacube.Datacube", *args, **kwargs) -> None: raise ConfigException(f"Could not find flags low_res product {pqn} for layer {self.product.name} in datacube") self.pq_low_res_products.append(pq_product) + # Resolve band alias if necessary. + if self.main_products: + canonical_band = self.product.band_idx.band(self.pq_band) + if canonical_band and canonical_band != self.pq_band: + self.pq_band = canonical_band + # pyre-ignore[16] self.info_mask: int = ~0 # A (hopefully) representative product diff --git a/datacube_ows/ows_configuration.py b/datacube_ows/ows_configuration.py index 42fb1d848..fbff298b7 100644 --- a/datacube_ows/ows_configuration.py +++ b/datacube_ows/ows_configuration.py @@ -921,17 +921,22 @@ def parse_product_names(self, cfg): raise ConfigException(f"'low_res_product_names' entry in non-multi-product layer {self.name} - use 'low_res_product_name' only") def parse_pq_names(self, cfg): + main_product = False if "dataset" in cfg: raise ConfigException(f"The 'dataset' entry in the flags section is no longer supported. Please refer to the documentation for the correct format (layer {self.name})") if "product" in cfg: pq_names = (cfg["product"],) else: pq_names = (self.product_name,) + main_product = (pq_names[0] == self.product_name) if "low_res_product" in cfg: pq_low_res_names = (cfg.get("low_res_product"),) - else: + elif main_product: pq_low_res_names = self.low_res_product_names + else: + pq_low_res_names = pq_names + if "products" in cfg: raise ConfigException(f"'products' entry in flags section of non-multi-product layer {self.name} - use 'product' only") if "low_res_products" in cfg: @@ -939,6 +944,7 @@ def parse_pq_names(self, cfg): return { "pq_names": pq_names, "pq_low_res_names": pq_low_res_names, + "main_products": main_product } @@ -959,11 +965,14 @@ def parse_product_names(self, cfg): raise ConfigException(f"'low_res_product_name' entry in multi-product layer {self.name} - use 'low_res_product_names' only") def parse_pq_names(self, cfg): + main_products = False if "datasets" in cfg: raise ConfigException(f"The 'datasets' entry in the flags section is no longer supported. Please refer to the documentation for the correct format (layer {self.name})") if "products" in cfg: pq_names = tuple(cfg["products"]) + main_products = pq_names == self.product_names else: + main_products =True pq_names = self.product_names if "low_res_products" in cfg: @@ -977,6 +986,7 @@ def parse_pq_names(self, cfg): return { "pq_names": pq_names, "pq_low_res_names": pq_low_res_names, + "main_products": main_products, } def dataset_groupby(self): From c1da066177c9d9afd5ff752fb39023094b49f266 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Tue, 18 Oct 2022 12:47:47 +1100 Subject: [PATCH 2/8] whitespace for flake8. --- datacube_ows/ows_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datacube_ows/ows_configuration.py b/datacube_ows/ows_configuration.py index fbff298b7..039f904fa 100644 --- a/datacube_ows/ows_configuration.py +++ b/datacube_ows/ows_configuration.py @@ -972,7 +972,7 @@ def parse_pq_names(self, cfg): pq_names = tuple(cfg["products"]) main_products = pq_names == self.product_names else: - main_products =True + main_products = True pq_names = self.product_names if "low_res_products" in cfg: From 0a2a3cddbb502225abd632603dd24576167af12f Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Tue, 18 Oct 2022 15:07:48 +1100 Subject: [PATCH 3/8] Bug fixes and test coverage. --- datacube_ows/config_utils.py | 14 ++++++++------ integration_tests/cfg/ows_test_cfg.py | 6 +++--- tests/test_styles.py | 1 + 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/datacube_ows/config_utils.py b/datacube_ows/config_utils.py index 4109c19b1..3d6eedfac 100644 --- a/datacube_ows/config_utils.py +++ b/datacube_ows/config_utils.py @@ -650,7 +650,8 @@ def __init__(self, cfg: CFG_DICT, product_cfg: "datacube_ows.ows_configuration.O self.pq_names = pq_names["pq_names"] self.pq_low_res_names = pq_names["pq_low_res_names"] self.main_products = pq_names["main_products"] - self.pq_band = cfg["band"] # N.B. May be an alias if on main_products + self.pq_band = cfg["band"] + self.canonical_band_name = self.pq_band # Update for aliasing on make_ready if "fuse_func" in cfg: self.pq_fuse_func: Optional[FunctionWrapper] = FunctionWrapper(self.product, cast(Mapping[str, Any], cfg["fuse_func"])) else: @@ -688,11 +689,12 @@ def make_ready(self, dc: "datacube.Datacube", *args, **kwargs) -> None: # Resolve band alias if necessary. if self.main_products: - canonical_band = self.product.band_idx.band(self.pq_band) - if canonical_band and canonical_band != self.pq_band: - self.pq_band = canonical_band + try: + self.canonical_band_name = self.product.band_idx.band(self.pq_band) + except ConfigException: + pass - # pyre-ignore[16] + # pyre-ignore[16] self.info_mask: int = ~0 # A (hopefully) representative product product = self.pq_products[0] @@ -730,7 +732,7 @@ def __init__(self, flag_band: FlagBand, super().__init__({}) self.layer = layer self.bands: Set[str] = set() - self.bands.add(flag_band.pq_band) + self.bands.add(flag_band.canonical_band_name) self.flag_bands = {flag_band.pq_band: flag_band} self.product_names = tuple(flag_band.pq_names) self.ignore_time = flag_band.pq_ignore_time diff --git a/integration_tests/cfg/ows_test_cfg.py b/integration_tests/cfg/ows_test_cfg.py index a518e07ac..74d2ecff9 100644 --- a/integration_tests/cfg/ows_test_cfg.py +++ b/integration_tests/cfg/ows_test_cfg.py @@ -62,7 +62,7 @@ "nbart_nir_2": ["nbar_nir_2", "nir2", "nbart_nir_2"], "nbart_swir_2": ["nbar_swir_2", "swir_2", "nbart_swir_2"], "nbart_swir_3": ["nbar_swir_3", "swir_3", "nbart_swir_3"], - "fmask": ["fmask"], + "fmask": ["fmask", "fmask_alias"], } @@ -70,7 +70,7 @@ s2_nrt_fmask = [ { - "band": "fmask", + "band": "fmask_alias", "values": [0, 2, 3], "invert": True, }, @@ -807,7 +807,7 @@ }, "flags": [ { - "band": "fmask", + "band": "fmask_alias", "products": ["s2a_ard_granule", "s2b_ard_granule"], "ignore_time": False, "ignore_info_flags": [] diff --git a/tests/test_styles.py b/tests/test_styles.py index 527ced7b1..82be5ead3 100644 --- a/tests/test_styles.py +++ b/tests/test_styles.py @@ -44,6 +44,7 @@ def __init__(self, pq_name, band): self.pq_products = [FakeODCProduct(pq_name)] self.pq_low_res_products = None self.pq_band = band + self.canonical_band_name = band self.pq_names = [pq_name] self.pq_ignore_time = False self.pq_manual_merge = False From 34d81a53ac1b5bbb5891eb8ae1d1d44b8c1ece3a Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Tue, 18 Oct 2022 15:26:09 +1100 Subject: [PATCH 4/8] Pin Jinja2>=3.2.0 for Py3.10 compatibility. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index d2e90a255..17e919b05 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ 'Pillow', 'Babel', 'Flask-Babel', + 'Jinja2>=3.2.0', # Python 3.10 compatibility 'psycopg2', 'python_dateutil', 'pytz', From a37e895a745a15cf5e7bd769ce1af90739f4878d Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Tue, 18 Oct 2022 15:30:50 +1100 Subject: [PATCH 5/8] Revert - Jinja2 fix not released yet. --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 17e919b05..d2e90a255 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,6 @@ 'Pillow', 'Babel', 'Flask-Babel', - 'Jinja2>=3.2.0', # Python 3.10 compatibility 'psycopg2', 'python_dateutil', 'pytz', From 800fe666bb28b92088bbc4accbf741f96c0b9708 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Tue, 18 Oct 2022 18:55:32 +1100 Subject: [PATCH 6/8] Fix bug in OWSFlagBand.make_ready. --- datacube_ows/config_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datacube_ows/config_utils.py b/datacube_ows/config_utils.py index 3d6eedfac..a3983e010 100644 --- a/datacube_ows/config_utils.py +++ b/datacube_ows/config_utils.py @@ -699,7 +699,7 @@ def make_ready(self, dc: "datacube.Datacube", *args, **kwargs) -> None: # A (hopefully) representative product product = self.pq_products[0] try: - meas = product.lookup_measurements([self.pq_band])[self.pq_band] + meas = product.lookup_measurements([self.canonical_band_name])[self.canonical_band_name] except KeyError: raise ConfigException( f"Band {self.pq_band} does not exist in product {product.name} - cannot be used as a flag band for layer {self.product.name}.") From 99aaf0bd81d95052fa7bd2eac1a650bbff0c9ea2 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Tue, 18 Oct 2022 18:56:11 +1100 Subject: [PATCH 7/8] Patch OWSFlagBandStandalone --- datacube_ows/config_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datacube_ows/config_utils.py b/datacube_ows/config_utils.py index a3983e010..be9f3b91c 100644 --- a/datacube_ows/config_utils.py +++ b/datacube_ows/config_utils.py @@ -625,6 +625,7 @@ class OWSFlagBandStandalone: """ def __init__(self, band: str) -> None: self.pq_band = band + self.canonical_band_name = band self.pq_names: List["datacube.model.DatasetType"] = [] self.pq_ignore_time = False self.pq_manual_merge = False From 761446bc7dfbeeac23d6863fdbc6a2a0e609054d Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Wed, 19 Oct 2022 10:18:34 +1100 Subject: [PATCH 8/8] Update documentation. --- docs/cfg_layers.rst | 6 +++++- .../cfg/translations/en/LC_MESSAGES/ows_cfg.po | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/cfg_layers.rst b/docs/cfg_layers.rst index e5c4a3dd1..9a0af50e3 100644 --- a/docs/cfg_layers.rst +++ b/docs/cfg_layers.rst @@ -1003,8 +1003,12 @@ The name of the measurement band to be used for style-based masking. Pixel-quality bitmask bands or enumeration flag bands can be used, although bitmask bands are better supported and are recommended where possible. +If the flag product(s) is/are the same as the main data product(s), then +an alias from the `bands dictionary <#bands-dictionary-bands>`_ may be used. + Note that it is not possible to combine flag bands from separate products -if they have the same band name. +if they have the same band name (unless one of the products is the main product +and a band alias is used.) Required. diff --git a/integration_tests/cfg/translations/en/LC_MESSAGES/ows_cfg.po b/integration_tests/cfg/translations/en/LC_MESSAGES/ows_cfg.po index 32bf2f773..16be0be30 100644 --- a/integration_tests/cfg/translations/en/LC_MESSAGES/ows_cfg.po +++ b/integration_tests/cfg/translations/en/LC_MESSAGES/ows_cfg.po @@ -6,7 +6,7 @@ msgid "" msgstr "" "Project-Id-Version: Over-ridden: aardvark 2022-03-24T23:29:57.407805\n" "Report-Msgid-Bugs-To: test@example.com\n" -"POT-Creation-Date: 2022-08-26 13:23+1000\n" +"POT-Creation-Date: 2022-10-18 13:28+1100\n" "PO-Revision-Date: 2022-03-24 23:33+0000\n" "Last-Translator: FULL NAME \n" "Language: en\n"