From 845d6e51ab0addb1eed4ea0a1c59183766be6044 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Wed, 8 May 2024 15:37:48 +0200 Subject: [PATCH 1/7] Correct bypassing of all HDF5-file parsers for files which do not have the magic number that is used for HDF5 container file formats --- pynxtools_em/reader.py | 24 +++++------ pynxtools_em/subparsers/hfive_oxford.py | 17 ++++---- pynxtools_em/subparsers/nxs_pyxem.py | 56 ++++++++++++++----------- 3 files changed, 52 insertions(+), 45 deletions(-) diff --git a/pynxtools_em/reader.py b/pynxtools_em/reader.py index 9fb6d9f..f2b872e 100644 --- a/pynxtools_em/reader.py +++ b/pynxtools_em/reader.py @@ -19,23 +19,19 @@ from os import getcwd from time import perf_counter_ns -from typing import Tuple, Any -import numpy as np +from typing import Any, Tuple +import numpy as np from pynxtools.dataconverter.readers.base.reader import BaseReader -from pynxtools_em.utils.io_case_logic import ( - EmUseCaseSelector, -) + +from pynxtools_em.concepts.nxs_concepts import NxEmAppDef +from pynxtools_em.subparsers.nxs_imgs import NxEmImagesSubParser # from pynxtools_em.subparsers.nxs_mtex import NxEmNxsMTexSubParser from pynxtools_em.subparsers.nxs_pyxem import NxEmNxsPyxemSubParser -from pynxtools_em.subparsers.nxs_imgs import NxEmImagesSubParser -# from pynxtools_em.subparsers.nxs_nion import NxEmZippedNionProjectSubParser -from pynxtools_em.subparsers.rsciio_velox import RsciioVeloxSubParser ## from pynxtools_em.utils.default_plots import NxEmDefaultPlotResolver # from pynxtools_em.geometry.convention_mapper import NxEmConventionMapper - # remaining subparsers to be implemented and merged into this one # from pynxtools.dataconverter.readers.em_om.utils.orix_ebsd_parser \ # import NxEmOmOrixEbsdParser @@ -45,12 +41,16 @@ # import NxEmOmZipEbsdParser # from pynxtools.dataconverter.readers.em_om.utils.em_nexus_plots \ # import em_om_default_plot_generator - from pynxtools_em.subparsers.oasis_config_reader import ( NxEmNomadOasisConfigurationParser, ) from pynxtools_em.subparsers.oasis_eln_reader import NxEmNomadOasisElnSchemaParser -from pynxtools_em.concepts.nxs_concepts import NxEmAppDef + +# from pynxtools_em.subparsers.nxs_nion import NxEmZippedNionProjectSubParser +from pynxtools_em.subparsers.rsciio_velox import RsciioVeloxSubParser +from pynxtools_em.utils.io_case_logic import ( + EmUseCaseSelector, +) class EMReader(BaseReader): @@ -146,7 +146,7 @@ def read( # if resolved_path != "": # nxs_plt.annotate_default_plot(template, resolved_path) - debugging = True # print(template) + debugging = False if debugging: print("Reporting state of template before passing to HDF5 writing...") for keyword in template.keys(): diff --git a/pynxtools_em/subparsers/hfive_oxford.py b/pynxtools_em/subparsers/hfive_oxford.py index 51bca1b..7418ffb 100644 --- a/pynxtools_em/subparsers/hfive_oxford.py +++ b/pynxtools_em/subparsers/hfive_oxford.py @@ -17,21 +17,22 @@ # """(Sub-)parser mapping concepts and content from Oxford Instruments *.h5oina files on NXem.""" -import numpy as np -import h5py from typing import Dict + +import h5py +import numpy as np from diffpy.structure import Lattice, Structure +from pynxtools_em.examples.ebsd_database import ( + FLIGHT_PLAN, + REGULAR_TILING, + SQUARE_GRID, +) from pynxtools_em.subparsers.hfive_base import HdfFiveBaseParser from pynxtools_em.utils.hfive_utils import ( - read_strings_from_dataset, format_euler_parameterization, + read_strings_from_dataset, ) -from pynxtools_em.examples.ebsd_database import ( - SQUARE_GRID, - REGULAR_TILING, - FLIGHT_PLAN, -) # HEXAGONAL_GRID class HdfFiveOxfordReader(HdfFiveBaseParser): diff --git a/pynxtools_em/subparsers/nxs_pyxem.py b/pynxtools_em/subparsers/nxs_pyxem.py index bfa7338..66a35be 100644 --- a/pynxtools_em/subparsers/nxs_pyxem.py +++ b/pynxtools_em/subparsers/nxs_pyxem.py @@ -32,43 +32,42 @@ # task for the community and instead focus here on showing a more diverse example # towards more interoperability between the different tools in the community +import mmap import os -import numpy as np -from PIL import Image as pil -from orix import plot import matplotlib.pyplot as plt # in the hope that this closes figures with orix plot +import numpy as np +from orix import plot from orix.quaternion import Rotation from orix.quaternion.symmetry import get_point_group from orix.vector import Vector3d +from PIL import Image as pil +from pynxtools_em.concepts.nxs_image_r_set import NxImageRealSpaceSet +from pynxtools_em.subparsers.hfive_apex import HdfFiveEdaxApexReader +from pynxtools_em.subparsers.hfive_bruker import HdfFiveBrukerEspritReader +from pynxtools_em.subparsers.hfive_dreamthreed import HdfFiveDreamThreedReader +from pynxtools_em.subparsers.hfive_ebsd import HdfFiveCommunityReader +from pynxtools_em.subparsers.hfive_edax import HdfFiveEdaxOimAnalysisReader +from pynxtools_em.subparsers.hfive_emsoft import HdfFiveEmSoftReader +from pynxtools_em.subparsers.hfive_oxford import HdfFiveOxfordReader +from pynxtools_em.utils.get_scan_points import ( + get_scan_point_axis_values, + get_scan_point_coords, + hexagonal_grid, + square_grid, + threed, +) +from pynxtools_em.utils.get_sqr_grid import ( + get_scan_points_with_mark_data_discretized_on_sqr_grid, +) from pynxtools_em.utils.hfive_utils import read_strings_from_dataset from pynxtools_em.utils.hfive_web_constants import ( - HFIVE_WEB_MAXIMUM_ROI, HFIVE_WEB_MAXIMUM_RGB, + HFIVE_WEB_MAXIMUM_ROI, ) from pynxtools_em.utils.hfive_web_utils import hfive_web_decorate_nxdata from pynxtools_em.utils.image_processing import thumbnail -from pynxtools_em.utils.get_sqr_grid import ( - get_scan_points_with_mark_data_discretized_on_sqr_grid, -) -from pynxtools_em.utils.get_scan_points import ( - square_grid, - hexagonal_grid, - threed, - get_scan_point_axis_values, - get_scan_point_coords, -) - -from pynxtools_em.subparsers.hfive_oxford import HdfFiveOxfordReader -from pynxtools_em.subparsers.hfive_bruker import HdfFiveBrukerEspritReader -from pynxtools_em.subparsers.hfive_edax import HdfFiveEdaxOimAnalysisReader -from pynxtools_em.subparsers.hfive_apex import HdfFiveEdaxApexReader -from pynxtools_em.subparsers.hfive_ebsd import HdfFiveCommunityReader -from pynxtools_em.subparsers.hfive_emsoft import HdfFiveEmSoftReader -from pynxtools_em.subparsers.hfive_dreamthreed import HdfFiveDreamThreedReader -from pynxtools_em.concepts.nxs_image_r_set import NxImageRealSpaceSet - PROJECTION_VECTORS = [Vector3d.xvector(), Vector3d.yvector(), Vector3d.zvector()] PROJECTION_DIRECTIONS = [ @@ -128,7 +127,9 @@ def __init__(self, entry_id: int = 1, file_path: str = "", verbose: bool = False def parse(self, template: dict) -> dict: hfive_parser_type = self.identify_hfive_type() if hfive_parser_type is None: - print(f"{self.file_path} does not match any of the supported HDF5 formats") + print( + f"Parser PyXem/NeXus/HDF5 finds that {self.file_path} does not match any of the supported HDF5 formats" + ) return template print( f"Parsing via {self.__class__.__name__} content type {hfive_parser_type}..." @@ -200,6 +201,11 @@ def parse(self, template: dict) -> dict: def identify_hfive_type(self): """Identify if HDF5 file matches a known format for which a subparser exists.""" + with open(self.file_path, "rb", 0) as file: + s = mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) + magic = s.read(4) + if magic != rb"\89HDF": + return None # tech partner formats used for measurement hdf = HdfFiveOxfordReader(self.file_path) if hdf.supported is True: From de648b2e141c5a80337af262d80b2797b612aff5 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Wed, 8 May 2024 16:33:47 +0200 Subject: [PATCH 2/7] Reduce prints to stdout --- pynxtools_em/subparsers/image_base.py | 6 ++- .../subparsers/image_png_protochips.py | 31 +++++++------- pynxtools_em/subparsers/image_tiff_tfs.py | 41 +++++++++++-------- pynxtools_em/subparsers/nxs_pyxem.py | 2 +- .../subparsers/oasis_config_reader.py | 5 +-- pynxtools_em/subparsers/oasis_eln_reader.py | 5 +-- 6 files changed, 48 insertions(+), 42 deletions(-) diff --git a/pynxtools_em/subparsers/image_base.py b/pynxtools_em/subparsers/image_base.py index 1516ff3..bdad908 100644 --- a/pynxtools_em/subparsers/image_base.py +++ b/pynxtools_em/subparsers/image_base.py @@ -17,12 +17,13 @@ # """Parent class for all tech partner-specific image parsers for mapping on NXem.""" -import numpy as np from typing import Dict, List +import numpy as np + class ImgsBaseParser: - def __init__(self, file_path: str = ""): + def __init__(self, file_path: str = "", verbose=False): # self.supported_version = VERSION_MANAGEMENT # self.version = VERSION_MANAGEMENT # tech_partner the company which designed this format @@ -31,6 +32,7 @@ def __init__(self, file_path: str = ""): # writer_name the specific name of the tech_partner's (typically proprietary) software self.prfx = None self.tmp: Dict = {} + self.verbose = verbose if file_path is not None and file_path != "": self.file_path = file_path else: diff --git a/pynxtools_em/subparsers/image_png_protochips.py b/pynxtools_em/subparsers/image_png_protochips.py index 254bfcd..b2e2e49 100644 --- a/pynxtools_em/subparsers/image_png_protochips.py +++ b/pynxtools_em/subparsers/image_png_protochips.py @@ -17,36 +17,37 @@ # """Subparser for exemplar reading of raw PNG files collected on a TEM with Protochip heating_chip.""" +import datetime import mmap import re +from typing import Dict, List +from zipfile import ZipFile + +import flatdict as fd import numpy as np import xmltodict -import datetime -import flatdict as fd -from typing import Dict, List from PIL import Image -from zipfile import ZipFile +from pynxtools_em.concepts.concept_mapper import ( + add_specific_metadata, + variadic_path_to_specific_path, +) from pynxtools_em.config.image_png_protochips_cfg import ( - specific_to_variadic, + AXON_AUX_DYNAMIC_TO_NX_EM, + AXON_CHIP_DYNAMIC_TO_NX_EM, AXON_DETECTOR_STATIC_TO_NX_EM, - AXON_STAGE_STATIC_TO_NX_EM, AXON_STAGE_DYNAMIC_TO_NX_EM, - AXON_CHIP_DYNAMIC_TO_NX_EM, - AXON_AUX_DYNAMIC_TO_NX_EM, + AXON_STAGE_STATIC_TO_NX_EM, AXON_VARIOUS_DYNAMIC_TO_NX_EM, -) -from pynxtools_em.concepts.concept_mapper import ( - variadic_path_to_specific_path, - add_specific_metadata, + specific_to_variadic, ) from pynxtools_em.subparsers.image_base import ImgsBaseParser -from pynxtools_em.utils.xml_utils import flatten_xml_to_dict from pynxtools_em.utils.get_file_checksum import ( - get_sha256_of_file_content, DEFAULT_CHECKSUM_ALGORITHM, + get_sha256_of_file_content, ) from pynxtools_em.utils.sorting import sort_ascendingly_by_second_argument_iso8601 +from pynxtools_em.utils.xml_utils import flatten_xml_to_dict class ProtochipsPngSetSubParser(ImgsBaseParser): @@ -329,7 +330,7 @@ def process_event_data_em_metadata(self, template: dict) -> dict: """Add respective metadata.""" # contextualization to understand how the image relates to the EM session print( - f"Mapping some of the Protochips-specific metadata on respective NeXus concept instance" + f"Mapping some of the Protochips metadata on respective NeXus concepts..." ) # individual PNGs in self.file_path may include time/date information in the file name # surplus eventually AXON-specific identifier it seems useful though to sort these diff --git a/pynxtools_em/subparsers/image_tiff_tfs.py b/pynxtools_em/subparsers/image_tiff_tfs.py index b012f8f..68278c7 100644 --- a/pynxtools_em/subparsers/image_tiff_tfs.py +++ b/pynxtools_em/subparsers/image_tiff_tfs.py @@ -18,31 +18,32 @@ """Subparser for harmonizing ThermoFisher-specific content in TIFF files.""" import mmap -import numpy as np -import flatdict as fd from typing import Dict + +import flatdict as fd +import numpy as np from PIL import Image from PIL.TiffTags import TAGS -from pynxtools_em.subparsers.image_tiff import TiffSubParser -from pynxtools_em.subparsers.image_tiff_tfs_concepts import ( - get_fei_parent_concepts, - get_fei_childs, -) +from pynxtools_em.concepts.concept_mapper import add_specific_metadata from pynxtools_em.config.image_tiff_tfs_cfg import ( TFS_APERTURE_STATIC_TO_NX_EM, TFS_DETECTOR_STATIC_TO_NX_EM, - TFS_VARIOUS_STATIC_TO_NX_EM, TFS_OPTICS_DYNAMIC_TO_NX_EM, - TFS_STAGE_DYNAMIC_TO_NX_EM, TFS_SCAN_DYNAMIC_TO_NX_EM, + TFS_STAGE_DYNAMIC_TO_NX_EM, TFS_VARIOUS_DYNAMIC_TO_NX_EM, + TFS_VARIOUS_STATIC_TO_NX_EM, +) +from pynxtools_em.subparsers.image_tiff import TiffSubParser +from pynxtools_em.subparsers.image_tiff_tfs_concepts import ( + get_fei_childs, + get_fei_parent_concepts, ) from pynxtools_em.utils.image_utils import ( - sort_ascendingly_by_second_argument, if_str_represents_float, + sort_ascendingly_by_second_argument, ) -from pynxtools_em.concepts.concept_mapper import add_specific_metadata class TfsTiffSubParser(TiffSubParser): @@ -87,7 +88,7 @@ def check_if_tiff_tfs(self): def get_metadata(self): """Extract metadata in TFS specific tags if present.""" - print("Reporting the tags found in this TIFF file...") + print("Parsing TIFF tags...") # for an overview of tags # https://www.loc.gov/preservation/digital/formats/content/tiff_tags.shtml # with Image.open(self.file_path, mode="r") as fp: @@ -104,9 +105,10 @@ def get_metadata(self): pos = s.find(bytes(f"[{concept}]", "utf8")) # != -1 if pos != -1: tfs_parent_concepts_byte_offset[concept] = pos - else: - print(f"Instance of concept [{concept}] was not found !") - print(tfs_parent_concepts_byte_offset) + # else: + # print(f"Instance of concept [{concept}] was not found !") + if self.verbose: + print(tfs_parent_concepts_byte_offset) sequence = [] # decide I/O order in which metadata for childs of parent concepts will be read for key, value in tfs_parent_concepts_byte_offset.items(): @@ -114,7 +116,8 @@ def get_metadata(self): sequence.append((key, value)) # tuple of parent_concept name and byte offset sequence = sort_ascendingly_by_second_argument(sequence) - print(sequence) + if self.verbose: + print(sequence) idx = 0 for parent, byte_offset in sequence: @@ -183,7 +186,9 @@ def process_into_template(self, template: dict) -> dict: def process_event_data_em_data(self, template: dict) -> dict: """Add respective heavy data.""" # default display of the image(s) representing the data collected in this event - print(f"Writing TFS/FEI TIFF image as a onto the respective NeXus concept") + print( + f"Writing TFS/FEI TIFF image data to the respective NeXus concept instances..." + ) # read image in-place with Image.open(self.file_path, mode="r") as fp: nparr = np.array(fp) @@ -310,7 +315,7 @@ def add_various_dynamic_metadata(self, template: dict) -> dict: def process_event_data_em_metadata(self, template: dict) -> dict: """Add respective metadata.""" # contextualization to understand how the image relates to the EM session - print(f"Mapping some of the TFS/FEI metadata concepts onto NeXus concepts") + print(f"Mapping some of the TFS/FEI metadata on respective NeXus concepts...") self.add_aperture_static_metadata(template) self.add_detector_static_metadata(template) self.add_various_static_metadata(template) diff --git a/pynxtools_em/subparsers/nxs_pyxem.py b/pynxtools_em/subparsers/nxs_pyxem.py index 66a35be..9659cc2 100644 --- a/pynxtools_em/subparsers/nxs_pyxem.py +++ b/pynxtools_em/subparsers/nxs_pyxem.py @@ -204,7 +204,7 @@ def identify_hfive_type(self): with open(self.file_path, "rb", 0) as file: s = mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) magic = s.read(4) - if magic != rb"\89HDF": + if magic != b"\x89HDF": return None # tech partner formats used for measurement hdf = HdfFiveOxfordReader(self.file_path) diff --git a/pynxtools_em/subparsers/oasis_config_reader.py b/pynxtools_em/subparsers/oasis_config_reader.py index 3c21184..2a5b2c5 100644 --- a/pynxtools_em/subparsers/oasis_config_reader.py +++ b/pynxtools_em/subparsers/oasis_config_reader.py @@ -30,9 +30,8 @@ import flatdict as fd import yaml - -from pynxtools_em.config.oasis_cfg import EM_EXAMPLE_CSYS_TO_NEXUS from pynxtools_em.concepts.concept_mapper import variadic_path_to_specific_path +from pynxtools_em.config.oasis_cfg import EM_EXAMPLE_CSYS_TO_NEXUS class NxEmNomadOasisConfigurationParser: @@ -40,7 +39,7 @@ class NxEmNomadOasisConfigurationParser: def __init__(self, file_path: str, entry_id: int, verbose: bool = False): print( - f"Extracting data from deployment-specific configuration file: {file_path}" + f"Extracting data from deployment-specific configuration file {file_path} ..." ) if ( file_path.rsplit("/", 1)[-1].endswith(".oasis.specific.yaml") diff --git a/pynxtools_em/subparsers/oasis_eln_reader.py b/pynxtools_em/subparsers/oasis_eln_reader.py index 1e3dddc..2674faa 100644 --- a/pynxtools_em/subparsers/oasis_eln_reader.py +++ b/pynxtools_em/subparsers/oasis_eln_reader.py @@ -20,13 +20,12 @@ import flatdict as fd import yaml - +from pynxtools_em.concepts.concept_mapper import variadic_path_to_specific_path from pynxtools_em.config.eln_cfg import ( EM_EXAMPLE_ENTRY_TO_NEXUS, EM_EXAMPLE_SAMPLE_TO_NEXUS, EM_EXAMPLE_USER_TO_NEXUS, ) -from pynxtools_em.concepts.concept_mapper import variadic_path_to_specific_path class NxEmNomadOasisElnSchemaParser: @@ -37,7 +36,7 @@ class NxEmNomadOasisElnSchemaParser: """ def __init__(self, file_path: str, entry_id: int, verbose: bool = False): - print(f"Extracting data from ELN file: {file_path}") + print(f"Extracting data from ELN file {file_path} ...") if ( file_path.rsplit("/", 1)[-1].startswith("eln_data") or file_path.startswith("eln_data") From c5508bd231f3ab08f06e5041b3c83dd1cf5d2951 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Thu, 9 May 2024 02:08:38 +0200 Subject: [PATCH 3/7] Added harmonized cfg as in atom probe, reactivated ZipNionSwiftParser and checked that it loops non zip.nion through --- pynxtools_em/concepts/concept_mapper.py | 6 +- pynxtools_em/concepts/mapping_functors.py | 346 ++++++++++++++++++ pynxtools_em/concepts/nxs_concepts.py | 55 +-- pynxtools_em/config/eln_cfg.py | 49 ++- pynxtools_em/config/oasis_cfg.py | 6 +- pynxtools_em/config/rsciio_velox_cfg.py | 4 +- pynxtools_em/reader.py | 8 +- pynxtools_em/subparsers/nxs_nion.py | 51 +-- .../subparsers/oasis_config_reader.py | 35 +- pynxtools_em/subparsers/oasis_eln_reader.py | 114 ++---- pynxtools_em/utils/interpret_boolean.py | 38 ++ pynxtools_em/utils/versioning.py | 34 ++ 12 files changed, 564 insertions(+), 182 deletions(-) create mode 100644 pynxtools_em/concepts/mapping_functors.py create mode 100644 pynxtools_em/utils/interpret_boolean.py create mode 100644 pynxtools_em/utils/versioning.py diff --git a/pynxtools_em/concepts/concept_mapper.py b/pynxtools_em/concepts/concept_mapper.py index 8a9ad49..c2201cf 100644 --- a/pynxtools_em/concepts/concept_mapper.py +++ b/pynxtools_em/concepts/concept_mapper.py @@ -17,10 +17,14 @@ # """Utilities for working with NeXus concepts encoded as Python dicts in the concepts dir.""" +# these mapping functions are considered deprecated and +# should be replaced with those from mapping_functors !! + from datetime import datetime -import pytz + import flatdict as fd import numpy as np +import pytz from pynxtools_em.utils.string_conversions import string_to_number diff --git a/pynxtools_em/concepts/mapping_functors.py b/pynxtools_em/concepts/mapping_functors.py new file mode 100644 index 0000000..c88a677 --- /dev/null +++ b/pynxtools_em/concepts/mapping_functors.py @@ -0,0 +1,346 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Utilities for working with NeXus concepts encoded as Python dicts in the concepts dir.""" + +from datetime import datetime + +import flatdict as fd +import numpy as np +import pytz + +from pynxtools_em.utils.get_file_checksum import get_sha256_of_file_content +from pynxtools_em.utils.interpret_boolean import try_interpret_as_boolean +from pynxtools_em.utils.string_conversions import rchop, string_to_number + + +def variadic_path_to_specific_path(path: str, instance_identifier: list): + """Transforms a variadic path to an actual path with instances.""" + if (path is not None) and (path != ""): + narguments = path.count("*") + if narguments == 0: # path is not variadic + return path + if len(instance_identifier) >= narguments: + tmp = path.split("*") + if len(tmp) == narguments + 1: + nx_specific_path = "" + for idx in range(0, narguments): + nx_specific_path += f"{tmp[idx]}{instance_identifier[idx]}" + idx += 1 + nx_specific_path += f"{tmp[-1]}" + return nx_specific_path + return None + + +def add_specific_metadata( + concept_mapping: dict, orgmeta: fd.FlatDict, identifier: list, template: dict +) -> dict: + """Map specific concept src on specific NeXus concept trg. + + concept_mapping: translation dict how trg and src are to be mapped + orgmeta: instance data of src concepts + identifier: list of identifier to resolve variadic paths + template: instance data resulting from a resolved src to trg concept mapping + """ + if "prefix_trg" in concept_mapping: + variadic_prefix_trg = concept_mapping["prefix_trg"] + elif "prefix" in concept_mapping: + variadic_prefix_trg = concept_mapping["prefix"] + else: + raise KeyError(f"Neither prefix nor prefix_trg found in concept_mapping!") + + if "prefix_src" in concept_mapping: + prefix_src = concept_mapping["prefix_src"] + else: + prefix_src = "" + + # process all mapping functors + # (in graphical programming these are also referred to as filters or nodes), i.e. + # an agent that gets some input does some (maybe abstract mapping) and returns an output + # as the mapping can be abstract we call it functor + if "use" in concept_mapping: + for entry in concept_mapping["use"]: + if isinstance(entry, tuple): + if len(entry) == 2: + if isinstance(entry[1], str) and entry[1] == "": + continue + trg = variadic_path_to_specific_path( + f"{variadic_prefix_trg}/{entry[0]}", identifier + ) + template[f"{trg}"] = entry[1] + if "map" in concept_mapping: + for entry in concept_mapping["map"]: + if isinstance(entry, str): + if f"{prefix_src}{entry}" not in orgmeta: + continue + if ( + isinstance(orgmeta[f"{prefix_src}{entry}"], str) + and orgmeta[f"{prefix_src}{entry}"] == "" + ): + continue + trg = variadic_path_to_specific_path( + f"{variadic_prefix_trg}/{entry}", identifier + ) + template[f"{trg}"] = orgmeta[f"{prefix_src}{entry}"] + if isinstance(entry, tuple): + if len(entry) == 2: + if isinstance(entry[0], str): + if f"{prefix_src}{entry[1]}" not in orgmeta: + continue + if (orgmeta[f"{prefix_src}{entry[1]}"], str) and orgmeta[ + f"{prefix_src}{entry[1]}" + ] == "": + continue + trg = variadic_path_to_specific_path( + f"{variadic_prefix_trg}/{entry[0]}", identifier + ) + template[f"{trg}"] = orgmeta[f"{prefix_src}{entry[1]}"] + if "map_to_str" in concept_mapping: + for entry in concept_mapping["map_to_str"]: + if isinstance(entry, str): + if f"{prefix_src}{entry}" not in orgmeta: + continue + if ( + isinstance(orgmeta[f"{prefix_src}{entry}"], str) + and orgmeta[f"{prefix_src}{entry}"] == "" + ): + continue + trg = variadic_path_to_specific_path( + f"{variadic_prefix_trg}/{entry}", identifier + ) + template[f"{trg}"] = orgmeta[f"{prefix_src}{entry}"] + if isinstance(entry, tuple): + if len(entry) == 2: + if all(isinstance(elem, str) for elem in entry): + if f"{prefix_src}{entry[1]}" not in orgmeta: + continue + if ( + isinstance(orgmeta[f"{prefix_src}{entry[1]}"], str) + and orgmeta[f"{prefix_src}{entry[1]}"] == "" + ): + continue + trg = variadic_path_to_specific_path( + f"{variadic_prefix_trg}/{entry[0]}", identifier + ) + template[f"{trg}"] = orgmeta[f"{prefix_src}{entry[1]}"] + if "map_to_bool" in concept_mapping: + for entry in concept_mapping["map_to_bool"]: + if isinstance(entry, str): + if f"{prefix_src}{entry[0]}" not in orgmeta: + continue + trg = variadic_path_to_specific_path( + f"{variadic_prefix_trg}/{entry[0]}", identifier + ) + template[f"{trg}"] = try_interpret_as_boolean( + orgmeta[f"{prefix_src}{entry[0]}"] + ) + if isinstance(entry, tuple): + if len(entry) == 2: + if all(isinstance(elem, str) for elem in entry): + if f"{prefix_src}{entry[1]}" not in orgmeta: + continue + trg = variadic_path_to_specific_path( + f"{variadic_prefix_trg}/{entry[0]}", identifier + ) + template[f"{trg}"] = try_interpret_as_boolean( + orgmeta[f"{prefix_src}{entry[1]}"] + ) + if "map_to_real" in concept_mapping: + for entry in concept_mapping["map_to_real"]: + if isinstance(entry, str): + if isinstance(entry[0], str): + if f"{prefix_src}{entry[0]}" not in orgmeta: + continue + if ( + isinstance(orgmeta[f"{prefix_src}{entry[0]}"], str) + and orgmeta[f"{prefix_src}{entry[0]}"] == "" + ): + continue + trg = variadic_path_to_specific_path( + f"{variadic_prefix_trg}/{entry[0]}", identifier + ) + template[f"{trg}"] = string_to_number( + orgmeta[f"{prefix_src}{entry[0]}"] + ) + if isinstance(entry, tuple): + if len(entry) == 2: + if all(isinstance(elem, str) for elem in entry): + if f"{prefix_src}{entry[1]}" not in orgmeta: + continue + if ( + isinstance(orgmeta[f"{prefix_src}{entry[0]}"], str) + and orgmeta[f"{prefix_src}{entry[1]}"] == "" + ): + continue + trg = variadic_path_to_specific_path( + f"{variadic_prefix_trg}/{entry[0]}", identifier + ) + template[f"{trg}"] = string_to_number( + orgmeta[f"{prefix_src}{entry[1]}"] + ) + elif isinstance(entry[0], str) and isinstance(entry[1], list): + if not all( + ( + isinstance(value, str) + and f"{prefix_src}{value}" in orgmeta + ) + for value in entry[1] + ): + continue + if not all( + ( + isinstance(orgmeta[f"{prefix_src}{value}"], str) + and orgmeta[f"{prefix_src}{value}"] != "" + ) + for value in entry[1] + ): + continue + trg = variadic_path_to_specific_path( + f"{variadic_prefix_trg}/{entry[0]}", identifier + ) + res = [] + for value in entry[1]: + res.append( + string_to_number(orgmeta[f"{prefix_src}{value}"]) + ) + template[f"{trg}"] = np.asarray(res, np.float64) + if "map_to_real_and_multiply" in concept_mapping: + for entry in concept_mapping["map_to_real_and_multiply"]: + if isinstance(entry, tuple): + if len(entry) == 3: + if ( + isinstance(entry[0], str) + and isinstance(entry[1], str) + and isinstance(entry[2], float) + ): + if f"{prefix_src}{entry[1]}" not in orgmeta: + continue + if ( + isinstance(orgmeta[f"{prefix_src}{entry[1]}"], str) + and orgmeta[f"{prefix_src}{entry[1]}"] == "" + ): + continue + trg = variadic_path_to_specific_path( + f"{variadic_prefix_trg}/{entry[0]}", identifier + ) + template[f"{trg}"] = entry[2] * string_to_number( + orgmeta[f"{prefix_src}{entry[1]}"] + ) + if "map_to_real_and_join" in concept_mapping: + for entry in concept_mapping["map_to_real_and_join"]: + if isinstance(entry, tuple): + if len(entry) == 2: + if isinstance(entry[0], str) and isinstance(entry[1], list): + if not all( + ( + isinstance(value, str) + and f"{prefix_src}{value}" in orgmeta + ) + for value in entry[1] + ): + continue + if not all( + ( + isinstance(orgmeta[f"{prefix_src}{value}"], str) + and orgmeta[f"{prefix_src}{value}"] != "" + ) + for value in entry[1] + ): + continue + res = [] + for value in entry[1]: + res.append( + string_to_number(orgmeta[f"{prefix_src}{value}"]) + ) + trg = variadic_path_to_specific_path( + f"{variadic_prefix_trg}/{entry[0]}", identifier + ) + template[f"{trg}"] = np.asarray(res) + # we may need to be more specific with the return datatype here, currently default python float + if "unix_to_iso8601" in concept_mapping: + for entry in concept_mapping["unix_to_iso8601"]: + if isinstance(entry, tuple): + if ( + 2 <= len(entry) <= 3 + ): # trg, src, timestamp or empty string (meaning utc) + if all(isinstance(elem, str) for elem in entry): + if f"{prefix_src}{entry[1]}" not in orgmeta: + continue + if ( + isinstance(orgmeta[f"{prefix_src}{entry[1]}"], str) + and orgmeta[f"{prefix_src}{entry[1]}"] == "" + ): + continue + tzone = "UTC" + if len(entry) == 3: + # if not isinstance(entry[2], str): + # raise TypeError(f"{tzone} needs to be of type string!") + tzone = entry[2] + if tzone not in pytz.all_timezones: + raise ValueError( + f"{tzone} is not a timezone in pytz.all_timezones!" + ) + trg = variadic_path_to_specific_path( + f"{variadic_prefix_trg}/{entry[0]}", identifier + ) + template[f"{trg}"] = datetime.fromtimestamp( + int(orgmeta[f"{prefix_src}{entry[1]}"]), + tz=pytz.timezone(tzone), + ).isoformat() + if "join_str" in concept_mapping: # currently also joining empty strings + for entry in concept_mapping["join_str"]: + if isinstance(entry, tuple): + if len(entry) == 2: + if isinstance(entry[0], str) and isinstance(entry[1], list): + if not all( + ( + isinstance(value, str) + and f"{prefix_src}{value}" in orgmeta + ) + for value in entry[1] + ): + continue + trg = variadic_path_to_specific_path( + f"{variadic_prefix_trg}/{entry[0]}", identifier + ) + res = [] + for value in entry[1]: + res.append(orgmeta[f"{prefix_src}{value}"]) + template[f"{trg}"] = " ".join(res) + if "sha256" in concept_mapping: + for entry in concept_mapping["sha256"]: + if isinstance(entry, tuple): + if len(entry) == 2: + if not all(isinstance(elem, str) for elem in entry): + continue + if f"{prefix_src}{entry[1]}" not in orgmeta: + continue + if orgmeta[f"{prefix_src}{entry[1]}"] == "": + continue + trg = variadic_path_to_specific_path( + f"{variadic_prefix_trg}/{entry[0]}", identifier + ) + with open(orgmeta[f"{prefix_src}{entry[1]}"], "rb") as fp: + template[f"{rchop(trg, 'checksum')}checksum"] = ( + get_sha256_of_file_content(fp) + ) + template[f"{rchop(trg, 'checksum')}type"] = "file" + template[f"{rchop(trg, 'checksum')}path"] = orgmeta[ + f"{prefix_src}{entry[1]}" + ] + template[f"{rchop(trg, 'checksum')}algorithm"] = "sha256" + return template diff --git a/pynxtools_em/concepts/nxs_concepts.py b/pynxtools_em/concepts/nxs_concepts.py index eee464b..44769fc 100644 --- a/pynxtools_em/concepts/nxs_concepts.py +++ b/pynxtools_em/concepts/nxs_concepts.py @@ -17,25 +17,38 @@ # """Implement NeXus-specific groups and fields to document software and versions used.""" -from pynxtools_em.concepts.concept_mapper import variadic_path_to_specific_path +from pynxtools_em.concepts.mapping_functors import ( + add_specific_metadata, + variadic_path_to_specific_path, +) +from pynxtools_em.utils.versioning import ( + NX_EM_ADEF_NAME, + NX_EM_EXEC_NAME, + NX_EM_EXEC_VERSION, +) +EM_APPDEF_TO_NEXUS = { + "prefix_trg": "/ENTRY[entry*]", + "use": [ + ("definition", NX_EM_ADEF_NAME), + ( + "definition/@version", + "Redundant, see metadata in NXroot header, the specific version of pynxtools has only one specific set of definitions with its version.", + ), + ], +} -PYNXTOOLS_EM_VERSION = "Redundant, see metadata in NXroot header" -PYNXTOOLS_EM_URL = "Redundant, see metadata in NXroot header" - -NXEM_NAME = "NXem" -NXEM_VERSION = "Redundant, see metadata in NXroot header" -NXEM_URL = "Redundant, see metadata in NXroot header" -EM_APPDEF = { - "prefix": "/ENTRY[entry*]", +EM_PYNX_TO_NEXUS = { + "prefix_trg": "/ENTRY[entry*]/profiling", "use": [ - ("PROGRAM[program1]/program", "pynxtools/dataconverter/readers/em"), - ("PROGRAM[program1]/program/@version", PYNXTOOLS_EM_VERSION), - ("PROGRAM[program1]/program/@url", PYNXTOOLS_EM_URL), - ("definition", NXEM_NAME), - ("definition/@version", NXEM_VERSION), - ("definition/@url", NXEM_URL), + ("PROGRAM[program1]/program", NX_EM_EXEC_NAME), + ("PROGRAM[program1]/program/@version", NX_EM_EXEC_VERSION), + # ("definition", NX_EM_ADEF_NAME), + # ( + # "definition/@version", + # "Redundant, see metadata in NXroot header, the specific version of pynxtools has only one specific set of definitions with its version.", + # ), ], } @@ -50,15 +63,11 @@ def parse( self, template: dict, entry_id: int = 1, cmd_line_args: tuple = () ) -> dict: """Parse application definition.""" - variadic_prefix = EM_APPDEF["prefix"] - for entry in EM_APPDEF["use"]: - if isinstance(entry, tuple) and len(entry) == 2: - trg = variadic_path_to_specific_path( - f"{variadic_prefix}/{entry[0]}", [entry_id] - ) - template[trg] = entry[1] + identifier = [entry_id] + add_specific_metadata(EM_APPDEF_TO_NEXUS, {}, identifier, template) + add_specific_metadata(EM_PYNX_TO_NEXUS, {}, identifier, template) - if cmd_line_args != [] and all(isinstance(item, str) for item in cmd_line_args): + if cmd_line_args != () and all(isinstance(item, str) for item in cmd_line_args): template[f"/ENTRY[entry{entry_id}]/profiling/command_line_call"] = ( cmd_line_args ) diff --git a/pynxtools_em/config/eln_cfg.py b/pynxtools_em/config/eln_cfg.py index 154ee54..e9eda55 100644 --- a/pynxtools_em/config/eln_cfg.py +++ b/pynxtools_em/config/eln_cfg.py @@ -30,28 +30,35 @@ # conversion is required while map_does not assume this # and instead does the conversion also -EM_EXAMPLE_ENTRY_TO_NEXUS = { - "prefix": "/ENTRY[entry*]", - "load": ["experiment_alias"], - "iso8601": ["start_time"], +EM_ENTRY_TO_NEXUS = { + "prefix_trg": "/ENTRY[entry*]", + "prefix_src": "entry/", + "map_to_str": [ + "experiment_alias", + "start_time", + "end_time", + "experiment_description", + ], } -EM_EXAMPLE_SAMPLE_TO_NEXUS = { - "prefix": "/ENTRY[entry*]/sample", - "load": ["method", "atom_types"], - "iso8601": ["preparation_date"], +EM_SAMPLE_TO_NEXUS = { + "prefix_trg": "/ENTRY[entry*]/sample", + "prefix_src": "sample/", + "map_to_str": [("thickness/@units", "thickness/unit")], + "map": [ + "method", + "name", + "atom_types", + "preparation_date", + ("thickness", "thickness/value"), + ], } -EM_EXAMPLE_USER_TO_NEXUS = { - "prefix": "/ENTRY[entry*]/USER[user*]", - "use": [ - ("IDENTIFIER[identifier]/identifier", "orcid"), - ("IDENTIFIER[identifier]/service", "orcid"), - ("IDENTIFIER[identifier]/is_persistent", False), - ], - "load": [ +EM_USER_TO_NEXUS = { + "prefix_trg": "/ENTRY[entry*]/USER[user*]", + "map": [ "name", "affiliation", "address", @@ -60,3 +67,13 @@ "role", ], } + + +EM_USER_IDENTIFIER_TO_NEXUS = { + "prefix_trg": "/ENTRY[entry*]/USER[user*]", + "use": [ + ("IDENTIFIER[identifier]/identifier", "orcid"), + ("IDENTIFIER[identifier]/service", "orcid"), + ("IDENTIFIER[identifier]/is_persistent", True), + ], +} diff --git a/pynxtools_em/config/oasis_cfg.py b/pynxtools_em/config/oasis_cfg.py index daf5a41..59c2753 100644 --- a/pynxtools_em/config/oasis_cfg.py +++ b/pynxtools_em/config/oasis_cfg.py @@ -22,9 +22,9 @@ # import datetime as dt # f"{dt.datetime.now(dt.timezone.utc).isoformat().replace('+00:00', 'Z')}", -EM_EXAMPLE_CSYS_TO_NEXUS = { - "prefix": "/ENTRY[entry*]/coordinate_system_set/COORDINATE_SYSTEM[coordinate_system*]", - "load": [ +EM_CSYS_TO_NEXUS = { + "prefix_trg": "/ENTRY[entry*]/coordinate_system_set/COORDINATE_SYSTEM[coordinate_system*]", + "map": [ "alias", "type", "handedness", diff --git a/pynxtools_em/config/rsciio_velox_cfg.py b/pynxtools_em/config/rsciio_velox_cfg.py index 31ceb65..dda234c 100644 --- a/pynxtools_em/config/rsciio_velox_cfg.py +++ b/pynxtools_em/config/rsciio_velox_cfg.py @@ -109,8 +109,8 @@ } # we do not know whether the angle is radiant or degree, in all examples # the instance values are very small so can be both :( needs clarification -# we also cannot document this into the NeXus file like @units = "Check this" -# because then the dataconverter (rightly so complains) that the string "Check this" +# we also cannot document this into the NeXus file like @units = "check this" +# because then the dataconverter (rightly so complains) that the string "check this" # is not a proper unit for an instance of NX_VOLTAGE diff --git a/pynxtools_em/reader.py b/pynxtools_em/reader.py index f2b872e..6026547 100644 --- a/pynxtools_em/reader.py +++ b/pynxtools_em/reader.py @@ -26,6 +26,7 @@ from pynxtools_em.concepts.nxs_concepts import NxEmAppDef from pynxtools_em.subparsers.nxs_imgs import NxEmImagesSubParser +from pynxtools_em.subparsers.nxs_nion import ZipNionProjectSubParser # from pynxtools_em.subparsers.nxs_mtex import NxEmNxsMTexSubParser from pynxtools_em.subparsers.nxs_pyxem import NxEmNxsPyxemSubParser @@ -45,8 +46,6 @@ NxEmNomadOasisConfigurationParser, ) from pynxtools_em.subparsers.oasis_eln_reader import NxEmNomadOasisElnSchemaParser - -# from pynxtools_em.subparsers.nxs_nion import NxEmZippedNionProjectSubParser from pynxtools_em.subparsers.rsciio_velox import RsciioVeloxSubParser from pynxtools_em.utils.io_case_logic import ( EmUseCaseSelector, @@ -123,9 +122,8 @@ def read( nxs_pyxem = NxEmNxsPyxemSubParser(entry_id, case.dat[0], verbose=False) nxs_pyxem.parse(template) - # nxs_nion = NxEmZippedNionProjectSubParser(entry_id, case.dat[0], verbose=False) - # nxs_nion.parse(template) - # TODO::check correct loop through! + nxs_nion = ZipNionProjectSubParser(entry_id, case.dat[0], verbose=False) + nxs_nion.parse(template) # for dat_instance in case.dat_parser_type: # print(f"Process pieces of information in {dat_instance} tech partner file...") diff --git a/pynxtools_em/subparsers/nxs_nion.py b/pynxtools_em/subparsers/nxs_nion.py index 9d8f528..242db1b 100644 --- a/pynxtools_em/subparsers/nxs_nion.py +++ b/pynxtools_em/subparsers/nxs_nion.py @@ -20,17 +20,16 @@ # pylint: disable=no-member -import mmap -import yaml import json +import mmap +from typing import Dict, List +from zipfile import ZipFile + import flatdict as fd -import numpy as np import h5py import nion.swift.model.NDataHandler as nsnd -from zipfile import ZipFile -from typing import Dict, List - -from pynxtools_em.utils.nion_utils import uuid_to_file_name +import numpy as np +import yaml # from pynxtools_em.utils.swift_generate_dimscale_axes \ # import get_list_of_dimension_scale_axes @@ -41,20 +40,24 @@ # from pynxtools_em.swift_to_nx_image_real_space \ # import NxImageRealSpaceDict from pynxtools_em.utils.get_file_checksum import ( - get_sha256_of_file_content, DEFAULT_CHECKSUM_ALGORITHM, + get_sha256_of_file_content, ) +from pynxtools_em.utils.nion_utils import uuid_to_file_name -class NxEmZippedNionProjectSubParser: +class ZipNionProjectSubParser: """Parse zip-compressed archive of a nionswift project with its content.""" - def __init__(self, entry_id: int = 1, input_file_path: str = ""): + def __init__( + self, entry_id: int = 1, input_file_path: str = "", verbose: bool = False + ): """Class wrapping swift parser.""" if input_file_path is not None and input_file_path != "": self.file_path = input_file_path - else: - raise ValueError(f"{__name__} needs proper instantiation !") + if not self.file_path.endswith(".zip.nion"): + self.supported = False + return if entry_id > 0: self.entry_id = entry_id else: @@ -72,6 +75,7 @@ def __init__(self, entry_id: int = 1, input_file_path: str = ""): # just get the *.h5 files irrespective whether parsed later or not self.configure() self.supported = False + self.verbose = verbose def configure(self): self.tmp["cfg"]: Dict = {} @@ -83,6 +87,12 @@ def configure(self): def check_if_zipped_nionswift_project_file(self, verbose=False): """Inspect the content of the compressed project file to check if supported.""" + if self.supported is False: + print( + f"Parser ZipNionProject finds no content in {self.file_path} that it supports" + ) + return False + with open(self.file_path, "rb", 0) as fp: s = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ) magic = s.read(8) @@ -245,13 +255,14 @@ def process_ndata(self, file_hdl, full_path, template, verbose=False): # this should be done more elegantly by just writing the # data directly into the template and not creating another copy # TODO::only during inspection - return template - + """ self.map_to_nexus(flat_metadata_dict, data_arr, nx_concept_name, template) del flat_metadata_dict del data_arr del nx_concept_name return template + """ + return template def process_hfive(self, file_hdl, full_path, template: dict, verbose=False): """Handle reading and processing of opened *.h5 inside the ZIP file.""" @@ -369,11 +380,9 @@ def parse_project_file(self, template: dict, verbose=False) -> dict: def parse(self, template: dict, verbose=False) -> dict: """Parse NOMAD OASIS relevant data and metadata from swift project.""" - print( - "Parsing in-place from zip-compressed nionswift project (nsproj + directory)..." - ) - if self.check_if_zipped_nionswift_project_file(verbose) is False: - return template - - self.parse_project_file(template, verbose) + if self.check_if_zipped_nionswift_project_file(verbose): + print( + "Parsing in-place from zip-compressed nionswift project (nsproj + directory)..." + ) + self.parse_project_file(template, verbose) return template diff --git a/pynxtools_em/subparsers/oasis_config_reader.py b/pynxtools_em/subparsers/oasis_config_reader.py index 2a5b2c5..41ceb38 100644 --- a/pynxtools_em/subparsers/oasis_config_reader.py +++ b/pynxtools_em/subparsers/oasis_config_reader.py @@ -27,11 +27,13 @@ # ("/ENTRY[entry*]/USER[user*]/name, "load", "name") # if pair load the value pointed to by src and copy into trg +import pathlib + import flatdict as fd import yaml -from pynxtools_em.concepts.concept_mapper import variadic_path_to_specific_path -from pynxtools_em.config.oasis_cfg import EM_EXAMPLE_CSYS_TO_NEXUS +from pynxtools_em.concepts.mapping_functors import add_specific_metadata +from pynxtools_em.config.oasis_cfg import EM_CSYS_TO_NEXUS class NxEmNomadOasisConfigurationParser: @@ -42,14 +44,14 @@ def __init__(self, file_path: str, entry_id: int, verbose: bool = False): f"Extracting data from deployment-specific configuration file {file_path} ..." ) if ( - file_path.rsplit("/", 1)[-1].endswith(".oasis.specific.yaml") - or file_path.endswith(".oasis.specific.yml") + pathlib.Path(file_path).name.endswith(".oasis.specific.yaml") + or pathlib.Path(file_path).name.endswith(".oasis.specific.yml") ) and entry_id > 0: self.entry_id = entry_id self.file_path = file_path with open(self.file_path, "r", encoding="utf-8") as stream: self.yml = fd.FlatDict(yaml.safe_load(stream), delimiter="/") - if verbose is True: + if verbose: for key, val in self.yml.items(): print(f"key: {key}, val: {val}") else: @@ -69,23 +71,12 @@ def parse_reference_frames(self, template: dict) -> dict: if csys_dict == {}: continue identifier = [self.entry_id, csys_id] - variadic_prefix = EM_EXAMPLE_CSYS_TO_NEXUS["prefix"] - for key in csys_dict: - for entry in EM_EXAMPLE_CSYS_TO_NEXUS["load"]: - if isinstance(entry, str): - if key == entry: - trg = variadic_path_to_specific_path( - f"{variadic_prefix}/{entry}", identifier - ) - template[trg] = csys_dict[entry] - break - if isinstance(entry, tuple) and len(entry) == 2: - if key == entry[1]: - trg = variadic_path_to_specific_path( - f"{variadic_prefix}/{entry[0]}", identifier - ) - template[trg] = csys_dict[entry[1]] - break + add_specific_metadata( + EM_CSYS_TO_NEXUS, + fd.FlatDict(csys_dict), + identifier, + template, + ) csys_id += 1 return template diff --git a/pynxtools_em/subparsers/oasis_eln_reader.py b/pynxtools_em/subparsers/oasis_eln_reader.py index 2674faa..3ed255e 100644 --- a/pynxtools_em/subparsers/oasis_eln_reader.py +++ b/pynxtools_em/subparsers/oasis_eln_reader.py @@ -17,14 +17,17 @@ # """Parser generic ELN content serialized as eln_data.yaml to NeXus NXem.""" +import pathlib + import flatdict as fd import yaml -from pynxtools_em.concepts.concept_mapper import variadic_path_to_specific_path +from pynxtools_em.concepts.mapping_functors import add_specific_metadata from pynxtools_em.config.eln_cfg import ( - EM_EXAMPLE_ENTRY_TO_NEXUS, - EM_EXAMPLE_SAMPLE_TO_NEXUS, - EM_EXAMPLE_USER_TO_NEXUS, + EM_ENTRY_TO_NEXUS, + EM_SAMPLE_TO_NEXUS, + EM_USER_IDENTIFIER_TO_NEXUS, + EM_USER_TO_NEXUS, ) @@ -38,14 +41,14 @@ class NxEmNomadOasisElnSchemaParser: def __init__(self, file_path: str, entry_id: int, verbose: bool = False): print(f"Extracting data from ELN file {file_path} ...") if ( - file_path.rsplit("/", 1)[-1].startswith("eln_data") - or file_path.startswith("eln_data") + pathlib.Path(file_path).name.endswith("eln_data.yaml") + or pathlib.Path(file_path).name.endswith("eln_data.yml") ) and entry_id > 0: self.entry_id = entry_id self.file_path = file_path with open(self.file_path, "r", encoding="utf-8") as stream: self.yml = fd.FlatDict(yaml.safe_load(stream), delimiter="/") - if verbose is True: + if verbose: for key, val in self.yml.items(): print(f"key: {key}, value: {val}") else: @@ -55,63 +58,14 @@ def __init__(self, file_path: str, entry_id: int, verbose: bool = False): def parse_entry(self, template: dict) -> dict: """Copy data from entry section into template.""" - src = "entry" identifier = [self.entry_id] - if src in self.yml: - if isinstance(self.yml[src], fd.FlatDict): - for key in self.yml[src]: - variadic_prefix = EM_EXAMPLE_ENTRY_TO_NEXUS["prefix"] - for entry in EM_EXAMPLE_ENTRY_TO_NEXUS["load"]: - if isinstance(entry, str) and key == entry: - trg = variadic_path_to_specific_path( - f"{variadic_prefix}/{entry}", identifier - ) - template[trg] = self.yml[src][entry] - break - elif isinstance(entry, tuple) and len(entry) == 2: - if key == entry[1]: - trg = variadic_path_to_specific_path( - f"{variadic_prefix}/{entry[0]}", identifier - ) - template[trg] = self.yml[src][entry[1]] - break - for entry in EM_EXAMPLE_ENTRY_TO_NEXUS["iso8601"]: - if isinstance(entry, str) and key == entry: - trg = variadic_path_to_specific_path( - f"{variadic_prefix}/{entry}", identifier - ) - template[trg] = self.yml[src][entry].isoformat() + add_specific_metadata(EM_ENTRY_TO_NEXUS, self.yml, identifier, template) return template def parse_sample(self, template: dict) -> dict: """Copy data from entry section into template.""" - src = "sample" identifier = [self.entry_id] - if src in self.yml: - if isinstance(self.yml[src], fd.FlatDict): - for key in self.yml[src]: - variadic_prefix = EM_EXAMPLE_SAMPLE_TO_NEXUS["prefix"] - for entry in EM_EXAMPLE_SAMPLE_TO_NEXUS["load"]: - if isinstance(entry, str) and key == entry: - trg = variadic_path_to_specific_path( - f"{variadic_prefix}/{entry}", identifier - ) - template[trg] = self.yml[src][entry] - break - elif isinstance(entry, tuple) and len(entry) == 2: - if key == entry[1]: - trg = variadic_path_to_specific_path( - f"{variadic_prefix}/{entry[0]}", identifier - ) - template[trg] = self.yml[src][entry[1]] - break - for entry in EM_EXAMPLE_SAMPLE_TO_NEXUS["iso8601"]: - if isinstance(entry, str) and key == entry: - trg = variadic_path_to_specific_path( - f"{variadic_prefix}/{entry}", identifier - ) - template[trg] = self.yml[src][entry].isoformat() - break + add_specific_metadata(EM_SAMPLE_TO_NEXUS, self.yml, identifier, template) return template def parse_user(self, template: dict) -> dict: @@ -126,37 +80,19 @@ def parse_user(self, template: dict) -> dict: if user_dict == {}: continue identifier = [self.entry_id, user_id] - variadic_prefix = EM_EXAMPLE_USER_TO_NEXUS["prefix"] - for key in user_dict: - if key != "orcid": - for entry in EM_EXAMPLE_USER_TO_NEXUS["load"]: - if isinstance(entry, str) and key == entry: - trg = variadic_path_to_specific_path( - f"{variadic_prefix}/{entry}", identifier - ) - template[trg] = user_dict[entry] - break - elif isinstance(entry, tuple) and len(entry) == 2: - if key == entry[1]: - trg = variadic_path_to_specific_path( - f"{variadic_prefix}/{entry[0]}", - identifier, - ) - template[trg] = user_dict[entry[1]] - break - else: - trg = variadic_path_to_specific_path( - f"{variadic_prefix}", identifier - ) - template[f"{trg}/IDENTIFIER[identifier]/identifier"] = ( - user_dict["orcid"] - ) - template[f"{trg}/IDENTIFIER[identifier]/service"] = ( - "orcid" - ) - template[ - f"{trg}/IDENTIFIER[identifier]/is_persistent" - ] = False + add_specific_metadata( + EM_USER_TO_NEXUS, + fd.FlatDict(user_dict), + identifier, + template, + ) + if "orcid" in user_dict: + add_specific_metadata( + EM_USER_IDENTIFIER_TO_NEXUS, + fd.FlatDict(user_dict), + identifier, + template, + ) user_id += 1 return template diff --git a/pynxtools_em/utils/interpret_boolean.py b/pynxtools_em/utils/interpret_boolean.py new file mode 100644 index 0000000..a3f75ec --- /dev/null +++ b/pynxtools_em/utils/interpret_boolean.py @@ -0,0 +1,38 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Interpret different human-readable forms of a boolean statement to boolean.""" + +HUMAN_BOOLEAN_STATEMENT = { + "0": False, + "1": True, + "n": False, + "y": True, + "no": False, + "yes": True, + "false": False, + "true": True, +} + + +def try_interpret_as_boolean(arg: str) -> bool: + """Try to interpret a human string statement if boolean be strict.""" + if arg.lower() in HUMAN_BOOLEAN_STATEMENT: + return HUMAN_BOOLEAN_STATEMENT[arg.lower()] + raise KeyError( + f"try_to_interpret_as_boolean argument {arg} does not yield key even for {arg.lower()}!" + ) diff --git a/pynxtools_em/utils/versioning.py b/pynxtools_em/utils/versioning.py new file mode 100644 index 0000000..ca33130 --- /dev/null +++ b/pynxtools_em/utils/versioning.py @@ -0,0 +1,34 @@ +# +# Copyright The NOMAD Authors. +# +# This file is part of NOMAD. See https://nomad-lab.eu for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Utility tool constants and versioning.""" + +from pynxtools_em.utils.get_gitrepo_commit import get_repo_last_commit + +NX_EM_ADEF_NAME = "NXem" +NX_EM_EXEC_NAME = "pynxtools-em/reader.py" + + +def get_em_exec_version() -> str: + tag = get_repo_last_commit() + if tag is not None: + return f"https://github.com/FAIRmat-NFDI/pynxtools-em/commit/{tag}" + else: + return f"https://github.com/FAIRmat-NFDI/pynxtools-em/commit/ UNKNOWN COMMIT !!" + + +NX_EM_EXEC_VERSION = get_em_exec_version() From 33d47fc04a8aafd10e068243047e8bc41cb05add Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Thu, 9 May 2024 03:28:08 +0200 Subject: [PATCH 4/7] Added an updated default plot resolver with priority voting --- pynxtools_em/reader.py | 23 +++---- pynxtools_em/utils/nx_default_plots.py | 90 +++++++++++++++++++------- 2 files changed, 78 insertions(+), 35 deletions(-) diff --git a/pynxtools_em/reader.py b/pynxtools_em/reader.py index 6026547..466801e 100644 --- a/pynxtools_em/reader.py +++ b/pynxtools_em/reader.py @@ -30,6 +30,14 @@ # from pynxtools_em.subparsers.nxs_mtex import NxEmNxsMTexSubParser from pynxtools_em.subparsers.nxs_pyxem import NxEmNxsPyxemSubParser +from pynxtools_em.subparsers.oasis_config_reader import ( + NxEmNomadOasisConfigurationParser, +) +from pynxtools_em.subparsers.oasis_eln_reader import NxEmNomadOasisElnSchemaParser +from pynxtools_em.subparsers.rsciio_velox import RsciioVeloxSubParser +from pynxtools_em.utils.io_case_logic import ( + EmUseCaseSelector, +) ## from pynxtools_em.utils.default_plots import NxEmDefaultPlotResolver # from pynxtools_em.geometry.convention_mapper import NxEmConventionMapper @@ -40,16 +48,7 @@ # import NxEmOmMtexEbsdParser # from pynxtools.dataconverter.readers.em_om.utils.zip_ebsd_parser \ # import NxEmOmZipEbsdParser -# from pynxtools.dataconverter.readers.em_om.utils.em_nexus_plots \ -# import em_om_default_plot_generator -from pynxtools_em.subparsers.oasis_config_reader import ( - NxEmNomadOasisConfigurationParser, -) -from pynxtools_em.subparsers.oasis_eln_reader import NxEmNomadOasisElnSchemaParser -from pynxtools_em.subparsers.rsciio_velox import RsciioVeloxSubParser -from pynxtools_em.utils.io_case_logic import ( - EmUseCaseSelector, -) +from pynxtools_em.utils.nx_default_plots import NxEmDefaultPlotResolver class EMReader(BaseReader): @@ -131,7 +130,9 @@ def read( # # elif case.dat_parser_type == "zip": # # zip_parser = NxEmOmZipEbsdParser(case.dat[0], entry_id) # # zip_parser.parse(template) - # em_default_plot_generator(template, 1) + + nxplt = NxEmDefaultPlotResolver() + nxplt.priority_select(template) # run_block = False # if run_block is True: diff --git a/pynxtools_em/utils/nx_default_plots.py b/pynxtools_em/utils/nx_default_plots.py index 049fd32..a7dbf28 100644 --- a/pynxtools_em/utils/nx_default_plots.py +++ b/pynxtools_em/utils/nx_default_plots.py @@ -17,7 +17,8 @@ # """Logics and functionality to identify and annotate a default plot NXem.""" -import h5py +from typing import Dict + import numpy as np @@ -27,29 +28,70 @@ class NxEmDefaultPlotResolver: def __init__(self): pass - def annotate_default_plot(self, template: dict, plot_nxpath: str = "") -> dict: - """Write path to the default plot from root to plot_nxpath.""" - if plot_nxpath != "": - print(plot_nxpath) - tmp = plot_nxpath.split("/") - print(tmp) - for idx in np.arange(0, len(tmp)): - if tmp[idx] != "": - if idx != 0: - template[f'{"/".join(tmp[0:idx])}/@default'] = tmp[idx] + def decorate_path_to_default_plot(self, template: dict, nxpath: str) -> dict: + """Write @default attribute to point to the default plot.""" + # an example for nxpath + # "/ENTRY[entry1]/atom_probe/ranging/mass_to_charge_distribution/mass_spectrum" + if nxpath.count("/") == 0: + return template + path = nxpath.split("/") + trg = "/" + for idx in np.arange(0, len(path) - 1): + symbol_s = path[idx + 1].find("[") + symbol_e = path[idx + 1].find("]") + if 0 <= symbol_s < symbol_e: + template[f"{trg}@default"] = f"{path[idx + 1][symbol_s + 1:symbol_e]}" + trg += f"{path[idx + 1][symbol_s + 1:symbol_e]}/" + else: + template[f"{trg}@default"] = f"{path[idx + 1]}" + trg += f"{path[idx + 1]}/" return template - def nxs_mtex_get_nxpath_to_default_plot( - self, entry_id: int = 1, nxs_mtex_file_name: str = "" - ) -> str: - """Find a path to a default plot (i.e. NXdata instance) if any.""" - h5r = h5py.File(nxs_mtex_file_name, "r") - if f"/entry{entry_id}/roi1/ebsd/indexing/roi" in h5r: - h5r.close() - return f"/entry{entry_id}/roi1/ebsd/indexing/roi" - h5r.close() - return "" - - def parse(self, template: dict, entry_id: int = 1) -> dict: - """Pass because for *.nxs.mtex all data are already in the copy of the output.""" + def priority_select(self, template: dict, entry_id: int = 1) -> dict: + """Inspects all NXdata instances that could serve as default plots and picks one.""" + # find candidates + candidates: Dict = {} + for votes in [1, 2, 3]: + candidates[votes] = [] + + dtyp_vote = [ + ("IMAGE_R_SET", "image", 1), + ("IMAGE_C_SET", "image", 2), + ("SPECTRUM_SET", "spectrum", 3), + ] + for tpl in dtyp_vote: + for key in template.keys(): + for dimensionality in ["zerod", "oned", "twod", "threed"]: + head = f"{tpl[0]}[" + idx_head = key.find(head) + tail = f"]/{tpl[1]}_{dimensionality}" + idx_tail = key.find(tail) + # TODO: better use a regex + if idx_head is None or idx_tail is None: + continue + if 0 < idx_head < idx_tail: + keyword = f"{key[0:idx_tail + len(tail)]}" + if keyword not in candidates[tpl[2]]: + candidates[tpl[2]].append(keyword) + break + + for votes in [1, 2, 3]: + print(f"NXdata instances with priority {votes}:") + for entry in candidates[votes]: + print(entry) + + # maybe we want to sort + + has_default_plot = False + for votes in [3, 2, 1]: + if len(candidates[votes]) > 0: + self.decorate_path_to_default_plot(template, candidates[votes][0]) + print( + f"Decorating {candidates[votes][0]} as the default plot for H5Web ..." + ) + has_default_plot = True + break + + if not has_default_plot: + print("WARNING::No default plot!") return template From 617fa6f1ef490021963e4ed827b1c52181c0e3f3 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Fri, 10 May 2024 11:24:04 +0200 Subject: [PATCH 5/7] Fixed incorrect entering of nionswift reader, fixed unnecessary print statements during EBSD map regridding, added support for default plot annotations for EBSD ipf and roi NXdata instances --- pynxtools_em/subparsers/nxs_nion.py | 47 +++++++++++------------- pynxtools_em/utils/get_sqr_grid.py | 18 +++++----- pynxtools_em/utils/io_case_logic.py | 5 +-- pynxtools_em/utils/nx_default_plots.py | 50 ++++++++++++++++++-------- 4 files changed, 68 insertions(+), 52 deletions(-) diff --git a/pynxtools_em/subparsers/nxs_nion.py b/pynxtools_em/subparsers/nxs_nion.py index 242db1b..9bec4b7 100644 --- a/pynxtools_em/subparsers/nxs_nion.py +++ b/pynxtools_em/subparsers/nxs_nion.py @@ -50,14 +50,11 @@ class ZipNionProjectSubParser: """Parse zip-compressed archive of a nionswift project with its content.""" def __init__( - self, entry_id: int = 1, input_file_path: str = "", verbose: bool = False + self, entry_id: int = 1, input_file_path: str = "", verbose: bool = True ): """Class wrapping swift parser.""" if input_file_path is not None and input_file_path != "": self.file_path = input_file_path - if not self.file_path.endswith(".zip.nion"): - self.supported = False - return if entry_id > 0: self.entry_id = entry_id else: @@ -76,6 +73,7 @@ def __init__( self.configure() self.supported = False self.verbose = verbose + self.check_if_zipped_nionswift_project_file() def configure(self): self.tmp["cfg"]: Dict = {} @@ -85,18 +83,18 @@ def configure(self): self.tmp["cfg"]["spectrum_id"] = 1 self.tmp["meta"]: Dict = {} - def check_if_zipped_nionswift_project_file(self, verbose=False): + def check_if_zipped_nionswift_project_file(self): """Inspect the content of the compressed project file to check if supported.""" - if self.supported is False: + if not self.file_path.endswith(".zip.nion"): print( f"Parser ZipNionProject finds no content in {self.file_path} that it supports" ) - return False + return with open(self.file_path, "rb", 0) as fp: s = mmap.mmap(fp.fileno(), 0, access=mmap.ACCESS_READ) magic = s.read(8) - if verbose is True: + if self.verbose: fp.seek(0, 2) eof_byte_offset = fp.tell() print( @@ -117,7 +115,7 @@ def check_if_zipped_nionswift_project_file(self, verbose=False): ): with zip_file_hdl.open(file) as fp: magic = fp.read(8) - if verbose is True: + if self.verbose: fp.seek(0, 2) eof_byte_offset = fp.tell() print( @@ -129,7 +127,7 @@ def check_if_zipped_nionswift_project_file(self, verbose=False): elif file.endswith(".ndata"): with zip_file_hdl.open(file) as fp: magic = fp.read(8) - if verbose is True: + if self.verbose: fp.seek(0, 2) eof_byte_offset = fp.tell() print( @@ -141,7 +139,7 @@ def check_if_zipped_nionswift_project_file(self, verbose=False): elif file.endswith(".nsproj"): with zip_file_hdl.open(file) as fp: magic = fp.read(8) - if verbose is True: + if self.verbose: fp.seek(0, 2) eof_byte_offset = fp.tell() print( @@ -156,24 +154,23 @@ def check_if_zipped_nionswift_project_file(self, verbose=False): print( "Test 2 failed, UUID keys of *.ndata and *.h5 files in project are not disjoint!" ) - return False + return if len(self.proj_file_dict.keys()) != 1: print( "Test 3 failed, he project contains either no or more than one nsproj file!" ) - return False + return print( f"Content in zip-compressed nionswift project {self.file_path} passed all tests" ) self.supported = True - if verbose is True: + if self.verbose: for key, val in self.proj_file_dict.items(): print(f"nsprj: ___{key}___{val}___") for key, val in self.ndata_file_dict.items(): print(f"ndata: ___{key}___{val}___") for key, val in self.hfive_file_dict.items(): print(f"hfive: ___{key}___{val}___") - return True def update_event_identifier(self): """Advance and reset bookkeeping of event data em and data instances.""" @@ -193,7 +190,7 @@ def map_to_nexus(self, meta, arr, concept_name, template): # TODO:: return template - def process_ndata(self, file_hdl, full_path, template, verbose=False): + def process_ndata(self, file_hdl, full_path, template): """Handle reading and processing of opened *.ndata inside the ZIP file.""" # assure that we start reading that file_hdl/pointer from the beginning... file_hdl.seek(0) @@ -225,7 +222,7 @@ def process_ndata(self, file_hdl, full_path, template, verbose=False): """ flat_metadata_dict = fd.FlatDict(metadata_dict, delimiter="/") - if verbose is True: + if self.verbose: print(f"Flattened content of this metadata.json") for key, value in flat_metadata_dict.items(): print(f"ndata, metadata.json, flat: ___{key}___{value}___") @@ -264,7 +261,7 @@ def process_ndata(self, file_hdl, full_path, template, verbose=False): """ return template - def process_hfive(self, file_hdl, full_path, template: dict, verbose=False): + def process_hfive(self, file_hdl, full_path, template: dict): """Handle reading and processing of opened *.h5 inside the ZIP file.""" flat_metadata_dict = {} """ @@ -286,7 +283,7 @@ def process_hfive(self, file_hdl, full_path, template: dict, verbose=False): """ flat_metadata_dict = fd.FlatDict(metadata_dict, delimiter="/") - if verbose is True: + if self.verbose: print(f"Flattened content of this metadata.json") for key, value in flat_metadata_dict.items(): print(f"hfive, data, flat: ___{key}___{value}___") @@ -314,7 +311,7 @@ def process_hfive(self, file_hdl, full_path, template: dict, verbose=False): """ return template - def parse_project_file(self, template: dict, verbose=False) -> dict: + def parse_project_file(self, template: dict) -> dict: """Parse lazily from compressed NionSwift project (nsproj + directory).""" nionswift_proj_mdata = {} with ZipFile(self.file_path) as zip_file_hdl: @@ -324,7 +321,7 @@ def parse_project_file(self, template: dict, verbose=False) -> dict: yaml.safe_load(file_hdl), delimiter="/" ) # TODO::inspection phase, maybe with yaml to file? - if verbose is True: + if self.verbose: print(f"Flattened content of {proj_file_name}") for ( key, @@ -357,7 +354,6 @@ def parse_project_file(self, template: dict, verbose=False) -> dict: file_hdl, self.ndata_file_dict[key], template, - verbose, ) elif key in self.hfive_file_dict.keys(): print( @@ -372,17 +368,16 @@ def parse_project_file(self, template: dict, verbose=False) -> dict: file_hdl, self.hfive_file_dict[key], template, - verbose, ) else: print(f"Key {key} has no corresponding data file") return template - def parse(self, template: dict, verbose=False) -> dict: + def parse(self, template: dict) -> dict: """Parse NOMAD OASIS relevant data and metadata from swift project.""" - if self.check_if_zipped_nionswift_project_file(verbose): + if self.supported: print( "Parsing in-place from zip-compressed nionswift project (nsproj + directory)..." ) - self.parse_project_file(template, verbose) + self.parse_project_file(template) return template diff --git a/pynxtools_em/utils/get_sqr_grid.py b/pynxtools_em/utils/get_sqr_grid.py index d16fc8d..fa5505c 100644 --- a/pynxtools_em/utils/get_sqr_grid.py +++ b/pynxtools_em/utils/get_sqr_grid.py @@ -23,13 +23,13 @@ from scipy.spatial import KDTree from pynxtools_em.examples.ebsd_database import SQUARE_GRID -from pynxtools_em.utils.get_scan_points import threed, square_grid, hexagonal_grid +from pynxtools_em.utils.get_scan_points import hexagonal_grid, square_grid, threed def get_scan_points_with_mark_data_discretized_on_sqr_grid( src_grid: dict, max_edge_length: int ) -> dict: - """Inspect grid_type, dimensionality, point locations, and mark src_grida, map then.""" + """Inspect grid_type, dimensionality, point locations, and mark src_grid, map then.""" is_threed = threed(src_grid) req_keys = ["grid_type", "tiling", "flight_plan"] dims = ["x", "y"] @@ -113,7 +113,7 @@ def get_scan_points_with_mark_data_discretized_on_sqr_grid( ) # TODO:: if scan_point_{dim} are calibrated this approach # here would shift the origin to 0, 0 implicitly which may not be desired - print(f"trg_xy {trg_xy}, shape {np.shape(trg_xy)}") + print(f"trg_xy np.shape {np.shape(trg_xy)}") tree = KDTree( np.column_stack((src_grid["scan_point_x"], src_grid["scan_point_y"])) ) @@ -123,7 +123,7 @@ def get_scan_points_with_mark_data_discretized_on_sqr_grid( del d del tree - # rebuild src_grid container with only the relevant src_grida selected from src_grid + # rebuild src_grid container with only the relevant src_grid selected from src_grid for key in src_grid.keys(): if key == "euler": trg_grid[key] = np.empty((np.shape(trg_xy)[0], 3), np.float32) @@ -170,10 +170,10 @@ def get_scan_points_with_mark_data_discretized_on_sqr_grid( trg_grid[key] = src_grid[key] # print(f"WARNING:: src_grid[{key}] is mapped as is on trg_grid[{key}] !") # print(f"final np.shape(trg_grid[{key}]) {np.shape(trg_grid[key])}") - else: - print( - f"WARNING:: src_grid[{key}] is not yet mapped on trg_grid[{key}] !" - ) + # else: + # print( + # f"WARNING:: src_grid[{key}] is not yet mapped on trg_grid[{key}] !" + # ) trg_grid["n_x"] = trg_nxy[0] trg_grid["n_y"] = trg_nxy[1] trg_grid["s_x"] = trg_sxy @@ -185,5 +185,5 @@ def get_scan_points_with_mark_data_discretized_on_sqr_grid( else: raise ValueError( f"The 3D discretization is currently not implemented because " - f"we do not know of any large enough dataset the test it !" + f"we do not know of any large enough dataset to test it !" ) diff --git a/pynxtools_em/utils/io_case_logic.py b/pynxtools_em/utils/io_case_logic.py index fe18505..1cab37c 100644 --- a/pynxtools_em/utils/io_case_logic.py +++ b/pynxtools_em/utils/io_case_logic.py @@ -19,7 +19,7 @@ # pylint: disable=no-member,duplicate-code,too-many-branches -from typing import Tuple, Dict, List +from typing import Dict, List, Tuple VALID_FILE_NAME_SUFFIX_CONFIG = [".yaml", ".yml"] VALID_FILE_NAME_SUFFIX_DATA = [ @@ -27,13 +27,14 @@ ".tiff", ".tif", ".zip.axon", + ".zip.nion", ".edaxh5", ".h5", ".hdf5", ".nxs", ".dream3d", ] -# ".dm3", ".dm4", ".mtex", ".nion"] +# ".dm3", ".dm4", ".mtex"] class EmUseCaseSelector: diff --git a/pynxtools_em/utils/nx_default_plots.py b/pynxtools_em/utils/nx_default_plots.py index a7dbf28..4c3eac2 100644 --- a/pynxtools_em/utils/nx_default_plots.py +++ b/pynxtools_em/utils/nx_default_plots.py @@ -49,9 +49,10 @@ def decorate_path_to_default_plot(self, template: dict, nxpath: str) -> dict: def priority_select(self, template: dict, entry_id: int = 1) -> dict: """Inspects all NXdata instances that could serve as default plots and picks one.""" - # find candidates + # find candidates for interesting default plots with some priority + # priority ipf map > roi overview > spectra > complex image > real image candidates: Dict = {} - for votes in [1, 2, 3]: + for votes in [1, 2, 3, 4, 5]: candidates[votes] = [] dtyp_vote = [ @@ -59,31 +60,50 @@ def priority_select(self, template: dict, entry_id: int = 1) -> dict: ("IMAGE_C_SET", "image", 2), ("SPECTRUM_SET", "spectrum", 3), ] - for tpl in dtyp_vote: - for key in template.keys(): + for key in template.keys(): + for tpl in dtyp_vote: for dimensionality in ["zerod", "oned", "twod", "threed"]: head = f"{tpl[0]}[" idx_head = key.find(head) tail = f"]/{tpl[1]}_{dimensionality}" idx_tail = key.find(tail) # TODO: better use a regex - if idx_head is None or idx_tail is None: - continue - if 0 < idx_head < idx_tail: - keyword = f"{key[0:idx_tail + len(tail)]}" - if keyword not in candidates[tpl[2]]: - candidates[tpl[2]].append(keyword) - break + if idx_head is not None and idx_tail is not None: + if 0 < idx_head < idx_tail: + keyword = f"{key[0:idx_tail + len(tail)]}" + if keyword not in candidates[tpl[2]]: + candidates[tpl[2]].append(keyword) + break - for votes in [1, 2, 3]: + # find ebsd ipf map + idx_head = key.find("/ROI[") + tail = "/ebsd/indexing/phaseID[phase1]/ipfID[ipf1]/map" + idx_tail = key.find(tail) + if idx_head is not None and idx_tail is not None: + if 0 < idx_head < idx_tail: + keyword = key[0 : idx_tail + len(tail)] + if keyword not in candidates[5]: + candidates[5].append(keyword) + continue + # find ebsd roi map + tail = "/ebsd/indexing/roi" + idx_tail = key.find(tail) + if idx_head is not None and idx_tail is not None: + if 0 < idx_head < idx_tail: + keyword = key[0 : idx_tail + len(tail)] + if keyword not in candidates[4]: + candidates[4].append(keyword) + + # one could think about more fine-grained priority voting, e.g. based on + # image descriptors or shape of the data behind a key in template + + for votes in [1, 2, 3, 4, 5]: print(f"NXdata instances with priority {votes}:") for entry in candidates[votes]: print(entry) - # maybe we want to sort - has_default_plot = False - for votes in [3, 2, 1]: + for votes in [5, 4, 3, 2, 1]: if len(candidates[votes]) > 0: self.decorate_path_to_default_plot(template, candidates[votes][0]) print( From 731444ec3f6f16cb01faaad9aa6ee2cd4afca348 Mon Sep 17 00:00:00 2001 From: mkuehbach Date: Fri, 10 May 2024 12:08:49 +0200 Subject: [PATCH 6/7] Refactored examples --- dev-requirements.txt | 250 +++++++++++- examples/HowToUseTutorial.ipynb | 230 +++++++++++ examples/README.md | 19 + examples/README_Ebsd.md | 37 -- examples/README_Nion.md | 24 -- examples/README_Spctrscpy.md | 24 -- examples/Write.NXem_ebsd.Example.1.ipynb | 377 ------------------ examples/Write.NXem_nion.Example.1.ipynb | 287 ------------- examples/Write.NXem_spctrscpy.Example.1.ipynb | 349 ---------------- examples/eln_data.yaml | 14 +- examples/eln_data_em_nion.yaml | 63 --- examples/eln_data_em_om.yaml | 65 --- examples/eln_data_em_spctrscpy.yaml | 64 --- examples/image_png_protochips_to_nexus.ods | Bin 11254 -> 0 bytes examples/image_tiff_tfs_to_nexus.ods | Bin 11631 -> 0 bytes pynxtools_em/concepts/nxs_object.py | 4 +- pynxtools_em/reader.py | 11 - pynxtools_em/subparsers/nxs_nion.py | 2 - pynxtools_em/utils/get_scan_points.py | 6 +- pynxtools_em/utils/get_sqr_grid.py | 2 - pynxtools_em/utils/image_processing.py | 2 - pynxtools_em/utils/io_case_logic.py | 2 - pynxtools_em/utils/nion_utils.py | 2 - pynxtools_em/utils/nx_default_plots.py | 18 +- pyproject.toml | 3 + 25 files changed, 518 insertions(+), 1337 deletions(-) create mode 100644 examples/HowToUseTutorial.ipynb create mode 100644 examples/README.md delete mode 100644 examples/README_Ebsd.md delete mode 100644 examples/README_Nion.md delete mode 100644 examples/README_Spctrscpy.md delete mode 100644 examples/Write.NXem_ebsd.Example.1.ipynb delete mode 100644 examples/Write.NXem_nion.Example.1.ipynb delete mode 100644 examples/Write.NXem_spctrscpy.Example.1.ipynb delete mode 100644 examples/eln_data_em_nion.yaml delete mode 100644 examples/eln_data_em_om.yaml delete mode 100644 examples/eln_data_em_spctrscpy.yaml delete mode 100644 examples/image_png_protochips_to_nexus.ods delete mode 100644 examples/image_tiff_tfs_to_nexus.ods diff --git a/dev-requirements.txt b/dev-requirements.txt index fe6567a..2497c28 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,6 +4,16 @@ # # pip-compile --extra=dev --extra=docs --output-file=dev-requirements.txt pyproject.toml # +anyio==4.3.0 + # via + # httpx + # jupyter-server +argon2-cffi==23.1.0 + # via jupyter-server +argon2-cffi-bindings==21.2.0 + # via argon2-cffi +arrow==1.3.0 + # via isoduration asciitree==0.3.3 # via zarr ase==3.22.1 @@ -12,12 +22,29 @@ asteval==0.9.32 # via lmfit asttokens==2.4.1 # via stack-data +async-lru==2.0.4 + # via jupyterlab +attrs==23.2.0 + # via + # jsonschema + # referencing babel==2.14.0 - # via mkdocs-material + # via + # jupyterlab-server + # mkdocs-material +beautifulsoup4==4.12.3 + # via nbconvert +bleach==6.1.0 + # via nbconvert build==1.2.1 # via pip-tools certifi==2024.2.2 - # via requests + # via + # httpcore + # httpx + # requests +cffi==1.16.0 + # via argon2-cffi-bindings charset-normalizer==3.3.2 # via requests click==8.1.7 @@ -34,7 +61,9 @@ cloudpickle==3.0.0 colorama==0.4.6 # via mkdocs-material comm==0.2.2 - # via ipykernel + # via + # ipykernel + # ipywidgets contourpy==1.2.0 # via matplotlib cycler==0.12.1 @@ -52,6 +81,8 @@ decorator==5.1.1 # via # ipyparallel # ipython +defusedxml==0.7.1 + # via nbconvert diffpy-structure==3.1.0 # via # diffsims @@ -73,10 +104,14 @@ fabio==2023.10.0 # silx fasteners==0.19 # via zarr +fastjsonschema==2.19.1 + # via nbformat flatdict==4.0.1 # via pynxtools-em (pyproject.toml) fonttools==4.50.0 # via matplotlib +fqdn==1.5.1 + # via jsonschema fsspec==2024.3.1 # via # dask @@ -87,11 +122,17 @@ ghp-import==2.1.0 # via mkdocs greenlet==3.0.3 # via sqlalchemy +h11==0.14.0 + # via httpcore +h5grove==2.1.0 + # via jupyterlab-h5web h5py==3.10.0 # via # fabio + # h5grove # hdf5plugin # hyperspy + # jupyterlab-h5web # kikuchipy # nionswift # orix @@ -101,12 +142,20 @@ h5py==3.10.0 # silx hdf5plugin==4.4.0 # via fabio +httpcore==1.0.5 + # via httpx +httpx==0.27.0 + # via jupyterlab hyperspy==1.7.6 # via # kikuchipy # pyxem idna==3.6 - # via requests + # via + # anyio + # httpx + # jsonschema + # requests imageio==2.27.0 # via # hyperspy @@ -123,7 +172,12 @@ importlib-metadata==7.1.0 iniconfig==2.0.0 # via pytest ipykernel==6.29.4 - # via ipyparallel + # via + # ipyparallel + # jupyter + # jupyter-console + # jupyterlab + # qtconsole ipyparallel==8.7.0 # via hyperspy ipython==8.23.0 @@ -131,24 +185,88 @@ ipython==8.23.0 # hyperspy # ipykernel # ipyparallel + # ipywidgets + # jupyter-console +ipywidgets==8.1.2 + # via jupyter +isoduration==20.11.0 + # via jsonschema jedi==0.19.1 # via ipython jinja2==3.1.3 # via # hyperspy + # jupyter-server + # jupyterlab + # jupyterlab-server # mkdocs # mkdocs-macros-plugin # mkdocs-material + # nbconvert joblib==1.3.2 # via scikit-learn +json5==0.9.25 + # via jupyterlab-server +jsonpointer==2.4 + # via jsonschema +jsonschema[format-nongpl]==4.22.0 + # via + # jupyter-events + # jupyterlab-server + # nbformat +jsonschema-specifications==2023.12.1 + # via jsonschema +jupyter==1.0.0 + # via pynxtools-em (pyproject.toml) jupyter-client==8.6.1 # via # ipykernel # ipyparallel + # jupyter-console + # jupyter-server + # nbclient + # qtconsole +jupyter-console==6.6.3 + # via jupyter jupyter-core==5.7.2 # via # ipykernel # jupyter-client + # jupyter-console + # jupyter-server + # jupyterlab + # nbclient + # nbconvert + # nbformat + # qtconsole +jupyter-events==0.10.0 + # via jupyter-server +jupyter-lsp==2.2.5 + # via jupyterlab +jupyter-server==2.14.0 + # via + # jupyter-lsp + # jupyterlab + # jupyterlab-h5web + # jupyterlab-server + # notebook + # notebook-shim +jupyter-server-terminals==0.5.3 + # via jupyter-server +jupyterlab==4.1.8 + # via + # notebook + # pynxtools-em (pyproject.toml) +jupyterlab-h5web==12.1.0 + # via pynxtools-em (pyproject.toml) +jupyterlab-pygments==0.3.0 + # via nbconvert +jupyterlab-server==2.27.1 + # via + # jupyterlab + # notebook +jupyterlab-widgets==3.0.10 + # via ipywidgets kikuchipy==0.9.0 # via pynxtools-em (pyproject.toml) kiwisolver==1.4.5 @@ -174,6 +292,7 @@ markupsafe==2.1.5 # via # jinja2 # mkdocs + # nbconvert matplotlib==3.8.3 # via # ase @@ -194,6 +313,8 @@ mergedeep==1.3.4 # via # mkdocs # pynxtools +mistune==3.0.2 + # via nbconvert mkdocs==1.5.3 # via # mkdocs-macros-plugin @@ -215,6 +336,17 @@ mypy-extensions==1.0.0 # via mypy natsort==8.4.0 # via hyperspy +nbclient==0.10.0 + # via nbconvert +nbconvert==7.16.4 + # via + # jupyter + # jupyter-server +nbformat==5.10.4 + # via + # jupyter-server + # nbclient + # nbconvert nest-asyncio==1.6.0 # via ipykernel networkx==3.2.1 @@ -235,6 +367,12 @@ nionutils==0.4.10 # nionswift # nionswift-io # nionui +notebook==7.1.3 + # via jupyter +notebook-shim==0.2.4 + # via + # jupyterlab + # notebook numba==0.59.1 # via # diffsims @@ -257,6 +395,7 @@ numpy==1.26.4 # dask # diffsims # fabio + # h5grove # h5py # hyperspy # imageio @@ -294,16 +433,26 @@ orix==0.11.1 # diffsims # kikuchipy # pyxem +orjson==3.10.3 + # via h5grove +overrides==7.7.0 + # via jupyter-server packaging==24.0 # via # build # dask # hyperspy # ipykernel + # jupyter-server + # jupyterlab + # jupyterlab-server # matplotlib # mkdocs + # nbconvert # pooch # pytest + # qtconsole + # qtpy # scikit-image # silx # xarray @@ -314,6 +463,8 @@ pandas==2.2.1 # via # pynxtools # xarray +pandocfilters==1.5.1 + # via nbconvert parso==0.8.3 # via jedi partd==1.4.1 @@ -352,8 +503,12 @@ pooch==1.8.1 # orix prettytable==3.10.0 # via hyperspy +prometheus-client==0.20.0 + # via jupyter-server prompt-toolkit==3.0.43 - # via ipython + # via + # ipython + # jupyter-console psutil==5.9.8 # via # diffsims @@ -361,17 +516,24 @@ psutil==5.9.8 # ipyparallel # pyxem ptyprocess==0.7.0 - # via pexpect + # via + # pexpect + # terminado pure-eval==0.2.2 # via stack-data pycifrw==4.4.6 # via diffpy-structure +pycparser==2.22 + # via cffi pyfai==2024.2.0 # via pyxem pygments==2.17.2 # via # ipython + # jupyter-console # mkdocs-material + # nbconvert + # qtconsole pymdown-extensions==10.8 # via mkdocs-material pynxtools==0.2.1 @@ -388,6 +550,7 @@ python-box==6.1.0 # via rosettasciio python-dateutil==2.9.0.post0 # via + # arrow # ghp-import # hyperspy # ipyparallel @@ -396,6 +559,8 @@ python-dateutil==2.9.0.post0 # mkdocs-macros-plugin # pandas # rosettasciio +python-json-logger==2.0.7 + # via jupyter-events pytz==2024.1 # via # nionswift @@ -406,6 +571,7 @@ pyyaml==6.0.1 # via # dask # hyperspy + # jupyter-events # kikuchipy # mkdocs # mkdocs-macros-plugin @@ -420,15 +586,40 @@ pyzmq==25.1.2 # ipykernel # ipyparallel # jupyter-client + # jupyter-console + # jupyter-server + # qtconsole +qtconsole==5.5.2 + # via jupyter +qtpy==2.4.1 + # via qtconsole +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications + # jupyter-events regex==2024.4.16 # via mkdocs-material requests==2.31.0 # via # hyperspy + # jupyterlab-server # mkdocs-material # pooch +rfc3339-validator==0.1.4 + # via + # jsonschema + # jupyter-events +rfc3986-validator==0.1.1 + # via + # jsonschema + # jupyter-events rosettasciio==0.4 # via pynxtools-em (pyproject.toml) +rpds-py==0.18.1 + # via + # jsonschema + # referencing ruff==0.3.4 # via pynxtools-em (pyproject.toml) scikit-image==0.22.0 @@ -456,13 +647,23 @@ scipy==1.12.0 # scikit-learn # sparse # xraydb +send2trash==1.8.3 + # via jupyter-server silx==2.0.1 # via pyfai six==1.16.0 # via # asttokens + # bleach # diffpy-structure # python-dateutil + # rfc3339-validator +sniffio==1.3.1 + # via + # anyio + # httpx +soupsieve==2.5 + # via beautifulsoup4 sparse==0.15.1 # via hyperspy sqlalchemy==2.0.29 @@ -473,12 +674,19 @@ sympy==1.12 # via hyperspy termcolor==2.4.0 # via mkdocs-macros-plugin +terminado==0.18.1 + # via + # jupyter-server + # jupyter-server-terminals threadpoolctl==3.4.0 # via scikit-learn tifffile==2024.2.12 # via + # h5grove # hyperspy # scikit-image +tinycss2==1.3.0 + # via nbconvert toolz==0.12.1 # via # dask @@ -489,6 +697,10 @@ tornado==6.4 # ipykernel # ipyparallel # jupyter-client + # jupyter-server + # jupyterlab + # notebook + # terminado tqdm==4.66.2 # via # diffsims @@ -503,9 +715,18 @@ traitlets==5.14.2 # ipykernel # ipyparallel # ipython + # ipywidgets # jupyter-client + # jupyter-console # jupyter-core + # jupyter-events + # jupyter-server + # jupyterlab # matplotlib-inline + # nbclient + # nbconvert + # nbformat + # qtconsole traits==6.4.3 # via # hyperspy @@ -514,10 +735,13 @@ transforms3d==0.4.1 # via # diffsims # pyxem +types-python-dateutil==2.9.0.20240316 + # via arrow types-pyyaml==6.0.12.20240311 # via pynxtools-em (pyproject.toml) typing-extensions==4.10.0 # via + # h5grove # ipython # mypy # pint @@ -530,6 +754,8 @@ tzlocal==5.2 # pynxtools-em (pyproject.toml) uncertainties==3.1.7 # via lmfit +uri-template==1.3.0 + # via jsonschema urllib3==2.2.1 # via requests watchdog==4.0.0 @@ -538,8 +764,18 @@ wcwidth==0.2.13 # via # prettytable # prompt-toolkit +webcolors==1.13 + # via jsonschema +webencodings==0.5.1 + # via + # bleach + # tinycss2 +websocket-client==1.8.0 + # via jupyter-server wheel==0.43.0 # via pip-tools +widgetsnbextension==4.0.10 + # via ipywidgets xarray==2024.3.0 # via pynxtools xmltodict==0.13.0 diff --git a/examples/HowToUseTutorial.ipynb b/examples/HowToUseTutorial.ipynb new file mode 100644 index 0000000..5c52bf1 --- /dev/null +++ b/examples/HowToUseTutorial.ipynb @@ -0,0 +1,230 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# How to convert electron microscopy (meta)data to NeXus/HDF5" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The aim of this tutorial is to guide users how to create a NeXus/HDF5 file to parse and normalize pieces of information
\n", + "from typical file formats of the electron microscopy community into a common form. The tool assures that this NeXus file
\n", + "matches to the NXem application definition. Such documented conceptually, the file can be used for sharing electron
\n", + "microscopy research with others (colleagues, project partners, the public), for uploading a summary of the (meta)data to
\n", + "public repositories and thus avoid additional work that typically comes with having to write documentation of metadata
\n", + "in such repositories by hand but use a research data management system like NOMAD Oasis instead.
\n", + "\n", + "The benefit of the data normalization that pynxtools-em performs is that all pieces of information are represents in the
\n", + "same conceptual way with the benefit that most of the so far required format conversions when interfacing with software
\n", + "from the technology partners or scientific community are no longer necessary.
" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### **Step 1:** Check that packages are installed and working in your local Python environment." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check the result of the query below specifically that `jupyterlab_h5web` and `pynxtools` are installed in your environment.
\n", + "Note that next to the name pynxtools you should see the directory in which it is installed. Otherwise, make sure that you follow
\n", + "the instructions in the `README` files: \n", + "- How to set up a development environment as in the main README \n", + "- Lauch the jupyter lab from this environement as in the README of folder `examples`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! pip list | grep \"h5py\\|nexus\\|jupyter\\|jupyterlab_h5web\\|pynxtools\\|pynxtools-em\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Set the pynxtools directory and start H5Web for interactive exploring of HDF5 files." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import zipfile as zp\n", + "import numpy as np\n", + "from jupyterlab_h5web import H5Web\n", + "print(f\"Current working directory: {os.getcwd()}\")\n", + "print(f\"So-called base, home, or root directory of the pynxtools: {os.getcwd().replace('/examples/em', '')}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### **Step 2:** Use your own data or download an example" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "Please note that the metadata inside the provided em.oasis.specific.yaml and eln_data_apm.yaml files
\n", + "contain exemplar values. These do not necessarily reflect the conditions when the raw data of example
\n", + "above-mentioned were collected by the scientists. Instead, these file are meant to be edited by you,
\n", + "either and preferably programmatically e.g. using output from an electronic lab notebook or manually.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This example shows the types of files from which the parser collects and normalizes pieces of information:
\n", + "* **eln_data.yaml** metadata collected with an electronic lab notebook (ELN) such as a NOMAD Oasis custom schema
\n", + "* **em.oasis.specific.yaml** frequently used metadata that are often the same for many datasets to avoid having to
\n", + " type it every time in ELN templates. This file can be considered a configuration file whereby e.g. coordinate system
\n", + " conventions can be injected or details about the atom probe instrument communicated if that is part of frequently used
\n", + " lab equipment. The benefit of such an approach is that eventual all relevant metadata to an instrument can be read from\n", + " this configuration file via guiding the user e.g. through the ELN with an option to select the instrument.
\n", + "* **collected data** in community, technology partner format with images, spectra, and other metadata.
\n", + "\n", + "The tool several of the currently frequently used file formats of the electron microscopy community. Given that there is
\n", + "though a large number of these and different versions users should also be aware that we had to prioritize the implementation
\n", + "strongly. We cannot implement every request to add support for further formats or additional pieces of information in those
\n", + "formats we currently do support with the resources in the FAIRmat project. Nevertheless, please raise an issue to document
\n", + "where we should place our priorities.
\n", + "Consult the reference part of the documentation to get a detailed view on how specific formats are supported.
" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### **Step 3:** Run the parser" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "eln_data = [\"eln_data.yaml\"]\n", + "deployment_specific = [\"em.oasis.specific.yaml\"]\n", + "tech_partner = [\"CHANGEME YOUR TECH PARTNER FILE (e.g. EMD, Nion, etc.)\"]\n", + "output_file_name = [\"em.nxs\"]\n", + "for case_id in np.arange(0, 1):\n", + " ELN = eln_data[case_id]\n", + " CFG = deployment_specific[case_id]\n", + " DATA = tech_partner[case_id]\n", + " OUTPUT = output_file_name[case_id]\n", + "\n", + " # CHANGEME activate the following line\n", + " # ! dataconverter convert $ELN $CFG $DATA --reader em --nxdl NXem --output $OUTPUT" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "### **Step 4:** Inspect the NeXus/HDF5 file using H5Web." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# CHANGEME activate the following line to view the data\n", + "# H5Web(OUTPUT)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The NeXus file an also be viewed with H5Web by opening it via the file explorer panel to the left side of this Jupyter lab window." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Conclusions:\n", + "***" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This tutorial showed how you can call the pynxtools-em parser via a jupyter notebook. This opens many possibilities
\n", + "like processing the results further with Python such as through e.g. conda on your local computer, a virtual environment
\n", + "or to interface with community software to do further processing of your data.
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Contact person for pynxtools-apm and related examples in FAIRmat:\n", + "Dr.-Ing. Markus Kühbach, 2024/05/10
\n", + "\n", + "### Funding\n", + "FAIRmat is a consortium on research data management which is part of the German NFDI.
\n", + "The project is funded by the Deutsche Forschungsgemeinschaft (DFG, German Research Foundation) – project 460197019." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..d8c4014 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,19 @@ +## em reader + +**Write.NXem.Example.1.ipynb** provides an example how the em parser/reader/data extractor +can be used as a standalone tool for converting data and metadata from different sources into an NXem-formatted NeXus/HDF5 file. + +**eln_data.yaml** is a YAML/text file which contains relevant data which are not +contained typically in files from technology partners. These data have been collected +either by editing the file manually or by using an electronic lab notebook (ELN), +such as the NOMAD ELN. Every other ELN can be used with this parser provided that +this ELN writes its data into a YAML file with the same keywords and structure as is +exemplified in the above-mentioned YAML file. + +**em.oasis.specific.yaml** is a YAML/text file which contains additional metadata that are +not necessarily coming from an ELN but which should be considered during parsing. +One use case where this file is ideal is for passing frequent metadata that are often the +same and thus can be offloaded once from an ELN to make such ELN templates more +focused on metadata that do change frequently. A typical example is a lab where always the +same microscope is used, many static details of the microscope are expected to end up +in the static section of the NXem application definition i.e. */NXem/ENTRY/measurement/em_lab* diff --git a/examples/README_Ebsd.md b/examples/README_Ebsd.md deleted file mode 100644 index eb665bf..0000000 --- a/examples/README_Ebsd.md +++ /dev/null @@ -1,37 +0,0 @@ -## em_om reader - -This is an example how the em_om parser/reader/data extractor can be used as a -standalone tool to convert data and metadata from different sources into an -NXem_ebsd-formatted NeXus/HDF5 file. Further details to the functionalities of the -parser are documented in the parsers sub-directory: - -``` -pynxtools/pynxtools/dataconverter/readers/em_om -``` - -**Write.NXem_ebsd.Example.1.ipynb** is the Jupyter notebook which exemplies -how the parser can be used as a standalone version, i.e. without NOMAD. - -**eln_data_em_om.yaml** is a YAML/text file which contains relevant data which are not -contained typically in files from technology partners. These data have been collected -either by editing the file manually or by using an electronic lab notebook (ELN), -such as the NOMAD ELN. -A few example files from real electron backscatter diffraction measurements are -offered as downloads to run the example with the above-mentioned Juypter notebook. - -Every other ELN can be used with this parser provided that this ELN writes its data -into a YAML file with the same keywords and structure as is exemplified in the -above-mentioned YAML file. - -### Map data from Matlab/MTex to NXem_ebsd -The download material includes several \*.mtex files. These reflect that the -em_om parser can handle data which have been generated with the Matlab/MTex -texture tool box. These \*.mtex files are currently an exchange file format -to map data from the MTex internal representation to something which the em_om -parser understands and can map to NXem_ebsd. You are very much invited to test this -feature. The feature requires, though, a specific extension of the MTex toolbox, -which is an MTex/Matlab script surplus a wrapper for the HDF5 library to enable -writing HDF5 files with more flexible data structures and attributes. -We tested this extension with Matlab>=2021 and MTex>=5.8.2. -Please contact Markus Kühbach directly if you are interested in testing this feature -and contribute thereby to make EBSD data more interoperable. \ No newline at end of file diff --git a/examples/README_Nion.md b/examples/README_Nion.md deleted file mode 100644 index 324fd50..0000000 --- a/examples/README_Nion.md +++ /dev/null @@ -1,24 +0,0 @@ -## em_nion reader - -This is an example how the em_nion parser/reader/data extractor can be used as a standalone -tool to convert data and metadata from a compressed nionswift project into an NXem-formatted -NeXus/HDF5 file. Further details to the functionalities of the parser are documented -in the parsers sub-directory: - -``` -pynxtools/pynxtools/dataconverter/readers/em_nion -``` - -**Write.NXem_nion.Example.1.ipynb** is the Jupyter notebook which exemplies -how the parser can be used as a standalone version, i.e. without NOMAD. - -**eln_data_em_nion.yaml** is a YAML/text file which contains relevant data which are not -contained typically in files from technology partners. These data have been collected -either by editing the file manually or by using an electronic lab notebook (ELN), -such as the NOMAD ELN. -A few example files from real atom probe reconstructions and ranging definitions are -offered as downloads to run the example with the above-mentioned Juypter notebook. - -Every other ELN can be used with this parser provided that this ELN writes its data -into a YAML file with the same keywords and structure as is exemplified in the -above-mentioned YAML file. \ No newline at end of file diff --git a/examples/README_Spctrscpy.md b/examples/README_Spctrscpy.md deleted file mode 100644 index 7b9a31c..0000000 --- a/examples/README_Spctrscpy.md +++ /dev/null @@ -1,24 +0,0 @@ -## em_spctrscpy reader - -This is an example how the em_spctrscpy parser/reader/data extractor can be used as a -standalone tool to convert data and metadata from different sources into an NXem-formatted -NeXus/HDF5 file. Further details to the functionalities of the parser are documented -in the parsers sub-directory: - -``` -pynxtools/pynxtools/dataconverter/readers/em_spctrscpy -``` - -**Write.NXem.Example.1.ipynb** is the Jupyter notebook which exemplies -how the parser can be used as a standalone version, i.e. without NOMAD. - -**eln_data_em_spctrscpy.yaml** is a YAML/text file which contains relevant data which are not -contained typically in files from technology partners. These data have been collected -either by editing the file manually or by using an electronic lab notebook (ELN), -such as the NOMAD ELN. -A few example files from real electron microscopy measurements are offered as downloads -to run the example with the above-mentioned Juypter notebook. - -Every other ELN can be used with this parser provided that this ELN writes its data -into a YAML file with the same keywords and structure as is exemplified in the -above-mentioned YAML file. \ No newline at end of file diff --git a/examples/Write.NXem_ebsd.Example.1.ipynb b/examples/Write.NXem_ebsd.Example.1.ipynb deleted file mode 100644 index 7f5afeb..0000000 --- a/examples/Write.NXem_ebsd.Example.1.ipynb +++ /dev/null @@ -1,377 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using dataconverter/em_om for mapping EBSD/Orientation Microscopy to NeXus/HDF5/NXem_ebsd" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [] - }, - "source": [ - "### **Step 1:** Check that packages are installed and working in your local Python environment." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check the result of the query below specifically that `jupyterlab_h5web` and `pynxtools` are installed in your environment.
\n", - "Note that next to the name pynxtools you should see the directory in which it is installed. Otherwise, make sure that you follow
\n", - "the instructions in the `README` files: \n", - "- How to set up a development environment as in the main README \n", - "- Lauch the jupyter lab from this environement as in the README of folder `examples`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "! pip list | grep \"h5py\\|nexus\\|jupyter\" && jupyter serverextension list && jupyter labextension list && python -V" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Set the pynxtools directory and start H5Web for interactive exploring of HDF5 files." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from jupyterlab_h5web import H5Web\n", - "print(f\"Current working directory: {os.getcwd()}\")\n", - "print(f\"So-called base, home, or root directory of the pynxtools: {os.getcwd().replace('/examples/em_om', '')}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### **Step 2:** Download EM-OM-specific example data or use your own datasets." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [] - }, - "source": [ - "Example data can be found on Zenodo https://dx.doi.org/10.5281/zenodo.7885531." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import zipfile as zp" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "! curl --output em_om_sprint14_01.zip https://zenodo.org/record/7885531/files/em_om_sprint14_01.zip\n", - "! curl --output em_om_sprint14_02.zip https://zenodo.org/record/7885531/files/em_om_sprint14_02.zip" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "zp.ZipFile(\"em_om_sprint14_01.zip\").extractall(path=\"\", members=None, pwd=None)\n", - "zp.ZipFile(\"em_om_sprint14_02.zip\").extractall(path=\"\", members=None, pwd=None)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "These files should serve exclusively as examples. The dataconverter for EM-OM always requires at least one file:\n", - "\n", - "**When you only would like to store conventions**:
\n", - "* **YAML file with metadata** (either edited manually or via an ELN).
\n", - " The eln_data_em_om.yaml file in the example can be edited with a text editor.\n", - "\n", - "**When you would like to store conventions surplus stack of simulated Kikuchi images**:
\n", - "* **YAML file with metadata**
\n", - "* **ZIP archive with images all using the same format, rectangular dimensions, and file type**
\n", - "\n", - "**When you would like to store conventions surplus H5OINA EBSD mapping**:
\n", - "* **YAML file with metadata**
\n", - "* **H5OINA file as obtained from Oxford Instrument AZTec (>=5.0)**
\n", - "\n", - "**When you would like to store conventions surplus have transcoded EBSD results from MTex**:
\n", - "* **YAML file with metadata**
\n", - "* **\\*.mtex file as obtained by running the specific MTex to NeXus transcoder** (see second info box below for details)
" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "
\n", - "For GUI-based editing, a NOMAD OASIS instance is needed.
\n", - "
" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "
\n", - "Please note that the metadata inside the provided eln_data_em_om.yaml file are example values.
\n", - "These reflect not necessarily the conditions when the raw data for the example were collected!
\n", - "The file is meant to be edited by you if you work with datasets others than the here provided!
\n", - "
" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "
\n", - "Feel free to contact the maintainer of this example to learn more about the parsing capabilities of SEM/EBSD data in NOMAD.
\n", - "We have also a draft version which supports importing results from MatLab/MTex and DREAM.3D. We would like to get in contact
\n", - "to document and develop these further, ideally using as diverse examples as possible, maybe also including one of your examples?
\n", - "
" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [] - }, - "source": [ - "### **Step 3:** Run the EBSD-specific dataconverter on the example data." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we run our parser. The --reader flag takes the em_om reader, the --nxdl flag takes the application definition for this technique NXem_ebsd." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### **Step 3a:** Optionally see the command line help of the dataconverter." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "! dataconverter --help" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### **Step 3b:** Optionally explore all paths which NXem provides." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "! dataconverter --nxdl NXem_ebsd --reader em_om --generate-template" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### **Step 3c**: Convert the files in the example into an NXem_ebsd-compliant NeXus/HDF5 file." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "
\n", - "In what follows we will show several of the examples that have been implemented for SEM/EBSD.
\n", - "
" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Only conventions:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# parser-nexus/tests/data/tools/dataconverter/readers/em/\n", - "output_file_name = [\"em_om.case0.nxs\"]\n", - "! dataconverter \\\n", - "--reader em_om \\\n", - "--nxdl NXem_ebsd \\\n", - "--input-file eln_data_em_om.yaml \\\n", - "--output \"em_om.case0.nxs\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Conventions and different data sources:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true, - "tags": [] - }, - "outputs": [], - "source": [ - "#parser-nexus/tests/data/tools/dataconverter/readers/em_om/\n", - "import numpy as np\n", - "eln_data_file_name = [\"eln_data_em_om.yaml\"]\n", - "input_data_file_name = [\"PrcShanghaiShi.EBSPs70deg.zip\",\n", - " \"H5OINA_examples_Specimen_1_Map_EDS_+_EBSD_Map_Data_2.h5oina\",\n", - " \"Forsterite.ctf.mtex\",\n", - " \"SmallIN100_Final.dream3d\"]\n", - "output_file_name = [\"em_om.case1.nxs\",\n", - " \"em_om.case2.nxs\",\n", - " \"em_om.case3e.nxs\",\n", - " \"em_om.case4.nxs\"]\n", - "for case_id in np.arange(0, 3 + 1):\n", - " ELN = eln_data_file_name[0]\n", - " INPUT = input_data_file_name[case_id]\n", - " OUTPUT = output_file_name[case_id]\n", - "\n", - " ! dataconverter --reader em_om --nxdl NXem_ebsd --input-file $ELN --input-file $INPUT --output $OUTPUT" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The key take home message is that the command above-specified triggers the automatic creation of the HDF5 file. This *.nxs file, is an HDF5 file." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [] - }, - "source": [ - "### **Step 4:** Inspect the HDF5/NeXus file using H5Web" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# H5Web(OUTPUT)\n", - "H5Web(\"em_om.case0.nxs\")\n", - "# H5Web(\"em_om.case1.nxs\")\n", - "# H5Web(\"em_om.case2.nxs\")\n", - "# H5Web(\"em_om.case3e.nxs\")\n", - "# H5Web(\"em_om.case4.nxs\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here is where the general template ends. Continue with filling in the notebook with your own post-processing of this *.nxs file." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "***" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Contact person for this example in FAIRmat:\n", - "Markus Kühbach" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Contact person for the apm reader and related examples in FAIRmat:\n", - "Markus Kühbach, 2023/08/31
\n", - "\n", - "### Funding\n", - "FAIRmat is a consortium on research data management which is part of the German NFDI.
\n", - "The project is funded by the Deutsche Forschungsgemeinschaft (DFG, German Research Foundation) – project 460197019." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - }, - "vscode": { - "interpreter": { - "hash": "fb9c1bd38c7663f011bf442e740f4844d912585f80182885cfafffa0a1ff37a6" - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/Write.NXem_nion.Example.1.ipynb b/examples/Write.NXem_nion.Example.1.ipynb deleted file mode 100644 index 0d48dea..0000000 --- a/examples/Write.NXem_nion.Example.1.ipynb +++ /dev/null @@ -1,287 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using dataconverter/em_nion for mapping content of a nionswift project to NeXus/HDF5/NXem" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [] - }, - "source": [ - "### **Step 1:** Check that packages are installed and working in your local Python environment." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check the result of the query below specifically that `jupyterlab_h5web` and `pynxtools` are installed in your environment.
\n", - "Note that next to the name pynxtools you should see the directory in which it is installed. Otherwise, make sure that you follow
\n", - "the instructions in the `README` files: \n", - "- How to set up a development environment as in the main README \n", - "- Lauch the jupyter lab from this environement as in the README of folder `examples`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "! pip list | grep \"h5py\\|nexus\\|jupyter\" && jupyter serverextension list && jupyter labextension list && python -V" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Set the pynxtools directory and start H5Web for interactive exploring of HDF5 files." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from jupyterlab_h5web import H5Web\n", - "print(f\"Current working directory: {os.getcwd()}\")\n", - "print(f\"So-called base, home, or root directory of the pynxtools: {os.getcwd().replace('/examples/em_nion', '')}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### **Step 2:** Download Nionswift-specific example data or try out with one of your own datasets." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [] - }, - "source": [ - "Example data can be found on Zenodo http://dx.doi.org/10.5281/zenodo.7986279." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import zipfile as zp" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "! wget https://www.zenodo.org/record/7986279/files/ger_berlin_haas_nionswift_multimodal.zip" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "zp.ZipFile(\"ger_berlin_haas_nionswift_multimodal.zip\").extractall(path=\"\", members=None, pwd=None)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "These files should serve exclusively as examples. The dataconverter em_nion for nionswift project always requires a pair of files:\n", - "* A **YAML file with metadata** (either edited manually/or generated via an ELN).
\n", - " The eln_data_apm.yaml file in the example can be edited with a text editor.
\n", - "* A **compressed zip.nionswift file** with the mime type/file name ending \\*.nionswift suffix.
\n", - " This includes the *.nsproj file and a directory or directory nest with *.ndata and *.h5 files
\n", - " which were generated from nionswift.
" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "
\n", - "For GUI-based editing, a NOMAD OASIS instance is needed.
\n", - "
" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "
\n", - "Please note that the metadata inside the provided eln_data_em_nion.yaml file contains example values.
\n", - "These reflect not necessarily the conditions when the raw data for the example were collected!
\n", - "The file is meant to be edited by you if you work with datasets others than the here provided!
\n", - "
" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [] - }, - "source": [ - "### **Step 3:** Run the nionswift-specific dataconverter on the example data." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we run our parser. The --reader flag takes the em_nion reader (em_nion), the --nxdl flag takes the application definition for this technique which is currently NXem.
" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### **Step 3a:** Optionally see the command line help of the dataconverter." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "! dataconverter --help" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### **Step 3b:** Optionally explore all paths which NXapm provides." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# to inspect what can/should all be in the NeXus file\n", - "! dataconverter --nxdl NXem --generate-template" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### **Step 3c**: Convert the files in the example into an NXapm-compliant NeXus/HDF5 file." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "#parser-nexus/tests/data/tools/dataconverter/readers/em_om/\n", - "eln_data_file_name = [\"eln_data_em_nion.yaml\"]\n", - "swift_proj_file_name = [\"2022-02-18_Metadata_Kuehbach.zip.nionswift\"]\n", - "output_file_name = [\"nion.case1.nxs\"]\n", - "for case_id in [0]:\n", - " ELN = eln_data_file_name[0]\n", - " SWIFT = swift_proj_file_name[case_id]\n", - " OUTPUT = output_file_name[case_id]\n", - "\n", - " ! dataconverter --reader em_nion --nxdl NXem --input-file $ELN --input-file $SWIFT --output $OUTPUT" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The key take home message is that the command above-specified triggers the automatic creation of the HDF5 file. This *.nxs file, is an HDF5 file." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [] - }, - "source": [ - "### **Step 4:** Inspect the NeXus/HDF5 file using H5Web." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# H5Web(OUTPUT)\n", - "H5Web(\"nion.case1.nxs\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You can also visualize the .nxs file by double clicking on it in the file explorer panel to the left side of your jupyter lab screen in the browser." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Contact person for the em_nion reader and related examples in FAIRmat:\n", - "Markus Kühbach, 2023/08/31
\n", - "\n", - "### Funding\n", - "FAIRmat is a consortium on research data management which is part of the German NFDI.
\n", - "The project is funded by the Deutsche Forschungsgemeinschaft (DFG, German Research Foundation) – project 460197019." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/Write.NXem_spctrscpy.Example.1.ipynb b/examples/Write.NXem_spctrscpy.Example.1.ipynb deleted file mode 100644 index 3b57b7f..0000000 --- a/examples/Write.NXem_spctrscpy.Example.1.ipynb +++ /dev/null @@ -1,349 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using dataconverter/em_spctrscpy for mapping EDX(S) and EELS spectroscopy as well as electron microscopy images to NeXus/HDF5/NXem" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [] - }, - "source": [ - "### **Step 1:** Check that packages are installed and working in your local Python environment." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check the result of the query below specifically that `jupyterlab_h5web` and `pynxtools` are installed in your environment.
\n", - "Note that next to the name pynxtools you should see the directory in which it is installed. Otherwise, make sure that you follow
\n", - "the instructions in the `README` files: \n", - "- How to set up a development environment as in the main README \n", - "- Lauch the jupyter lab from this environement as in the README of folder `examples`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "! pip list | grep \"h5py\\|nexus\\|jupyter\" && jupyter serverextension list && jupyter labextension list && python -V" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Set the pynxtools directory and start H5Web for interactive exploring of HDF5 files." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from jupyterlab_h5web import H5Web\n", - "print(f\"Current working directory: {os.getcwd()}\")\n", - "print(f\"So-called base, home, or root directory of the pynxtools: {os.getcwd().replace('/examples/em_spctrscpy', '')}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### **Step 2:** Download EM-SPCTRSCPY-specific example data or use your own dataset." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [] - }, - "source": [ - "Example data can be found on Zenodo https://zenodo.org/record/7908429." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import zipfile as zp" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "! curl --output EM.Various.Datasets.1.zip https://zenodo.org/record/7908429/files/EM.Various.Datasets.1.zip" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "zp.ZipFile(\"EM.Various.Datasets.1.zip\").extractall(path=\"\", members=None, pwd=None)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "These files should serve exclusively as examples. The dataconverter for EM-SPCTRSCPY always requires a pair of files:\n", - "* A **YAML file with metadata** (either edited manually or via an ELN).
\n", - " The eln_data_em_spctrscpy.yaml file in the example can be edited with a text editor.
\n", - "* A **community or technology partner / vendor file** with the data from the microscope session.
\n", - " Hyperspy is used for parsing, tests were performed for DM3, BCF, and Velox EMD files.
" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "
\n", - "For GUI-based editing, a NOMAD OASIS instance is needed.
\n", - "
" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "
\n", - "Please note that the metadata inside the provided eln_data_em_spctrscpy.yaml file contains example values.
\n", - "These reflect not necessarily the conditions when the raw data for the example were collected!
\n", - "The file is meant to be edited by you if you work with datasets others than the here provided!
\n", - "
" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [] - }, - "source": [ - "### **Step 3:** Run the EM-spectroscopy-specific dataconverter on the example data." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we run our parser. The --reader flag takes the em_spctrscpy reader, the --nxdl flag takes the application definition for this technique NXem.
" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### **Step 3a:** Optionally see the command line help of the dataconverter." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "! dataconverter --help" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### **Step 3b:** Optionally explore all paths which NXem provides." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "! dataconverter --nxdl NXem --reader em_spctrscpy --generate-template" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### **Step 3c**: Convert the files in the example into an NXem-compliant NeXus/HDF5 file." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "#parser-nexus/tests/data/tools/dataconverter/readers/em_spctrscpy/\n", - "eln_data_file_name = [\"eln_data_em_spctrscpy.yaml\"]\n", - "input_data_file_name = [\"46_ES-LP_L1_brg.bcf\",\n", - " \"1613_Si_HAADF_610_kx.emd\",\n", - " \"EELS_map_2_ROI_1_location_4.dm3\"]\n", - "output_file_name = [\"em_sp.case1.nxs\",\n", - " \"em_sp.case2.nxs\",\n", - " \"em_sp.case3.nxs\"]\n", - "for case_id in [0, 1, 2]:\n", - " ELN = eln_data_file_name[0]\n", - " INPUT = input_data_file_name[case_id]\n", - " OUTPUT = output_file_name[case_id]\n", - "\n", - " ! dataconverter --reader em_spctrscpy --nxdl NXem --input-file $ELN --input-file $INPUT --output $OUTPUT" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The key take home message is that the command above-specified triggers the automatic creation of the HDF5 file. This *.nxs file, is an HDF5 file." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "tags": [] - }, - "source": [ - "### **Step 4:** Inspect the NeXus/HDF5 file using H5Web" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# H5Web(OUTPUT)\n", - "H5Web(\"em_sp.case1.nxs\")\n", - "# H5Web(\"em_sp.case2.nxs\")\n", - "# H5Web(\"em_sp.case3.nxs\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### **Optional:** Generate synthetic data for testing and development purposes" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The apm reader has a functionality to generate synthetic dataset which are meant for pursuing code development." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "! dataconverter --reader em_spctrscpy --nxdl NXem --input-file synthesize1 --output em_sp.case0.nxs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "H5Web(\"em_sp.case0.nxs\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here is where the general template ends. Continue with filling in the notebook with your own post-processing of this *.nxs file." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "***" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Contact person for this example in FAIRmat:\n", - "Markus Kühbach" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Contact person for the apm reader and related examples in FAIRmat:\n", - "Markus Kühbach, 2023/08/31
\n", - "\n", - "### Funding\n", - "FAIRmat is a consortium on research data management which is part of the German NFDI.
\n", - "The project is funded by the Deutsche Forschungsgemeinschaft (DFG, German Research Foundation) – project 460197019." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - }, - "vscode": { - "interpreter": { - "hash": "fb9c1bd38c7663f011bf442e740f4844d912585f80182885cfafffa0a1ff37a6" - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/eln_data.yaml b/examples/eln_data.yaml index 91ee87d..7454d67 100644 --- a/examples/eln_data.yaml +++ b/examples/eln_data.yaml @@ -1,10 +1,16 @@ entry: - experiment_alias: Testing experiment_alias - start_time: 2024-04-08T11:11:11.000Z + experiment_alias: test + start_time: "2024-05-08T12:56:00+00:00" + end_time: "2024-05-08T12:56:00+00:00" + experiment_description: "

Normal

\n

Bold

\n

Italics

" sample: method: experiment - atom_types: Xe, Yb, Zn - preparation_date: 2024-04-08T11:11:11.000Z + name: test + atom_types: Yb, Gd + preparation_date: "2024-05-08T13:06:00+00:00" + thickness: + value: 100.0 + unit: m user: - name: MarkusK orcid: "0000" diff --git a/examples/eln_data_em_nion.yaml b/examples/eln_data_em_nion.yaml deleted file mode 100644 index 455816c..0000000 --- a/examples/eln_data_em_nion.yaml +++ /dev/null @@ -1,63 +0,0 @@ -em_lab: - detector: - - local_name: Should better be inferred in the future from a Nion microscope instance description instead of having to add it every time manually. - ebeam_column: - aberration_correction: - applied: true - aperture_em: - - name: C1 - value: 4 - electron_source: - emitter_type: field_emission - voltage: - unit: V - value: 200000 - fabrication: - capabilities: '---' - identifier: ChristophKochGroupsNionMicroscope - model: NionHermes200 - vendor: Nion Co. - instrument_name: Nion Hermes 200kV - location: Berlin - optical_system_em: - beam_current: - unit: A - value: 1.2e-11 - beam_current_description: estimated - magnification: 610000 - semi_convergence_angle: - unit: rad - value: 0.2 - stage_lab: - description: double tilt - name: nothing -entry: - attr_version: nexus-fairmat-proposal successor of 9636feecb79bb32b828b1a9804269573256d7696 - definition: NXem - end_time: '2023-05-25T14:44:00+01:00' - experiment_description: BenediktHaas_MultiModal_ImagingMode_TestDataSet - experiment_identifier: 2022-04-18-BenediktHaas - program: nionswift - program__attr_version: 0.16.8, this is what Markus used for implementing the nionswift project file reader but was it also the version used by Benedikt? - start_time: '2023-05-25T14:44:00+01:00' -sample: - atom_types: - - Cu - description: test - method: experiment - name: Cu - preparation_date: '2023-05-25T14:44:00+01:00' - sample_history: unknown - short_title: Cu - thickness: - unit: m - value: 2.0e-08 -user: -- name: MarkusK - orcid: '0000' -- email: '----' - name: Benedikt -- name: Sherjeel - affiliation: HU Berlin -- name: Christoph - role: B1 task leader diff --git a/examples/eln_data_em_om.yaml b/examples/eln_data_em_om.yaml deleted file mode 100644 index 383d15d..0000000 --- a/examples/eln_data_em_om.yaml +++ /dev/null @@ -1,65 +0,0 @@ -entry: - attr_version: nexus-fairmat-proposal successor of 9636feecb79bb32b828b1a9804269573256d7696 - definition: NXem_ebsd - workflow_identifier: test_id - workflow_description: test_description - start_time: 2023-02-04T14:43:00.000Z - end_time: 2023-02-04T14:43:00.000Z - program: nexusutils/dataconverter/readers/em_om.py - program__attr_version: undefined -user: -- name: Markus Kühbach -commercial_on_the_fly_indexing: - program: TSL EDAX - program__attr_version: v6.0 - results_file: test.ang - results_file__attr_version: n/a -measurement: - origin: undefined - origin__attr_version: should be collected if possible automatically by parser or RDMS - path: undefined -calibration: - origin: undefined - origin__attr_version: should be collected if possible automatically by parser or RDMS - path: undefined -rotation_conventions: - three_dimensional_rotation_handedness: counter_clockwise - rotation_convention: passive - euler_angle_convention: zxz - axis_angle_convention: rotation_angle_on_interval_zero_to_pi - sign_convention: p_minus_one -processing_reference_frame: - reference_frame_type: right_handed_cartesian - xaxis_direction: east - xaxis_alias: rolling direction (RD) - yaxis_direction: south - yaxis_alias: transverse direction (TD) - zaxis_direction: undefined - zaxis_alias: normal direction (ND) - origin: front_top_left -sample_reference_frame: - reference_frame_type: right_handed_cartesian - xaxis_direction: east - yaxis_direction: south - zaxis_direction: undefined - origin: front_top_left -detector_reference_frame: - reference_frame_type: right_handed_cartesian - xaxis_direction: east - yaxis_direction: south - zaxis_direction: undefined - origin: front_top_left -gnomonic_projection: - gnomonic_projection_reference_frame: - reference_frame_type: right_handed_cartesian - xaxis_direction: east - yaxis_direction: south - zaxis_direction: undefined - origin: in_the_pattern_centre - pattern_centre: - xaxis_boundary_convention: top - xaxis_normalization_direction: east - yaxis_boundary_convention: top - yaxis_normalization_direction: south -indexing: - method: hough_transform diff --git a/examples/eln_data_em_spctrscpy.yaml b/examples/eln_data_em_spctrscpy.yaml deleted file mode 100644 index 68d5d37..0000000 --- a/examples/eln_data_em_spctrscpy.yaml +++ /dev/null @@ -1,64 +0,0 @@ -em_lab: - detector: - - local_name: EDS detector - ebeam_column: - aperture_em: - - name: condensed - value: 4 - aberration_correction: - applied: true - electron_source: - emitter_type: field_emission - # cold_cathode_ - name: some - voltage: - unit: V - value: 200000 - instrument_name: TALOS - location: EPFL - fabrication: - capabilities: many - identifier: undisclosed - model: unknown - vendor: TALOS - optical_system_em: - beam_current: - unit: A - value: 1.2e-11 - beam_current_description: estimated - magnification: 610000 - semi_convergence_angle: - unit: rad - value: 0.2 - stage_lab: - description: nothing - name: double tilt -entry: - attr_version: nexus-fairmat-proposal successor of 9636feecb79bb32b828b1a9804269573256d7696 - definition: NXem - end_time: '2022-08-11T14:23:00+00:00' - experiment_description: '1613' - experiment_identifier: '1613' - program: Velox - program__attr_version: v6.8 - start_time: '2022-08-03T14:23:00+00:00' -sample: - atom_types: - - Al - - Nd - - O - description: test - method: experiment - name: 1613_Si - preparation_date: '2022-08-11T14:25:00+00:00' - sample_history: unknown - short_title: '1613' - thickness: - unit: m - value: 2.0e-08 -user: -- name: MarkusK - orcid: '0000' - orcid_platform: orcid -- email: '-----' - name: MarkusS diff --git a/examples/image_png_protochips_to_nexus.ods b/examples/image_png_protochips_to_nexus.ods deleted file mode 100644 index 323fbc10c6c2670bcf24cc41ebc5cc5197084f5a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11254 zcmb7K1z6Ne*Iz;bk(BP1URt_QLK+EamZe#i-6aJ zEqc9r?|Xfp_d7hh`)B6VoSFIUnLVSSf`(260AKjVM1LM*wR%;C0hZYL-hV(A2OaRfsU+;AuiY;Fm+0fP}5f79uE z8a-x=4gg%e$dEd9Y+M{IAm+9Xa3JExBR3Rc9ipLj59bEO4I~SWlA^p80Dyvg1ejr> zBHz7Z#O441bS5Qv86Eeejn^Bn44#_`UH50}V{Q**FS316F8_-D+;2-ns?KYYM4y!D zYw7suc@rQP)@a0yha-NGNd2X~+)Ve}Rhymj=txXUsm5Kut)KL|z#CWNoTHa+gPvQ8 zt%ItDFG?;pLgS4O{+NF38(@EI-Pk-PnP>x=`5f6r{DO!o&$>al1#{dJJXmlsjlJ`D zBGmp8-`7J1K!<}0u*C-``U9jX&;cLl08!Ubd<8H8na3o)DdyFL{j-yj_+L2I`fKie1LH636pNQn!t8p{(knvAj?%t9}$f-Th>=k$aFjH6AuX z5kNDAKgmF#wBc-SfXON?=CUSe^HAvlma%_~DHCw$K*{0Jv$6L#<=%ZhJlH7+A#Y%T z?9}4tLOMB$iibH*UE||pV<>GK*mn{Or|nfeHf*QJF1Smt5txKNJG%|=rLyd{9BNQ+)$si=3O3jRfW z=R|d_T&lvgUR%Rrw}bCKm1^PR+g_Ct4oPpM$F#}Y9HZ7O3JK9qVH71k`Ik9q?vdSwp#*LCjMuUad~N zoS~JP>A7-H3?MT@97Aq?QfOt{(fT&0_iiVv51D~|mTX7$qOjyM^$3l^ud`6$O1FZ^ z0-9ZB;|Ihqn!H(W5Tu|pQ)+=--=1`I(0<*o7@g$sFiUI_?Ef0&eZF&SB7KshLW~~b zD0^^#w$Wz_w-0C!wOe2HEMffT6lJPvHK0E)C zPj1PW!s@5Vj!y=lGvdL~I*o(jiq`E)_GZecQk7b~$oUnpjFFdey~M}nch%3@@l8)f z1hWV~UMqB7z;AT3O_YXv_J>q=8a|VpTPQ}+De=QlW2D!dPR~>}^>yMB?S}6jGQdh$ zPHtQ~4>2NGej1kFJQUP$KF&&UB#fybp}n8<$IHNv4H-cvl=JH13U^8~1~5%uB;=Ul zC9h4vrn^fk%2Pi?O+n83b4hcFu5t&)9IyZJg{jDmUW&4wQqg zYeM7#H5%p9n7dL!z=_3U-_PFN;|UdH1} zJ2CgzXSfc>P({wM1W`5n?9Ao_Nt#oq&er{(?8`7I!dkXo;!o4PpA>&PZq-_L|1UAuo5l6?%Rh-WQ-1~Z;XfiYh zc&>~kF%hO|+g+6FMsARnrW@58pMTvJJ2ah{tE?=5GWH#MWK){+Z`2Ph%t7@m-Az7N zj%a^$m;gSGuk3kTMtzB`p@M-K>hh-J01W`xqXqze4;R=-xBxjp5LYSB6YFdkK^l4UFidBtHxRrCQd^jnHTj&{$8IopV`Xoyz4ygPp{vahyT^GQ_td-5l zUPc}=TbtyFifvZbJjX4G`P|8;?$BrvmtQT{N5g)k#_swbj_de72#wQ0F~LJSDe8+g zE^6G6a6{ZGIlR-im3hD=C%L#kJUF@~aszvvG~Z!49e1`hFn=Y%DNTvB%gQpWD`E`{ z#i)Tgb0u+eZ8x$$)bFzDPM6tDqzpS$EmbzU4YVly|&I< zgqM3Vj-2v?<%M0AS}6|WY?KD!=jYxkD{92mIWU<{Bd*UAB(7z4VTR$JZOWDsoEn83?++?>PDPT2bX)CGMG+nRA*HVNsdYKP^MAab3&FT?bsC2Xg+jPjZ|BONp15@&cc($ zV}o%T5MEzE7tB27$cWCziac9de`44;^ph_iuV8Pf`}PYP!JNAAz!h#>YW6z>)Vv+Y zTyNdMkS|Q}w$K)4@|nmj!<6N=rmVp1cgz#24VLVbZWwbI2?V3Jme?5*XyWh2ETzhD zJP=*5Fya#|7PJrQq9cipGaL>`qw4LYFE3j&oBAB&lwl^IDwJ-Y)giN?A0$M2tkxIR z$`HL9d>Yk%Hgtr2gY~k5^Sw>VVS1%44Gwv0@0*qwCOH=8g+=*=u+F!lN#rXdIg<;d z?r zkk2fBwtMe4ko~%7GcKLaW6(z#F5Y_j!cR_+1v$58-jpxhCxUD3$92qepeU{1( zi&#~dBY4Tf1MM1j4vpU*0|1HSg{c*(J3u0Oy2mfu0b1R@vOtFVh>A zg?qo^A+kx&eOXzT5PrEAmq1`IyVMP`Siexp$>t2r;8v9T)ZJB_G({h8U7=IQ)*kjW zqm0}M?|7txydJ#lg@u{q6&(^@B)uvivPTp6{1JvA)v>>-oGFiOCf_(dPkcEalCsqVJw*sGU}1*K#ntDEq+BmG^+~LZLc@ z@75lZhiQ9-V&O)tC6%e=mtpy5d*w5|=3WuLZz1hR`!`+H*7EKbe;l6XeW~Y_cp#~M zV7qDYnKam@2LvR`2)2u~=_*y(<_Qwi)_`NaCJvUcT9zmX@K>lPcEW~*o?r?FI%w_$GcM_&D9V;Rg?SWH^wKuo`}|%ruM%k&4s(!zP58n zD2NjW%?l2Kn%t!rSam%TV3L!AQ2smTRIrU*3&c8rzeY&G1@iy-lL5L_hA9nR`p!y`XP zZd@(_)Jqqwyt9g(f?2q62GQP~IZURq9VnAtA5=NF_C7;5D?|_DLY0OHM>(EGMnTi# zY(y7jOG1?W1l_i6NGQY$`xUlMY77gP*QLD5#lze?_gjsyA9<7F6hcbQi-)D`X=l2f z!2u6G-o~y)muL=uVtvh>rIWfTonVybi$&U|>Yk_ChYh@xb>l#TbBoDo>zLYAGqt-M zgtwK0)xtV%d2#E7q1;6k!>HQ0XYcE`8VMr})% z)_9YH@DFARs!(-L)z3ixC%+7{{UKxklzI3?h z#^cVisrbugD?6XFA@^ucpGRq)muT2?sw=kc>Pr?;5=r89qH+Dg>0f0QUmGROuWIIk z>93cT(EP#7Er+tdg;S^eMze3bFiK5@2^#5s6!x+6nh5}CVS>n3pK&!Jf_`Ov4v_Mt zjXZL`doSnRH3;Ai4P1_!M;_Bn@d-s_UfL`f;E}W^-fFyB{djHDgF`N!Q_D$u?GBwg z_Int?$;>CfA=TVPRQ)F1^n0p|a?QE!IPHjR+a7opy7Pek@Z=BwIj116eQO9RBB zA{~>G3^*;U--N_a?xBhrz_id9X07BK3bGWNc)DuFUoA}rAH0rKKD(g}-<~VHGy$ol zxApL7eHPiFb{RG1Co5oYIe7KWS_z+6q_>5T*Z@nch#1hUmy&Fs4xj0XZyl~x4)<## zpjZ{^J9qDQ?lnuk=U>RFU;}wBVx%)YUkoF>)eN-z!(-!k_&<8hn)3D%hxndxJohQm zJ@hN@dwh~-iLR|$g5itu=93+rklD@rUb2xklhoNKcxR@sFYN_yo^LgG0L+{L;}9ak zEHMVosBtO90{BwhCJjX^7I0o6ku*LmEI880eGHJxdoR4$Xejp(F5H4WukehNnFiyU za-kIS0uRbc#}iTmlhqPm6fR#9@`w9Tm$jO+eWI!Fo@%5oo$9xQ_RTBYaz|-MSbUFD z-ZrTH)z=x_J-P!MlcFSyY6fa{Zya4tal(CME6<0pc>}$r5Rh;u!P(t^bQ$DvpA8=D zX`dR?kW*ag_8K)Goc?f3MG9qcF401yuiZ7xa~zE#y8La?In{dy0nur~ z*E1Q(2mS)NwI0U)I4g8|A;8&m?|JcveoKucf62%nXvL zOi%Dj(Y-#*eT)jS8Qn&|oJNSN=TYza1nETWnn>Z;G_fY~aO4YT559e5 z&Ak2fBN48%9%grSftpR|&M-CF{c&1tZXi}axUHnP)O`g3~|>XGI%qfscJMwztCo5=p2>t zdHd+eEy`NxyDu)%H*=cUvC2MHEzCDRxK1iw^KAaDVLN;9S3zP-TvQ!&JvKbx$D$j_ zMpEP-T_5As`Vw!nFM31-NQ_YubE8iy14R2Cwl%OeUd3MsexT`nfaXc9ew;n_=F4_fw@_pUoW0LjzyC`NW4D$RT@P8Qe8C1 z`t{=M=Yt|Onm3)7VFrsa??0d5qwag6uQFowX$IVPW8*9d8jII!JnY!0czjp`QP*X@ zW=l8X7$7-r%Jt%H3a8^AF$AnZw_QrdUWx0 z9X%xV7<-A%In{Gh&J&jjZIF5y98my(<(0-{9sFzk9z#PG$x!7uKX_eey) zmzf^|y^|_St$f|#4wa#|Y?T;`9Ll|@0(2HCky!*!{rUsak)V>j2N)k{h&x9+$ml0@ zv3kq>Hcq&E4yx=t1c}noP4C^|7xipaeTr5n$>gOmRs4DAu#Dh&8;8Qpq2;G7>*!^? z^cD`4QYI#3;YBAAXC;$ILpBgW%H^?$Nv~mhA;bD`U3f`TMSWv)O|v6pqq@5KX37o! zqiRaga@++6P}PM3Zmau# z)vBUi0UBPSjSS!&)EzxOeB{q+@>C%S}vf9 zc-hJ?M!v_ou`x~^YG99P$|sv#9a%g_Tk~iO2!sULv@B3>tCKiq5|KY=E~xJ-U-A_5 z&$`%zKMHHik=wN*KWAR6f6K@@!6;kY(R68H1~07V5mwX{t5kmSsa{``>y8o6LXD{B zX+C`B6gs0hM070y%Rt~+HSV$yW9xd&WNU9h!ZBoR1IDvmL+1WMseB9~B&BBB6qmQt z&oA>}l~X;ytaG9~64T$hV$o7b`&j>M|Ewz)x0SXj9YZ~T zduQv{R3Ulp4((MDFvEF`h>`#|MF@=WULNX?iWTSS2e0O&EI{CTAQQD`WA zoWh@sJZxXRz7-n42=lA+w73-Vjm{MegCk1~5)8cDJPdST2*}A2`SY9v!vmeWT*3@e zk~hSitgLK7V9_7d2QFlp!ki8%3W1A$?~-6}fk8yS)hR?F=8j;vC;}w?(B^YcF2&gC!=;r3e?Z(gT1hWS6iin5+zx6WwY047x7a!CG=J1Wn5(ET0 zfUim-a3C)?FYpKNm52YUQ&+sdy5!{K^p~NlF?=85H*2FS z0U&w#f!07vbA&mUt1Z}#k%8{dIQ|lFJ_gAj(Y{~5MQbR{jQ9wXd zfJggJAO3^m?}8u?kOLSl$t&~^_CMRdGh2fpV5ELzl+=-PrT%TD0tL8{i$Q?t0mRmw z?q~DW6q-U|PIh1r0w}=CE8xh>B;G%f!x6|)z!A0}x^K}( zm|Hl2xj;@X5Cn4PFnn(WfgO;u_PtF&Nbt{ACyOhKKiYX1fRf^WE+WzIlm6qszifSL z{{7ElS5-SW!UGwct6e-};0S*ujDPXC*^mCtYu9Wt=yM@gkhSL;Vl%4?@v(A_7++tH zp7(j_;>C#}RzRz-swu>?5s}SSVC_b`Q5;9y*x=cjThT5{*iQlx#xlaF2<`~T!ai)9 zvEO>z`0xVO6%!!uPP3QW$)laDWB4o!t^0m|&-*|_OraG;LJ@1$Hdt}G$N+DJ@*ayP z#OMVf$f2+f$uEf4j>aB;M9V@8e4u<$2}i+h}(CW;oFJj}@`uz!skAo7r&x32%@8)NpIog*!g z@K<%@S_MzeOpfl9+FaCmR1|PThaY5>6=^8=EoIagq`x{z+DpvWWyPyc6T|W2Z}x(u z-of}pDMLhp8>P!YKawJ`AgO>ESvS?oU0>i=*kXpGLfS%%<4>$jI=x)V`Kp>v8B8!{Rqe z>XE1UI-KcV;`gmNi2IwcX;)4Lnv)L4J8U1cY_<2(icY?0NLv2NZIP7fr^}yQWkeu& z-WJM}RUEjYn5vWN_4zaE!w3!Z>uoTsL9@UWllz&n_XmPomy$}2-|b6}jq-~dy2Nga z2!vNkvvu*ZL3T&O6YT2~tV?&roaB*mw)pWE7)vt_{+gkTnP?8&7JiIl<&s!?RYVe$X{WH-G zvOk1e@V2Ilv;9bGYx~I-!hI_bC54zaH8v8TXHqTC#xP;c-#g8Bu>1O3v2HJqeRkB9q0^{8^4@W4$}Fop7$x0ki!@vJ?tYMRpra)GqJqv& zL@TY6WuKJjQ!f~V!?Jc`8I)Uiq~gzip1)d&W^XlfzA*X1eEeXGYz>cW?ly0g*1)N1 z5I$iHpZeRV+&L$nc!Nc#b&Dd#EnE`+ZRX(a_O&%{C)rN7yQC2GXV6P>FXOQXvXcH= z+XltcY2C-wmbwHu&Fq;%)F4@QcMqS#)KlA&id2;kbCibp_l&2ZdMwf7$=0xQlWdzF z?9q?Is>y*6yj&IKf$dTh^~go+_#5goTd;D!nGKTyE`zrsnNC3Sy7A(CCiIxKuV*4` zD=%+`5RI{_?JFCyvU?eN4!rmu-lEtOG={Ys79pIp&R`djI*}!xr=bxq{$y^%Ivqi>X2Nbm_}H{}JgH zo~Q-2E`-n-d}PGujzfq2H0@WjQQ4=Kj4W(5;0J7p2ZH-2q=DD2!C1R%?NV9HbunfYG&rIr;IEX z-{qf+sC>0yPSqJ&p4+M;0AgeARYPh7o$p>!IBq+NdE$sm7XTzOKOzIA)Q3% zjA&kJ`3>`Gt;V2gy67j;;aK;(0}%dHb!sZ_`Q!F9MOxD~7COuthyByV&9*rr1}0{M zZ)SfsnQtuJH1ePGdSTCa-JfB7 z617sfOZ(tn$yN9&4{bhNLtduwM<(0fzruf{;mC;nMHI*9 zgv_YEr*}Y*3k>#W9T4r?I=zbJ)orCK`hCD?k-h&MxUHkPH5d+*wM96ZL*f4=YX3G% zS5bqSL%a-JdT6?Mzr0lKSXuLToTYCc-r#v@pY=x zQS3I%Sg)+-dJNxS9#PdS97oFyZUfgJu$-!oj72XjBER0ISZ|8yf)nED(R)9yDn>lJ zNAb#69CvT1iAG;)I@Wz_KAHT!llJ;a&X=<0Lrcw>&NmK6B&9WSY8XO&FNCgB#qu$M zVakVn7l&U`!!km9bradPeO}%@l084Z5oda}|8T$bAefQ+PXT!YPD2F+l?d?r8+XWH z|3W|R-~ChSpLgnhis&F8e|N*~hn63+{AG7d<~=^ncY7 z_b*z0!s`F3rT1U7{Dj%>T7Jyw)d+rI{b~(g>o6f^;`1pfm!~-6f!O z{ublb`+mHi_xbmo{p|UiGv}(Ax#oMWnQPRQuA-6x0GI%PUSRaS5SZ{oAOHZk{3ELX zc2GMA!ovXqc5r|}&A|w$Jsji;w*cCMouSS^dj|;I!rt7)4gyC2ogJJYU<+q!2n3=2 zH<@=+F{g&E0sxo)Ye<>e)-HBta4;0+3_|?81vTKHu5L&kzoqda>sFzs| z&1*EBYp1s;6kKYJ3%4joOb%{rhMuikNO`#nkCIoM=Wf2;&vjjBs8kl^%uM&=MHj40 zKr80T*)QK1s|;Olz`aHzKq;K>AlcuqJ?>v}kd4FKV|QX@cx&;N(}jbRcWq^)q)3tE z!bTiuA#ean^1wg0*2x9@rlKlT$ zeTLn%=Q%kwv32mpb$m)PsJ z(*heTM>MMas03>Kg$v;}%TZ$%OMC|o-j7(4tW!+?Un5>d;YBl(TXZfx^kwjr#IIM3 za#}J}-G;-hsl9kE%o42+d~`*ZU{XW2VGaX#gCc#sTm+*=&B(8so`>y#2M zeO6UQd)qrl=h59;npMN@G^RVf$E>Ud6G_8jqoR^~-V6O&7!e2FI}fjY5d-w^0`d}e zYkLN+TH|KqU>xF$H$7Kf+6BxDn-pUd9Y4*)pi7}KEq~-Yob2gOdvIcSvfrye1yo+W zVqAl9=m^#I2oF%m@4Fhg>>Wol^Jda|`ZGsEI=#3 zq{~^WD5ol0AY&T%9SSfpF~7mf<(r=c7kT3hK_H-TD`%j)9c)ow$$3nHuwkn(6VZ;c zF(U_20GAkh%`*Xm>r*GtUj?wmduFrQ6<)}&(?sCi5_`f<1#@n}Aa z4C^ypoVG|BD&37m;cF;rg~FlsqN3mr@vQPk^L++vWFi&yqcBy$m#jCRxVmv)8>t=G z)gd?v1a>~WJ0(FH64o0yhZD{#3a(dO?oyR7d{h4jBWG1Vdk}whnD+h#8k66cwlv{* zPPrd-o?uy-f|UyRD@=f6YicJQgMBrvSr3p5wtf+6bA9c0M}okb(FjO-3e_722ZDZN zvX-Yqv#amd=XLM;In|0&Lf&zvpR2DidE<@PB9&?awHYf&Gt0D6t*A)DyNbxRqUs$! zo4d>iOucL&x*GeB{OH{)!i-}Kg>0VQt@p3w!oxAsWM!UJ-BwIc^}nFPeQdr^=<$A1 z8VaH@H_6VNj^k*Vh}GD`@_8>}8uCm&Hb{MtFC?R1LO>&4(KEO4%4zY-6)M*it(HWz zrB$7)YI}A`RhlDEqP)R(c2AnL29>n2gC-V)K`NZ>eban9P>VmmS>zc&um`4 zCVNk8=6lBG*$5C@o~19OxUq#<1;gIRt@b}5!7orD%8+VXk|ot0pYp(adPa*fE8yT3 zlB(DA0xm+ch5AlTG)VG}g;bS*0U=9PO_DvG!^=ia;!IiXs~3FiT77;vDVt7IQqdZ$ z@6|0bDcjfvH}}-M>fO%JPi>mmhh=Ssvi;Q@o$=lc-wr0C7iH*IHV2%^`-VTJscV`< zceeN#0)xV2!#Wj(-m=AEmqf!EN{MH#SJTo&DkcREo&#ZzYZH=%aGp+c4F|hrSE~(c zlfRxObZ_{gbDK4QR#IaXUR_E2e$_2kN|vlP;A|%8A~0jOa|`_uMjTyn&~yn?mE4?~ zKs}w6AqgZrf0lmK&wG~S!AIHiiT4C22~o8S&qn;pWxY|G+aD48QaGx~FCD^tr(R61k_+@0sB5^|B$HsBl`3_Fj~KW@b9;|( z`!X7#%uDQs2XbVcXkg0ZT-Z+)%qz?WbAx!V+Bcw|J>#{>RX%;BYTlIuKbrGD#aJe3 z5jV{dv&*O89^anSJWsMP+%Z_+bsJv0r=&X{<+JnX5W6+)SYq4CxS)}@f!UifF44C1 zY!2Yw+7Y0^*$CL1dQ+vuB$GG@D|xcKucW70@V3Cy`7 z()z2tJ3DV3=vC<&8@1pDg`~=uoa9)vFF;)h1M{r+1bl&v683KZkk6NK$)A7&2MHYJ z_He{y(1FB_&2%rV>!MGZE}p8*e$lq;hP_$6CPSla+BU*c6co^cUwx3H39LRD`IJ*T z#Q2J5Lap&p&Dpy)S3onI-F7)5&rG< zZ|7cqQe3^i^Yo725navw&u^vaPWc1%61|CRzgC*cw6D>973a+1l^|4hbRjIo91Pee zkVVwqn7y)|sDHH{YjrypAHzn`}cZEg7UAz&E~Jt)0-DmqrY@bvu5tM@3HcOz!uk@f>s z_HA*@W|o|~FRG*0JLEG*Umum0#OpsC$i3>c_zavPo|&ScF8{8sK&NYnAqAK3g-K<7 zLiu_{WoFWuL_EiOGD(JRLcOVr&L|GcTY2RJifx}|fykF9LqfFc36Z)gogJ!V2_TP> zFSeE+<7Yz%v-8IQ`VhC#NuRy7@)v%M8hbNr7cbI|CJVGoNIx^yJ zHowt1(qJw@%OXb|+2h@cE5at1KQo-KTRosRGY2;qRlPHRp)Urcgwk_*5BR9bFg&Lr zQ+9Kix3%7k{_<&9qiO`9H=SjzhTd$X5uYyJn?$4)u2e~l?w4NjCT9JuS1(o+Y2t06 zd}hYcQb~D^@ny{i%P>8E$wax7q2NBzMsp~<8Cv%Nw%}?`|+zE_C+gk);W(Wn>N*(F1rO* z%@5M#KvV8rZqeEpS&I0qLLP**BDQg8Tr?NOu$Kpn1r8-$CHVYHN08J>95az@6^69SE}D{;4LPVgkWUX=sC z!vLUE3uiJqX*+c}O2=9k$A8(X!tPv!J4wOHDfezE$E%acO{9it^`V9;59?~y$=vfH ze~!J!&q2ZD_`PWh&N`HLEn;GTtMK;-U zo_DbIsdM6d>F(#f8V|49S9P0P8}&Xq#;ndxaUF_0Ngt9Ole=`6AU95$ynO7Ga!!}h zRnB%eUt-^7=od2sLwT#7u;s_^pa~qE@SeL~bU)xN(D#71g+|9wjs|ct=VQF!l*Oyz z#M|pKRH@St?82|QE93ffnKh<+pR0?m#-bK0v{n5H!V0fDQr=M6vv8@|=CTQASNrjf;Q5j$nu0=W? zS3xkp?wp~VG6Edk(1CkqCPy8~PrT30o z2d;I$fgZK>0(IPgd8|4)+6^R7`%0L|Iw_Dpp;@@rC|3e!;}F3M zS&nPVNnHCPlu)TJcdhN#EsLBMSDL8ySF3w!6y2Nbnp!W)6r8l#Z=ZE`uzaN^@9o>_ zJs%;ib)xdr2Ib5@=YFyU@EXMo9TX3GSu+&4J_6XQ> zib8bn3n8=5lUHL`%O4Xut@QM#5Df}ri*v;eCpm8Fw^MVjA>xGToQueoW~8r%eYmD< z>ASAd{EWV*5RA=YwN}wjBc{$=f^LCl+(*TkqD5z0g!jqc6N6vRoh@~SD{a3=T!f*D z){j<+sYq(?%5HxsjJNHqht;xVZLn-=(d@GzlzSpleJM?WW*uCg{*dG8wKS56Zs@X> zRPySF$ZoOJg*Vj8Q#kLAj?v~O`~rlgfl&h$czioZ$012h&q-4pekTCZ5gish#qMkw zOv*TUSH)qc{mg2<2*>eJHhH{wADVf%w*q3;{xhR$(^+8D(iR#G1#qTPyZX+-(RNT` z=;=(i<-9)MClzxOoa)(y2`@Oi3LdYfn%c(R-EaXpcAskjG(fka;5#rCF*H2iM$;O4 zkq7NK)2aQRw(d=jrdXID1T5Ccrm!6!1fp#gkoK3hpWZ0C7gj!ZlKD6`M{D_h6+Su7 zxT|G*6GQJggSNH0z)4-6AERlN|Cwr&IzW`?ZElX>?RVxqG-`oKb~vR2*HJ=H2K6o2 zb#J4_5ZHzdS8UrHZ|ppx(;g3AGq4!Hk4cGd5#AS!jltgSMKmDuu?mNho65~jyy!LN z)msZ+*c(*!M*?pai>7NTp2^B?s}6)=7)_J#iFKuP`M%D1g>N)iLh!t`+`GL-Dgt`wI2^$3w`@*&d+d07F90Jh9dmg z%=tI=JqmrzPshsd=6x)B0A4s7pQkg#@W0v?`AD`fDJBJ!ISL@W9{0tF02JOfW~EB=*&C@Z6U#W zL*+qSjMDEM2=gXW=N8Yyx3%PKiTfN+j2NxmGlv)pHUv;pZ&VOfyfjl%5ogpBGELu3 z3URF`zw-9V#_qybwR0B~neB2?N5krD`?_&5mSDw~8PBy+$4JhDv~AME`_RU2Wsvw7 znRe#fftE@>Vppg=i@Q?l!!j2VL*c_MgOcCu$#bg&&SkBm08P1XK=u%QIw6>x0QwOf z3n)U2M$Z=pER~}jB$Afx#=9O)&SFr(ob5um65B6>$@zW*cT8YS@%{2R^%P7;;?wwY z!aS&hJi_H<4VvAv$y*@2UBQ20-H)Fcs&?Hla?mkLq<}9g1Nzzh+l$n ze!Egh-r89{_x9SMx4gAm8|LQunta&tn_dk?sA*U@c@!2;|8bBtI7=>6f{|ICoCu3Y z^o8&_p>enuy^p+9I6Np-y~4-z0J)l(RoxJ19m>h(G0_&5WXt81t3bsnNPTLK(R4R7 z8#nmWa?TDztakrNepAG&p|*Y4nZp|ct2qbVEA3igINlX7-ngEI7R|Fv+S|FSo>=_v zZA{;N1-^D8m!sRXT(mt=M%^J2vvM?*74%fQa>ZNT-<62-#%KRH>Ph-2jEu>fGehX| zX{NLOE(|KG6+)8+0cazyVp*wMq;@33DTytuio?Y%=_gU5Lq~;Unw{1a$MzrKUKg-( zcNe%8qjU4r9u*g(h{G3SmTlYkX@$MH%ziRPjT+K7vZEP*x&=YBywmO^g482$|F-_H zq;i|7;r-djDcmdyI3N?V-GuRyl=b)+HoV`D~6&;MvSd&Ut0A+O_M~``cmD_wj z$Y9Yzwx!bU(7MP!dRl*>;e3M1bg&EDtTQiOps#Q}A~rDljQwV@kA+6rD;(1BQLSXf zsT)(OfHFTctn%3pa)V^jmoq?ze45tetwONJYt`d!Nw1h zD4p8`$Kbht*a%s61{1IGVAO=K@$l!ll7XoEcNFt8IvyTV>M4v~RW1z4>7i%0u0zL) z%G}v5=`1e5qF^!NGSKypxEJaUo|C%@(XO&ii^LwL4aUJLxSsM!>g}3yl)9bhYmN{0 z$qz_org(b|(oGJF(z_%-ePCKD)yAvHj$_-gOpTGBvf-cqPaQ zK;M00$|x^dybZ{8aM~9PhnctCFFO8j!bdCaIH3e=W!{#pcBguEz zIC-cG=*!QRFprB2z2{zMcq{fOE*>7|8$i?V8+)&oM0XL8pRDaPDeM4EoJ2`peT>&$ z6GnG_K+l=keba6gVlFOnoVK;%9i|qvv!+1)YfJkJfE64`i>jx8|>LvmWLc zFVGZd8x*1L*HuY10LFY`q+U#y-P<^R^xSt}E764s1@l8qgQ<&)pF!sscS+HBP^2l? ze~8_y{iCm5xOBfZ2GFS79OVH(PbY~6O>1|Rn1u7S;Y|kt_pWSyr%611##1abZmJl%U^y2Yv;tawnk+0@8h?~~)1LU1+bCTX=26`Orkxdso8w%*1o5eeJS%U_5<{>p`BqB!?7t|~QKA!Mc79Z^O=MH)803u^<$4%_ZDrpeD7iXx z;2UtS(iwUqClWQnD|1Pd;=vUN0b)tpOxpDNd0lRY=-y<|tl#3j!s*S%8}%q;zDkMd`luqsB7~m2=J)z@N50B z;Xf$;&S>so4ud#L@bLVD{73zFVk-z7f|QT!B{k$&seSWQ5I+#<4E#(waHu=oujR#B{B}G{GA>Y<1(&eUUrK(-`!4@4r*}pmTj7j=n$vyjeFWGH2H`Td zcY!0s8Tc5!SDHg$$Wi-VCv=PVN3FftrN*E2+zcQIu^&z(@_o>MKK-TZTlK&1hF+#i zoe>_$&bge$llpcssX~P3zKtlwFW1-;0Hb!binsLlDnurg+;qyvechS}O`@V;@v!4{ zdl+{B0Lw>s&(rXgVOwvPSOz75xu?rntcF!k2vZu@gadIEz9mr%mJ-G1lauK>r325S z0WvjW;#rTbY=*bW9-5mspWxr*69!RG!0AWQ7!?=NvT!@AA1;Ztcd>V;u2ehf+>Io*G zBy=n{G)}@<$G>7xyrS)~rT(+j!V%!!s-I%o<9Q zlH<7}NNMP)hKQE+P5J;7$Nt;f@`-hFIQ6U2ooYS-&OmJyCd%mzI+20n+yWi>(3sc8 z<$c=k96_WjX(tY_**^E{ft6Noj9taf`KJAiUamDqHAuQ7R(xQkrt%JC*k~1c=H#J6 z(_^slxYu(W=b(++<)Pk&h=V&PJ0oJ)$a_vqMmQ0)PrSUi>BE*c>`sqK-g9GW42@8+ zrAV%hB@-51>D)HUSZwP@}adA+}Gj^`wdHfg;3?56D^FX!AmrbF~%gFGk$hMkU(>H2ew4h2+E#~C4Hs>>M zF;y#8vYrMk+U2F*#x%EEFQGb3_H8}1BBKd)X-gYGv$V`WUA~NRBO-ybw-MYg3PP3@ zGPN_kHy2S1qSaB+-#TISnT9MID?gV}?hR{RU<03otp?=hS-DrN-JEr{rh*1&#)2(T zi9kujQfp_T2kQMnJ&}5~QkAfTK3IPt`B_=h z^I3fz`<<6Ifi^EP*>jr20m4IC@=14+TG&=vnL={VeI|w0iuK3Cu<1=_ruV9`gCcGP zOtwDITB-9i(0$efj|sS6F*jHOwdypxPHA{o@;Ya{$&Ref=r*)H&Y%xr(T*WWx|@dbMhKyv+*%cJhh!xC=deReLT$EW2PCs zc3l?dv~&K>E6?KG2#iJHH*>4y57!k0?{>rSjT121#3z$g`4d%Riq1Uzgw)hMA{HH4 zR3{nMl}d-FTI0-kW%FuzG{PReALE+|Y=!|UMPWRTw1^3$DwsYcVrt)%@Xx(FEF^H@ zYKw2E{86B*U0r`?8Qtnwt{;l=sjDWmwG{NWV11hBBQ1A>ikZ!ejDu8sdL|Y~Mliib zKMtRTniG9ii7+8*VBlmbICd)vIDttzd(L-Qlb;vtjWRu3YE*qY;j7Ds zd0I()a?>P_OTR_<`4%0xX1E}a2{nFoW7S!+hmVq&Eng#T zC80uH`>+Dhn(%RW!j!lR2qym4POXC z5||`0Tcoe77>(T%isN?qECOog6)Zov2HTeb+Kr$1vwCqWzAA-l$@a56_x7E@lI_$|vgqj#aJyEpJ(GquV2Jmn z%YvZtfvM-`5G_z}s=CnLTO1Q5c5W7eCJAo|0?;x=X$r#RsO-q6GR5hXrEjAA zNvS@GUdQ>1v9Bj;wbK_*?K}OxW#k@`trjVx0|0f`|64}xGL>%!hC?kO&Iq8Bg=Keq zH#~@&FnCKW1ky7i1wun}y4g`}^R!9efgT#@OA7H+Lv*~Z#`D)}PY34<^!nkX6Q-hwxkaL*i;rvCCy&Xm#!s zUWVai2i70akf)e}kiquf4@Dt+?5{$Fyt{1=w}e{#h>o0jK?r+$m;)FNfr0*=kNOXM z$dJa$3F7Q*26pmNA5=E#WUd-w!0cX4w1kzT#}%?~bZy*!3>S^nLi%hO%| zY=|Yy9vOW8H0I|jbz9KWQoYT(yo%WO z=&;rZZ*sCstn}6_GTQZnW1ajbr6I%CUUcy2h}XOH=^eTY!76nkAoL-IZ(U0x3$BV`i)bL<^5O*;x8+ z*Z~fyzSK=xEwANz3}7=4s%Yd7U(F4B3#r{>IZ+$@6gM}I{Jf3d#1++rBqh?Lb}gs64_kjg7b3y@RRGOp}*t*} zGyVNr@ssnXoxd~y0GAWzSAjDBC%XI&lYbxHUwi<-B~bn1Q8Ya{k1= zOW^!f#{VGcUtsjVHS-dUeidGp|7_+j5dHT?Uc%_FV*Ljx{{gE1F6{3nz4(K$Um*Lx zN*VuylwY9x?^6Em1w_ap_~$p~cM$$x dict: """Inspects all NXdata instances that could serve as default plots and picks one.""" # find candidates for interesting default plots with some priority # priority ipf map > roi overview > spectra > complex image > real image + # TODO: some of the here used idx_head, idx_tail string mangling could be + # made likely better with using a regex candidates: Dict = {} - for votes in [1, 2, 3, 4, 5]: + priorities = [1, 2, 3, 4, 5] + for votes in priorities: candidates[votes] = [] dtyp_vote = [ @@ -67,7 +70,6 @@ def priority_select(self, template: dict, entry_id: int = 1) -> dict: idx_head = key.find(head) tail = f"]/{tpl[1]}_{dimensionality}" idx_tail = key.find(tail) - # TODO: better use a regex if idx_head is not None and idx_tail is not None: if 0 < idx_head < idx_tail: keyword = f"{key[0:idx_tail + len(tail)]}" @@ -94,16 +96,16 @@ def priority_select(self, template: dict, entry_id: int = 1) -> dict: if keyword not in candidates[4]: candidates[4].append(keyword) - # one could think about more fine-grained priority voting, e.g. based on + # TODO:one could think about more fine-grained priority voting, e.g. based on # image descriptors or shape of the data behind a key in template + # but this will likely escalate into a discussion about personal preferences + # and particularity details - for votes in [1, 2, 3, 4, 5]: - print(f"NXdata instances with priority {votes}:") - for entry in candidates[votes]: - print(entry) + for votes in priorities: + print(f"{len(candidates[votes])} NXdata instances with priority {votes}") has_default_plot = False - for votes in [5, 4, 3, 2, 1]: + for votes in priorities[::-1]: if len(candidates[votes]) > 0: self.decorate_path_to_default_plot(template, candidates[votes][0]) print( diff --git a/pyproject.toml b/pyproject.toml index 1986124..4c38daf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,9 @@ dev = [ "pytest", "types-pyyaml", "pip-tools", + "jupyter", + "jupyterlab", + "jupyterlab-h5web" ] [project.entry-points."pynxtools.reader"] From 1e6a66a9fe61a9386ffad56cf8c512eb6ae2f694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20K=C3=BChbach?= Date: Fri, 10 May 2024 12:13:41 +0200 Subject: [PATCH 7/7] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 89ecdcc..f676f1a 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ git clone https://github.com/FAIRmat-NFDI/pynxtools-em.git --branch main --recur cd pynxtools_em python -m pip install --upgrade pip python -m pip install -e . -python -m pip install -e ".[dev]" +python -m pip install -e ".[dev,docs]" ```