updates fileEMDVelox's metadata extraction

ncempy/io/__init__.py

@@ -1,12 +1,12 @@
+from pathlib import Path
+
 from . import dm
 from . import ser
 from . import emd
 from . import mrc
 from . import emdVelox
 from . import smv
-
-from pathlib import Path
-
+from . import dectris
 
 def read(filename, dsetNum=0):
     """
@@ -40,11 +40,27 @@ def read(filename, dsetNum=0):
         out = ser.serReader(filename)
     elif suffix in ('.dm3', '.dm4'):
         out = dm.dmReader(filename)
-    elif suffix in ('.emd', '.h5', '.hdf5'):
+    elif suffix in ('.h5', '.hdf5'):
+        try:
+            # Try Berkeley EMD
+            out = emd.emdReader(filename, dsetNum)
+        except emd.NoEmdDataSets:
+            # Try Dectris Arina
+            out = dectris.dectrisReader(filename)
+        except:
+            print('ncempy.read: Unknown H5 file.')
+            raise
+    elif suffix == '.emd':
         try:
+            # Try Berkeley EMD
             out = emd.emdReader(filename, dsetNum)
         except emd.NoEmdDataSets:
-            out = emdVelox.emdVeloxReader(filename, dsetNum)
+            # Try Velox EMD
+            try:
+                out = emdVelox.emdVeloxReader(filename, dsetNum)
+            except KeyError:
+                print('ncempy.read: Unknown EMD file.')
+                raise
     elif suffix in ('.mrc', '.rec', '.st', '.ali'):
         out = mrc.mrcReader(filename)
     elif suffix in ('.smv', '.img'):

ncempy/io/dectris.py

@@ -0,0 +1,198 @@
+"""
+This module provides an interface to Dectris Arina data sets
+"""
+
+from pathlib import Path
+import h5py
+import numpy as np
+import hdf5plugin
+
+class fileDECTRIS:
+    """ Class to represent Dectris Arina data sets
+
+        Attributes
+        ----------
+        raw_shape : list
+            The shape of the raw data. This is three-dimensional: [num_frames, frameY, frameX].
+        data_shape : list
+            The four-dimensional shape of the dataset. By default, the 
+            scanned region is square.
+        file_hdl : h5py.File
+            The h5py file handle which provides direct access to the underlying hdf5 file structure.
+        data_type : numpy.dtype
+            The data type of the values in the data set.
+        """
+    def __init__(self, filename, bad_pixels=None, verbose=False):
+        """ Initialize a data set by opening the master file and determining the file size
+
+            Parameters
+            ----------
+            filename : str or pathlib.Path or file object
+                The HDF5 master file to open.
+            verbose : bool, default False
+                If True, prints out debugging information
+        """
+
+        self._verbose = verbose
+        self.raw_shape = [0, 0, 0] # shape of data on disk
+        self.data_shape = [0, 0, 0, 0] # the shape of the final 4D dataset
+        self.file_hdl = None
+        self.data_dtype = None
+        self.bad_pixel_value = bad_pixels
+
+        # Pixels to remove automatically
+        # self.bad_pixels = ((49, 75), (93,118), (95,119), (108, 57)) # NCEM bad pixels
+
+        if hasattr(filename, 'read'):
+            try:
+                self.file_path = Path(filename.name)
+                self.file_name = self.file_path.name
+            except AttributeError:
+                self.file_path = None
+                self.file_name = None
+        else:
+            # check filename type, change to pathlib.Path
+            if isinstance(filename, str):
+                filename = Path(filename)
+            elif isinstance(filename, Path):
+                pass
+            else:
+                raise TypeError('Filename is supposed to be a string or pathlib.Path or file object')
+            self.file_path = Path(filename)
+            self.file_name = self.file_path.name
+
+        # Try opening the file
+        try:
+            self.file_hdl = h5py.File(filename, 'r')
+            assert self.file_hdl['/entry/data']
+        except:
+            print('Error opening file: "{}"'.format(filename))
+            raise
+
+        # if this is a HDF5 file
+        if self.file_hdl:
+            # Find the initial shape of the data set
+            for v in self.file_hdl['/entry/data'].values():
+                self.raw_shape[0] = self.raw_shape[0] + v.shape[0]
+                self.raw_shape[1] = v.shape[1]
+                self.raw_shape[2] = v.shape[2]
+                self.data_dtype = v.dtype
+
+    def __del__(self):
+        """ Destructor for EMD file object.
+
+        """
+        # close the file
+        # if(not self.file_hdl.closed):
+        self.file_hdl.close()
+
+    def __enter__(self):
+        """Implement python's with statement for context managers.
+
+        """
+        return self
+
+    def __exit__(self, exception_type, exception_value, traceback):
+        """Implement python's with statement fr context managers.
+        and close the file via __del__()
+        """
+        self.__del__()
+        return None
+
+    def getDataset(self, remove_bad_pixels=False, assume_shape=None):
+        """ Read the data from the HDF5 files
+
+        Parameters
+        ----------
+        remove_bad_pixels : bool, default False
+            If True, _remove_bad_pixels function is called after the data is loaded.
+        assume_shape : tuple, optional
+            If this is set, then this tuple is used as the scanning shape overriding 
+            the assumption of a square real space scanning grid
+        """
+        # Pre allocate space
+        data = np.zeros(self.raw_shape, dtype=self.data_dtype)
+        # Read in the data in all linked files
+        ii = 0
+        for v in self.file_hdl['/entry/data'].values():
+            data[ii:ii+v.shape[0]] = v[:]
+            ii += v.shape[0]
+
+        if assume_shape:
+            self.data_shape = (assume_shape[0], assume_shape[1],
+                               data.shape[1], data.shape[2])
+        else:
+            # Reshape assuming square
+            shape_square = int((data.shape[0])**0.5)
+            assert data.shape[0] == shape_square**2
+            self.data_shape = (shape_square, shape_square,
+                               data.shape[1], data.shape[2])
+        data = data.reshape(self.data_shape)
+        if remove_bad_pixels:
+            self._remove_bad_pixels()
+
+        data_out = {}
+        data_out['data'] = data
+        return data_out
+
+    def getMetadata(self):
+        """ The dectris Arina files sometimes output an extra file with 
+        metadata in it. This checks for that file and reads the meta data
+        if if exists. The units are assumed to be nanometers.
+
+        Returns
+        -------
+        : dict
+            Meta data as a dictionary
+
+        """
+
+        filename_parts = self.file_path.stem.split('_')
+        metadata_file_path = self.file_path.parent / Path('_'.join(filename_parts[0:-1])).with_suffix('.h5')
+        if metadata_file_path.exists():
+            try:
+                metadata = {}
+                with h5py.File(metadata_file_path, 'r') as f0:
+                    for k,v in f0["STEM Metadata"].attrs.items():
+                        metadata[k] = v
+
+                pixel_size0 = metadata["Pixel Size"] # convert to ncempy standard
+                metadata['pixelSize'] = (pixel_size0, pixel_size0)
+                metadata['pixelUnit'] = ('n_m', 'n_m')
+
+                return metadata
+            except:
+                raise
+
+    def remove_bad_pixels(self, data, value=0, bad_pixels=None):
+        """ Some pixels are known to be very high or very low. This function will replace the 
+        pixel values.
+
+        Parameters
+        ----------
+        data : numpy.ndarray
+            The 4D-STEM data set
+        value : int or float
+            The value to replace the bad pixels by.
+        bad_pixels : numpy.ndarray
+            A m by 2 ndarray where m is the number of bad pixels and the locations
+            are specified in order for frame axis 2 and 3.
+
+        """        
+        if bad_pixels:
+            self.bad_pixels = bad_pixels
+        for bad in self.bad_pixels:
+            data[:, :, bad[0], bad[1]] = value
+
+def dectrisReader(file_name):
+    if isinstance(file_name, str):
+        file_name = Path(file_name)
+
+    with fileDECTRIS(file_name) as f1:  # open the file and init the class
+        im1 = f1.getDataset()  # read in the dataset
+        md = f1.getMetadata()
+        if md:
+            extra_metadata = {'pixelSize': md['pixelSize'], 'pixelUnit':md['pixelUnit'], 'filename': f1.file_name}
+            im1.update(extra_metadata)
+    return im1
+

ncempy/io/dm.py

@@ -386,19 +386,29 @@ def parseHeader(self):
         else:  # this file only contains tags (such as a GTG file)
             self.thumbnail = False
 
-    def getMetadata(self, index):
+    def getMetadata(self, index, metadata_keys=None):
         """ Get the useful metadata in the file. This parses the allTags dictionary and retrieves only the useful
         information about hte experimental parameters. This is a (useful) subset of the information contains in the
         allTags attribute.
 
         Note: some DM files contain extra information called the Tecnai Microscope Info. This is added to the metadata
         dictionary as a string.
 
+        The "good" keys include:
+        ['Calibrations', 'Acquisition', 'DataBar', 'EELS', 'Meta Data', 'Microscope Info', '4Dcamera Parameters', 'Session Info']
+        To extract other metadata not found in the above list use the metadata_keys input to this function.
 
         Parameters
         ----------
         index : int
             The number of the dataset to get the metadata from.
+        metadata_keys : list or tuple
+            Extra keys in a list or tuple to extract from the DM tags as metadata.
+
+        Returns
+        -------
+        : dict
+            A subset of the DM tags returned as a dictionary with useful meatdata about the experiment.
         """
         # The first dataset is usually a thumbnail. Test for this and skip the thumbnail automatically
         # metadata indexing starts at 1 but the index keyword starts at 0
@@ -416,6 +426,11 @@ def getMetadata(self, index):
         # Most of the useful keys. Two other keys Tecnai.Microscope Info is treated specially below
         good_keys = ['Calibrations', 'Acquisition', 'DataBar', 'EELS', 'Meta Data', 'Microscope Info', '4Dcamera Parameters', 'Session Info']
 
+        # Add extra keys in case the user wants to extract other metadata
+        if metadata_keys:
+            assert isinstance(metadata_keys, (list, tuple))
+            good_keys.extend(metadata_keys)
+
         # Determine useful meta data
         prefix1 = '.ImageList.{}.ImageTags.'.format(index)
         prefix2 = '.ImageList.{}.ImageData.'.format(index)
@@ -891,7 +906,6 @@ def writeTags(self, new_folder_path_for_tags=None):
 
         # Change output path
         if new_folder_path_for_tags:
-            print('choosing different path')
             out_directory = Path(new_folder_path_for_tags)
         else:
             out_directory = self.file_path.parent

ncempy/io/emd.py

@@ -511,7 +511,7 @@ def put_comment(self, msg, timestamp=None):
 
         # create timestamp if missing
         if not timestamp:
-            timestamp = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S (UTC)')
+            timestamp = datetime.datetime.now(datetime.UTC).strftime('%Y-%m-%d %H:%M:%S (UTC)')
         else:
             # try to convert given timestamp to string
             try:
@@ -527,7 +527,38 @@ def put_comment(self, msg, timestamp=None):
         else:
             # create new entry
             self.comments.attrs[timestamp] = msg
+
+    def getMetadata(self, group):
+        """Get the useful metdata (experimental information) available in the file. These
+        are the attrs of the user, microscope, and sample groups. The metdata is returned
+        as a dictionary. 
 
+        Parameters
+        ----------
+        group: h5py._hl.group.Group or int
+            Reference to the HDF5 group to load. If int is used then the item corresponding to self.list_emds
+            is loaded
+
+        Returns
+        -------
+        : dict
+            A dictionary of meta data keys and values.
+        """
+        meta_data = {}
+        try:
+            meta_data.update(self.user.attrs)
+        except AttributeError:
+            pass
+        try:
+            meta_data.update(self.microscope.attrs)
+        except AttributeError:
+            pass
+        try:
+            meta_data.update(self.sample.attrs)
+        except AttributeError:
+            pass
+
+        return meta_data
 
 def defaultDims(data, pixel_size=None, pixel_unit=None):
     """ A helper function that can generate a properly setup dim tuple