Data structure

halodrops.cfg

@@ -1,2 +1,11 @@
 [MANDATORY]
 data_directory = ./example_data
+
+[helper.paths.Flight.__init__]
+path_structure = {platform}/Level_0/{flight}
+
+[helper.paths.Platform.__init__]
+path_structure = {platform}/Level_0
+
+[processor.Sonde.add_path_structure]
+path_structure = {platform}/Level_0/{flight}

src/halodrops/helper/paths.py

@@ -20,26 +20,33 @@ class Platform:
     """
 
     def __init__(
-        self, data_directory, platform_id, platform_directory_name=None
+        self,
+        data_directory,
+        platform_id,
+        platform_directory_name=None,
+        path_structure="{platform}/Level_0",
     ) -> None:
         self.platform_id = platform_id
         self.platform_directory_name = platform_directory_name
         self.data_directory = data_directory
+        self.path_structure = path_structure
         self.flight_ids = self.get_flight_ids()
 
     def get_flight_ids(self):
-        """Returns a list of flight IDs for the given platform directory"""
+        """Returns a list of flight IDs for the given platform and level directory"""
         if self.platform_directory_name is None:
             platform_dir = os.path.join(self.data_directory, self.platform_id)
         else:
             platform_dir = os.path.join(
                 self.data_directory, self.platform_directory_name
             )
-
         flight_ids = []
-        for flight_dir in os.listdir(platform_dir):
-            if os.path.isdir(os.path.join(platform_dir, flight_dir)):
-                flight_ids.append(flight_dir)
+
+        dir_with_flights = self.path_structure.format(platform=platform_dir)
+        print(dir_with_flights)
+        for flight_id in os.listdir(dir_with_flights):
+            if os.path.isdir(os.path.join(dir_with_flights, flight_id)):
+                flight_ids.append(flight_id)
         return flight_ids
 
 
@@ -52,7 +59,11 @@ class Flight:
     """
 
     def __init__(
-        self, data_directory, flight_id, platform_id, platform_directory_name=None
+        self,
+        data_directory,
+        flight_id,
+        platform_id,
+        path_structure="{platform}/Level_0/{flight}",
     ):
         """Creates an instance of Paths object for a given flight
 
@@ -78,26 +89,34 @@ def __init__(
         `l1dir`
             Path to Level-1 data directory
         """
+
+        self.path_structure = path_structure
+        self.data_directory = data_directory
+
         self.logger = logging.getLogger("halodrops.helper.paths.Paths")
-        if platform_directory_name is None:
-            platform_directory_name = platform_id
-        self.flight_idpath = os.path.join(
-            data_directory, platform_directory_name, flight_id
-        )
+
         self.flight_id = flight_id
         self.platform_id = platform_id
-        self.l1dir = os.path.join(self.flight_idpath, "Level_1")
-        self.l0dir = os.path.join(self.flight_idpath, "Level_0")
+        flight_dir = os.path.join(
+            self.data_directory,
+            self.path_structure.format(
+                platform=self.platform_id, flight=self.flight_id
+            ),
+        )
+        self.flight_idpath = flight_dir
+        self.l0_dir = flight_dir
+        self.l1_dir = flight_dir.replace("Level_0", "Level_1")
+        self.l2_dir = flight_dir.replace("Level_0", "Level_2")
 
         self.logger.info(
-            f"Created Path Instance: {self.flight_idpath=}; {self.flight_id=}; {self.l1dir=}"
+            f"Created Path Instance: {self.flight_idpath=}; {self.flight_id=}; {self.l1_dir=}"
         )
 
     def get_all_afiles(self):
         """Returns a list of paths to all A-files for the given directory
         and also sets it as attribute named 'afiles_list'
         """
-        a_files = glob.glob(os.path.join(self.l0dir, "A*"))
+        a_files = glob.glob(os.path.join(self.l0_dir, "A*"))
         self.afiles_list = a_files
         return a_files
 
@@ -111,7 +130,8 @@ def quicklooks_path(self):
         `str`
             Path to quicklooks directory
         """
-        quicklooks_path_str = os.path.join(self.flight_idpath, "Quicklooks")
+        quicklooks_path_str = self.l0_dir.replace("Level_0", "Quicklooks")
+
         if pp(quicklooks_path_str).exists():
             self.logger.info(f"Path exists: {quicklooks_path_str=}")
         else:
@@ -133,15 +153,12 @@ def populate_sonde_instances(self) -> Dict:
             launch_detect = rr.check_launch_detect_in_afile(a_file)
             sonde_id = rr.get_sonde_id(a_file)
             launch_time = rr.get_launch_time(a_file)
-
             Sondes[sonde_id] = Sonde(sonde_id, launch_time=launch_time)
             Sondes[sonde_id].add_launch_detect(launch_detect)
             Sondes[sonde_id].add_flight_id(self.flight_id)
             Sondes[sonde_id].add_platform_id(self.platform_id)
             Sondes[sonde_id].add_afile(a_file)
-            if launch_detect:
-                Sondes[sonde_id].add_postaspenfile()
-                Sondes[sonde_id].add_aspen_ds()
+            Sondes[sonde_id].add_level_dir()
 
         object.__setattr__(self, "Sondes", Sondes)
 

src/halodrops/pipeline.py

@@ -222,8 +222,8 @@ def create_and_populate_flight_object(
                 platform_objects[platform].data_directory,
                 flight_id,
                 platform,
-                platform_objects[platform].platform_directory_name,
             )
+
             output["sondes"].update(flight.populate_sonde_instances())
     return output["platforms"], output["sondes"]
 
@@ -373,11 +373,20 @@ def run_pipeline(pipeline: dict, config: configparser.ConfigParser):
         "apply": create_and_populate_flight_object,
         "output": ["platforms", "sondes"],
     },
-    "qc": {
+    "create_L1": {
         "intake": "sondes",
         "apply": iterate_Sonde_method_over_dict_of_Sondes_objects,
         "functions": [
             "filter_no_launch_detect",
+            "run_aspen",
+            "add_aspen_ds",
+        ],
+        "output": "sondes",
+    },
+    "qc": {
+        "intake": "sondes",
+        "apply": iterate_Sonde_method_over_dict_of_Sondes_objects,
+        "functions": [
             "detect_floater",
             "profile_fullness",
             "near_surface_coverage",

src/halodrops/processor.py

@@ -3,6 +3,7 @@
 import datetime
 from typing import Any, Optional, List
 import os
+import subprocess
 
 import numpy as np
 import xarray as xr
@@ -106,13 +107,26 @@ def add_afile(self, path_to_afile: str) -> None:
         object.__setattr__(self, "afile", path_to_afile)
         return self
 
-    def add_postaspenfile(self, path_to_postaspenfile: str = None) -> None:
-        """Sets attribute with path to post-ASPEN file of the sonde
+    def add_level_dir(self):
+        if not hasattr(self, "afile"):
+            raise ValueError("No afile in sonde. Cannot continue")
+        l0_dir = os.path.dirname(self.afile)
+        l1_dir = l0_dir.replace("Level_0", "Level_1")
+        l2_dir = l0_dir.replace("Level_0", "Level_2")
+        l3_dir = l0_dir.replace("Level_0", "Level_3")
+
+        object.__setattr__(self, "l0_dir", l0_dir)
+        object.__setattr__(self, "l1_dir", l1_dir)
+        object.__setattr__(self, "l2_dir", l2_dir)
+        object.__setattr__(self, "l3_dir", l3_dir)
+
+    def run_aspen(self, path_to_postaspenfile: str = None) -> None:
+        """Runs aspen and sets attribute with path to post-ASPEN file of the sonde
 
         If the A-file path is known for the sonde, i.e. if the attribute `path_to_afile` exists,
         then the function will attempt to look for a post-ASPEN file of the same date-time as in the A-file's name.
         Sometimes, the post-ASPEN file might not exist (e.g. because launch was not detected), and in
-        such cases, an exception will be raised.
+        such cases, ASPEN will run in a docker image and create the file.
 
         If the A-file path is not known for the sonde, the function will expect the argument
         `path_to_postaspenfile` to be not empty.
@@ -122,47 +136,42 @@ def add_postaspenfile(self, path_to_postaspenfile: str = None) -> None:
         path_to_postaspenfile : str, optional
             The path to the post-ASPEN file. If not provided, the function will attempt to construct the path from the `afile` attribute.
 
-        Raises
-        ------
-        ValueError
-            If the `afile` attribute does not exist when `path_to_postaspenfile` is not provided.
-            If the post-ASPEN file does not exist at the constructed or provided path, and launch was detected in the A-file.
-            If the launch was not detected in the A-file.
-
         Attributes Set
         --------------
         postaspenfile : str
             The path to the post-ASPEN file. This attribute is set if the file exists at the constructed or provided path.
         """
 
+        l0_dir = self.l0_dir  # os.path.dirname(self.afile)
+        aname = os.path.basename(self.afile)
+        dname = "D" + aname[1:]
+        l1_dir = self.l1_dir
+        l1_name = dname.split(".")[0] + "QC.nc"
+
         if path_to_postaspenfile is None:
-            if hasattr(self, "afile"):
-                path_to_l1dir = os.path.dirname(self.afile)[:-1] + "1"
-                postaspenfile = (
-                    "D" + os.path.basename(self.afile).split(".")[0][1:] + "QC.nc"
-                )
-                path_to_postaspenfile = os.path.join(path_to_l1dir, postaspenfile)
-                if os.path.exists(path_to_postaspenfile):
-                    object.__setattr__(self, "postaspenfile", path_to_postaspenfile)
-                else:
-                    if rr.check_launch_detect_in_afile(self.afile):
-                        raise ValueError(
-                            f"The post-ASPEN file for {self.serial_id} with filename {postaspenfile} does not exist. Therefore, I am not setting the `postaspenfile` attribute. I checked and found that launch was detected for {self.serial_id}."
-                        )
-                    else:
-                        raise ValueError(
-                            f"Launch not detected for {self.serial_id}. Therefore, {postaspenfile} does not exist and I am not setting the `postaspenfile` attribute."
-                        )
-            else:
-                raise ValueError("The attribute `path_to_afile` doesn't exist.")
+            path_to_postaspenfile = os.path.join(l1_dir, l1_name)
+
+        if not os.path.exists(path_to_postaspenfile):
+            os.makedirs(l1_dir, exist_ok=True)
+            subprocess.run(
+                [
+                    "docker",
+                    "run",
+                    "--rm",
+                    "--mount",
+                    f"type=bind,source={l0_dir},target=/input",
+                    "--mount",
+                    f"type=bind,source={l1_dir},target=/output",
+                    "ghcr.io/atmdrops/aspenqc:4.0.2",
+                    "-i",
+                    f"/input/{dname}",
+                    "-n",
+                    f"/output/{l1_name}",
+                ],
+                check=True,
+            )
 
-        else:
-            if os.path.exists(path_to_postaspenfile):
-                object.__setattr__(self, "postaspenfile", path_to_postaspenfile)
-            else:
-                raise ValueError(
-                    f"The post-ASPEN file for your provided {path_to_postaspenfile=} does not exist. Therefore, I am not setting the `postaspenfile` attribute."
-                )
+        object.__setattr__(self, "postaspenfile", path_to_postaspenfile)
         return self
 
     def add_aspen_ds(self) -> None:
@@ -917,7 +926,7 @@ def write_l2(self, l2_dir: str = None):
         """
 
         if l2_dir is None:
-            l2_dir = os.path.dirname(self.afile)[:-1] + "2"
+            l2_dir = self.l2_dir
 
         if not os.path.exists(l2_dir):
             os.makedirs(l2_dir)
@@ -941,7 +950,7 @@ def add_l2_ds(self, l2_dir: str = None):
             Returns the sonde object with the L2 dataset added as an attribute.
         """
         if l2_dir is None:
-            l2_dir = os.path.dirname(self.afile)[:-1] + "2"
+            self.l2_dir
 
         object.__setattr__(
             self, "l2_ds", xr.open_dataset(os.path.join(l2_dir, self.l2_filename))

tests/test_paths.py

@@ -1,18 +1,42 @@
 from halodrops.helper import paths
 import os
+import pytest
 
-main_data_directory = "../sample"
-platform = "HALO"
-flightdate = "20200101"
-l1_path = os.path.join(main_data_directory, platform, flightdate, "Level_1")
-quicklooks_path = os.path.join(main_data_directory, platform, flightdate, "Quicklooks")
+main_data_directory = "./example_data"
+platform_id = "HALO"
+flightdate = "20200119"
+path_structure = "{platform}/Level_0/{flight}"
+platform_path_structure = "{platform}/Level_0"
 
-object = paths.Flight(main_data_directory, flightdate, platform)
+l1_path = os.path.join(main_data_directory, platform_id, "Level_1", flightdate)
 
+quicklooks_path = os.path.join(
+    main_data_directory, platform_id, "Quicklooks", flightdate
+)
 
-def test_l1_path():
-    assert object.l1dir == l1_path
 
+@pytest.fixture
+def flight():
+    flight = paths.Flight(main_data_directory, flightdate, platform_id, path_structure)
+    return flight
 
-def test_quicklooks_path():
-    assert object.quicklooks_path() == quicklooks_path
+
+@pytest.fixture
+def platform():
+    platform = paths.Platform(
+        main_data_directory, platform_id, path_structure=platform_path_structure
+    )
+    return platform
+
+
+def test_get_flight_ids(platform):
+    flight_ids = platform.get_flight_ids()
+    assert flight_ids[0] == flightdate
+
+
+def test_l1_path(flight):
+    assert flight.l1_dir == l1_path
+
+
+def test_quicklooks_path(flight):
+    assert flight.quicklooks_path() == quicklooks_path