From 78c1a000d7c9e92c99c66e5a35214d6f1afbbba7 Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Tue, 14 Jul 2026 10:06:55 -0500 Subject: [PATCH 1/3] ADD: Feature to use previous x labels --- CODEBOOK.md | 26 ++++++++++++++------------ lars/nepho/inference.py | 13 +++++++++++-- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/CODEBOOK.md b/CODEBOOK.md index 4a64512..fe8a7e8 100644 --- a/CODEBOOK.md +++ b/CODEBOOK.md @@ -23,14 +23,14 @@ The purpose of this section is to label radar imagery for warm-season precipitat |--------------------------|-------|-------------------------------------| | *reflectivity* | dBZ | Intensity of returned radar signal | | n_gates_50dBZ | percent | The percentage of gates greater than 50 dBZ | - +| n_gates_30dBZ | percent | The percentage of gates greater than 30 dBZ | ### 2.2 Image Format - **Spatial resolution:** 1 km by 1 km - **Temporal resolution:** 10-minute intervals -- **Projection:** Polar coordinates projected onto a 256x256 image -- **Color scale:** ChaseSpectral colormap with vmin=-10 and vmax=60 +- **Projection:** Polar coordinates projected onto a 2048x2048 image +- **Color scale:** NWSRef colormap with vmin=-20 and vmax=80 --- @@ -42,10 +42,10 @@ Each image or region-of-interest must be assigned exactly one primary class. | Label | Description | |------------------------|-----------------------------------------------------------------------------| -| No Precipitation | No significant return; background noise only. The image will only have blue and black colors. The percentage of gates greater than 50 dBZ must not exceed 0.005 percent. If it does exceed 0.005 percent, then classify as Isolated Convection. | -| Stratiform Precipitation | The image must have no pink colors. Green, yellow and red colors are present in a widespread blob. The percentage of gates greater than 50 dBZ must not exceed 0.035 percent. If it does exceed 0.035 percent, then classify as a mesoscale convective system. | -| Isolated Convection | The image must have regions of yellow, red and pink colors. These dark red and pink regions must be separated by regions of black and blue, with no connection to other dark red and pink regions through yellow regions. Over half of the image must be blue or black. The percentage of gates with reflectivity greater than 30 dBZ must not exceed 1.3 percent. If it does exceed 1.3 percent, then classify as a mesoscale convective system. | -| Mesoscale Convective System | A string or connected cluster of dark red and pink colors must be present in the image. This string can take on a curved structure. There can be more than one such string or cluster in the image. The dark red and pink colors in the clusters must be connected by yellow regions. | +| No Precipitation | No significant return; background noise only. The image will be over 80 percent blue colors. The percentage of gates greater than 50 dBZ must not exceed 0.005 percent. If it does exceed 0.005 percent, then classify as Isolated Convection. | +| Stratiform Precipitation | The image must have no dark red or pink colors. Green, yellow, and light red colors are present in a widespread blob. The percentage of gates greater than 50 dBZ must not exceed 0.035 percent. If it does exceed 0.035 percent, then classify as a mesoscale convective system. | +| Isolated Convection | The image must have regions of yellow and dark red colors. These dark red regions must be separated by regions of blue, with no connection to other dark red and pink regions through yellow regions. Over half of the image must be blue or black. The percentage of gates with reflectivity greater than 30 dBZ must not exceed 1.3 percent. If it does exceed 1.3 percent, then classify as a mesoscale convective system. | +| Mesoscale Convective System | A string or connected cluster of dark red colors must be present in the image. This string can take on a curved structure. There can be more than one such string or cluster in the image. The dark red colors in the clusters must be connected by yellow or green regions. | | Ambiguous | Cannot be classified with confidence. | > **Note on enforcement.** Only the quantitative reflectivity thresholds in the @@ -69,9 +69,11 @@ Each image or region-of-interest must be assigned exactly one primary class. ## 6. Annotator Guidelines The bullets in this section are passed verbatim to automated labelling models, -so they must be self-contained for a single image with no external context. +so they must be self-contained for a single image with no external context, outside of +the label of the previous timestep. - If two or more categories are present in regions of the image, classify with the most widespread category in the image. +- When in doubt, default to the class of the image(s) preceding it in time. ### 6.1 Human Annotators Only @@ -103,10 +105,10 @@ independently, with no temporal context and no access to the example gallery. | Class | Example Image | Notes | |--------------------------|-----------------------------------|--------------------------| -| Stratiform Precipitation | [![Stratiform Precipitation](examples/01_stratiform.png)](examples/01_stratiform.png) | Widespread yellows and reds | -| Mesoscale Convective System | [![Mesoscale Convective System](examples/01_mcs.png)](examples/01_mcs.png) | Multiple lines of pinks and reds | -| Isolated Convection | [![Isolated Convection](examples/01_isolated.png)](examples/01_isolated.png) | Isolated reds not inter-connected | -| No Precipitation | [![No Precipitation](examples/01_clutter.png)](examples/01_clutter.png) | No greens, yellows, reds or pinks | +| Stratiform Precipitation | [![Stratiform Precipitation](../examples/01_stratiform.png)](examples/01_stratiform.png) | Widespread yellows and reds | +| Mesoscale Convective System | [![Mesoscale Convective System](../examples/01_mcs.png)](examples/01_mcs.png) | Multiple lines of pinks and reds | +| Isolated Convection | [![Isolated Convection](../examples/01_isolated.png)](examples/01_isolated.png) | Isolated reds not inter-connected | +| No Precipitation | [![No Precipitation](../examples/01_clutter.png)](examples/01_clutter.png) | No greens, yellows, reds or pinks | --- diff --git a/lars/nepho/inference.py b/lars/nepho/inference.py index 4bbc835..ba1a3d0 100644 --- a/lars/nepho/inference.py +++ b/lars/nepho/inference.py @@ -493,7 +493,8 @@ async def label_radar_data(radar_df, model, categories=None, guidelines=None, mlflow_experiment=None, mlflow_run_name=None, mlflow_tracking_uri=None, codebook_path=None, site="Bankhead National Forest", - verbose=True, vmin=None, vmax=None, model_output_dir=None): + verbose=True, vmin=None, vmax=None, model_output_dir=None, + use_previous_labels=False): """ Label radar data using a given model. @@ -535,6 +536,8 @@ async def label_radar_data(radar_df, model, categories=None, guidelines=None, ``colormap_from_codebook``; otherwise they fall back to ``DEFAULT_VMIN`` (-20) and ``DEFAULT_VMAX`` (60). model_output_dir: str: Directory to save model outputs. + use_previous_labels: bool or int: If True, the function will use the previous *use_previous_labels* + labels as an additional input to the model for labeling. This can be useful if the model is being used to refine or validate existing labels. Returns ------- @@ -576,9 +579,15 @@ async def label_radar_data(radar_df, model, categories=None, guidelines=None, for fi in radar_df["file_path"].values: time = radar_df.loc[radar_df["file_path"] == fi, "time"].values[0] + cur_index = radar_df.index[radar_df["file_path"] == fi][0] prompt_with_time = prompt + f"Please provide just the category label for the radar image taken at time {time}." prompt_with_time = prompt_with_time + "Do not provide your reasoning for your selection, just the category." - + if use_previous_labels: + for i in range(use_previous_labels): + if cur_index - i - 1 >= 0: + prev_label = radar_df.loc[cur_index - i - 1, "label"] + prompt_with_time += f" The label for the previous radar image taken at time {radar_df.loc[cur_index - i - 1, 'time']} is {prev_label}." + output_model = await model.chat(prompt_with_time, images=[fi]) # Find the category label in the output output_model = output_model.strip() From 80a7b69c2839ed2a8a1e20fcc05b79ba36df0833 Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Tue, 14 Jul 2026 10:09:25 -0500 Subject: [PATCH 2/3] ADD: Two codebooks: one for ChaseSpectral and one for NWSRef --- example_codebooks/CODEBOOK.md | 126 +++++++++++++++++++++++++++ example_codebooks/CODEBOOK_NWSRef.md | 126 +++++++++++++++++++++++++++ 2 files changed, 252 insertions(+) create mode 100644 example_codebooks/CODEBOOK.md create mode 100644 example_codebooks/CODEBOOK_NWSRef.md diff --git a/example_codebooks/CODEBOOK.md b/example_codebooks/CODEBOOK.md new file mode 100644 index 0000000..0f18694 --- /dev/null +++ b/example_codebooks/CODEBOOK.md @@ -0,0 +1,126 @@ +# Radar Image Labelling Codebook + +A reference guide for annotators labelling radar imagery with LARS. Use this codebook to ensure consistent, reproducible labels across all annotators and sessions. + +--- + +## 1. Overview + +The purpose of this section is to label radar imagery for warm-season precipitation + +- **Radar type:** ARM CSAPR2 +- **Data source:** csapr2cfr.a1 datastream +- **Geographic scope:** Bankhead National Forest +- **Labelling task:** scene classification + +--- + +## 2. Data Description + +### 2.1 Input Fields + +| Field | Units | Description | +|--------------------------|-------|-------------------------------------| +| *reflectivity* | dBZ | Intensity of returned radar signal | +| n_gates_50dBZ | percent | The percentage of gates greater than 50 dBZ | + + +### 2.2 Image Format + +- **Spatial resolution:** 1 km by 1 km +- **Temporal resolution:** 10-minute intervals +- **Projection:** Polar coordinates projected onto a 256x256 image +- **Color scale:** ChaseSpectral colormap with vmin=-10 and vmax=60 + +--- + +## 3. Label Classes + +Each image or region-of-interest must be assigned exactly one primary class. + +### 3.1 Primary Classes + +| Label | Description | +|------------------------|-----------------------------------------------------------------------------| +| No Precipitation | No significant return; background noise only. The image will only have blue and black colors. The percentage of gates greater than 50 dBZ must not exceed 0.005 percent. If it does exceed 0.005 percent, then classify as Isolated Convection. | +| Stratiform Precipitation | The image must have no pink colors. Green, yellow and red colors are present in a widespread blob. The percentage of gates greater than 50 dBZ must not exceed 0.035 percent. If it does exceed 0.035 percent, then classify as a mesoscale convective system. | +| Isolated Convection | The image must have regions of yellow, red and pink colors. These dark red and pink regions must be separated by regions of black and blue, with no connection to other dark red and pink regions through yellow regions. Over half of the image must be blue or black. The percentage of gates with reflectivity greater than 30 dBZ must not exceed 1.3 percent. If it does exceed 1.3 percent, then classify as a mesoscale convective system. | +| Mesoscale Convective System | A string or connected cluster of dark red and pink colors must be present in the image. This string can take on a curved structure. There can be more than one such string or cluster in the image. The dark red and pink colors in the clusters must be connected by yellow regions. | +| Ambiguous | Cannot be classified with confidence. | + +> **Note on enforcement.** Only the quantitative reflectivity thresholds in the +> descriptions above (e.g. "percentage of gates greater than X dBZ must not +> exceed Y percent") are validated automatically, and only when the +> corresponding `pct_gates_*` / `n_gates_*` columns are present in the data. +> Spatial and topological criteria — "separated by regions of…", "connected +> cluster", "curved structure", "widespread blob", "over half of the image" — +> are judged by the annotator or model and are **not** checked programmatically. + +--- + +## 5. Labelling Procedure + +1. Use :code:`lars.preprocessing.preprocess_radar_data` to generate images and a .csv file +2. The csv file will label all categories as UNKNOWN. This is just a placeholder for hand labelling. +3. According to the criteria above, label all images in the 'file_path' column of the .csv file. + +--- + +## 6. Annotator Guidelines + +The bullets in this section are passed verbatim to automated labelling models, +so they must be self-contained for a single image with no external context. + +- If two or more categories are present in regions of the image, classify with the most widespread category in the image. + +### 6.1 Human Annotators Only + +These apply to human annotators and the review process. They are intentionally +kept out of the bullet list above because an automated model labels each image +independently, with no temporal context and no access to the example gallery. + +- When in doubt, default to the class of the image preceding it in time. +- Use the provided example gallery (Section 8) to calibrate your judgement. +- Inter-annotator agreement should be checked periodically; raise disagreements with the team lead. + + +--- + +## 7. Quality Control + +| Check | Method | +|-------|--------| +| Completeness | All images have a primary label | +| Consistency | Random sample reviewed by second annotator | +| Agreement metric | Cohen's κ computed per annotator pair | +| Outlier review | Labels deviating from model predictions flagged for review | + +--- + +## 8. Example Gallery + +*(Attach or link representative images for each primary class here.)* + +| Class | Example Image | Notes | +|--------------------------|-----------------------------------|--------------------------| +| Stratiform Precipitation | [![Stratiform Precipitation](../examples/01_stratiform.png)](examples/01_stratiform.png) | Widespread yellows and reds | +| Mesoscale Convective System | [![Mesoscale Convective System](../examples/01_mcs.png)](examples/01_mcs.png) | Multiple lines of pinks and reds | +| Isolated Convection | [![Isolated Convection](../examples/01_isolated.png)](examples/01_isolated.png) | Isolated reds not inter-connected | +| No Precipitation | [![No Precipitation](../examples/01_clutter.png)](examples/01_clutter.png) | No greens, yellows, reds or pinks | + +--- + +## 9. Changelog + +| Version | Date | Author | Changes | +|---------|------|--------|---------| +| 1.0 | 2025-04-23 | Robert Jackson | Initial release | +| 1.1 | 2026-07-13 | Robert Jackson | Calibrated reflectivity coverage thresholds against 1,212 hand-labelled CSAPR2 scenes: No Precipitation pct_gates_50dBZ 0.002→0.005 (clears observed No-Precip envelope); Stratiform pct_gates_50dBZ 0.02→0.035 (Youden-optimal vs MCS); Isolated Convection pct_gates_30dBZ 1.0→1.3 (Youden-optimal vs MCS). | + +--- + +## 10. References + +- Rinehart, R. E. (2004). *Radar for Meteorologists* (4th ed.). +- American Meteorological Society Glossary: https://glossary.ametsoc.org + diff --git a/example_codebooks/CODEBOOK_NWSRef.md b/example_codebooks/CODEBOOK_NWSRef.md new file mode 100644 index 0000000..bb4b328 --- /dev/null +++ b/example_codebooks/CODEBOOK_NWSRef.md @@ -0,0 +1,126 @@ +# Radar Image Labelling Codebook + +A reference guide for annotators labelling radar imagery with LARS. Use this codebook to ensure consistent, reproducible labels across all annotators and sessions. + +--- + +## 1. Overview + +The purpose of this section is to label radar imagery for warm-season precipitation + +- **Radar type:** ARM CSAPR2 +- **Data source:** csapr2cfr.a1 datastream +- **Geographic scope:** Bankhead National Forest +- **Labelling task:** scene classification + +--- + +## 2. Data Description + +### 2.1 Input Fields + +| Field | Units | Description | +|--------------------------|-------|-------------------------------------| +| *reflectivity* | dBZ | Intensity of returned radar signal | +| n_gates_50dBZ | percent | The percentage of gates greater than 50 dBZ | +| n_gates_30dBZ | percent | The percentage of gates greater than 30 dBZ | + +### 2.2 Image Format + +- **Spatial resolution:** 1 km by 1 km +- **Temporal resolution:** 10-minute intervals +- **Projection:** Polar coordinates projected onto a 2048x2048 image +- **Color scale:** NWSRef colormap with vmin=-20 and vmax=80 + +--- + +## 3. Label Classes + +Each image or region-of-interest must be assigned exactly one primary class. + +### 3.1 Primary Classes + +| Label | Description | +|------------------------|-----------------------------------------------------------------------------| +| No Precipitation | No significant return; background noise only. The image will be over 80 percent blue colors. The percentage of gates greater than 50 dBZ must not exceed 0.005 percent. If it does exceed 0.005 percent, then classify as Isolated Convection. | +| Stratiform Precipitation | The image must have no dark red or pink colors. Green, yellow, and light red colors are present in a widespread blob. The percentage of gates greater than 50 dBZ must not exceed 0.035 percent. If it does exceed 0.035 percent, then classify as a mesoscale convective system. | +| Isolated Convection | The image must have regions of yellow and dark red colors. These dark red regions must be separated by regions of blue, with no connection to other dark red and pink regions through yellow regions. Over half of the image must be blue or black. The percentage of gates with reflectivity greater than 30 dBZ must not exceed 1.3 percent. If it does exceed 1.3 percent, then classify as a mesoscale convective system. | +| Mesoscale Convective System | A string or connected cluster of dark red colors must be present in the image. This string can take on a curved structure. There can be more than one such string or cluster in the image. The dark red colors in the clusters must be connected by yellow or green regions. | +| Ambiguous | Cannot be classified with confidence. | + +> **Note on enforcement.** Only the quantitative reflectivity thresholds in the +> descriptions above (e.g. "percentage of gates greater than X dBZ must not +> exceed Y percent") are validated automatically, and only when the +> corresponding `pct_gates_*` / `n_gates_*` columns are present in the data. +> Spatial and topological criteria — "separated by regions of…", "connected +> cluster", "curved structure", "widespread blob", "over half of the image" — +> are judged by the annotator or model and are **not** checked programmatically. + +--- + +## 5. Labelling Procedure + +1. Use :code:`lars.preprocessing.preprocess_radar_data` to generate images and a .csv file +2. The csv file will label all categories as UNKNOWN. This is just a placeholder for hand labelling. +3. According to the criteria above, label all images in the 'file_path' column of the .csv file. + +--- + +## 6. Annotator Guidelines + +The bullets in this section are passed verbatim to automated labelling models, +so they must be self-contained for a single image with no external context. + +- If two or more categories are present in regions of the image, classify with the most widespread category in the image. + +### 6.1 Human Annotators Only + +These apply to human annotators and the review process. They are intentionally +kept out of the bullet list above because an automated model labels each image +independently, with no temporal context and no access to the example gallery. + +- When in doubt, default to the class of the image preceding it in time. +- Use the provided example gallery (Section 8) to calibrate your judgement. +- Inter-annotator agreement should be checked periodically; raise disagreements with the team lead. + + +--- + +## 7. Quality Control + +| Check | Method | +|-------|--------| +| Completeness | All images have a primary label | +| Consistency | Random sample reviewed by second annotator | +| Agreement metric | Cohen's κ computed per annotator pair | +| Outlier review | Labels deviating from model predictions flagged for review | + +--- + +## 8. Example Gallery + +*(Attach or link representative images for each primary class here.)* + +| Class | Example Image | Notes | +|--------------------------|-----------------------------------|--------------------------| +| Stratiform Precipitation | [![Stratiform Precipitation](../examples/01_stratiform.png)](examples/01_stratiform.png) | Widespread yellows and reds | +| Mesoscale Convective System | [![Mesoscale Convective System](../examples/01_mcs.png)](examples/01_mcs.png) | Multiple lines of pinks and reds | +| Isolated Convection | [![Isolated Convection](../examples/01_isolated.png)](examples/01_isolated.png) | Isolated reds not inter-connected | +| No Precipitation | [![No Precipitation](../examples/01_clutter.png)](examples/01_clutter.png) | No greens, yellows, reds or pinks | + +--- + +## 9. Changelog + +| Version | Date | Author | Changes | +|---------|------|--------|---------| +| 1.0 | 2025-04-23 | Robert Jackson | Initial release | +| 1.1 | 2026-07-13 | Robert Jackson | Calibrated reflectivity coverage thresholds against 1,212 hand-labelled CSAPR2 scenes: No Precipitation pct_gates_50dBZ 0.002→0.005 (clears observed No-Precip envelope); Stratiform pct_gates_50dBZ 0.02→0.035 (Youden-optimal vs MCS); Isolated Convection pct_gates_30dBZ 1.0→1.3 (Youden-optimal vs MCS). | + +--- + +## 10. References + +- Rinehart, R. E. (2004). *Radar for Meteorologists* (4th ed.). +- American Meteorological Society Glossary: https://glossary.ametsoc.org + From 4c8edbc16c7978970b1e06496a3470abb64ff320 Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Tue, 14 Jul 2026 10:26:12 -0500 Subject: [PATCH 3/3] FIX: Add parser for color coverage. --- lars/nepho/inference.py | 21 +++++++++++++++++++++ tests/test_validation_tracking.py | 7 ++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/lars/nepho/inference.py b/lars/nepho/inference.py index ba1a3d0..106e96e 100644 --- a/lars/nepho/inference.py +++ b/lars/nepho/inference.py @@ -325,6 +325,10 @@ def colormap_from_codebook(codebook_path): rf"\b(?:over|more\s+than)\s+half\s+(?:of\s+the\s+image\s+)?must\s+be\s+{_COLOR_LIST_RE}\b", re.IGNORECASE, ) +_PERCENT_DOMINANCE_RE = re.compile( + r"\b(?:over|more\s+than)\s+(\d+(?:\.\d+)?)\s*percent\b", + re.IGNORECASE, +) def _extract_colors(text): @@ -435,6 +439,23 @@ def color_criteria_from_codebook(codebook_path, }) continue + percent_m = _PERCENT_DOMINANCE_RE.search(sentence) + if percent_m: + colors = _extract_colors(sentence) + color_max_hi = max( + (COLOR_DBZ_RANGE[c][1] or 999) + for c in colors if c in COLOR_DBZ_RANGE + ) + if color_max_hi < 999: + rules.append({ + "kind": "max_pct_above", + "field": f"pct_gates_{int(color_max_hi)}dbz", + "value": 50.0, + "colors": colors, + "phrase": sentence, + }) + continue + m = _COLOR_PRESENCE_RE.search(sentence) if m: colors_text = next((g for g in m.groups() if g), "") diff --git a/tests/test_validation_tracking.py b/tests/test_validation_tracking.py index fe12196..2ed19c9 100644 --- a/tests/test_validation_tracking.py +++ b/tests/test_validation_tracking.py @@ -16,6 +16,7 @@ def test_color_criteria_from_codebook_covers_all_labels(): from lars.nepho.inference import color_criteria_from_codebook rules = color_criteria_from_codebook(CODEBOOK_PATH) + print(rules) assert set(rules) == { "No Precipitation", "Stratiform Precipitation", @@ -69,9 +70,9 @@ def test_colormap_from_codebook_parses_name_and_bounds(): from lars.nepho.inference import colormap_from_codebook cmap = colormap_from_codebook(CODEBOOK_PATH) - assert cmap["colormap"] == "ChaseSpectral" - assert cmap["vmin"] == -10 - assert cmap["vmax"] == 60 + assert cmap["colormap"] == "NWSRef" + assert cmap["vmin"] == -20 + assert cmap["vmax"] == 80 def test_colormap_from_codebook_falls_back_to_defaults():