Use obstype SUB_EXP for raw and reduced stackframes

CHANGES.md

@@ -1,6 +1,12 @@
 Versions
 ========
 
+1.38.1 (2026-08-25)
+-------------------
+
+- Expect smartstack stackframes to have obstype=SUB_EXP
+  (filenames use n00 for raw, n09 for reduced)
+
 1.38.0 (2026-08-25)
 -------------------
 

banzai/settings.py

@@ -49,8 +49,12 @@
                             'READNOISE': ['configuration_mode', 'binning']}
 
 LAST_STAGE = {'BIAS': 'banzai.trim.Trimmer',
-              'DARK': 'banzai.uncertainty.PoissonInitializer', 'SKYFLAT': 'banzai.dark.DarkSubtractor',
-              'SINISTRO': 'banzai.mosaic.MosaicCreator', 'STANDARD': None, 'EXPOSE': None, 'EXPERIMENTAL': None}
+              'DARK': 'banzai.uncertainty.PoissonInitializer',
+              'SKYFLAT': 'banzai.dark.DarkSubtractor',
+              'STANDARD': None,
+              'EXPOSE': None,
+              'SUB_EXP': None,
+              'EXPERIMENTAL': None}
 
 EXTRA_STAGES = {'BIAS': ['banzai.bias.BiasMasterLevelSubtractor', 'banzai.bias.BiasComparer'],
                 'DARK': ['banzai.dark.DarkNormalizer', 'banzai.dark.DarkTemperatureChecker',
@@ -59,6 +63,7 @@
                             'banzai.qc.PatternNoiseDetector', 'banzai.flats.FlatComparer'],
                 'STANDARD': None,
                 'EXPOSE': None,
+                'SUB_EXP': None,
                 'EXPERIMENTAL': None}
 
 CALIBRATION_STACKER_STAGES = {'BIAS': ['banzai.bias.BiasMaker'],
@@ -137,12 +142,14 @@
 # Proposal ids for data that should be public instantly. Should all be lowercase
 PUBLIC_PROPOSALS = ['calibrate', 'standard', '*standards', '*epo*', 'pointing', 'tso*']
 
-SUPPORTED_FRAME_TYPES = ['BPM', 'READNOISE', 'BIAS', 'DARK', 'SKYFLAT', 'EXPOSE', 'STANDARD', 'EXPERIMENTAL']
+SUPPORTED_FRAME_TYPES = ['BPM', 'READNOISE', 'BIAS', 'DARK', 'SKYFLAT', 'EXPOSE', 'SUB_EXP', 'STANDARD',
+                         'EXPERIMENTAL']
 
 REDUCED_DATA_EXTENSION_ORDERING = {'BIAS': ['SCI', 'BPM', 'ERR'],
                                    'DARK': ['SCI', 'BPM', 'ERR'],
                                    'SKYFLAT': ['SCI', 'BPM', 'ERR'],
                                    'EXPOSE': ['SCI', 'CAT', 'BPM', 'ERR'],
+                                   'SUB_EXP': ['SCI', 'CAT', 'BPM', 'ERR'],
                                    'STANDARD': ['SCI', 'CAT', 'BPM', 'ERR'],
                                    'EXPERIMENTAL': ['SCI', 'CAT', 'BPM', 'ERR']}
 

banzai/smartstack_products.py

@@ -115,11 +115,12 @@ def apply_smartstack_metadata(output_frame, input_images, stackframes, moluid):
         utstop = newest_image.primary_hdu.meta.get('UTSTOP')
 
     # The inherited L1MEAN/L1MEDIAN/L1SIGMA describe the first input, not the stack. Remeasure with the
-    # same background-map method as photometry.SourceDetector so e45 values are comparable to e09 ones
+    # same background-map method as photometry.SourceDetector so e45 values are comparable to n09 ones
     # (raw-pixel stats would inflate L1SIGMA with pixel noise; the map measures background variation).
     background_cards = background_header_cards(estimate_background(output_frame['SCI'].data))
 
     for header in headers:
+        header['OBSTYPE'] = ('EXPOSE', 'Observation type')
         header['EXPTIME'] = (total_exptime, '[s] Total exposure time')
         header['DATE-OBS'] = (date_utils.date_obs_to_string(earliest_dateobs), '[UTC] Earliest observation time')
         header['NCOMBINE'] = (n_stacked, 'Number of images combined')
@@ -146,8 +147,8 @@ def build_stacked_frame(stackframes, runtime_context, moluid):
         raise ValueError('Cannot build a smartstack without stackframes')
     stackframes = sorted(stackframes, key=lambda stackframe: stackframe.stack_num)
     input_images = open_stackframe_images(stackframes, runtime_context)
-    base, rlevel, file_extension = input_images[0].filename.rpartition('-e09')
-    if rlevel != '-e09' or file_extension not in ('.fits', '.fits.fz'):
+    base, rlevel, file_extension = input_images[0].filename.rpartition('-n09')
+    if rlevel != '-n09' or file_extension not in ('.fits', '.fits.fz'):
         raise ValueError(f'Could not parse smartstack input filename: {input_images[0].filename}')
     output_filename = f'{base}-e{SMARTSTACK_REDUCTION_LEVEL}{file_extension}'
     output_frame = init_smartstack_frame(input_images[0], output_filename)

banzai/tests/site_e2e/README.md

@@ -22,9 +22,9 @@ These tests validate the full system by:
 5. **Cache Drift Reconciliation** - The download worker restores local cache
    filepaths when replicated database state drifts.
 
-6. **Smartstack Reduction and Stacking** - Raw stackframes published to the stack
-   queue are reduced to `e09` using the cached calibrations, then stacked into
-   `e45` products with JPEG previews and shipper messages.
+6. **Smartstack Reduction and Stacking** - The tests cover `n00` -> `n09` ->
+   `e45`: raw stackframes are reduced using the cached calibrations, then
+   stacked into final products with JPEG previews and shipper messages.
 
 7. **Replication Slot Reuse** - Cache initialization succeeds when a publisher
    replication slot already exists.

banzai/tests/site_e2e/test_site_e2e.py

@@ -60,6 +60,22 @@ def check():
         assert p.stat().st_size > 0, f"Cached file is empty: {p}"
 
 
+def _prepare_stackframe(source_path, destination_filename, moluid, stack_num, frmtotal):
+    """Copy the archive seed to an n00 member and add Smartstack metadata."""
+    frame_path = RAW_DIR / destination_filename
+    shutil.copy2(str(source_path), str(frame_path))
+    with fits.open(str(frame_path), mode='update') as hdul:
+        for hdu in hdul:
+            if 'OBSTYPE' not in hdu.header:
+                continue
+            hdu.header['OBSTYPE'] = 'SUB_EXP'
+            hdu.header['MOLUID'] = moluid
+            hdu.header['MOLFRNUM'] = stack_num
+            hdu.header['FRMTOTAL'] = frmtotal
+            hdu.header['STACK'] = 'T'
+    return frame_path
+
+
 # Broker the host-side test process publishes to; the shared banzai-rabbitmq
 # container is exposed on localhost:5672 (containers reach it via host.docker.internal).
 BROKER_URL = 'amqp://localhost:5672'
@@ -111,7 +127,7 @@ def _drain_ship_probe(queue_name=SHIP_PROBE_QUEUE, broker_url=BROKER_URL):
 def _corrupt_file(path):
     """Overwrite an on-disk file that may be owned by a container UID with garbage.
 
-    The reduced e09 files are written from inside the site containers, so the host
+    The reduced n09 files are written from inside the site containers, so the host
     test user may not own them. Overwrite through a throwaway container that mounts
     the reduced-output tree at the same absolute path (mirrors
     ``conftest._clean_data_dir``), which runs as root and always succeeds.
@@ -287,16 +303,12 @@ def test_12_stackframe_stack_completes(self, site_deployment):
 
         raw_dir = RAW_DIR
         src_path = raw_dir / RAW_FRAME_FILENAME
-        stackframe_path = raw_dir / 'lsc0m476-sq34-20260121-0301-e00.fits.fz'
 
         assert src_path.exists(), f"Raw frame not found: {src_path}"
-        shutil.copy2(str(src_path), str(stackframe_path))
-
-        with fits.open(str(stackframe_path), mode='update') as hdul:
-            hdul['SCI'].header['MOLUID'] = 'mol-e2e-test'
-            hdul['SCI'].header['MOLFRNUM'] = 1
-            hdul['SCI'].header['FRMTOTAL'] = 1
-            hdul['SCI'].header['STACK'] = 'T'
+        stackframe_path = _prepare_stackframe(
+            src_path, 'lsc0m476-sq34-20260121-0301-n00.fits.fz',
+            'mol-e2e-test', stack_num=1, frmtotal=1,
+        )
 
         body = json.dumps({
             'fits_file': str(stackframe_path),
@@ -333,10 +345,10 @@ def check():
 
     @pytest.mark.e2e_site_reduction
     def test_12b_reduction_used_cached_calibrations(self, site_deployment):
-        """Verify reduced e09 stackframes used calibrations from the local cache and DB."""
-        reduced_files = [path for pattern in ('*-e09.fits', '*-e09.fits.fz')
+        """Verify reduced n09 stackframes used calibrations from the local cache and DB."""
+        reduced_files = [path for pattern in ('*-n09.fits', '*-n09.fits.fz')
                          for path in OUTPUT_DIR.rglob(pattern)]
-        assert reduced_files, f"No reduced e09 files found under {OUTPUT_DIR}"
+        assert reduced_files, f"No reduced n09 files found under {OUTPUT_DIR}"
 
         cal_header_keys = {'L1IDBIAS': 'bias', 'L1IDDARK': 'dark', 'L1IDFLAT': 'flat'}
         cached_files = {p.name for p in CACHE_DIR.rglob('*.fits.fz')}
@@ -392,13 +404,10 @@ def test_13_smartstack_completes(self, site_deployment):
             stack_queue = os.environ.get('STACK_QUEUE_NAME', 'banzai_stack_queue')
 
             def publish_stackframe(stack_num, is_last):
-                frame_path = raw_dir / f'lsc0m476-sq34-20260121-{stack_num:04d}-e00.fits.fz'
-                shutil.copy2(str(src_path), str(frame_path))
-                with fits.open(str(frame_path), mode='update') as hdul:
-                    hdul['SCI'].header['MOLUID'] = moluid
-                    hdul['SCI'].header['MOLFRNUM'] = stack_num
-                    hdul['SCI'].header['FRMTOTAL'] = 3
-                    hdul['SCI'].header['STACK'] = 'T'
+                frame_path = _prepare_stackframe(
+                    src_path, f'lsc0m476-sq34-20260121-{stack_num:04d}-n00.fits.fz',
+                    moluid, stack_num=stack_num, frmtotal=3,
+                )
                 body = json.dumps({
                     'fits_file': str(frame_path),
                     'last_frame': is_last,
@@ -448,6 +457,7 @@ def stack_complete():
 
             with fits.open(e45_path) as hdul:
                 for header in (hdul[0].header, hdul['SCI'].header):
+                    assert header['OBSTYPE'] == 'EXPOSE'
                     assert header['NCOMBINE'] == 3
                     assert [header[f'IMCOM{index:03d}'] for index in range(1, 4)] == input_basenames
                     assert 'IMCOM004' not in header
@@ -500,7 +510,7 @@ def stack_complete():
     def test_14_poison_stack_self_heals(self, site_deployment):
         """A stack with a corrupted reduced input self-heals to 'error'; a sibling completes.
 
-        After the poison stack's first frame reduces, its e09 file is corrupted so
+        After the poison stack's first frame reduces, its n09 file is corrupted so
         finalize cannot open it. Finalize failure is a *caught* exception (not a
         violent crash), so the worker keeps running: it burns one claimed attempt per
         tick and, after the FINALIZE_BACKOFF_SECONDS ladder (set short in site_e2e.env),
@@ -516,21 +526,18 @@ def test_14_poison_stack_self_heals(self, site_deployment):
         sibling_moluid = 'mol-e2e-poison-sibling'
 
         def publish_stackframe(moluid, stack_num, frmtotal, is_last, filename):
-            frame_path = raw_dir / filename
-            shutil.copy2(str(src_path), str(frame_path))
-            with fits.open(str(frame_path), mode='update') as hdul:
-                hdul['SCI'].header['MOLUID'] = moluid
-                hdul['SCI'].header['MOLFRNUM'] = stack_num
-                hdul['SCI'].header['FRMTOTAL'] = frmtotal
-                hdul['SCI'].header['STACK'] = 'T'
+            frame_path = _prepare_stackframe(
+                src_path, filename, moluid, stack_num=stack_num,
+                frmtotal=frmtotal,
+            )
             body = json.dumps({
                 'fits_file': str(frame_path),
                 'last_frame': is_last,
             })
             publish_raw_string_to_queue(stack_queue, body)
 
         # 1. Publish the poison stack's first frame and wait for it to reduce.
-        publish_stackframe(poison_moluid, 1, 2, False, 'lsc0m476-sq34-20260121-0101-e00.fits.fz')
+        publish_stackframe(poison_moluid, 1, 2, False, 'lsc0m476-sq34-20260121-0101-n00.fits.fz')
 
         def poison_frame_reduced():
             with dbs.get_session(LOCAL_DB_ADDRESS, site_deploy=True) as session:
@@ -542,16 +549,16 @@ def poison_frame_reduced():
 
         # Generous timeout: the reduction path makes network calls (WCS/photometry over
         # VPN) and can stall well past a single frame's typical ~20s under contention.
-        e09_path = poll_until(poison_frame_reduced, timeout=600, interval=5)
-        assert e09_path, "Poison stack's first frame never reduced"
+        n09_path = poll_until(poison_frame_reduced, timeout=600, interval=5)
+        assert n09_path, "Poison stack's first frame never reduced"
 
-        # 2. Corrupt the reduced e09 so finalize cannot open it.
-        _corrupt_file(e09_path)
+        # 2. Corrupt the reduced n09 so finalize cannot open it.
+        _corrupt_file(n09_path)
 
         # 3. Complete the poison stack, and publish a healthy sibling in the same window.
-        publish_stackframe(poison_moluid, 2, 2, True, 'lsc0m476-sq34-20260121-0102-e00.fits.fz')
-        publish_stackframe(sibling_moluid, 1, 2, False, 'lsc0m476-sq34-20260121-0201-e00.fits.fz')
-        publish_stackframe(sibling_moluid, 2, 2, True, 'lsc0m476-sq34-20260121-0202-e00.fits.fz')
+        publish_stackframe(poison_moluid, 2, 2, True, 'lsc0m476-sq34-20260121-0102-n00.fits.fz')
+        publish_stackframe(sibling_moluid, 1, 2, False, 'lsc0m476-sq34-20260121-0201-n00.fits.fz')
+        publish_stackframe(sibling_moluid, 2, 2, True, 'lsc0m476-sq34-20260121-0202-n00.fits.fz')
 
         # 4. The poison stack exhausts the attempt ladder and lands in 'error'.
         def poison_errored():
@@ -615,21 +622,18 @@ def restarted():
         stack_queue = os.environ.get('STACK_QUEUE_NAME', 'banzai_stack_queue')
 
         def publish_stackframe(stack_num, is_last, filename):
-            frame_path = raw_dir / filename
-            shutil.copy2(str(src_path), str(frame_path))
-            with fits.open(str(frame_path), mode='update') as hdul:
-                hdul['SCI'].header['MOLUID'] = moluid
-                hdul['SCI'].header['MOLFRNUM'] = stack_num
-                hdul['SCI'].header['FRMTOTAL'] = 2
-                hdul['SCI'].header['STACK'] = 'T'
+            frame_path = _prepare_stackframe(
+                src_path, filename, moluid, stack_num=stack_num,
+                frmtotal=2,
+            )
             body = json.dumps({
                 'fits_file': str(frame_path),
                 'last_frame': is_last,
             })
             publish_raw_string_to_queue(stack_queue, body)
 
-        publish_stackframe(1, False, 'lsc0m476-sq34-20260121-0401-e00.fits.fz')
-        publish_stackframe(2, True, 'lsc0m476-sq34-20260121-0402-e00.fits.fz')
+        publish_stackframe(1, False, 'lsc0m476-sq34-20260121-0401-n00.fits.fz')
+        publish_stackframe(2, True, 'lsc0m476-sq34-20260121-0402-n00.fits.fz')
 
         def stack_complete():
             with dbs.get_session(LOCAL_DB_ADDRESS, site_deploy=True) as session:

banzai/tests/test_frames.py

@@ -203,11 +203,11 @@ def test_init_poisson_uncertainties():
 
 
 def test_get_output_filename():
-    test_frame = FakeLCOObservationFrame(file_path='test_image_00.fits')
-    test_context = FakeContext(frame_class=FakeLCOObservationFrame)
+    test_frame = FakeLCOObservationFrame(file_path='test_image_n00.fits')
+    test_context = FakeContext(frame_class=FakeLCOObservationFrame, reduction_level='9')
     filename = test_frame.get_output_filename(test_context)
 
-    assert filename == 'test_image_91.fits.fz'
+    assert filename == 'test_image_n09.fits.fz'
 
 
 def test_section_transformation():

banzai/tests/test_image_utils.py

@@ -6,10 +6,13 @@
 pytestmark = pytest.mark.image_utils
 
 
-def test_image_can_be_processed_known_obstype():
-    image = FakeLCOObservationFrame([FakeCCDData(meta={'OBSTYPE': 'BIAS'})])
+def test_sub_exp_image_can_be_processed_with_full_pipeline_settings():
+    image = FakeLCOObservationFrame([FakeCCDData(meta={'OBSTYPE': 'SUB_EXP'})])
+    context = FakeContext()
 
-    assert image_utils.image_can_be_processed(image, FakeContext())
+    assert image_utils.image_can_be_processed(image, context)
+    assert (context.LAST_STAGE['SUB_EXP'], context.EXTRA_STAGES['SUB_EXP']) == (None, None)
+    assert context.REDUCED_DATA_EXTENSION_ORDERING['SUB_EXP'] == context.REDUCED_DATA_EXTENSION_ORDERING['EXPOSE']
 
 
 def test_image_cannot_be_processed_unknown_obstype():

banzai/tests/test_smart_stacking.py

@@ -525,7 +525,7 @@ def _make_fits_header(**overrides):
         return h
 
     @staticmethod
-    def _make_mock_image(output_dir='/data/processed', output_filename='frame-e09.fits'):
+    def _make_mock_image(output_dir='/data/processed', output_filename='frame-n09.fits'):
         """Build a mock image returned by run_pipeline_stages."""
         img = MagicMock()
         img.get_output_directory.return_value = output_dir
@@ -549,7 +549,7 @@ def test_process_stackframe_logs_timeout_rejection(self, mock_run_stages, mock_l
                         'smartstack_moluid': 'mol-xyz',
                         'smartstack_camera': 'cam1',
                         'smartstack_stack_num': 1,
-                        'smartstack_filepath': '/data/processed/frame-e09.fits',
+                        'smartstack_filepath': '/data/processed/frame-n09.fits',
                         'smartstack_status': 'timeout'},
         )
 
@@ -584,7 +584,7 @@ def test_process_stackframe_upserts_stackframe_and_does_not_notify_redis(
             stack_num=1,
             frmtotal=5,
             camera='cam1',
-            filepath='/data/processed/frame-e09.fits',
+            filepath='/data/processed/frame-n09.fits',
             is_last=expected_is_last,
             dateobs=datetime.datetime(2024, 1, 1, 0, 0, 0),
         )
@@ -614,7 +614,7 @@ def _run_stages(*args, **kwargs):
             process_stackframe(body, runtime_context)
 
         mock_upsert.assert_called_once()
-        assert mock_upsert.call_args.kwargs['filepath'] == '/data/processed/frame-e09.fits'
+        assert mock_upsert.call_args.kwargs['filepath'] == '/data/processed/frame-n09.fits'
         redis_module.Redis.from_url.assert_not_called()
 
     @patch('banzai.scheduling.stage_utils.run_pipeline_stages')