From 2474f5f60e1351f5636722b72a26c07f2763dc01 Mon Sep 17 00:00:00 2001 From: Tim Beccue Date: Sat, 22 Aug 2026 00:40:04 -0600 Subject: [PATCH 1/4] Support SUB_EXP smartstack frames --- banzai/settings.py | 8 ++- banzai/smartstack_products.py | 7 +- banzai/tests/site_e2e/test_site_e2e.py | 92 ++++++++++++------------ banzai/tests/test_frames.py | 6 +- banzai/tests/test_image_utils.py | 9 ++- banzai/tests/test_smart_stacking.py | 8 +-- banzai/tests/test_smartstack_products.py | 45 ++++++------ 7 files changed, 95 insertions(+), 80 deletions(-) diff --git a/banzai/settings.py b/banzai/settings.py index 52f55a7b6..bf2e10d45 100644 --- a/banzai/settings.py +++ b/banzai/settings.py @@ -50,7 +50,8 @@ LAST_STAGE = {'BIAS': 'banzai.trim.Trimmer', 'DARK': 'banzai.uncertainty.PoissonInitializer', 'SKYFLAT': 'banzai.dark.DarkSubtractor', - 'SINISTRO': 'banzai.mosaic.MosaicCreator', 'STANDARD': None, 'EXPOSE': None, 'EXPERIMENTAL': None} + 'SINISTRO': 'banzai.mosaic.MosaicCreator', 'STANDARD': None, 'EXPOSE': None, 'SUB_EXP': None, + 'EXPERIMENTAL': None} EXTRA_STAGES = {'BIAS': ['banzai.bias.BiasMasterLevelSubtractor', 'banzai.bias.BiasComparer'], 'DARK': ['banzai.dark.DarkNormalizer', 'banzai.dark.DarkTemperatureChecker', @@ -59,6 +60,7 @@ 'banzai.qc.PatternNoiseDetector', 'banzai.flats.FlatComparer'], 'STANDARD': None, 'EXPOSE': None, + 'SUB_EXP': None, 'EXPERIMENTAL': None} CALIBRATION_STACKER_STAGES = {'BIAS': ['banzai.bias.BiasMaker'], @@ -137,12 +139,14 @@ # Proposal ids for data that should be public instantly. Should all be lowercase PUBLIC_PROPOSALS = ['calibrate', 'standard', '*standards', '*epo*', 'pointing', 'tso*'] -SUPPORTED_FRAME_TYPES = ['BPM', 'READNOISE', 'BIAS', 'DARK', 'SKYFLAT', 'EXPOSE', 'STANDARD', 'EXPERIMENTAL'] +SUPPORTED_FRAME_TYPES = ['BPM', 'READNOISE', 'BIAS', 'DARK', 'SKYFLAT', 'EXPOSE', 'SUB_EXP', 'STANDARD', + 'EXPERIMENTAL'] REDUCED_DATA_EXTENSION_ORDERING = {'BIAS': ['SCI', 'BPM', 'ERR'], 'DARK': ['SCI', 'BPM', 'ERR'], 'SKYFLAT': ['SCI', 'BPM', 'ERR'], 'EXPOSE': ['SCI', 'CAT', 'BPM', 'ERR'], + 'SUB_EXP': ['SCI', 'CAT', 'BPM', 'ERR'], 'STANDARD': ['SCI', 'CAT', 'BPM', 'ERR'], 'EXPERIMENTAL': ['SCI', 'CAT', 'BPM', 'ERR']} diff --git a/banzai/smartstack_products.py b/banzai/smartstack_products.py index cb1c58950..257263832 100644 --- a/banzai/smartstack_products.py +++ b/banzai/smartstack_products.py @@ -115,11 +115,12 @@ def apply_smartstack_metadata(output_frame, input_images, stackframes, moluid): utstop = newest_image.primary_hdu.meta.get('UTSTOP') # The inherited L1MEAN/L1MEDIAN/L1SIGMA describe the first input, not the stack. Remeasure with the - # same background-map method as photometry.SourceDetector so e45 values are comparable to e09 ones + # same background-map method as photometry.SourceDetector so e45 values are comparable to n09 ones # (raw-pixel stats would inflate L1SIGMA with pixel noise; the map measures background variation). background_cards = background_header_cards(estimate_background(output_frame['SCI'].data)) for header in headers: + header['OBSTYPE'] = ('EXPOSE', 'Observation type') header['EXPTIME'] = (total_exptime, '[s] Total exposure time') header['DATE-OBS'] = (date_utils.date_obs_to_string(earliest_dateobs), '[UTC] Earliest observation time') header['NCOMBINE'] = (n_stacked, 'Number of images combined') @@ -146,8 +147,8 @@ def build_stacked_frame(stackframes, runtime_context, moluid): raise ValueError('Cannot build a smartstack without stackframes') stackframes = sorted(stackframes, key=lambda stackframe: stackframe.stack_num) input_images = open_stackframe_images(stackframes, runtime_context) - base, rlevel, file_extension = input_images[0].filename.rpartition('-e09') - if rlevel != '-e09' or file_extension not in ('.fits', '.fits.fz'): + base, rlevel, file_extension = input_images[0].filename.rpartition('-n09') + if rlevel != '-n09' or file_extension not in ('.fits', '.fits.fz'): raise ValueError(f'Could not parse smartstack input filename: {input_images[0].filename}') output_filename = f'{base}-e{SMARTSTACK_REDUCTION_LEVEL}{file_extension}' output_frame = init_smartstack_frame(input_images[0], output_filename) diff --git a/banzai/tests/site_e2e/test_site_e2e.py b/banzai/tests/site_e2e/test_site_e2e.py index 6c2e0607e..24e34459d 100644 --- a/banzai/tests/site_e2e/test_site_e2e.py +++ b/banzai/tests/site_e2e/test_site_e2e.py @@ -60,6 +60,22 @@ def check(): assert p.stat().st_size > 0, f"Cached file is empty: {p}" +def _prepare_stackframe(source_path, destination_filename, moluid, stack_num, frmtotal): + """Copy the archive seed to an n00 member and add Smartstack metadata.""" + frame_path = RAW_DIR / destination_filename + shutil.copy2(str(source_path), str(frame_path)) + with fits.open(str(frame_path), mode='update') as hdul: + for hdu in hdul: + if 'OBSTYPE' not in hdu.header: + continue + hdu.header['OBSTYPE'] = 'SUB_EXP' + hdu.header['MOLUID'] = moluid + hdu.header['MOLFRNUM'] = stack_num + hdu.header['FRMTOTAL'] = frmtotal + hdu.header['STACK'] = 'T' + return frame_path + + # Broker the host-side test process publishes to; the shared banzai-rabbitmq # container is exposed on localhost:5672 (containers reach it via host.docker.internal). BROKER_URL = 'amqp://localhost:5672' @@ -111,7 +127,7 @@ def _drain_ship_probe(queue_name=SHIP_PROBE_QUEUE, broker_url=BROKER_URL): def _corrupt_file(path): """Overwrite an on-disk file that may be owned by a container UID with garbage. - The reduced e09 files are written from inside the site containers, so the host + The reduced n09 files are written from inside the site containers, so the host test user may not own them. Overwrite through a throwaway container that mounts the reduced-output tree at the same absolute path (mirrors ``conftest._clean_data_dir``), which runs as root and always succeeds. @@ -287,16 +303,12 @@ def test_12_stackframe_stack_completes(self, site_deployment): raw_dir = RAW_DIR src_path = raw_dir / RAW_FRAME_FILENAME - stackframe_path = raw_dir / 'lsc0m476-sq34-20260121-0301-e00.fits.fz' assert src_path.exists(), f"Raw frame not found: {src_path}" - shutil.copy2(str(src_path), str(stackframe_path)) - - with fits.open(str(stackframe_path), mode='update') as hdul: - hdul['SCI'].header['MOLUID'] = 'mol-e2e-test' - hdul['SCI'].header['MOLFRNUM'] = 1 - hdul['SCI'].header['FRMTOTAL'] = 1 - hdul['SCI'].header['STACK'] = 'T' + stackframe_path = _prepare_stackframe( + src_path, 'lsc0m476-sq34-20260121-0301-n00.fits.fz', + 'mol-e2e-test', stack_num=1, frmtotal=1, + ) body = json.dumps({ 'fits_file': str(stackframe_path), @@ -333,10 +345,10 @@ def check(): @pytest.mark.e2e_site_reduction def test_12b_reduction_used_cached_calibrations(self, site_deployment): - """Verify reduced e09 stackframes used calibrations from the local cache and DB.""" - reduced_files = [path for pattern in ('*-e09.fits', '*-e09.fits.fz') + """Verify reduced n09 stackframes used calibrations from the local cache and DB.""" + reduced_files = [path for pattern in ('*-n09.fits', '*-n09.fits.fz') for path in OUTPUT_DIR.rglob(pattern)] - assert reduced_files, f"No reduced e09 files found under {OUTPUT_DIR}" + assert reduced_files, f"No reduced n09 files found under {OUTPUT_DIR}" cal_header_keys = {'L1IDBIAS': 'bias', 'L1IDDARK': 'dark', 'L1IDFLAT': 'flat'} cached_files = {p.name for p in CACHE_DIR.rglob('*.fits.fz')} @@ -392,13 +404,10 @@ def test_13_smartstack_completes(self, site_deployment): stack_queue = os.environ.get('STACK_QUEUE_NAME', 'banzai_stack_queue') def publish_stackframe(stack_num, is_last): - frame_path = raw_dir / f'lsc0m476-sq34-20260121-{stack_num:04d}-e00.fits.fz' - shutil.copy2(str(src_path), str(frame_path)) - with fits.open(str(frame_path), mode='update') as hdul: - hdul['SCI'].header['MOLUID'] = moluid - hdul['SCI'].header['MOLFRNUM'] = stack_num - hdul['SCI'].header['FRMTOTAL'] = 3 - hdul['SCI'].header['STACK'] = 'T' + frame_path = _prepare_stackframe( + src_path, f'lsc0m476-sq34-20260121-{stack_num:04d}-n00.fits.fz', + moluid, stack_num=stack_num, frmtotal=3, + ) body = json.dumps({ 'fits_file': str(frame_path), 'last_frame': is_last, @@ -448,6 +457,7 @@ def stack_complete(): with fits.open(e45_path) as hdul: for header in (hdul[0].header, hdul['SCI'].header): + assert header['OBSTYPE'] == 'EXPOSE' assert header['NCOMBINE'] == 3 assert [header[f'IMCOM{index:03d}'] for index in range(1, 4)] == input_basenames assert 'IMCOM004' not in header @@ -500,7 +510,7 @@ def stack_complete(): def test_14_poison_stack_self_heals(self, site_deployment): """A stack with a corrupted reduced input self-heals to 'error'; a sibling completes. - After the poison stack's first frame reduces, its e09 file is corrupted so + After the poison stack's first frame reduces, its n09 file is corrupted so finalize cannot open it. Finalize failure is a *caught* exception (not a violent crash), so the worker keeps running: it burns one claimed attempt per tick and, after the FINALIZE_BACKOFF_SECONDS ladder (set short in site_e2e.env), @@ -516,13 +526,10 @@ def test_14_poison_stack_self_heals(self, site_deployment): sibling_moluid = 'mol-e2e-poison-sibling' def publish_stackframe(moluid, stack_num, frmtotal, is_last, filename): - frame_path = raw_dir / filename - shutil.copy2(str(src_path), str(frame_path)) - with fits.open(str(frame_path), mode='update') as hdul: - hdul['SCI'].header['MOLUID'] = moluid - hdul['SCI'].header['MOLFRNUM'] = stack_num - hdul['SCI'].header['FRMTOTAL'] = frmtotal - hdul['SCI'].header['STACK'] = 'T' + frame_path = _prepare_stackframe( + src_path, filename, moluid, stack_num=stack_num, + frmtotal=frmtotal, + ) body = json.dumps({ 'fits_file': str(frame_path), 'last_frame': is_last, @@ -530,7 +537,7 @@ def publish_stackframe(moluid, stack_num, frmtotal, is_last, filename): publish_raw_string_to_queue(stack_queue, body) # 1. Publish the poison stack's first frame and wait for it to reduce. - publish_stackframe(poison_moluid, 1, 2, False, 'lsc0m476-sq34-20260121-0101-e00.fits.fz') + publish_stackframe(poison_moluid, 1, 2, False, 'lsc0m476-sq34-20260121-0101-n00.fits.fz') def poison_frame_reduced(): with dbs.get_session(LOCAL_DB_ADDRESS, site_deploy=True) as session: @@ -542,16 +549,16 @@ def poison_frame_reduced(): # Generous timeout: the reduction path makes network calls (WCS/photometry over # VPN) and can stall well past a single frame's typical ~20s under contention. - e09_path = poll_until(poison_frame_reduced, timeout=600, interval=5) - assert e09_path, "Poison stack's first frame never reduced" + n09_path = poll_until(poison_frame_reduced, timeout=600, interval=5) + assert n09_path, "Poison stack's first frame never reduced" - # 2. Corrupt the reduced e09 so finalize cannot open it. - _corrupt_file(e09_path) + # 2. Corrupt the reduced n09 so finalize cannot open it. + _corrupt_file(n09_path) # 3. Complete the poison stack, and publish a healthy sibling in the same window. - publish_stackframe(poison_moluid, 2, 2, True, 'lsc0m476-sq34-20260121-0102-e00.fits.fz') - publish_stackframe(sibling_moluid, 1, 2, False, 'lsc0m476-sq34-20260121-0201-e00.fits.fz') - publish_stackframe(sibling_moluid, 2, 2, True, 'lsc0m476-sq34-20260121-0202-e00.fits.fz') + publish_stackframe(poison_moluid, 2, 2, True, 'lsc0m476-sq34-20260121-0102-n00.fits.fz') + publish_stackframe(sibling_moluid, 1, 2, False, 'lsc0m476-sq34-20260121-0201-n00.fits.fz') + publish_stackframe(sibling_moluid, 2, 2, True, 'lsc0m476-sq34-20260121-0202-n00.fits.fz') # 4. The poison stack exhausts the attempt ladder and lands in 'error'. def poison_errored(): @@ -615,21 +622,18 @@ def restarted(): stack_queue = os.environ.get('STACK_QUEUE_NAME', 'banzai_stack_queue') def publish_stackframe(stack_num, is_last, filename): - frame_path = raw_dir / filename - shutil.copy2(str(src_path), str(frame_path)) - with fits.open(str(frame_path), mode='update') as hdul: - hdul['SCI'].header['MOLUID'] = moluid - hdul['SCI'].header['MOLFRNUM'] = stack_num - hdul['SCI'].header['FRMTOTAL'] = 2 - hdul['SCI'].header['STACK'] = 'T' + frame_path = _prepare_stackframe( + src_path, filename, moluid, stack_num=stack_num, + frmtotal=2, + ) body = json.dumps({ 'fits_file': str(frame_path), 'last_frame': is_last, }) publish_raw_string_to_queue(stack_queue, body) - publish_stackframe(1, False, 'lsc0m476-sq34-20260121-0401-e00.fits.fz') - publish_stackframe(2, True, 'lsc0m476-sq34-20260121-0402-e00.fits.fz') + publish_stackframe(1, False, 'lsc0m476-sq34-20260121-0401-n00.fits.fz') + publish_stackframe(2, True, 'lsc0m476-sq34-20260121-0402-n00.fits.fz') def stack_complete(): with dbs.get_session(LOCAL_DB_ADDRESS, site_deploy=True) as session: diff --git a/banzai/tests/test_frames.py b/banzai/tests/test_frames.py index 906acef71..d79c20466 100644 --- a/banzai/tests/test_frames.py +++ b/banzai/tests/test_frames.py @@ -203,11 +203,11 @@ def test_init_poisson_uncertainties(): def test_get_output_filename(): - test_frame = FakeLCOObservationFrame(file_path='test_image_00.fits') - test_context = FakeContext(frame_class=FakeLCOObservationFrame) + test_frame = FakeLCOObservationFrame(file_path='test_image_n00.fits') + test_context = FakeContext(frame_class=FakeLCOObservationFrame, reduction_level='9') filename = test_frame.get_output_filename(test_context) - assert filename == 'test_image_91.fits.fz' + assert filename == 'test_image_n09.fits.fz' def test_section_transformation(): diff --git a/banzai/tests/test_image_utils.py b/banzai/tests/test_image_utils.py index c06521094..923382491 100644 --- a/banzai/tests/test_image_utils.py +++ b/banzai/tests/test_image_utils.py @@ -6,10 +6,13 @@ pytestmark = pytest.mark.image_utils -def test_image_can_be_processed_known_obstype(): - image = FakeLCOObservationFrame([FakeCCDData(meta={'OBSTYPE': 'BIAS'})]) +def test_sub_exp_image_can_be_processed_with_full_pipeline_settings(): + image = FakeLCOObservationFrame([FakeCCDData(meta={'OBSTYPE': 'SUB_EXP'})]) + context = FakeContext() - assert image_utils.image_can_be_processed(image, FakeContext()) + assert image_utils.image_can_be_processed(image, context) + assert (context.LAST_STAGE['SUB_EXP'], context.EXTRA_STAGES['SUB_EXP']) == (None, None) + assert context.REDUCED_DATA_EXTENSION_ORDERING['SUB_EXP'] == context.REDUCED_DATA_EXTENSION_ORDERING['EXPOSE'] def test_image_cannot_be_processed_unknown_obstype(): diff --git a/banzai/tests/test_smart_stacking.py b/banzai/tests/test_smart_stacking.py index abcab17ae..6b668d77a 100644 --- a/banzai/tests/test_smart_stacking.py +++ b/banzai/tests/test_smart_stacking.py @@ -525,7 +525,7 @@ def _make_fits_header(**overrides): return h @staticmethod - def _make_mock_image(output_dir='/data/processed', output_filename='frame-e09.fits'): + def _make_mock_image(output_dir='/data/processed', output_filename='frame-n09.fits'): """Build a mock image returned by run_pipeline_stages.""" img = MagicMock() img.get_output_directory.return_value = output_dir @@ -549,7 +549,7 @@ def test_process_stackframe_logs_timeout_rejection(self, mock_run_stages, mock_l 'smartstack_moluid': 'mol-xyz', 'smartstack_camera': 'cam1', 'smartstack_stack_num': 1, - 'smartstack_filepath': '/data/processed/frame-e09.fits', + 'smartstack_filepath': '/data/processed/frame-n09.fits', 'smartstack_status': 'timeout'}, ) @@ -584,7 +584,7 @@ def test_process_stackframe_upserts_stackframe_and_does_not_notify_redis( stack_num=1, frmtotal=5, camera='cam1', - filepath='/data/processed/frame-e09.fits', + filepath='/data/processed/frame-n09.fits', is_last=expected_is_last, dateobs=datetime.datetime(2024, 1, 1, 0, 0, 0), ) @@ -614,7 +614,7 @@ def _run_stages(*args, **kwargs): process_stackframe(body, runtime_context) mock_upsert.assert_called_once() - assert mock_upsert.call_args.kwargs['filepath'] == '/data/processed/frame-e09.fits' + assert mock_upsert.call_args.kwargs['filepath'] == '/data/processed/frame-n09.fits' redis_module.Redis.from_url.assert_not_called() @patch('banzai.scheduling.stage_utils.run_pipeline_stages') diff --git a/banzai/tests/test_smartstack_products.py b/banzai/tests/test_smartstack_products.py index fd2351ef4..4be8d3485 100644 --- a/banzai/tests/test_smartstack_products.py +++ b/banzai/tests/test_smartstack_products.py @@ -29,7 +29,7 @@ def open(self, file_info, runtime_context): def stackframe(stack_num, filepath=None): if filepath is None: - filepath = f'/tmp/cpt1m010-fa16-20240706-{stack_num:04d}-e09.fits' + filepath = f'/tmp/cpt1m010-fa16-20240706-{stack_num:04d}-n09.fits' return SimpleNamespace( filepath=filepath, stack_num=stack_num, @@ -37,10 +37,10 @@ def stackframe(stack_num, filepath=None): def make_frame(filename, value=1.0, exptime=10.0, date_obs='2024-07-06T00:00:00.000', molfrnum=1, - frmtotal=3, frame_class=FakeLCOObservationFrame, saturate=65535.0, maxlin=60000.0, + frmtotal=3, obstype='SUB_EXP', frame_class=FakeLCOObservationFrame, saturate=65535.0, maxlin=60000.0, rdnoise=8.0): common_header = { - 'OBSTYPE': 'EXPOSE', + 'OBSTYPE': obstype, 'DATE-OBS': date_obs, 'EXPTIME': exptime, 'PROPID': 'standard', @@ -97,8 +97,8 @@ def combine_images(images, output_image, nsigma, method): def test_build_stacked_frame_structure(monkeypatch): input_images = [ - make_frame('/tmp/cpt1m010-fa16-20240706-0031-e09.fits', value=2.0), - make_frame('/tmp/cpt1m010-fa16-20240706-0032-e09.fits', value=3.0), + make_frame('/tmp/cpt1m010-fa16-20240706-0031-n09.fits', value=2.0), + make_frame('/tmp/cpt1m010-fa16-20240706-0032-n09.fits', value=3.0), ] stackframes = [stackframe(31, '/tmp/31.fits'), stackframe(32, '/tmp/32.fits')] monkeypatch.setattr(products, 'open_stackframe_images', lambda rows, context: input_images) @@ -119,9 +119,9 @@ def test_build_stacked_frame_structure(monkeypatch): def test_metadata(): output_frame = make_frame('/tmp/cpt1m010-fa16-20240706-0031-e45.fits', value=7.0) input_images = [ - make_frame('/tmp/cpt1m010-fa16-20240706-0033-e09.fits', exptime=20.0, + make_frame('/tmp/cpt1m010-fa16-20240706-0033-n09.fits', exptime=20.0, date_obs='2024-07-06T00:01:00.000'), - make_frame('/tmp/cpt1m010-fa16-20240706-0031-e09.fits', exptime=10.0, + make_frame('/tmp/cpt1m010-fa16-20240706-0031-n09.fits', exptime=10.0, date_obs='2024-07-06T00:00:00.000'), ] stackframes = [stackframe(33), stackframe(31)] @@ -141,16 +141,19 @@ def test_metadata(): assert header['DATE-OBS'] == date_utils.date_obs_to_string(input_images[1].dateobs) assert header['NCOMBINE'] == 2 assert header['MOLUID'] == 'mol-1' + assert header['OBSTYPE'] == 'EXPOSE' assert 'DATE' in header assert 'MOLFRNUM' not in header - assert header['IMCOM001'] == 'cpt1m010-fa16-20240706-0031-e09.fits' - assert header['IMCOM002'] == 'cpt1m010-fa16-20240706-0033-e09.fits' + assert header['IMCOM001'] == 'cpt1m010-fa16-20240706-0031-n09.fits' + assert header['IMCOM002'] == 'cpt1m010-fa16-20240706-0033-n09.fits' assert 'Images combined to create smartstack image:' in header['HISTORY'] @pytest.mark.parametrize('file_extension', ['.fits', '.fits.fz']) def test_build_thumbnail_metadata(file_extension): - output_frame = make_frame(f'/tmp/cpt1m010-fa16-20240706-0031-e45{file_extension}', exptime=30.0) + output_frame = make_frame( + f'/tmp/cpt1m010-fa16-20240706-0031-e45{file_extension}', exptime=30.0, obstype='EXPOSE' + ) for header in (output_frame.primary_hdu.meta, output_frame['SCI'].meta): header['DAY-OBS'] = '20240706' header['MOLUID'] = 'mol-1' @@ -182,7 +185,7 @@ def test_build_thumbnail_metadata(file_extension): def test_build_thumbnail_metadata_rejects_missing_required_primary_value(): - output_frame = make_frame('/tmp/cpt1m010-fa16-20240706-0031-e45.fits') + output_frame = make_frame('/tmp/cpt1m010-fa16-20240706-0031-e45.fits', obstype='EXPOSE') for header in (output_frame.primary_hdu.meta, output_frame['SCI'].meta): header['MOLUID'] = 'mol-1' header['NCOMBINE'] = 1 @@ -195,8 +198,8 @@ def test_build_thumbnail_metadata_rejects_missing_required_primary_value(): @pytest.mark.parametrize('file_extension', ['.fits', '.fits.fz']) def test_output_names_stay_fixed_as_stack_grows(monkeypatch, file_extension): images_by_stack_num = { - 1: make_frame(f'/tmp/cpt1m010-fa16-20240706-0031-e09{file_extension}', value=1.0), - 2: make_frame(f'/tmp/cpt1m010-fa16-20240706-0033-e09{file_extension}', value=1.0), + 1: make_frame(f'/tmp/cpt1m010-fa16-20240706-0031-n09{file_extension}', value=1.0), + 2: make_frame(f'/tmp/cpt1m010-fa16-20240706-0033-n09{file_extension}', value=1.0), } monkeypatch.setattr( products, @@ -218,8 +221,8 @@ def test_build_stacked_frame_rejects_empty(): @pytest.mark.parametrize('filename', [ - '/tmp/cpt1m010-fa16-20240706-0031-e08.fits', - '/tmp/cpt1m010-fa16-20240706-0031-e09.txt', + '/tmp/cpt1m010-fa16-20240706-0031-n08.fits', + '/tmp/cpt1m010-fa16-20240706-0031-n09.txt', ]) def test_build_stacked_frame_rejects_invalid_filename(monkeypatch, filename): input_image = make_frame(filename) @@ -231,7 +234,7 @@ def test_build_stacked_frame_rejects_invalid_filename(monkeypatch, filename): @pytest.mark.parametrize('written_suffix', ['', '.fz'], ids=['fits', 'fpack']) def test_run_final_returns_paths_and_writes(monkeypatch, tmp_path, written_suffix): - output_frame = make_frame('/tmp/cpt1m010-fa16-20240706-0031-e45.fits', value=1.0) + output_frame = make_frame('/tmp/cpt1m010-fa16-20240706-0031-e45.fits', value=1.0, obstype='EXPOSE') context = FakeContext(processed_path=str(tmp_path)) output_dir = output_frame.get_output_directory(context) written_filename = output_frame.filename + written_suffix @@ -257,7 +260,7 @@ def test_run_final_returns_paths_and_writes(monkeypatch, tmp_path, written_suffi def test_run_preview_publishes_null_fits(monkeypatch, tmp_path): - output_frame = make_frame('/tmp/cpt1m010-fa16-20240706-0031-e45.fits', value=1.0) + output_frame = make_frame('/tmp/cpt1m010-fa16-20240706-0031-e45.fits', value=1.0, obstype='EXPOSE') for header in (output_frame.primary_hdu.meta, output_frame['SCI'].meta): header['MOLUID'] = 'mol-1' header['NCOMBINE'] = 3 @@ -298,8 +301,8 @@ def test_open_stackframe_images_raises_on_unopenable(): def test_write_and_reopen_smoke(monkeypatch, tmp_path): input_images = [ - make_frame('/tmp/cpt1m010-fa16-20240706-0031-e09.fits', value=2.0, frame_class=LCOObservationFrame), - make_frame('/tmp/cpt1m010-fa16-20240706-0032-e09.fits', value=3.0, frame_class=LCOObservationFrame), + make_frame('/tmp/cpt1m010-fa16-20240706-0031-n09.fits', value=2.0, frame_class=LCOObservationFrame), + make_frame('/tmp/cpt1m010-fa16-20240706-0032-n09.fits', value=3.0, frame_class=LCOObservationFrame), ] stackframes = [stackframe(31, '/tmp/31.fits'), stackframe(32, '/tmp/32.fits')] # Deliberately wrong reduction_level: run_final must stamp RLEVEL 45 regardless of the caller's context. @@ -326,8 +329,8 @@ def test_write_and_reopen_smoke(monkeypatch, tmp_path): assert header['NCOMBINE'] == 2 assert header['MOLUID'] == 'mol-1' assert 'MOLFRNUM' not in header - assert header['IMCOM001'] == 'cpt1m010-fa16-20240706-0031-e09.fits' - assert header['IMCOM002'] == 'cpt1m010-fa16-20240706-0032-e09.fits' + assert header['IMCOM001'] == 'cpt1m010-fa16-20240706-0031-n09.fits' + assert header['IMCOM002'] == 'cpt1m010-fa16-20240706-0032-n09.fits' assert 'IMCOM003' not in header assert header['SATURATE'] == pytest.approx(2.0 * 65535.0) assert header['MAXLIN'] == pytest.approx(2.0 * 60000.0) From aa1660f16050fea55c9f836b8d2555d8a20cab1c Mon Sep 17 00:00:00 2001 From: Tim Beccue Date: Sat, 22 Aug 2026 01:12:20 -0600 Subject: [PATCH 2/4] Document SUB_EXP smartstack frames --- banzai/tests/site_e2e/README.md | 6 ++-- docs/smartstacking_architecture.md | 50 +++++++++++++++--------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/banzai/tests/site_e2e/README.md b/banzai/tests/site_e2e/README.md index 89c566b87..584f98b16 100644 --- a/banzai/tests/site_e2e/README.md +++ b/banzai/tests/site_e2e/README.md @@ -22,9 +22,9 @@ These tests validate the full system by: 5. **Cache Drift Reconciliation** - The download worker restores local cache filepaths when replicated database state drifts. -6. **Smartstack Reduction and Stacking** - Raw stackframes published to the stack - queue are reduced to `e09` using the cached calibrations, then stacked into - `e45` products with JPEG previews and shipper messages. +6. **Smartstack Reduction and Stacking** - The tests cover `n00` -> `n09` -> + `e45`: raw stackframes are reduced using the cached calibrations, then + stacked into final products with JPEG previews and shipper messages. 7. **Replication Slot Reuse** - Cache initialization succeeds when a publisher replication slot already exists. diff --git a/docs/smartstacking_architecture.md b/docs/smartstacking_architecture.md index df277de94..fd73f7c23 100644 --- a/docs/smartstacking_architecture.md +++ b/docs/smartstacking_architecture.md @@ -4,13 +4,15 @@ Smartstacks are exposures made from combining a series of shorter exposures. The benefits of this procedure are reduced tracking error and the ability to provide previews of the stacking progress (currently only jpgs) for realtime users as the stack is being processed. Since smartstacks require lots of files, creating the stacks in AWS risks saturating the bandwidth available to observatories. Therefore, smartstacks are designed to be processed with a banzai instance running directly at site, and only the final smartstack products will be sent to the archive. +With the development of smartstacks, we also introduce a new obstype `SUB_EXP`, which is used for the raw and reduced individual stackframes. Raw `SUB_EXP` frames use the suffix `-n00.fits`, and reduced frames use `-n09.fits`. These are only stored temporarily at site, and never sent to the archive. + ### Smartstack limitations Some important caveats, to help understand when smartstacks might not be a useful alternative to regular exposures. - BANZAI does not align or resample stack members, so sources that move between frames will smear. -- Reproducability is limited. Final e45 products include the frames that were stacked, but these are not sent to the archive and are regularly cleaned from disks at site (after approx. 2 weeks). If the time since e45 creation is less than the site's file retention policy *and* the `STACK_RETENTION_DAYS` param that controls the local db cleanup, stacks can be recreated; otherwise not. -- The combined product's RDNOISE is scaled by sqrt(N) for a stack of N farmes, compared to a single frame with the same total exposure. +- Reproducibility is limited. Only final `e45` products are sent to the archive, while the raw `n00` and reduced `n09` stackframes remain at site and are regularly cleaned from site disks. If the time since e45 creation is less than the site's file retention policy *and* the `STACK_RETENTION_DAYS` param that controls the local banzai db cleanup, stacks can be recreated; otherwise not. +- The combined product's RDNOISE is scaled by sqrt(N) for a stack of N frames, compared to a single frame with the same total exposure. - No auto or manual frame rejection capabilities. Also worth noting: the preview frames are visually stretched, lower-resolution jpg previews that overwrite each other as the stack is updated. They are intended as quick feedback for realtime observers, and not for any sort of data analysis. @@ -21,39 +23,39 @@ This page describes the Smartstack path in `docker-compose-site.yml`. The progra The reduction level suffixes used: -- `e00` is a raw exposure. -- `e09` is an exposure reduced using BANZAI's default ordered reduction steps. -- `e45` is the combined Smartstack product. +- `n00` is a raw stackframe with obstype=SUB_EXP. +- `n09` is a reduced stackframe, following BANZAI's default ordered reduction steps. +- `e45` is the combined Smartstack product, with obstype=EXPOSE. -## The whole path +## Full smartstack process: ```mermaid flowchart LR Site["Site software
(outside this repository)"] Listener["Listener"] Reducer["Reduction worker"] - DB[("PostgreSQL
e09 paths and stack progress")] + DB[("PostgreSQL
n09 paths and stack progress")] Stacker["Stack worker
rebuild preview or final"] Shipper["Shipper
(outside this repository)"] - Files[("Shared files
e00, e09, e45, and JPEGs")] + Files[("Shared files
n00, n09, e45, and JPEGs")] Site -->|"raw path over RabbitMQ"| Listener Listener -->|"Celery task through Redis"| Reducer - Reducer -->|"record successful e09"| DB + Reducer -->|"record successful n09"| DB DB <-->|"read and update"| Stacker Stacker -->|"product paths over RabbitMQ"| Shipper - Site -->|"write e00"| Files - Reducer <-->|"read e00 and write e09"| Files - Stacker <-->|"read e09 and write products"| Files + Site -->|"write n00"| Files + Reducer <-->|"read n00 and write n09"| Files + Stacker <-->|"read n09 and write products"| Files Shipper -->|"open products"| Files ``` -1. Site software writes a raw `e00` FITS file and sends its absolute path to RabbitMQ `banzai_stack_queue`. +1. Site software writes a raw `n00` FITS file and sends its absolute path to RabbitMQ `banzai_stack_queue`. 2. The listener checks the message and publishes a Celery reduction task to Redis. -3. A Celery worker runs the default BANZAI ordered reduction steps for the raw `e00`, and writes an `e09` file as output. By default the reductions use super calibration frames from the central AWS BANZAI instance that have been cached locally. -4. After that succeeds, the worker saves the `e09` path and group information in PostgreSQL. -5. A stacking process checks PostgreSQL about every five seconds. It opens the recorded `e09` files and makes a preview or final product when needed. +3. A Celery worker runs the default BANZAI ordered reduction steps for the raw `n00`, and writes the `n09` file as output. By default the reductions use super calibration frames from the central AWS BANZAI instance that have been cached locally. +4. After that succeeds, the worker saves the `n09` path and stack information in PostgreSQL. +5. A stacking process checks PostgreSQL about every five seconds. It opens the `n09` files and makes a preview or final product when needed. 6. BANZAI sends the product paths through RabbitMQ to the shipper. ### Local calibration cache @@ -75,7 +77,7 @@ The listener checks that the two required message keys exist. It does not check PostgreSQL has two Smartstack tables: - `stacks` has one row for each `MOLUID`. It stores the camera, expected frame count, status, preview count, timeout clock, and final retry information. -- `stackframes` has one row for each `(MOLUID, MOLFRNUM)`. It stores the reduced `e09` path and the information needed to build and send products. +- `stackframes` has one row for each `(MOLUID, MOLFRNUM)`. It stores the reduced `n09` path and the information needed to build and send products. A frame is added only after its reduction succeeds. Failed reductions do not appear in these tables, so a timeout product may contain fewer frames than the producer sent. @@ -101,7 +103,7 @@ The complete check runs before the timeout check. The timeout clock uses the tim The finished states behave differently: - A later successful reduction reopens a `complete` or `error` stack. -- A `timeout` stack never reopens. Later reductions still write their `e09` files, but their database rows are rejected and they are not combined. +- A `timeout` stack never reopens. Later reductions still write their `n09` files, but their database rows are rejected and they are not combined. After `STACK_RETENTION_DAYS`, cleanup removes the `stackframes` rows for old finished stacks. It also removes old `complete` and `error` stack rows. It keeps `timeout` stack rows so late frames continue to be rejected. @@ -109,11 +111,11 @@ Cleanup does not delete FITS or JPEG files; these are regularly cleaned as part ## Products -Every preview and final product is rebuilt from the recorded `e09` files. BANZAI does not keep a partly combined image between checks. +Every preview and final product is rebuilt from the recorded `n09` files. BANZAI does not keep a partly combined image between checks. For each build, BANZAI: -1. Sorts the rows by frame number and opens the `e09` files. +1. Sorts the rows by frame number and opens the `n09` files. 2. Uses the lowest-numbered file as the header and filename template. 3. Combines matching `SCI` pixels in row sections to limit memory use, using a sum with 3-sigma rejection. 4. Updates the FITS information that describes the whole stack and writes an ordered `IMCOMnnn` list of input files. @@ -122,7 +124,7 @@ Masks and uncertainties are combined along with the image data. Smartstack does not align or resample images. The input arrays must already line up and have compatible shapes. Counts are not adjusted for different exposure times; BANZAI only logs a warning when exposure times differ by more than one percent. -Product creation also expects input filenames ending in `-e09.fits` or `-e09.fits.fz`. +Input filenames are expected to end in `-n09.fits` or `-n09.fits.fz`. - **Preview:** waits for logical frame 1 so the filename does not change. It writes a small JPEG of up to 300 pixels and a large JPEG of up to 900 pixels. Each preview replaces the same files. The shipper message has `fits: null` and includes `thumbnail_metadata`. - **Final:** writes an `e45` FITS and replaces the same two JPEGs. A timed-out stack that never received frame 1 uses its lowest-numbered frame instead. @@ -130,7 +132,7 @@ Product creation also expects input filenames ending in `-e09.fits` or `-e09.fit For example: ```text -input: cpt1m010-fa16-20240706-0031-e09.fits +input: cpt1m010-fa16-20240706-0031-n09.fits final: cpt1m010-fa16-20240706-0031-e45.fits JPEGs: cpt1m010-fa16-20240706-0031-e45-small_thumbnail.jpg cpt1m010-fa16-20240706-0031-e45-large_thumbnail.jpg @@ -142,12 +144,10 @@ Products are written under: ////processed/ ``` -The shipper message is plain-text JSON. It contains absolute paths, not file data. When BANZAI publishes each preview or final product, it creates a fresh `instrument_enqueue_timestamp` for that Shipper handoff. The reduction worker, stacking service, and external shipper must see the reduced-data directory at the same absolute path. +The shipper message is plain-text JSON. It contains absolute paths, not file data. When BANZAI publishes each preview or final product, it creates a fresh `instrument_enqueue_timestamp` for that shipper handoff. The reduction worker, stacking service, and external shipper must see the reduced-data directory at the same absolute path. Preview paths are not snapshots. A later preview can replace a JPEG before the shipper opens the path. -This repository proves the BANZAI message format, not the deployed shipper or archive behavior. A `complete` or `timeout` status means RabbitMQ confirmed the final product message. It does not mean the files were uploaded or accepted by the archive. - ## Failure Modes Malformed input and messages with missing keys are logged, acknowledged, and discarded. From 42f73179ccbabf16925426d76550770d38ba5c2e Mon Sep 17 00:00:00 2001 From: Tim Beccue Date: Tue, 25 Aug 2026 15:56:23 -0600 Subject: [PATCH 3/4] Bump version to 1.38.1 --- CHANGES.md | 6 ++++++ pyproject.toml | 2 +- uv.lock | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c46ff17eb..cc27a11c4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,12 @@ Versions ======== +1.38.1 (2026-08-25) +------------------- + +- Expect smartstack stackframes to have obstype=SUB_EXP + (filenames use n00 for raw, n09 for reduced) + 1.38.0 (2026-08-25) ------------------- diff --git a/pyproject.toml b/pyproject.toml index 9bb3a2267..b03e38920 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "lco-banzai" requires-python = ">=3.11,<4" -version = "1.38.0" +version = "1.38.1" description = "Python data reduction package for LCOGT data" authors = [ diff --git a/uv.lock b/uv.lock index 8146c961f..f59c3d91a 100644 --- a/uv.lock +++ b/uv.lock @@ -1176,7 +1176,7 @@ wheels = [ [[package]] name = "lco-banzai" -version = "1.38.0" +version = "1.38.1" source = { editable = "." } dependencies = [ { name = "alembic" }, From f6ea990391c4195d9df0ec36cfb58f3a85716673 Mon Sep 17 00:00:00 2001 From: Tim Beccue Date: Tue, 1 Sep 2026 14:54:59 -0600 Subject: [PATCH 4/4] Remove SINISTRO from obstype last_stage config --- banzai/settings.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/banzai/settings.py b/banzai/settings.py index bf2e10d45..6422b208a 100644 --- a/banzai/settings.py +++ b/banzai/settings.py @@ -49,8 +49,11 @@ 'READNOISE': ['configuration_mode', 'binning']} LAST_STAGE = {'BIAS': 'banzai.trim.Trimmer', - 'DARK': 'banzai.uncertainty.PoissonInitializer', 'SKYFLAT': 'banzai.dark.DarkSubtractor', - 'SINISTRO': 'banzai.mosaic.MosaicCreator', 'STANDARD': None, 'EXPOSE': None, 'SUB_EXP': None, + 'DARK': 'banzai.uncertainty.PoissonInitializer', + 'SKYFLAT': 'banzai.dark.DarkSubtractor', + 'STANDARD': None, + 'EXPOSE': None, + 'SUB_EXP': None, 'EXPERIMENTAL': None} EXTRA_STAGES = {'BIAS': ['banzai.bias.BiasMasterLevelSubtractor', 'banzai.bias.BiasComparer'],