From 5538c9a20c4093f0dddc26e406bb3b2430e579bb Mon Sep 17 00:00:00 2001 From: Jeff Levesque Date: Sun, 20 Sep 2026 21:52:13 -0400 Subject: [PATCH 1/5] #82: archive-links.js, name the archive's files and judge what answers --- jsx/import/general/archive-links.js | 99 +++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 jsx/import/general/archive-links.js diff --git a/jsx/import/general/archive-links.js b/jsx/import/general/archive-links.js new file mode 100644 index 0000000..877c611 --- /dev/null +++ b/jsx/import/general/archive-links.js @@ -0,0 +1,99 @@ +/** + * archive-links.js: which archived performance files a stream might have, and + * which of them are really there. + * + * `/stream//alarm` offers the raw ingest performance metrics as csv, a + * file per year or per month. The list used to be INVENTED: the page counted + * from a start year to today and built a url per step, so every link was a + * guess that an object sat at that path. Thirty-one of the sixty-five it + * generated were guesses that were wrong. + * + * Nothing catches that, because a path with no object behind it does not 404. + * The site answers an unmatched path with the single-page app's shell -- HTTP + * 200, `text/html`, about half a kilobyte -- and the anchors carry `download`, + * so the browser saves that shell under the name it was asked for. A reader + * clicking `2026.csv` got a file called `2026.csv` full of ``. + * + * So this module does two things and the page does the asking between them: + * name the candidates, and judge an answer. What it cannot do is publish a file + * that was never published -- a candidate with nothing behind it is dropped + * from the list, not repaired. + * + * Note: judged on the CONTENT TYPE rather than the status. Every dead path + * answers 200, so a status check passes all of them and changes nothing. + */ + +// +// where each stream's archive lives, how far back it goes, and whether it is +// filed by year or by month. +// +// Note: the two stock market streams are deliberately absent. Nothing is +// published for either -- '/ingest/stockmarket/', '/ingest/article/ +// stockmarket/', '/ingest/article/stocksplit/' and a month-nested variant +// were all tried against the live site and none resolves -- so the page +// has nothing to offer and should say so rather than offer four links +// that download the app's shell. Give one an entry here when its csvs +// start being published. +// +const ARCHIVES = { + bls: { path: 'ingest/article/bls', since: 2024, by: 'year' }, + sec: { path: 'ingest/article/sec', since: 2024, by: 'month' }, + usnationalweather: { path: 'ingest/article/weather', since: 2024, by: 'month' }, +}; + +/** + * every file `stream` might have published, newest first. + * + * Note: the month bound applies to the CURRENT year alone. It used to cap every + * year at the month it happens to be now, so in September the archive hid + * October, November and December of 2024 and 2025 -- six real files, on + * the day this was written, withheld because of the date on the reader's + * clock. + * + * Note: `today` is an argument so a test can state the date rather than work + * around it. The page passes nothing. + */ +export function archiveCandidates(stream, base, today = new Date()) { + const archive = ARCHIVES[String(stream).toLowerCase()]; + + if (!archive || !base) { + return []; + } + + const thisYear = today.getFullYear(); + const thisMonth = today.getMonth() + 1; + const out = []; + + for (let year = thisYear; year >= archive.since; year--) { + if (archive.by === 'year') { + out.push({ + href: `${base}/${archive.path}/${year}.csv`, + label: `${year}.csv`, + }); + continue; + } + + const last = year === thisYear ? thisMonth : 12; + + for (let month = 1; month <= last; month++) { + const mm = String(month).padStart(2, '0'); + + out.push({ + href: `${base}/${archive.path}/${year}/${mm}.csv`, + label: `${mm}/${year}.csv`, + }); + } + } + + return out; +} + +/** + * whether an answer is the file that was asked for, or the app's shell wearing + * its name. + */ +export function published(contentType) { + return !!contentType && !/text\/html/i.test(contentType); +} + +export { ARCHIVES }; From f770f5e4b74c20df4729fd079208c4e9b57cb5fc Mon Sep 17 00:00:00 2001 From: Jeff Levesque Date: Sun, 20 Sep 2026 21:52:14 -0400 Subject: [PATCH 2/5] #82: alarm.jsx, offer the archive files that are there, under their names --- jsx/import/layout/stream/alarm.jsx | 209 +++++++++++++---------------- 1 file changed, 92 insertions(+), 117 deletions(-) diff --git a/jsx/import/layout/stream/alarm.jsx b/jsx/import/layout/stream/alarm.jsx index 8d886a3..90aea3d 100644 --- a/jsx/import/layout/stream/alarm.jsx +++ b/jsx/import/layout/stream/alarm.jsx @@ -35,6 +35,7 @@ import { useParams } from 'react-router-dom'; import { ErrorBoundary } from 'react-error-boundary'; import ErrorFallback from '../../formatter/boundary-error.jsx'; import streamName from '../../general/stream-name.js'; +import { archiveCandidates, published } from '../../general/archive-links.js'; import { datalakeUrl, DATASETS } from '../../general/api-url.js'; class StreamAlarm extends Component { @@ -65,12 +66,19 @@ class StreamAlarm extends Component { expand_archive_stockmarket: false, expand_archive_usnationalweather: false, expand_archive_bls: false, - expand_archive_sec: false + expand_archive_sec: false, + // + // per stream: absent until asked, then the files that really exist. + // See loadArchive -- this list used to be a date loop, and half of + // what it produced was not there. + // + archive: {} } this.callbackGetData = this.callbackGetData.bind(this); this.downloadData = this.downloadData.bind(this); this.handleArchiveClick = this.handleArchiveClick.bind(this); + this.loadArchive = this.loadArchive.bind(this); } componentDidMount() { @@ -97,6 +105,47 @@ class StreamAlarm extends Component { handleArchiveClick(stream=null) { stream = stream ? stream : this.state.stream; this.setState({ [`expand_archive_${stream}`]: ! this.state[`expand_archive_${stream}`] }); + this.loadArchive(stream); + } + + /** + * ask which of a stream's archive files are really published. + * + * On EXPANSION rather than on load, and once per stream. The sublinks sit + * inside a Collapse with `unmountOnExit`, so they do not exist until a + * reader opens that stream -- which is what makes asking affordable: at + * most twenty-seven requests, on a click, for one stream, instead of a + * hundred on every page view. + * + * Note: HEAD, so nothing is downloaded to find out whether it is there. + * + * Note: a request that fails outright is treated as 'not published', the + * same as one answering the app's shell. Either way there is nothing + * to offer, and a link that might work is the thing being removed. + */ + loadArchive(stream) { + const key = String(stream).toLowerCase(); + + if (this.state.archive[key]) { + return; + } + + const candidates = archiveCandidates(key, this.state.performance_link); + + this.setState((state) => ({ archive: { ...state.archive, [key]: [] } })); + + if (!candidates.length) { + return; + } + + Promise.all(candidates.map((file) => fetch(file.href, { method: 'HEAD' }) + .then((answer) => (published(answer.headers.get('content-type')) ? file : null)) + .catch(() => null))) + .then((found) => { + this.setState((state) => ({ + archive: { ...state.archive, [key]: found.filter(Boolean) }, + })); + }); } // @@ -331,7 +380,7 @@ class StreamAlarm extends Component { const notice = ( <> {` - To subscribe to ${this.state.stream} ${term}, + To subscribe to ${streamName(this.state.stream)} ${term}, `} you must accept the terms and conditions. @@ -345,124 +394,50 @@ class StreamAlarm extends Component { var alarm_count = parseInt(this.state.total_source); } - const max_year = new Date().getFullYear(); - - if (stream === 'usnationalweather') { - var min_year = 2024; - var download_prefix = `${this.state.performance_link}/ingest/article/weather`; - } else if (['stockmarket', 'stockmarketstocksplit'].includes(stream)) { - var min_year = 2023; - var download_prefix = `${this.state.performance_link}/ingest/${stream}`; - } else if (stream === 'bls') { - var min_year = 2024; - var download_prefix = [ - `${this.state.performance_link}/ingest/article/bls` - ] - } else if (stream === 'sec') { - var min_year = 2024; - var download_prefix = [ - `${this.state.performance_link}/ingest/article/sec` - ] - } - - const links = []; - if (Array.isArray(download_prefix)) { - for (let index = 0; index < download_prefix.length; index++) { - const item = download_prefix[index]; - const stream = item.split('/').pop(); - const sublinks = []; - - for (let i = max_year; i >= min_year; i--) { - if (['sec', 'weather'].includes(stream)) { - for (let j = 1; j <= this.state.mm; j++) { - const month = (j).toLocaleString( - undefined, - {minimumIntegerDigits: 2} - ); - - sublinks.push( - - - + // + // the archive column: the files this stream really published, which the + // page has asked about -- see loadArchive. Empty until it has, and + // empty for good on a stream that publishes nothing. + // + const key = String(stream).toLowerCase(); + const found = this.state.archive[key]; + const links = [ +
+ { + this.handleArchiveClick(key); + }}> + + {this.state[`expand_archive_${key}`] ? : } + + + + {found && found.length + ? found.map((file) => ( + + + - ); - } - } else { - sublinks.push( - - - - - - ); - } - } - links.push( -
- { - this.handleArchiveClick(stream); - }}> - - {this.state[`expand_archive_${stream}`] ? : } - - - {sublinks} - -
- ); - } - } else { - const stream = download_prefix.split('/').pop() === 'stockmarketstocksplit' - ? 'stocksplit' - : download_prefix.split('/').pop(); - const sublinks = []; - - for (let i = max_year; i >= min_year; i--) { - if (['sec', 'weather'].includes(stream)) { - for (let j = 1; j <= this.state.mm; j++) { - const month = (j).toLocaleString( - undefined, - {minimumIntegerDigits: 2} - ); - - sublinks.push( - - - + )) + : ( + + - - ); - } - } else { - sublinks.push( - - - - - - ); - } - } - - if (sublinks.length > 0) { - links.push( -
- { - this.handleArchiveClick(stream); - }}> - - {this.state[`expand_archive_${stream}`] ? : } - - - {sublinks} - -
- ); - } - } + )} + + +
, + ]; - const archive_text = `Download raw ${this.state.stream} ingest performance metrics`; + const archive_text = `Download raw ${streamName(this.state.stream)} ingest performance metrics`; const tool_tip = ! isMobile ? ( {` - The ${this.state.stream} ingest stream runs ${ingest_interval}. + The ${streamName(this.state.stream)} ingest stream runs ${ingest_interval}. ${ingest_content_1}. `} {isMobile ? null : summary_graphic} From 5b2624bc6429de3c078a153f169e227a8c254e77 Mon Sep 17 00:00:00 2001 From: Jeff Levesque Date: Sun, 20 Sep 2026 21:52:14 -0400 Subject: [PATCH 3/5] #82: archive-links.test.js, pin the candidates and the content-type check --- jsx/__tests__/general/archive-links.test.js | 154 ++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 jsx/__tests__/general/archive-links.test.js diff --git a/jsx/__tests__/general/archive-links.test.js b/jsx/__tests__/general/archive-links.test.js new file mode 100644 index 0000000..62b0d27 --- /dev/null +++ b/jsx/__tests__/general/archive-links.test.js @@ -0,0 +1,154 @@ +/** + * archive-links.test.js: which archived performance files a stream might have. + * + * The page used to count from a start year to today and build a url per step, + * so every link was a guess. Thirty-one of the sixty-five it produced were + * guesses that were wrong, and a wrong one did not 404 -- it downloaded the + * app's shell under the name of the file it was asked for. + * + * So the two halves worth holding here are the shape of the candidates and the + * judgement on an answer. What is deliberately NOT here is the asking: that + * belongs to the page, and has its own cases in alarm.test.jsx. + * + * Note: every case states the date rather than reading the clock. The old + * month bound was the current month applied to every year, which is a bug + * a suite running in September cannot see and one running in January + * cannot miss. + */ + +import { archiveCandidates, published, ARCHIVES } from '../../import/general/archive-links.js'; + +const BASE = 'https://example.com/artifact/performance'; +const SEPTEMBER = new Date(2026, 8, 20); +const JANUARY = new Date(2026, 0, 4); + +const labels = (...args) => archiveCandidates(...args).map((f) => f.label); + +describe('which files a stream might have', () => { + it('offers a file per year for a stream filed by year', () => { + expect(labels('bls', BASE, SEPTEMBER)).toEqual(['2026.csv', '2025.csv', '2024.csv']); + }); + + it('offers a file per month for a stream filed by month', () => { + const sec = labels('sec', BASE, SEPTEMBER); + + expect(sec[0]).toBe('01/2026.csv'); + expect(sec).toContain('09/2026.csv'); + }); + + it('stops the CURRENT year at the current month', () => { + expect(labels('sec', BASE, SEPTEMBER)).not.toContain('10/2026.csv'); + }); + + it('runs a PAST year to december', () => { + // + // the bug this replaces: the month bound was the current month applied + // to every year, so in September the archive hid October, November and + // December of every year before this one -- six real files on the day + // it was found, withheld because of the date on the reader's clock. + // + const sec = labels('sec', BASE, SEPTEMBER); + + ['10/2025.csv', '11/2025.csv', '12/2025.csv'].forEach((file) => { + expect(sec).toContain(file); + }); + }); + + it('does not read the clock for a year it is not in', () => { + // + // the same list, asked for in January, still has all of 2025 + // + expect(labels('sec', BASE, JANUARY)).toContain('12/2025.csv'); + expect(labels('sec', BASE, JANUARY)).not.toContain('02/2026.csv'); + }); + + it('newest first', () => { + expect(labels('bls', BASE, SEPTEMBER)[0]).toBe('2026.csv'); + }); + + it('builds an absolute href under the artifact base', () => { + // + // the yearly links used to be RELATIVE -- href={`${i}.csv`}, with no + // prefix at all -- so on /stream/bls/alarm they resolved to + // /stream/bls/2026.csv and never reached the artifact host. Every bls + // link on the page was pointing at the wrong origin. + // + expect(archiveCandidates('bls', BASE, SEPTEMBER)[0].href) + .toBe(`${BASE}/ingest/article/bls/2026.csv`); + }); + + it('pads a single-digit month in both the href and the label', () => { + const january = archiveCandidates('sec', BASE, SEPTEMBER)[0]; + + expect(january.href).toBe(`${BASE}/ingest/article/sec/2026/01.csv`); + expect(january.label).toBe('01/2026.csv'); + }); + + it('reads a stream id in any casing', () => { + // + // the application links to this page with 'BLS', not 'bls' + // + expect(labels('BLS', BASE, SEPTEMBER)).toEqual(labels('bls', BASE, SEPTEMBER)); + }); + + it.each([ + ['stockmarket'], + ['stockmarketstocksplit'], + ])('offers nothing for %s, which publishes nothing', (stream) => { + // + // four paths were tried against the live site for each and none + // resolves. A stream with no archive should offer no links rather than + // four that download the app's shell. + // + expect(archiveCandidates(stream, BASE, SEPTEMBER)).toEqual([]); + }); + + it('offers nothing for a stream it does not know', () => { + expect(archiveCandidates('no-such-stream', BASE, SEPTEMBER)).toEqual([]); + }); + + it('offers nothing without somewhere to look', () => { + expect(archiveCandidates('bls', '', SEPTEMBER)).toEqual([]); + expect(archiveCandidates('bls', undefined, SEPTEMBER)).toEqual([]); + }); + + it('survives a stream that is not a string', () => { + expect(archiveCandidates(undefined, BASE, SEPTEMBER)).toEqual([]); + expect(archiveCandidates(null, BASE, SEPTEMBER)).toEqual([]); + }); + + it('starts each stream at its own first year', () => { + Object.keys(ARCHIVES).forEach((stream) => { + const oldest = labels(stream, BASE, SEPTEMBER).pop(); + + expect(oldest).toContain(String(ARCHIVES[stream].since)); + }); + }); +}); + +describe('whether an answer is the file or the app wearing its name', () => { + it('accepts what the archive actually serves', () => { + expect(published('binary/octet-stream')).toBe(true); + }); + + it('accepts a csv served as one', () => { + expect(published('text/csv')).toBe(true); + }); + + it('rejects the app shell', () => { + // + // the whole point. A path with nothing behind it answers 200 and + // text/html, and `download` saves that under the name asked for -- so + // the judgement cannot be on the status, which is 200 either way. + // + expect(published('text/html')).toBe(false); + expect(published('text/html; charset=utf-8')).toBe(false); + expect(published('TEXT/HTML')).toBe(false); + }); + + it('rejects an answer that carries no type at all', () => { + expect(published(null)).toBe(false); + expect(published(undefined)).toBe(false); + expect(published('')).toBe(false); + }); +}); From 368dbb901466c0c0dc17833679b7bfaf9bd0a4f4 Mon Sep 17 00:00:00 2001 From: Jeff Levesque Date: Sun, 20 Sep 2026 21:52:14 -0400 Subject: [PATCH 4/5] #82: alarm.test.jsx, the archive is asked for, and the page no longer crashes --- jsx/__tests__/layout/stream/alarm.test.jsx | 283 ++++++++++++++------- 1 file changed, 188 insertions(+), 95 deletions(-) diff --git a/jsx/__tests__/layout/stream/alarm.test.jsx b/jsx/__tests__/layout/stream/alarm.test.jsx index 0db544d..b6a400c 100644 --- a/jsx/__tests__/layout/stream/alarm.test.jsx +++ b/jsx/__tests__/layout/stream/alarm.test.jsx @@ -147,6 +147,18 @@ describe('every stream id the application links to', () => { // exactly the ids layout/stream/stream.jsx puts in the url. There is no // sixth stream; this is the complete set of links to this page. // + // These used to CRASH. The archive column read a `download_prefix` that no + // branch assigned for a capitalised id, `.split()` threw inside the same + // render() that would have created this page's ErrorBoundary -- so the + // boundary never mounted, the error escaped to the one in layout/page.jsx, + // and the whole site went down, navigation included. Every link from + // /stream to an alarm page did this. + // + // The column no longer reads a prefix at all: it lower-cases the id, asks + // what that stream publishes, and offers nothing when the answer is + // nothing. A stream it does not recognise is the same case as one that + // publishes nothing, which is why an unknown id renders too. + // const LINKED = [ 'StockMarket', 'StockMarketStockSplit', @@ -155,51 +167,35 @@ describe('every stream id the application links to', () => { 'SEC', ]; - it.each(LINKED)('/stream/%s/alarm throws during render', (stream) => { - const error = crashFrom(stream); - - expect(error).toBeInstanceOf(TypeError); - expect(error.message).toMatch(/split/); + it.each(LINKED)('/stream/%s/alarm renders', (stream) => { + expect(crashFrom(stream)).toBeNull(); }); - it('fails on the download prefix, which no branch assigned', () => { + it('renders the same page whatever the casing', () => { // - // the specific failure, so a change to the surrounding code that moves - // the crash somewhere else does not quietly keep this test passing. + // the crux of the old defect: same page, same route, same stream, and + // the only difference was the case of the url segment. // - expect(crashFrom('StockMarket').message) - .toMatch(/Cannot read propert.* of undefined \(reading 'split'\)/); - }); - - it('is a casing problem and nothing else', () => { - // - // the crux. Same page, same route, same stream -- the only difference is - // the case of the url segment. - // - expect(crashFrom('StockMarket')).toBeInstanceOf(TypeError); + expect(crashFrom('StockMarket')).toBeNull(); expect(crashFrom('stockmarket')).toBeNull(); }); - it('takes the page down rather than showing the error fallback', () => { + it('renders for a stream that does not exist', () => { // - // alarm.jsx wraps its output in an ErrorBoundary, but the throw happens - // in the same render() that would have created it, so the boundary is - // never mounted and cannot catch its own parent. The error escapes to - // whatever boundary is above -- in the running app that is the one in - // layout/page.jsx, which replaces the ENTIRE page, navigation included. + // indistinguishable from a mis-cased known one, and it should be: a + // stream with no archive is a stream with no archive. // - crashFrom('StockMarket'); - - expect(screen.queryByRole('alert')).not.toBeInTheDocument(); - expect(screen.queryByText('Something went wrong:')).not.toBeInTheDocument(); + expect(crashFrom('no-such-stream')).toBeNull(); }); - it('also throws for a stream that does not exist', () => { + it('shows the archive heading rather than taking the page down', () => { // - // same root cause: no else branch. An unknown stream is indistinguishable - // from a mis-cased known one. + // the old failure replaced the ENTIRE page. This asserts the opposite + // of what it used to: the page is here. // - expect(crashFrom('no-such-stream')).toBeInstanceOf(TypeError); + renderAlarm('StockMarket'); + + expect(screen.getByText('Latest Archive')).toBeInTheDocument(); }); }); @@ -465,139 +461,236 @@ describe('the ticker count arriving from the worker', () => { }); describe('the archive list', () => { + // + // the list is no longer invented from a date loop. The page asks which + // files a stream really published -- HEAD per candidate, on expansion -- + // and offers the ones that answered as a file. + // + // A missing object does NOT 404 here: the site answers an unmatched path + // with the app's shell, 200 and text/html, and the anchors carry + // `download`, so a dead link used to save half a kilobyte of markup under + // the name `2026.csv`. That is why these answer with a content type and why + // the judgement is on the type rather than the status. + // + const CSV = 'binary/octet-stream'; + const SHELL = 'text/html'; + + function answering(typeFor) { + global.fetch = jest.fn((url) => Promise.resolve({ + headers: { get: () => typeFor(String(url)) }, + })); + + return global.fetch; + } + function archiveToggle() { // - // the collapsed row is the only ListItemButton rendered before expansion. + // the collapsed row is the only ListItemButton rendered before expansion // return document.querySelector('.left-column .MuiListItemButton-root'); } + const offered = () => [...document.querySelectorAll('.left-column a[download]')] + .map((a) => a.textContent); + + afterEach(() => { + delete global.fetch; + }); + it('is collapsed until it is clicked', () => { + answering(() => CSV); renderAlarm('bls'); expect(screen.queryByText(`${THIS_YEAR}.csv`)).not.toBeInTheDocument(); }); - it('expands to a csv per year when clicked', async () => { + it('asks nothing until a reader expands it', () => { + // + // the whole reason asking is affordable: it costs a click, not a page + // view. + // + const fetcher = answering(() => CSV); + renderAlarm('bls'); + + expect(fetcher).not.toHaveBeenCalled(); + }); + + it('asks with HEAD, so nothing is downloaded to find out', async () => { + const fetcher = answering(() => CSV); renderAlarm('bls'); await userEvent.click(archiveToggle()); + // the HEAD answers land after the click, so let them settle + await act(async () => {}); - expect(screen.getByText(`${THIS_YEAR}.csv`)).toBeInTheDocument(); - expect(screen.getByText('2024.csv')).toBeInTheDocument(); + expect(fetcher.mock.calls.every(([, init]) => init.method === 'HEAD')).toBe(true); }); - it('collapses again on a second click', async () => { + it('offers a file that is published', async () => { + answering(() => CSV); renderAlarm('bls'); await userEvent.click(archiveToggle()); - expect(screen.getByText('2024.csv')).toBeInTheDocument(); + // the HEAD answers land after the click, so let them settle + await act(async () => {}); + + expect(offered()).toContain(`${THIS_YEAR}.csv`); + }); + + it('does not offer one that answers with the app shell', async () => { + // + // the case the old list got wrong thirty-one times over + // + answering((url) => (url.includes('2024') ? CSV : SHELL)); + renderAlarm('bls'); await userEvent.click(archiveToggle()); + // the HEAD answers land after the click, so let them settle + await act(async () => {}); - expect(screen.queryByText('2024.csv')).not.toBeInTheDocument(); + expect(offered()).toEqual(['2024.csv']); }); - it('counts back to the stream\'s own first year', async () => { + it('judges the content type rather than the status', async () => { // - // bls and sec start in 2024, the stock streams in 2023. The list is - // built from the current year down, so it grows by one row every January. + // every dead path answers 200, so a status check would pass all of them + // and change nothing at all. // + answering(() => SHELL); renderAlarm('bls'); await userEvent.click(archiveToggle()); + // the HEAD answers land after the click, so let them settle + await act(async () => {}); - const years = []; - for (let year = THIS_YEAR; year >= 2024; year--) { - years.push(`${year}.csv`); - } - - years.forEach(label => expect(screen.getByText(label)).toBeInTheDocument()); - expect(screen.queryByText('2023.csv')).not.toBeInTheDocument(); + expect(offered()).toEqual([]); + expect(screen.getByText('Nothing published yet')).toBeInTheDocument(); }); - it('goes back to 2023 for the stock-market stream', async () => { - renderAlarm('stockmarket'); + it('drops a candidate whose request fails outright', async () => { + global.fetch = jest.fn(() => Promise.reject(new Error('offline'))); + renderAlarm('bls'); await userEvent.click(archiveToggle()); + // the HEAD answers land after the click, so let them settle + await act(async () => {}); - expect(screen.getByText('2023.csv')).toBeInTheDocument(); - expect(screen.queryByText('2022.csv')).not.toBeInTheDocument(); + expect(offered()).toEqual([]); }); it('breaks the year down by month for sec', async () => { - // - // sec and weather publish monthly rather than yearly, so their rows are - // 'MM/YYYY.csv' and run to the CURRENT month only. - // + answering(() => CSV); renderAlarm('sec'); await userEvent.click(archiveToggle()); + // the HEAD answers land after the click, so let them settle + await act(async () => {}); - const month = String(new Date().getMonth() + 1).padStart(2, '0'); - expect(screen.getByText(`01/${THIS_YEAR}.csv`)).toBeInTheDocument(); - expect(screen.getByText(`${month}/${THIS_YEAR}.csv`)).toBeInTheDocument(); + expect(offered()).toContain(`01/${THIS_YEAR}.csv`); }); - it.each([ - ['bls', 'bls'], - ['sec', 'sec'], - ['stockmarket', 'stockmarket'], - ['stockmarketstocksplit', 'stocksplit'], - ['usnationalweather', 'weather'], - ])('%s labels its archive row "%s"', (stream, label) => { + it('offers a past year in full, not truncated at this month', async () => { // - // the label is the last path segment of the artifact prefix, not the - // stream id -- which is why usnationalweather reads 'weather' and the - // split stream is shortened back to 'stocksplit'. + // the month bound is the CURRENT month and used to cap every year, so + // in September the archive hid October to December of 2024 and 2025 -- + // real files, withheld because of the date on the reader's clock. // - renderAlarm(stream); + answering(() => CSV); + renderAlarm('sec'); + + await userEvent.click(archiveToggle()); + // the HEAD answers land after the click, so let them settle + await act(async () => {}); - expect(within(document.querySelector('.left-column')).getByText(label)) - .toBeInTheDocument(); + expect(offered()).toContain(`12/${THIS_YEAR - 1}.csv`); + expect(offered()).not.toContain(`12/${THIS_YEAR}.csv`); }); - it('toggles a state key the constructor never declared', async () => { + it('offers nothing for a stream that publishes nothing', async () => { // - // WORTH KNOWING: the constructor seeds expand_archive_usnationalweather, - // but the key actually toggled is built from the archive LABEL, so this - // stream uses expand_archive_weather and the declared field is dead. It - // works only because toggling an undefined field with ! yields true. + // both stock market streams. Nothing is published for either, so the + // page says so rather than offering four links to the app's shell. // - renderAlarm('usnationalweather'); + const fetcher = answering(() => CSV); + renderAlarm('stockmarket'); await userEvent.click(archiveToggle()); + // the HEAD answers land after the click, so let them settle + await act(async () => {}); - const month = String(new Date().getMonth() + 1).padStart(2, '0'); - expect(screen.getByText(`${month}/${THIS_YEAR}.csv`)).toBeInTheDocument(); + expect(fetcher).not.toHaveBeenCalled(); + expect(screen.getByText('Nothing published yet')).toBeInTheDocument(); }); - it('puts the react key on the wrong element', async () => { - // - // DEFECT, and the reason renderAlarm() has to filter console output: the - // repeated element is the wrapper, but the key is set on the - // ListItemButton nested INSIDE it, so React sees an array of unkeyed - // anchors and warns. + it('collapses again on a second click', async () => { + answering(() => CSV); + renderAlarm('bls'); + + await userEvent.click(archiveToggle()); + // the HEAD answers land after the click, so let them settle + await act(async () => {}); + expect(offered().length).toBeGreaterThan(0); + + await userEvent.click(archiveToggle()); + // the HEAD answers land after the click, so let them settle + await act(async () => {}); + + expect(offered()).toEqual([]); + }); + + it('asks once, not again on every expansion', async () => { + const fetcher = answering(() => CSV); + renderAlarm('bls'); + + await userEvent.click(archiveToggle()); + // the HEAD answers land after the click, so let them settle + await act(async () => {}); + const first = fetcher.mock.calls.length; + + await userEvent.click(archiveToggle()); + // the HEAD answers land after the click, so let them settle + await act(async () => {}); + await userEvent.click(archiveToggle()); + // the HEAD answers land after the click, so let them settle + await act(async () => {}); + + expect(fetcher.mock.calls.length).toBe(first); + }); + + it.each([ + ['bls', 'Bureau of Labor Statistics'], + ['sec', 'SEC Filings'], + ['stockmarket', 'S&P 500'], + ['stockmarketstocksplit', 'Stock Splits'], + ['usnationalweather', 'US Weather Alerts'], + ])('labels the %s row with its name, not its id', (stream, label) => { // - // The warning itself is not asserted here. React deduplicates it per - // owner component, so it appears exactly once per module registry -- - // whichever test renders this page first absorbs it, which would make an - // assertion on it depend on test order. The structure that causes it is - // stable, so that is what gets pinned. + // stream-name.js exists to keep identifiers out of the page and carries + // the reasoning for each of these. This column printed the raw id. // + answering(() => CSV); + renderAlarm(stream); + + expect(screen.getByText(label)).toBeInTheDocument(); + }); + + it('puts the react key on the anchor it repeats', async () => { + answering(() => CSV); renderAlarm('bls'); await userEvent.click(archiveToggle()); + // the HEAD answers land after the click, so let them settle + await act(async () => {}); - const anchors = document.querySelectorAll('.left-column .MuiCollapse-root a'); - + const anchors = [...document.querySelectorAll('.left-column a[download]')]; expect(anchors.length).toBeGreaterThan(0); - anchors.forEach(anchor => { + anchors.forEach((anchor) => { expect(anchor.querySelector('.MuiListItemButton-root')).toBeInTheDocument(); }); }); }); + describe('the archive help tooltip', () => { it('is offered on a desktop viewport', () => { renderAlarm('bls'); From 6214e19f940b99e7306a1b23b4808300d82b17e8 Mon Sep 17 00:00:00 2001 From: Jeff Levesque Date: Sun, 20 Sep 2026 21:52:14 -0400 Subject: [PATCH 5/5] #82: performance.md, document the csv archive behind the endpoint --- documentation/api/performance.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/documentation/api/performance.md b/documentation/api/performance.md index 5401292..70bf1c7 100644 --- a/documentation/api/performance.md +++ b/documentation/api/performance.md @@ -38,6 +38,38 @@ It stacks each source's successes by bucket for the chart, and computes the stre health and ingest coverage from the same rows. See [Ingest coverage](../application/ingest-coverage.md). +## The archive behind it + +The same measurement is also published as static csv, a file per year or per +month, under `https://www.jefflevesque.com/artifact/performance/ingest/`. The +alarm page for each stream links to them in its *Latest Archive* column. + +The two are not copies of each other, and each holds what the other cannot: + +| | the archive | this endpoint | +|---|---|---| +| a row is | one ingest event | one bucket, summarised | +| columns | `group_by`, `window_start`, `total_success`, `total_fail`, `window_every` | the first four, plus `_mean` and `_max` for each total | +| window | a whole year, historical | trails from now | + +So this endpoint cannot answer for March 2024, and the archive cannot answer for +this morning. + +| Stream | Where | Filed | +|---|---|---| +| `bls` | `ingest/article/bls/.csv` | by year, from 2024 | +| `sec` | `ingest/article/sec//.csv` | by month, from 2024 | +| `usnationalweather` | `ingest/article/weather//.csv` | by month, from 2024 | +| `stockmarket`, `stockmarketstocksplit` | -- | nothing published yet | + +Not every file in that range exists. A path with nothing behind it does **not** +answer 404: the site serves the single-page app's shell for any unmatched path, +with a 200 and `content-type: text/html`. So a reader saving one of these +programmatically should judge the **content type**, not the status -- a status +check accepts every miss, and the file lands on disk as html under a `.csv` +name. The alarm page checks each candidate this way before offering it, in +[`jsx/import/general/archive-links.js`](https://github.com/jeff1evesque/jefflevesque.com/blob/master/jsx/import/general/archive-links.js). + ## Errors | Status | `report` |