From 6ad0d56e503340e87c93a33be2112a9d07f7d5a6 Mon Sep 17 00:00:00 2001 From: lucasrodes Date: Wed, 20 May 2026 18:22:10 +0200 Subject: [PATCH 01/11] =?UTF-8?q?=F0=9F=8E=89=F0=9F=A4=96=20Add=20Federico?= =?UTF-8?q?-Tena=20V2=20population=20dataset=20(1991=20borders,=202026)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Snapshot + meadow + garden steps for the published Federico-Tena World Population Historical Database (V2.0, DOI 10.21950/GW7SOZ), which supersedes the 2024-01-19 working-paper Excel. Annual estimates for all polities 1800-1938 at 1991 borders. Co-Authored-By: Claude Opus 4.7 (1M context) --- dag/demography.yml | 6 ++ .../federico_tena_population.countries.json | 1 + .../federico_tena_population.meta.yml | 31 ++++++ .../2026-05-20/federico_tena_population.py | 34 +++++++ .../2026-05-20/federico_tena_population.py | 96 +++++++++++++++++++ .../2026-05-20/federico_tena_population.py | 24 +++++ .../federico_tena_population.tab.dvc | 32 +++++++ 7 files changed, 224 insertions(+) create mode 100644 etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.countries.json create mode 100644 etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.meta.yml create mode 100644 etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.py create mode 100644 etl/steps/data/meadow/ggdc/2026-05-20/federico_tena_population.py create mode 100644 snapshots/ggdc/2026-05-20/federico_tena_population.py create mode 100644 snapshots/ggdc/2026-05-20/federico_tena_population.tab.dvc diff --git a/dag/demography.yml b/dag/demography.yml index 6d0ae11ef4c..fde6b3f2a52 100644 --- a/dag/demography.yml +++ b/dag/demography.yml @@ -130,6 +130,12 @@ steps: - data://meadow/ggdc/2024-01-19/maddison_federico_paper: - snapshot://ggdc/2024-01-19/maddison_federico_paper.xlsx + # Federico–Tena World Population Historical Database — V2 (1991 borders, 2026 update). + # Published Dataverse release that supersedes the 2024-01-19 working-paper Excel. + data://garden/ggdc/2026-05-20/federico_tena_population: + - data://meadow/ggdc/2026-05-20/federico_tena_population: + - snapshot://ggdc/2026-05-20/federico_tena_population.tab + # # Population of the Republic of Ireland territory between 1841 and 2006 (Central Statistics Office of Ireland) # diff --git a/etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.countries.json b/etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.countries.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.countries.json @@ -0,0 +1 @@ +{} diff --git a/etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.meta.yml b/etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.meta.yml new file mode 100644 index 00000000000..2b373eece28 --- /dev/null +++ b/etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.meta.yml @@ -0,0 +1,31 @@ +# NOTE: To learn more about the fields, hover over their names. +definitions: + common: + presentation: + topic_tags: + - Population Growth + +dataset: + update_period_days: 365 + +tables: + federico_tena_population: + variables: + population: + title: Population + unit: people + short_unit: "" + description_short: |- + Estimated population, harmonized to 1991 political borders, as compiled by Federico and Tena-Junguito from first-hand sources and country-specific literature. + display: + numDecimalPlaces: 0 + continent: + title: Continent (source label) + unit: "" + description_short: |- + Continent label as recorded in the Federico–Tena source spreadsheet. + historical_name: + title: Historical name (source label) + unit: "" + description_short: |- + Historical or alternative polity name as recorded in the Federico–Tena source spreadsheet, where different from the 1991-borders country name. diff --git a/etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.py b/etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.py new file mode 100644 index 00000000000..7f2bb304646 --- /dev/null +++ b/etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.py @@ -0,0 +1,34 @@ +"""Harmonize country names in the Federico–Tena V2 (1991 borders) population table.""" + +from etl.helpers import PathFinder + +paths = PathFinder(__file__) + + +def run() -> None: + # + # Load inputs. + # + ds_meadow = paths.load_dataset("federico_tena_population") + tb = ds_meadow.read("federico_tena_population") + + # + # Process data. + # + # Harmonize country names against the OWID standard list. Unmapped entries are + # written to federico_tena_population.countries.json on first run for review. + tb = paths.regions.harmonize_names( + tb=tb, + country_col="country", + countries_file=paths.country_mapping_path, + ) + + # We keep the source-side continent label and historical name as auxiliary + # columns so downstream steps can inspect them, but they are not indicators. + tb = tb.format(["country", "year"], short_name=paths.short_name) + + # + # Save outputs. + # + ds_garden = paths.create_dataset(tables=[tb], default_metadata=ds_meadow.metadata) + ds_garden.save() diff --git a/etl/steps/data/meadow/ggdc/2026-05-20/federico_tena_population.py b/etl/steps/data/meadow/ggdc/2026-05-20/federico_tena_population.py new file mode 100644 index 00000000000..d02b623695e --- /dev/null +++ b/etl/steps/data/meadow/ggdc/2026-05-20/federico_tena_population.py @@ -0,0 +1,96 @@ +"""Load the Federico–Tena V2 (1991 borders) tab snapshot and reshape it to a long table. + +Source file layout (tab-separated): + Row 0: continent labels per column ("POPULATION" | "AFRICA" | ... | "BG" | "AMERICA" | ...) + Row 1: country names at 1991 borders ("Borders 1991" | "Burundi" | ...) + Row 2: alternative / historical names ("(000's omitted)" | "" | "French Somalia" | ...) + Rows 3..141: year (1800–1938) and population values in thousands + Rows 142+: blank/footer rows ("WORD COUNTRY Nº" enumeration) + +The continent label "BG" marks blank separator columns between continents — they are dropped. +""" + +import owid.catalog.processing as pr +from owid.catalog import Table + +from etl.helpers import PathFinder + +paths = PathFinder(__file__) + +# Continent labels that separate empty buffer columns in the source spreadsheet. +SEPARATOR_LABEL = "BG" + +# Expected year range of the data block. +YEAR_MIN = 1800 +YEAR_MAX = 1938 + + +def run() -> None: + # + # Load inputs. + # + snap = paths.load_snapshot("federico_tena_population.tab") + + # Read the whole sheet as raw text — the first 3 rows are a multi-row header. + tb_raw = snap.read_csv(sep="\t", header=None, dtype=str, na_values=[""], keep_default_na=False) + + # + # Process data. + # + tb = _reshape_to_long(tb_raw) + + # Values are reported in thousands of people; convert to absolute count. + tb["population"] = (pr.to_numeric(tb["population"], errors="raise") * 1000).round().astype("int64") + + # Sanity checks. + assert tb["year"].min() == YEAR_MIN, f"unexpected min year: {tb['year'].min()}" + assert tb["year"].max() == YEAR_MAX, f"unexpected max year: {tb['year'].max()}" + assert tb["population"].ge(0).all(), "negative population values" + assert tb["country"].notna().all(), "missing country labels" + + tables = [tb.format(["country", "year"], short_name=paths.short_name)] + + # + # Save outputs. + # + ds_meadow = paths.create_dataset(tables=tables, default_metadata=snap.metadata) + ds_meadow.save() + + +def _reshape_to_long(tb_raw: Table) -> Table: + """Pivot the wide source layout into (country, year, population, continent).""" + # Header rows. + continents = tb_raw.iloc[0].tolist() + countries_1991 = tb_raw.iloc[1].tolist() + historical_names = tb_raw.iloc[2].tolist() + + # Data block: rows where the first column parses as a 4-digit year. + body = tb_raw.iloc[3:].copy() + body = body[body[0].astype(str).str.match(r"^\d{4}$", na=False)] + body[0] = body[0].astype(int) + body = body[body[0].between(YEAR_MIN, YEAR_MAX)] + + # Identify country columns (skip column 0 = year, and skip continent separators). + country_cols = [] + for i in range(1, tb_raw.shape[1]): + if continents[i] == SEPARATOR_LABEL: + continue + if not countries_1991[i] or countries_1991[i] == "": + continue + country_cols.append(i) + + # Melt to long form. + body = body.rename(columns={0: "year"}) + long = body.melt(id_vars="year", value_vars=country_cols, var_name="col_idx", value_name="population") + + # Map the column index to its country and continent labels. + long["country"] = long["col_idx"].map(lambda i: countries_1991[i]) + long["continent"] = long["col_idx"].map(lambda i: continents[i]) + long["historical_name"] = long["col_idx"].map(lambda i: historical_names[i] or None) + long = long.drop(columns=["col_idx"]) + + # Drop rows with no observation (Dataverse exports empty cells as ""). + long = long[long["population"].astype(str).str.len() > 0] + long = long.dropna(subset=["population"]) + + return long[["country", "year", "population", "continent", "historical_name"]] diff --git a/snapshots/ggdc/2026-05-20/federico_tena_population.py b/snapshots/ggdc/2026-05-20/federico_tena_population.py new file mode 100644 index 00000000000..767d7ccfb86 --- /dev/null +++ b/snapshots/ggdc/2026-05-20/federico_tena_population.py @@ -0,0 +1,24 @@ +"""Script to create a snapshot of the Federico–Tena World Population Historical Database (V2, 1991 borders).""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +def main(upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"ggdc/{SNAPSHOT_VERSION}/federico_tena_population.tab") + + # Download data from source (url_download in the .dvc), add file to DVC and upload to S3. + snap.create_snapshot(upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/ggdc/2026-05-20/federico_tena_population.tab.dvc b/snapshots/ggdc/2026-05-20/federico_tena_population.tab.dvc new file mode 100644 index 00000000000..ff8d22b81f5 --- /dev/null +++ b/snapshots/ggdc/2026-05-20/federico_tena_population.tab.dvc @@ -0,0 +1,32 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: World Population Historical Database (1991 borders) + description: |- + Annual population estimates for all polities from 1800 to 1938, harmonized to 1991 political borders. The Federico–Tena World Population Historical Database is the result of re-estimating historical population series using first-hand sources and country-specific literature; the 2026 update revises the Africa series using Patrick Manning's African Population Database 1850–1960 and other regional sources. Values are reported in thousands of people. + date_published: "2026-01-26" + version_producer: "V2 (2026 update, 1991 borders)" + + # Citation + producer: Federico and Tena-Junguito + citation_full: |- + Federico, Giovanni and Antonio Tena-Junguito (2025). Federico–Tena World Population Historical Database: World Population borders 1991 (2026), V2. doi:10.21950/GW7SOZ. Building on Federico, G. and Tena-Junguito, A. (2023). "How many people on Earth? World Population 1800-1938". Working Papers in Economic History 23-02, Instituto Figuerola, Universidad Carlos III de Madrid. + attribution: Federico–Tena (2026) + attribution_short: Federico–Tena + + # Files + url_main: https://www.uc3m.es/research/federico-tena-population + url_download: https://edatos.consorciomadrono.es/api/access/datafile/51077 + date_accessed: "2026-05-20" + + # License + license: + name: CC BY 4.0 + url: https://creativecommons.org/licenses/by/4.0/ + +outs: + - md5: 9d635a855627e9619371c42306068e8d + size: 673520 + path: federico_tena_population.tab From 41ecd4916f933c18cefc424972d56407fa1dbbc7 Mon Sep 17 00:00:00 2001 From: lucasrodes Date: Wed, 20 May 2026 18:23:40 +0200 Subject: [PATCH 02/11] =?UTF-8?q?=F0=9F=8E=89=F0=9F=A4=96=20Add=20Federico?= =?UTF-8?q?-Tena=20Quality=20Assessment=20snapshot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion file to the population dataset: per-(country, year) reliability classes A-E from 1800-1938 (DOI 10.21950/U6AANV). Snapshot only for now; meadow/garden integration to follow. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../federico_tena_population_quality.py | 24 ++++++++++++++ .../federico_tena_population_quality.xlsx.dvc | 32 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 snapshots/ggdc/2026-05-20/federico_tena_population_quality.py create mode 100644 snapshots/ggdc/2026-05-20/federico_tena_population_quality.xlsx.dvc diff --git a/snapshots/ggdc/2026-05-20/federico_tena_population_quality.py b/snapshots/ggdc/2026-05-20/federico_tena_population_quality.py new file mode 100644 index 00000000000..c039e8a51dd --- /dev/null +++ b/snapshots/ggdc/2026-05-20/federico_tena_population_quality.py @@ -0,0 +1,24 @@ +"""Script to create a snapshot of the Federico–Tena Quality Assessment file (2025, V2).""" + +from pathlib import Path + +import click + +from etl.snapshot import Snapshot + +# Version for current snapshot dataset. +SNAPSHOT_VERSION = Path(__file__).parent.name + + +@click.command() +@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") +def main(upload: bool) -> None: + # Create a new snapshot. + snap = Snapshot(f"ggdc/{SNAPSHOT_VERSION}/federico_tena_population_quality.xlsx") + + # Download data from source (url_download in the .dvc), add file to DVC and upload to S3. + snap.create_snapshot(upload=upload) + + +if __name__ == "__main__": + main() diff --git a/snapshots/ggdc/2026-05-20/federico_tena_population_quality.xlsx.dvc b/snapshots/ggdc/2026-05-20/federico_tena_population_quality.xlsx.dvc new file mode 100644 index 00000000000..0476941ce77 --- /dev/null +++ b/snapshots/ggdc/2026-05-20/federico_tena_population_quality.xlsx.dvc @@ -0,0 +1,32 @@ +# Learn more at: +# http://docs.owid.io/projects/etl/architecture/metadata/reference/ +meta: + origin: + # Data product / Snapshot + title: World Population Historical Database — Quality Assessment + description: |- + Companion quality-assessment file to the Federico–Tena World Population Historical Database. Classifies each (polity, year) cell from 1800 to 1938 into five reliability classes: A (fully trustworthy estimate), B (interpolation between censuses), C (colonial statistics or estimates), D (interpolation of C), and E (lowest reliability). The country list in this file is documented at 1938 historical borders, so it does not align one-to-one with the 1991-borders companion dataset. + date_published: "2025-01-28" + version_producer: "V2 (2025, 1938 borders)" + + # Citation + producer: Federico and Tena-Junguito + citation_full: |- + Federico, Giovanni and Antonio Tena-Junguito (2025). Federico–Tena World Population Historical Database: Quality Assessment Population (2025), V2. doi:10.21950/U6AANV. Universidad Carlos III de Madrid – e-Datos Consorcio Madroño Dataverse. + attribution: Federico–Tena (2025) — Quality Assessment + attribution_short: Federico–Tena QA + + # Files + url_main: https://www.uc3m.es/research/federico-tena-population + url_download: https://edatos.consorciomadrono.es/api/access/datafile/34065 + date_accessed: "2026-05-20" + + # License + license: + name: CC BY 4.0 + url: https://creativecommons.org/licenses/by/4.0/ + +outs: + - md5: 57067ba3e1d6482a8233863ae6cf36dd + size: 140770 + path: federico_tena_population_quality.xlsx From c927da44c107f199218da8b4736f98838bb6f915 Mon Sep 17 00:00:00 2001 From: lucasrodes Date: Wed, 20 May 2026 18:31:51 +0200 Subject: [PATCH 03/11] =?UTF-8?q?=F0=9F=94=A8=F0=9F=A4=96=20Federico-Tena?= =?UTF-8?q?=20V2:=20move=20under=20demography=20namespace,=20wire=20Qualit?= =?UTF-8?q?y=20Assessment=20as=20second=20table?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename namespace ggdc/ → demography/ (F-T is published by UC3M Madroño, not GGDC; demography/ matches the topical bucket used by the rest of the OMM-relevant steps). - Drop the canonical-snapshot .py scripts; the .dvc + url_download is enough. - Meadow now produces two tables: the main 1991-borders population series and the per-(polity, year) Quality Assessment classes (A-E, plus NE = not estimated) at 1938 borders. - Garden harmonizes the main population table; QA table country names are kept at the source's 1938 historical borders (we expose a current_name_hint column from the parenthetical labels). Mapping QA to 1991 borders is left to the OMM consumer. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../federico_tena_population.countries.json | 0 .../federico_tena_population.meta.yml | 0 .../2026-05-20/federico_tena_population.py | 0 .../2026-05-20/federico_tena_population.py | 142 ++++++++++++++++++ .../2026-05-20/federico_tena_population.py | 96 ------------ .../federico_tena_population.tab.dvc | 0 .../federico_tena_population_quality.xlsx.dvc | 0 .../2026-05-20/federico_tena_population.py | 24 --- .../federico_tena_population_quality.py | 24 --- 9 files changed, 142 insertions(+), 144 deletions(-) rename etl/steps/data/garden/{ggdc => demography}/2026-05-20/federico_tena_population.countries.json (100%) rename etl/steps/data/garden/{ggdc => demography}/2026-05-20/federico_tena_population.meta.yml (100%) rename etl/steps/data/garden/{ggdc => demography}/2026-05-20/federico_tena_population.py (100%) create mode 100644 etl/steps/data/meadow/demography/2026-05-20/federico_tena_population.py delete mode 100644 etl/steps/data/meadow/ggdc/2026-05-20/federico_tena_population.py rename snapshots/{ggdc => demography}/2026-05-20/federico_tena_population.tab.dvc (100%) rename snapshots/{ggdc => demography}/2026-05-20/federico_tena_population_quality.xlsx.dvc (100%) delete mode 100644 snapshots/ggdc/2026-05-20/federico_tena_population.py delete mode 100644 snapshots/ggdc/2026-05-20/federico_tena_population_quality.py diff --git a/etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.countries.json b/etl/steps/data/garden/demography/2026-05-20/federico_tena_population.countries.json similarity index 100% rename from etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.countries.json rename to etl/steps/data/garden/demography/2026-05-20/federico_tena_population.countries.json diff --git a/etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.meta.yml b/etl/steps/data/garden/demography/2026-05-20/federico_tena_population.meta.yml similarity index 100% rename from etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.meta.yml rename to etl/steps/data/garden/demography/2026-05-20/federico_tena_population.meta.yml diff --git a/etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.py b/etl/steps/data/garden/demography/2026-05-20/federico_tena_population.py similarity index 100% rename from etl/steps/data/garden/ggdc/2026-05-20/federico_tena_population.py rename to etl/steps/data/garden/demography/2026-05-20/federico_tena_population.py diff --git a/etl/steps/data/meadow/demography/2026-05-20/federico_tena_population.py b/etl/steps/data/meadow/demography/2026-05-20/federico_tena_population.py new file mode 100644 index 00000000000..97b04d754c9 --- /dev/null +++ b/etl/steps/data/meadow/demography/2026-05-20/federico_tena_population.py @@ -0,0 +1,142 @@ +"""Load Federico–Tena V2 (1991 borders) and the companion Quality Assessment file. + +Two snapshots, two meadow tables: + +- ``federico_tena_population``: annual population 1800–1938 at 1991 borders. + Source layout (tab-separated): + Row 0: continent labels per column ("POPULATION" | "AFRICA" | ... | "BG" | ...) + Row 1: country names at 1991 borders + Row 2: alternative / historical names (column 0 holds "(000's omitted)") + Rows 3..141: year (1800–1938) and population values in thousands + +- ``federico_tena_population_quality``: per (polity, year) reliability classes A–E (plus + "NE" = not estimated), at 1938 historical borders (note: different from the main table's + 1991 borders, so country lists don't align one-to-one). Source layout (xlsx, sheet + "Quality Assessment"): + Row 0: continent labels + Row 1: country name at 1938 borders + Row 2: 1991 / current-name hint in parentheses + Rows 3..141: year and quality class + +The continent label "BG" marks blank separator columns and is dropped. +""" + +import owid.catalog.processing as pr +from owid.catalog import Table + +from etl.helpers import PathFinder + +paths = PathFinder(__file__) + +SEPARATOR_LABEL = "BG" +YEAR_MIN = 1800 +YEAR_MAX = 1938 +QA_SHEET = "Quality Assessment" +QA_VALID_CLASSES = {"A", "B", "C", "D", "E", "NE"} + + +def run() -> None: + # + # Load inputs. + # + snap_pop = paths.load_snapshot("federico_tena_population.tab") + snap_qa = paths.load_snapshot("federico_tena_population_quality.xlsx") + + tb_pop_raw = snap_pop.read_csv(sep="\t", header=None, dtype=str, na_values=[""], keep_default_na=False) + tb_qa_raw = snap_qa.read_excel(sheet_name=QA_SHEET, header=None, dtype=str) + + # + # Process data. + # + tb_pop = _reshape_population(tb_pop_raw) + tb_pop["population"] = (pr.to_numeric(tb_pop["population"], errors="raise") * 1000).round().astype("int64") + _check_year_range(tb_pop, "population") + assert tb_pop["population"].ge(0).all(), "negative population values" + + tb_qa = _reshape_quality(tb_qa_raw) + _check_year_range(tb_qa, "quality_class") + unexpected = set(tb_qa["quality_class"].unique()) - QA_VALID_CLASSES + assert not unexpected, f"unexpected quality classes: {unexpected}" + + tables = [ + tb_pop.format(["country", "year"], short_name="federico_tena_population"), + tb_qa.format(["country", "year"], short_name="federico_tena_population_quality"), + ] + + # + # Save outputs. + # + ds_meadow = paths.create_dataset(tables=tables, default_metadata=snap_pop.metadata) + ds_meadow.save() + + +def _reshape_population(tb_raw: Table) -> Table: + """Reshape the wide tab layout into (country, year, population, continent, historical_name).""" + continents = tb_raw.iloc[0].tolist() + countries_1991 = tb_raw.iloc[1].tolist() + historical_names = tb_raw.iloc[2].tolist() + + body = tb_raw.iloc[3:].copy() + body = body[body[0].astype(str).str.match(r"^\d{4}$", na=False)] + body[0] = body[0].astype(int) + body = body[body[0].between(YEAR_MIN, YEAR_MAX)] + + country_cols = [ + i for i in range(1, tb_raw.shape[1]) if continents[i] != SEPARATOR_LABEL and countries_1991[i] not in (None, "") + ] + + body = body.rename(columns={0: "year"}) + long = body.melt(id_vars="year", value_vars=country_cols, var_name="col_idx", value_name="population") + + long["country"] = long["col_idx"].map(lambda i: str(countries_1991[i]).strip()) + long["continent"] = long["col_idx"].map(lambda i: continents[i]) + long["historical_name"] = long["col_idx"].map(lambda i: (historical_names[i] or "").strip() or None) + long = long.drop(columns=["col_idx"]) + + long = long[long["population"].astype(str).str.len() > 0] + long = long.dropna(subset=["population"]) + + return long[["country", "year", "population", "continent", "historical_name"]] + + +def _reshape_quality(tb_raw: Table) -> Table: + """Reshape the QA wide xlsx layout into (country, year, quality_class, continent, current_name_hint). + + Country names here are 1938 historical labels. Row 2 of the source contains a parenthetical + current-name hint (e.g. "(Lesotho)" under "Basutoland") which we surface as ``current_name_hint``. + """ + continents = tb_raw.iloc[0].tolist() + historical_names = tb_raw.iloc[1].tolist() + current_hints = tb_raw.iloc[2].tolist() + + body = tb_raw.iloc[3:].copy() + body = body[body[0].astype(str).str.match(r"^\d{4}$", na=False)] + body[0] = body[0].astype(int) + body = body[body[0].between(YEAR_MIN, YEAR_MAX)] + + country_cols = [ + i + for i in range(1, tb_raw.shape[1]) + if continents[i] not in (None, "", SEPARATOR_LABEL) and historical_names[i] not in (None, "") + ] + + body = body.rename(columns={0: "year"}) + long = body.melt(id_vars="year", value_vars=country_cols, var_name="col_idx", value_name="quality_class") + + long["country"] = long["col_idx"].map(lambda i: str(historical_names[i]).strip()) + long["continent"] = long["col_idx"].map(lambda i: continents[i]) + long["current_name_hint"] = long["col_idx"].map( + lambda i: (current_hints[i] or "").strip().strip("()").strip() or None + ) + long = long.drop(columns=["col_idx"]) + + long["quality_class"] = long["quality_class"].astype(str).str.strip() + long = long[long["quality_class"].str.len() > 0] + + return long[["country", "year", "quality_class", "continent", "current_name_hint"]] + + +def _check_year_range(tb: Table, value_col: str) -> None: + assert tb["year"].min() == YEAR_MIN, f"unexpected min year in {value_col}: {tb['year'].min()}" + assert tb["year"].max() == YEAR_MAX, f"unexpected max year in {value_col}: {tb['year'].max()}" + assert tb["country"].notna().all(), f"missing country labels in {value_col}" diff --git a/etl/steps/data/meadow/ggdc/2026-05-20/federico_tena_population.py b/etl/steps/data/meadow/ggdc/2026-05-20/federico_tena_population.py deleted file mode 100644 index d02b623695e..00000000000 --- a/etl/steps/data/meadow/ggdc/2026-05-20/federico_tena_population.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Load the Federico–Tena V2 (1991 borders) tab snapshot and reshape it to a long table. - -Source file layout (tab-separated): - Row 0: continent labels per column ("POPULATION" | "AFRICA" | ... | "BG" | "AMERICA" | ...) - Row 1: country names at 1991 borders ("Borders 1991" | "Burundi" | ...) - Row 2: alternative / historical names ("(000's omitted)" | "" | "French Somalia" | ...) - Rows 3..141: year (1800–1938) and population values in thousands - Rows 142+: blank/footer rows ("WORD COUNTRY Nº" enumeration) - -The continent label "BG" marks blank separator columns between continents — they are dropped. -""" - -import owid.catalog.processing as pr -from owid.catalog import Table - -from etl.helpers import PathFinder - -paths = PathFinder(__file__) - -# Continent labels that separate empty buffer columns in the source spreadsheet. -SEPARATOR_LABEL = "BG" - -# Expected year range of the data block. -YEAR_MIN = 1800 -YEAR_MAX = 1938 - - -def run() -> None: - # - # Load inputs. - # - snap = paths.load_snapshot("federico_tena_population.tab") - - # Read the whole sheet as raw text — the first 3 rows are a multi-row header. - tb_raw = snap.read_csv(sep="\t", header=None, dtype=str, na_values=[""], keep_default_na=False) - - # - # Process data. - # - tb = _reshape_to_long(tb_raw) - - # Values are reported in thousands of people; convert to absolute count. - tb["population"] = (pr.to_numeric(tb["population"], errors="raise") * 1000).round().astype("int64") - - # Sanity checks. - assert tb["year"].min() == YEAR_MIN, f"unexpected min year: {tb['year'].min()}" - assert tb["year"].max() == YEAR_MAX, f"unexpected max year: {tb['year'].max()}" - assert tb["population"].ge(0).all(), "negative population values" - assert tb["country"].notna().all(), "missing country labels" - - tables = [tb.format(["country", "year"], short_name=paths.short_name)] - - # - # Save outputs. - # - ds_meadow = paths.create_dataset(tables=tables, default_metadata=snap.metadata) - ds_meadow.save() - - -def _reshape_to_long(tb_raw: Table) -> Table: - """Pivot the wide source layout into (country, year, population, continent).""" - # Header rows. - continents = tb_raw.iloc[0].tolist() - countries_1991 = tb_raw.iloc[1].tolist() - historical_names = tb_raw.iloc[2].tolist() - - # Data block: rows where the first column parses as a 4-digit year. - body = tb_raw.iloc[3:].copy() - body = body[body[0].astype(str).str.match(r"^\d{4}$", na=False)] - body[0] = body[0].astype(int) - body = body[body[0].between(YEAR_MIN, YEAR_MAX)] - - # Identify country columns (skip column 0 = year, and skip continent separators). - country_cols = [] - for i in range(1, tb_raw.shape[1]): - if continents[i] == SEPARATOR_LABEL: - continue - if not countries_1991[i] or countries_1991[i] == "": - continue - country_cols.append(i) - - # Melt to long form. - body = body.rename(columns={0: "year"}) - long = body.melt(id_vars="year", value_vars=country_cols, var_name="col_idx", value_name="population") - - # Map the column index to its country and continent labels. - long["country"] = long["col_idx"].map(lambda i: countries_1991[i]) - long["continent"] = long["col_idx"].map(lambda i: continents[i]) - long["historical_name"] = long["col_idx"].map(lambda i: historical_names[i] or None) - long = long.drop(columns=["col_idx"]) - - # Drop rows with no observation (Dataverse exports empty cells as ""). - long = long[long["population"].astype(str).str.len() > 0] - long = long.dropna(subset=["population"]) - - return long[["country", "year", "population", "continent", "historical_name"]] diff --git a/snapshots/ggdc/2026-05-20/federico_tena_population.tab.dvc b/snapshots/demography/2026-05-20/federico_tena_population.tab.dvc similarity index 100% rename from snapshots/ggdc/2026-05-20/federico_tena_population.tab.dvc rename to snapshots/demography/2026-05-20/federico_tena_population.tab.dvc diff --git a/snapshots/ggdc/2026-05-20/federico_tena_population_quality.xlsx.dvc b/snapshots/demography/2026-05-20/federico_tena_population_quality.xlsx.dvc similarity index 100% rename from snapshots/ggdc/2026-05-20/federico_tena_population_quality.xlsx.dvc rename to snapshots/demography/2026-05-20/federico_tena_population_quality.xlsx.dvc diff --git a/snapshots/ggdc/2026-05-20/federico_tena_population.py b/snapshots/ggdc/2026-05-20/federico_tena_population.py deleted file mode 100644 index 767d7ccfb86..00000000000 --- a/snapshots/ggdc/2026-05-20/federico_tena_population.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Script to create a snapshot of the Federico–Tena World Population Historical Database (V2, 1991 borders).""" - -from pathlib import Path - -import click - -from etl.snapshot import Snapshot - -# Version for current snapshot dataset. -SNAPSHOT_VERSION = Path(__file__).parent.name - - -@click.command() -@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") -def main(upload: bool) -> None: - # Create a new snapshot. - snap = Snapshot(f"ggdc/{SNAPSHOT_VERSION}/federico_tena_population.tab") - - # Download data from source (url_download in the .dvc), add file to DVC and upload to S3. - snap.create_snapshot(upload=upload) - - -if __name__ == "__main__": - main() diff --git a/snapshots/ggdc/2026-05-20/federico_tena_population_quality.py b/snapshots/ggdc/2026-05-20/federico_tena_population_quality.py deleted file mode 100644 index c039e8a51dd..00000000000 --- a/snapshots/ggdc/2026-05-20/federico_tena_population_quality.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Script to create a snapshot of the Federico–Tena Quality Assessment file (2025, V2).""" - -from pathlib import Path - -import click - -from etl.snapshot import Snapshot - -# Version for current snapshot dataset. -SNAPSHOT_VERSION = Path(__file__).parent.name - - -@click.command() -@click.option("--upload/--skip-upload", default=True, type=bool, help="Upload dataset to Snapshot") -def main(upload: bool) -> None: - # Create a new snapshot. - snap = Snapshot(f"ggdc/{SNAPSHOT_VERSION}/federico_tena_population_quality.xlsx") - - # Download data from source (url_download in the .dvc), add file to DVC and upload to S3. - snap.create_snapshot(upload=upload) - - -if __name__ == "__main__": - main() From 4f0fbc40098cc30c6afc9a7395473c3ab001cd95 Mon Sep 17 00:00:00 2001 From: lucasrodes Date: Wed, 20 May 2026 19:21:47 +0200 Subject: [PATCH 04/11] =?UTF-8?q?=F0=9F=93=8A=20Update=20Federico-Tena=20p?= =?UTF-8?q?opulation=20and=20quality=20assessment=20datasets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dag/demography.yml | 9 ++++-- .../federico_tena_population.meta.yml | 18 ++++++++++++ .../2026-05-20/federico_tena_population.py | 28 ++++++++++++------- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/dag/demography.yml b/dag/demography.yml index fde6b3f2a52..793eee58ea8 100644 --- a/dag/demography.yml +++ b/dag/demography.yml @@ -132,9 +132,12 @@ steps: # Federico–Tena World Population Historical Database — V2 (1991 borders, 2026 update). # Published Dataverse release that supersedes the 2024-01-19 working-paper Excel. - data://garden/ggdc/2026-05-20/federico_tena_population: - - data://meadow/ggdc/2026-05-20/federico_tena_population: - - snapshot://ggdc/2026-05-20/federico_tena_population.tab + # Two tables: main population series (1991 borders) and the per-(country, year) + # Quality Assessment file (1938 borders). + data://garden/demography/2026-05-20/federico_tena_population: + - data://meadow/demography/2026-05-20/federico_tena_population: + - snapshot://demography/2026-05-20/federico_tena_population.tab + - snapshot://demography/2026-05-20/federico_tena_population_quality.xlsx # # Population of the Republic of Ireland territory between 1841 and 2006 (Central Statistics Office of Ireland) diff --git a/etl/steps/data/garden/demography/2026-05-20/federico_tena_population.meta.yml b/etl/steps/data/garden/demography/2026-05-20/federico_tena_population.meta.yml index 2b373eece28..d99772e407a 100644 --- a/etl/steps/data/garden/demography/2026-05-20/federico_tena_population.meta.yml +++ b/etl/steps/data/garden/demography/2026-05-20/federico_tena_population.meta.yml @@ -29,3 +29,21 @@ tables: unit: "" description_short: |- Historical or alternative polity name as recorded in the Federico–Tena source spreadsheet, where different from the 1991-borders country name. + + federico_tena_population_quality: + variables: + quality_class: + title: Quality assessment class + unit: "" + description_short: |- + Reliability class assigned by Federico and Tena-Junguito to each (polity, year) population estimate. A: fully trustworthy estimate. B: interpolation between censuses. C: colonial statistics or estimates. D: interpolation of C. E: lowest reliability. NE: not estimated. + continent: + title: Continent (source label) + unit: "" + description_short: |- + Continent label as recorded in the Federico–Tena Quality Assessment spreadsheet. + current_name_hint: + title: Current-name hint (source label) + unit: "" + description_short: |- + Where the Federico–Tena Quality Assessment file lists a current-name hint in parentheses next to the 1938 historical name (e.g. "(Lesotho)" under "Basutoland"), it is reproduced here without the parentheses. diff --git a/etl/steps/data/garden/demography/2026-05-20/federico_tena_population.py b/etl/steps/data/garden/demography/2026-05-20/federico_tena_population.py index 7f2bb304646..02dd75cdd30 100644 --- a/etl/steps/data/garden/demography/2026-05-20/federico_tena_population.py +++ b/etl/steps/data/garden/demography/2026-05-20/federico_tena_population.py @@ -1,4 +1,9 @@ -"""Harmonize country names in the Federico–Tena V2 (1991 borders) population table.""" +"""Federico–Tena V2 population + Quality Assessment. + +Two tables: the main population series at 1991 borders (harmonized) and the per-(country, year) +reliability classes (kept at the source's 1938 historical borders — country lists do not align +one-to-one with the main table, so we don't force harmonization here). +""" from etl.helpers import PathFinder @@ -10,25 +15,28 @@ def run() -> None: # Load inputs. # ds_meadow = paths.load_dataset("federico_tena_population") - tb = ds_meadow.read("federico_tena_population") + tb_pop = ds_meadow.read("federico_tena_population") + tb_qa = ds_meadow.read("federico_tena_population_quality") # # Process data. # - # Harmonize country names against the OWID standard list. Unmapped entries are - # written to federico_tena_population.countries.json on first run for review. - tb = paths.regions.harmonize_names( - tb=tb, + # Population table is at 1991 borders — harmonize against OWID's standard country list. + # Unmapped entries are written to federico_tena_population.countries.json on first run. + tb_pop = paths.regions.harmonize_names( + tb=tb_pop, country_col="country", countries_file=paths.country_mapping_path, ) + tb_pop = tb_pop.format(["country", "year"], short_name="federico_tena_population") - # We keep the source-side continent label and historical name as auxiliary - # columns so downstream steps can inspect them, but they are not indicators. - tb = tb.format(["country", "year"], short_name=paths.short_name) + # Quality assessment table is at 1938 borders — keep source labels untouched for now. + # Mapping these to 1991 borders is non-trivial (former colonies, partitioned states, etc.) + # and we'd rather expose them as-is until the OMM consumer decides how to use them. + tb_qa = tb_qa.format(["country", "year"], short_name="federico_tena_population_quality") # # Save outputs. # - ds_garden = paths.create_dataset(tables=[tb], default_metadata=ds_meadow.metadata) + ds_garden = paths.create_dataset(tables=[tb_pop, tb_qa], default_metadata=ds_meadow.metadata) ds_garden.save() From b4a28fae387c8b61a8b91914fb715963fe5f2817 Mon Sep 17 00:00:00 2001 From: lucasrodes Date: Wed, 27 May 2026 22:20:35 +0200 Subject: [PATCH 05/11] federico & tena dataset --- .../federico_tena_population.countries.json | 233 +++++++++++++++++- .../2026-05-20/federico_tena_population.py | 35 ++- .../2026-05-20/federico_tena_population.py | 29 +++ .../federico_tena_population_quality.py | 29 +++ 4 files changed, 316 insertions(+), 10 deletions(-) create mode 100644 snapshots/demography/2026-05-20/federico_tena_population.py create mode 100644 snapshots/demography/2026-05-20/federico_tena_population_quality.py diff --git a/etl/steps/data/garden/demography/2026-05-20/federico_tena_population.countries.json b/etl/steps/data/garden/demography/2026-05-20/federico_tena_population.countries.json index 0967ef424bc..adaf02c4152 100644 --- a/etl/steps/data/garden/demography/2026-05-20/federico_tena_population.countries.json +++ b/etl/steps/data/garden/demography/2026-05-20/federico_tena_population.countries.json @@ -1 +1,232 @@ -{} +{ + "Afghanistan": "Afghanistan", + "Albania": "Albania", + "Algeria": "Algeria", + "American Samoa": "American Samoa", + "Andorra": "Andorra", + "Angola": "Angola", + "Anguilla": "Anguilla", + "Antigua and Barbuda": "Antigua and Barbuda", + "Argentina": "Argentina", + "Armenia": "Armenia", + "Aruba": "Aruba", + "Australia": "Australia", + "Austria": "Austria", + "Azerbaijan": "Azerbaijan", + "Bahamas": "Bahamas", + "Bahrain": "Bahrain", + "Bangladesh": "Bangladesh", + "Barbados": "Barbados", + "Belarus": "Belarus", + "Belgium": "Belgium", + "Belize": "Belize", + "Benin": "Benin", + "Bermuda": "Bermuda", + "Bhutan": "Bhutan", + "Bolivia (Plurinational State of)": "Bolivia", + "Bosnia and Herzegovina": "Bosnia and Herzegovina", + "Botswana": "Botswana", + "Brazil": "Brazil", + "British Virgin Islands": "British Virgin Islands", + "Brunei Darussalam": "Brunei", + "Bulgaria": "Bulgaria", + "Burkina Faso": "Burkina Faso", + "Burundi": "Burundi", + "Cambodia": "Cambodia", + "Cameroon": "Cameroon", + "Canada": "Canada", + "Cape Verde": "Cape Verde", + "Cayman Islands": "Cayman Islands", + "Central African Republic": "Central African Republic", + "Chad": "Chad", + "Channel Islands": "Channel Islands", + "Chile": "Chile", + "China": "China", + "China, Hong Kong SAR": "Hong Kong", + "China, Macao SAR": "Macao", + "Colombia": "Colombia", + "Comoros": "Comoros", + "Congo": "Congo", + "Cook Islands": "Cook Islands", + "Costa Rica": "Costa Rica", + "Croatia": "Croatia", + "Cuba": "Cuba", + "Cyprus": "Cyprus", + "Czech Republic": "Czechia", + "Côte d'Ivoire": "Cote d'Ivoire", + "Dem. People's Republic of Korea": "North Korea", + "Democratic Republic of the Congo": "Democratic Republic of Congo", + "Denmark": "Denmark", + "Djibouti": "Djibouti", + "Dominica": "Dominica", + "Dominican Republic": "Dominican Republic", + "Ecuador": "Ecuador", + "Egypt": "Egypt", + "El Salvador": "El Salvador", + "Equatorial Guinea": "Equatorial Guinea", + "Eritrea": "Eritrea", + "Estonia": "Estonia", + "Ethiopia": "Ethiopia", + "Faeroe Islands": "Faroe Islands", + "Falkland Islands (Malvinas)": "Falkland Islands", + "Fiji": "Fiji", + "Finland": "Finland", + "France": "France", + "French Guiana": "French Guiana", + "French Polynesia": "French Polynesia", + "Gabon": "Gabon", + "Gambia": "Gambia", + "Georgia": "Georgia", + "Germany": "Germany", + "Ghana": "Ghana", + "Gibraltar": "Gibraltar", + "Greece": "Greece", + "Greenland": "Greenland", + "Grenada": "Grenada", + "Guadeloupe": "Guadeloupe", + "Guam": "Guam", + "Guatemala": "Guatemala", + "Guinea": "Guinea", + "Guinea-Bissau": "Guinea-Bissau", + "Guyana": "Guyana", + "Haiti": "Haiti", + "Holy See": "Vatican", + "Honduras": "Honduras", + "Hungary": "Hungary", + "Iceland": "Iceland", + "India": "India", + "Indonesia": "Indonesia", + "Iran (Islamic Republic of)": "Iran", + "Iraq": "Iraq", + "Ireland": "Ireland", + "Isle of Man": "Isle of Man", + "Israel": "Israel", + "Italy": "Italy", + "Jamaica": "Jamaica", + "Japan": "Japan", + "Jordan": "Jordan", + "Kazakhstan": "Kazakhstan", + "Kenya": "Kenya", + "Kiribati": "Kiribati", + "Kuwait": "Kuwait", + "Kyrgyzstan": "Kyrgyzstan", + "Lao People's Democratic Republic": "Laos", + "Latvia": "Latvia", + "Lebanon": "Lebanon", + "Lesotho": "Lesotho", + "Liberia": "Liberia", + "Libyan Arab Jamahiriya": "Libya", + "Liechtenstein": "Liechtenstein", + "Lithuania": "Lithuania", + "Luxembourg": "Luxembourg", + "Madagascar": "Madagascar", + "Malawi": "Malawi", + "Malaysia": "Malaysia", + "Maldives": "Maldives", + "Mali": "Mali", + "Malta": "Malta", + "Marshall Islands": "Marshall Islands", + "Martinique": "Martinique", + "Mauritania": "Mauritania", + "Mauritius": "Mauritius", + "Mayotte": "Mayotte", + "Mexico": "Mexico", + "Micronesia (Fed. States of)": "Micronesia (country)", + "Monaco": "Monaco", + "Mongolia": "Mongolia", + "Montenegro": "Montenegro", + "Montserrat": "Montserrat", + "Morocco": "Morocco", + "Mozambique": "Mozambique", + "Myanmar": "Myanmar", + "Namibia": "Namibia", + "Nauru": "Nauru", + "Nepal": "Nepal", + "Netherlands": "Netherlands", + "Netherlands Antilles": "Netherlands Antilles", + "New Caledonia": "New Caledonia", + "New Zealand": "New Zealand", + "Nicaragua": "Nicaragua", + "Niger": "Niger", + "Nigeria": "Nigeria", + "Niue": "Niue", + "Northern Mariana Islands": "Northern Mariana Islands", + "Norway": "Norway", + "Occupied Palestinian Territory": "Palestine", + "Oman": "Oman", + "Pakistan": "Pakistan", + "Palau": "Palau", + "Panama": "Panama", + "Papua New Guinea": "Papua New Guinea", + "Paraguay": "Paraguay", + "Peru": "Peru", + "Philippines": "Philippines", + "Poland": "Poland", + "Portugal": "Portugal", + "Puerto Rico": "Puerto Rico", + "Qatar": "Qatar", + "Republic of Korea": "South Korea", + "Republic of Moldova": "Moldova", + "Romania": "Romania", + "Russian Federation": "Russia", + "Rwanda": "Rwanda", + "Réunion": "Reunion", + "Saint Helena": "Saint Helena", + "Saint Kitts and Nevis": "Saint Kitts and Nevis", + "Saint Lucia": "Saint Lucia", + "Saint Pierre and Miquelon": "Saint Pierre and Miquelon", + "Saint Vincent and the Grenadines": "Saint Vincent and the Grenadines", + "Samoa": "Samoa", + "San Marino": "San Marino", + "Sao Tome and Principe": "Sao Tome and Principe", + "Saudi Arabia": "Saudi Arabia", + "Senegal": "Senegal", + "Serbia": "Serbia", + "Seychelles": "Seychelles", + "Sierra Leone": "Sierra Leone", + "Singapore": "Singapore", + "Slovakia": "Slovakia", + "Slovenia": "Slovenia", + "Solomon Islands": "Solomon Islands", + "Somalia": "Somalia", + "South Africa": "South Africa", + "Spain": "Spain", + "Sri Lanka": "Sri Lanka", + "Sudan": "Sudan", + "Suriname": "Suriname", + "Swaziland": "Eswatini", + "Sweden": "Sweden", + "Switzerland": "Switzerland", + "Syrian Arab Republic": "Syria", + "TFYR Macedonia": "North Macedonia", + "Taiwan": "Taiwan", + "Tajikistan": "Tajikistan", + "Thailand": "Thailand", + "Timor-Leste": "East Timor", + "Togo": "Togo", + "Tokelau": "Tokelau", + "Tonga": "Tonga", + "Trinidad and Tobago": "Trinidad and Tobago", + "Tunisia": "Tunisia", + "Turkey": "Turkey", + "Turkmenistan": "Turkmenistan", + "Turks and Caicos Islands": "Turks and Caicos Islands", + "Tuvalu": "Tuvalu", + "Uganda": "Uganda", + "Ukraine": "Ukraine", + "United Arab Emirates": "United Arab Emirates", + "United Kingdom": "United Kingdom", + "United Republic of Tanzania": "Tanzania", + "United States Virgin Islands": "United States Virgin Islands", + "United States of America": "United States", + "Uruguay": "Uruguay", + "Uzbekistan": "Uzbekistan", + "Vanuatu": "Vanuatu", + "Venezuela (Bolivarian Republic of)": "Venezuela", + "Viet Nam": "Vietnam", + "Wallis and Futuna Islands": "Wallis and Futuna", + "Western Sahara": "Western Sahara", + "Yemen": "Yemen", + "Zambia": "Zambia", + "Zimbabwe": "Zimbabwe" +} diff --git a/etl/steps/data/meadow/demography/2026-05-20/federico_tena_population.py b/etl/steps/data/meadow/demography/2026-05-20/federico_tena_population.py index 97b04d754c9..906f7055353 100644 --- a/etl/steps/data/meadow/demography/2026-05-20/federico_tena_population.py +++ b/etl/steps/data/meadow/demography/2026-05-20/federico_tena_population.py @@ -18,19 +18,36 @@ Row 2: 1991 / current-name hint in parentheses Rows 3..141: year and quality class -The continent label "BG" marks blank separator columns and is dropped. +Both sheets interleave real polity columns with blank spacer columns (row-0 codes like "BG", "DG", +"FG", "IB"…"IN") and, on the right, aggregate/summary columns (continent totals and World totals, +e.g. "WORLD Bod 1991", "World Bord Histor."). We keep only columns whose row-0 label is one of the +five real continents and whose country name (row 1) is non-blank — this drops every spacer and every +aggregate column in one go (more robust than blacklisting the ever-growing set of spacer codes). """ import owid.catalog.processing as pr +import pandas as pd from owid.catalog import Table from etl.helpers import PathFinder paths = PathFinder(__file__) -SEPARATOR_LABEL = "BG" +CONTINENTS = {"AFRICA", "AMERICA", "ASIA", "EUROPE", "OCEANIA"} YEAR_MIN = 1800 YEAR_MAX = 1938 + + +def _blank(x) -> bool: + """True for None, NaN, or whitespace-only cells (header cells come through as str or NaN).""" + return x is None or (isinstance(x, float) and pd.isna(x)) or str(x).strip() == "" + + +def _clean(x): + """Stripped string, or None if blank.""" + return None if _blank(x) else str(x).strip() + + QA_SHEET = "Quality Assessment" QA_VALID_CLASSES = {"A", "B", "C", "D", "E", "NE"} @@ -82,7 +99,7 @@ def _reshape_population(tb_raw: Table) -> Table: body = body[body[0].between(YEAR_MIN, YEAR_MAX)] country_cols = [ - i for i in range(1, tb_raw.shape[1]) if continents[i] != SEPARATOR_LABEL and countries_1991[i] not in (None, "") + i for i in range(1, tb_raw.shape[1]) if continents[i] in CONTINENTS and not _blank(countries_1991[i]) ] body = body.rename(columns={0: "year"}) @@ -90,11 +107,11 @@ def _reshape_population(tb_raw: Table) -> Table: long["country"] = long["col_idx"].map(lambda i: str(countries_1991[i]).strip()) long["continent"] = long["col_idx"].map(lambda i: continents[i]) - long["historical_name"] = long["col_idx"].map(lambda i: (historical_names[i] or "").strip() or None) + long["historical_name"] = long["col_idx"].map(lambda i: _clean(historical_names[i])) long = long.drop(columns=["col_idx"]) - long = long[long["population"].astype(str).str.len() > 0] long = long.dropna(subset=["population"]) + long = long[long["population"].astype(str).str.strip() != ""] return long[["country", "year", "population", "continent", "historical_name"]] @@ -115,9 +132,7 @@ def _reshape_quality(tb_raw: Table) -> Table: body = body[body[0].between(YEAR_MIN, YEAR_MAX)] country_cols = [ - i - for i in range(1, tb_raw.shape[1]) - if continents[i] not in (None, "", SEPARATOR_LABEL) and historical_names[i] not in (None, "") + i for i in range(1, tb_raw.shape[1]) if continents[i] in CONTINENTS and not _blank(historical_names[i]) ] body = body.rename(columns={0: "year"}) @@ -126,10 +141,12 @@ def _reshape_quality(tb_raw: Table) -> Table: long["country"] = long["col_idx"].map(lambda i: str(historical_names[i]).strip()) long["continent"] = long["col_idx"].map(lambda i: continents[i]) long["current_name_hint"] = long["col_idx"].map( - lambda i: (current_hints[i] or "").strip().strip("()").strip() or None + lambda i: (_clean(current_hints[i]) or "").strip("()").strip() or None ) long = long.drop(columns=["col_idx"]) + # drop blank / not-estimated-as-blank cells (kept "NE" cells are explicit and valid) + long = long.dropna(subset=["quality_class"]) long["quality_class"] = long["quality_class"].astype(str).str.strip() long = long[long["quality_class"].str.len() > 0] diff --git a/snapshots/demography/2026-05-20/federico_tena_population.py b/snapshots/demography/2026-05-20/federico_tena_population.py new file mode 100644 index 00000000000..cb4034a8cb8 --- /dev/null +++ b/snapshots/demography/2026-05-20/federico_tena_population.py @@ -0,0 +1,29 @@ +"""Snapshot of the Federico–Tena World Population Historical Database (V2, 1991 borders). + +The source dataverse (edatos.consorciomadrono.es) sits behind an Anubis JS proof-of-work bot +wall, so the file cannot be fetched unattended (an automated download stores the challenge HTML +page instead of the data). Download the file manually from the DOI landing page — choosing +"Original File Format" — and pass the local path: + + python snapshots/demography/2026-05-20/federico_tena_population.py --path-to-file /federico_tena_population.tab + +DOI: 10.21950/GW7SOZ (datafile 51077, original tab-separated). +""" + +import click + +from etl.helpers import PathFinder + +paths = PathFinder(__file__) + + +@click.command() +@click.option("--upload/--skip-upload", default=True, help="Upload snapshot to S3.") +@click.option("--path-to-file", prompt=True, type=str, help="Path to the locally downloaded data file.") +def run(path_to_file: str, upload: bool) -> None: + snap = paths.init_snapshot() + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + run() diff --git a/snapshots/demography/2026-05-20/federico_tena_population_quality.py b/snapshots/demography/2026-05-20/federico_tena_population_quality.py new file mode 100644 index 00000000000..78c6a5cca9d --- /dev/null +++ b/snapshots/demography/2026-05-20/federico_tena_population_quality.py @@ -0,0 +1,29 @@ +"""Snapshot of the Federico–Tena Quality Assessment file (companion to the V2 population database). + +The source dataverse (edatos.consorciomadrono.es) sits behind an Anubis JS proof-of-work bot +wall, so the file cannot be fetched unattended (an automated download stores the challenge HTML +page instead of the data). Download the file manually from the DOI landing page — choosing +"Original File Format" — and pass the local path: + + python snapshots/demography/2026-05-20/federico_tena_population_quality.py --path-to-file /federico_tena_population_quality.xlsx + +DOI: 10.21950/U6AANV (datafile 34065, .xlsx, sheet "Quality Assessment"). +""" + +import click + +from etl.helpers import PathFinder + +paths = PathFinder(__file__) + + +@click.command() +@click.option("--upload/--skip-upload", default=True, help="Upload snapshot to S3.") +@click.option("--path-to-file", prompt=True, type=str, help="Path to the locally downloaded data file.") +def run(path_to_file: str, upload: bool) -> None: + snap = paths.init_snapshot() + snap.create_snapshot(filename=path_to_file, upload=upload) + + +if __name__ == "__main__": + run() From 22b903b5472e20cefcf217b1c663ba575c188f5e Mon Sep 17 00:00:00 2001 From: lucasrodes Date: Thu, 28 May 2026 21:22:04 +0200 Subject: [PATCH 06/11] new omm (in test) --- dag/demography.yml | 19 + .../demography/2026-05-28/population.meta.yml | 259 +++++ .../demography/2026-05-28/population.py | 891 ++++++++++++++++++ .../garden/demography/2026-05-28/utils.py | 115 +++ .../demography/2026-05-28/population.py | 42 + 5 files changed, 1326 insertions(+) create mode 100644 etl/steps/data/garden/demography/2026-05-28/population.meta.yml create mode 100644 etl/steps/data/garden/demography/2026-05-28/population.py create mode 100644 etl/steps/data/garden/demography/2026-05-28/utils.py create mode 100644 etl/steps/data/grapher/demography/2026-05-28/population.py diff --git a/dag/demography.yml b/dag/demography.yml index 793eee58ea8..d2b883eed8f 100644 --- a/dag/demography.yml +++ b/dag/demography.yml @@ -32,6 +32,25 @@ steps: data://grapher/demography/2024-07-15/population: - data://garden/demography/2024-07-15/population + # Federico-Tena variant of the OMM (owid-issues#1232 — F-T replaces Gapminder for 1800-1938; + # 1939-1949 bridged by per-country linear interpolation between F-T(1938) and UN WPP(1950)). + # Mirrors the 2024-07-15 step's output schema so consumers don't need code changes. + data://garden/demography/2026-05-28/population: + # HYDE 3.3 + - data://garden/hyde/2024-01-02/all_indicators + # Federico-Tena V2 (1991 borders, 2026) + - data://garden/demography/2026-05-20/federico_tena_population + # Gapminder Systema Globalis — used for former states (USSR, Czechoslovakia, ...) only + - data://open_numbers/open_numbers/latest/gapminder__systema_globalis + # UN WPP (2024) + - data://garden/un/2024-07-12/un_wpp + # Auxiliary + - data://garden/regions/2023-01-01/regions + - data://garden/wb/2025-07-01/income_groups + - data://garden/faostat/2024-03-14/faostat_rl_auxiliary + data://grapher/demography/2026-05-28/population: + - data://garden/demography/2026-05-28/population + # WPP ######################################## # WPP (2024) data://meadow/un/2024-07-12/un_wpp: diff --git a/etl/steps/data/garden/demography/2026-05-28/population.meta.yml b/etl/steps/data/garden/demography/2026-05-28/population.meta.yml new file mode 100644 index 00000000000..402107fe7ab --- /dev/null +++ b/etl/steps/data/garden/demography/2026-05-28/population.meta.yml @@ -0,0 +1,259 @@ +definitions: + others: + different_sources_processing: |- + ### Combination of different sources + We construct our long-run population data by combining multiple sources: + + - 10,000 BCE–1799: historical estimates by HYDE (v3.3). + + - 1800–1949: historical estimates by Gapminder (v7). + + - 1950–2023: population records from the United Nations World Population Prospects (2024 revision). + + different_sources_gr_processing: |- + ### Combination of different sources + We construct our long-run data by combining multiple sources: + + - 10,000 BCE–1799: historical estimates by HYDE (v3.3). Growth rate estimated over 50-year periods. + + - 1800–1949: historical estimates by Gapminder (v7). Growth rate estimated over 1-year periods. + + - 1950–2023: population records from the United Nations World Population Prospects (2024 revision). We use the UN's published growth rates directly (based on mid-year population estimates). + + ### Display filtering + To reduce noise in sparse historical data, growth rates are selectively displayed: + + - 1700–1799: Only 100-year intervals (1700, 1800) + - 1800–1899: Only 100-year intervals (1800, 1900) + - 1900–1949: Only 5-year intervals (1900, 1905, 1910, etc.) + - 1950 onwards: All years (annual data) + + regional_aggregates_processing: |- + **Geographical aggregates** + + - For most years, we calculate aggregates by summing the population of member countries. + - We do this based on [our definition of continents](https://ourworldindata.org/world-region-map-definitions#our-world-in-data) and the [World Bank’s income groups](https://ourworldindata.org/grapher/world-bank-income-groups). + - The only exception is before 1800, where we use HYDE's estimates for continents (but not income groups). + + For most of the years, we've estimated regional aggregates by summing the population of countries in each region. We've relied on [our continents](https://ourworldindata.org/world-region-map-definitions#our-world-in-data) and [World Bank income group definitions](https://ourworldindata.org/grapher/world-bank-income-groups). The only exception is before 1800, where we've used HYDE's estimates on continents (but not income groups). + + **World** + - Before 1800: we use data from HYDE. + - 1800-1950: we estimate the global population by summing all available countries in the dataset. + - After 1950, we rely on estimates from the United Nations World Population Prospects. + +tables: + # Table with reduced metadata (to be used when estimating per-capita or other derived indicators) + # We do this to avoid showing complete source information in the chart footers. + population: + title: Population + variables: + population: + title: Population + description_short: &population-description |- + Population by country, available from 10,000 BCE to 2100, based on data and estimates from different sources. + unit: people + short_unit: "" + + world_pop_share: + title: Share of world population + description_short: &population-share-description |- + Share of the world's population by country, available from 10,000 BCE to 2100, based on data and estimates from different sources. + unit: "%" + short_unit: "%" + + source: + title: Source + description_short: &population-source-description |- + Name of the source for a specific data point. The name includes a short name for the source and a link. + unit: "" + + # Table with full metadata (to be used when showing the population time series, e.g. the data page for the population indicator) + population_original: + common: + presentation: + topic_tags: + - Population Growth + attribution: HYDE (2023); Gapminder (2022); UN WPP (2024) + processing_level: major + description_key: &population-description-key + - Population is the most commonly used metric throughout Our World in Data. It is used directly to understand population growth over time, and indirectly to calculate per-capita indicators, making it easier to compare countries of different sizes. + - |- + We construct this indicator by combining multiple sources covering different periods. + - HYDE v3.3 (2023): historical estimates from 10,000 BCE to 1799. + - Gapminder v7 (2022): for 1800-1949. + - UN World Population Prospects (2024): for 1950 onwards, including 2100 projections. + - Gapminder Systema Globalis (2023): additional source for former countries (Yugoslavia, USSR, etc.) + - Breaks in the data may occur at the boundaries between sources due to their methodological differences. + - You can read more about the sources and methodology in our [dedicated article](https://ourworldindata.org/population-sources). We also provide a table of sources showing the source we use for each country-year. + - We calculate geographical aggregates (continents, income groups, etc.) by summing individual country populations. For years before 1800, we rely directly on HYDE's values for continents to ensure historical consistency. + description_processing: &population-description-proc |- + {definitions.others.different_sources_processing} + + - 2024-2100: Projections based on Medium variant by the UN World Population Prospects (2024 revision). + + {definitions.others.regional_aggregates_processing} + title: Population + variables: + population: + title: Population + description_short: *population-description + unit: people + display: + numDecimalPlaces: 0 + world_pop_share: + title: Share of world population + description_short: *population-share-description + unit: "%" + short_unit: "%" + source: + title: Source + description_short: *population-source-description + unit: "" + + # Population density + population_density: + common: + presentation: + topic_tags: + - Population Growth + attribution: HYDE (2023); Gapminder (2022); UN WPP (2024); UN FAO (2024) + variables: + population_density: + title: Population density + unit: people per km² + description_short: | + Population per square kilometer by country, available from 10,000 BCE to 2100, based on data and estimates from different sources. + processing_level: major + description_processing: |- + We have estimated the population density by using population estimates from multiple sources and land area estimates by the Food and Agriculture Organization of the United Nations. + + We obtain it by dividing the population estimates by the land area estimates. + + {tables.population_original.common.description_processing} + + # Population density + population_growth_rate: + common: + presentation: + topic_tags: + - Population Growth + attribution: HYDE (2023); Gapminder (2022); UN WPP (2024) + variables: + growth_rate: + title: Population growth rate + description_short: | + Average exponential rate of growth of the population over a given period. It is calculated as ln(P2/P1) where P1 and P2 are the populations on subsequent years. Available from 1700 to 2100, based on data and estimates from different sources. + unit: "%" + short_unit: "%" + processing_level: major + description_processing: |- + {definitions.others.different_sources_gr_processing} + + - 2024-2100: Projections based on Medium variant by the UN World Population Prospects (2024 revision). Growth rate estimated over 1-year periods. + + {definitions.others.regional_aggregates_processing} + + # Historical data + historical: + common: + presentation: + topic_tags: + - Population Growth + attribution: HYDE (2023); Gapminder (2022); UN WPP (2024) + processing_level: major + description_processing: |- + {definitions.others.different_sources_processing} + + {definitions.others.regional_aggregates_processing} + + variables: + population_historical: + title: |- + {tables.population.variables.population.title} (historical) + description_short: |- + Population by country, available from 10,000 BCE to 2023, based on data and estimates from different sources. + description_key: *population-description-key + unit: "{tables.population.variables.population.unit}" + display: + name: Population + numDecimalPlaces: 0 + + presentation: + title_variant: Long-run data + attribution_short: "HYDE, Gapminder, UN" + + world_pop_share_historical: + title: |- + {tables.population.variables.world_pop_share.title} (historical) + description_short: |- + Share of the world's population by country, available from 10,000 BCE to 2023, based on data and estimates from different sources. + unit: "{tables.population.variables.world_pop_share.unit}" + short_unit: "{tables.population.variables.world_pop_share.short_unit}" + + population_density_historical: + title: |- + {tables.population_density.variables.population_density.title} (historical) + description_short: |- + Population per square kilometer by country, available from 10,000 BCE to 2023, based on data and estimates from different sources. + unit: "{tables.population_density.variables.population_density.unit}" + + growth_rate_historical: + title: |- + {tables.population_growth_rate.variables.growth_rate.title} (historical) + description_short: |- + Average exponential rate of growth of the population over a given period. It is calculated as ln(P2/P1) where P1 and P2 are the populations on subsequent years. Available from 1700 to 2023, based on data and estimates from different sources. + description_processing: |- + {definitions.others.different_sources_gr_processing} + + {definitions.others.regional_aggregates_processing} + unit: "{tables.population_growth_rate.variables.growth_rate.unit}" + short_unit: "{tables.population_growth_rate.variables.growth_rate.short_unit}" + + # Projection data + projections: + common: + presentation: + topic_tags: + - Population Growth + attribution: UN WPP (2024) + processing_level: major + description_processing: *population-description-proc + + variables: + population_projection: + title: |- + {tables.population.variables.population.title} (projections) + description_short: |- + Population by country, available from 10,000 BCE to 2023, based on data and estimates from different sources. + unit: "{tables.population.variables.population.unit}" + display: + numDecimalPlaces: 0 + world_pop_share_projection: + title: |- + {tables.population.variables.world_pop_share.title} (projections) + description_short: |- + Share of the world's population by country, available from 10,000 BCE to 2023, based on data and estimates from different sources. + unit: "{tables.population.variables.world_pop_share.unit}" + short_unit: "{tables.population.variables.world_pop_share.short_unit}" + population_density_projection: + title: |- + {tables.population_density.variables.population_density.title} (projections) + description_short: |- + Population per square kilometer by country, available from 10,000 BCE to 2023, based on data and estimates from different sources. + unit: "{tables.population_density.variables.population_density.unit}" + growth_rate_projection: + title: |- + {tables.population_growth_rate.variables.growth_rate.title} (projections) + description_short: |- + Average exponential rate of growth of the population over a given period. It is calculated as ln(P2/P1) where P1 and P2 are the populations on subsequent years. Available from 2024 to 2100, based on UN medium scenario projections. + unit: "{tables.population_growth_rate.variables.growth_rate.unit}" + short_unit: "{tables.population_growth_rate.variables.growth_rate.short_unit}" + +dataset: + title: Population + update_period_days: 730 + description: |- + Our World in Data builds and maintains a long-run dataset on population by country, region, and for the world, based on various sources. + + You can find more information on these sources and how our time series is constructed on this page: https://ourworldindata.org/population-sources diff --git a/etl/steps/data/garden/demography/2026-05-28/population.py b/etl/steps/data/garden/demography/2026-05-28/population.py new file mode 100644 index 00000000000..334a3ce2cd9 --- /dev/null +++ b/etl/steps/data/garden/demography/2026-05-28/population.py @@ -0,0 +1,891 @@ +"""Long-run population OMM — Federico-Tena variant (assesses the swap proposed in +owid-issues#1232; see reports/population_ft/). + +Stitching: + - HYDE 3.3 for years < 1800 (unchanged from 2024-07-15) + - Federico-Tena V2 (1991 borders) for 1800-1938 (replaces Gapminder v7) + - linear interpolation for 1939-1949 per country (between F-T(1938) & WPP(1950)) + - UN WPP (2024) for >= 1950 (unchanged) + - Gapminder Systema Globalis for former states (USSR, Czechoslovakia, …) — unchanged + +Notes: + - "Gapminder SG" stands for "Gapminder Systema Globalis". + - On regional estimates: + - Continents: re-estimated for WPP and F-T; HYDE uses original values. + - Income groups: TODO + - World: re-estimated. +""" + +import json + +import numpy as np +import owid.catalog.processing as pr +import pandas as pd +from owid.catalog import Dataset, License, Origin, Table +from utils import ( + COUNTRIES_FORMER_EQUIVALENTS, + GAP_HI, + GAP_LO, + GAPMINDER_SG_COUNTRIES, + GAPMINDER_SG_COUNTRIES_FORMER, + GAPMINDER_SG_ORIGINS, + SOURCES_NAMES, + YEAR_END_FT, + YEAR_END_WPP, + YEAR_START_FT, + YEAR_START_GAPMINDER, + YEAR_START_HYDE, + YEAR_START_WPP, + YEAR_START_WPP_PROJ, +) + +from etl.data_helpers import geo +from etl.helpers import PathFinder + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) +# Coluns relevant +COLUMS_RELEVANT_POP = [ + "country", + "year", + "population", +] +COLUMNS_INDEX = [ + "country", + "year", +] + +# Known overlaps between historical and successor regions in the FAOSTAT land area dataset. +KNOWN_OVERLAPS_IN_LAND_AREA_DATA = [{year: {"Netherlands Antilles", "Aruba"} for year in range(1961, 2011)}] + + +def run() -> None: + # + # Load inputs. + # + # Load UN WPP dataset. + ds_un = paths.load_dataset("un_wpp") + tb_un = ds_un.read("population") + tb_un_gr = ds_un.read("growth_rate") + # Load HYDE dataset. + ds_hyde = paths.load_dataset("all_indicators") + tb_hyde = ds_hyde.read("all_indicators") + # Load Federico-Tena dataset (replaces Gapminder for 1800-1938) + ds_ft = paths.load_dataset("federico_tena_population") + tb_ft = ds_ft.read("federico_tena_population") + # Load Gapminder SG dataset + ds_gapminder_sg = paths.load_dataset(short_name="gapminder__systema_globalis", channel="open_numbers") + tb_gapminder_sg = ds_gapminder_sg.read("total_population_with_projections") + + # Load auxiliary datasets: + # * Regions + ds_regions = paths.load_dataset("regions") + tb_regions = ds_regions.read("regions", reset_index=False) + # * Income groups + ds_income_groups = paths.load_dataset("income_groups") + # * Land area (FAOSTAT RL) + ds_land_area = paths.load_dataset("faostat_rl_auxiliary") + tb_land_area = ds_land_area.read("faostat_rl_auxiliary") + + # + # Process data. + # + # Format tables + tb_hyde = format_hyde(tb=tb_hyde) + tb_ft = format_ft(tb_ft) + tb_un = format_wpp(tb_un, "population", "uint64") + tb_un_gr = format_wpp(tb_un_gr, "growth_rate", "float32") + tb_gapminder_sg, tb_gapminder_sg_former = format_gapminder_sg(tb_gapminder_sg) + + # Build the 1939-1949 interpolation rows for every F-T country that also has a 1950 UN WPP value. + tb_ft_interp = make_ft_interp(tb_ft, tb_un) + + # Concat tables + tb = pr.concat( + [tb_hyde, tb_ft, tb_ft_interp, tb_un, tb_gapminder_sg], + ignore_index=True, + ) + + # Make table + tb = ( + tb.pipe(select_source) + .astype( + { + "year": int, + "population": "uint64", + "source": "string", + } + ) + .pipe(add_regions, ds_regions, ds_income_groups) + ) + + tb = ( + tb.pipe(add_world) + .pipe(add_historical_regions, tb_gapminder_sg_former, tb_regions) + .pipe(fix_anomalies) + .astype( + { + "year": "int64", + "population": "uint64", + "source": "category", + } + ) + .pipe(add_world_population_share) + ) + + # Add population growth rate + tb_growth_rate = make_table_growth_rate( + tb_population=tb, + tb_un_gr=tb_un_gr, + ) + + # Add population density + # NOTE: The regions and income groups datasets are used to create region aggregates for land area data. + tb_density = make_table_density( + tb_population=tb, + tb_land_area=tb_land_area, + ds_regions=ds_regions, + ds_income_groups=ds_income_groups, + ) + + # Create auxiliary table + tb_auxiliary = generate_auxiliary_table(tb) + + # Create table with historical & projection data + tbs = [ + tb.drop(columns=["source"]), + tb_density, + tb_growth_rate, + ] + tb_historical = make_table_historical(tbs) + tb_projection = make_table_projection(tbs) + + # Format tables + tb = tb.format(COLUMNS_INDEX, short_name="population_original") + tb_auxiliary = tb_auxiliary.format(COLUMNS_INDEX, short_name="population") + tb_density = tb_density.format(COLUMNS_INDEX, short_name="population_density") + tb_growth_rate = tb_growth_rate.format(COLUMNS_INDEX, short_name="population_growth_rate") + tb_historical = tb_historical.format(COLUMNS_INDEX, short_name="historical") + tb_projection = tb_projection.format(COLUMNS_INDEX, short_name="projections") + + # + # Save outputs. + # + tables = [ + tb, + tb_auxiliary, + tb_density, + tb_growth_rate, + tb_historical, + tb_projection, + ] + + # Create a new garden dataset with the same metadata as the meadow dataset. + ds_garden = paths.create_dataset(tables=tables) + + # Save changes in the new garden dataset. + ds_garden.save() + + +############################################################################################# +# FORMAT SOURCE DATA ######################################################################## +############################################################################################# + + +###################### +# HYDE ############### +###################### +def format_hyde(tb: Table) -> Table: + """Format UN WPP table.""" + # Rename columns, sort rows + columns_rename = { + "country": "country", + "year": "year", + "popc_c": "population", + } + # Rename columns + tb = tb.rename(columns=columns_rename, errors="raise").loc[:, COLUMS_RELEVANT_POP] + + # Exclude HYDE-specific countries + countries_exclude = [ + "Asia (excl. China and India)", + "South America (excl. Brazil)", + "Europe (excl. Russia)", + ] + tb = tb.loc[~tb.country.isin(countries_exclude)] + + # Set source identifier + tb["source"] = "hyde" + return tb + + +###################### +# Gapminder ########## +###################### +def format_ft(tb: Table) -> Table: + """Format the Federico-Tena V2 population table (already harmonized in its garden step).""" + # F-T garden output is (country, year, population, continent, historical_name) — keep core cols + tb = tb.loc[:, COLUMS_RELEVANT_POP].copy() + tb["source"] = "ft" + return tb + + +def make_ft_interp(tb_ft: Table, tb_un: Table) -> Table: + """Linear-interpolate the 1939-1949 gap (F-T(1938) → UN WPP(1950)) per country. + + For each country present in both F-T (at 1938) and UN WPP (at 1950) we generate yearly rows + 1939-1949 with `population = F-T(1938) + (WPP(1950) - F-T(1938)) * (y - 1938) / 12`. + """ + ft_1938 = ( + tb_ft.loc[tb_ft["year"] == YEAR_END_FT, ["country", "population"]] + .set_index("country")["population"] + .astype(float) + ) + wpp_1950 = ( + tb_un.loc[tb_un["year"] == YEAR_START_WPP, ["country", "population"]] + .set_index("country")["population"] + .astype(float) + ) + common = ft_1938.index.intersection(wpp_1950.index) + rows = [] + span = YEAR_START_WPP - YEAR_END_FT # 12 + for c in common: + v0, v1 = ft_1938[c], wpp_1950[c] + for yr in range(GAP_LO, GAP_HI + 1): + rows.append((c, yr, int(round(v0 + (v1 - v0) * (yr - YEAR_END_FT) / span)))) + if not rows: + return Table(pd.DataFrame(columns=COLUMS_RELEVANT_POP + ["source"])) + out = Table(pd.DataFrame(rows, columns=COLUMS_RELEVANT_POP)) + out["source"] = "ft_interp" + # carry F-T's population metadata so origins propagate to the interpolated rows + out["population"].metadata = tb_ft["population"].metadata.copy() + return out + + +###################### +# UN WPP ############# +###################### +def format_wpp(tb: Table, column_indicator: str, indicator_dtype: str) -> Table: + """Format UN WPP table.""" + # Only keep data for general population (all sexes, all ages, etc.) + tb = tb.loc[ + ((tb["sex"] == "all") & (tb["age"] == "all") & (tb["variant"].isin(["estimates", "medium"]))), + ["country", "year", "variant", column_indicator], + ] + + # Sanity checks IN + assert tb.loc[tb["variant"] == "estimates", "year"].min() == YEAR_START_WPP, ( + f"Unexpected start year for WPP estimates. Should be {YEAR_START_WPP}!" + ) + assert tb.loc[tb["variant"] == "estimates", "year"].max() == YEAR_START_WPP_PROJ - 1, ( + f"Unexpected end year for WPP estimates. Should be {YEAR_START_WPP_PROJ - 1}!" + ) + assert tb.loc[tb["variant"] == "medium", "year"].min() == YEAR_START_WPP_PROJ, ( + f"Unexpected start year for WPP projections. Should be {YEAR_START_WPP_PROJ}!" + ) + assert tb.loc[tb["variant"] == "medium", "year"].max() == YEAR_END_WPP, ( + f"Unexpected end year for WPP projections. Should be {YEAR_END_WPP}!" + ) + + # Rename columns, sort rows + tb = ( + tb.loc[:, ["country", "year", column_indicator]] + .assign(source="unwpp") + .astype( + { + "source": "string", + "country": "string", + column_indicator: indicator_dtype, + "year": "uint64", + } + ) + .sort_values(["country", "year"]) + .reset_index(drop=True) + ) + + # Exclude countries + countries_exclude = [ + "Northern America", + "Latin America & Caribbean", + "Land-locked developing countries (LLDC)", + "Latin America and the Caribbean", + "Least developed countries", + "Less developed regions", + "Less developed regions, excluding China", + "Less developed regions, excluding least developed countries", + "More developed regions", + "Small island developing states (SIDS)", + "High-income countries", + "Low-income countries", + "Lower-middle-income countries", + "Upper-middle-income countries", + ] + tb = tb.loc[~tb.country.isin(countries_exclude)] + + # Sanity checks OUT + assert tb.groupby(["country", "year"])[column_indicator].count().max() == 1 + + return tb + + +###################### +# Gapminder SG ####### +###################### +def format_gapminder_sg(tb: Table) -> tuple[Table, Table]: + """Format Gapminder Systema Globalis table.""" + columns_rename = { + "country": "country", + "time": "year", + "total_population_with_projections": "population", + } + + def _core_formatting(tb: Table, country_rename: dict[str, str]) -> Table: + ## rename countries + tb["country"] = tb["geo"].map(country_rename) + ## rename columns + tb = tb.rename(columns=columns_rename, errors="raise").loc[:, COLUMS_RELEVANT_POP] + # Set source identifier + tb["source"] = "gapminder_sg" + # add origins + tb["population"].metadata.origins = GAPMINDER_SG_ORIGINS + return tb + + # Data on former countries + ## only keep former country data + tb_former: Table = tb.loc[tb["geo"].isin(GAPMINDER_SG_COUNTRIES_FORMER)].copy() + + # core formatting: column and country rename, add source, metadata + tb_former = _core_formatting( + tb=tb_former, + country_rename={code: data["name"] for code, data in GAPMINDER_SG_COUNTRIES_FORMER.items()}, + ) + + ## filter years: only keep former countries until they disappear + for _, data in GAPMINDER_SG_COUNTRIES_FORMER.items(): + tb_former = tb_former.loc[~((tb_former["country"] == data["name"]) & (tb_former["year"] > data["end"]))] + + # Complement + ## filter countries + tb = tb.loc[tb["geo"].isin(GAPMINDER_SG_COUNTRIES)] + + # core formatting: column and country rename, add source, metadata + tb = _core_formatting( + tb=tb, + country_rename=GAPMINDER_SG_COUNTRIES, + ) + + return tb, tb_former + + +############################################################################################# +# Combine and process data ################################################################## +############################################################################################# + + +###################### +# Select source ##### +###################### +def select_source(tb: Table) -> Table: + """Select the best source for each country/year. + + Prioritisation: + - For 1800-2100: WPP > F-T (incl. F-T interpolation) > HYDE. + - Prior to 1800: HYDE. + + If a country has F-T data, we drop HYDE for that country in 1800-1949 (so the F-T series + runs unbroken, with the interpolation block bridging 1939-1949). + """ + paths.log.info("selecting source...") + tb = tb.loc[tb["population"] > 0] + + # If a country has UN WPP data, drop all non-UN data >=1950 + has_un_data = set(tb.loc[tb["source"] == "unwpp", "country"]) + tb = tb.loc[~((tb["country"].isin(has_un_data)) & (tb["year"] >= YEAR_START_WPP) & (tb["source"] != "unwpp"))] + + # If a country has F-T data, drop non-F-T data between 1800 and 1949 (keep both "ft" and + # "ft_interp" — they cover 1800-1938 and 1939-1949 respectively for the same country). + ft_sources = {"ft", "ft_interp"} + has_ft_data = set(tb.loc[tb["source"] == "ft", "country"]) + tb = tb.loc[ + ~( + tb["country"].isin(has_ft_data) + & (tb["year"] >= YEAR_START_FT) + & (tb["year"] < YEAR_START_WPP) + & ~tb["source"].isin(ft_sources) + ) + ] + + # Check if all countries have only one row per year + _ = tb.set_index(["country", "year"], verify_integrity=True) + + # map to source full names + tb["source"] = tb["source"].map(SOURCES_NAMES) + return tb + + +###################### +# Add regions / World +###################### +def add_regions(tb: Table, ds_regions: Dataset, ds_income_groups: Dataset) -> Table: + """Add continents and income groups.""" + paths.log.info("population: adding regions...") + # 0/ Initial definitions + ## Country data needed to estimate region aggregates + continents_required = { + "Asia": ["China", "India", "Indonesia", "Pakistan", "Bangladesh"], + "Africa": ["Nigeria", "Ethiopia", "Egypt"], + "North America": ["United States", "Canada", "Mexico"], + "South America": ["Brazil", "Argentina", "Colombia", "Peru"], + "Oceania": ["Australia", "New Zealand"], + "Europe": ["Russia", "Germany", "France", "United Kingdom", "Italy", "Spain"], + } + regions_required = { + "High-income countries": [ + "United States", + "Japan", + "Germany", + "France", + "United Kingdom", + "Italy", + "South Korea", + "Russia", + ], + "Upper-middle-income countries": [ + "China", + "Brazil", + "Indonesia", + "Mexico", + "Iran", + ], + "Lower-middle-income countries": [ + "India", + "Pakistan", + "Nigeria", + "Bangladesh", + "Philippines", + "Egypt", + "Kenya", + "Vietnam", + ], + "Low-income countries": [ + # "Ethiopia", Currently not classified as low-income by the World Bank, but historically it was. + "Democratic Republic of Congo", + "Uganda", + ], + "European Union (27)": ["Germany", "France", "Italy", "Spain"], + } + continents = list(continents_required.keys()) + regions = list(regions_required.keys()) + + def _aggregate(tb, regions_required_countries): + regions_ = list(regions_required_countries.keys()) + tb_agg = geo.add_regions_to_table( + tb=tb, + regions=regions_, + aggregations={"population": "sum", "source": lambda x: "; ".join(sorted(set(x)))}, # ty: ignore + ds_regions=ds_regions, + ds_income_groups=ds_income_groups, + num_allowed_nans_per_year=None, + frac_allowed_nans_per_year=0.2, + countries_that_must_have_data=regions_required_countries, + ) + tb_agg = tb_agg.loc[tb_agg["country"].isin(regions_)] + tb_agg = tb_agg.dropna(subset=["population"]) + return tb_agg + + # Exclude regions and keep continents (<1800), exclude regions+continents (≥1800) + tb = tb.loc[ + ~((tb["year"] < YEAR_START_GAPMINDER) & (tb["country"].isin(regions))) + & ~((tb["year"] >= YEAR_START_GAPMINDER) & tb["country"].isin(regions + continents)) + ].copy() + + # keep sources per countries, remove from tb + # remove from tb: otherwise geo.add_region_aggregates will add this column too + # sources = tb.loc[:, ["country", "year", "source"]].copy() + # tb = tb.drop(columns=["source"]) + + # Get rid of historical regions + ## Otherwise this are counted when estimating region values, and may lead to double-counts + tb_regions = ds_regions["regions"] + historical_regions = set(tb_regions.loc[tb_regions["is_historical"], "name"]) + tb_agg = tb.loc[~tb["country"].isin(historical_regions)].copy() + + # 1/ Estimate aggregates for HYDE ------ + ## <1800 and exclude regions (keep continents) + tb_agg_hyde = tb_agg.loc[tb_agg["year"] < YEAR_START_GAPMINDER].copy() + # re-estimate region aggregates + tb_agg_hyde = _aggregate( + tb=tb_agg_hyde, + regions_required_countries=regions_required, + ) + + # 2/ Estimate aggregates post-HYDE (Gapminder + WPP) ------ + ## ≥1800 and exclude regions + continents + tb_agg = tb_agg.loc[tb_agg["year"] >= YEAR_START_GAPMINDER].copy() + + # Interpolate table + # Keep track of original sources for merging back later + sources_agg = tb_agg.loc[:, ["country", "year", "source"]].copy() + + tb_agg = geo.interpolate_table( + tb_agg.loc[tb_agg["year"] >= YEAR_START_GAPMINDER, ["country", "year", "population"]], + "country", + "year", + ) + + # Add source column back to interpolated data + tb_agg = tb_agg.merge(sources_agg, on=["country", "year"], how="left") + ## Previous interpolation left some sources empty (NA) + tb_agg.loc[(tb_agg["year"] < YEAR_START_FT) & tb_agg["source"].isna(), "source"] = SOURCES_NAMES["hyde"] + # NA rows in the 1800-1949 block were filled by interpolation of F-T values — label them as F-T + tb_agg.loc[ + (tb_agg["year"] >= YEAR_START_FT) & (tb_agg["year"] < YEAR_START_WPP) & tb_agg["source"].isna(), "source" + ] = SOURCES_NAMES["ft"] + tb_agg.loc[(tb_agg["year"] >= YEAR_START_WPP) & tb_agg["source"].isna(), "source"] = SOURCES_NAMES["unwpp"] + assert tb_agg.notna().all().all(), ( + f"Some rows still have missing values! Columns without NaN? {tb_agg.notna().all()}" + ) + # re-estimate region aggregates + tb_agg = _aggregate( + tb=tb_agg, + regions_required_countries=regions_required | continents_required, + ) + + # 3/ Combine aggregates with original data + tb = pr.concat([tb, tb_agg, tb_agg_hyde], ignore_index=True) + + # Ensure all rows have source + assert tb["source"].notna().all(), "Some rows do not have a source!" + vals = tb.loc[tb["country"].isin(continents) & (tb["year"] < YEAR_START_GAPMINDER), "source"].unique() + assert len(vals) == 1 and vals[0] == SOURCES_NAMES["hyde"], ( + f"Unexpected sources for continents before {YEAR_START_GAPMINDER}!" + ) + return tb + + +def add_world(tb: Table) -> Table: + """Add world aggregate. + + We do this by adding the values for all continents. + + HYDE and UN already provide estimates on world population. Therefore, we only estimate the + world population for period in between: 1800 - 1950. + """ + paths.log.info("adding World...") + + # Sanity checks + ## Min year of 'World' for source UN WPP + year_min_un = tb.loc[(tb["country"] == "World") & (tb["source"] == SOURCES_NAMES["unwpp"]), "year"].min() + year_max_un = tb.loc[(tb["country"] == "World") & (tb["source"] == SOURCES_NAMES["unwpp"]), "year"].max() + assert ( + (year_min_un == YEAR_START_WPP) & (year_max_un == YEAR_END_WPP) # This is the year that the UN data starts. + ), "World data found in UN WPP outside of [1950, 2100]!" + ## Min year of 'World' for source HYDE + year_min_hyde = tb.loc[(tb["country"] == "World") & (tb["source"] == SOURCES_NAMES["hyde"]), "year"].min() + year_max_hyde = tb.loc[(tb["country"] == "World") & (tb["source"] == SOURCES_NAMES["hyde"]), "year"].max() + assert ( + (year_min_hyde == YEAR_START_HYDE) & (year_max_hyde == 1940) # This is the year that the UN data starts. + ), "World data found in HYDE outside of [-10000, 1940]!" + + # Filter 'World' in HYDE for period [1800, 1950] + # tb_world = tb.loc[ + # (tb["country"] == "World") & (tb["year"] > YEAR_START_HYDE) & (tb["year"] < YEAR_START_GAPMINDER) + # ].copy() + + # Estimate World using regions + continents = [ + "Europe", + "Asia", + "North America", + "South America", + "Africa", + "Oceania", + ] + # Estimate "World" population for years without UN WPP data. Previously we avoided doing it so for HYDE, but then we could have sum(regions) != World for that period, which is odd. + mask = (tb["year"] < YEAR_START_WPP) & (tb["year"] >= YEAR_START_GAPMINDER) + tb_world = ( + tb[(tb["country"].isin(continents)) & mask] + .groupby("year", as_index=False)["population"] + .sum(numeric_only=True) + .assign(country="World") + ) + + # Remove 'World' from tb + tb = tb.loc[~(mask & (tb["country"] == "World"))] + + # Merge original tb (without World pre-WPP) with World estimates (pre-WPP) + tb = pr.concat([tb, tb_world], ignore_index=True).sort_values(["country", "year"]) + + # add sources for world + tb.loc[tb["country"] == "World", "source"] = "; ".join(sorted(SOURCES_NAMES.values())) + return tb + + +## Add historical regions +def add_historical_regions(tb: Table, tb_gm: Table, tb_regions: Table) -> Table: + """Add historical regions. + + Historical regions are added using different techniques: + + 1. Systema Globalis from Gapminder contains historical regions. We add them to the data. These include + Yugoslavia, USSR, etc. Note that this is added after regions and world regions have been obtained, to avoid double counting. + 2. Add historical regions by grouping and summing current countries. + """ + # 1. Add from Systema Globalis + paths.log.info("loading data (Gapminder Systema Globalis)") + # Add to main table + tb_gm["source"] = SOURCES_NAMES["gapminder_sg"] + tb = pr.concat([tb, tb_gm], ignore_index=True) + + # 2. Add historical regions by grouping and summing current countries. + for code in COUNTRIES_FORMER_EQUIVALENTS: + # Get former country name and end year (dissolution) + former_country_name = tb_regions.loc[code, "name"] + end_year = tb_regions.loc[code, "end_year"] + # Sanity check: former country not already in table! remember that we are creating it now + assert former_country_name not in set(tb["country"]), ( + f"{former_country_name} already in table (either import it via Systema Globalis or manual aggregation)!" + ) + # Get list of country successors (equivalent of former state with nowadays' countries) and end year (dissolution of former state) + codes_successors = json.loads(tb_regions.loc[code, "successors"]) + countries_successors = tb_regions.loc[codes_successors, "name"].tolist() + # Filter table accordingly + tb_suc = tb[(tb["year"] <= end_year) & (tb["country"].isin(countries_successors))] + # Filter rows (only preserve years where all countries have data) + year_filter = tb_suc.groupby("year")["country"].nunique() == len(countries_successors) + year_filter = year_filter[year_filter].index.tolist() + tb_suc = tb_suc[tb_suc["year"].isin(year_filter)] + # Perform operations + tb_suc = tb_suc.groupby("year", as_index=False, observed=True).agg( + {"population": sum, "source": lambda x: "; ".join(sorted(set(x)))} + ) + tb_suc["country"] = former_country_name + # Add to main table + tb = pr.concat([tb, tb_suc], ignore_index=True) + return tb + + +def fix_anomalies(tb: Table) -> Table: + """Make sure that all rows make sense. + + - Remove rows with population = 0. + - Remove datapoints for the Netherland Antilles after 2010 (it was dissolved then), as HYDE has data after that year. + """ + paths.log.info("filter rows...") + # remove datapoints with population = 0 + tb = tb.loc[tb["population"] > 0].copy() + # remove datapoints for the Netherland Antilles after 2010 (it was dissolved then) + tb = tb.loc[~((tb["country"] == "Netherlands Antilles") & (tb["year"] > 2010))] + # remove datapoints for the Netherland Antilles after 2010 (it was dissolved then) + tb = tb.loc[~((tb["country"] == "Serbia and Montenegro") & (tb["year"] > 2006))] + return tb + + +def add_world_population_share(tb: Table) -> Table: + """Obtain world's population share for each country/region and year.""" + paths.log.info("adding world population share...") + # Add a metric "% of world population" + tb_world = tb.loc[tb["country"] == "World", ["year", "population"]].rename(columns={"population": "world_pop"}) + tb = tb.merge(tb_world, on="year", how="left") + tb["world_pop_share"] = 100 * tb["population"].div(tb["world_pop"]) + tb = tb.drop(columns="world_pop") + return tb + + +def generate_auxiliary_table(tb: Table) -> Table: + """Generate an identical table, with a change in the origins. + + This is to be able to show a simpler attribution in charts where population is used as a secondary indicator, and hence there is very little relevance in showing the full attribution. + """ + tb_auxiliary = tb.copy().update_metadata(short_name="population") + + # Origins from original table + origins_raw = tb["population"].metadata.origins + date_accessed = max(origin.date_accessed for origin in origins_raw) + + # Add origins + origins = [ + Origin( + producer="Various sources", + title="Population", + # NEW-OMM variant (Federico-Tena replaces Gapminder for 1800-1938; gap interp 1939-1949). + # The "(NEW)" tag is intentional — it lets chart footers visually distinguish prod + # (which still says "...sources (2024)") from staging at a glance during the A/B. + attribution="Population based on various sources (NEW)", + attribution_short="Population (NEW)", + citation_full="The long-run data on population is based on various sources, described on this page: https://ourworldindata.org/population-sources", + url_main="https://ourworldindata.org/population-sources", + date_accessed=date_accessed, + date_published=paths.version, + description=( + "Our World in Data builds and maintains a long-run dataset on population by country, region, and for the world, based on various sources.\n\n" + + "You can find more information on these sources and how our time series is constructed on this page: https://ourworldindata.org/population-sources" + ), + license=License(name="CC BY 4.0", url="https://creativecommons.org/licenses/by/4.0/"), + ) + ] + ## Add to indicators + for col in tb_auxiliary.columns: + tb_auxiliary[col].origins = origins + + return tb_auxiliary + + +###################### +# Population density +###################### +def make_table_density( + tb_population: Table, tb_land_area: Table, ds_regions: Dataset, ds_income_groups: Dataset +) -> Table: + """Create a table with population density data.""" + paths.log.info("build population density table") + + # Add region aggregates to land area data. + # NOTE: This wasn't done in the corresponding FAOSTAT RL auxiliary step to avoid circular dependencies. + tb_land_area = geo.add_regions_to_table( + tb=tb_land_area, + ds_regions=ds_regions, + ds_income_groups=ds_income_groups, + num_allowed_nans_per_year=None, + frac_allowed_nans_per_year=None, + accepted_overlaps=KNOWN_OVERLAPS_IN_LAND_AREA_DATA, + ) + + # We use land area of countries as they are defined today (latest reported value) + column_area = "land_area__00006601__area__005110__hectares" + tb_land_area = ( + tb_land_area.loc[:, [column_area, "country", "year"]] + .rename(columns={column_area: "area"}) + .sort_values(["country", "year"]) + .drop_duplicates(subset=["country"], keep="last") + .drop(columns=["year"]) + ) + + # Merge tables + tb = tb_population.merge(tb_land_area, on="country", how="inner") + # Drop NaN (no data for area) + tb = tb.dropna(subset=["area"]) + # Estimate population density as population / land_area(in km2) + tb["population_density"] = tb["population"] / (0.01 * tb["area"]) # 0.01 to convert from hectares to km2 + # Select relevant columns, order them, set index + tb = tb.loc[:, ["country", "year", "population_density"]] + return tb + + +######################### +# Population growth rate +######################### +def make_table_growth_rate(tb_population: Table, tb_un_gr: Table) -> Table: + """Build population growth rate table. + + For 1950-2100 (WPP period), we use the UN WPP growth rate directly rather than estimating it + from our population series. This avoids minor discrepancies caused by: + - Our population values being rounded to integers (uint64), which introduces small errors + in the log-ratio calculation. + - The UN computing growth rates from more precise mid-year population estimates, not from + the rounded annual figures published in their population table. + + For pre-1950, we estimate growth rate from our composite population series using: + growth_rate = 100 * ln(P_t / P_{t-1}) / (t - t_{t-1}) + """ + # --- Pre-1950: estimate from population series --- + tb_estimated = _estimate_growth_rate(tb_population) + + # --- 1950-2100: use UN WPP growth rate directly --- + tb_un = tb_un_gr.loc[:, ["country", "year", "growth_rate"]].copy() + + # For countries/regions present in UN data, use UN values; estimate only for the rest + un_countries = set(tb_un["country"]) + mask_covered_by_un = tb_estimated["country"].isin(un_countries) & (tb_estimated["year"] >= YEAR_START_WPP) + tb_estimated = tb_estimated.loc[~mask_covered_by_un] + + # Combine: estimated (pre-1950 + non-UN countries post-1950) with UN WPP (1950-2100) + tb = pr.concat([tb_estimated, tb_un], ignore_index=True) + tb = tb.sort_values(["country", "year"]).reset_index(drop=True) + + # Remove unnecessary metadata fields to avoid errors + tb["growth_rate"].metadata.presentation = None + return tb + + +def _estimate_growth_rate(tb_population: Table) -> Table: + """Estimate population growth rate from the population series. + + Uses the continuous growth formula: growth_rate = 100 * ln(P_t / P_{t-1}) / (t - t_{t-1}) + + Processing: + 1. Calculate growth rates for all consecutive year pairs from 1700 onwards + 2. Smooth source-transition years (1800, 1950) by averaging neighbors + 3. Apply selective display filtering to reduce noise in sparse historical data: + - 1700-1800: Only 100-year intervals (1700, 1800) + - 1800-1900: Only 100-year intervals (1800, 1900) + - 1900-1950: Only 5-year intervals (1900, 1905, 1910, ...) + - 1950+: All years (annual data from UN WPP) + """ + tb = tb_population.sort_values(by=["country", "year"]).copy() + + # Creating the 'previous_population' and 'previous_year' columns + tb["previous_population"] = tb.groupby("country")["population"].shift(1) + tb["previous_year"] = tb.groupby("country")["year"].shift(1) + + # Only since 1700 + tb = tb.loc[tb["year"] >= 1700] + + # Drop rows without previous year + tb = tb.dropna(subset=["previous_year"]) + + # Estimate population growth rate for all consecutive years + tb["growth_rate"] = 100 * ( + np.log(tb["population"] / tb["previous_population"]) / (tb["year"] - tb["previous_year"]) + ) + + # Smooth source-transition years (1800: HYDE→Gapminder, 1950: Gapminder→WPP) + # The raw log-ratio at these boundaries reflects the discontinuity between sources, + # not actual demographic change. We replace with the average of neighbors. + tb["previous_growth_rate"] = tb.groupby("country")["growth_rate"].shift(1) + tb["next_growth_rate"] = tb.groupby("country")["growth_rate"].shift(-1) + + for transition_year in [1800, 1950]: + mask = tb["year"] == transition_year + tb.loc[mask, "growth_rate"] = (tb.loc[mask, "previous_growth_rate"] + tb.loc[mask, "next_growth_rate"]) / 2 + + # Apply selective display filtering to reduce noise in sparse historical data + # Keep only specific year intervals for pre-1950 data, all years for 1950+ + filter_mask = ( + ((tb["year"] >= 1700) & (tb["year"] < 1800) & (tb["year"] % 100 == 0)) + | ((tb["year"] >= 1800) & (tb["year"] < 1900) & (tb["year"] % 100 == 0)) + | ((tb["year"] >= 1900) & (tb["year"] < 1950) & (tb["year"] % 5 == 0)) + | (tb["year"] >= 1950) + ) + tb.loc[~filter_mask, "growth_rate"] = np.nan + + # Keep relevant columns + tb = tb.loc[:, ["country", "year", "growth_rate"]] + return tb + + +########################## +# Historical & Projections +########################## +def make_table_historical(tbs: list[Table]) -> Table: + """Create a table with historical data.""" + tbs_ = [] + for tb in tbs: + tb_ = tb.loc[tb["year"] < YEAR_START_WPP_PROJ].copy() + tbs_.append(tb_) + tb = pr.multi_merge(tbs_, on=COLUMNS_INDEX, how="outer") + + tb.columns = [f"{col}_historical" if col not in COLUMNS_INDEX else col for col in tb.columns] + return tb + + +def make_table_projection(tbs: list[Table]) -> Table: + """Create a table with projection data.""" + tbs_ = [] + for tb in tbs: + tb_ = tb.loc[tb["year"] >= YEAR_START_WPP_PROJ].copy() + tbs_.append(tb_) + tb = pr.multi_merge(tbs_, on=COLUMNS_INDEX, how="outer") + + tb.columns = [f"{col}_projection" if col not in COLUMNS_INDEX else col for col in tb.columns] + return tb diff --git a/etl/steps/data/garden/demography/2026-05-28/utils.py b/etl/steps/data/garden/demography/2026-05-28/utils.py new file mode 100644 index 00000000000..d2dd1b4928f --- /dev/null +++ b/etl/steps/data/garden/demography/2026-05-28/utils.py @@ -0,0 +1,115 @@ +from owid.catalog import License, Origin + +# Year boundaries +YEAR_START_HYDE = -10000 +YEAR_START_FT = 1800 # Federico-Tena replaces Gapminder for 1800-1938 +YEAR_END_FT = 1938 +GAP_LO = 1939 # bridged by linear interpolation +GAP_HI = 1949 +YEAR_START_WPP = 1950 +YEAR_START_WPP_PROJ = 2024 +YEAR_END_WPP = 2100 + +# Backwards-compatible alias used elsewhere as the start of the post-HYDE block +YEAR_START_GAPMINDER = YEAR_START_FT + +# sources names +# this dictionary maps source short names to complete source names +SOURCES_NAMES = { + "unwpp": "United Nations - World Population Prospects (2024) (https://population.un.org/wpp/downloads?folder=Standard%20Projections&group=Population)", + "ft": "Federico-Tena World Population Database — V2, 1991 borders (2026) (https://edatos.consorciomadrono.es/dataset.xhtml?persistentId=doi:10.21950/GW7SOZ)", + "ft_interp": "Linear interpolation between Federico-Tena (1938) and UN WPP (1950)", + "gapminder_sg": "Gapminder - Systema Globalis (2023) (https://github.com/open-numbers/ddf--gapminder--systema_globalis)", + "hyde": "HYDE v3.3 (2023) (https://public.yoda.uu.nl/geo/UU01/AEZZIT.html)", +} + +# Gapminder Systema Globalis contains data on the following countries which can +# complement the other sources. That is, contains older data which other sources don't have. +GAPMINDER_SG_COUNTRIES = { + "akr_a_dhe": "Akrotiri and Dhekelia", + "bmu": "Bermuda", + "vgb": "British Virgin Islands", + "cym": "Cayman Islands", + "cok": "Cook Islands", + "nld_curacao": "Curacao", + "pyf": "French Polynesia", + "gib": "Gibraltar", + "gum": "Guam", + "gbg": "Guernsey", + "gbm": "Isle of Man", + "jey": "Jersey", + "kos": "Kosovo", + "mac": "Macao", + "mnp": "Northern Mariana Islands", + "stbar": "Saint Barthelemy", + "shn": "Saint Helena", + "stmar": "Saint Martin (French part)", + "sxm": "Sint Maarten (Dutch part)", + "tkl": "Tokelau", + "wlf": "Wallis and Futuna", + "ssd": "South Sudan", +} +# former countries (1): sourced from gapminder systema globalis +# to translate code to name: +# 1. use https://github.com/open-numbers/ddf--gapminder--systema_globalis/blob/master/ddf--entities--geo--country.csv +# 2. then ensure namings are aligned with our reference dataset +GAPMINDER_SG_COUNTRIES_FORMER = { + "cheslo": { + "name": "Czechoslovakia", + "end": 1993, + }, + "deu_west": { + "name": "West Germany", + "end": 1990, + }, + "deu_east": { + "name": "East Germany", + "end": 1990, + }, + "eri_a_eth": { + "name": "Ethiopia (former)", + "end": 1993, + }, + # "scg": { + # "name": "Serbia and Montenegro", + # "end": 2006, + # }, + # "ussr": { + # "name": "USSR", + # "end": 1991, + # }, + "yem_north": { + "name": "Yemen Arab Republic", + "end": 1990, + }, + "yem_south": { + "name": "Yemen People's Republic", + "end": 1990, + }, + "yug": { + "name": "Yugoslavia", + "end": 1992, + }, +} +# former countries (2): sourced from our regions dataset +## These countries are added by aggregating their successors' values, and using regions. +COUNTRIES_FORMER_EQUIVALENTS = {"OWID_USS"} + +# Gapminder SG origins +GAPMINDER_SG_ORIGINS = [ + Origin( + producer="Gapminder", + title="Systema Globalis", + citation_full="Gapminder - Systema Globalis (2023)", + url_main="https://github.com/open-numbers/ddf--gapminder--systema_globalis", + attribution="Gapminder - Systema Globalis (2022)", + attribution_short="Gapminder", + date_accessed="2023-03-31", + date_published="2023-02-21", # ty: ignore + description="Data by Gapminder on population and other indicators. It provides data on former countries and regions.", + license=License( + name="CC BY 4.0", + url="https://github.com/open-numbers/ddf--gapminder--systema_globalis", + ), + ) +] diff --git a/etl/steps/data/grapher/demography/2026-05-28/population.py b/etl/steps/data/grapher/demography/2026-05-28/population.py new file mode 100644 index 00000000000..a2f9caffbae --- /dev/null +++ b/etl/steps/data/grapher/demography/2026-05-28/population.py @@ -0,0 +1,42 @@ +"""Load a garden dataset and create a grapher dataset.""" + +from etl.helpers import PathFinder, create_dataset + +# Get paths and naming conventions for current step. +paths = PathFinder(__file__) + + +# Each variable has data from 10,000 BCE until 2100. We create two new versions for each variables: +# - Historical: data from 10,000 BCE until YEAR_THRESHOLD - 1. +# - Projection: data from YEAR_THRESHOLD until 2100. +YEAR_THRESHOLD = 2024 + + +def run(dest_dir: str) -> None: + # + # Load inputs. + # + # Load garden dataset. + ds_garden = paths.load_dataset("population") + + # Read table from garden dataset. + tb_original = ds_garden["population_original"].update_metadata(short_name="population") + + # Set origins on `source` + tb_original.source.m.origins = tb_original.population.m.origins + + # + # Save outputs. + # + tables = [ + tb_original, + ds_garden["population_density"], + ds_garden["population_growth_rate"], + ds_garden["historical"], + ds_garden["projections"], + ] + # Create a new grapher dataset with the same metadata as the garden dataset. + ds_grapher = create_dataset(dest_dir, tables=tables, default_metadata=ds_garden.metadata) + + # Save changes in the new grapher dataset. + ds_grapher.save() From 91320fc06cfd342424bed9de3641a2d8bc3c0282 Mon Sep 17 00:00:00 2001 From: lucasrodes Date: Thu, 28 May 2026 21:22:14 +0200 Subject: [PATCH 07/11] downstream dependencies in dag --- dag/emissions.yml | 6 ++++-- dag/health.yml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dag/emissions.yml b/dag/emissions.yml index 73df557d305..b32c787780d 100644 --- a/dag/emissions.yml +++ b/dag/emissions.yml @@ -62,7 +62,8 @@ steps: data://garden/gcp/2025-11-13/global_carbon_budget: - data://garden/wb/2025-07-01/income_groups - data://garden/regions/2023-01-01/regions - - data://garden/demography/2024-07-15/population + # owid-issues#1232: F-T population OMM for staging A/B (was demography/2024-07-15/population) + - data://garden/demography/2026-05-28/population - data://garden/ggdc/2024-04-26/maddison_project_database - data://garden/energy/2026-05-05/primary_energy_consumption - data://meadow/gcp/2025-11-13/global_carbon_budget: @@ -93,7 +94,8 @@ steps: - snapshot://emissions/2025-12-04/national_contributions_temperature_response.csv - snapshot://emissions/2025-12-04/national_contributions_annual_emissions.csv - snapshot://emissions/2025-12-04/national_contributions_cumulative_emissions.csv - - data://garden/demography/2024-07-15/population + # owid-issues#1232: F-T population OMM for staging A/B (was demography/2024-07-15/population) + - data://garden/demography/2026-05-28/population - data://garden/regions/2023-01-01/regions - data://garden/wb/2025-07-01/income_groups # diff --git a/dag/health.yml b/dag/health.yml index 65796dc4233..c3ada02a0c5 100644 --- a/dag/health.yml +++ b/dag/health.yml @@ -573,7 +573,8 @@ steps: - data://garden/health/2025-02-19/measles_historical: - data://meadow/health/2025-02-19/measles_historical: - snapshot://health/2025-02-19/measles_historical.csv - - data://garden/demography/2024-07-15/population + # owid-issues#1232: F-T population OMM for staging A/B (was demography/2024-07-15/population) + - data://garden/demography/2026-05-28/population # # Cancer survival rates (GCO) @@ -595,7 +596,8 @@ steps: - data://garden/health/2025-02-20/measles_deaths_public_health_reports: - data://meadow/health/2025-02-20/measles_deaths_public_health_reports: - snapshot://health/2025-02-20/measles_deaths_public_health_reports.csv - - data://garden/demography/2024-07-15/population + # owid-issues#1232: F-T population OMM for staging A/B (was demography/2024-07-15/population) + - data://garden/demography/2026-05-28/population # # US - reported diphtheria cases - long-run From 328111d8c34e6d60128a3dd821eaa2a0700706b0 Mon Sep 17 00:00:00 2001 From: lucasrodes Date: Thu, 28 May 2026 21:52:26 +0200 Subject: [PATCH 08/11] change dependencies --- dag/democracy.yml | 12 ++++++------ dag/natural_disasters.yml | 4 +++- dag/war.yml | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/dag/democracy.yml b/dag/democracy.yml index 1a1ed8b04a1..2e63fba87a2 100644 --- a/dag/democracy.yml +++ b/dag/democracy.yml @@ -4,14 +4,14 @@ steps: - data://meadow/democracy/2026-03-17/vdem: - snapshot://democracy/2026-03-17/vdem.zip - data://garden/regions/2023-01-01/regions - - data://garden/demography/2024-07-15/population + - data://garden/demography/2026-05-28/population data://grapher/democracy/2026-03-17/vdem: - data://garden/democracy/2026-03-17/vdem # Episodes of Regime Transition (2026) data://grapher/democracy/2026-03-18/ert: - data://garden/democracy/2026-03-18/ert: - - data://garden/demography/2024-07-15/population + - data://garden/demography/2026-05-28/population - data://meadow/democracy/2026-03-18/ert: - snapshot://democracy/2026-03-18/ert.csv - data://garden/regions/2023-01-01/regions @@ -19,7 +19,7 @@ steps: # Freedom House (2026) data://grapher/democracy/2026-03-23/fh: - data://garden/democracy/2026-03-23/fh: - - data://garden/demography/2024-07-15/population + - data://garden/demography/2026-05-28/population - data://meadow/democracy/2026-03-23/fh: - snapshot://democracy/2026-03-23/fh_ratings.xlsx - snapshot://democracy/2024-05-16/fh_scores.xlsx @@ -32,7 +32,7 @@ steps: - data://meadow/democracy/2026-03-27/bti: - snapshot://democracy/2026-03-27/bti.xlsx - data://garden/regions/2023-01-01/regions - - data://garden/demography/2024-07-15/population + - data://garden/demography/2026-05-28/population # Lexical Index (2026) data://grapher/democracy/2026-04-02/lexical_index: @@ -40,7 +40,7 @@ steps: - data://garden/regions/2023-01-01/regions - data://meadow/democracy/2026-04-02/lexical_index: - snapshot://democracy/2026-04-02/lexical_index.xlsx - - data://garden/demography/2024-07-15/population + - data://garden/demography/2026-05-28/population # EIU (2025) data://grapher/democracy/2025-03-05/eiu: @@ -51,7 +51,7 @@ steps: - snapshot://democracy/2024-05-22/eiu_2022.csv - snapshot://democracy/2024-05-22/eiu_2023.csv - snapshot://democracy/2025-03-05/eiu_2024.csv - - data://garden/demography/2024-07-15/population + - data://garden/demography/2026-05-28/population - data://garden/regions/2023-01-01/regions # Polity (2020) diff --git a/dag/natural_disasters.yml b/dag/natural_disasters.yml index f726a335e20..69164ba787c 100644 --- a/dag/natural_disasters.yml +++ b/dag/natural_disasters.yml @@ -7,7 +7,9 @@ steps: - data://meadow/emdat/2026-04-30/natural_disasters: - snapshot://emdat/2026-04-30/natural_disasters.xlsx - data://garden/worldbank_wdi/2026-02-27/wdi - - data://garden/demography/2024-07-15/population + # owid-issues#1232: F-T population OMM for staging A/B (was demography/2024-07-15/population — + # which is itself now F-T-swapped on this branch, but we pin to the new dated step for clarity) + - data://garden/demography/2026-05-28/population - data://garden/regions/2023-01-01/regions - data://garden/wb/2025-07-01/income_groups # diff --git a/dag/war.yml b/dag/war.yml index 158ba9bbdf2..da8aca2bf33 100644 --- a/dag/war.yml +++ b/dag/war.yml @@ -32,7 +32,7 @@ steps: - snapshot://war/2025-06-13/ucdp_one_sided.zip data://garden/war/2025-06-13/ucdp: - data://meadow/war/2025-06-13/ucdp - - data://garden/demography/2024-07-15/population + - data://garden/demography/2026-05-28/population - data://garden/geography/2025-06-26/geoboundaries_cgaz - data://garden/countries/2024-08-27/gleditsch data://grapher/war/2025-06-13/ucdp: @@ -46,7 +46,7 @@ steps: - data://meadow/war/2025-06-13/ucdp - data://garden/war/2025-06-13/ucdp - data://garden/geography/2025-06-26/geoboundaries_cgaz - - data://garden/demography/2024-07-15/population + - data://garden/demography/2026-05-28/population - data://garden/countries/2024-08-27/gleditsch - data://meadow/war/latest/ucdp_ced: - snapshot://war/latest/ucdp_ced_v25_01_25_12.csv From df1d1c9dc228e07780dda6104e701bd9eb32b77d Mon Sep 17 00:00:00 2001 From: lucasrodes Date: Fri, 5 Jun 2026 21:11:15 +0200 Subject: [PATCH 09/11] =?UTF-8?q?=F0=9F=94=A8=F0=9F=A4=96=20Drop=20Gapmind?= =?UTF-8?q?er=20Systema=20Globalis=20from=20new=20population=20OMM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-05-28 step is now fully Gapminder-free: SG is no longer used as a complement source for small territories nor for former states (Czechoslovakia, Yugoslavia, E/W Germany, the Yemens, Ethiopia (former)). USSR is unaffected (built from successors). Also fixes the stale meta.yml copy that still described the Gapminder-v7 source mix. Co-Authored-By: Claude Opus 4.8 (1M context) --- dag/demography.yml | 5 +- .../demography/2026-05-28/population.meta.yml | 19 ++-- .../demography/2026-05-28/population.py | 92 +++--------------- .../garden/demography/2026-05-28/utils.py | 95 +------------------ 4 files changed, 32 insertions(+), 179 deletions(-) diff --git a/dag/demography.yml b/dag/demography.yml index d2b883eed8f..f959bf6753a 100644 --- a/dag/demography.yml +++ b/dag/demography.yml @@ -34,14 +34,15 @@ steps: # Federico-Tena variant of the OMM (owid-issues#1232 — F-T replaces Gapminder for 1800-1938; # 1939-1949 bridged by per-country linear interpolation between F-T(1938) and UN WPP(1950)). + # Fully Gapminder-free: unlike 2024-07-15, Gapminder Systema Globalis is also dropped (former + # states such as Czechoslovakia/Yugoslavia/E-W Germany are no longer included; USSR remains, + # built from successors). # Mirrors the 2024-07-15 step's output schema so consumers don't need code changes. data://garden/demography/2026-05-28/population: # HYDE 3.3 - data://garden/hyde/2024-01-02/all_indicators # Federico-Tena V2 (1991 borders, 2026) - data://garden/demography/2026-05-20/federico_tena_population - # Gapminder Systema Globalis — used for former states (USSR, Czechoslovakia, ...) only - - data://open_numbers/open_numbers/latest/gapminder__systema_globalis # UN WPP (2024) - data://garden/un/2024-07-12/un_wpp # Auxiliary diff --git a/etl/steps/data/garden/demography/2026-05-28/population.meta.yml b/etl/steps/data/garden/demography/2026-05-28/population.meta.yml index 402107fe7ab..427fe6711dc 100644 --- a/etl/steps/data/garden/demography/2026-05-28/population.meta.yml +++ b/etl/steps/data/garden/demography/2026-05-28/population.meta.yml @@ -6,7 +6,9 @@ definitions: - 10,000 BCE–1799: historical estimates by HYDE (v3.3). - - 1800–1949: historical estimates by Gapminder (v7). + - 1800–1938: historical estimates by the Federico-Tena World Population Database (v2, 1991 borders). + + - 1939–1949: linear interpolation per country between Federico-Tena (1938) and UN WPP (1950). - 1950–2023: population records from the United Nations World Population Prospects (2024 revision). @@ -16,7 +18,7 @@ definitions: - 10,000 BCE–1799: historical estimates by HYDE (v3.3). Growth rate estimated over 50-year periods. - - 1800–1949: historical estimates by Gapminder (v7). Growth rate estimated over 1-year periods. + - 1800–1949: historical estimates by the Federico-Tena World Population Database (v2; 1939–1949 interpolated to UN WPP 1950). Growth rate estimated over 1-year periods. - 1950–2023: population records from the United Nations World Population Prospects (2024 revision). We use the UN's published growth rates directly (based on mid-year population estimates). @@ -74,16 +76,15 @@ tables: presentation: topic_tags: - Population Growth - attribution: HYDE (2023); Gapminder (2022); UN WPP (2024) + attribution: HYDE (2023); Federico–Tena (2026); UN WPP (2024) processing_level: major description_key: &population-description-key - Population is the most commonly used metric throughout Our World in Data. It is used directly to understand population growth over time, and indirectly to calculate per-capita indicators, making it easier to compare countries of different sizes. - |- We construct this indicator by combining multiple sources covering different periods. - HYDE v3.3 (2023): historical estimates from 10,000 BCE to 1799. - - Gapminder v7 (2022): for 1800-1949. + - Federico-Tena World Population Database v2 (2026): for 1800-1938. Values for 1939-1949 are linearly interpolated between Federico-Tena (1938) and UN WPP (1950). - UN World Population Prospects (2024): for 1950 onwards, including 2100 projections. - - Gapminder Systema Globalis (2023): additional source for former countries (Yugoslavia, USSR, etc.) - Breaks in the data may occur at the boundaries between sources due to their methodological differences. - You can read more about the sources and methodology in our [dedicated article](https://ourworldindata.org/population-sources). We also provide a table of sources showing the source we use for each country-year. - We calculate geographical aggregates (continents, income groups, etc.) by summing individual country populations. For years before 1800, we rely directly on HYDE's values for continents to ensure historical consistency. @@ -117,7 +118,7 @@ tables: presentation: topic_tags: - Population Growth - attribution: HYDE (2023); Gapminder (2022); UN WPP (2024); UN FAO (2024) + attribution: HYDE (2023); Federico–Tena (2026); UN WPP (2024); UN FAO (2024) variables: population_density: title: Population density @@ -138,7 +139,7 @@ tables: presentation: topic_tags: - Population Growth - attribution: HYDE (2023); Gapminder (2022); UN WPP (2024) + attribution: HYDE (2023); Federico–Tena (2026); UN WPP (2024) variables: growth_rate: title: Population growth rate @@ -160,7 +161,7 @@ tables: presentation: topic_tags: - Population Growth - attribution: HYDE (2023); Gapminder (2022); UN WPP (2024) + attribution: HYDE (2023); Federico–Tena (2026); UN WPP (2024) processing_level: major description_processing: |- {definitions.others.different_sources_processing} @@ -181,7 +182,7 @@ tables: presentation: title_variant: Long-run data - attribution_short: "HYDE, Gapminder, UN" + attribution_short: "HYDE, Federico–Tena, UN" world_pop_share_historical: title: |- diff --git a/etl/steps/data/garden/demography/2026-05-28/population.py b/etl/steps/data/garden/demography/2026-05-28/population.py index 334a3ce2cd9..cdb63867f99 100644 --- a/etl/steps/data/garden/demography/2026-05-28/population.py +++ b/etl/steps/data/garden/demography/2026-05-28/population.py @@ -6,10 +6,14 @@ - Federico-Tena V2 (1991 borders) for 1800-1938 (replaces Gapminder v7) - linear interpolation for 1939-1949 per country (between F-T(1938) & WPP(1950)) - UN WPP (2024) for >= 1950 (unchanged) - - Gapminder Systema Globalis for former states (USSR, Czechoslovakia, …) — unchanged Notes: - - "Gapminder SG" stands for "Gapminder Systema Globalis". + - Unlike 2024-07-15, this step does NOT use Gapminder Systema Globalis. Consequences: + - Former states sourced from it (Czechoslovakia, Yugoslavia, East/West Germany, + Yemen Arab Republic, Yemen People's Republic, Ethiopia (former)) are no longer + in the dataset. USSR is unaffected (built by aggregating successors). + - Small territories lose the few gap-fill points SG provided where no other source + has data (e.g. Guernsey, Jersey, Kosovo 1800/1820; Bermuda 1730; Macao 1555/1640). - On regional estimates: - Continents: re-estimated for WPP and F-T; HYDE uses original values. - Income groups: TODO @@ -26,9 +30,6 @@ COUNTRIES_FORMER_EQUIVALENTS, GAP_HI, GAP_LO, - GAPMINDER_SG_COUNTRIES, - GAPMINDER_SG_COUNTRIES_FORMER, - GAPMINDER_SG_ORIGINS, SOURCES_NAMES, YEAR_END_FT, YEAR_END_WPP, @@ -73,9 +74,6 @@ def run() -> None: # Load Federico-Tena dataset (replaces Gapminder for 1800-1938) ds_ft = paths.load_dataset("federico_tena_population") tb_ft = ds_ft.read("federico_tena_population") - # Load Gapminder SG dataset - ds_gapminder_sg = paths.load_dataset(short_name="gapminder__systema_globalis", channel="open_numbers") - tb_gapminder_sg = ds_gapminder_sg.read("total_population_with_projections") # Load auxiliary datasets: # * Regions @@ -95,14 +93,13 @@ def run() -> None: tb_ft = format_ft(tb_ft) tb_un = format_wpp(tb_un, "population", "uint64") tb_un_gr = format_wpp(tb_un_gr, "growth_rate", "float32") - tb_gapminder_sg, tb_gapminder_sg_former = format_gapminder_sg(tb_gapminder_sg) # Build the 1939-1949 interpolation rows for every F-T country that also has a 1950 UN WPP value. tb_ft_interp = make_ft_interp(tb_ft, tb_un) # Concat tables tb = pr.concat( - [tb_hyde, tb_ft, tb_ft_interp, tb_un, tb_gapminder_sg], + [tb_hyde, tb_ft, tb_ft_interp, tb_un], ignore_index=True, ) @@ -121,7 +118,7 @@ def run() -> None: tb = ( tb.pipe(add_world) - .pipe(add_historical_regions, tb_gapminder_sg_former, tb_regions) + .pipe(add_historical_regions, tb_regions) .pipe(fix_anomalies) .astype( { @@ -328,55 +325,6 @@ def format_wpp(tb: Table, column_indicator: str, indicator_dtype: str) -> Table: return tb -###################### -# Gapminder SG ####### -###################### -def format_gapminder_sg(tb: Table) -> tuple[Table, Table]: - """Format Gapminder Systema Globalis table.""" - columns_rename = { - "country": "country", - "time": "year", - "total_population_with_projections": "population", - } - - def _core_formatting(tb: Table, country_rename: dict[str, str]) -> Table: - ## rename countries - tb["country"] = tb["geo"].map(country_rename) - ## rename columns - tb = tb.rename(columns=columns_rename, errors="raise").loc[:, COLUMS_RELEVANT_POP] - # Set source identifier - tb["source"] = "gapminder_sg" - # add origins - tb["population"].metadata.origins = GAPMINDER_SG_ORIGINS - return tb - - # Data on former countries - ## only keep former country data - tb_former: Table = tb.loc[tb["geo"].isin(GAPMINDER_SG_COUNTRIES_FORMER)].copy() - - # core formatting: column and country rename, add source, metadata - tb_former = _core_formatting( - tb=tb_former, - country_rename={code: data["name"] for code, data in GAPMINDER_SG_COUNTRIES_FORMER.items()}, - ) - - ## filter years: only keep former countries until they disappear - for _, data in GAPMINDER_SG_COUNTRIES_FORMER.items(): - tb_former = tb_former.loc[~((tb_former["country"] == data["name"]) & (tb_former["year"] > data["end"]))] - - # Complement - ## filter countries - tb = tb.loc[tb["geo"].isin(GAPMINDER_SG_COUNTRIES)] - - # core formatting: column and country rename, add source, metadata - tb = _core_formatting( - tb=tb, - country_rename=GAPMINDER_SG_COUNTRIES, - ) - - return tb, tb_former - - ############################################################################################# # Combine and process data ################################################################## ############################################################################################# @@ -622,30 +570,20 @@ def add_world(tb: Table) -> Table: ## Add historical regions -def add_historical_regions(tb: Table, tb_gm: Table, tb_regions: Table) -> Table: - """Add historical regions. - - Historical regions are added using different techniques: +def add_historical_regions(tb: Table, tb_regions: Table) -> Table: + """Add historical regions by grouping and summing current countries. - 1. Systema Globalis from Gapminder contains historical regions. We add them to the data. These include - Yugoslavia, USSR, etc. Note that this is added after regions and world regions have been obtained, to avoid double counting. - 2. Add historical regions by grouping and summing current countries. + NOTE: Unlike the 2024-07-15 step, historical regions are no longer sourced from Gapminder + Systema Globalis — only successor-aggregation (currently USSR) remains. This is added after + regions and world aggregates have been obtained, to avoid double counting. """ - # 1. Add from Systema Globalis - paths.log.info("loading data (Gapminder Systema Globalis)") - # Add to main table - tb_gm["source"] = SOURCES_NAMES["gapminder_sg"] - tb = pr.concat([tb, tb_gm], ignore_index=True) - - # 2. Add historical regions by grouping and summing current countries. + # Add historical regions by grouping and summing current countries. for code in COUNTRIES_FORMER_EQUIVALENTS: # Get former country name and end year (dissolution) former_country_name = tb_regions.loc[code, "name"] end_year = tb_regions.loc[code, "end_year"] # Sanity check: former country not already in table! remember that we are creating it now - assert former_country_name not in set(tb["country"]), ( - f"{former_country_name} already in table (either import it via Systema Globalis or manual aggregation)!" - ) + assert former_country_name not in set(tb["country"]), f"{former_country_name} already in table!" # Get list of country successors (equivalent of former state with nowadays' countries) and end year (dissolution of former state) codes_successors = json.loads(tb_regions.loc[code, "successors"]) countries_successors = tb_regions.loc[codes_successors, "name"].tolist() diff --git a/etl/steps/data/garden/demography/2026-05-28/utils.py b/etl/steps/data/garden/demography/2026-05-28/utils.py index d2dd1b4928f..1ace76b953b 100644 --- a/etl/steps/data/garden/demography/2026-05-28/utils.py +++ b/etl/steps/data/garden/demography/2026-05-28/utils.py @@ -1,5 +1,3 @@ -from owid.catalog import License, Origin - # Year boundaries YEAR_START_HYDE = -10000 YEAR_START_FT = 1800 # Federico-Tena replaces Gapminder for 1800-1938 @@ -19,97 +17,12 @@ "unwpp": "United Nations - World Population Prospects (2024) (https://population.un.org/wpp/downloads?folder=Standard%20Projections&group=Population)", "ft": "Federico-Tena World Population Database — V2, 1991 borders (2026) (https://edatos.consorciomadrono.es/dataset.xhtml?persistentId=doi:10.21950/GW7SOZ)", "ft_interp": "Linear interpolation between Federico-Tena (1938) and UN WPP (1950)", - "gapminder_sg": "Gapminder - Systema Globalis (2023) (https://github.com/open-numbers/ddf--gapminder--systema_globalis)", "hyde": "HYDE v3.3 (2023) (https://public.yoda.uu.nl/geo/UU01/AEZZIT.html)", } -# Gapminder Systema Globalis contains data on the following countries which can -# complement the other sources. That is, contains older data which other sources don't have. -GAPMINDER_SG_COUNTRIES = { - "akr_a_dhe": "Akrotiri and Dhekelia", - "bmu": "Bermuda", - "vgb": "British Virgin Islands", - "cym": "Cayman Islands", - "cok": "Cook Islands", - "nld_curacao": "Curacao", - "pyf": "French Polynesia", - "gib": "Gibraltar", - "gum": "Guam", - "gbg": "Guernsey", - "gbm": "Isle of Man", - "jey": "Jersey", - "kos": "Kosovo", - "mac": "Macao", - "mnp": "Northern Mariana Islands", - "stbar": "Saint Barthelemy", - "shn": "Saint Helena", - "stmar": "Saint Martin (French part)", - "sxm": "Sint Maarten (Dutch part)", - "tkl": "Tokelau", - "wlf": "Wallis and Futuna", - "ssd": "South Sudan", -} -# former countries (1): sourced from gapminder systema globalis -# to translate code to name: -# 1. use https://github.com/open-numbers/ddf--gapminder--systema_globalis/blob/master/ddf--entities--geo--country.csv -# 2. then ensure namings are aligned with our reference dataset -GAPMINDER_SG_COUNTRIES_FORMER = { - "cheslo": { - "name": "Czechoslovakia", - "end": 1993, - }, - "deu_west": { - "name": "West Germany", - "end": 1990, - }, - "deu_east": { - "name": "East Germany", - "end": 1990, - }, - "eri_a_eth": { - "name": "Ethiopia (former)", - "end": 1993, - }, - # "scg": { - # "name": "Serbia and Montenegro", - # "end": 2006, - # }, - # "ussr": { - # "name": "USSR", - # "end": 1991, - # }, - "yem_north": { - "name": "Yemen Arab Republic", - "end": 1990, - }, - "yem_south": { - "name": "Yemen People's Republic", - "end": 1990, - }, - "yug": { - "name": "Yugoslavia", - "end": 1992, - }, -} -# former countries (2): sourced from our regions dataset +# Former countries, sourced from our regions dataset. ## These countries are added by aggregating their successors' values, and using regions. +## NOTE: Unlike the 2024-07-15 step, no former states are imported from Gapminder Systema +## Globalis (Czechoslovakia, Yugoslavia, East/West Germany, the Yemens, Ethiopia (former) +## are therefore no longer in the dataset). COUNTRIES_FORMER_EQUIVALENTS = {"OWID_USS"} - -# Gapminder SG origins -GAPMINDER_SG_ORIGINS = [ - Origin( - producer="Gapminder", - title="Systema Globalis", - citation_full="Gapminder - Systema Globalis (2023)", - url_main="https://github.com/open-numbers/ddf--gapminder--systema_globalis", - attribution="Gapminder - Systema Globalis (2022)", - attribution_short="Gapminder", - date_accessed="2023-03-31", - date_published="2023-02-21", # ty: ignore - description="Data by Gapminder on population and other indicators. It provides data on former countries and regions.", - license=License( - name="CC BY 4.0", - url="https://github.com/open-numbers/ddf--gapminder--systema_globalis", - ), - ) -] From f14b87cea75661f314670a6b8a4c38bf715996fb Mon Sep 17 00:00:00 2001 From: lucasrodes Date: Wed, 10 Jun 2026 14:56:12 +0200 Subject: [PATCH 10/11] =?UTF-8?q?=F0=9F=93=8A=20Update=20historical=20regi?= =?UTF-8?q?ons=20logic=20to=20include=20optional=20successors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../demography/2026-05-28/population.py | 35 ++++++++++++------- .../garden/demography/2026-05-28/utils.py | 26 +++++++++++--- 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/etl/steps/data/garden/demography/2026-05-28/population.py b/etl/steps/data/garden/demography/2026-05-28/population.py index cdb63867f99..5f66a9fcd5d 100644 --- a/etl/steps/data/garden/demography/2026-05-28/population.py +++ b/etl/steps/data/garden/demography/2026-05-28/population.py @@ -30,6 +30,7 @@ COUNTRIES_FORMER_EQUIVALENTS, GAP_HI, GAP_LO, + OPTIONAL_SUCCESSORS, SOURCES_NAMES, YEAR_END_FT, YEAR_END_WPP, @@ -571,28 +572,38 @@ def add_world(tb: Table) -> Table: ## Add historical regions def add_historical_regions(tb: Table, tb_regions: Table) -> Table: - """Add historical regions by grouping and summing current countries. + """Add historical regions by summing the population of their present-day successor countries. NOTE: Unlike the 2024-07-15 step, historical regions are no longer sourced from Gapminder - Systema Globalis — only successor-aggregation (currently USSR) remains. This is added after - regions and world aggregates have been obtained, to avoid double counting. + Systema Globalis — they are rebuilt purely by successor-aggregation (USSR, Czechoslovakia, + Yugoslavia, Ethiopia (former); see COUNTRIES_FORMER_EQUIVALENTS). Only states whose territory is + the union of whole present-day countries can be reconstructed this way; East/West Germany and the + two Yemens cannot and are intentionally absent. This runs after region and world aggregates have + been obtained, so these entities never enter those sums (avoids double counting). + + A successor listed in OPTIONAL_SUCCESSORS is summed when it has data but is not required every + year (e.g. Kosovo, which only has data from 1950 and is otherwise subsumed in Federico-Tena's + Serbia). Years are kept iff every *required* (non-optional) successor has data. """ - # Add historical regions by grouping and summing current countries. for code in COUNTRIES_FORMER_EQUIVALENTS: # Get former country name and end year (dissolution) former_country_name = tb_regions.loc[code, "name"] end_year = tb_regions.loc[code, "end_year"] # Sanity check: former country not already in table! remember that we are creating it now assert former_country_name not in set(tb["country"]), f"{former_country_name} already in table!" - # Get list of country successors (equivalent of former state with nowadays' countries) and end year (dissolution of former state) + # Get successors (equivalent of the former state with nowadays' countries), splitting out + # any optional ones (summed when present, not required to gate the year). codes_successors = json.loads(tb_regions.loc[code, "successors"]) - countries_successors = tb_regions.loc[codes_successors, "name"].tolist() - # Filter table accordingly - tb_suc = tb[(tb["year"] <= end_year) & (tb["country"].isin(countries_successors))] - # Filter rows (only preserve years where all countries have data) - year_filter = tb_suc.groupby("year")["country"].nunique() == len(countries_successors) - year_filter = year_filter[year_filter].index.tolist() - tb_suc = tb_suc[tb_suc["year"].isin(year_filter)] + optional_codes = OPTIONAL_SUCCESSORS.get(code, set()) + required_codes = [c for c in codes_successors if c not in optional_codes] + countries_all = tb_regions.loc[codes_successors, "name"].tolist() + countries_required = tb_regions.loc[required_codes, "name"].tolist() + # Filter to successor rows up to the former state's dissolution year + tb_suc = tb[(tb["year"] <= end_year) & (tb["country"].isin(countries_all))] + # Keep only years where every required successor has data (optional ones are added if present) + n_required = tb_suc[tb_suc["country"].isin(countries_required)].groupby("year")["country"].nunique() + valid_years = n_required[n_required == len(countries_required)].index + tb_suc = tb_suc[tb_suc["year"].isin(valid_years)] # Perform operations tb_suc = tb_suc.groupby("year", as_index=False, observed=True).agg( {"population": sum, "source": lambda x: "; ".join(sorted(set(x)))} diff --git a/etl/steps/data/garden/demography/2026-05-28/utils.py b/etl/steps/data/garden/demography/2026-05-28/utils.py index 1ace76b953b..10b0b94aa16 100644 --- a/etl/steps/data/garden/demography/2026-05-28/utils.py +++ b/etl/steps/data/garden/demography/2026-05-28/utils.py @@ -21,8 +21,24 @@ } # Former countries, sourced from our regions dataset. -## These countries are added by aggregating their successors' values, and using regions. -## NOTE: Unlike the 2024-07-15 step, no former states are imported from Gapminder Systema -## Globalis (Czechoslovakia, Yugoslavia, East/West Germany, the Yemens, Ethiopia (former) -## are therefore no longer in the dataset). -COUNTRIES_FORMER_EQUIVALENTS = {"OWID_USS"} +## These former states are rebuilt by summing the population of their present-day successor +## countries (so they need no Gapminder Systema Globalis import). Only states whose territory is +## the union of *whole* present-day countries can be reconstructed this way — East/West Germany +## and the two Yemens cannot (each maps to a single modern country, Germany / Yemen, so summing +## can't split them) and are therefore intentionally absent. +COUNTRIES_FORMER_EQUIVALENTS = { + "OWID_USS", # USSR -> 15 post-Soviet republics + "OWID_CZS", # Czechoslovakia -> Czechia + Slovakia + "OWID_YGS", # Yugoslavia -> Bosnia, Croatia, N. Macedonia, Montenegro, Kosovo, Serbia, Slovenia + "OWID_ERE", # Ethiopia (former) -> Ethiopia + Eritrea +} + +# Successors that should be summed when present but NOT required to have data every year. +## Kosovo only has data from 1950 (UN WPP); before then it is already included in Federico-Tena's +## "Serbia" (Kosovo was not a separate polity at 1991 borders). Requiring it would either drop all +## pre-1950 years of Yugoslavia or, if forced, double-count it. Treating it as optional gives the +## correct sum in both eras: pre-1950 from the 6 core successors (Serbia already covers Kosovo), +## 1950+ with Kosovo added (UN WPP's Serbia excludes it). +OPTIONAL_SUCCESSORS = { + "OWID_YGS": {"OWID_KOS"}, # Kosovo +} From f83c896eb972ed171212f79e132bffebcc424657 Mon Sep 17 00:00:00 2001 From: lucasrodes Date: Tue, 7 Jul 2026 17:39:17 +0200 Subject: [PATCH 11/11] =?UTF-8?q?=F0=9F=93=8A=20Update=20HYDE=20version=20?= =?UTF-8?q?to=203.5=20in=20population=20datasets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dag/demography.yml | 4 +-- .../demography/2026-05-28/population.meta.yml | 14 +++++----- .../demography/2026-05-28/population.py | 28 +++++++++++++++---- .../garden/demography/2026-05-28/utils.py | 2 +- 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/dag/demography.yml b/dag/demography.yml index 6bd75d6add4..48eba919d0e 100644 --- a/dag/demography.yml +++ b/dag/demography.yml @@ -39,8 +39,8 @@ steps: # built from successors). # Mirrors the 2024-07-15 step's output schema so consumers don't need code changes. data://garden/demography/2026-05-28/population: - # HYDE 3.3 - - data://garden/hyde/2024-01-02/all_indicators + # HYDE 3.5 + - data://garden/hyde/2026-06-08/all_indicators # Federico-Tena V2 (1991 borders, 2026) - data://garden/demography/2026-05-20/federico_tena_population # UN WPP (2024) diff --git a/etl/steps/data/garden/demography/2026-05-28/population.meta.yml b/etl/steps/data/garden/demography/2026-05-28/population.meta.yml index 427fe6711dc..5998790b43b 100644 --- a/etl/steps/data/garden/demography/2026-05-28/population.meta.yml +++ b/etl/steps/data/garden/demography/2026-05-28/population.meta.yml @@ -4,7 +4,7 @@ definitions: ### Combination of different sources We construct our long-run population data by combining multiple sources: - - 10,000 BCE–1799: historical estimates by HYDE (v3.3). + - 10,000 BCE–1799: historical estimates by HYDE (v3.5). - 1800–1938: historical estimates by the Federico-Tena World Population Database (v2, 1991 borders). @@ -16,7 +16,7 @@ definitions: ### Combination of different sources We construct our long-run data by combining multiple sources: - - 10,000 BCE–1799: historical estimates by HYDE (v3.3). Growth rate estimated over 50-year periods. + - 10,000 BCE–1799: historical estimates by HYDE (v3.5). Growth rate estimated over 50-year periods. - 1800–1949: historical estimates by the Federico-Tena World Population Database (v2; 1939–1949 interpolated to UN WPP 1950). Growth rate estimated over 1-year periods. @@ -76,13 +76,13 @@ tables: presentation: topic_tags: - Population Growth - attribution: HYDE (2023); Federico–Tena (2026); UN WPP (2024) + attribution: HYDE (2025); Federico–Tena (2026); UN WPP (2024) processing_level: major description_key: &population-description-key - Population is the most commonly used metric throughout Our World in Data. It is used directly to understand population growth over time, and indirectly to calculate per-capita indicators, making it easier to compare countries of different sizes. - |- We construct this indicator by combining multiple sources covering different periods. - - HYDE v3.3 (2023): historical estimates from 10,000 BCE to 1799. + - HYDE v3.5 (2025): historical estimates from 10,000 BCE to 1799. - Federico-Tena World Population Database v2 (2026): for 1800-1938. Values for 1939-1949 are linearly interpolated between Federico-Tena (1938) and UN WPP (1950). - UN World Population Prospects (2024): for 1950 onwards, including 2100 projections. - Breaks in the data may occur at the boundaries between sources due to their methodological differences. @@ -118,7 +118,7 @@ tables: presentation: topic_tags: - Population Growth - attribution: HYDE (2023); Federico–Tena (2026); UN WPP (2024); UN FAO (2024) + attribution: HYDE (2025); Federico–Tena (2026); UN WPP (2024); UN FAO (2024) variables: population_density: title: Population density @@ -139,7 +139,7 @@ tables: presentation: topic_tags: - Population Growth - attribution: HYDE (2023); Federico–Tena (2026); UN WPP (2024) + attribution: HYDE (2025); Federico–Tena (2026); UN WPP (2024) variables: growth_rate: title: Population growth rate @@ -161,7 +161,7 @@ tables: presentation: topic_tags: - Population Growth - attribution: HYDE (2023); Federico–Tena (2026); UN WPP (2024) + attribution: HYDE (2025); Federico–Tena (2026); UN WPP (2024) processing_level: major description_processing: |- {definitions.others.different_sources_processing} diff --git a/etl/steps/data/garden/demography/2026-05-28/population.py b/etl/steps/data/garden/demography/2026-05-28/population.py index 5f66a9fcd5d..f1b5cc9ab17 100644 --- a/etl/steps/data/garden/demography/2026-05-28/population.py +++ b/etl/steps/data/garden/demography/2026-05-28/population.py @@ -2,7 +2,7 @@ owid-issues#1232; see reports/population_ft/). Stitching: - - HYDE 3.3 for years < 1800 (unchanged from 2024-07-15) + - HYDE 3.5 for years < 1800 (2024-07-15 used HYDE 3.3) - Federico-Tena V2 (1991 borders) for 1800-1938 (replaces Gapminder v7) - linear interpolation for 1939-1949 per country (between F-T(1938) & WPP(1950)) - UN WPP (2024) for >= 1950 (unchanged) @@ -42,6 +42,7 @@ ) from etl.data_helpers import geo +from etl.data_helpers.misc import interpolate_table from etl.helpers import PathFinder # Get paths and naming conventions for current step. @@ -476,10 +477,27 @@ def _aggregate(tb, regions_required_countries): # Keep track of original sources for merging back later sources_agg = tb_agg.loc[:, ["country", "year", "source"]].copy() - tb_agg = geo.interpolate_table( - tb_agg.loc[tb_agg["year"] >= YEAR_START_GAPMINDER, ["country", "year", "population"]], - "country", - "year", + tb_agg = tb_agg.loc[tb_agg["year"] >= YEAR_START_GAPMINDER, ["country", "year", "population"]] + first_year_observed = tb_agg.groupby("country", observed=True)["year"].min() + + # Fill HYDE's decadal pre-1950 points to annual values so region aggregates can be built for + # every year. NOTE: interpolation must stay strictly within each country's own data range + # (`full_range_entity` + interior-only). Extrapolating beyond it (the old + # `geo.interpolate_table` default, `limit_direction="both"`) back-filled entities whose data + # starts in 1950 (South Sudan, Kosovo, Jersey, ...) flat across 1800-1949 inside region + # aggregates — double-counting people already included in F-T's 1991-borders parents + # (e.g. Kosovo inside Serbia). + tb_agg = interpolate_table( + tb_agg, + entity_col="country", + time_col="year", + time_mode="full_range_entity", + limit_area="inside", + ) + first_year_interp = tb_agg.groupby("country", observed=True)["year"].min() + assert first_year_interp.ge(first_year_observed.reindex(first_year_interp.index)).all(), ( + "Interpolation created rows before a country's first actual data point — this back-fills " + "post-1950 entities into pre-1950 region aggregates." ) # Add source column back to interpolated data diff --git a/etl/steps/data/garden/demography/2026-05-28/utils.py b/etl/steps/data/garden/demography/2026-05-28/utils.py index 10b0b94aa16..babebda724c 100644 --- a/etl/steps/data/garden/demography/2026-05-28/utils.py +++ b/etl/steps/data/garden/demography/2026-05-28/utils.py @@ -17,7 +17,7 @@ "unwpp": "United Nations - World Population Prospects (2024) (https://population.un.org/wpp/downloads?folder=Standard%20Projections&group=Population)", "ft": "Federico-Tena World Population Database — V2, 1991 borders (2026) (https://edatos.consorciomadrono.es/dataset.xhtml?persistentId=doi:10.21950/GW7SOZ)", "ft_interp": "Linear interpolation between Federico-Tena (1938) and UN WPP (1950)", - "hyde": "HYDE v3.3 (2023) (https://public.yoda.uu.nl/geo/UU01/AEZZIT.html)", + "hyde": "HYDE v3.5 (2025) (https://doi.org/10.24416/UU01-F45D44)", } # Former countries, sourced from our regions dataset.