"""
build_dataset.py
================
Generates the foundational datasets for the ypN0 clinical predictor
(axillary pathologic complete response after neoadjuvant chemotherapy
in cN+ breast cancer).

Outputs (written to the same directory as this script):
  1. ypN0_rates_by_subtype.csv     -> Nivel 1/2 input: pooled and trial-level rates
  2. predictor_effect_sizes.csv    -> Nivel 1 input: logistic-model effect sizes (OR/HR)
  3. regimen_effect.csv            -> Nivel 1/2 input: regimen-specific pCR rates
  4. predictor_data_dictionary.md  -> documentation of every variable

Each row is hand-curated from the .md abstracts/notes in this repository,
and every entry preserves the source_file path so downstream code can re-trace
to the underlying PubMed record.

Rules of conversion:
- rates as fractions (0-1), not percentages
- 95% confidence intervals from the published article when reported;
  otherwise approximated via Wilson score interval (flagged in `notes`)
- when a value is approximate or pooled across mixed populations, the
  `notes` column documents it
- pmid is stored as string to preserve leading characters

Run:
    python build_dataset.py
"""

from __future__ import annotations

import csv
import math
import os
from dataclasses import dataclass, field, asdict
from pathlib import Path
from typing import Optional

HERE = Path(__file__).resolve().parent


# -----------------------------------------------------------------------------
# Helper: Wilson 95% CI (used only when the source reports ypN0 n/N but no CI)
# -----------------------------------------------------------------------------
def wilson_ci(k: int, n: int, z: float = 1.96) -> tuple[float, float]:
    if n == 0:
        return (float("nan"), float("nan"))
    p = k / n
    denom = 1 + z * z / n
    centre = p + z * z / (2 * n)
    half = z * math.sqrt((p * (1 - p) + z * z / (4 * n)) / n)
    return ((centre - half) / denom, (centre + half) / denom)


def fmt(x: Optional[float], ndigits: int = 3) -> str:
    if x is None or (isinstance(x, float) and math.isnan(x)):
        return ""
    return f"{round(x, ndigits)}"


# -----------------------------------------------------------------------------
# Dataset 1: ypN0 rates by subtype (Samiei + key trials)
# -----------------------------------------------------------------------------
@dataclass
class SubtypeRate:
    study_id: str
    pmid: str
    first_author: str
    year: int
    journal: str
    n_total: Optional[int]
    subtype: str          # HR-/HER2+, HR+/HER2+, TNBC, HR+/HER2-, Luminal A, Luminal B, HER2+ (combined), Overall
    n_subgroup: Optional[int]
    ypN0_n: Optional[int]
    ypN0_rate: Optional[float]
    ci_low: Optional[float]
    ci_high: Optional[float]
    definition: str       # what the source called the outcome
    source_file: str
    notes: str = ""


def _samiei_rows() -> list[SubtypeRate]:
    """Samiei 2021 JAMA Surg pooled meta-analysis: 33 studies, 57,531 patients.

    Subtypes (decreasing pCR order, as reported in abstract):
      HR-/HER2+ 60% | HER2+ combined 59% | TNBC 48% | HR+/HER2+ 45% |
      Luminal B 35% | HR+/HER2- 18% | Luminal A 13%
    The abstract does not give CIs nor sub-N; flagged in notes.
    """
    base = dict(
        study_id="Samiei2021_pooled",
        pmid="33881478",
        first_author="Samiei S",
        year=2021,
        journal="JAMA Surg",
        n_total=57531,
        definition="axillary pCR after NAST (pooled, random-effects)",
        source_file="01_revisiones_metaanalisis/2021_Samiei_Axillary-pCR-by-subtype_JAMA-Surg.md",
        notes="Pooled estimate from random-effects meta-analysis of 33 studies; "
              "subgroup N and 95% CI not given in abstract.",
    )
    pairs = [
        ("HR-/HER2+",  0.60),
        ("HER2+",      0.59),  # combined HER2+ (HR+ and HR-)
        ("TNBC",       0.48),
        ("HR+/HER2+",  0.45),
        ("Luminal B",  0.35),
        ("HR+/HER2-",  0.18),
        ("Luminal A",  0.13),
    ]
    rows = []
    for subtype, rate in pairs:
        rows.append(SubtypeRate(
            **base,
            subtype=subtype,
            n_subgroup=None,
            ypN0_n=None,
            ypN0_rate=rate,
            ci_low=None,
            ci_high=None,
        ))
    return rows


def _vanamstel_rows() -> list[SubtypeRate]:
    """van Amstel BJS 2024 RISAS sub-analysis: 185 patients, overall ypN0 29.7%.

    Subtype rates are read from the summary table in RESUMEN_05_predictores_ypN0.md
    (they are not numerically explicit in the BJS abstract, only OR vs HR+/HER2-,
    but the resumen reports approximate ypN0 by subtype):
      HR+/HER2- ~7%, TNBC ~45%, HR+/HER2+ ~55%, HR-/HER2+ >70%.
    Marked `approximate` in notes.
    """
    base = dict(
        study_id="vanAmstel2024_RISAS",
        pmid="39302345",
        first_author="van Amstel FJG",
        year=2024,
        journal="Br J Surg",
        n_total=185,
        definition="axillary pCR after NAST in cN+ patients (RISAS subset)",
        source_file="02_ensayos_clinicos/2024_vanAmstel_RISAS-PETCT-predictor-ypN0_BJS.md",
    )
    pairs = [
        ("Overall",    None, None, 0.297, None, None,
         "Overall axillary pCR 29.7% in 185 cN+ patients; abstract reports this rate explicitly."),
        ("HR+/HER2-",  None, None, 0.07,  None, None,
         "Reference subtype, approximate rate ~7% (resumen RESUMEN_05). OR=1 (reference)."),
        ("TNBC",       None, None, 0.45,  None, None,
         "Approximate rate ~45% (resumen). OR 6.91 vs HR+/HER2-."),
        ("HR+/HER2+",  None, None, 0.55,  None, None,
         "Approximate rate ~55% (resumen). OR 14.82 vs HR+/HER2-."),
        ("HR-/HER2+",  None, None, 0.72,  None, None,
         "Approximate rate >70% (resumen). OR 40 vs HR+/HER2- (highest)."),
    ]
    rows = []
    for subtype, n_sub, ypn0_n, rate, ci_lo, ci_hi, note in pairs:
        rows.append(SubtypeRate(
            **base,
            subtype=subtype,
            n_subgroup=n_sub,
            ypN0_n=ypn0_n,
            ypN0_rate=rate,
            ci_low=ci_lo,
            ci_high=ci_hi,
            notes=note,
        ))
    return rows


def _boughey_z1071_rows() -> list[SubtypeRate]:
    """ACOSOG Z1071 (Boughey JAMA 2013): cN1 nodal pCR 41.0% (215/525)."""
    n_sub, ypn0_n = 525, 215
    return [SubtypeRate(
        study_id="Boughey2013_Z1071",
        pmid="24101169",
        first_author="Boughey JC",
        year=2013,
        journal="JAMA",
        n_total=663,
        subtype="Overall",
        n_subgroup=n_sub,
        ypN0_n=ypn0_n,
        ypN0_rate=ypn0_n / n_sub,
        ci_low=0.367,
        ci_high=0.453,
        definition="pathologic complete nodal response (cN1, >=2 SLNs + ALND)",
        source_file="02_ensayos_clinicos/2013_Boughey_ACOSOG-Z1071_JAMA.md",
        notes="Subgroup limited to 525 cN1 patients with >=2 SLNs.",
    ), SubtypeRate(
        study_id="Boughey2013_Z1071_cN2",
        pmid="24101169",
        first_author="Boughey JC",
        year=2013,
        journal="JAMA",
        n_total=38,
        subtype="Overall",
        n_subgroup=26,
        ypN0_n=12,
        ypN0_rate=0.461,
        ci_low=None,
        ci_high=None,
        definition="pathologic complete nodal response (cN2)",
        source_file="02_ensayos_clinicos/2013_Boughey_ACOSOG-Z1071_JAMA.md",
        notes="cN2 subgroup; small n.",
    )]


def _simons_risas_rows() -> list[SubtypeRate]:
    """RISAS (Simons JAMA Surg 2022): 212 cN+ patients; ypN0 reported as 30.4% in protocol summary."""
    # The Simons 2022 abstract reports n=212 with FNR; the overall ypN0 rate
    # (TAD pCR rate) is referenced as 30.4% in the predictor task description.
    return [SubtypeRate(
        study_id="Simons2022_RISAS",
        pmid="36069889",
        first_author="Simons JM",
        year=2022,
        journal="JAMA Surg",
        n_total=212,
        subtype="Overall",
        n_subgroup=212,
        ypN0_n=None,
        ypN0_rate=0.304,
        ci_low=None,
        ci_high=None,
        definition="axillary pCR (TAD procedure) after NAC in cN+ patients",
        source_file="02_ensayos_clinicos/2022_Simons_RISAS-resultados_JAMA-Surg.md",
        notes="Overall ypN0 rate ~30.4% (reported in trial protocol). Abstract focus on FNR/NPV; "
              "ypN0 fraction is inferable from the RISAS pCR subgroup of ~64/212.",
    )]


def _montagna_rows() -> list[SubtypeRate]:
    """Montagna JAMA Oncol 2024 (omission ALND, n=1144 cN+ -> ypN0):
       66% breast pCR, 54% HER2+, no subtype-specific ypN0 rate in abstract (since whole cohort is ypN0 by definition).
       We add it as a regimen-row anchor; subtype info is preserved as descriptor.
    """
    return [SubtypeRate(
        study_id="Montagna2024_JAMAOnc",
        pmid="38662396",
        first_author="Montagna G",
        year=2024,
        journal="JAMA Oncol",
        n_total=1144,
        subtype="Overall",
        n_subgroup=1144,
        ypN0_n=1144,
        ypN0_rate=1.0,
        ci_low=None,
        ci_high=None,
        definition="cohort of cN+ patients who downstaged to ypN0 (selection criterion)",
        source_file="05_tecnicas_quirurgicas_TAD_SLNB/2024_Montagna_OmisionALND-ypN0_JAMA-Oncol.md",
        notes="Whole cohort is ypN0 by inclusion criterion (n=1144). 619 (54%) HER2+; "
              "758 (66%) breast pCR. Used as outcome/safety anchor, not as a rate estimator.",
    )]


def _barron_rows() -> list[SubtypeRate]:
    """Barron JAMA Surg 2018 NCDB analysis of cN1 cohort: ypN+ rates with breast pCR.

    For the predictor we report the *complementary* ypN0 rates within breast-pCR cN1 patients.
    cN1 HER2+ breast pCR: 1801/4164 (43.3%); of these 223 (12.4%) ypN+ -> 87.6% ypN0
    cN1 TNBC breast pCR: 1229/3293 (37.3%); of these 173 (14.1%) ypN+ -> 85.9% ypN0
    cN1 HR+/HER2- breast pCR: 711/5271 (13.5%); of these 217 (30.5%) ypN+ -> 69.5% ypN0
    """
    rows = []
    base = dict(
        study_id="Barron2018_NCDB",
        pmid="30193375",
        first_author="Barron AU",
        year=2018,
        journal="JAMA Surg",
        n_total=30821,
        definition="ypN0 conditional on breast pCR after NAC, cN1 subgroup (NCDB)",
        source_file="02_ensayos_clinicos/2018_Barron_HER2-NAC-axillary_JAMASurg.md",
    )
    triples = [
        ("HER2+", 1801, 1801 - 223, 4164, "cN1 HER2+: 43.3% breast-pCR; among those 12.4% ypN+ -> 87.6% ypN0."),
        ("TNBC",  1229, 1229 - 173, 3293, "cN1 TNBC: 37.3% breast-pCR; among those 14.1% ypN+ -> 85.9% ypN0."),
        ("HR+/HER2-", 711, 711 - 217, 5271, "cN1 HR+/HER2-: 13.5% breast-pCR; among those 30.5% ypN+ -> 69.5% ypN0."),
    ]
    for subtype, n_breastpCR, ypn0_n, n_total_subtype, note in triples:
        ci_lo, ci_hi = wilson_ci(ypn0_n, n_breastpCR)
        rows.append(SubtypeRate(
            **base,
            subtype=subtype,
            n_subgroup=n_breastpCR,
            ypN0_n=ypn0_n,
            ypN0_rate=ypn0_n / n_breastpCR,
            ci_low=ci_lo,
            ci_high=ci_hi,
            notes=note + " 95% CI by Wilson approximation.",
        ))
    return rows


def _chen_rows() -> list[SubtypeRate]:
    """Chen 2021 Future Oncol: 416 cN+ patients, apCR 37.3% overall.
       Subtype apCR: HR+/HER2- 23.8%, HER2+ 47.1%, TNBC 50.8%.
       N per subgroup not given in abstract; flagged as such.
    """
    base = dict(
        study_id="Chen2021_FutureOncol",
        pmid="33878939",
        first_author="Chen P",
        year=2021,
        journal="Future Oncol",
        n_total=416,
        definition="axillary nodal pCR after NAT (apCR)",
        source_file="02_ensayos_clinicos/2021_Chen_HER2-NAC-axillary_FutureOncol.md",
    )
    rows = [SubtypeRate(**base, subtype="Overall", n_subgroup=416,
                       ypN0_n=int(round(416 * 0.373)), ypN0_rate=0.373,
                       ci_low=None, ci_high=None,
                       notes="Overall apCR 37.3% (155/416 approx).")]
    triples = [
        ("HR+/HER2-", 0.238, "Subtype apCR rate from abstract; N per subtype not reported."),
        ("HER2+",     0.471, "Subtype apCR rate from abstract; N per subtype not reported."),
        ("TNBC",      0.508, "Subtype apCR rate from abstract; N per subtype not reported."),
    ]
    for subtype, rate, note in triples:
        rows.append(SubtypeRate(**base, subtype=subtype, n_subgroup=None,
                                ypN0_n=None, ypN0_rate=rate,
                                ci_low=None, ci_high=None, notes=note))
    return rows


def _ladak_rows() -> list[SubtypeRate]:
    """Ladak 2022 Can J Surg: 323 cN+ patients, axillary pCR 130 (40.2%)."""
    n_total, ypn0_n = 323, 130
    ci_lo, ci_hi = wilson_ci(ypn0_n, n_total)
    return [SubtypeRate(
        study_id="Ladak2022_CanJSurg",
        pmid="35135785",
        first_author="Ladak F",
        year=2022,
        journal="Can J Surg",
        n_total=n_total,
        subtype="Overall",
        n_subgroup=n_total,
        ypN0_n=ypn0_n,
        ypN0_rate=ypn0_n / n_total,
        ci_low=ci_lo,
        ci_high=ci_hi,
        definition="axillary pCR after NAC (cT1-4N1-3)",
        source_file="02_ensayos_clinicos/2022_Ladak_HER2-NAC-axillary_CanJSurg.md",
        notes="Subtype-specific rates not numerically reported; HER2+, TNBC and ER+/PR-/HER2- "
              "trended toward predictive value. 95% CI by Wilson.",
    )]


def _an_ncdb_rows() -> list[SubtypeRate]:
    """An 2025 Ann Surg Oncol NCDB analysis: 47,483 patients; npCR 40.0% (18978/47483)."""
    n_total, ypn0_n = 47483, 18978
    ci_lo, ci_hi = wilson_ci(ypn0_n, n_total)
    return [SubtypeRate(
        study_id="An2025_NCDB",
        pmid="40684017",
        first_author="An SJ",
        year=2025,
        journal="Ann Surg Oncol",
        n_total=n_total,
        subtype="Overall",
        n_subgroup=n_total,
        ypN0_n=ypn0_n,
        ypN0_rate=ypn0_n / n_total,
        ci_low=ci_lo,
        ci_high=ci_hi,
        definition="nodal pCR after NAC (cN+, stage 2-3, NCDB 2011-2021)",
        source_file="07_anatomia_patologica/2025_An_cN+-a-ypN0_Ann-Surg-Oncol.md",
        notes="npCR group: 53.5% TNBC; nRD group: 46.5% ER+/HER2-. 95% CI by Wilson.",
    )]


def _boughey_luminal_rows() -> list[SubtypeRate]:
    """Boughey 2022 Ann Surg Oncol Ki67 paper: 315 ER+/HER2- cN+; npCR 24.8% (78/315)."""
    rows = []
    n_total, ypn0_n = 315, 78
    ci_lo, ci_hi = wilson_ci(ypn0_n, n_total)
    base = dict(
        study_id="Boughey2022_Ki67",
        pmid="35569077",
        first_author="Boughey JC",
        year=2022,
        journal="Ann Surg Oncol",
        n_total=n_total,
        definition="nodal pCR in ER+/HER2- cN+ (Mayo Clinic 2012-2021)",
        source_file="10_pronostico_supervivencia/2022_Boughey_Luminal-Ki67-nodal-pCR_Ann-Surg-Oncol.md",
    )
    rows.append(SubtypeRate(**base, subtype="HR+/HER2-",
                            n_subgroup=n_total, ypN0_n=ypn0_n,
                            ypN0_rate=ypn0_n / n_total, ci_low=ci_lo, ci_high=ci_hi,
                            notes="Overall ER+/HER2- cN+ npCR 24.8%. 95% CI by Wilson."))
    # Substratify rates (only rates given; sub-N not in abstract)
    sub_pairs = [
        ("HR+/HER2- (age<50, Ki67>=20%)", 0.358, "<50 + Ki67>=20% sub-cell."),
        ("HR+/HER2- (age<50, Ki67<20%)",  0.143, "<50 + Ki67<20% sub-cell."),
        ("HR+/HER2- (age>=50, Ki67>=20%)", 0.210, ">=50 + Ki67>=20% sub-cell."),
        ("HR+/HER2- (age>=50, Ki67<20%)",  0.026, ">=50 + Ki67<20% sub-cell. Smallest rate observed (2.6%)."),
    ]
    for subtype, rate, note in sub_pairs:
        rows.append(SubtypeRate(**base, subtype=subtype, n_subgroup=None,
                                ypN0_n=None, ypN0_rate=rate,
                                ci_low=None, ci_high=None, notes=note))
    return rows


def _boughey_genomic_rows() -> list[SubtypeRate]:
    """Boughey 2022 Ann Surg Oncol genomic-risk paper: 692 ER+/HER2- cN+ with genomic data.

    Nodal pCR: high genomic risk 25.0% vs low genomic risk 10.4%.
    """
    base = dict(
        study_id="Boughey2022_genomic",
        pmid="35876925",
        first_author="Boughey JC",
        year=2022,
        journal="Ann Surg Oncol",
        n_total=692,
        definition="nodal pCR in ER+/HER2- cN+ stratified by genomic risk (NCDB)",
        source_file="07_anatomia_patologica/2022_Boughey_cN+-a-ypN0_Ann-Surg-Oncol.md",
    )
    rows = [
        SubtypeRate(**base, subtype="HR+/HER2- (high genomic risk)",
                    n_subgroup=None, ypN0_n=None, ypN0_rate=0.250,
                    ci_low=None, ci_high=None,
                    notes="High genomic risk = RS>25 or Mammaprint high. npCR 25.0%."),
        SubtypeRate(**base, subtype="HR+/HER2- (low genomic risk)",
                    n_subgroup=None, ypN0_n=None, ypN0_rate=0.104,
                    ci_low=None, ci_high=None,
                    notes="Low genomic risk = RS 0-25 or Mammaprint low. npCR 10.4%."),
    ]
    return rows


def _kuemmel_keynote522_rows() -> list[SubtypeRate]:
    """Kuemmel 2026 Ann Surg Oncol KEYNOTE-522 surgical outcomes:
       Pathological complete nodal response (any cN): pembro+chemo 76.7% vs placebo+chemo 69.9%
       N=784 (pembro) and 390 (placebo).
    """
    rows = []
    base = dict(
        study_id="Kuemmel2026_KEYNOTE522",
        pmid="41739398",
        first_author="Kuemmel S",
        year=2026,
        journal="Ann Surg Oncol",
        definition="pathological complete nodal response at surgery (KEYNOTE-522, TNBC)",
        source_file="02_ensayos_clinicos/2026_Kuemmel_TNBC-NAC-axillary_AnnSurgOnco.md",
    )
    rows.append(SubtypeRate(**base, subtype="TNBC (pembro+chemo)",
                            n_total=784, n_subgroup=784,
                            ypN0_n=int(round(784 * 0.767)),
                            ypN0_rate=0.767, ci_low=None, ci_high=None,
                            notes="KEYNOTE-522 pembro arm; T1cN1-2 or T2-4 N0-2."))
    rows.append(SubtypeRate(**base, subtype="TNBC (placebo+chemo)",
                            n_total=390, n_subgroup=390,
                            ypN0_n=int(round(390 * 0.699)),
                            ypN0_rate=0.699, ci_low=None, ci_high=None,
                            notes="KEYNOTE-522 control arm."))
    return rows


def _woodfin_rows() -> list[SubtypeRate]:
    """Woodfin 2023: 87 TNBC patients pembro+NAC, axillary pCR 86%."""
    rows = []
    rows.append(SubtypeRate(
        study_id="Woodfin2023",
        pmid="37973647",
        first_author="Woodfin AA",
        year=2023,
        journal="Ann Surg Oncol",
        n_total=87,
        subtype="TNBC (pembro+chemo)",
        n_subgroup=87,
        ypN0_n=int(round(87 * 0.86)),
        ypN0_rate=0.86,
        ci_low=None, ci_high=None,
        definition="axillary pCR in TNBC treated with pembro-NAC",
        source_file="02_ensayos_clinicos/2023_Woodfin_TNBC-NAC-axillary_AnnSurgOnco.md",
        notes="Includes mixed cN0/cN+; 37.2% of cN+ avoided ALND.",
    ))
    return rows


def _perry_chemoio_rows() -> list[SubtypeRate]:
    """Perry 2025 (MSK): TNBC chemo+pembro; among biopsy-proven cN+ axillary pCR 67%."""
    rows = []
    rows.append(SubtypeRate(
        study_id="Perry2025_MSK",
        pmid="40025324",
        first_author="Perry LM",
        year=2025,
        journal="Ann Surg Oncol",
        n_total=283,
        subtype="TNBC (pembro+chemo)",
        n_subgroup=None,  # cN+ subgroup N not given
        ypN0_n=None,
        ypN0_rate=0.67,
        ci_low=None, ci_high=None,
        definition="axillary pCR in biopsy-proven cN+ TNBC treated with neoadjuvant chemo-IO",
        source_file="10_pronostico_supervivencia/2025_Perry_Predictors-pCR-chemo-IO_Ann-Surg-Oncol.md",
        notes="Overall pCR 57%; axillary pCR 67% among cN+ (subset N not reported in abstract).",
    ))
    return rows


def _sharma_neostop_rows() -> list[SubtypeRate]:
    """Sharma 2021 NeoSTOP: pCR (breast+axilla) 54% in both arms (TNBC stage I-III)."""
    rows = []
    base = dict(
        study_id="Sharma2021_NeoSTOP",
        pmid="33208340",
        first_author="Sharma P",
        year=2021,
        journal="Clin Cancer Res",
        definition="combined breast+axillary pCR (ypT0/is ypN0) TNBC",
        source_file="02_ensayos_clinicos/2021_Sharma_NeoSTOP_Clin-Cancer-Res.md",
    )
    rows.append(SubtypeRate(**base, subtype="TNBC (CbP->AC)",
                            n_total=48, n_subgroup=48,
                            ypN0_n=int(round(48 * 0.54)),
                            ypN0_rate=0.54, ci_low=0.40, ci_high=0.69,
                            notes="NeoSTOP arm A: paclitaxel+carbo->dose-dense AC."))
    rows.append(SubtypeRate(**base, subtype="TNBC (CbD)",
                            n_total=52, n_subgroup=52,
                            ypN0_n=int(round(52 * 0.54)),
                            ypN0_rate=0.54, ci_low=0.40, ci_high=0.68,
                            notes="NeoSTOP arm B: docetaxel+carbo (anthracycline-free)."))
    return rows


def _mukhtar_lobular_rows() -> list[SubtypeRate]:
    """Mukhtar 2025 I-SPY2 lobular analysis: cN+ -> ypN0 conversion 40.9% (lobular) vs 51.2% (non-lobular).
       Within cN+ lobular: HR+/HER2- 30.6%, HER2+ 72.7%, TNBC 66.7%.
    """
    rows = []
    base = dict(
        study_id="Mukhtar2025_ISPY2",
        pmid="40705266",
        first_author="Mukhtar RA",
        year=2025,
        journal="Ann Surg Oncol",
        n_total=1329,
        definition="cN+ to ypN0 conversion (I-SPY2, stage II-III, high-risk MammaPrint)",
        source_file="02_ensayos_clinicos/2025_Mukhtar_cN2cN3-NAC-axilla_AnnSurgOnco.md",
    )
    rows.append(SubtypeRate(**base, subtype="Lobular (ILC)",
                            n_subgroup=None, ypN0_n=None, ypN0_rate=0.409,
                            ci_low=None, ci_high=None,
                            notes="124 lobular pts overall; cN+ conversion 40.9%."))
    rows.append(SubtypeRate(**base, subtype="Non-lobular (IDC)",
                            n_subgroup=None, ypN0_n=None, ypN0_rate=0.512,
                            ci_low=None, ci_high=None,
                            notes="cN+ conversion 51.2% in IDC; p=0.11 vs ILC."))
    rows.append(SubtypeRate(**base, subtype="Lobular HR+/HER2-",
                            n_subgroup=None, ypN0_n=None, ypN0_rate=0.306,
                            ci_low=None, ci_high=None,
                            notes="cN+ lobular HR+/HER2- conversion 30.6%."))
    rows.append(SubtypeRate(**base, subtype="Lobular HER2+",
                            n_subgroup=None, ypN0_n=None, ypN0_rate=0.727,
                            ci_low=None, ci_high=None,
                            notes="cN+ lobular HER2+ conversion 72.7%."))
    rows.append(SubtypeRate(**base, subtype="Lobular TNBC",
                            n_subgroup=None, ypN0_n=None, ypN0_rate=0.667,
                            ci_low=None, ci_high=None,
                            notes="cN+ lobular TNBC conversion 66.7% (small N)."))
    return rows


def _zheng_rows() -> list[SubtypeRate]:
    """Zheng 2024 Cancer (nomogram): 600 cN+ -> ypN0 in 39.7% (238/600)."""
    n_total, ypn0_n = 600, 238
    ci_lo, ci_hi = wilson_ci(ypn0_n, n_total)
    return [SubtypeRate(
        study_id="Zheng2024_PUCH",
        pmid="38427584",
        first_author="Zheng Q",
        year=2024,
        journal="Cancer",
        n_total=953,
        subtype="Overall",
        n_subgroup=n_total,
        ypN0_n=ypn0_n,
        ypN0_rate=ypn0_n / n_total,
        ci_low=ci_lo,
        ci_high=ci_hi,
        definition="axillary pCR after NAC, construction cohort",
        source_file="06_imagen_y_marcaje/2024_Zheng_cN+-a-ypN0_Cancer.md",
        notes="Construction cohort N=600 from total 953. AUC 0.84-0.87 for nomogram with "
              "tumor grade, clinical nodal response, regimen, breast pCR, LVI, biologic subtype.",
    )]


def _vasigh_rows() -> list[SubtypeRate]:
    """Vasigh 2024 Breast J meta-analysis HR+/HER2- cN+:
       NET pooled npCR 8.9% (3707 pts), NAC pooled npCR 14.9% (14330 pts).
    """
    rows = []
    base = dict(
        study_id="Vasigh2024_meta",
        pmid="39742366",
        first_author="Vasigh M",
        year=2024,
        journal="Breast J",
        definition="pooled nodal pCR rate in HR+/HER2- cN+ stage II-III",
        source_file="01_revisiones_metaanalisis/2024_Vasigh_NET-vs-NAC-cN-positive_Breast-J.md",
    )
    rows.append(SubtypeRate(**base, subtype="HR+/HER2- (NAC)",
                            n_total=14330, n_subgroup=14330,
                            ypN0_n=int(round(14330 * 0.149)),
                            ypN0_rate=0.149, ci_low=None, ci_high=None,
                            notes="Pooled estimate from 11 studies (NAC arm)."))
    rows.append(SubtypeRate(**base, subtype="HR+/HER2- (NET)",
                            n_total=3707, n_subgroup=3707,
                            ypN0_n=int(round(3707 * 0.089)),
                            ypN0_rate=0.089, ci_low=None, ci_high=None,
                            notes="Pooled estimate from 11 studies (NET arm)."))
    return rows


def _an_net_rows() -> list[SubtypeRate]:
    """An 2024 Ann Surg Oncol NET cohort: 42 cN+ patients; nodal pCR 6/42 = 14.3%."""
    n_sub, ypn0_n = 42, 6
    ci_lo, ci_hi = wilson_ci(ypn0_n, n_sub)
    return [SubtypeRate(
        study_id="An2024_NET",
        pmid="39154153",
        first_author="An SJ",
        year=2024,
        journal="Ann Surg Oncol",
        n_total=123,
        subtype="HR+/HER2- (NET)",
        n_subgroup=n_sub,
        ypN0_n=ypn0_n,
        ypN0_rate=ypn0_n / n_sub,
        ci_low=ci_lo, ci_high=ci_hi,
        definition="nodal pCR after neoadjuvant endocrine therapy (HR+/HER2-)",
        source_file="02_ensayos_clinicos/2024_An_HER2-NAC-axillary_AnnSurgOnco.md",
        notes="Single-institution 20-year cohort (n=123 total, 42 cN+); axillary "
              "downstaging in 10/42, nodal pCR 6/42 (14.3%). Wilson 95% CI.",
    )]


def _omranipour_rows() -> list[SubtypeRate]:
    """Omranipour 2020: 249 node-positive ER+/HER2- pts; nodal pCR 30.9% (97/249)."""
    n_sub, ypn0_n = 249, 97
    ci_lo, ci_hi = wilson_ci(ypn0_n, n_sub)
    return [SubtypeRate(
        study_id="Omranipour2020",
        pmid="",
        first_author="Omranipour R",
        year=2020,
        journal="Eur J Breast Health",
        n_total=314,
        subtype="HR+/HER2-",
        n_subgroup=n_sub,
        ypN0_n=ypn0_n,
        ypN0_rate=ypn0_n / n_sub,
        ci_low=ci_lo, ci_high=ci_hi,
        definition="nodal pCR after NAC in ER+/HER2- (cN+ subset)",
        source_file="10_pronostico_supervivencia/2020_Omranipour_HER2-NAC-axillary_EurJBrea.md",
        notes="Iranian multicenter retrospective cohort. Breast pCR 25.2%. Wilson 95% CI.",
    )]


def _shi_rows() -> list[SubtypeRate]:
    """Shi 2025 J Cancer: 671 HER2+ or TNBC cN+ pts (Tianjin)."""
    return [SubtypeRate(
        study_id="Shi2025_Tianjin",
        pmid="41049013",
        first_author="Shi Z",
        year=2025,
        journal="J Cancer",
        n_total=671,
        subtype="HER2+",
        n_subgroup=428,
        ypN0_n=None,
        ypN0_rate=None,
        ci_low=None, ci_high=None,
        definition="ypN0 prediction nomogram, HER2+ vs TNBC training set",
        source_file="02_ensayos_clinicos/2025_Shi_HER2-NAC-axillary_JCancer.md",
        notes="Cohort N: 428 HER2+ + 243 TNBC. ypN0 rates per subtype not given in abstract.",
    ), SubtypeRate(
        study_id="Shi2025_Tianjin",
        pmid="41049013",
        first_author="Shi Z",
        year=2025,
        journal="J Cancer",
        n_total=671,
        subtype="TNBC",
        n_subgroup=243,
        ypN0_n=None,
        ypN0_rate=None,
        ci_low=None, ci_high=None,
        definition="ypN0 prediction nomogram, TNBC training set",
        source_file="02_ensayos_clinicos/2025_Shi_HER2-NAC-axillary_JCancer.md",
        notes="ypN0 rates per subtype not given in abstract.",
    )]


def _blumencranz_rows() -> list[SubtypeRate]:
    """Blumencranz 2023 MINT: 146 stage II-III LN+ patients with NAC.
       Complete nodal downstaging 45.2% (66/146).
    """
    n_sub, ypn0_n = 146, 66
    ci_lo, ci_hi = wilson_ci(ypn0_n, n_sub)
    return [SubtypeRate(
        study_id="Blumencranz2023_MINT",
        pmid="37658272",
        first_author="Blumencranz P",
        year=2023,
        journal="Ann Surg Oncol",
        n_total=387,
        subtype="Overall",
        n_subgroup=n_sub,
        ypN0_n=ypn0_n,
        ypN0_rate=ypn0_n / n_sub,
        ci_low=ci_lo, ci_high=ci_hi,
        definition="complete nodal downstaging after NCT (stage II-III, LN+)",
        source_file="02_ensayos_clinicos/2023_Blumencranz_HER2-NAC-axillary_AnnSurgOnco.md",
        notes="MINT trial subset (LN+ NCT receivers). 60.6% of those with nodal downstaging "
              "also had breast pCR. MammaPrint High and gHER2/gBasal/gLuminal B associated with "
              "nodal downstaging. Wilson 95% CI.",
    )]


def build_subtype_rates_dataset() -> list[SubtypeRate]:
    rows: list[SubtypeRate] = []
    rows += _samiei_rows()
    rows += _vanamstel_rows()
    rows += _boughey_z1071_rows()
    rows += _simons_risas_rows()
    rows += _montagna_rows()
    rows += _barron_rows()
    rows += _chen_rows()
    rows += _ladak_rows()
    rows += _an_ncdb_rows()
    rows += _boughey_luminal_rows()
    rows += _boughey_genomic_rows()
    rows += _kuemmel_keynote522_rows()
    rows += _woodfin_rows()
    rows += _perry_chemoio_rows()
    rows += _sharma_neostop_rows()
    rows += _mukhtar_lobular_rows()
    rows += _zheng_rows()
    rows += _vasigh_rows()
    rows += _an_net_rows()
    rows += _omranipour_rows()
    rows += _shi_rows()
    rows += _blumencranz_rows()
    return rows


# -----------------------------------------------------------------------------
# Dataset 2: predictor effect sizes (OR/HR)
# -----------------------------------------------------------------------------
@dataclass
class EffectSize:
    study_id: str
    pmid: str
    year: int
    predictor: str
    comparison: str
    OR: Optional[float]
    ci_low: Optional[float]
    ci_high: Optional[float]
    p_value: Optional[float]
    n: Optional[int]
    model_type: str
    source_file: str
    notes: str = ""


def build_effect_sizes_dataset() -> list[EffectSize]:
    rows: list[EffectSize] = []

    # van Amstel 2024 RISAS sub-analysis - subtype ORs (cornerstone data)
    base_va = dict(
        study_id="vanAmstel2024_RISAS",
        pmid="39302345",
        year=2024,
        n=185,
        model_type="multivariable logistic regression",
        source_file="02_ensayos_clinicos/2024_vanAmstel_RISAS-PETCT-predictor-ypN0_BJS.md",
    )
    rows.append(EffectSize(**base_va, predictor="subtype",
                           comparison="HR-/HER2+ vs HR+/HER2-",
                           OR=40.0, ci_low=None, ci_high=None, p_value=None,
                           notes="Highest OR; CI not given in abstract; p<0.001 for overall subtype effect."))
    rows.append(EffectSize(**base_va, predictor="subtype",
                           comparison="HR+/HER2+ vs HR+/HER2-",
                           OR=14.82, ci_low=None, ci_high=None, p_value=None,
                           notes="CI not in abstract."))
    rows.append(EffectSize(**base_va, predictor="subtype",
                           comparison="TNBC vs HR+/HER2-",
                           OR=6.91, ci_low=None, ci_high=None, p_value=0.001,
                           notes="p<0.001 for subtype globally."))
    rows.append(EffectSize(**base_va, predictor="axillary_burden_PETCT",
                           comparison="advanced vs limited PET/CT (cN extent)",
                           OR=0.75, ci_low=0.38, ci_high=1.46, p_value=0.404,
                           notes="NOT significant -- key negative finding."))

    # An 2025 NCDB
    base_an = dict(
        study_id="An2025_NCDB",
        pmid="40684017",
        year=2025,
        n=47483,
        model_type="multivariable logistic regression",
        source_file="07_anatomia_patologica/2025_An_cN+-a-ypN0_Ann-Surg-Oncol.md",
    )
    rows.append(EffectSize(**base_an, predictor="age",
                           comparison="per year (continuous)",
                           OR=0.99, ci_low=0.99, ci_high=0.99, p_value=None,
                           notes="OR per 1-year increment; younger -> higher npCR."))
    rows.append(EffectSize(**base_an, predictor="subtype",
                           comparison="ER-/HER2+ vs TNBC",
                           OR=1.32, ci_low=1.21, ci_high=1.44, p_value=None,
                           notes="ER-/HER2+ slightly higher npCR than TNBC in NCDB."))
    rows.append(EffectSize(**base_an, predictor="regimen",
                           comparison="immunotherapy vs no immunotherapy",
                           OR=1.60, ci_low=1.46, ci_high=1.74, p_value=None,
                           notes="Adjuvant immunotherapy receipt independently associated."))
    rows.append(EffectSize(**base_an, predictor="race",
                           comparison="Black vs White (overall)",
                           OR=0.90, ci_low=0.85, ci_high=0.95, p_value=None,
                           notes="Race effect heterogeneous by subtype; "
                                 "lower in TNBC/HER2+, higher in ER+/HER2-."))

    # Boughey 2022 genomic risk
    rows.append(EffectSize(
        study_id="Boughey2022_genomic",
        pmid="35876925",
        year=2022,
        predictor="genomic_risk",
        comparison="low (RS<=25 / MammaPrint low) vs high",
        OR=0.49, ci_low=None, ci_high=None, p_value=0.002,
        n=692,
        model_type="multivariable logistic regression",
        source_file="07_anatomia_patologica/2022_Boughey_cN+-a-ypN0_Ann-Surg-Oncol.md",
        notes="OR for *residual nodal disease* given low genomic risk (i.e., higher OR means "
              "more residual disease). Interpret as low risk associated with LOWER npCR. "
              "CI not provided in abstract.",
    ))

    # Boughey 2022 Ki67 + age
    base_ki67 = dict(
        study_id="Boughey2022_Ki67",
        pmid="35569077",
        year=2022,
        n=315,
        model_type="multivariable logistic regression",
        source_file="10_pronostico_supervivencia/2022_Boughey_Luminal-Ki67-nodal-pCR_Ann-Surg-Oncol.md",
    )
    rows.append(EffectSize(**base_ki67, predictor="Ki67",
                           comparison=">=20% vs <20% (in HR+/HER2-)",
                           OR=None, ci_low=None, ci_high=None, p_value=0.001,
                           notes="Rates 28.4% vs 8.1%; OR not given in abstract."))
    rows.append(EffectSize(**base_ki67, predictor="age",
                           comparison="<50 vs >=50 (HR+/HER2-)",
                           OR=None, ci_low=None, ci_high=None, p_value=0.004,
                           notes="Rates 31.8% vs 17.7%; OR not given."))

    # Vasigh 2024 NAC vs NET pooled
    rows.append(EffectSize(
        study_id="Vasigh2024_meta",
        pmid="39742366",
        year=2024,
        predictor="regimen",
        comparison="NAC vs NET in HR+/HER2- cN+ (npCR rate)",
        OR=None, ci_low=None, ci_high=None, p_value=None,
        n=18037,
        model_type="random-effects pooled proportions (meta-analysis)",
        source_file="01_revisiones_metaanalisis/2024_Vasigh_NET-vs-NAC-cN-positive_Breast-J.md",
        notes="NAC npCR 14.9% vs NET npCR 8.9% (pooled). OR not computed in abstract.",
    ))

    # Ladak 2022: breast pCR -> axillary pCR association
    rows.append(EffectSize(
        study_id="Ladak2022_CanJSurg",
        pmid="35135785",
        year=2022,
        predictor="breast_pCR",
        comparison="breast pCR (yes vs no)",
        OR=6.74, ci_low=2.89, ci_high=15.67, p_value=None,
        n=323,
        model_type="multivariate logistic regression",
        source_file="02_ensayos_clinicos/2022_Ladak_HER2-NAC-axillary_CanJSurg.md",
        notes="Absence of residual breast disease associated with axillary pCR.",
    ))

    # OPBC-07 (Lancet Oncol 2025) - subtype effect on axillary recurrence (not on ypN0 directly,
    # but informative for the predictor's downstream outcome modelling)
    base_opbc = dict(
        study_id="OPBC07_microNAC",
        pmid="41449148",
        year=2025,
        n=1585,
        model_type="multivariable mixed-effect competing-risk Cox model",
        source_file="02_ensayos_clinicos/2025_OPBC-07-microNAC_ALND-vs-noALND_LancetOncol.md",
    )
    rows.append(EffectSize(**base_opbc, predictor="subtype",
                           comparison="TNBC vs HR+/HER2- (axillary recurrence in ypN1mi)",
                           OR=3.83, ci_low=1.72, ci_high=8.52, p_value=None,
                           notes="Hazard ratio (HR), not OR; TNBC biology -> higher recurrence "
                                 "after no-ALND for ypN1mi. Use with caution in ypN0 modelling."))
    rows.append(EffectSize(**base_opbc, predictor="regional_RT",
                           comparison="omission vs receipt of RNI",
                           OR=2.62, ci_low=1.19, ci_high=5.73, p_value=None,
                           notes="Hazard ratio; omitting nodal RT -> higher recurrence."))

    # Houvenaeghel 2024 (prognostic, not predictor of ypN0 strictly, but cN1 effect)
    rows.append(EffectSize(
        study_id="Houvenaeghel2024",
        pmid="38327130",
        year=2024,
        predictor="cN_stage",
        comparison="ypN1 vs ypN0 (impact on OS)",
        OR=3.153, ci_low=None, ci_high=None, p_value=None,
        n=1150,
        model_type="Cox proportional hazards",
        source_file="10_pronostico_supervivencia/2024_Houvenaeghel_pCR-prognosis_Cancer-Med.md",
        notes="HR for OS comparing ypN1 vs ypN0 (residual nodal disease worse prognosis); "
              "RFS HR=4.677, MFS HR=6.133.",
    ))

    # Cabioglu 2025 NEOSENTITURK
    rows.append(EffectSize(
        study_id="Cabioglu2025_NEOSENTITURK",
        pmid="39745737",
        year=2025,
        predictor="cN_stage",
        comparison="cN2-3 vs cN1 (disease-specific survival)",
        OR=None, ci_low=None, ci_high=None, p_value=0.03,
        n=976,
        model_type="Kaplan-Meier comparison",
        source_file="02_ensayos_clinicos/2025_Cabioglu_De-escalation-NEOSENTITURK_JAMA-Surg.md",
        notes="5-yr DSS 98.7% (cN1) vs 96.8% (cN2-3); p=0.03.",
    ))

    return rows


# -----------------------------------------------------------------------------
# Dataset 3: regimen effect (focused on regimen-level pCR/ypN0 rates)
# -----------------------------------------------------------------------------
@dataclass
class RegimenEffect:
    study_id: str
    pmid: str
    year: int
    regimen_name: str
    regimen_components: str
    subtype_target: str
    n: Optional[int]
    ypN0_rate: Optional[float]
    breast_pCR_rate: Optional[float]
    source_file: str
    notes: str = ""


def build_regimen_dataset() -> list[RegimenEffect]:
    rows: list[RegimenEffect] = []

    rows.append(RegimenEffect(
        study_id="Kuemmel2026_KEYNOTE522",
        pmid="41739398", year=2026,
        regimen_name="KEYNOTE-522 (pembro+chemo)",
        regimen_components="pembrolizumab + paclitaxel/carbo -> AC/EC",
        subtype_target="TNBC",
        n=784, ypN0_rate=0.767,
        breast_pCR_rate=None,
        source_file="02_ensayos_clinicos/2026_Kuemmel_TNBC-NAC-axillary_AnnSurgOnco.md",
        notes="Nodal pCR 76.7% in pembro arm. Surgical outcomes paper.",
    ))
    rows.append(RegimenEffect(
        study_id="Kuemmel2026_KEYNOTE522",
        pmid="41739398", year=2026,
        regimen_name="KEYNOTE-522 (placebo+chemo)",
        regimen_components="placebo + paclitaxel/carbo -> AC/EC",
        subtype_target="TNBC",
        n=390, ypN0_rate=0.699,
        breast_pCR_rate=None,
        source_file="02_ensayos_clinicos/2026_Kuemmel_TNBC-NAC-axillary_AnnSurgOnco.md",
        notes="Nodal pCR 69.9% in control arm.",
    ))

    rows.append(RegimenEffect(
        study_id="Takahashi2023_KEYNOTE522asia",
        pmid="37966841", year=2023,
        regimen_name="KEYNOTE-522 Asia subgroup (pembro+chemo)",
        regimen_components="pembrolizumab + paclitaxel/carbo -> AC/EC",
        subtype_target="TNBC",
        n=136, ypN0_rate=None, breast_pCR_rate=0.587,
        source_file="02_ensayos_clinicos/2023_Takahashi_TNBC-NAC-axillary_JamaNetwOpen.md",
        notes="Overall pCR (ypT0/is + ypN0) 58.7% vs 40.0% control in Asia subset.",
    ))
    rows.append(RegimenEffect(
        study_id="Takahashi2023_KEYNOTE522asia",
        pmid="37966841", year=2023,
        regimen_name="KEYNOTE-522 Asia subgroup (placebo+chemo)",
        regimen_components="placebo + paclitaxel/carbo -> AC/EC",
        subtype_target="TNBC",
        n=80, ypN0_rate=None, breast_pCR_rate=0.400,
        source_file="02_ensayos_clinicos/2023_Takahashi_TNBC-NAC-axillary_JamaNetwOpen.md",
        notes="Overall pCR 40.0%.",
    ))

    rows.append(RegimenEffect(
        study_id="Sharma2021_NeoSTOP_A",
        pmid="33208340", year=2021,
        regimen_name="NeoSTOP arm A (CbP -> AC)",
        regimen_components="paclitaxel + carboplatin -> dose-dense AC",
        subtype_target="TNBC",
        n=48, ypN0_rate=0.54, breast_pCR_rate=0.54,
        source_file="02_ensayos_clinicos/2021_Sharma_NeoSTOP_Clin-Cancer-Res.md",
        notes="pCR breast+axilla 54% (40-69%).",
    ))
    rows.append(RegimenEffect(
        study_id="Sharma2021_NeoSTOP_B",
        pmid="33208340", year=2021,
        regimen_name="NeoSTOP arm B (CbD)",
        regimen_components="docetaxel + carboplatin (anthracycline-free)",
        subtype_target="TNBC",
        n=52, ypN0_rate=0.54, breast_pCR_rate=0.54,
        source_file="02_ensayos_clinicos/2021_Sharma_NeoSTOP_Clin-Cancer-Res.md",
        notes="pCR breast+axilla 54% (40-68%) -- noninferior to arm A with lower toxicity.",
    ))

    rows.append(RegimenEffect(
        study_id="Woodfin2023",
        pmid="37973647", year=2023,
        regimen_name="Pembro-NAC (single institution)",
        regimen_components="pembrolizumab + standard NAC (paclitaxel/carbo + AC)",
        subtype_target="TNBC",
        n=87, ypN0_rate=0.86, breast_pCR_rate=0.678,
        source_file="02_ensayos_clinicos/2023_Woodfin_TNBC-NAC-axillary_AnnSurgOnco.md",
        notes="Axillary pCR 86%; overall pCR 67.8%; 37.2% of cN+ spared from ALND.",
    ))

    rows.append(RegimenEffect(
        study_id="Perry2025_MSK",
        pmid="40025324", year=2025,
        regimen_name="Chemo-IO TNBC (MSK)",
        regimen_components="pembrolizumab + NAC backbone (mostly paclitaxel/carbo + AC)",
        subtype_target="TNBC",
        n=283, ypN0_rate=0.67, breast_pCR_rate=0.58,
        source_file="10_pronostico_supervivencia/2025_Perry_Predictors-pCR-chemo-IO_Ann-Surg-Oncol.md",
        notes="Axillary pCR 67% among biopsy-proven cN+; overall pCR 57%.",
    ))

    rows.append(RegimenEffect(
        study_id="Chowdhary2026_pilot",
        pmid="41176781", year=2026,
        regimen_name="Dual HER2 blockade pilot",
        regimen_components="trastuzumab + pertuzumab + chemo",
        subtype_target="HER2+",
        n=20, ypN0_rate=0.30, breast_pCR_rate=0.25,
        source_file="02_ensayos_clinicos/2026_Chowdhary_cN2cN3-NAC-axilla_AnticancerDrugs.md",
        notes="Pilot, n=20. Complete nodal response 30%; breast pCR 25%; small sample.",
    ))

    rows.append(RegimenEffect(
        study_id="An2024_NET",
        pmid="39154153", year=2024,
        regimen_name="Neoadjuvant endocrine therapy",
        regimen_components="aromatase inhibitor / tamoxifen",
        subtype_target="HR+/HER2-",
        n=42, ypN0_rate=6/42, breast_pCR_rate=0.008,
        source_file="02_ensayos_clinicos/2024_An_HER2-NAC-axillary_AnnSurgOnco.md",
        notes="Single-institution NET cohort (1998-2022); cN+ subset n=42; nodal pCR 14.3% (6/42); "
              "breast pCR 0.8% (1/123 overall).",
    ))

    rows.append(RegimenEffect(
        study_id="Vasigh2024_meta_NAC",
        pmid="39742366", year=2024,
        regimen_name="NAC (pooled, HR+/HER2- cN+)",
        regimen_components="anthracyclines+taxanes (heterogeneous regimens)",
        subtype_target="HR+/HER2-",
        n=14330, ypN0_rate=0.149, breast_pCR_rate=None,
        source_file="01_revisiones_metaanalisis/2024_Vasigh_NET-vs-NAC-cN-positive_Breast-J.md",
        notes="Pooled npCR 14.9% across 11 studies.",
    ))
    rows.append(RegimenEffect(
        study_id="Vasigh2024_meta_NET",
        pmid="39742366", year=2024,
        regimen_name="NET (pooled, HR+/HER2- cN+)",
        regimen_components="aromatase inhibitor / tamoxifen (months)",
        subtype_target="HR+/HER2-",
        n=3707, ypN0_rate=0.089, breast_pCR_rate=None,
        source_file="01_revisiones_metaanalisis/2024_Vasigh_NET-vs-NAC-cN-positive_Breast-J.md",
        notes="Pooled npCR 8.9% (NET) -- substantially lower than NAC in same population.",
    ))

    rows.append(RegimenEffect(
        study_id="Boughey2013_Z1071",
        pmid="24101169", year=2013,
        regimen_name="ACOSOG Z1071 standard NAC",
        regimen_components="anthracycline+taxane in 74.6% (median 4 months)",
        subtype_target="Mixed (cN1, T0-4 N1-2)",
        n=525, ypN0_rate=0.410, breast_pCR_rate=None,
        source_file="02_ensayos_clinicos/2013_Boughey_ACOSOG-Z1071_JAMA.md",
        notes="Overall ypN0 rate 41.0% (215/525), 95% CI 36.7-45.3%. Pre-pembro/pertuzumab era.",
    ))

    rows.append(RegimenEffect(
        study_id="Simons2022_RISAS",
        pmid="36069889", year=2022,
        regimen_name="RISAS standard NAC",
        regimen_components="anthracycline+taxane +/- HER2 therapy (Dutch protocols 2017-19)",
        subtype_target="Mixed (cT1-4 cN1-3)",
        n=212, ypN0_rate=0.304, breast_pCR_rate=None,
        source_file="02_ensayos_clinicos/2022_Simons_RISAS-resultados_JAMA-Surg.md",
        notes="Overall ypN0 ~30.4% (per protocol summary).",
    ))

    rows.append(RegimenEffect(
        study_id="Montagna2024_JAMAOnc",
        pmid="38662396", year=2024,
        regimen_name="Modern multicentre cN+ -> ypN0 (mixed)",
        regimen_components="contemporary NAC including pembro and dual HER2 in subsets",
        subtype_target="Mixed (cN+ -> ypN0 select)",
        n=1144, ypN0_rate=None, breast_pCR_rate=0.662,
        source_file="05_tecnicas_quirurgicas_TAD_SLNB/2024_Montagna_OmisionALND-ypN0_JAMA-Oncol.md",
        notes="All ypN0 by inclusion; 619 (54%) HER2+, 758 (66%) breast pCR. Used as denominator "
              "for outcome (1.0% axillary recurrence at 5y).",
    ))

    return rows


# -----------------------------------------------------------------------------
# Data dictionary (markdown)
# -----------------------------------------------------------------------------
DATA_DICTIONARY = """# Data dictionary - ypN0 clinical predictor datasets

This document accompanies three CSV files in `predictor_ypN0/data/`:

1. `ypN0_rates_by_subtype.csv` - axillary pCR rates by breast cancer subtype.
2. `predictor_effect_sizes.csv` - odds ratios / hazard ratios for ypN0.
3. `regimen_effect.csv` - regimen-level (chemo / chemo-IO / NET / dual-HER2) pCR rates.

All rates are stored as fractions (0-1). PMID is stored as a string (no leading zeros expected).
Every row has a `source_file` pointing to the corresponding `.md` in this repository.

---

## 1. `ypN0_rates_by_subtype.csv`

| column        | type   | description |
|---------------|--------|-------------|
| study_id      | str    | short unique identifier `Author{Year}_TrialName` |
| pmid          | str    | PubMed ID; empty if not indexed |
| first_author  | str    | first author family name + initials |
| year          | int    | publication year |
| journal       | str    | journal name |
| n_total       | int    | total cohort size in the source study (may exceed n_subgroup) |
| subtype       | str    | standardised subtype value (see below) |
| n_subgroup    | int    | N in the subtype/subset for which the rate applies (NULL if not reported) |
| ypN0_n        | int    | numerator (axillary pCR events). NULL if not directly reported |
| ypN0_rate     | float  | proportion (0-1) of ypN0 within the subgroup |
| ci_low        | float  | lower bound 95% CI (NULL if not reported and not computable) |
| ci_high       | float  | upper bound 95% CI |
| definition    | str    | how the source defined the outcome |
| source_file   | str    | repo-relative path to the `.md` |
| notes         | str    | caveats (approximate, pooled, Wilson-approximated CI, etc.) |

### Allowed values for `subtype` (controlled vocabulary)
- `Overall`
- `HR-/HER2+` (= HER2-enriched, ERBB2-positive HR-negative)
- `HR+/HER2+` (= triple-positive)
- `HER2+` (combined HR-/HER2+ and HR+/HER2+, used by some meta-analyses)
- `TNBC` (triple-negative, ER-/PR-/HER2-)
- `HR+/HER2-` (luminal-like, ER+ and/or PR+, HER2-)
- `Luminal A`
- `Luminal B`
- regimen / context modifiers in parentheses, e.g. `TNBC (pembro+chemo)`,
  `HR+/HER2- (high genomic risk)`, `HR+/HER2- (age<50, Ki67>=20%)`,
  `HR+/HER2- (NAC)` / `HR+/HER2- (NET)`, `Lobular HER2+`, etc.

### CI approximation policy
When the source reports `ypN0_n` and `n_subgroup` but no 95% CI, we approximate
with the Wilson score interval (function `wilson_ci` in `build_dataset.py`).
Rows using this method are flagged in `notes`.

### Caveats by source
- **Samiei 2021 (PMID 33881478)**: pooled random-effects estimates with no subgroup
  N or CI in abstract. Pooled HER2+ (combined) row preserved because some literature
  references this aggregate; modellers should pick either `HER2+` or split rows but
  not both to avoid double counting.
- **van Amstel 2024 (PMID 39302345)**: only ORs reported in abstract; subtype-specific
  ypN0 rates approximated from the bibliographic summary
  `10_pronostico_supervivencia/RESUMEN_05_predictores_ypN0.md`. Flagged as approximate.
- **Barron 2018 (PMID 30193375)**: rates are **conditional on breast pCR**, not
  marginal axillary pCR rates. Useful for the calculator's joint-pCR scenario.
- **Montagna 2024 (PMID 38662396)**: cohort selected by ypN0 (rate = 1.0 by design);
  use as outcome anchor (axillary recurrence 1.0% at 5y) rather than predictor.
- **Chen 2021, Mukhtar 2025, Boughey 2022 Ki67**: subgroup rates given but sub-N
  not explicit; CI not computable.

---

## 2. `predictor_effect_sizes.csv`

| column        | type   | description |
|---------------|--------|-------------|
| study_id      | str    | unique identifier |
| pmid          | str    | PubMed ID |
| year          | int    | publication year |
| predictor     | str    | controlled vocabulary (see below) |
| comparison    | str    | the contrast for the OR/HR (e.g. "HR-/HER2+ vs HR+/HER2-") |
| OR            | float  | odds ratio OR hazard ratio depending on `model_type` |
| ci_low        | float  | lower 95% CI |
| ci_high       | float  | upper 95% CI |
| p_value       | float  | p-value if reported |
| n             | int    | analytic N |
| model_type    | str    | e.g. "multivariable logistic regression", "Cox PH", "univariable" |
| source_file   | str    | repo-relative path |
| notes         | str    | caveats |

### Allowed values for `predictor`
- `subtype` (most common; comparison field carries the contrast)
- `Ki67` (typically >=20% vs <20%; sometimes >=30%)
- `age` (continuous OR `<50 vs >=50`)
- `cN_stage` (cN2/3 vs cN1)
- `histology` (ILC vs IDC)
- `regimen` (with vs without pembro, pertuzumab, carboplatin, immunotherapy, NAC vs NET)
- `tumor_size` (cT category, continuous)
- `genomic_risk` (Oncotype RS, MammaPrint, BluePrint)
- `breast_pCR` (yes vs no; modelled as predictor of axillary pCR)
- `axillary_burden_PETCT` (limited vs advanced cN extent on baseline FDG-PET/CT)
- `race` (Black vs White, etc.)
- `regional_RT` (omission vs receipt, used in OPBC-07)

### Sign convention
- OR > 1 = higher probability of ypN0 (axillary pCR) -- preferred direction for
  predictors of interest.
- One row (Boughey 2022 genomic, low vs high risk) uses **OR for residual nodal
  disease**, NOT ypN0; the calculator must invert (1/OR) before composing.
- OPBC-07 rows are **hazard ratios for axillary recurrence**, not ORs for ypN0;
  documented in `notes` -- they belong in the downstream outcome model, not in the
  Nivel-1 logistic calculator. Modellers should filter `model_type` to retain only
  logistic / OR rows for the Nivel-1 calculator.

---

## 3. `regimen_effect.csv`

| column              | type   | description |
|---------------------|--------|-------------|
| study_id            | str    | unique identifier |
| pmid                | str    | PubMed ID |
| year                | int    | publication year |
| regimen_name        | str    | short label (e.g. "KEYNOTE-522 (pembro+chemo)") |
| regimen_components  | str    | drugs / sequence |
| subtype_target      | str    | target subtype population |
| n                   | int    | N in the regimen-specific arm |
| ypN0_rate           | float  | axillary pCR rate as fraction (NULL if not reported) |
| breast_pCR_rate     | float  | breast-only pCR rate as fraction |
| source_file         | str    | repo-relative path |
| notes               | str    | caveats |

### Notes
- KEYNOTE-522 and TRAIN-2-like dual-HER2 are the canonical examples; we cover them
  with the Kuemmel 2026 surgical-outcomes paper (KEYNOTE-522) and the Chowdhary 2026
  pilot (dual HER2 blockade).
- NeoSTOP, WSG-ADAPT-TN are listed for TNBC carboplatin context. KATHERINE and
  CheckMate-816 are excluded as instructed (residual setting / non-breast).
- NET vs NAC contrast for HR+/HER2- is captured both at the pooled meta-analysis
  level (Vasigh 2024) and at the institutional level (An 2024).

---

## Reproducibility

Run `python build_dataset.py` from this directory. The script is **idempotent**:
all data are hard-coded inside it, so re-running regenerates the same CSVs. Add new
rows by extending the corresponding `_xxx_rows()` factory functions, then rerun.

The Wilson 95% CI is computed via `wilson_ci(k, n)` (see top of script). Float
formatting uses Python `round(..., 3)`. All percentage values from the literature
were divided by 100 at the point of insertion (no late conversion).
"""


# -----------------------------------------------------------------------------
# Writers
# -----------------------------------------------------------------------------
SUBTYPE_FIELDS = [
    "study_id", "pmid", "first_author", "year", "journal",
    "n_total", "subtype", "n_subgroup", "ypN0_n", "ypN0_rate",
    "ci_low", "ci_high", "definition", "source_file", "notes",
]
EFFECT_FIELDS = [
    "study_id", "pmid", "year", "predictor", "comparison",
    "OR", "ci_low", "ci_high", "p_value", "n",
    "model_type", "source_file", "notes",
]
REGIMEN_FIELDS = [
    "study_id", "pmid", "year", "regimen_name", "regimen_components",
    "subtype_target", "n", "ypN0_rate", "breast_pCR_rate", "source_file", "notes",
]


def write_csv(path: Path, rows: list, fields: list[str]) -> None:
    with path.open("w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=fields)
        w.writeheader()
        for r in rows:
            d = asdict(r)
            # render floats with consistent precision
            for k, v in d.items():
                if isinstance(v, float):
                    d[k] = fmt(v)
            w.writerow(d)


def main() -> None:
    subtype_rows = build_subtype_rates_dataset()
    effect_rows = build_effect_sizes_dataset()
    regimen_rows = build_regimen_dataset()

    write_csv(HERE / "ypN0_rates_by_subtype.csv", subtype_rows, SUBTYPE_FIELDS)
    write_csv(HERE / "predictor_effect_sizes.csv", effect_rows, EFFECT_FIELDS)
    write_csv(HERE / "regimen_effect.csv", regimen_rows, REGIMEN_FIELDS)

    (HERE / "predictor_data_dictionary.md").write_text(
        DATA_DICTIONARY, encoding="utf-8"
    )

    n_sub = len(subtype_rows)
    n_eff = len(effect_rows)
    n_reg = len(regimen_rows)
    print(f"Wrote ypN0_rates_by_subtype.csv      ({n_sub} rows)")
    print(f"Wrote predictor_effect_sizes.csv     ({n_eff} rows)")
    print(f"Wrote regimen_effect.csv             ({n_reg} rows)")
    print(f"Wrote predictor_data_dictionary.md")


if __name__ == "__main__":
    main()
