"""Triage script for breast-cancer NAC ypN0 PMIDs."""
import json
import os
import re
import csv
from glob import glob

# Inline batch-6 (returned directly to assistant context, not as a file)
INLINE_ARTICLES_RAW = r'''{"articles":[{"identifiers":{"pmid":"34229943","doi":"10.1016/j.clbc.2021.06.003","pii":"S1526-8209(21)00148-8"},"title":"Regional Nodal Irradiation for Clinically Node-Positive Breast Cancer Patients With Pathologic Negative Nodes After Neoadjuvant Chemotherapy.","abstract":"Neoadjuvant chemotherapy (NAC) is increasingly used for operable breast cancer (BC). Appropriate radiation therapy (RT) fields (ie, whole breast [WB] +/- regional nodal irradiation [RNI]) in patients who were clinically node positive (cN1) but convert to pathologically node negative (ypN0) after NAC are unknown and the subject of the accruing NSABP B-51 trial. We sought to compare outcomes between WB RT with or without RNI following breast conservation and sentinel lymph node biopsy (SLNB) alone in cN1, ypN0 women following NAC.","doi":"10.1016/j.clbc.2021.06.003","journal":{"title":"Clinical breast cancer"},"publication_date":{"year":"2021"},"authors":[{"last_name":"Schlafstein"}]},{"identifiers":{"pmid":"41193475","pmc":"PMC12589602","doi":"10.1038/s41523-025-00819-7"},"title":"Rewriting the radiation playbook: NSABP B-51 and the rise of pathologic precision.","abstract":"The NSABP B-51/RTOG 1304 trial represents a paradigm shift in breast cancer management, showing that patients achieving nodal clearance after neoadjuvant chemotherapy derive no added benefit from regional nodal irradiation.","doi":"10.1038/s41523-025-00819-7","journal":{"title":"NPJ breast cancer"},"publication_date":{"year":"2025"},"authors":[{"last_name":"Habibi"}]},{"identifiers":{"pmid":"40421156","pmc":"PMC12104955","doi":"10.21037/tbcr-24-54"},"title":"Radiotherapy in breast cancer patients achieving nodal pathologic complete response after neoadjuvant therapy: a scoping review.","abstract":"Neoadjuvant chemotherapy (NAC) is widely accepted as standard treatment in breast cancer, particularly in cases of tumors with aggressive biology. NAC has increased the rates of breast-conserving surgery (BCS) and the conversion of positive axillary nodes detected at initial diagnosis (cN+ to ypN0).","doi":"10.21037/tbcr-24-54","journal":{"title":"Translational breast cancer research"},"publication_date":{"year":"2025"},"authors":[{"last_name":"Cavalcante"}]},{"identifiers":{"pmid":"29383405","doi":"10.1007/s00066-018-1270-x"},"title":"Individualization of post-mastectomy radiotherapy and regional nodal irradiation based on treatment response after neoadjuvant chemotherapy for breast cancer : A systematic review.","abstract":"To review the evidence regarding post-mastectomy radiotherapy (PMRT) and regional nodal irradiation (RNI) after neoadjuvant chemotherapy (NACT) for breast cancer, with a special focus on individualization of adjuvant radiotherapy based on treatment response.","doi":"10.1007/s00066-018-1270-x","journal":{"title":"Strahlentherapie und Onkologie"},"publication_date":{"year":"2018"},"authors":[{"last_name":"Krug"}]},{"identifiers":{"pmid":"34465118","doi":"10.33699/PIS.2021.100.6.285-294"},"title":"Role of the radiologist during neoadjuvant systemic therapy for breast cancer.","abstract":"Neoadjuvant therapy (NT) is one of the possible oncological treatment strategies for breast cancer. Its aim is to achieve down-staging of the tumour in the breast and axilla and thus the possibility of converting mastectomy to a breast-conserving procedure, and also to allow for a less burdensome and more targeted operation of the axillary lymph nodes.","doi":"10.33699/PIS.2021.100.6.285-294","journal":{"title":"Rozhl Chir"},"publication_date":{"year":"2021"},"authors":[{"last_name":"Houserkova"}]},{"identifiers":{"pmid":"25458227","doi":"10.1016/j.ejrad.2014.09.020"},"title":"Noninvasive nodal restaging in clinically node positive breast cancer patients after neoadjuvant systemic therapy: a systematic review.","abstract":"To provide a systematic review of studies comparing the diagnostic performance of noninvasive techniques and axillary lymph node dissection in the identification of initially node positive patients with pathological complete response of axillary lymph nodes to neoadjuvant systemic therapy.","doi":"10.1016/j.ejrad.2014.09.020","journal":{"title":"European journal of radiology"},"publication_date":{"year":"2014"},"authors":[{"last_name":"Schipper"}]}]}'''

TR_DIR = r'C:\Users\dfduq\.claude\projects\C--Users-dfduq-OneDrive-Documents-ganglio-sergioc\fb63df9b-3a14-44fb-97e5-1adb5839edd4\tool-results'
OUT_DIR = r'C:\Users\dfduq\OneDrive\Documents\ganglio sergioc\indice_metadatos'

# All metadata files (excludes full_text_article)
files = sorted(glob(os.path.join(TR_DIR, 'mcp-4e55303b-*-get_article_metadata-*.txt')))
articles = {}
for f in files:
    try:
        with open(f, encoding='utf-8') as fh:
            d = json.load(fh)
        for a in d.get('articles', []):
            pmid = a['identifiers']['pmid']
            articles[pmid] = a
    except Exception as e:
        print(f'Error reading {f}: {e}')

# Also load inline batch (batch-6 was small enough to be returned directly)
inline = json.loads(INLINE_ARTICLES_RAW)
for a in inline['articles']:
    pmid = a['identifiers']['pmid']
    if pmid not in articles:
        articles[pmid] = a

with open(os.path.join(OUT_DIR, 'pmids.json')) as f:
    all_pmids = json.load(f)

print(f'Loaded {len(articles)} articles, expecting {len(all_pmids)}')
missing = [p for p in all_pmids if p not in articles]
print(f'Missing PMIDs: {missing}')

# Triage logic
# Score 5 = direct hit: NAC + axilla/SLNB/TAD + cN+→ypN0 / ypN status / clip / marker / radiotherapy post-NAC
# Score 4 = closely related: NAC breast + axillary management or related trial
# Score 3 = related/general NAC breast cancer
# Score 2 = tangential
# Score 1 = off-topic

KEY5_PHRASES = [
    r'\bypn0\b', r'\bcn\+', r'cn1', r'cn2', r'cn3',
    r'pathologic complete response.*(?:nodal|axill)',
    r'(?:nodal|axill).*pathologic complete response',
    r'axillary pcr', r'targeted axillary dissection', r'\btad\b',
    r'marked? (?:lymph)? ?node', r'mari procedure', r'risas',
    r'radioactive (?:iodine )?seed', r'magseed', r'magnetic seed',
    r'clipped node', r'clip(?:ped)? (?:lymph )?node',
    r'sentinel lymph node biopsy.*(?:neoadjuvant|after)',
    r'(?:neoadjuvant|after).{0,30}sentinel lymph node',
    r'acosog z1071', r'\bsentina\b', r'\bsn fnac\b', r'sn-fnac',
    r'\bganea[- ]?2\b', r'nsabp b-?51', r'rtog 1304', r'alliance a011202',
    r'\baxsana\b', r'\batnec\b', r'\bopbc\b',
    r'(?:cn\+|node[- ]positive|node positive).*(?:ypn0|conver|down ?stag)',
    r'(?:conver|down ?stag).{0,80}(?:ypn0|node[- ]?negative)',
    r'\brni\b.*neoadjuvant', r'regional nodal irradiation.*neoadjuvant',
    r'neoadjuvant.{0,40}(?:rni|regional nodal irradiation)',
    r'post[- ]?mastectomy radiotherapy.*neoadjuvant',
    r'neoadjuvant.{0,30}post[- ]?mastectomy radiotherapy',
    r'restaging.*(?:axill|node).*neoadjuvant',
    r'neoadjuvant.*restaging.*(?:axill|node)',
]
KEY3_PHRASES = [
    r'neoadjuvant', r'breast cancer', r'axill', r'sentinel', r'lymph node',
    r'mastectomy', r'breast[- ]conserv', r'\bpcr\b', r'response',
]
OTHER_TUMOR = [r'gastric', r'rectal', r'pancrea', r'esophag', r'colorect',
               r'\bbladder\b', r'cervical cancer', r'lung cancer',
               r'\bovari', r'penile', r'thyroid', r'melanoma', r'sarcoma',
               r'head and neck']

def score_and_categorize(art):
    title = (art.get('title') or '').lower()
    abstract = (art.get('abstract') or '').lower()
    mesh = ' '.join(art.get('mesh_terms', []) or []).lower()
    artypes = ' '.join(art.get('article_types', []) or []).lower()
    journal = (art.get('journal', {}).get('title') or '').lower()
    text = f'{title} {abstract} {mesh}'

    # Detect off-topic / other tumor
    is_other_tumor = any(re.search(p, text) for p in OTHER_TUMOR) and 'breast' not in text

    score5_hits = sum(1 for p in KEY5_PHRASES if re.search(p, text))
    score3_hits = sum(1 for p in KEY3_PHRASES if re.search(p, text))

    # Scoring
    if score5_hits >= 2 or ('ypn0' in text and 'breast' in text) or 'targeted axillary' in text or 'tad ' in text or re.search(r'\bcn1\b.*ypn0|cn\+.*ypn0', text):
        score = 5
    elif score5_hits == 1:
        score = 4
    elif 'neoadjuvant' in text and ('breast' in text) and ('axill' in text or 'sentinel' in text or 'lymph' in text or 'node' in text):
        score = 4
    elif 'neoadjuvant' in text and 'breast' in text:
        score = 3
    elif score3_hits >= 3 or is_other_tumor:
        score = 3 if is_other_tumor else 2
    else:
        score = 1

    # If other-tumor (not breast) AND about NAC/axillary nodes -> 04 folder, score 3
    if is_other_tumor and ('neoadjuvant' in text or 'nac' in text or 'preoperative chemo' in text) and ('node' in text or 'axill' in text or 'lymph' in text):
        score = 3
        category = '04_otros_tumores'
        return score, category

    # Categorization
    # 1) Reviews / meta-analyses
    if any(k in artypes for k in ['systematic review', 'meta-analysis', 'review']) or 'systematic review' in title or 'meta-analysis' in title or 'meta analysis' in title or 'scoping review' in title:
        # but classify true clinical trials first
        if 'randomized controlled trial' in artypes and 'review' not in title and 'meta-analysis' not in title:
            category = '02_ensayos_clinicos'
        else:
            category = '01_revisiones_metaanalisis'
    elif 'randomized controlled trial' in artypes or 'clinical trial' in artypes or re.search(r'phase (?:ii|iii|2|3) trial', text) or 'multicenter study' in artypes:
        category = '02_ensayos_clinicos'
    elif any(g in text for g in ['guideline', 'consensus', 'recommendation', 'st. gallen', 'st gallen', 'asco guideline', 'eusoma', 'nccn']) and 'guideline' in artypes + title + journal:
        category = '08_guias_clinicas'
    elif 'guideline' in artypes or 'practice guideline' in artypes:
        category = '08_guias_clinicas'
    elif any(p in text for p in ['radiotherapy', 'radiation therapy', 'radiation oncology', 'rni ', 'regional nodal irradiation', 'pmrt', 'post-mastectomy radio', 'postmastectomy radio']):
        category = '09_radioterapia_post_NAC'
    elif any(p in text for p in ['mari procedure', 'risas', 'magseed', 'magnetic seed', 'radioactive seed', 'i-125', 'i125', 'iodine seed', 'clip placement', 'clipped node', 'clip marker', 'tattoo']) or 'marker' in text or 'imaging' in text or 'ultrasound' in text or 'mri ' in text or 'pet' in text or 'fdg' in text or 'restaging' in text:
        category = '06_imagen_y_marcaje'
    elif any(p in text for p in ['targeted axillary dissection', 'tad', 'sentinel lymph node', 'slnb', 'axillary lymph node dissection', 'alnd', 'axillary dissection', 'axillary surgery', 'surgery of the axilla']):
        category = '05_tecnicas_quirurgicas_TAD_SLNB'
    elif any(p in text for p in ['pathology', 'pathologic', 'histopathology', 'isolated tumor cell', 'micromet', 'macromet']) and 'pathologic' in text and ('node' in text or 'lymph' in text):
        category = '07_anatomia_patologica'
    elif any(p in text for p in ['survival', 'prognos', 'recurrence-free', 'overall survival', 'dfs', 'rfs', 'disease-free']):
        category = '10_pronostico_supervivencia'
    elif is_other_tumor:
        category = '04_otros_tumores'
    else:
        category = '03_cancer_mama_axila'

    return score, category

# Build CSV rows
rows = []
for pmid in all_pmids:
    a = articles.get(pmid)
    if not a:
        rows.append({
            'pmid': pmid, 'year': '', 'first_author': '',
            'title': 'METADATA NOT FOUND', 'journal': '',
            'relevance_score': '', 'category_folder': '',
            'has_pmcid': '', 'doi': ''
        })
        continue
    year = a.get('publication_date', {}).get('year', '')
    authors = a.get('authors', []) or []
    first_author = ''
    if authors:
        au = authors[0]
        first_author = au.get('last_name', '') or au.get('fore_name', '')
    title = (a.get('title') or '').strip()
    journal = a.get('journal', {}).get('iso_abbreviation') or a.get('journal', {}).get('title') or ''
    pmcid = a.get('identifiers', {}).get('pmc', '')
    doi = a.get('doi') or a.get('identifiers', {}).get('doi', '') or ''
    score, category = score_and_categorize(a)
    rows.append({
        'pmid': pmid, 'year': year, 'first_author': first_author,
        'title': title.replace('"', "'").replace('\n', ' ')[:300],
        'journal': journal,
        'relevance_score': score, 'category_folder': category,
        'has_pmcid': 'yes' if pmcid else 'no', 'doi': doi
    })

# Write CSV
csv_path = os.path.join(OUT_DIR, 'triage_pmids.csv')
with open(csv_path, 'w', encoding='utf-8', newline='') as fh:
    w = csv.DictWriter(fh, fieldnames=['pmid', 'year', 'first_author', 'title', 'journal', 'relevance_score', 'category_folder', 'has_pmcid', 'doi'])
    w.writeheader()
    for r in rows:
        w.writerow(r)

print(f'Wrote {csv_path}')

# Summary stats
from collections import Counter
score_counts = Counter(r['relevance_score'] for r in rows)
cat_counts = Counter(r['category_folder'] for r in rows)
print('Score distribution:', dict(score_counts))
print('Category distribution:', dict(cat_counts))

# Top 20 most relevant, grouped by category
relevant = sorted([r for r in rows if r['relevance_score'] in (4, 5)],
                  key=lambda r: (-int(r['relevance_score']) if r['relevance_score'] else 0, r['category_folder']))
top20 = relevant[:20]

excluded = [r for r in rows if r['relevance_score'] in (1, 2)]

# Build markdown
md = []
md.append('# Triage Resumen - PMIDs Ganglio Centinela post-NAQ')
md.append('')
md.append(f'**Total PMIDs procesados:** {len(rows)} (PMIDs unicos)')
md.append('')
md.append('## Conteo por score de relevancia')
md.append('')
md.append('| Score | Conteo |')
md.append('|------:|-------:|')
for s in [5, 4, 3, 2, 1, '']:
    c = score_counts.get(s, 0)
    label = str(s) if s != '' else 'sin metadata'
    md.append(f'| {label} | {c} |')
md.append('')
md.append('## Conteo por carpeta/categoria')
md.append('')
md.append('| Categoria | Conteo |')
md.append('|-----------|-------:|')
for k in sorted(cat_counts.keys()):
    md.append(f'| {k} | {cat_counts[k]} |')
md.append('')
md.append('## Top 20 PMIDs mas relevantes (score 5 y 4) agrupados por categoria')
md.append('')
# Group
by_cat = {}
for r in top20:
    by_cat.setdefault(r['category_folder'], []).append(r)
for cat in sorted(by_cat.keys()):
    md.append(f'### {cat}')
    md.append('')
    for r in by_cat[cat]:
        md.append(f"- **PMID {r['pmid']}** (score {r['relevance_score']}, {r['year']}, {r['first_author']}) - {r['title']} - *{r['journal']}* - DOI: {r['doi']}")
    md.append('')

# Add full list of score-5 if more than 20
all_5 = [r for r in rows if r['relevance_score'] == 5]
all_4 = [r for r in rows if r['relevance_score'] == 4]
md.append(f'## Resumen ampliado')
md.append('')
md.append(f'- PMIDs con score 5: {len(all_5)}')
md.append(f'- PMIDs con score 4: {len(all_4)}')
md.append(f'- PMIDs con score 3: {score_counts.get(3, 0)}')
md.append(f'- PMIDs con score 2: {score_counts.get(2, 0)}')
md.append(f'- PMIDs con score 1: {score_counts.get(1, 0)}')
md.append('')

md.append('## PMIDs a excluir (score 1-2)')
md.append('')
if not excluded:
    md.append('_Ninguno_')
else:
    md.append('| PMID | Score | Titulo | Categoria |')
    md.append('|------|------:|--------|-----------|')
    for r in excluded:
        md.append(f"| {r['pmid']} | {r['relevance_score']} | {r['title'][:120]} | {r['category_folder']} |")
md.append('')

if missing:
    md.append('## PMIDs sin metadata recuperada')
    md.append('')
    for p in missing:
        md.append(f'- {p}')
    md.append('')

md_path = os.path.join(OUT_DIR, 'TRIAGE_RESUMEN.md')
with open(md_path, 'w', encoding='utf-8') as fh:
    fh.write('\n'.join(md))
print(f'Wrote {md_path}')
print(f'Total score 5: {len(all_5)}, score 4: {len(all_4)}')
print(f'Excluded (1-2): {len(excluded)}')
