#!/usr/bin/env python3
"""Describe held-out residuals by nearest OIS pillar distance, without a model rerun.

Run with Python 3.12.14: uv run --python 3.12.14 python3 summarize_residuals.py
The distance is a descriptive end-date coordinate, not a measure of financial
risk or similarity. Keep spot-start and forward-start cohorts separate.
"""
import csv
from datetime import date
import hashlib
import json
import math
from pathlib import Path
import platform
import statistics
import sys

ROOT = Path(__file__).resolve().parent
COHORTS = ('spot start, other maturity', 'forward start')
BANDS = ('0–30 days', '31–90 days', '>90 days')


def summarize(rows, nodes):
    pillars = [date.fromisoformat(row['pillar_date']) for row in nodes
               if row['instrument'] not in ('O/N', 'T/N')]
    if not pillars:
        raise ValueError('no OIS pillar dates')
    groups = {(cohort, band): [] for cohort in COHORTS for band in BANDS}
    for row in rows:
        cohort = row['group']
        if cohort == 'spot start, input tenor':
            continue
        if cohort not in COHORTS:
            raise ValueError('unrecognized cohort')
        distance = min(abs((date.fromisoformat(row['expiration']) - node).days) for node in pillars)
        values = [abs(float(row['residual_A_bp'])), abs(float(row['residual_B_bp']))]
        if not all(math.isfinite(value) for value in values):
            raise ValueError('nonfinite residual')
        band = BANDS[0 if distance <= 30 else 1 if distance <= 90 else 2]
        groups[(cohort, band)].append((distance, *values))
    result = []
    for (cohort, band), values in groups.items():
        if not values:
            continue
        entry = {'cohort': cohort, 'distance_band': band, 'n': len(values),
                 'minimum_distance_days': min(row[0] for row in values),
                 'maximum_distance_days': max(row[0] for row in values)}
        for column, method in ((1, 'A'), (2, 'B')):
            observations = [row[column] for row in values]
            q1, median, q3 = statistics.quantiles(observations, n=4, method='inclusive') if len(observations) > 1 else [observations[0]] * 3
            entry[method] = {'median_abs_residual_bp': median, 'q1_abs_residual_bp': q1,
                             'q3_abs_residual_bp': q3, 'iqr_abs_residual_bp': q3 - q1}
        result.append(entry)
    return result


def main():
    names = ['oos_residuals.csv', 'nodes.csv', 'run_info.json']
    paths = {name: ROOT / 'out' / name for name in names}
    with paths['oos_residuals.csv'].open() as stream:
        rows = list(csv.DictReader(stream))
    with paths['nodes.csv'].open() as stream:
        nodes = list(csv.DictReader(stream))
    result = {
        'as_of': json.loads(paths['run_info.json'].read_text())['as_of'],
        'python': platform.python_version(), 'dependencies': 'Python standard library only',
        'script_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
        'input_sha256': {name: hashlib.sha256(path.read_bytes()).hexdigest() for name, path in paths.items()},
        'distance_definition': 'Absolute calendar days from print expiration to the nearest used OIS pillar_date; O/N and T/N excluded.',
        'population': 'Both held-out cohorts; excludes spot-start prints at input tenors. Forward-start and spot-start cohorts remain separate.',
        'limitations': 'Exploratory descriptive bins; end-date distance does not measure start-date differences, financial risk, independence or generalization. Residuals share the original quote adjustment.',
        'quartiles': 'Linear interpolation of empirical order statistics (inclusive).',
        'summary': summarize(rows, nodes),
    }
    result['heldout_count'] = sum(row['n'] for row in result['summary'])
    (ROOT / 'out/residual-distance.json').write_text(json.dumps(result, indent=2, allow_nan=False) + '\n')
    lines = ['# Descriptive held-out residual summaries', '',
             f"Input as-of: {result['as_of']}. Population: {result['population']}", '',
             result['distance_definition'], '', result['limitations'], '',
             '| Cohort | End-date distance | Prints (count) | A median absolute residual (bp) | A IQR (bp) | B median absolute residual (bp) | B IQR (bp) |',
             '|---|---|---:|---:|---:|---:|---:|']
    for row in result['summary']:
        a, b = row['A'], row['B']
        lines.append(f"| {row['cohort']} | {row['distance_band']} | {row['n']} | {a['median_abs_residual_bp']:.3f} | {a['iqr_abs_residual_bp']:.3f} | {b['median_abs_residual_bp']:.3f} | {b['iqr_abs_residual_bp']:.3f} |")
    lines += ['', 'Source: `summarize_residuals.py`, `out/oos_residuals.csv`, `out/nodes.csv` and `out/run_info.json`; exact input/script hashes, Python version and unrounded quantities are in `out/residual-distance.json`.', '',
              'This supplemental calculation does not replace historical `out/SHA256SUMS` or claim a new financial-model run.']
    (ROOT / 'out/residual-distance.md').write_text('\n'.join(lines) + '\n')
    print(json.dumps({'python': result['python'], 'heldout_count': result['heldout_count'], 'script_sha256': result['script_sha256']}), file=sys.stderr)


if __name__ == '__main__':
    main()
