#!/usr/bin/env python3
"""Reproduction and benchmark checks for the curves built by bootstrap.py.

Each check compares the curve with something that was not used to build it:

  1. Known answers from the paper: the two example curves of Hagan and West
     (2006), pp. 102 and 107, whose capitalisation factors and discrete
     forwards are published; on both, method B must recover every discrete
     forward and stay positive (the first curve is the one on which every
     cubic spline in the paper produces negative forwards).
  2. Properties the paper requires of method B (2008, p. 76, conditions
     (i)-(iv)) on the curve of record, and flat inputs giving a flat curve.
  3. Two implementations of method A: QuantLib's PiecewiseLogLinearDiscount
     against the log-linear bootstrap in hagan_west.py, which shares no
     curve code with QuantLib.
  4. Two pricers: the par-rate formula in hagan_west.py against QuantLib's
     OIS helpers on the QuantLib curve, and QuantLib's swap engine against
     method B's own repricing.
  5. Two implementations of the interpolation of method B: hagan_west.py
     against QuantLib's ConvexMonotoneInterpolation at the paper's setting
     (quadraticity 0, monotonicity 1), fed the same node data.
  6. Out-of-sample repricing: every USD SOFR OIS print of the day that starts
     on or after spot, close-adjusted (out/trades.csv), priced on both
     curves. Prints at tenors that are not curve inputs, and forward-starting
     prints, were never seen by the bootstrap. The print rates are
     close-adjusted with the intraday model of derive_quotes.py, whose path is
     fitted on spot-starting prints at standard tenors; a second row uses only
     prints from the last two traded hours at their raw rates, so that the
     result does not rest on that model.

Usage:
    uv run --with QuantLib==1.43 --with scipy --with matplotlib python3 verify.py
Writes: out/verify.md, out/verify.json, out/oos_residuals.csv, out/fig_oos_residuals.png
Exits non-zero if any check fails its stated tolerance.
"""
from __future__ import annotations

import csv
import json
import math
import sys

import QuantLib as ql

import bootstrap as bs
import hagan_west as hw

OUT = bs.OUT
LATE_HOUR = 15   # New York hour from which the close adjustment is at most 0.2 bp (out/intraday_path.csv)
CHECKS: list[dict] = []


def check(name: str, measured: float, tolerance: float, unit: str, detail: str = "", at_least: bool = False) -> None:
    """Pass when measured <= tolerance, or measured > tolerance when at_least is set."""
    ok = measured > tolerance if at_least else measured <= tolerance
    CHECKS.append({"check": name, "measured": measured, "tolerance": tolerance, "unit": unit,
                   "rule": "greater than" if at_least else "at most", "pass": bool(ok), "detail": detail})


def paper_examples():
    """Hagan and West (2006): p. 102 and p. 107 tables (term, continuous yield, capitalisation factor, discrete forward).

    The p. 102 table prints a continuous yield of 4.40 % at 4 years, but its own
    capitalisation factor there, 1.22140276 = exp(0.20), and the discrete
    forward it prints for 1-4 years, 4.33 %, both imply 5.00 %. The check
    therefore takes each yield from the capitalisation factor (ln C / t) and
    reports any printed yield that disagrees with it."""
    ex1 = [(0.1, 0.081, 1.00813289, None), (1, 0.07, 1.07250818, 0.0688), (4, 0.044, 1.22140276, 0.0433),
           (9, 0.07, 1.87761058, 0.0860), (20, 0.04, 2.22554093, 0.0155), (30, 0.03, 2.45960311, 0.0100)]
    ex2 = [(0.1, 0.06, 1.006018, 0.0600), (1, 0.06, 1.061837, 0.0600), (2, 0.05, 1.105171, 0.0400), (3, 0.05, 1.161834, 0.0500),
           (4, 0.05, 1.221403, 0.0500), (9, 0.065, 1.794991, 0.0770), (20, 0.06, 3.320117, 0.0559), (30, 0.06, 6.049647, 0.0600)]
    for label, ex in (("p. 102 (cubics go negative)", ex1), ("p. 107 (tamer curve)", ex2)):
        t = [0.0] + [row[0] for row in ex]
        r = [math.log(row[2]) / row[0] for row in ex]
        misprints = [f"{row[0]}Y printed {100 * row[1]:.2f} %, factor implies {100 * ri:.2f} %"
                     for row, ri in zip(ex, r) if abs(row[1] - ri) > 0.00005]
        fd = [(r[i] * t[i + 1] - (r[i - 1] * t[i] if i else 0.0)) / (t[i + 1] - t[i]) for i in range(len(r))]
        fd_err = max(abs(f - row[3]) for f, row in zip(fd, ex) if row[3] is not None)
        check(f"Paper example {label}: discrete forwards from the capitalisation factors against the printed ones",
              fd_err, 0.00005, "abs", "tolerance is half a unit of the printed precision (0.01 %); "
              + ("; ".join(misprints) or "printed yields agree with the factors"))
        mc = hw.MonotoneConvex(t, fd, positive=True)
        grid = [t[-1] * k / 30000 for k in range(1, 30001)]
        low = min(mc.forward(s) for s in grid)
        check(f"Paper example {label}: minimum instantaneous forward of method B on (0, 30]", low, 0.0, "rate", at_least=True)
        rec = max(abs((mc.integral(t[i + 1]) - mc.integral(t[i])) / (t[i + 1] - t[i]) - fd[i]) for i in range(len(fd)))
        check(f"Paper example {label}: every discrete forward recovered by method B", rec, 1e-14, "rate")


def properties(base):
    c = base.hw_curve
    t, fd = c.t, c.fd
    # (i) the discrete forward of each interval is recovered, by numerical quadrature of f
    worst = 0.0
    for i in range(1, len(t)):
        m = 2000
        h = (t[i] - t[i - 1]) / m
        quad = sum(c.forward(t[i - 1] + (k + 0.5) * h) for k in range(m)) * h
        worst = max(worst, abs(quad / (t[i] - t[i - 1]) - fd[i - 1]))
    check("Method B, condition (i): midpoint-rule average of f on each interval equals its discrete forward", worst, 1e-7, "rate")
    # (ii) positivity, (iii) continuity at every node
    dense = [t[-1] * k / 200000 for k in range(1, 200001)]
    check("Method B, condition (ii): minimum instantaneous forward", min(c.forward(s) for s in dense), 0.0, "rate", at_least=True)
    eps = 1e-9
    check("Method B, condition (iii): largest jump of f across a node", max(abs(c.forward(x + eps) - c.forward(x - eps)) for x in t[1:]),
          1e-6, "rate")
    # (iv) monotone on intervals whose neighbouring discrete forwards are monotone
    bad = 0
    for i in range(2, len(t) - 1):
        a, b, d = fd[i - 2], fd[i - 1], fd[i]
        if a < b < d or a > b > d:
            xs = [t[i - 1] + (t[i] - t[i - 1]) * k / 200 for k in range(201)]
            v = [c.forward(x) for x in xs]
            steps = [v[k + 1] - v[k] for k in range(200)]
            if (a < b < d and min(steps) < -1e-12) or (a > b > d and max(steps) > 1e-12):
                bad += 1
    check("Method B, condition (iv): intervals that break local monotonicity", bad, 0, "count")
    # Round trip: price every instrument on a flat 4 % continuously compounded curve,
    # bootstrap from those prices, and recover the flat curve (both methods).
    flat = hw.LogLinearDiscount([0.0, 60.0], [0.04])
    quotes = [(s_.name, hw.ois_par_rate(flat, s_)) for s_ in base.swaps]
    dep = {d.name: (flat.discount(d.start) / flat.discount(d.end) - 1.0) / d.accrual for d in base.deposits}
    fc = bs.CurveSet(quotes, dep["O/N"], tn_rate=dep["T/N"])
    grid = [bs.CAL.advance(bs.AS_OF, i, ql.Months) for i in range(1, 361)]
    for k in bs.METHODS:
        check(f"Round trip: flat 4 % curve recovered by method {k}, max abs zero-rate error (monthly to 30Y)",
              max(abs(fc.zero(k, d) - 0.04) for d in grid) * 1e4, 1e-6, "bp",
              "each deposit and swap is priced off the flat curve and bootstrapped back")


class _TimeCurve:
    """QuantLib curve seen through hagan_west.py's interface (discount by year fraction)."""
    def __init__(self, curve):
        self.curve = curve

    def discount(self, s):
        return self.curve.discount(s, True)


def two_implementations(base, monthly, days):
    ll = hw.bootstrap(base.deposits, base.swaps, "loglinear")
    diff = max(abs(ll.discount(bs.T(d)) - base.df("A", d)) for d in days)
    check("Method A: QuantLib against hagan_west.py log-linear bootstrap, max abs discount-factor difference (daily)", diff, 1e-9, "abs")
    par = max(abs(hw.ois_par_rate(_TimeCurve(base.ql_curve), s) - h.impliedQuote()) * 1e4
              for s, h in zip(base.swaps, base.helpers[2:]))
    check("Pricer: hagan_west.py par rate against QuantLib OISRateHelper.impliedQuote on the QuantLib curve", par, 1e-6, "bp")
    # QuantLib's swap engine on a daily discount-factor curve taken from method B
    grid = bs.business_days(bs.AS_OF, bs.CAL.advance(max(base.pillars), 2, ql.Years))
    curve_b = ql.DiscountCurve(grid, [base.df("B", d) for d in grid], bs.DC)
    curve_b.enableExtrapolation()
    h = ql.YieldTermStructureHandle(curve_b)
    worst = 0.0
    for tenor, rate in base.quotes:
        sw = ql.MakeOIS(ql.Period(tenor), ql.Sofr(h), 0.0, settlementDays=2, paymentLag=bs.PAYMENT_LAG,
                        paymentFrequency=ql.Annual, paymentCalendar=bs.CAL, discountingTermStructure=h)
        worst = max(worst, abs(sw.fairRate() - rate) * 1e4)
    check("Method B through QuantLib's swap engine: max abs par-rate error of the inputs", worst, 1e-4, "bp",
          "QuantLib values method B's curve on a daily discount-factor grid (log-linear between business days)")
    # interpolation cross-check against QuantLib's ConvexMonotoneInterpolation at the paper's setting
    c = base.hw_curve
    qlc = ql.ConvexMonotoneInterpolation(ql.Array(c.t), ql.Array([0.0] + c.fd), 0.0, 1.0, False)
    xs = [c.t[-1] * k / 20000 for k in range(1, 20000)]
    d = max(abs(qlc(x, True) - c.forward(x)) for x in xs) * 1e4
    check("Method B interpolation: hagan_west.py against QuantLib ConvexMonotoneInterpolation(0, 1), max abs forward difference", d, 1e-6, "bp",
          "same node times and discrete forwards; QuantLib's forcePositive floors only the end points, the paper's collar is not binding here")


def out_of_sample(base):
    handles = {}
    for k in bs.METHODS:
        grid = bs.business_days(bs.AS_OF, bs.CAL.advance(bs.AS_OF, 32, ql.Years))
        crv = ql.DiscountCurve(grid, [base.df(k, d) for d in grid], bs.DC)
        crv.enableExtrapolation()
        handles[k] = ql.YieldTermStructureHandle(crv)
    rows = []
    with open(OUT / "trades.csv") as f:
        for r in csv.DictReader(f):
            eff, exp = ql.DateParser.parseISO(r["effective"]), ql.DateParser.parseISO(r["expiration"])
            if exp <= eff:
                continue
            fair = {}
            for k in bs.METHODS:
                sw = ql.MakeOIS(ql.Period(), ql.Sofr(handles[k]), 0.0, effectiveDate=eff, terminationDate=exp,
                                paymentLag=bs.PAYMENT_LAG, paymentFrequency=ql.Annual, paymentCalendar=bs.CAL,
                                discountingTermStructure=handles[k])
                fair[k] = sw.fairRate()
            start = "spot" if eff == bs.CAL.advance(bs.AS_OF, 2, ql.Days) else "forward"
            if start == "spot" and r["in_bootstrap_set"] == "True":
                group = "spot start, input tenor"
            elif start == "spot":
                group = "spot start, other maturity"
            else:
                group = "forward start"
            rate = float(r["close_adjusted_rate"])
            rows.append({"effective": r["effective"], "expiration": r["expiration"], "group": group,
                         "years": bs.T(exp), "hour_ny": int(r["hour_ny"]), "fixed_rate_pct": 100 * float(r["fixed_rate"]),
                         "close_adjusted_rate_pct": 100 * rate,
                         "residual_A_bp": (rate - fair["A"]) * 1e4, "residual_B_bp": (rate - fair["B"]) * 1e4})
    with open(OUT / "oos_residuals.csv", "w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=list(rows[0]))
        w.writeheader()
        w.writerows(rows)

    def stats(xs):
        s = sorted(xs)
        q = lambda p: s[min(len(s) - 1, int(p * (len(s) - 1) + 0.5))]
        return {"n": len(s), "median": q(0.5), "p25": q(0.25), "p75": q(0.75),
                "share_within_1bp": sum(1 for x in s if abs(x) <= 1.0) / len(s),
                "share_within_2bp": sum(1 for x in s if abs(x) <= 2.0) / len(s)}
    summary = {}
    for g in ("spot start, input tenor", "spot start, other maturity", "forward start", "all"):
        sel = [r for r in rows if g == "all" or r["group"] == g]
        summary[g] = {k: stats([r[f"residual_{k}_bp"] for r in sel]) for k in bs.METHODS}
    # Robustness: the same residuals using only prints executed in the last two
    # traded hours, at their raw (unadjusted) rate, where the close adjustment
    # is small, so the result does not rest on the intraday model.
    late = [r for r in rows if r["hour_ny"] >= LATE_HOUR]
    summary["held out, last two hours, unadjusted"] = {
        k: stats([r["residual_B_bp"] + (r["fixed_rate_pct"] - r["close_adjusted_rate_pct"]) * 100 if k == "B"
                  else r["residual_A_bp"] + (r["fixed_rate_pct"] - r["close_adjusted_rate_pct"]) * 100
                  for r in late if r["group"] != "spot start, input tenor"]) for k in bs.METHODS}
    held_out = [r for r in rows if r["group"] != "spot start, input tenor"]
    # Paired comparison of the two methods on the same held-out prints, by maturity.
    def paired(sel):
        a = [abs(r["residual_A_bp"]) for r in sel]
        b = [abs(r["residual_B_bp"]) for r in sel]
        b_closer = sum(1 for x, y in zip(a, b) if y < x - 1e-9)
        a_closer = sum(1 for x, y in zip(a, b) if x < y - 1e-9)
        n = b_closer + a_closer
        k = min(b_closer, a_closer)
        p = min(1.0, 2 * sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n) if n else 1.0
        d = sorted(x - y for x, y in zip(a, b))
        return {"n": len(sel), "B_closer": b_closer, "A_closer": a_closer, "sign_test_p": p,
                "median_abs_A_minus_abs_B_bp": d[len(d) // 2] if d else 0.0,
                "A_within_1bp": sum(1 for x in a if x <= 1.0), "B_within_1bp": sum(1 for y in b if y <= 1.0)}
    last_node = bs.T(max(base.pillars))
    buckets = {"under 1Y": (0, 1), "1Y to 10Y": (1, 10), "10Y to the last node": (10, last_node), "beyond the last node": (last_node, 99)}
    comparison = {}
    for g in ("spot start, other maturity", "forward start"):
        for label, (lo, hi) in buckets.items():
            sel = [r for r in held_out if r["group"] == g and lo < r["years"] <= hi]
            if sel:
                comparison[f"{g}, {label}"] = paired(sel)
    comparison["all held-out prints"] = paired(held_out)
    # Prints at the same maturity are not independent observations: repeat the
    # short-end comparison with one observation per distinct expiration date
    # (the median absolute residual of that date's prints), and on raw rates.
    short = [r for r in held_out if r["group"] == "spot start, other maturity" and r["years"] <= 1]
    by_date = {}
    for r in short:
        by_date.setdefault(r["expiration"], []).append(r)
    med = lambda xs: sorted(xs)[len(xs) // 2]
    per_date = [{"residual_A_bp": med([abs(r["residual_A_bp"]) for r in v]), "residual_B_bp": med([abs(r["residual_B_bp"]) for r in v])}
                for v in by_date.values()]
    comparison["spot start, other maturity, under 1Y, one per expiration date"] = paired(per_date)
    raw = [{"residual_A_bp": r["residual_A_bp"] + (r["fixed_rate_pct"] - r["close_adjusted_rate_pct"]) * 100,
            "residual_B_bp": r["residual_B_bp"] + (r["fixed_rate_pct"] - r["close_adjusted_rate_pct"]) * 100} for r in short]
    comparison["spot start, other maturity, under 1Y, raw rates"] = paired(raw)
    six_to_twelve = [r for r in short if 0.5 < r["years"] <= 1]
    comparison["spot start, other maturity, 6M to 1Y (exploratory)"] = paired(six_to_twelve)
    med_b = sorted(abs(r["residual_B_bp"]) for r in held_out)[len(held_out) // 2]
    med_a = sorted(abs(r["residual_A_bp"]) for r in held_out)[len(held_out) // 2]
    check("Out-of-sample: median absolute residual of prints the bootstrap never saw, method B", med_b, 1.0, "bp",
          f"{len(held_out)} prints; residual is close-adjusted print rate minus the curve's par rate; method A: {med_a:.3f} bp")

    import matplotlib
    matplotlib.use("Agg")
    import matplotlib.pyplot as plt
    fig, ax = plt.subplots(figsize=(9, 4.2), dpi=150, facecolor=bs.C_SURF)
    ax.set_facecolor(bs.C_SURF)
    for s in ("top", "right"):
        ax.spines[s].set_visible(False)
    colours = {"spot start, input tenor": bs.C_INK, "spot start, other maturity": bs.C_A, "forward start": bs.C_B}
    for g, col in colours.items():
        sel = [r for r in rows if r["group"] == g]
        ax.scatter([r["years"] for r in sel], [r["residual_B_bp"] for r in sel], s=6, color=col, alpha=0.6, label=f"{g} ({len(sel)})")
    ax.axhline(0, color=bs.C_AXIS, linewidth=0.8)
    ax.set_ylim(-6, 6)
    ax.grid(True, color=bs.C_GRID, linewidth=0.6)
    ax.tick_params(colors=bs.C_MUTED, labelsize=9)
    ax.set_title("Close-adjusted print rate minus method B par rate, every print of 2026-09-18 starting on or after spot",
                 color=bs.C_INK, fontsize=10, loc="left")
    ax.set_ylabel("Residual (bp)", color=bs.C_MUTED, fontsize=9)
    ax.set_xlabel("Years from as-of date to the print's expiration", color=bs.C_MUTED, fontsize=9)
    ax.legend(frameon=False, fontsize=8, labelcolor=bs.C_INK, markerscale=2)
    clipped = sum(1 for r in rows if abs(r["residual_B_bp"]) > 6)
    fig.text(0.01, 0.005, f"Source: out/trades.csv (DTCC PPD 2026-09-18), verify.py; {clipped} prints beyond ±6 bp not shown",
             color=bs.C_MUTED, fontsize=7.5)
    fig.tight_layout(rect=(0, 0.03, 1, 1))
    fig.savefig(OUT / "fig_oos_residuals.png", facecolor=bs.C_SURF)
    plt.close(fig)
    return summary, clipped, comparison


def house_tolerance_bp() -> float:
    """The repricing tolerance from the house evidence policy (one place for the number)."""
    import re
    text = (bs.HERE.parent.parent / "house" / "evidence-policy.md").read_text()
    return float(re.search(r"absolute error of at most ([0-9.]+) basis point", text).group(1))


def repricing(base):
    tol = house_tolerance_bp()
    for k in bs.METHODS:
        worst = max(abs(r[3]) for r in base.repricing(k))
        check(f"Repricing: every input within the house tolerance, method {k}, largest abs error", worst, tol, "bp",
              "tolerance read from src/house/evidence-policy.md")
    fx = json.loads((OUT / "sofr_fixing.json").read_text())
    realised = fx.get("realised_after_as_of", {})
    gap = max(abs(v - fx["sofr_percent"]) for v in realised.values()) * 100 if realised else float("nan")
    check("Anchor assumption, ex post: largest gap between the fixings realised for 2026-09-18 and 2026-09-21 and the assumed level",
          gap, 0.0, "bp", "the snapshot holds the fixings published after the as-of date")


def main():
    ql.Settings.instance().evaluationDate = bs.AS_OF
    quotes, on_rate = bs.load_inputs()
    base = bs.CurveSet(quotes, on_rate)
    monthly = [bs.CAL.advance(bs.AS_OF, i, ql.Months) for i in range(1, 361)]
    days = bs.business_days(bs.AS_OF, max(base.pillars))
    repricing(base)
    paper_examples()
    properties(base)
    two_implementations(base, monthly, days)
    oos, clipped, comparison = out_of_sample(base)

    (OUT / "verify.json").write_text(json.dumps({"checks": CHECKS, "out_of_sample": oos, "paired_comparison": comparison, "oos_prints_beyond_6bp": clipped}, indent=2))
    with open(OUT / "verify.md", "w") as f:
        f.write("Table: reproduction and benchmark checks (verify.py).\n\n| Check | Measured | Tolerance | Result |\n|---|---:|---:|:---:|\n")
        for c in CHECKS:
            f.write(f"| {c['check']}{' (' + c['detail'] + ')' if c['detail'] else ''} | {c['measured']:.3g} {c['unit']} | {c['rule']} {c['tolerance']:.3g} | {'pass' if c['pass'] else 'FAIL'} |\n")
        f.write("\nTable: out-of-sample residuals, close-adjusted print rate minus curve par rate (bp).\n\n")
        f.write("| Prints | n | A median | A p25 / p75 | A within 1 bp | B median | B p25 / p75 | B within 1 bp |\n|---|---:|---:|---:|---:|---:|---:|---:|\n")
        for g, v in oos.items():
            a, b = v["A"], v["B"]
            f.write(f"| {g} | {a['n']} | {a['median']:+.2f} | {a['p25']:+.2f} / {a['p75']:+.2f} | {a['share_within_1bp']:.0%} | "
                    f"{b['median']:+.2f} | {b['p25']:+.2f} / {b['p75']:+.2f} | {b['share_within_1bp']:.0%} |\n")
        f.write("\nSource: out/verify.json, out/oos_residuals.csv, verify.py.\n")
        f.write("\nTable: paired comparison on the held-out prints: which method's par rate is closer to each print, by years to the print's expiration.\n\n"
                "| Prints | n | B closer | A closer | Sign-test p (two-sided) | Median of abs(A) minus abs(B) (bp) | Within 1 bp: A | Within 1 bp: B |\n|---|---:|---:|---:|---:|---:|---:|---:|\n")
        for g, v in comparison.items():
            f.write(f"| {g} | {v['n']} | {v['B_closer']} | {v['A_closer']} | {("<0.001" if v['sign_test_p'] < 0.001 else f"{v['sign_test_p']:.3f}")} | {v['median_abs_A_minus_abs_B_bp']:+.3f} | {v['A_within_1bp']} | {v['B_within_1bp']} |\n")
        f.write("\nSource: out/verify.json, verify.py. A positive median means B's residual is smaller. The one-per-expiration row treats each "
                "distinct expiration date as one observation (the median absolute residual of its prints); the raw-rates row uses unadjusted "
                "print rates. Eight maturity buckets are tested, so a p-value near 0.05 in one bucket is weak evidence on its own.\n")
    print((OUT / "verify.md").read_text())
    failed = [c["check"] for c in CHECKS if not c["pass"]]
    if failed:
        print("FAILED:", *failed, sep="\n  ")
        sys.exit(1)


if __name__ == "__main__":
    main()
