"""Stage 1 v4 checks (step 6). A failure stops the run: exit code 1, and run_all.sh (set -e) builds no page.

  python check.py        (run from stage1-v4/code after v4.py)

Writes stage1-v4/checks_v4.csv.
"""
import hashlib
import subprocess
import sys

import numpy as np
import pandas as pd

from common import ROOT, RQ, V4, num, rd

L = pd.read_csv(V4 / "layers_v4.csv", index_col=0)
SH = pd.read_csv(V4 / "shortages_v4.csv", index_col=0)
OB = pd.read_csv(V4 / "own_blocks_v4.csv")
ROWS = pd.read_csv(V4 / "rows_v4.csv")
UNE = pd.read_csv(V4 / "unexplained_v4.csv")
VR = pd.read_csv(V4 / "variants_v4.csv")
RK = pd.read_csv(V4 / "ranking_v4.csv", index_col=0)
PK = pd.read_csv(V4 / "picks_v4.csv")
SR = pd.read_csv(V4 / "searches_v4.csv")
REQ = rd(RQ / "requirements.csv")
checks = []


def check(name, ok, detail):
    checks.append(dict(check=name, ok=bool(ok), detail=str(detail)[:600]))


# 1. each layer: today + on-course growth + tier 1 + tier 2 = target
bad = [j for j in L.index if abs(L.loc[j, "today"] + L.loc[j, "growth"] + L.loc[j, "tier1"] + L.loc[j, "tier2"] - L.loc[j, "target"]) > 0.5]
check("each layer: today + on-course growth + tier 1 + tier 2 = its target", not bad, bad or f"{len(L)} layers")

# 2. layers sum to 448.5T (= 2 x today)
tt = L.target.sum()
check("layers sum to 448.5T (= 2 x today)", round(tt / 1000, 1) == 448.5 and abs(tt - 2 * L.today.sum()) < 1,
      f"{tt:,.1f} bn = 2 x {L.today.sum():,.1f}")

# 3. every missing dollar is in a named shortage or on the unexplained list
miss_layers = (L.tier1.clip(lower=0) + L.tier2.clip(lower=0)).sum()
rows_ok = all(abs(r.explained_bn + r.unexplained_bn - r.missing_bn) < 0.5 for r in ROWS.itertuples())
expect = {(j, str(t)) for j in L.index for t in (1, 2) if L.loc[j, f"tier{t}"] > 1e-9}
have = {(r.layer_id, str(r.tier)) for r in ROWS.itertuples()}
une_match = abs(UNE.unexplained_bn.sum() - ROWS.unexplained_bn.sum()) < 0.5
named = all((r.explained_bn <= 1e-6) or (isinstance(r.binds_first, str) and r.binds_first) for r in ROWS.itertuples())
check("every missing dollar is in a named shortage or on the unexplained list",
      rows_ok and expect == have and une_match and named and abs(ROWS.missing_bn.sum() - miss_layers) < 0.5,
      f"missing {miss_layers:,.1f} = explained {ROWS.explained_bn.sum():,.1f} + unexplained {ROWS.unexplained_bn.sum():,.1f}; {len(ROWS)} rows")
check("unexplained output carries a size and no name", "binds_first" not in UNE.columns and (UNE.unexplained_bn > 0).all(),
      f"{len(UNE)} rows, {UNE.unexplained_bn.sum():,.1f} bn")

# 4. no shortage blocks more output than exists in the layers it serves (each cell, and in total), every variant
served = {}
for k in SH.index:
    if k.startswith("BUY-"):
        served[k] = {k[4:]}
    else:
        x = REQ[REQ.input_id == k]
        served[k] = {r.layer_id for r in x.itertuples() if num(r.required_base, 0) > 0}
row_amt = {(r.layer_id, str(r.tier)): r.missing_bn for r in ROWS.itertuples()}
over_cell, over_tot, outside = [], [], []
for (k, v), g in OB.groupby(["shortage_id", "variant"]):
    for r in g.itertuples():
        if r.blocked_bn > row_amt.get((r.layer_id, str(r.tier)), 0) + 0.5:
            over_cell.append(f"{k}/{v} {r.layer_id} t{r.tier}")
        if r.layer_id not in served[k]:
            outside.append(f"{k}/{v} {r.layer_id}")
    cap = sum(L.loc[j, "tier1"] + L.loc[j, "tier2"] for j in served[k] if j in L.index)
    if g.blocked_bn.sum() > cap + 0.5:
        over_tot.append(f"{k}/{v} {g.blocked_bn.sum():,.0f} > {cap:,.0f}")
check("no shortage blocks more output than exists in the layers it serves (every cell, every total, every variant)",
      not over_cell and not over_tot and not outside, (over_cell + over_tot + outside)[:10] or f"{OB.groupby(['shortage_id', 'variant']).ngroups} shortage-variants")
k22 = OB[(OB.shortage_id == "K22") & (OB.variant == "base")]
c01 = k22[k22.layer_id == "C01"].blocked_bn.sum()
c01_miss = L.loc["C01", "tier1"] + L.loc["C01", "tier2"]
il = rd(RQ / "input_layer_v3.csv")
v3_c01 = num(il[(il.input_id == "K22") & (il.layer_id == "C01")].output_depending_ppp_bn.iloc[0])
check("cropland: what it blocks in food at home stays inside food at home's missing output and target",
      c01 <= c01_miss + 0.5 and c01_miss <= L.loc["C01", "target"],
      f"v4 cropland blocks {c01:,.1f} bn in C01 (missing {c01_miss:,.1f}, target {L.loc['C01', 'target']:,.1f}); "
      f"in all {k22.blocked_bn.sum():,.1f} bn across {k22.layer_id.nunique()} layers whose crop footprint it serves "
      f"(v3 put {v3_c01:,.0f} bn of C01 and 45.4T counted once across all layers on it, by proportional shares)")
# the same cell test applied to v3's proportional numbers, for comparison (information: v3's files are not changed)
lay3 = rd(RQ / "layers_v3.csv").set_index("layer_id")
v3_over = []
for r in il.itertuples():
    m3 = num(lay3.loc[r.layer_id, "tier1"]) + num(lay3.loc[r.layer_id, "tier2"])
    if num(r.output_depending_ppp_bn, 0) > m3 + 0.5:
        v3_over.append(f"{r.input_id} {r.layer_id}")
check("information: v3's proportional 'output depending' cells that exceed their layer's missing output (v4 fixes this)",
      True, f"{len(v3_over)} cells: " + ", ".join(v3_over[:12]))

# 5. nothing counted twice in any total
own_sum = OB[OB.variant == "base"].groupby("shortage_id").blocked_bn.sum()
tot_ok = all(abs(own_sum.get(k, 0.0) - SH.loc[k, "blocked_total"]) < 0.5 for k in SH.index)
union_ok = abs(ROWS.explained_bn.sum() + ROWS.unexplained_bn.sum() - miss_layers) < 0.5 and ROWS.explained_bn.sum() <= own_sum.sum() + 0.5
row_max = all(r.explained_bn <= r.missing_bn + 0.5 for r in ROWS.itertuples())
dup_rows = ROWS.duplicated(["layer_id", "tier"]).any()
dup_cells = OB.duplicated(["shortage_id", "variant", "layer_id", "tier"]).any()
check("nothing counted twice: each row once, each shortage's total = its own cells, explained = union per row (not a sum of shortages)",
      tot_ok and union_ok and row_max and not dup_rows and not dup_cells,
      f"sum of shortages on their own {own_sum.sum():,.1f} bn (overlapping, never added into a total); explained union {ROWS.explained_bn.sum():,.1f} bn")

# 6. step 3: every shortage over 1T tested under at least two other ways, SOLID only if it binds in all
bad3 = []
for k in SH.index:
    if SH.loc[k, "blocked_total"] <= 1000:
        continue
    v = VR[VR.shortage_id == k]
    if SH.loc[k, "verdict"] == "SOLID" and k != "K11" and (len(v) < 3 or not v.binds.all()):
        bad3.append(f"{k} SOLID but {len(v) - 1} alternatives / binds {v.binds.sum()} of {len(v)}")
    if SH.loc[k, "verdict"] not in ("SOLID", "SCENARIO"):
        bad3.append(f"{k} no verdict")
    if SH.loc[k, "verdict"] == "SCENARIO" and not str(SH.loc[k, "depends_on"]).strip():
        bad3.append(f"{k} SCENARIO without its assumption")
    if not (SH.loc[k, "blocked_low"] <= SH.loc[k, "blocked_total"] + 0.5 <= SH.loc[k, "blocked_high"] + 1):
        bad3.append(f"{k} range")
check("every shortage over 1T is tested: range, SOLID only if it binds in every way, SCENARIO names its assumption", not bad3,
      bad3 or f"{(SH.blocked_total > 1000).sum()} shortages over 1T")

# 7. labels and UNTESTED rows
lab_ok = SH.label.isin(["S", "E", "A"]).all() and ROWS.missing_label.isin(["S", "E", "A"]).all()
unt = SH[(SH.label == "A") & ((SH.status != "UNTESTED") | (SH.test.fillna("").str.len() < 10))]
check("every shortage labelled; each resting on an assumed size or rate is UNTESTED with its cheapest test", lab_ok and unt.empty,
      list(unt.index) or f"{(SH.status == 'UNTESTED').sum()} UNTESTED")

# 8. ranking and picks follow the rules
pk_layers = PK.main_layer.value_counts()
se_solid = PK.solid_se.sum()
n_se_all = ((RK.verdict == "SOLID") & RK.label.isin(["S", "E"]) & (RK.blocked_total > 0)).sum()
check("six picks: at most two per layer; at least three SOLID [S]/[E] (or every such shortage when fewer exist)",
      len(PK) == 6 and pk_layers.max() <= 2 and se_solid >= min(3, n_se_all), f"{se_solid} SOLID [S]/[E] of {n_se_all}; {PK.rule.iloc[0]}")
check("ranking counts each shortage on its own (no sharing rule) and shows the order under every scenario",
      all(abs(RK.loc[k, "blocked_total"] - SH.loc[k, "blocked_total"]) < 0.5 for k in RK.index) and
      all(c in RK.columns for c in [f"rank_S{i}" for i in range(7)]), f"{len(RK)} shortages, {int(RK.rank_changes.sum())} change rank across scenarios")

# 9. every searched number is listed with its source
check("every web search listed with its source", SR.url.str.startswith("http").all() and SR.source.str.len().gt(20).all(), f"{len(SR)} searches")

# 10. other folders untouched
def untouched(ref, root, tracked):
    """Files that differ from the reference, and files absent here that git does not hold (gitignored raw downloads or
    never-committed files, so a fresh clone lacks them). A missing file that git tracks counts as changed."""
    changed, absent = [], []
    for p, h in ref.items():
        f = root / p
        if not f.exists():
            (absent if tracked is not None and p not in tracked else changed).append(p)
        elif hashlib.sha1(f.read_bytes()).hexdigest() != h:
            changed.append(p)
    return changed, absent


ref = {}
for line in (V4 / "work" / "hashes_reference.txt").read_text().splitlines():
    h, p = line.split(None, 1)
    ref[p] = h
try:
    ls = subprocess.run(["git", "ls-files", "-z", "stage1-v1", "stage1-v2", "stage1-v3"], cwd=ROOT, capture_output=True, check=True)
    tracked = set(ls.stdout.decode().split("\0")) - {""}
except (OSError, subprocess.CalledProcessError):
    tracked = None
changed, absent = untouched(ref, ROOT, tracked)
check("stage1-v1, stage1-v2 and stage1-v3 untouched", not changed,
      changed[:5] or (f"{len(ref)} files" if not absent else
                      f"{len(ref) - len(absent)} files identical; {len(absent)} not in this checkout and not in git: " + ", ".join(absent)))

out = pd.DataFrame(checks)
out.to_csv(V4 / "checks_v4.csv", index=False)
for c in checks:
    print("PASS" if c["ok"] else "FAIL", c["check"], "|", c["detail"][:220])
if not out.ok.all():
    sys.exit("a check failed: stop")
