"""Stage 1 v4: what stops the missing output. Run from stage1-v4/code:  python v4.py

Writes, in stage1-v4/:
  layers_v4.csv      step 1: the v3 requirements table by layer
  variants_v4.csv    step 3: every way of producing the output used to test a shortage, with its source
  own_blocks_v4.csv  output each shortage blocks on its own, by variant, layer and tier
  shortages_v4.csv   step 2-3: each shortage, physical size, share of need met, output blocked (range), SOLID/SCENARIO
  rows_v4.csv        step 2: every missing row (layer x tier): the shortage that binds first and what each blocks
  unexplained_v4.csv step 2: output no input explains, by row, with its size and no name
  demand_v4.csv      step 4: demand check for the five largest layers
  ranking_v4.csv     step 5: shortages ranked by output blocked on their own, and the rank under every scenario
  picks_v4.csv       step 5: the six picks and why each beat the next one down
  agents_v4.csv      headline numbers for the page's agent streams
  searches_v4.csv    every web search used, with its source
No number is typed by hand except the two searched ratios, which are kept in searches_v4.csv with their sources.
"""
import math

import numpy as np
import pandas as pd

import model as M
from common import A4, A6, BL, INPUT_NAME, LAYER_NAME, RQ, V2, V3, V4, num, rd, worst

pd.set_option("display.width", 250)
L = M.layers()
I = M.inputs()
R = M.need_ratios()
D = M.labour_data()
LAY = list(L.index)
T2 = {j: max(0.0, L.loc[j, "tier2"]) for j in LAY}
T1 = {j: max(0.0, L.loc[j, "tier1"]) for j in LAY}
TX = {j: L.loc[j, "target"] for j in LAY}
MISS = {j: T1[j] + T2[j] for j in LAY}
HO, HP = I.loc["K18", "O"], I.loc["K18", "P"]
TODAY_T, TARGET_T = L.today.sum(), L.target.sum()

# ---------------------------------------------------------------- searched ratios (steps 2-4), kept with their sources
SEARCH = pd.DataFrame([
    dict(id="W1", query="Mueller 2012 Nature closing yield gaps nutrient water management production increase 45% 70% attainable yields",
         source="Mueller N.D. et al. (2012) Closing yield gaps through nutrient and water management, Nature 490:254-257, doi:10.1038/nature11420",
         url="https://ideas.repec.org/a/nat/nature/v490y2012i7419d10.1038_nature11420.html",
         value_low=0.45, value_high=0.70, label="S",
         used_for="K22 cropland, step 3: crop output per ha if yield gaps close to 100% of attainable yields (+45% to +70% for most crops)"),
    dict(id="W2", query="Smith Newhouse Freeland 2009 Health Affairs technology share of health spending growth 27 to 48 percent",
         source="Smith S., Newhouse J.P., Freeland M.S. (2009) Income, insurance, and technology: why does health spending outpace economic growth? Health Affairs 28(5):1276-1284",
         url="https://www.hks.harvard.edu/publications/income-insurance-and-technology-why-does-health-spending-outpace-economic-growth",
         value_low=0.27, value_high=0.48, label="S",
         used_for="K15 clinical approval, step 3: share of health spending growth that comes from new medical technology (27% to 48%)"),
])
SEARCH.to_csv(V4 / "searches_v4.csv", index=False)
W = SEARCH.set_index("id")


# ---------------------------------------------------------------- need functions
def lin(m):
    return lambda X: sum(m.get(j, 0.0) * X[j] for j in X)


def order_by(m):
    return [j for j in sorted(m, key=lambda j: -m[j]) if m[j] > 0]


def ratio(k, col="m"):
    x = R[R.input_id == k]
    return {a: (b if not (isinstance(b, float) and math.isnan(b)) else 0.0) for a, b in zip(x.layer_id, x[col])}


LAB_SRC = ("stage1-v3/requirements/requirements.csv (K18 work_ratio_2035); stage1-v3/repair/agent4_bottlenecks/work/model_memo.json (cog, "
           "gw_per_we); stage1-v3/repair/agent4_bottlenecks/work/need_by_layer.csv (K03, K04 on course); stage1-v3/requirements/model_constants.csv (MK13, MK14)")
trend_w = {}
nb = rd(A4 / "work" / "need_by_layer.csv")
for x in nb[nb.input_id == "K18"].itertuples():
    trend_w[x.layer_id] = num(x.intensity_unit_per_ppp_bn, 0.0)
D_TREND = dict(D, w=trend_w)
LAB_METHODS = {
    "base": (D, "base", "v3 split: humans spread over all tasks; the gap split cognitive (AI) / physical (robots) by each layer's high-skill share", "E"),
    "cogfirst": (D, "cogfirst", "AI agents take every cognitive task first; humans move to physical tasks; robots only for physical work beyond them", "E"),
    "humanscog": (D, "humanscog", "humans keep the cognitive tasks; AI adds no work; robots fill the whole gap", "E"),
    "trendprod": (D_TREND, "base", "v3 split, with output per worker on its trend (Agent 4 2035 intensity) instead of each sector's fastest 9-year growth", "E"),
}


def lab_fn(method, H, what):
    Dm, meth = LAB_METHODS[method][0], LAB_METHODS[method][1]
    return lambda X: M.labour(X, Dm, H, meth)[what]


def lab_order(method, what):
    Dm = LAB_METHODS[method][0]
    w, cog = Dm["w"], Dm["cog"]
    if what == "rob":
        sc = {j: w.get(j, 0) * (1 - cog[j]) for j in LAY}
    elif what in ("ai", "k05"):
        sc = {j: w.get(j, 0) * cog[j] for j in LAY}
    elif what == "k16":
        sc = {j: w.get(j, 0) * D["lic"].get(j, 0) for j in LAY}
    else:
        sc = {j: w.get(j, 0) for j in LAY}
    return order_by(sc)


# electricity: footprint (v3 ratio net of machine-labour electricity) + machine-labour electricity of the task split
lab_t = M.labour(TX, D, HP, "base")
Wt_t = lab_t["Wt"]
sh_t = max(0.0, Wt_t - HP) / Wt_t
EM_base_j = {j: D["w"].get(j, 0) * TX[j] * sh_t * ((1 - D["cog"][j]) * D["e_rob"] + D["cog"][j] * D["e_ai"]) for j in LAY}
fp_base = {j: (ratio("K01")[j] * TX[j] - EM_base_j[j]) / TX[j] if TX[j] else 0.0 for j in LAY}
fp_alt = {j: (ratio("K01", "m_alt")[j] * TX[j] - EM_base_j[j]) / TX[j] if TX[j] else 0.0 for j in LAY}

# 2026 intensities held (today's technology): each input's 2026 supply per unit of 2026 world output, with v3's layer pattern
def today_ratio(k):
    m = ratio(k)
    s, Rk = I.loc[k, "s2026"], I.loc[k, "R"]
    f = (s / TODAY_T) / (Rk / TARGET_T)
    return {j: m[j] * f for j in m}, f


# capital-output ratio on its 2013-23 trend: PWT capital growth (landcap GK_1323) against UN AMA world GDP growth 2013-23
lc = rd(RQ / "work" / "landcap.csv").set_index("id")
ama = pd.read_csv(BL / "work" / "ama_ppp.csv", index_col=0)
g_gdp_1323 = (ama.loc[2023, "Gross Domestic Product (GDP)"] / ama.loc[2013, "Gross Domestic Product (GDP)"]) ** (1 / 10) - 1
ky_trend_f = ((1 + num(lc.loc["GK_1323", "value"])) / (1 + g_gdp_1323)) ** 12      # 2023 -> 2035, relative to K/Y held
yld_base = (1 + num(lc.loc["GY_MAX9", "value"])) ** 9                              # base: crop output per ha at its fastest 9-year growth


def V(vid, desc, src, label, need_prec, need_oc, order, s_oc, s_prec, scen):
    return dict(vid=vid, desc=desc, src=src, label=label, need_prec=need_prec, need_oc=need_oc, order=order, s_oc=s_oc, s_prec=s_prec, scen=scen)


CES = ("K06", "K07", "K08")   # Agent 4 rule: CES substitution, elasticity 0.4 (v3's conversion of a missing unit into output)


def ces(Rk, mces):
    """CES inputs: need at the target is the true requirement Rk; each bn of output given up saves only mces units (the
    substitution elasticity 0.4 lets other inputs replace most of a missing unit), so the output lost is excess / mces."""
    return lambda X: Rk - sum(mces.get(j, 0.0) * (TX[j] - X[j]) for j in X)


def linear_variants(k, n_more=0):
    """base (v3 best-achievable ratio), v3's second sourced ratio, and, only where fewer than two other ways exist, the
    2026 intensity held (today's technology). n_more: alternatives the caller adds itself."""
    out = []
    m = ratio(k)
    mb = ratio(k, "m_ces") if k in CES else m
    note = " (CES substitution, elasticity 0.4: v3's missing/output-depending conversion)" if k in CES else ""
    out.append(V("base", "v3 ratio: " + R[R.input_id == k].ratio_kind.iloc[0] + note, "stage1-v3/requirements/requirements.csv (ratio_2035)"
                 + ("; input_layer_v3.csv (missing, output_depending_ppp_bn)" if k in CES else ""),
                 I.loc[k, "R_label"], *( (ces(I.loc[k, "R"], mb),) * 2 if k in CES else (lin(mb),) * 2), order_by(mb), I.loc[k, "O"], I.loc[k, "P"], "S0"))
    ma = ratio(k, "m_alt")
    altb = R[R.input_id == k].ratio_alt_basis.iloc[0]
    if sum(ma.values()) > 0 and altb not in ("", "same") and abs(sum(ma[j] * TX[j] for j in LAY) - I.loc[k, "R"]) > 1e-6 * I.loc[k, "R"]:
        mab = {j: mb[j] * (ma[j] / m[j]) if m[j] else 0.0 for j in m}
        Ra = sum(ma[j] * TX[j] for j in LAY)
        fa = ces(Ra, mab) if k in CES else lin(mab)
        out.append(V("v3alt", "v3 second ratio: " + altb + note, "stage1-v3/requirements/requirements.csv (ratio_alt_2035)", "E",
                     fa, fa, order_by(mab), I.loc[k, "O"], I.loc[k, "P"], "S4"))
    if len(out) - 1 + n_more < 2 and not math.isnan(I.loc[k, "s2026"]) and k not in ("K11", "K12", "K15"):
        mt, f = today_ratio(k)
        mtb = {j: mb[j] * f for j in mb}
        ft = ces(I.loc[k, "R"] * f, mtb) if k in CES else lin(mtb)
        out.append(V("today", f"2026 intensity held (today's technology, no efficiency gain): x{f:.2f} the v3 ratio" + note,
                     I.loc[k, "s2026_src"] + "; 2026 world output (repaired baseline)",
                     worst(I.loc[k, "s2026_label"], "E"), ft, ft, order_by(mtb), I.loc[k, "O"], I.loc[k, "P"], "S5"))
    return out


def variants(k):
    nm = INPUT_NAME[k][0]
    if k in ("K04", "K03", "K05", "K16"):
        what = {"K04": "rob", "K03": "ai", "K05": "k05", "K16": "k16"}[k]
        out = []
        for meth, (Dm, mm, desc, lab) in LAB_METHODS.items():
            scen = {"base": "S0", "cogfirst": "S1", "humanscog": "S2", "trendprod": "S3"}[meth]
            if k == "K05":
                fp = lambda H, meth=meth: (lambda X: lab_fn(meth, H, "ai")(X) * D["gw_per_we"])
                out.append(V(meth, desc, LAB_SRC, worst(lab, "A"), fp(HP), fp(HO), lab_order(meth, "ai"), I.loc[k, "O"], I.loc[k, "P"], scen))
            elif k == "K16":
                # machine work that current rules allow (outside the licensed share of tasks), held at its level for the target
                lt = M.labour(TX, Dm, HP, mm)
                on16 = sum(D["on03"].values()) + sum(D["on04"].values())
                s_on = on16 + lt["mw"] - lt["lic_on"]
                s_pr = on16 + lt["mw"] - lt["lic_prec"]
                need = lambda H, meth=meth: (lambda X: on16 + lab_fn(meth, H, "mw")(X))
                out.append(V(meth, desc + "; licensed share of each layer's machine work not allowed (BLS shares via A4v1), half after the reform precedent",
                             LAB_SRC + "; stage1-v3/requirements/code/inputs.py (LIC, LIC_REFORM)", "A", need(HP), need(HO),
                             lab_order(meth, "k16"), s_on, s_pr, scen))
            else:
                out.append(V(meth, desc, LAB_SRC, worst(lab, "A") if k == "K04" else lab, lab_fn(meth, HP, what), lab_fn(meth, HO, what),
                             lab_order(meth, what), I.loc[k, "O"], I.loc[k, "P"], scen))
        if k == "K16":   # task reallocation: humans take the licensed tasks, machines the rest
            licw = lambda X: sum(D["lic"].get(j, 0) * D["w"].get(j, 0) * X[j] for j in X)
            out.append(V("humanslicensed", "humans take the licensed tasks first, machines the rest: licensing binds only if licensed work exceeds all human work",
                         LAB_SRC + "; stage1-v3/requirements/code/inputs.py (LIC)", "E", licw, licw, lab_order("base", "k16"), HO, HP, "S6"))
        return out
    if k in ("K01", "K13", "K14"):
        out = []
        for meth, (Dm, mm, desc, lab) in LAB_METHODS.items():
            scen = {"base": "S0", "cogfirst": "S1", "humanscog": "S2", "trendprod": "S3"}[meth]
            f = lambda H, meth=meth: (lambda X: lin(fp_base)(X) + lab_fn(meth, H, "em")(X))
            out.append(V(meth, "IEA NZE footprint + machine-labour electricity, task split: " + desc,
                         "stage1-v3/requirements/requirements.csv (K01 ratio_2035, net of machine-labour electricity); " + LAB_SRC,
                         worst(I.loc[k, "R_label"], lab), f(HP), f(HO), order_by(fp_base), I.loc[k, "O"], I.loc[k, "P"], scen))
        fa = lambda H: (lambda X: lin(fp_alt)(X) + lab_fn("base", H, "em")(X))
        out.append(V("v3alt", "IEA STEPS 2035 footprint (v3 second ratio) + machine-labour electricity (v3 split)",
                     "stage1-v3/requirements/requirements.csv (K01 ratio_alt_2035)", "E", fa(HP), fa(HO), order_by(fp_alt), I.loc[k, "O"], I.loc[k, "P"], "S4"))
        return out
    if k == "K18":
        m = ratio(k)
        out = linear_variants(k, n_more=1)
        mt = {j: m[j] * (trend_w.get(j, 0) / D["w"][j]) if D["w"].get(j) else 0.0 for j in m}
        out.append(V("trendprod", "humans' share of work with output per worker on its trend (Agent 4 2035 intensity) instead of the fastest 9-year growth",
                     LAB_SRC, "E", lin(mt), lin(mt), order_by(mt), I.loc[k, "O"], I.loc[k, "P"], "S3"))
        return out
    if k == "K21":
        m = ratio(k)
        out = linear_variants(k, n_more=1)
        mt = {j: m[j] * ky_trend_f for j in m}
        out.append(V("kytrend", f"capital-output ratio on its 2013-23 trend: x{ky_trend_f:.3f} today's (PWT capital growth {num(lc.loc['GK_1323', 'value']):.2%} vs UN AMA GDP growth {g_gdp_1323:.2%})",
                     "stage1-v3/requirements/work/landcap.csv (GK_1323, PWT 11.0); stage1-v3/repair/baseline/work/ama_ppp.csv (UN AMA world GDP 2013, 2023)",
                     "E", lin(mt), lin(mt), order_by(mt), I.loc[k, "O"], I.loc[k, "P"], "S5"))
        return out
    if k == "K22":
        m = ratio(k)
        out = linear_variants(k, n_more=2)
        for tag, v in (("low", W.loc["W1", "value_low"]), ("high", W.loc["W1", "value_high"])):
            f = yld_base / (1 + v)
            mt = {j: m[j] * f for j in m}
            out.append(V(f"yieldgap_{tag}", f"yield gaps closed to attainable yields: crop output per ha +{v:.0%} on 2026 (x{f:.3f} the v3 ratio)",
                         "searches_v4.csv W1 (Mueller et al. 2012); stage1-v3/requirements/work/landcap.csv (GY_MAX9)", "E",
                         lin(mt), lin(mt), order_by(mt), I.loc[k, "O"], I.loc[k, "P"], "S6"))
        return out
    if k == "K15":
        m = ratio(k)
        out = linear_variants(k, n_more=2)
        P_, O_, Tc = I.loc[k, "P"], I.loc[k, "O"], TX["C05"]
        need = lambda X: X["C05"]
        for tag, v in (("low", W.loc["W2", "value_low"]), ("high", W.loc["W2", "value_high"])):
            # only the technology share of health output growth needs newly approved products; the rest is more of approved
            # care, allowed under current rules: the allowed output rises by (1 - share) of the growth beyond the current-rule path
            out.append(V(f"techshare_{tag}", f"only {v:.0%} of health output growth beyond the current-rule path comes from new technology needing approval; the rest is more of approved care",
                         "searches_v4.csv W2 (Smith, Newhouse, Freeland 2009)", "E", need, need, ["C05"],
                         O_ + (1 - v) * (Tc - O_), P_ + (1 - v) * (Tc - P_), "S6"))
        return out
    return linear_variants(k)


SHORT_IDS = [k for k in I.index if k in INPUT_NAME]
results = {}       # (k, vid) -> own block dict
VARS = {}
for k in SHORT_IDS:
    VARS[k] = variants(k)
    for v in VARS[k]:
        ob = M.own_block(v["need_prec"], v["need_oc"], v["order"], v["s_oc"], v["s_prec"], TX, T2, T1)
        ob["need_target"] = v["need_prec"](TX)
        results[(k, v["vid"])] = ob

# ---------------------------------------------------------------- step 4: demand for the five largest layers
dt = rd(BL / "demand_test.csv").set_index("id")
te = rd(RQ / "target_economy.csv").set_index("layer_id")
j6 = rd(A6 / "joint.csv")
ach = j6[j6.case == "achievable"].set_index("layer_id")
big5 = list(L.target.sort_values(ascending=False).index[:5])
inc_ratio = TARGET_T / TODAY_T
dem_rows, BUY = [], {}
obs = {"C12": "General government final consumption expenditure", "C13": "Construction (ISIC F)"}
WINDOWS = ((2014, 2024), (2004, 2014), (1994, 2004))


def eta_obs(col, a0, a1):
    return (math.log(ama.loc[a1, col] / ama.loc[a0, col]) /
            math.log(ama.loc[a1, "Gross Domestic Product (GDP)"] / ama.loc[a0, "Gross Domestic Product (GDP)"]))


ETA_ALT = {}
for j in big5:
    if j in obs:
        eta = eta_obs(obs[j], *WINDOWS[0])
        ETA_ALT[j] = [(f"{a0}-{a1}", eta_obs(obs[j], a0, a1)) for a0, a1 in WINDOWS[1:]]
        eta_src, eta_lab = f"stage1-v3/repair/baseline/work/ama_ppp.csv (UN AMA world, {obs[j]} vs GDP, 2014-2024)", "E"
        eps, eps_src = 0.0, "no market price (budgets or investment set quantity): no price response"
    else:
        rr = dt.loc[j]
        eta = math.log(num(rr.demand_ratio_at_today_price)) / math.log(num(rr.income_ratio_at_max))
        eta_src, eta_lab = "stage1-v3/repair/baseline/demand_test.csv (income elasticity from v1 bridge, eta)", rr.income_elasticity_label or "E"
        eps, eps_src = num(rr.price_elasticity), "stage1-v3/repair/baseline/demand_test.csv (price elasticity, Agent 2)"
    pf = num(ach.loc[j, "price_fall_of_product"], 0.0) if j in ach.index else 0.0
    price_ratio = 1 - pf
    dem = L.loc[j, "today"] * inc_ratio ** eta * price_ratio ** eps
    short = max(0.0, L.loc[j, "target"] - dem)
    dem_rows.append(dict(layer_id=j, layer=LAYER_NAME[j], today=L.loc[j, "today"], target=L.loc[j, "target"], extra_output=MISS[j],
                         income_ratio=inc_ratio, income_elasticity=eta, income_elasticity_label=eta_lab, income_elasticity_src=eta_src,
                         price_ratio_2035=price_ratio, price_ratio_src="stage1-v3/repair/agent6_cheap_inputs/joint.csv (achievable case, price_fall_of_product)",
                         price_elasticity=eps, price_elasticity_src=eps_src, demand_2035=dem, demand_label=worst(eta_lab, "E"),
                         buyers_short=short, verdict=("bought: demand at the 2035 price covers the target" if short <= 0 else
                                                     f"not all bought: buyers short of {short:,.0f} bn; buyers are the shortage")))
    if short > 0:
        BUY[j] = (short, worst(eta_lab, "E"), eta_src)
demand = pd.DataFrame(dem_rows)
demand.to_csv(V4 / "demand_v4.csv", index=False)
for j, (short, lab, src) in BUY.items():
    k = f"BUY-{j}"
    INPUT_NAME[k] = (f"Buyers for {LAYER_NAME[j].lower()}", "buyers")
    m = {jj: (1.0 if jj == j else 0.0) for jj in LAY}
    dem_j = demand.set_index("layer_id").loc[j, "demand_2035"]
    oc_fixed = {jj: TX[jj] for jj in LAY}
    need = lambda X, j=j: X[j]
    VARS[k] = [V("base", "demand at the 2035 price from the sourced income response (UN AMA world 2014-2024)", src, lab, need, need, [j], dem_j, dem_j, "S0")]
    drow = demand.set_index("layer_id").loc[j]
    for win, e in ETA_ALT.get(j, []):
        dj = L.loc[j, "today"] * inc_ratio ** e * drow.price_ratio_2035 ** drow.price_elasticity
        VARS[k].append(V(f"eta_{win}", f"demand from the UN AMA world income response {win} (elasticity {e:.2f})",
                         src.replace("2014-2024", win), "E", need, need, [j], dj, dj, "S4" if win.startswith("2004") else "S5"))
    for v in VARS[k]:
        results[(k, v["vid"])] = M.own_block(need, need, [j], v["s_oc"], v["s_prec"], TX, T2, T1)
        results[(k, v["vid"])]["need_target"] = TX[j]
    SHORT_IDS.append(k)
demand["income_elasticity_other_windows"] = demand.layer_id.map(lambda j: "; ".join(f"{w}: {e:.3f}" for w, e in ETA_ALT.get(j, [])))
demand.to_csv(V4 / "demand_v4.csv", index=False)

# ---------------------------------------------------------------- shortages table (steps 2-3)
def gr(a, b):
    return (b / a) ** (1 / 9) - 1 if a and b and a > 0 and b > 0 else np.nan


sh_rows, ob_rows, var_rows = [], [], []
for k in SHORT_IDS:
    base = results[(k, "base")]
    tot = {v["vid"]: results[(k, v["vid"])]["total"] for v in VARS[k]}
    lo, hi = min(tot.values()), max(tot.values())
    binds_all = all(t > 1e-6 for t in tot.values())
    tested_3 = base["total"] > 1000
    n_alt = len(VARS[k]) - 1
    identity = k in ("K11",)      # saving = investment at market prices: no way of producing the target changes it
    if not tested_3:
        verdict = "SCENARIO (binds only in another way of producing)" if hi > 1000 else "not tested (blocks 1T or less)"
    elif identity:
        verdict = "SOLID"
    elif n_alt < 2:
        verdict = "SCENARIO"
    else:
        verdict = "SOLID" if binds_all else "SCENARIO"
    dep = "; ".join(v["desc"] for v in VARS[k] if results[(k, v["vid"])]["total"] <= 1e-6)
    if not tested_3 and hi > 1000:
        dep = "binds only under: " + "; ".join(v["desc"] for v in VARS[k] if results[(k, v["vid"])]["total"] > 1000)
    if identity and tested_3:
        dep = "accounting identity (saving = investment in C13-C15 at market prices): binds under every way of producing; rests on the on-course saving rate (2015-24 mean, A)"
    elif verdict == "SCENARIO" and n_alt < 2:
        dep = "fewer than two sourced alternative ways found; " + (dep or "rests on the base ratio")
    is_inp = k in I.index
    need_t = base["need_target"]
    s_oc = VARS[k][0]["s_oc"]; s_pr = VARS[k][0]["s_prec"]
    lab_parts = [VARS[k][0]["label"]]
    if is_inp:
        lab_parts += [I.loc[k, "O_label"], I.loc[k, "P_label"]]
    # label of the blocked output: also A when a whole tier row (an [A] amount) is dropped
    rows_hit = [(j, t) for t, dd in (("2", base["t2"]), ("1", base["t1"])) for j, v in dd.items() if v > 1e-6]
    full_row = any(abs((T2 if t == "2" else T1)[j] - (base["t2"] if t == "2" else base["t1"])[j]) < 1e-6 for j, t in rows_hit)
    row_labs = [L.loc[j, "tier2_label" if t == "2" else "tier1_label"] for j, t in rows_hit] if full_row else []
    label = worst(*lab_parts, *row_labs)
    by_layer = {j: base["t2"][j] + base["t1"][j] for j in LAY}
    main = max(by_layer, key=by_layer.get) if base["total"] > 0 else ""
    untested = label == "A" or any(l == "A" for l in lab_parts)
    test = I.loc[k, "test"] if is_inp else ("UN AMA construction and government consumption vs GDP by region 1995-2024, to see whether the world elasticity holds as incomes rise" if k.startswith("BUY") else "")
    if is_inp and not test:
        test = {"K20": "IEA STEPS/NZE 2035 aluminium demand by end use (WEO Annex) against the EXIOBASE footprint, and 2026 smelter capacity by region",
                "K21": "PWT 11.0 capital-output ratios by country 1990-2023 for economies growing 5%+ a year (does K/Y fall during fast growth?)",
                "K22": "FAOSTAT crop output per ha and cropland by country 2000-2024, and the EXIOBASE crop footprint per $ of non-food output 1995-2022 (does it fall with income?)"}.get(k, "")
    gneed, gprec = (gr(I.loc[k, "s2026"], need_t), gr(I.loc[k, "s2026"], s_pr)) if is_inp else (np.nan, np.nan)
    sh_rows.append(dict(shortage_id=k, shortage=INPUT_NAME[k][0], category=INPUT_NAME[k][1], unit=(I.loc[k, "unit"] if is_inp else "bn 2026 int$ of output bought"),
                        need_2035=need_t, supply_on_course=s_oc, supply_precedent=s_pr,
                        share_met_on_course=s_oc / need_t if need_t else np.nan, share_met_precedent=s_pr / need_t if need_t else np.nan,
                        short_at_precedent=max(0.0, need_t - s_pr), short_on_course=max(0.0, need_t - s_oc),
                        blocked_tier2=sum(base["t2"].values()), blocked_tier1=sum(base["t1"].values()),
                        blocked_tier1_beyond_precedent=sum(base["t1_beyond_precedent"].values()),
                        tier1_competing_with_tier2=sum(base["t1_competing"].values()), blocked_total=base["total"],
                        excess_not_absorbed_on_course=base["left_oc"], excess_not_absorbed_precedent=base["left_prec"],
                        blocked_low=lo, blocked_high=hi, n_alternatives=n_alt, verdict=verdict, depends_on=dep,
                        label=label, status=("UNTESTED" if untested else "TESTED"), test=(test if untested else ""),
                        growth_needed_pa=gneed, growth_precedent_pa=gprec,
                        times_faster=(gneed / gprec if gprec and gprec > 0 and not math.isnan(gneed) else np.nan),
                        fastest_precedent=(I.loc[k, "fastest_precedent"] if is_inp else ""), main_layer=main,
                        src=("stage1-v3/requirements/inputs_v3.csv; requirements.csv" if is_inp else "demand_v4.csv")))
    for v in VARS[k]:
        ob = results[(k, v["vid"])]
        var_rows.append(dict(shortage_id=k, variant=v["vid"], scenario=v["scen"], way_of_producing=v["desc"], src=v["src"], label=v["label"],
                             need_2035=ob["need_target"], supply_on_course=v["s_oc"], supply_precedent=v["s_prec"],
                             blocked_tier2=sum(ob["t2"].values()), blocked_tier1=sum(ob["t1"].values()), blocked_total=ob["total"],
                             binds=ob["total"] > 1e-6))
        for t, dd in (("2", ob["t2"]), ("1", ob["t1"])):
            for j, val in dd.items():
                if val > 1e-9:
                    ob_rows.append(dict(shortage_id=k, variant=v["vid"], layer_id=j, tier=t, blocked_bn=val))
SH = pd.DataFrame(sh_rows).set_index("shortage_id")
OB = pd.DataFrame(ob_rows)
VR = pd.DataFrame(var_rows)

# ---------------------------------------------------------------- rows (step 2): binding shortage and unexplained output
row_rows, une_rows = [], []
for j in LAY:
    for t, amt in (("1", T1[j]), ("2", T2[j])):
        if amt <= 1e-9:
            continue
        x = OB[(OB.variant == "base") & (OB.layer_id == j) & (OB.tier == t)].copy()
        x["share_met"] = x.shortage_id.map(lambda k: SH.loc[k, "share_met_precedent" if t == "2" else "share_met_on_course"])
        x = x.sort_values(["blocked_bn", "share_met"], ascending=[False, True])
        explained = float(x.blocked_bn.max()) if len(x) else 0.0
        une = max(0.0, amt - explained)
        first = x.iloc[0] if len(x) else None
        k = first.shortage_id if first is not None else ""
        why = ""
        if t == "1" and k:
            why = (f"{SH.loc[k, 'shortage']} on course meets {SH.loc[k, 'share_met_on_course']:.0%} of need; at its fastest precedent "
                   f"({SH.loc[k, 'fastest_precedent'][:90] or 'record'}) it meets {SH.loc[k, 'share_met_precedent']:.0%}: on course it grows at its trend, not its record pace")
            if j in dt.index:
                why += f". Buyers (repaired baseline demand test at the max-output income): {dt.loc[j, 'verdict']}"
        elif t == "2" and k:
            g, gp = SH.loc[k, "growth_needed_pa"], SH.loc[k, "growth_precedent_pa"]
            why = (f"{SH.loc[k, 'shortage']} would have to grow {g:.1%} a year 2026-35 against {gp:.1%} at its fastest precedent ({SH.loc[k, 'times_faster']:.1f}x faster)"
                   if not (isinstance(g, float) and math.isnan(g)) else f"{SH.loc[k, 'shortage']}: buyers at the 2035 price fall short")
        phys = (first.blocked_bn * VARS[k][0]["need_prec"](TX) / TX[j] if False else np.nan) if k else np.nan
        row_rows.append(dict(layer_id=j, layer=LAYER_NAME[j], tier=t, missing_bn=amt, missing_label=L.loc[j, f"tier{t}_label"],
                             binds_first=k, binds_first_name=(SH.loc[k, "shortage"] if k else ""), binds_first_category=(SH.loc[k, "category"] if k else ""),
                             binds_first_blocks_bn=explained, binds_first_label=(SH.loc[k, "label"] if k else ""),
                             binds_first_verdict=(SH.loc[k, "verdict"] if k else ""),
                             others=("; ".join(f"{r.shortage_id} {r.blocked_bn:,.0f}" for r in x.iloc[1:].itertuples()) if len(x) > 1 else ""),
                             n_shortages=len(x), explained_bn=min(explained, amt), unexplained_bn=une, why=why))
        if une > 1e-6:
            une_rows.append(dict(layer_id=j, layer=LAYER_NAME[j], tier=t, unexplained_bn=une, label=L.loc[j, f"tier{t}_label"],
                                 note="no input's own numbers explain this output; named by no shortage"))
ROWS = pd.DataFrame(row_rows)
UNE = pd.DataFrame(une_rows, columns=["layer_id", "layer", "tier", "unexplained_bn", "label", "note"])

# ---------------------------------------------------------------- ranking (step 5) and scenario orders
SCEN = {"S0": "base", "S1": "AI takes cognitive tasks first", "S2": "humans keep cognitive tasks", "S3": "labour productivity on trend",
        "S4": "v3 second sourced ratios", "S5": "2026 intensities / K-Y trend", "S6": "land- and approval-saving technology, task reallocation"}
scen_tot = {}
for s in SCEN:
    for k in SHORT_IDS:
        v = next((v for v in VARS[k] if v["scen"] == s), VARS[k][0])
        scen_tot[(s, k)] = results[(k, v["vid"])]["total"]
rk = SH[(SH.blocked_total > 1e-6) | (SH.blocked_high > 1000)].copy()     # also shortages that bind only in another way of producing
rk["group_label"] = np.where(rk.label.isin(["S", "E"]), "S/E", "A")
rk["group_solid"] = np.where(rk.verdict == "SOLID", "SOLID", np.where(rk.verdict == "SCENARIO", "SCENARIO", "untested (<=1T)"))
rk = rk.sort_values("blocked_total", ascending=False)
rk["rank_all"] = range(1, len(rk) + 1)
for s in SCEN:
    vals = pd.Series({k: scen_tot[(s, k)] for k in rk.index})
    rk[f"rank_{s}"] = vals.rank(ascending=False, method="min").astype(int)
    rk[f"blocked_{s}"] = vals
rk["rank_changes"] = rk[[f"rank_{s}" for s in SCEN]].nunique(axis=1) > 1
for g in ("S/E", "A"):
    sub = rk[rk.group_label == g]
    rk.loc[sub.index, "rank_in_label_group"] = range(1, len(sub) + 1)
for g in rk.group_solid.unique():
    sub = rk[rk.group_solid == g]
    rk.loc[sub.index, "rank_in_solid_group"] = range(1, len(sub) + 1)

# picks: six, at least three SOLID and [S]/[E], at most two per layer; the largest blocked output that meets the rules
from itertools import combinations
cand = list(rk[rk.blocked_total > 1e-6].index)
SE_SOLID = [k for k in cand if rk.loc[k, "verdict"] == "SOLID" and rk.loc[k, "label"] in ("S", "E")]
NEED_SE = min(3, len(SE_SOLID))          # the brief asks for three; fewer exist only if the data hold fewer
best = None
for combo in combinations(cand[:16], 6):
    se_solid = sum(1 for k in combo if k in SE_SOLID)
    per = pd.Series([rk.loc[k, "main_layer"] for k in combo]).value_counts()
    if se_solid < NEED_SE or per.max() > 2:
        continue
    val = sum(rk.loc[k, "blocked_total"] for k in combo)
    if best is None or val > best[0]:
        best = (val, combo)
picks = list(best[1]) if best else []
PICK_RULE_NOTE = ("rule met: at least three SOLID [S]/[E]" if NEED_SE >= 3 else
                  f"rule not met: only {len(SE_SOLID)} shortages are both SOLID and [S]/[E] ({', '.join(SE_SOLID)}); all are picked")
rk["pick"] = rk.index.isin(picks)
pick_rows = []
for k in picks:
    pos = cand.index(k)
    nxt = cand[pos + 1] if pos + 1 < len(cand) else None
    se = k in SE_SOLID
    why = (f"blocks {rk.loc[k, 'blocked_total']:,.0f} bn on its own, {rk.loc[k, 'blocked_low']:,.0f}-{rk.loc[k, 'blocked_high']:,.0f} across the ways of producing; "
           f"{rk.loc[k, 'verdict']}" + (f" (depends on: {rk.loc[k, 'depends_on'][:120]})" if rk.loc[k, "verdict"] == "SCENARIO" else ""))
    if nxt:
        why += (f". Beat the next one down, {nxt} {rk.loc[nxt, 'shortage']} ({rk.loc[nxt, 'blocked_total']:,.0f} bn, {rk.loc[nxt, 'verdict']}), "
                f"by {rk.loc[k, 'blocked_total'] - rk.loc[nxt, 'blocked_total']:,.0f} bn")
    first_out = next((c for c in cand if c not in picks), None)
    if k == picks[-1] and first_out:
        why += f"; the first shortage left out is {first_out} {rk.loc[first_out, 'shortage']} ({rk.loc[first_out, 'blocked_total']:,.0f} bn)"
    pick_rows.append(dict(shortage_id=k, shortage=rk.loc[k, "shortage"], category=rk.loc[k, "category"], main_layer=rk.loc[k, "main_layer"], rule=PICK_RULE_NOTE,
                          blocked_total=rk.loc[k, "blocked_total"], blocked_low=rk.loc[k, "blocked_low"], blocked_high=rk.loc[k, "blocked_high"],
                          verdict=rk.loc[k, "verdict"], label=rk.loc[k, "label"], solid_se=se, rank_all=rk.loc[k, "rank_all"], why=why))
PICKS = pd.DataFrame(pick_rows)

# ---------------------------------------------------------------- agents (page column 1): headline numbers, v3/v4 where changed
RB = rd(BL / "totals.csv").set_index("id")
b4 = rd(A4 / "bottlenecks.csv").set_index("id")
ins, nin = rd(BL / "inside_on_course.csv"), rd(BL / "not_inside.csv")
cnt = lambda d, a: int(d.source_file.str.contains(a).sum())
nd = rd(V2 / "v1" / "agent3_needs" / "needs.csv").set_index("id")
i6 = rd(A6 / "inputs.csv")
ag = [
    ("a1_in", cnt(ins, "agent1"), "E", "stage1-v3/repair/baseline/inside_on_course.csv (Agent 1 rows), counted", "Agent 1 items already inside on course"),
    ("a1_all", cnt(ins, "agent1") + cnt(nin, "agent1"), "E", "stage1-v3/repair/baseline/inside_on_course.csv + not_inside.csv (Agent 1 rows)", "Agent 1 items checked against on course"),
    ("a2_energy", num(RB.loc["T16", "value_ppp_bn"]), RB.loc["T16", "value_ppp_bn_label"], "stage1-v3/repair/baseline/totals.csv (T16)", "Output the fastest sourced energy supply can power in 2035"),
    ("a3_needs", len(nd), "E", "stage1-v2/v1/agent3_needs/needs.csv (rows)", "Human needs mapped"),
    ("a3_health", num(nd.loc["N08", "people_unmet_m"]), nd.loc["N08", "people_unmet_m_label"], "stage1-v2/v1/agent3_needs/needs.csv (N08)", "People lacking adequate health care (millions)"),
    ("a4_bind", int((b4.status == "binding").sum()), "E", "stage1-v3/repair/agent4_bottlenecks/bottlenecks.csv (status = binding)", "Inputs binding at their fastest precedent (repaired Agent 4)"),
    ("a4_total", len(b4), "E", "stage1-v3/repair/agent4_bottlenecks/bottlenecks.csv (rows)", "Inputs Agent 4 tested"),
    ("a4_blk", num(b4.loc["K18", "output_blocked_ppp_bn"]), b4.loc["K18", "output_blocked_ppp_bn_label"], "stage1-v3/repair/agent4_bottlenecks/bottlenecks.csv (K18 output_blocked_ppp_bn)", "Output the human workforce blocks (repaired Agent 4)"),
    ("a5_in", cnt(ins, "agent5"), "E", "stage1-v3/repair/baseline/inside_on_course.csv (Agent 5 rows)", "Agent 5 capabilities already inside on course"),
    ("a5_caps", cnt(ins, "agent5") + cnt(nin, "agent5"), "E", "stage1-v3/repair/baseline/inside_on_course.csv + not_inside.csv (Agent 5 rows)", "Agent 5 capabilities checked"),
    ("a6_reach3", int((i6.reach_3x == "yes").sum()), "E", "stage1-v3/repair/agent6_cheap_inputs/inputs.csv (reach_3x = yes)", "Inputs that reach 3x cheaper by 2035 (repaired Agent 6)"),
    ("a6_n", len(i6), "E", "stage1-v3/repair/agent6_cheap_inputs/inputs.csv (rows)", "Inputs Agent 6 priced"),
    ("a6_ach", num(ach.loc["TOTAL", "net_created_ppp_bn"]), ach.loc["TOTAL", "net_created_ppp_bn_label"], "stage1-v3/repair/agent6_cheap_inputs/joint.csv (achievable TOTAL)", "Output created at sourced input prices, net of on course"),
]
pd.DataFrame(ag, columns=["key", "value", "label", "src", "what"]).to_csv(V4 / "agents_v4.csv", index=False)

# ---------------------------------------------------------------- write
lay_out = L.copy()
lay_out.insert(0, "layer", [LAYER_NAME[j] for j in L.index])
lay_out["src"] = "stage1-v3/requirements/layers_v3.csv; stage1-v3/repair/baseline/layers.csv (today)"
lay_out.to_csv(V4 / "layers_v4.csv")
VR.to_csv(V4 / "variants_v4.csv", index=False)
OB.to_csv(V4 / "own_blocks_v4.csv", index=False)
SH.to_csv(V4 / "shortages_v4.csv")
ROWS.to_csv(V4 / "rows_v4.csv", index=False)
UNE.to_csv(V4 / "unexplained_v4.csv", index=False)
rk.to_csv(V4 / "ranking_v4.csv")
PICKS.to_csv(V4 / "picks_v4.csv", index=False)
pd.DataFrame([dict(id=s, scenario=d) for s, d in SCEN.items()]).to_csv(V4 / "work" / "scenarios_v4.csv", index=False)

if __name__ == "__main__":
    print(L[["target", "today", "growth", "tier1", "tier2"]].sum().round(1).to_dict())
    print(SH[["blocked_tier2", "blocked_tier1", "blocked_total", "blocked_low", "blocked_high", "verdict", "label", "share_met_precedent", "main_layer"]].round(2).sort_values("blocked_total", ascending=False).to_string())
    print("explained", ROWS.explained_bn.sum().round(1), "unexplained", ROWS.unexplained_bn.sum().round(1), "missing", sum(MISS.values()).round(1))
    print(demand[["layer_id", "income_elasticity", "price_ratio_2035", "demand_2035", "target", "buyers_short"]].round(3).to_string())
    print(PICKS.to_string() if len(PICKS) else "NO PICKS MEET THE RULES")
