"""Check of the homothetic gap split against Engel effects.
World spend-weighted income (expenditure) elasticities from USDA ERS TB-1929 Appendix table 1 (144 countries), weighted
by ICP 2021 market-rate spending by country and category (same method as Agent 2's code/elasticity.py, which used
Appendix table 4). Each layer's required extra volume is scaled by (1+x)^eta - 1 with x the economy-wide real gain the
gap implies, then renormalised so layers still sum to the gap. Layers outside the household demand system get eta = 1.
Writes engel_check.csv.
"""
import json, re, unicodedata
from pathlib import Path

import pandas as pd

ROOT = Path(__file__).resolve().parents[2]
RAW = ROOT / "agent2_ratios" / "raw"
OUT = ROOT / "bridge"
def world_income_elasticities():
    txt = open(RAW / "usda_tb1929/tb1929.txt").read().splitlines()
    cols = ['food', 'clothing', 'housing', 'furnishings', 'health', 'transcomm', 'recreation', 'education', 'other']
    start = next(i for i, l in enumerate(txt) if l.strip().startswith('Appendix table 1') and i > 1300)
    end = next(i for i, l in enumerate(txt) if l.strip().startswith('Appendix table 2') and i > start)
    num = r'(-?\d\.\d{3})a?'
    rx = re.compile(r'^\s*([A-Z][A-Za-z\.,\'\-\(\) ]+?)\s+' + r'\s+'.join([num] * 9) + r'\s*$')
    el = {}
    for l in txt[start:end]:
        m = rx.match(l)
        if m:
            el[m.group(1).strip()] = [float(x) for x in m.groups()[1:]]


    def norm(s):
        s = unicodedata.normalize('NFKD', s).encode('ascii', 'ignore').decode().lower()
        s = re.sub(r'\(.*?\)', '', s)
        for a, b in [(' and ', ' '), ('&', ' '), ('rep.', 'republic'), ('dem.', 'democratic'), (',', ' '), ('.', ' '), ("'", '')]:
            s = s.replace(a, b)
        return ' '.join(s.split())


    d = json.load(open(RAW / 'wb_icp2021_countries/data.json'))['source']['data']
    imfc = set(json.load(open(RAW / 'imf_dm_countries/countries.json'))['countries'])
    ser2col = {'1101000': 'food', '1103000': 'clothing', '9060000': 'housing', '1105000': 'furnishings', '9080000': 'health',
               '1107000': 'transcomm', '1108000': 'transcomm', '9110000': 'recreation', '9120000': 'education', '9140000': 'other'}
    spend, names = {}, {}
    for r in d:
        v = {x['concept']: x['id'] for x in r['variable']}; n = {x['concept']: x['value'] for x in r['variable']}
        if r['value'] is None or v['Country'] not in imfc:
            continue
        names[v['Country']] = n['Country']
        spend.setdefault(v['Country'], {}).setdefault(v['Series'], r['value'])
    idx = {norm(k): k for k in el}
    alias = {'korea republic': 'korea', 'egypt arab republic': 'egypt', 'iran islamic republic': 'iran', 'russian federation': 'russia',
             'turkiye': 'turkey', 'viet nam': 'vietnam', 'slovak republic': 'slovakia', 'kyrgyz republic': 'kyrgyzstan',
             'hong kong sar china': 'hong kong', 'yemen republic': 'yemen'}
    match = {}
    for iso, nm in names.items():
        k = norm(nm); k2 = alias.get(k, k)
        for cand in (k, k2):
            if cand in idx:
                match[iso] = idx[cand]; break
        else:
            for kk in idx:
                if kk.split()[0] == k.split()[0] and len(k.split()[0]) > 4:
                    match[iso] = idx[kk]; break
    eta = {}
    for s, c in ser2col.items():
        j = cols.index(c)
        w = [(spend[i][s], el[match[i]][j]) for i in spend if i in match and spend[i].get(s)]
        eta[s] = sum(a * e for a, e in w) / sum(a for a, _ in w)
    lay_eta = {'C01': eta['1101000'], 'C02': eta['1103000'], 'C03': eta['9060000'], 'C04': eta['1105000'],
               'C05': eta['9080000'], 'C06': eta['1107000'], 'C07': eta['1108000'], 'C08': eta['9110000'],
               'C09': eta['9120000'], 'C10': eta['9140000'], 'C11': eta['9140000']}
    return lay_eta, len(el), len(match)


if __name__ == "__main__":
    lay_eta, n_el, n_match = world_income_elasticities()
    lay = pd.read_csv(OUT / "gap_by_layer.csv")
    macro = pd.read_csv(OUT / "bridge_macro.csv").set_index("id")
    GAP = float(macro.loc["BM05_base", "value"]); B35 = float(macro.loc["BM03_base", "value"])
    x = GAP / B35
    lay["eta"] = lay.id.map(lambda i: lay_eta.get(i, 1.0))
    raw = lay.share_base * ((1 + x) ** lay.eta - 1)
    lay["required_engel_ppp_bn"] = raw / raw.sum() * GAP
    lay["delta_vs_homothetic"] = lay.required_engel_ppp_bn - lay.required_homothetic_ppp_bn
    out = lay[["id", "capability", "eta", "required_homothetic_ppp_bn", "required_engel_ppp_bn", "delta_vs_homothetic"]].copy()
    out["eta_label"] = out.id.map(lambda i: "E" if i in lay_eta else "A")
    out["eta_src"] = "A2:raw/usda_tb1929 Appendix table 1;A2:raw/wb_icp2021_countries;code/engel_check.py"
    out.round(3).to_csv(OUT / "engel_check.csv", index=False)
    print(n_el, "countries parsed;", n_match, "matched")
    print(out.round(2).to_string())
