#!/usr/bin/env python3
"""Étude 3 — les signalements GEIPAN suivent-ils les lancements Starlink ? (voir PROTOCOLE.md, SHA-256 df9e63cf…)

Usage : python3 -I run.py <pylib numpy> <pylib openpyxl> <dossier données GEIPAN> <dossier GCAT (launch.tsv, satcat.tsv)> > resultats.json
Sortie : agrégats seulement (aucun cas, témoignage ni résumé).
"""
import csv, json, math, re, sys, unicodedata, collections
from datetime import date, datetime, timedelta, timezone
from zoneinfo import ZoneInfo

NP, OPX, GEIPAN, GCAT = sys.argv[1:5]
sys.path.insert(0, NP); sys.path.insert(0, OPX)
import numpy as np

PARIS = ZoneInfo("Europe/Paris")
START, END = date(2019, 1, 1), date(2026, 10, 31)
NDAY = (END - START).days + 1
SEED = 20261116
N_PRIM, N_EXPL = (2000, 1000) if not __import__("os").environ.get("ETUDE3_ESSAI") else (20, 10)  # essai rapide du code seulement
METRO = {f"{i:02d}" for i in range(1, 96)} | {"2A", "2B"}
MENTION = "ce document est extrait du site Internet GEIPAN. Informations protégées - Tous droits réservés © CNES (2026)"


def norm(s):
    s = unicodedata.normalize("NFD", str(s or "").lower())
    return "".join(c for c in s if unicodedata.category(c) != "Mn").strip()


def night_of_local(dtl):
    """Nuit = date du soir : heure de Paris moins 12 h."""
    return (dtl - timedelta(hours=12)).date()


def idx(d):
    return (d - START).days


# ----------------------------------------------------------------------------- GCAT
def read_tsv(path):
    with open(path, encoding="utf-8") as f:
        head = f.readline().lstrip("#").rstrip("\n").split("\t")
        return [dict(zip(head, [x.strip() for x in r])) for r in csv.reader(f, delimiter="\t") if r and not r[0].startswith("#")]


def gcat_night(s):
    """'2019 May 24 0230' (UTC) -> nuit de Paris. Sans heure : la date telle quelle."""
    p = s.replace("?", "").split()
    d = datetime.strptime(" ".join(p[:3]), "%Y %b %d")
    if len(p) >= 4 and re.match(r"^\d{4}", p[3]):
        hh, mm = int(p[3][:2]), int(p[3][2:4])
        t = datetime(d.year, d.month, d.day, hh, mm, tzinfo=timezone.utc).astimezone(PARIS)
        return night_of_local(t.replace(tzinfo=None))
    return d.date()


launches = [l for l in read_tsv(f"{GCAT}/launch.tsv") if l["LaunchCode"].startswith("O") and l["Launch_Tag"][:4].isdigit() and int(l["Launch_Tag"][:4]) >= 2018]
for l in launches:
    l["night"] = gcat_night(l["Launch_Date"])
is_sl = lambda l: any("Starlink" in l[k] for k in ("Mission", "Flight", "Group"))
TRIG = {
    "starlink": [l["night"] for l in launches if is_sl(l)],
    "orbital_hors_starlink": [l["night"] for l in launches if not is_sl(l)],
    "kourou": [l["night"] for l in launches if l["Launch_Site"] == "CSG"],
    "falcon9_hors_starlink": [l["night"] for l in launches if not is_sl(l) and l["LV_Type"].startswith("Falcon 9")],
}


def mass(s):
    for k in ("TotMass", "Mass", "DryMass"):
        try:
            v = float(s[k])
            if v > 0:
                return v
        except ValueError:
            pass
    return 0.0


reentries = []
for s in read_tsv(f"{GCAT}/satcat.tsv"):
    if s["Status"] == "R" and s["DDate"][:4].isdigit() and 2018 <= int(s["DDate"][:4]) <= 2026 and mass(s) >= 2000:
        try:
            reentries.append(gcat_night(s["DDate"]))
        except ValueError:
            pass
TRIG["rentree_2t"] = reentries

# ----------------------------------------------------------------------------- GEIPAN
import openpyxl


def read_xlsx(path):
    rows = list(openpyxl.load_workbook(path, read_only=True).worksheets[0].iter_rows(values_only=True))
    return [dict(zip(rows[0], r)) for r in rows[1:]]


def case_key(s):
    s = re.sub(r"^\[[^\]]*\]\s*", "", str(s).strip()); s = re.sub(r"\s+T\w*\d+$", "", s)
    return re.sub(r"\s+", " ", s).upper()


def parse_date(v):
    if isinstance(v, datetime):
        return v.date()
    if isinstance(v, date):
        return v
    s = str(v or "").strip()
    m = re.match(r"(\d{1,2})/(\d{1,2})/(\d{4})", s)
    if m:
        d, mo, y = map(int, m.groups())
    else:
        m = re.match(r"(\d{4})-(\d{2})-(\d{2})", s)
        if not m:
            return None
        y, mo, d = map(int, m.groups())
    try:
        return date(y, mo, d)
    except ValueError:
        return None


def parse_local(v):
    """Heure lisible et différente de 00:00 (00:00 = heure inconnue, comme dans l'étude 1)."""
    if isinstance(v, datetime):
        return None if (v.hour, v.minute) == (0, 0) else v.replace(tzinfo=None)
    m = re.match(r"(\d\d)/(\d\d)/(\d{4})\s+(\d{1,2})\s*[:hH]\s*(\d\d)", str(v or ""))
    if not m:
        return None
    d, mo, y, hh, mm = map(int, m.groups())
    if (hh, mm) == (0, 0):
        return None
    try:
        return datetime(y, mo, d, hh, mm)
    except ValueError:
        return None


def family(label, cls):
    n = norm(label)
    if cls in ("C", "D"):
        return "non_identifies_CD"
    if re.search(r"\biss\b|station spatiale", n):
        return "iss"
    if any(k in n for k in ("satellite", "starlink", "iridium")):
        return "satellites"
    if any(k in n for k in ("rentree", "bolide", "meteor", "lanceur", "fusee", "debris spatia", "etoile filante")):
        return "bolides_rentrees"
    if n.startswith("lanterne") or "lanterne" in n:
        return "lanternes"
    if (re.sub(r"\s*\(etoile\)$", "", n) in ("venus", "jupiter", "mars", "saturne", "mercure", "lune", "lune rousse", "etoile", "etoile brillante", "astronomique", "planete", "conjonction", "sirius", "capella", "arcturus", "vega", "rigel", "procyon", "betelgeuse")):
        return "astres"
    if "avion" in n or n in ("aeronef", "helicoptere") or n.startswith("escadrille") or n.startswith("drone"):
        return "avions_drones"
    if n.startswith("ballon") or n.startswith("montgolfi"):
        return "ballons"
    return "autres"


TEXT_FIELDS = ["Forme globale", "Trajectoire", "Nature de l'observation ou Termes descriptifs", "Caractéristique de l'observation",
               "Cadre de référence", "Nombre de phénomènes observés"]
MOTIFS = ["align", "en file", "a la file", "file indienne", "l'un derriere l'autre", "les uns derriere les autres", "queue leu leu", "train",
          "chapelet", "collier", "guirlande", "chaine", "en ligne", "ligne droite", "succession", "starlink"]


def motifs_of(t):
    s = " ".join(norm(t.get(k)) for k in TEXT_FIELDS)
    s = s.replace("’", "'")
    s = re.sub(r"en train d(e |')", " ", s)
    return [m for m in MOTIFS if m in s]


def multi(v):
    if isinstance(v, (int, float)):
        return v >= 3
    s = norm(v)
    m = re.search(r"\d+", s)
    if m:
        return int(m.group()) >= 3
    return any(k in s for k in ("plusieurs", "nombreu", "dizaine", "multiple", "beaucoup", "trois", "quatre", "cinq", "six", "sept", "huit", "dix"))


cases = read_xlsx(f"{GEIPAN}/export_cas_20261110.xlsx")
tems = read_xlsx(f"{GEIPAN}/export_temoignages_20261110.xlsx")
by_key = {case_key(c["Titre du Cas"]): c for c in cases}
stats = collections.Counter()

Y = collections.defaultdict(lambda: np.zeros(NDAY))
motif_count = collections.Counter()
for t in tems:
    loc = parse_local(t.get("Date et heure locale"))
    d = night_of_local(loc) if loc else parse_date(t.get("Date de l'observation")) or parse_date(t.get("Date et heure locale"))
    if d is None:
        stats["tem_sans_date"] += 1; continue
    if not (START <= d <= END):
        stats["tem_hors_periode"] += 1; continue
    stats["tem_periode"] += 1
    Y["tem_tous"][idx(d)] += 1
    c = by_key.get(case_key(t["Témoignage"]))
    dep = str(c["Code département"]).zfill(2) if c is not None and c["Code département"] is not None else ""
    if dep not in METRO:
        stats["tem_non_rattache_ou_hors_metropole"] += 1; continue
    stats["tem_metro"] += 1
    Y["H1_tem"][idx(d)] += 1
    ms = motifs_of(t)
    if ms:
        Y["H2_file"][idx(d)] += 1
        for m in ms:
            motif_count[m] += 1
    if multi(t.get("Nombre de phénomènes observés")):
        Y["tem_multi3"][idx(d)] += 1
    fam = family(c["Phénomène"], c["Classification"])
    if fam == "satellites":
        Y["tem_cas_satellites"][idx(d)] += 1

fam_count = collections.Counter()
for c in cases:
    d = parse_date(c["Date d'observation"])
    if d is None or not (START <= d <= END):
        continue
    dep = str(c["Code département"]).zfill(2) if c["Code département"] is not None else ""
    if dep not in METRO:
        continue
    fam = family(c["Phénomène"], c["Classification"])
    fam_count[fam] += 1
    Y["cas_tous"][idx(d)] += 1
    Y["cas_" + fam][idx(d)] += 1
Y["H3_cas_sat"] = Y["cas_satellites"]

# ----------------------------------------------------------------------------- covariables
days = [START + timedelta(i) for i in range(NDAY)]
ym = np.array([(d.year - 2019) * 12 + d.month - 1 for d in days])
dow = np.array([d.weekday() for d in days])


def dark_hours(d):
    """Durée de la nuit noire (Soleil sous −12°) à Paris, nuit du soir d (approximation de la déclinaison)."""
    n = (d + timedelta(days=1)).timetuple().tm_yday
    dec = math.radians(-23.44) * math.cos(2 * math.pi / 365 * (n + 10))
    phi, h0 = math.radians(48.85), math.radians(-12)
    x = (math.sin(h0) - math.sin(phi) * math.sin(dec)) / (math.cos(phi) * math.cos(dec))
    if x >= 1:
        return 0.0
    x = max(-1.0, x)
    return 24 - 2 * math.degrees(math.acos(x)) / 15


night = np.array([dark_hours(d) for d in days]); night_c = night - night.mean()


def window(trig_nights, lo, hi):
    """Indicateur : un déclencheur pendant les nuits d−hi … d−lo (J+lo à J+hi)."""
    e = np.zeros(NDAY)
    for n in trig_nights:
        for k in range(lo, hi + 1):
            i = idx(n) + k
            if 0 <= i < NDAY:
                e[i] += 1
    return e


ind = lambda e: (e > 0).astype(float)

# ----------------------------------------------------------------------------- Poisson à effets fixes
def fit(y, X, g, w=None, it=100):
    """Régression de Poisson pondérée à effets fixes année-mois (indice g), effets concentrés de façon exacte :
    alpha_m = log(sum w*y / sum w*exp(X beta)) sur le mois m ; Newton sur beta avec le hessien profilé. Renvoie (beta, mu)."""
    w = np.ones(len(y)) if w is None else w
    G = g.max() + 1
    beta = np.zeros(X.shape[1])
    Sy = np.bincount(g, w * y, minlength=G)
    for _ in range(it):
        ex = np.exp(np.clip(X @ beta, -30, 30))
        Se = np.bincount(g, w * ex, minlength=G)
        r = np.where(Se > 0, Sy / np.where(Se > 0, Se, 1), 0.0)
        mu = r[g] * ex
        Wm = w * mu
        grad = X.T @ (w * y - Wm)
        H = X.T @ (X * Wm[:, None])
        SW = np.bincount(g, Wm, minlength=G)
        M = np.zeros((G, X.shape[1]))
        np.add.at(M, g, X * Wm[:, None])
        ok = SW > 0
        H -= (M[ok] / SW[ok, None]).T @ M[ok]
        step = np.linalg.solve(H + 1e-10 * np.eye(len(beta)), grad)
        beta = beta + step
        if np.max(np.abs(step)) < 1e-10:
            break
    ex = np.exp(np.clip(X @ beta, -30, 30)); Se = np.bincount(g, w * ex, minlength=G)
    mu = np.where(Se > 0, Sy / np.where(Se > 0, Se, 1), 0.0)[g] * ex
    return beta, mu


_CACHE = {}


def design(y, expos, sample):
    """Colonnes : expositions, durée de nuit, jour de semaine (6) ; effets année-mois (mois avec au moins un signalement) via g."""
    key = (y.tobytes(), sample.tobytes())
    if key not in _CACHE:
        tot = collections.Counter()
        for i in np.where(sample)[0]:
            tot[ym[i]] += y[i]
        months = sorted(m for m in set(ym[sample]) if tot[m] > 0)
        s = sample & np.isin(ym, months)
        rows = np.where(s)[0]
        cols = [night_c] + [(dow == k).astype(float) for k in range(1, 7)]
        g = np.searchsorted(np.array(months), ym[rows])
        _CACHE[key] = (np.column_stack(cols)[rows], y[rows], rows, g)
    Xs, yy, rows, g = _CACHE[key]
    X = np.column_stack([e[rows] for e in expos] + [Xs]) if expos else Xs
    return X, yy, rows, g


def theta(y, expos, sample, w=None):
    X, yy, rows, g = design(y, expos, sample)
    b, _ = fit(yy, X, g, None if w is None else w[rows])
    return b[: len(expos)]


rng = np.random.default_rng(SEED)


def shifted(nights):
    out = []
    for n in nights:
        k = int(rng.integers(2, 9)) * (1 if rng.random() < .5 else -1)
        m = n + timedelta(weeks=k)
        if not (START <= m <= END):
            m = n - timedelta(weeks=k)
        out.append(m)
    return out


def boot_w(sample):
    months = np.unique(ym[sample])
    draw = rng.choice(months, size=len(months), replace=True)
    cnt = collections.Counter(draw.tolist())
    return np.array([cnt.get(m, 0) for m in ym], dtype=float)


def analyse(outcome, trig="starlink", lo=0, hi=7, sample=None, nperm=N_PRIM, nboot=N_PRIM, dose=False):
    y = Y[outcome]
    sample = np.ones(NDAY, bool) if sample is None else sample
    e = window(TRIG[trig], lo, hi); x = e if dose else ind(e)
    th = theta(y, [x], sample)[0]
    perm = []
    for _ in range(nperm):
        es = window(shifted(TRIG[trig]), lo, hi)
        perm.append(theta(y, [es if dose else ind(es)], sample)[0])
    boots = [theta(y, [x], sample, boot_w(sample))[0] for _ in range(nboot)]
    lo95, hi95 = np.percentile(boots, [2.5, 97.5])
    p = (1 + sum(1 for v in perm if v >= th)) / (1 + nperm)
    expo = sample & (x > 0)
    return {"outcome": outcome, "trigger": trig, "window": [lo, hi], "dose": dose, "n_events": int(y[sample].sum()),
            "n_nights_exposed": int(expo.sum()), "n_nights_unexposed": int((sample & (x == 0)).sum()),
            "events_exposed": int(y[expo].sum()), "events_unexposed": int(y[sample & (x == 0)].sum()),
            "rr": float(math.exp(th)), "ci95": [float(math.exp(lo95)), float(math.exp(hi95))], "p_one_sided": float(p),
            "perm_rr_median": float(math.exp(np.median(perm))), "perm_rr_q95": float(math.exp(np.percentile(perm, 95))),
            "nperm": nperm, "nboot": nboot}


def profile(outcome, trig="starlink", sample=None, nboot=N_EXPL):
    wins = [(-7, -1), (0, 1), (2, 3), (4, 7), (8, 14)]
    y = Y[outcome]; sample = np.ones(NDAY, bool) if sample is None else sample
    xs = [ind(window(TRIG[trig], a, b)) for a, b in wins]
    th = theta(y, xs, sample)
    bs = np.array([theta(y, xs, sample, boot_w(sample)) for _ in range(nboot)])
    return {"outcome": outcome, "trigger": trig, "windows": wins, "rr": [float(math.exp(v)) for v in th],
            "ci95": [[float(math.exp(np.percentile(bs[:, j], 2.5))), float(math.exp(np.percentile(bs[:, j], 97.5)))] for j in range(len(wins))]}


def placebo(outcome, sample=None, nboot=N_PRIM):
    y = Y[outcome]; sample = np.ones(NDAY, bool) if sample is None else sample
    xs = [ind(window(TRIG["starlink"], 0, 7)), ind(window(TRIG["starlink"], -7, -1))]
    th = theta(y, xs, sample)
    bs = np.array([theta(y, xs, sample, boot_w(sample)) for _ in range(nboot)])
    return {"outcome": outcome, "rr_post": float(math.exp(th[0])), "rr_pre": float(math.exp(th[1])),
            "ci95_post": [float(math.exp(v)) for v in np.percentile(bs[:, 0], [2.5, 97.5])],
            "ci95_pre": [float(math.exp(v)) for v in np.percentile(bs[:, 1], [2.5, 97.5])]}


def holm(ps):
    order = sorted(range(len(ps)), key=lambda i: ps[i]); adj = [0.0] * len(ps); run = 0.0
    for r, i in enumerate(order):
        run = max(run, min(1.0, (len(ps) - r) * ps[i])); adj[i] = run
    return adj


def verdict(r, thr):
    if r["ci95"][1] < 1:
        return "contraire"
    if r["p_holm"] < 0.05 and r["rr"] >= thr:
        return "soutenue"
    return "non démontrée"


# ----------------------------------------------------------------------------- calculs
out = {"meta": {"protocole_sha256": "df9e63cf3b3abb97489c0f1edf877a388b5a3efaaea530d7919e1e05e0704377", "seed": SEED, "periode": [START.isoformat(), END.isoformat()],
                "mention": MENTION, "sources": {"geipan": "exports cas et témoignages du 10/11/2026 (copie privée)",
                                                 "gcat": "J. McDowell, General Catalog of Artificial Space Objects, planet4589.org, CC BY (launch.tsv et satcat.tsv du 15/11/2026)"},
                "stats": dict(stats), "n_starlink": len([n for n in TRIG["starlink"] if START <= n <= END]),
                "n_triggers": {k: len([n for n in v if START <= n <= END]) for k, v in TRIG.items()}}}
sys.stderr.write(f"{dict(stats)}\n")

prim = {"H1": analyse("H1_tem"), "H2": analyse("H2_file"), "H3": analyse("H3_cas_sat")}
adj = holm([prim[k]["p_one_sided"] for k in ("H1", "H2", "H3")])
for k, a, thr in zip(("H1", "H2", "H3"), adj, (1.10, 1.25, 1.50)):
    prim[k]["p_holm"] = a; prim[k]["seuil_rr"] = thr; prim[k]["verdict"] = verdict(prim[k], thr)
out["principales"] = prim
sys.stderr.write("principales ok\n")

y1922 = np.array([d.year <= 2022 for d in days])
out["secondaires"] = {
    "2019_2022": {k: analyse(o, sample=y1922, nperm=N_EXPL, nboot=N_EXPL) for k, o in (("H1", "H1_tem"), ("H2", "H2_file"), ("H3", "H3_cas_sat"))},
    "dose": {k: analyse(o, dose=True, nperm=N_EXPL, nboot=N_EXPL) for k, o in (("H1", "H1_tem"), ("H2", "H2_file"), ("H3", "H3_cas_sat"))},
    "tem_tous": analyse("tem_tous", nperm=N_EXPL, nboot=N_EXPL),
    "placebo": {k: placebo(o) for k, o in (("H1", "H1_tem"), ("H2", "H2_file"), ("H3", "H3_cas_sat"))},
}
sys.stderr.write("secondaires ok\n")

expl = {"profil": {k: profile(o) for k, o in (("H1", "H1_tem"), ("H2", "H2_file"), ("H3", "H3_cas_sat"))},
        "familles": {f: analyse("cas_" + f, nperm=N_EXPL, nboot=N_EXPL) for f in sorted(fam_count) if fam_count[f] >= 20},
        "tem_multi3": analyse("tem_multi3", nperm=N_EXPL, nboot=N_EXPL),
        "autres_declencheurs": {t: analyse("H1_tem", trig=t, nperm=N_EXPL, nboot=N_EXPL) for t in ("orbital_hors_starlink", "kourou", "falcon9_hors_starlink")},
        "rentrees": {o: analyse(o, trig="rentree_2t", lo=0, hi=1, nperm=N_EXPL, nboot=N_EXPL) for o in ("H1_tem", "cas_bolides_rentrees")}}
out["exploratoires"] = expl
sys.stderr.write("exploratoires ok\n")

# ----------------------------------------------------------------------------- descriptifs (agrégats)
desc = {"motifs": dict(motif_count), "familles_cas": dict(fam_count), "totaux": {k: int(v.sum()) for k, v in Y.items()}}
e0 = ind(window(TRIG["starlink"], 0, 7))
desc["par_annee"] = {}
for yv in range(2019, 2027):
    s = np.array([d.year == yv for d in days])
    desc["par_annee"][yv] = {"nuits": int(s.sum()), "nuits_exposees": int((s & (e0 > 0)).sum()),
                             "starlink": sum(1 for n in TRIG["starlink"] if n.year == yv and n <= END),
                             **{k: int(Y[k][s].sum()) for k in ("H1_tem", "H2_file", "H3_cas_sat", "cas_tous")}}
desc["par_mois"] = []
for m in range(ym.max() + 1):
    s = ym == m
    d0 = days[int(np.where(s)[0][0])]
    desc["par_mois"].append({"mois": f"{d0.year}-{d0.month:02d}", "H1_tem": int(Y["H1_tem"][s].sum()), "H2_file": int(Y["H2_file"][s].sum()),
                             "H3_cas_sat": int(Y["H3_cas_sat"][s].sum()), "starlink": sum(1 for n in TRIG["starlink"] if (n.year - 2019) * 12 + n.month - 1 == m)})
# observé / attendu selon le nombre de nuits depuis le dernier lancement (modèle sans exposition)
last = np.full(NDAY, 10 ** 6)
for n in TRIG["starlink"]:
    for k in range(0, 60):
        i = idx(n) + k
        if 0 <= i < NDAY:
            last[i] = min(last[i], k)
bins = [(0, 0), (1, 1), (2, 2), (3, 3), (4, 5), (6, 7), (8, 14), (15, 10 ** 6)]
desc["depuis_dernier_lancement"] = {}
for o in ("H1_tem", "H2_file", "H3_cas_sat"):
    X, yy, rows, g = design(Y[o], [], np.ones(NDAY, bool)); b, mu = fit(yy, X, g)
    res = []
    for a, bb in bins:
        sel = (last[rows] >= a) & (last[rows] <= bb)
        O, E = float(yy[sel].sum()), float(mu[sel].sum())
        res.append({"de": a, "a": bb if bb < 10 ** 6 else None, "nuits": int(sel.sum()), "observe": int(O), "attendu": round(E, 2),
                    "ratio": round(O / E, 3) if E > 0 else None,
                    "ic95": [round(max(0.0, (math.sqrt(O) - 0.98) ** 2) / E, 3) if E > 0 and O > 0 else 0.0, round(((math.sqrt(O + 1) + 0.98) ** 2) / E, 3) if E > 0 else None]})
    desc["depuis_dernier_lancement"][o] = res
out["descriptifs"] = desc
json.dump(out, sys.stdout, ensure_ascii=False, indent=1, default=str)
