"""Head-to-head numbers for each pair of engines, and each engine against the merged list, from the public panel.

Run load.py from ../research/search-engine-agreement/ first, with JEVLIN_PANEL_DIR pointing at a download of the
JevlinAI/supersearch-tests dataset. Writes data.json beside this file."""
import pickle, sys, os, json, itertools, statistics
HERE = os.path.dirname(os.path.abspath(__file__))
D = os.environ.get('JEVLIN_PANEL_DIR', os.path.join(HERE, 'panel'))
sys.path.insert(0, D)
import score
items, calls = pickle.load(open(os.path.join(D, 'panel.pkl'), 'rb'))
E = ['exa', 'brave', 'parallel-search', 'perplexity-search']
SETS = ['devdex', 'obcode', 'seal']
def found(a, i, t):
    r = calls[a].get((i, 1, t))
    return None if r is None else bool(score.match_rank(score.dedupe([u for u in r['urls'] if u]), items[i], 'exact'))
def per_item(arms, ids=None):
    """Share of trials in which the answer page is in the top ten of any of `arms`, per question."""
    per = {}
    for i in (ids or items):
        v = []
        for t in (1, 2, 3):
            f = [found(a, i, t) for a in arms]
            if None in f: continue
            v.append(1.0 if any(f) else 0.0)
        if v: per[i] = statistics.mean(v)
    return per
def pct(per): return round(100 * statistics.mean(per.values()), 1)
solo = {a: per_item([a]) for a in E + ['balanced']}
out = {'source': 'https://huggingface.co/datasets/JevlinAI/supersearch-tests', 'collected': '2026-10-27', 'licence': 'CC-BY-4.0',
       'questions': len(items), 'alone': {a: pct(solo[a]) for a in solo}, 'pairs': {}, 'vs_merged': {}}
for a, b in itertools.combinations(E, 2):
    both = per_item([a, b]); d = score.boot_diff(solo[a], solo[b])
    always_a = sum(1 for i in solo[a] if solo[a][i] == 1 and solo[b].get(i) == 0)
    always_b = sum(1 for i in solo[b] if solo[b][i] == 1 and solo[a].get(i) == 0)
    row = {'a': a, 'b': b, 'a_alone': pct(solo[a]), 'b_alone': pct(solo[b]), 'either': pct(both), 'a_minus_b': d,
           'a_every_run_b_never': always_a, 'b_every_run_a_never': always_b,
           'gain_over_a': score.boot_diff(both, solo[a]), 'gain_over_b': score.boot_diff(both, solo[b]), 'by_set': {}}
    for s in SETS:
        ids = [i for i, it in items.items() if it['set'] == s]
        pa = per_item([a], ids); pb = per_item([b], ids)
        row['by_set'][s] = {'n': len(ids), 'a': pct(pa), 'b': pct(pb), 'either': pct(per_item([a, b], ids)), 'a_minus_b': score.boot_diff(pa, pb)}
    out['pairs'][a + '|' + b] = row
for a in E:
    d = score.boot_diff(solo['balanced'], solo[a]); row = {'merged': pct(solo['balanced']), 'alone': pct(solo[a]), 'merged_minus_alone': d, 'by_set': {}}
    for s in SETS:
        ids = [i for i, it in items.items() if it['set'] == s]
        pm = per_item(['balanced'], ids); pa = per_item([a], ids)
        row['by_set'][s] = {'n': len(ids), 'merged': pct(pm), 'alone': pct(pa), 'merged_minus_alone': score.boot_diff(pm, pa)}
    out['vs_merged'][a] = row
json.dump(out, open(os.path.join(HERE, 'data.json'), 'w'), indent=1)
print(json.dumps(out)[:200])
