{
 "source": "https://huggingface.co/datasets/JevlinAI/supersearch-tests",
 "collected": "2026-10-27",
 "analysed": "2026-11-10",
 "licence": "CC-BY-4.0",
 "router": {
  "questions": 485,
  "engines": {
   "exa": 57.5,
   "brave": 49.1,
   "parallel-search": 45.1,
   "perplexity-search": 44.5
  },
  "fixed_engine": "exa",
  "by_set_picks": {
   "seal": "exa",
   "devdex": "exa",
   "obcode": "exa"
  },
  "by_set_in_sample_best": {
   "devdex": {
    "exa": 63.5,
    "brave": 52.7,
    "parallel-search": 61.9,
    "perplexity-search": 54.4
   },
   "obcode": {
    "exa": 50.0,
    "brave": 18.9,
    "parallel-search": 18.9,
    "perplexity-search": 36.7
   },
   "seal": {
    "exa": 53.5,
    "brave": 49.9,
    "parallel-search": 34.9,
    "perplexity-search": 37.5
   }
  },
  "learned_picks_pct": {
   "exa": 65.8,
   "parallel-search": 16.8,
   "brave": 15.6,
   "perplexity-search": 1.8
  },
  "hit_at_10": {
   "fixed": 57.5,
   "by_set": 56.4,
   "learned": 57.1,
   "fast": 60.3,
   "balanced": 62.1
  },
  "vs_balanced": {
   "fixed": {
    "diff": 4.6,
    "ci": [
     1.4,
     7.7
    ]
   },
   "by_set": {
    "diff": 5.6,
    "ci": [
     2.8,
     8.5
    ]
   },
   "learned": {
    "diff": 5.0,
    "ci": [
     2.0,
     7.9
    ]
   }
  },
  "vs_fast": {
   "fixed": {
    "diff": 2.9,
    "ci": [
     -0.5,
     6.3
    ]
   },
   "by_set": {
    "diff": 3.9,
    "ci": [
     0.7,
     7.1
    ]
   },
   "learned": {
    "diff": 3.3,
    "ci": [
     0.1,
     6.5
    ]
   }
  },
  "when_fixed_misses": {
   "questions": 206,
   "share_pct": 42.5,
   "another_engine_had_it_pct": 38.3
  },
  "best_engine_share_pct": {
   "exa": 31.8,
   "brave": 25.2,
   "parallel-search": 22.3,
   "perplexity-search": 20.7
  },
  "only_one_engine_pct": 17.7
 },
 "per_arm": {
  "exa": {
   "calls": 4365,
   "empty_results_pct": 0.07,
   "cost_per_search": 0.007,
   "hit10": 57.5,
   "cost_per_1000_hits": 12.18
  },
  "exa-lite": {
   "calls": 4365,
   "empty_results_pct": 0.0,
   "cost_per_search": 0.004,
   "hit10": 49.7,
   "cost_per_1000_hits": 8.05
  },
  "brave": {
   "calls": 4365,
   "empty_results_pct": 0.09,
   "cost_per_search": 0.005,
   "hit10": 49.1,
   "cost_per_1000_hits": 10.17
  },
  "parallel-search": {
   "calls": 4365,
   "empty_results_pct": 1.44,
   "cost_per_search": 0.0039,
   "hit10": 45.1,
   "cost_per_1000_hits": 8.74
  },
  "perplexity-search": {
   "calls": 4365,
   "empty_results_pct": 0.44,
   "cost_per_search": 0.005,
   "hit10": 44.5,
   "cost_per_1000_hits": 11.19
  },
  "fast": {
   "calls": 4365,
   "empty_results_pct": 0.0,
   "cost_per_search": 0.014,
   "hit10": 60.3,
   "cost_per_1000_hits": 23.17
  },
  "balanced": {
   "calls": 4365,
   "empty_results_pct": 0.0,
   "cost_per_search": 0.0209,
   "hit10": 62.1,
   "cost_per_1000_hits": 33.66
  }
 },
 "midterms": {
  "source": "https://jevlin.ai/midterms",
  "races_called": 131,
  "median_minutes_all_engines": 120,
  "median_minutes_best_single_engine": 140
 }
}