{"id":"W3132704983","doi":"10.48550/arxiv.2102.12887","title":"Significant Improvements over the State of the Art? A Case Study of the MS MARCO Document Ranking Leaderboard","year":2021,"lang":"en","type":"preprint","venue":"arXiv (Cornell University)","topic":"Topic Modeling","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"ca_institutions":"Microsoft (Canada); University of Waterloo","funders":"","keywords":"Ranking (information retrieval); Metric (unit); Context (archaeology); Computer science; Artificial intelligence; State (computer science); Information retrieval; Order (exchange); sort; Ask price; Machine learning; Learning to rank; Algorithm; Engineering; History; Operations management; Economics","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0005706242,0.0002587773,0.0003244072,0.00007663656,0.000225223,0.0001080503,0.002979952,0.0000745376,0.00001257347],"category_scores_gemma":[0.00002555534,0.0001590052,0.0002895796,0.0005440884,0.0001473238,0.0001749635,0.005946186,0.0005962484,0.000001401518],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001588907,"about_ca_system_score_gemma":0.0002633076,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.003383025,"about_ca_topic_score_gemma":0.0009066789,"domain_scores_codex":[0.9976816,0.0005182683,0.0003883928,0.0008048532,0.0003215479,0.0002854026],"domain_scores_gemma":[0.9962152,0.0001268106,0.0006172596,0.002826748,0.0001644888,0.00004952683],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00005077591,0.0009068701,0.02453111,0.0003001926,0.0009655035,0.001731992,0.02889963,0.9287634,0.001306959,0.008078063,0.00009025696,0.0043752],"study_design_scores_gemma":[0.004517675,0.0003174582,0.02253013,0.000829329,0.0007965871,0.00007766478,0.019688,0.9215991,0.004652583,0.02360988,0.0001894115,0.001192117],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.9293134,0.00002623746,0.06858675,0.000114742,0.0006973341,0.001032781,0.000005268244,0.00002370707,0.0001998153],"genre_scores_gemma":[0.9988024,0.00001758249,0.0001437963,0.00008360331,0.00002197631,0.000003005038,4.224683e-7,0.00001184065,0.0009153892],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.06948902,"threshold_uncertainty_score":0.7411495,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.06168247512120693,"score_gpt":0.2001469899934812,"score_spread":0.1384645148722742,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}