{"id":"W2139516750","doi":"10.1145/2009916.2010039","title":"Repeatable and reliable search system evaluation using crowdsourcing","year":2011,"lang":"en","type":"article","venue":"","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":99,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Crowdsourcing; Computer science; Information retrieval; Data science; World Wide Web","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002086399,0.0001319189,0.0001601009,0.0001354162,0.0003707795,0.0002518532,0.0002783989,0.00006560865,0.00002237992],"category_scores_gemma":[0.00003468441,0.0001203853,0.00003289414,0.0003321987,0.00003893387,0.0005707896,0.0002502638,0.0001183311,0.00002300151],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0001197718,"about_ca_system_score_gemma":0.0001016408,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.001310932,"about_ca_topic_score_gemma":0.00000662361,"domain_scores_codex":[0.998388,0.00012171,0.0002419681,0.0004843432,0.0004024863,0.0003615234],"domain_scores_gemma":[0.9988931,0.00003764806,0.00006340525,0.0006707829,0.000214349,0.000120649],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"theoretical_or_conceptual","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0001145254,0.0004304968,0.0633124,0.001583098,0.0003135404,0.0003560808,0.05891041,0.0324355,0.136145,0.4995973,0.002216374,0.2045853],"study_design_scores_gemma":[0.0002700615,0.00003684668,0.001049862,0.0001400758,0.00002132964,0.0002122827,0.0009607874,0.973403,0.0233032,0.0003205424,0.00008433367,0.0001977161],"study_design_candidate":"simulation_or_modeling","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.5898843,0.0002311813,0.3601595,0.00002106563,0.0002406266,0.0002293218,2.275757e-7,0.0003427578,0.04889098],"genre_scores_gemma":[0.8978878,0.00000213775,0.1017106,0.00003688812,0.00004208704,0.000005050994,4.799237e-7,0.00001217786,0.0003028053],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.9409674,"threshold_uncertainty_score":0.4909169,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.08907501364983587,"score_gpt":0.2649148142627285,"score_spread":0.1758398006128926,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}