{"id":"W2139516750","doi":"10.1145/2009916.2010039","title":"Repeatable and reliable search system evaluation using crowdsourcing","year":2011,"lang":"en","type":"article","venue":"","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":99,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Crowdsourcing; Computer science; Information retrieval; Data science; World Wide Web","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.02372633,0.001889113,0.002087147,0.002848938,0.00182452,0.002588089,0.002222701,0.001892387,0.001422053],"category_scores_gemma":[0.05076407,0.000664027,0.0007741163,0.001795933,0.001544517,0.002801219,0.00302788,0.001381414,0.001158487],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002475567,"about_ca_system_score_gemma":0.002142604,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.007869883,"about_ca_topic_score_gemma":0.007547634,"domain_scores_codex":[0.9682514,0.01937322,0.001383181,0.002721047,0.007280934,0.0009902428],"domain_scores_gemma":[0.9328423,0.03885939,0.004470272,0.01144179,0.01061308,0.001773149],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.004781243,0.0033244,0.03823912,0.002238881,0.001641168,0.001649129,0.007807753,0.3158557,0.1065458,0.00913634,0.01839285,0.4903877],"study_design_scores_gemma":[0.001219732,0.004422674,0.02903271,0.0001804726,0.0003275778,0.0004610501,0.003492448,0.8450109,0.07410187,0.02067853,0.0204919,0.0005800893],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.6920934,0.0008773602,0.2822394,0.0009885819,0.0002452295,0.003964415,0.001210997,0.004151036,0.01422952],"genre_scores_gemma":[0.9194158,0.0001280241,0.07575525,0.0001518215,0.00005640556,0.001163582,0.0007538158,0.0002591109,0.002316075],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.02372633,"threshold_uncertainty_score":0.1254783,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.08907501364983587,"score_gpt":0.2649148142627285,"score_spread":0.1758398006128926,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}