{"id":"W2200523128","doi":"","title":"Evaluating Information Retrieval Systems With Multiple Non-Expert Assessors","year":2013,"lang":"en","type":"dissertation","venue":"UWSpace (University of Waterloo)","topic":"Mobile Crowdsensing and Crowdsourcing","field":"Computer Science","cited_by":5,"is_retracted":false,"has_abstract":true,"ca_institutions":"","funders":"University of Waterloo","keywords":"Ranking (information retrieval); Relevance (law); Computer science; Information retrieval; Key (lock); Expert system; Test (biology); Machine learning; Data mining; Artificial intelligence","routes":{"ca_aff":false,"ca_fund":true,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":true},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.0380567,0.001403254,0.002197949,0.002291781,0.001677627,0.003463869,0.002298962,0.002325211,0.00245583],"category_scores_gemma":[0.1015684,0.0007123091,0.001079069,0.001851137,0.001308387,0.004939539,0.003051366,0.002048422,0.001233121],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.002335954,"about_ca_system_score_gemma":0.002048928,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.005511354,"about_ca_topic_score_gemma":0.008071076,"domain_scores_codex":[0.9733071,0.01612108,0.002031599,0.003185268,0.004525573,0.0008293636],"domain_scores_gemma":[0.9068892,0.06590614,0.005744969,0.007440171,0.01210423,0.001915272],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"design_other","study_design_gemma":"observational","study_design_scores_codex":[0.004206687,0.001844067,0.07551784,0.001531899,0.001512969,0.0002998808,0.002601134,0.2379109,0.02489161,0.007298377,0.006342122,0.6360425],"study_design_scores_gemma":[0.0006359618,0.003272426,0.02543607,0.0001211726,0.0004200856,0.0002456603,0.001449536,0.9252684,0.02813099,0.01126877,0.003567485,0.000183532],"study_design_candidate":"observational","study_design_consensus":null,"genre_codex":"empirical","genre_gemma":"empirical","genre_scores_codex":[0.68705,0.001876961,0.296456,0.000999718,0.0001555964,0.001183126,0.0002386417,0.001316536,0.01072333],"genre_scores_gemma":[0.8054007,0.000270299,0.1907053,0.0002672611,0.0001024011,0.0003707967,0.0004471583,0.0000893148,0.002346718],"genre_candidate":"empirical","genre_consensus":"empirical","teacher_disagreement_score":0.0380567,"threshold_uncertainty_score":0.2012655,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01371373316032038,"score_gpt":0.2263860097607907,"score_spread":0.2126722766004703,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}