{"id":"W2041889759","doi":"10.1145/2641383.2641388","title":"Report on the CIKM workshop on living labs for information retrieval evaluation","year":2014,"lang":"en","type":"article","venue":"ACM SIGIR Forum","topic":"Personal Information Management and User Behavior","field":"Decision Sciences","cited_by":12,"is_retracted":false,"has_abstract":true,"ca_institutions":"University of Waterloo","funders":"","keywords":"Computer science; Session (web analytics); Scope (computer science); Information retrieval; World Wide Web; Data science","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"metacan-v3-hybrid-931329e0061c","candidate_categories":["metaresearch"],"consensus_categories":[],"category_scores_codex":[0.1313835,0.002041529,0.001885254,0.004075236,0.004207225,0.01539738,0.005870022,0.005347911,0.05353907],"category_scores_gemma":[0.1070019,0.001278042,0.002118471,0.003341681,0.002818521,0.01354732,0.01187131,0.008423231,0.02689306],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.006950194,"about_ca_system_score_gemma":0.01677747,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.01689512,"about_ca_topic_score_gemma":0.01919841,"domain_scores_codex":[0.9073242,0.05319436,0.003617652,0.005180269,0.02706976,0.00361369],"domain_scores_gemma":[0.8510875,0.04540773,0.003398457,0.01353335,0.0691098,0.01746322],"domain_codex":null,"domain_gemma":"evaluation","domain_candidate":"evaluation","domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.0005704341,0.00097708,0.001215835,0.0005287959,0.00007799309,0.0002237547,0.002314019,0.0009266277,0.00433672,0.01138062,0.7343128,0.2431353],"study_design_scores_gemma":[0.000347029,0.0006803913,0.004403114,0.0009251768,0.00008697081,0.0002369777,0.002547047,0.003090754,0.006174352,0.011988,0.9692771,0.0002430744],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"methods","genre_gemma":"other","genre_scores_codex":[0.0416996,0.0309155,0.3738366,0.2096103,0.04320622,0.02024086,0.01512725,0.01216605,0.2531976],"genre_scores_gemma":[0.1740914,0.01330113,0.3260956,0.03320405,0.009957948,0.02313273,0.0401485,0.009502309,0.3705663],"genre_candidate":"other","genre_consensus":null,"teacher_disagreement_score":0.8686165,"threshold_uncertainty_score":0.6948307,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.2454101908581725,"score_gpt":0.4366114750163633,"score_spread":0.1912012841581909,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}