{"id":"W4408184531","doi":"10.1145/3722449.3722461","title":"Report on the 1st Workshop on Large Language Model for Evaluation in Information Retrieval (LLM4Eval 2024) at SIGIR 2024","year":2024,"lang":"en","type":"article","venue":"ACM SIGIR Forum","topic":"Topic Modeling","field":"Computer Science","cited_by":7,"is_retracted":false,"has_abstract":true,"ca_institutions":"Microsoft (Canada); University of Waterloo","funders":"","keywords":"Information retrieval; Computer science; Natural language processing","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":[],"consensus_categories":[],"category_scores_codex":[0.002451456,0.0001979114,0.0001583721,0.0002376275,0.0001716763,0.000324136,0.0008161808,0.0001391265,0.00005865868],"category_scores_gemma":[0.00153213,0.0001482784,0.0001181783,0.0005147529,0.00001617919,0.0009011472,0.0004601538,0.0003314976,0.0001772923],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0004080797,"about_ca_system_score_gemma":0.000184448,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.000009309277,"about_ca_topic_score_gemma":0.00006822846,"domain_scores_codex":[0.997679,0.00006596007,0.0004992834,0.000461923,0.0008224896,0.0004713622],"domain_scores_gemma":[0.9978515,0.000578967,0.0001122862,0.001298232,0.0001010567,0.00005795115],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.0004651501,0.0002306312,0.0005535598,0.0002643599,0.0001279125,0.0001618611,0.02096893,0.2780218,0.0007723705,0.2647797,0.1971984,0.2364554],"study_design_scores_gemma":[0.0004014088,0.00005445247,0.00008077519,0.0001292897,0.00001314287,0.00001429546,0.0002249558,0.9779367,0.0004574696,0.01247832,0.008037632,0.0001714845],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2911738,0.0004946701,0.665692,0.02652935,0.004063607,0.00224593,0.00005552042,0.0003760726,0.00936895],"genre_scores_gemma":[0.9882663,0.00001290164,0.003074609,0.001754623,0.0001599423,0.000140704,0.00006842385,0.00001994597,0.006502475],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.6999151,"threshold_uncertainty_score":0.6046617,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.03542714885077752,"score_gpt":0.3152205595817855,"score_spread":0.279793410731008,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}