{"id":"W6891792210","doi":"10.48448/4nmw-gv06","title":"BenchIE^FL: A Manually Re-Annotated Fact-Based Open Information Extraction Benchmark","year":2024,"lang":"en","type":"other","venue":"Underline Science Inc.","topic":"","field":"","cited_by":0,"is_retracted":false,"has_abstract":true,"ca_institutions":"Université de Montréal","funders":"","keywords":"Benchmark (surveying); Field (mathematics); Information extraction; Natural language; Open source; Natural (archaeology)","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow","scholarly_communication","insufficient_payload"],"consensus_categories":["insufficient_payload"],"category_scores_codex":[0.002346516,0.0008012837,0.0006312786,0.003836457,0.0003793458,0.003051238,0.003183713,0.000521298,0.006865557],"category_scores_gemma":[0.0004645869,0.0007374377,0.0001335353,0.004343824,0.001136566,0.004067392,0.0008162783,0.001004569,0.04338586],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.001211065,"about_ca_system_score_gemma":0.00293373,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.002811285,"about_ca_topic_score_gemma":0.003188205,"domain_scores_codex":[0.9941841,0.0001275602,0.001035449,0.001260665,0.002349693,0.001042584],"domain_scores_gemma":[0.9965172,0.00008517162,0.001121441,0.001402072,0.0004652056,0.0004088625],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"not_applicable","study_design_gemma":"not_applicable","study_design_scores_codex":[0.00005704669,0.0001510807,0.00001863463,0.0001901971,0.00005781346,0.00002576026,0.0002607313,0.0005165821,0.001803489,0.001641526,0.984555,0.01072214],"study_design_scores_gemma":[0.000867462,0.0002210133,0.0001655068,0.0008474205,0.0001422745,0.00002413792,0.0004881385,0.03672794,0.0005974004,0.001082039,0.9577293,0.00110732],"study_design_candidate":"not_applicable","study_design_consensus":"not_applicable","genre_codex":"other","genre_gemma":"other","genre_scores_codex":[0.0001558678,0.0001867815,0.00394615,0.000730985,0.002265753,0.002584374,0.001320098,0.001654416,0.9871556],"genre_scores_gemma":[0.06458326,0.00006587167,0.05782215,0.002553723,0.001082793,0.0005275855,0.008932881,0.002798013,0.8616337],"genre_candidate":"other","genre_consensus":"other","teacher_disagreement_score":0.1255219,"threshold_uncertainty_score":0.9995077,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.02999519996806134,"score_gpt":0.3591961353819997,"score_spread":0.3292009354139384,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}