{"id":"W2946233956","doi":"10.1109/tse.2019.2918520","title":"Characterizing Crowds to Better Optimize Worker Recommendation in Crowdsourced Testing","year":2019,"lang":"en","type":"article","venue":"IEEE Transactions on Software Engineering","topic":"Software Engineering Research","field":"Computer Science","cited_by":41,"is_retracted":false,"has_abstract":true,"ca_institutions":"York University","funders":"National Key Research and Development Program of China; China Scholarship Council; National Natural Science Foundation of China","keywords":"Computer science; Crowds; Crowdsourcing; Task (project management); Context (archaeology); Software bug; Relevance (law); Test (biology); Machine learning; Software; Data science; Computer security; World Wide Web; Engineering","routes":{"ca_aff":true,"ca_fund":false,"ca_venue":false,"about_ca":false,"invisible_to_affiliation_only":false},"retraction":null,"screen":null,"direct_labels":[],"prediction":{"model_version":"codex-gemma-dda1882f352a","candidate_categories":["metaepi_narrow"],"consensus_categories":[],"category_scores_codex":[0.0004350434,0.0003115529,0.0002901903,0.0008795836,0.00007900145,0.0002248641,0.0006213834,0.0001335935,0.0001013054],"category_scores_gemma":[0.0002710604,0.0003635026,0.00008877985,0.001700357,0.000007510534,0.0006911882,0.00001635856,0.000692527,0.000361888],"about_ca_system_candidate":false,"about_ca_system_consensus":false,"about_ca_system_score_codex":0.0003180434,"about_ca_system_score_gemma":0.00004422693,"about_ca_topic_candidate":false,"about_ca_topic_consensus":false,"about_ca_topic_score_codex":0.00002282099,"about_ca_topic_score_gemma":0.00000173979,"domain_scores_codex":[0.9979115,0.00003673132,0.0003892443,0.0006423721,0.0003439772,0.000676211],"domain_scores_gemma":[0.9977195,0.001313503,0.00004176852,0.0006379824,0.00008392373,0.0002032855],"domain_codex":null,"domain_gemma":null,"domain_candidate":null,"domain_consensus":null,"study_design_codex":"simulation_or_modeling","study_design_gemma":"simulation_or_modeling","study_design_scores_codex":[0.00001702829,0.00006637619,0.00515839,0.000055991,0.00002727543,0.00001663612,0.0005635748,0.9194407,0.01202987,0.000007105806,0.00003510345,0.0625819],"study_design_scores_gemma":[0.002186112,0.0004151718,0.1208546,0.001207101,0.00001535975,0.00007445904,0.00004010432,0.807573,0.06255288,0.00001137307,0.003101713,0.001968092],"study_design_candidate":"simulation_or_modeling","study_design_consensus":"simulation_or_modeling","genre_codex":"methods","genre_gemma":"empirical","genre_scores_codex":[0.2874259,0.000005048821,0.7101633,0.0002042361,0.0009677418,0.0003144034,0.000003979471,0.0008981472,0.00001723158],"genre_scores_gemma":[0.721045,0.000001478677,0.2783916,0.0002329775,0.0000503746,0.0001200506,0.000002056159,0.0000608327,0.00009560864],"genre_candidate":"empirical","genre_consensus":null,"teacher_disagreement_score":0.4336191,"threshold_uncertainty_score":0.9998817,"prediction_status":"machine_predicted_unvalidated"},"machine_scores":{"provisional":true,"baseline":true,"maturity_gate_passed":false,"score_opus":0.01883341979320734,"score_gpt":0.2401437436804875,"score_spread":0.2213103238872801,"validation_status":"score_only:v0-immature-baseline","note":"Baseline scores from an immature model (maturity gate not passed). Scores rank; they never assert a category."}}