{"doi":"10.1101/2025.10.16.25337917","title":"Combining Clinician Expertise with Prompt Engineering enhances Small Language Models Reliability for Cancer Entity Recognition in Electronic Health Records","abstract":"Real-world data (RWD), largely stored in unstructured electronic health records (EHRs), are critical for understanding complex diseases like cancer. However, extracting structured information from these narratives is challenging due to linguistic variability, semantic complexity, and privacy concerns. This study evaluates the performance of four locally deployable and small language models (SLMs), LLaMA, Mistral, BioMistral, and MedLLaMA, for information extraction (IE) from Italian EHRs within the APOLLO 11 trial on non-small cell lung cancer (NSCLC). We examined three prompting strategies (zero-shot, few-shot, and annotated few-shot) across English and Italian, involving clinicians with varying expertise to assess prompt design's impact on accuracy. Results show that general-purpose models (e.g., LLaMA 3.1 8B) outperform biomedical models in most tasks, particularly in extracting binary features. Multiclass variables such as TNM staging, PD-L1, and ECOG were more difficult due to implicit language and lack of standardization. Few-shot prompting and native-language inputs significantly improved performance and reduced hallucinations. Clinical expertise enhanced consistency in annotation, particularly among students using annotated examples. The study confirms that privacy-preserving SLMs can be deployed locally for efficient and secure cancer data extraction. Findings highlight the need for hybrid systems combining SLMs with expert input and underline the importance of aligning clinical documentation practices with SLM capabilities. This is the first study to benchmark SLMs on Italian EHRs and investigate the role of clinical expertise in prompt engineering, offering valuable insights for the future integration of SLMs into real-world clinical workflows.","journal":"medRxiv","year":2025,"id":578677,"datarank":0.0,"base_score":0.0,"endowment":0.0,"self_citation_contribution":0.0,"citation_network_contribution":0.0,"self_endowment_contribution":0.0,"citer_contribution":0.0,"corpus_percentile":null,"corpus_rank":null,"citation_count":0,"citer_count":0,"citers_with_citation_signal":0,"citers_with_endowment":0,"datacite_reuse_total":0,"is_dataset":false,"is_dataset_confidence":0.9541,"is_data_producer":false,"deposit_databanks":null,"is_oa":true,"file_count":0,"downloads":0,"has_version_chain":false,"published_date":"2025-01-01","fair_score":null,"fair_percentile":null,"algorithm_id":"datarank_citation_only_1hop_v6","ranking_scope":"data_only","authors":[{"id":1488951,"name":"V. Peppoloni","orcid":null,"position":1,"is_corresponding":false},{"id":1488596,"name":"Laura Mazzeo","orcid":"0000-0001-9226-8861","position":2,"is_corresponding":false},{"id":1488597,"name":"Giuseppe Leone","orcid":"0009-0008-9931-9332","position":3,"is_corresponding":false},{"id":1488598,"name":"Luana Passos","orcid":"0000-0002-5470-7349","position":4,"is_corresponding":false},{"id":1027727,"name":"V. Miskovic","orcid":"0000-0003-1475-0243","position":5,"is_corresponding":false},{"id":1488952,"name":"Justin Armanini","orcid":null,"position":6,"is_corresponding":false},{"id":1488953,"name":"A. Ferrarin","orcid":null,"position":7,"is_corresponding":false},{"id":1201166,"name":"Isabella C. Wiest","orcid":"0009-0004-4848-614X","position":8,"is_corresponding":false},{"id":1369329,"name":"Fabian Wolf","orcid":"0000-0001-8842-3718","position":9,"is_corresponding":false},{"id":1488954,"name":"Giulia Montelatici","orcid":null,"position":10,"is_corresponding":false},{"id":1488599,"name":"Rebecca Romanò","orcid":"0000-0002-1644-1981","position":11,"is_corresponding":false},{"id":1488600,"name":"Ambrosini Paolo","orcid":"0009-0003-8200-2623","position":12,"is_corresponding":false},{"id":1488955,"name":"Tommaso Capoccia","orcid":null,"position":13,"is_corresponding":false},{"id":1488601,"name":"Stefano Natangelo","orcid":"0009-0002-3021-3085","position":14,"is_corresponding":false},{"id":1488602,"name":"Simone Rota","orcid":"0000-0002-6839-1488","position":15,"is_corresponding":false},{"id":1488956,"name":"Paola Andena","orcid":null,"position":16,"is_corresponding":false},{"id":1488957,"name":"Marta De Ponti","orcid":null,"position":17,"is_corresponding":false},{"id":1488603,"name":"Alessandra Russo","orcid":"0000-0002-6304-7928","position":18,"is_corresponding":false},{"id":1488958,"name":"Giulia Stasi","orcid":null,"position":19,"is_corresponding":false},{"id":1488604,"name":"L. Provenzano","orcid":"0000-0001-9208-2039","position":20,"is_corresponding":false},{"id":1488605,"name":"A. Spagnoletti","orcid":"0000-0002-5293-1849","position":21,"is_corresponding":false},{"id":1488959,"name":"Marco Meazza Prina","orcid":null,"position":22,"is_corresponding":false},{"id":1488606,"name":"Chiara Cavalli","orcid":"0000-0001-5568-8365","position":23,"is_corresponding":false},{"id":1488607,"name":"Claudia Giani","orcid":"0000-0001-8630-4037","position":24,"is_corresponding":false},{"id":1488960,"name":"Roberta Serino","orcid":null,"position":25,"is_corresponding":false},{"id":1488961,"name":"Michele Borracino","orcid":null,"position":26,"is_corresponding":false},{"id":1488962,"name":"C. Bonalume","orcid":null,"position":27,"is_corresponding":false},{"id":1488963,"name":"Rosa Maria Di Mauro","orcid":null,"position":28,"is_corresponding":false},{"id":1488964,"name":"C. Agosta","orcid":null,"position":29,"is_corresponding":false},{"id":1488965,"name":"Andra Diana Dumitrascu","orcid":null,"position":30,"is_corresponding":false},{"id":1488966,"name":"Giorgia Di Liberti","orcid":null,"position":31,"is_corresponding":false},{"id":1488608,"name":"Giulia Corrao","orcid":"0000-0002-5757-6073","position":32,"is_corresponding":false},{"id":1488609,"name":"Teresa Beninato","orcid":"0000-0003-0388-6655","position":33,"is_corresponding":false},{"id":1488610,"name":"Monica Ganzinelli","orcid":"0000-0002-0526-0835","position":34,"is_corresponding":false},{"id":1488611,"name":"Mario Occhipinti","orcid":"0000-0002-7169-5343","position":35,"is_corresponding":false},{"id":1488612,"name":"Marta Brambilla","orcid":"0000-0003-1148-7056","position":36,"is_corresponding":false},{"id":1488613,"name":"Claudia Proto","orcid":"0000-0003-0287-9787","position":37,"is_corresponding":false},{"id":110000,"name":"Jakob Nikolas Kather","orcid":"0000-0002-3730-5348","position":38,"is_corresponding":false},{"id":691162,"name":"Alessandra Pedrocchi","orcid":"0000-0001-9957-2786","position":39,"is_corresponding":false},{"id":226646,"name":"Filippo de Braud","orcid":"0000-0003-0103-730X","position":40,"is_corresponding":false},{"id":1488614,"name":"Giuseppe Lo Russo","orcid":"0000-0003-3224-2728","position":41,"is_corresponding":false},{"id":1488615,"name":"Paolo Baili","orcid":"0000-0003-0335-2418","position":42,"is_corresponding":false},{"id":1027730,"name":"Arsela Prelaj","orcid":"0000-0002-3863-088X","position":43,"is_corresponding":false},{"id":1488595,"name":"Federica Corso","orcid":"0000-0002-4947-9628","position":0,"is_corresponding":true}],"reference_count":43,"raw_metadata":{"citation_network_status":"fetched"},"created_at":"2026-07-19T02:58:24.957414Z","pmid":"41282683","pmcid":null,"fwci":null,"citation_percentile":null,"influential_citations":0,"oa_status":null,"license":null,"views":0,"total_file_size_bytes":0,"version_count":0,"fair_f":null,"fair_a":null,"fair_i":null,"fair_r":null,"fair_zscore":null,"fair_rationale":null,"fair_model":null,"fair_agent_version":null,"fair_fulltext_source":null,"fair_has_llm":null,"fair_computed_at":null,"clinical_trials":[],"software_tools":[],"db_accessions":[],"linked_datasets":[],"topics":[]}