{"doi":"10.1145/3626246.3653385","title":"Data-Juicer: A One-Stop Data Processing System for Large Language Models","abstract":null,"journal":"Companion of the 2024 International Conference on Management of Data","year":2024,"id":671711,"datarank":0.5456379239589579,"base_score":3.6375861597263857,"endowment":3.6375861597263857,"self_citation_contribution":0.5456379239589579,"citation_network_contribution":0.0,"self_endowment_contribution":0.5456379239589579,"citer_contribution":0.0,"corpus_percentile":null,"corpus_rank":null,"citation_count":37,"citer_count":0,"citers_with_citation_signal":0,"citers_with_endowment":0,"datacite_reuse_total":0,"is_dataset":false,"is_dataset_confidence":null,"is_data_producer":false,"deposit_databanks":null,"is_oa":false,"file_count":0,"downloads":0,"has_version_chain":false,"published_date":null,"fair_score":null,"fair_percentile":null,"algorithm_id":"datarank_citation_only_1hop_v6","ranking_scope":"data_only","authors":[{"id":1024391,"name":"Yilun Huang","orcid":"0009-0002-0267-1716","position":1,"is_corresponding":false},{"id":1754895,"name":"Zhijian Ma","orcid":"0009-0007-8674-5351","position":2,"is_corresponding":false},{"id":1754896,"name":"Hesen Chen","orcid":"0009-0009-3434-4809","position":3,"is_corresponding":false},{"id":1754897,"name":"Xuchen Pan","orcid":"0009-0002-0081-5405","position":4,"is_corresponding":false},{"id":1754898,"name":"Ce Ge","orcid":"0000-0003-4312-0152","position":5,"is_corresponding":false},{"id":1443611,"name":"Dawei Gao","orcid":"0000-0002-5525-8717","position":6,"is_corresponding":false},{"id":1754899,"name":"Yuexiang Xie","orcid":"0009-0005-6545-7882","position":7,"is_corresponding":false},{"id":1144471,"name":"Zhaoyang Liu","orcid":"0009-0007-8626-4558","position":8,"is_corresponding":false},{"id":1754900,"name":"Jinyang Gao","orcid":"0000-0001-8247-1196","position":9,"is_corresponding":false},{"id":1754901,"name":"Yaliang Li","orcid":"0000-0002-4204-6096","position":10,"is_corresponding":false},{"id":1754902,"name":"Bolin Ding","orcid":"0000-0003-1535-9692","position":11,"is_corresponding":false},{"id":1754903,"name":"Jingren Zhou","orcid":"0000-0002-4220-2634","position":12,"is_corresponding":false},{"id":1754894,"name":"Daoyuan Chen","orcid":"0000-0002-8015-2121","position":0,"is_corresponding":false}],"reference_count":0,"raw_metadata":{"has_enrichment":true,"resolved":true,"title":"Data-Juicer: A One-Stop Data Processing System for Large Language Models","abstract":"The immense evolution in Large Language Models (LLMs) has underscored the importance of massive, heterogeneous, and high-quality data. A data recipe is a mixture of data from different sources for training LLMs, which plays a vital role in LLMs' performance. Existing open-source tools for LLM data processing are mostly tailored for specific data recipes. To continuously uncover the potential of LLMs, incorporate data from new sources, and improve LLMs' performance, we build a new system named Data-Juicer, with which we can efficiently generate diverse data recipes, explore different possibilities in forming data mixtures, and evaluate their effects on model performance. Different from traditional data-analytics pipelines, Data-Juicer faces some unique challenges. Firstly, the possible data sources for forming data recipes are truly heterogeneous and massive with various qualities. Secondly, it is extremely expensive to precisely evaluate data recipes' impact on LLMs' performance. Thirdly, the end users of Data-Juicer, model developers, need sufficient flexibility to configure and evaluate different data recipes.","is_dataset_classified":null,"base_score":3.4657359027997265,"endowment":3.4657359027997265,"datacite_reuse_total":0,"file_count":0,"downloads":0,"views":0,"has_version_chain":false,"is_dataset":false,"is_oa":false,"pmid":"19965766","pmcid":null,"openalex_id":"https://openalex.org/W4398234485","authors":[],"funders":[],"total_grants":0,"fwci":10.3321,"citation_percentile":0.98873255,"influential_citations":0,"citation_trend":[{"year":2024,"count":4},{"year":2025,"count":19},{"year":2026,"count":8}],"oa_status":"closed","license":"https://www.acm.org/publications/policies/copyright_policy#Background","oa_locations":[{"url":"https://dl.acm.org/doi/10.1145/3626246.3653385","host_type":"publisher"},{"url":"https://dl.acm.org/doi/pdf/10.1145/3626246.3653385","host_type":"publisher"},{"url":"https://doi.org/10.1145/3626246.3653385","host_type":""}],"fields_of_study":["Topic Modeling","Natural Language Processing Techniques","Advanced Graph Neural Networks"],"mesh_terms":[],"keywords":["Computer science","Data modeling","Programming language","Database"],"sdg_mappings":[],"linked_datasets":[],"clinical_trials":[],"software_tools":[],"database_accessions":[],"source":"live","citation_network_status":"fetched"},"created_at":"2026-08-16T04:07:39.073489Z","pmid":null,"pmcid":null,"fwci":null,"citation_percentile":null,"influential_citations":0,"oa_status":null,"license":null,"views":0,"total_file_size_bytes":0,"version_count":0,"fair_f":null,"fair_a":null,"fair_i":null,"fair_r":null,"fair_zscore":null,"fair_rationale":null,"fair_model":null,"fair_agent_version":null,"fair_fulltext_source":null,"fair_has_llm":null,"fair_computed_at":null,"clinical_trials":[],"software_tools":[],"db_accessions":[],"linked_datasets":[],"topics":[]}