Copyright © 2026 Authors retain the copyright of this article. This article is an open access article distributed under the Creative Commons Attribution License which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.

@article{185585,
        author = {Brinda Sakhiya},
        title = {Architectural Comparative Analysis of Data Preprocessing Techniques for Large Language Models: From Linguistic Fundamentals to Scalable Cloud-Native Pipelines},
        journal = {International Journal of Innovative Research in Technology},
        year = {2025},
        volume = {12},
        number = {5},
        pages = {2066-2074},
        issn = {2349-6002},
        url = {https://ijirt.org/article?manuscript=185585},
        abstract = {The effectiveness of Large Language Models (LLMs)—including architectures like GPT, BERT, and PaLM—is fundamentally constrained by the quality of their training data. This paper presents a comprehensive, architectural, and comparative analysis of data preprocessing techniques required to transform raw, web-scale textual corpora into clean, standardized, and machine-readable input. We systematically evaluate key preprocessing methods (tokenization, normalization, morphological reduction, and data augmentation) across three distinct technological tiers: Natural Language Processing (NLP) libraries (NLTK, spaCy), Deep Learning frameworks (TensorFlow, PyTorch), and Cloud AI platforms (Google Cloud AI, Amazon SageMaker). Through simulated experimental results on a 106 -sample multilingual dataset, we demonstrate that robust preprocessing pipelines, particularly those combining optimized tools, achieve up to 12% higher downstream model accuracy and an 18% faster convergence time compared to models trained on unprocessed data. Furthermore, we expand the scope to include advanced data curation strategies such as toxicity screening, PII redaction, and MinHash Locality-Sensitive Hashing (LSH) deduplication, which are critical for ethical alignment and computational efficiency at petabyte scale. The findings emphasize that a hybrid architectural approach, leveraging the linguistic speed of spaCy with the scalability of cloud platforms, represents the optimal blueprint for future LLM MLOps pipelines, culminating in a crucial examination of specialized preprocessing for Retrieval-Augmented Generation (RAG) systems.},
        keywords = {Large Language Models, Data Preprocessing, NLP, Deep Learning, MLOps, Tokenization, Normalization, Data Augmentation, Cloud AI, RAG.},
        month = {October},
        }