Copyright © 2025 Authors retain the copyright of this article. This article is an open access article distributed under the Creative Commons Attribution License which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.

@article{180903,
        author = {Prateek Panigrahy},
        title = {Architectural patterns in ETL/ELT pipelines using SSIS, python, and cloud schedulers},
        journal = {International Journal of Innovative Research in Technology},
        year = {2025},
        volume = {12},
        number = {1},
        pages = {3696-3706},
        issn = {2349-6002},
        url = {https://ijirt.org/article?manuscript=180903},
        abstract = {Extract, Transform, Load (ETL) and Extract, Load, transform (ELT) pipelines are the data integration and operationalization building blocks for large-scale architecture in today's data-driven companies. This piece of writing delves into the shifting trends of ETL/ELT pipeline architecture based on SQL Server Integration Services (SSIS) integration, Alteryx, scripting with Python, and cloud-native platforms such as Apache Airflow, AWS Step Functions, and Google Cloud Composer. The evolution history of these tools is explored, focusing on the shift from monolithic, batch-oriented pipelines to event-based, modular, and scalable architectures. A comparison study sets out the trade-offs between GUI-centric tools, script-based workflows, and distributed schedulers, and hybrid pipeline design best practices are established in the areas of modularity, CI/CD, observability, and governance. A hybrid architecture pattern is presented for combining legacy systems with cloud-native paradigms. This paper provides researchers, architects, and practitioners with a reference model for architecting robust and interoperable pipelines of data that are accommodating in various enterprise environments.},
        keywords = {ETL, ELT, SSIS, Python, Apache Airflow, Cloud Composer, AWS Step Functions},
        month = {June},
        }