Copyright © 2026 Authors retain the copyright of this article. This article is an open access article distributed under the Creative Commons Attribution License which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.

@article{210578,
        author = {Suvarna Lahanu Ghogare},
        title = {Query-Level Deduplication for Optimizing Database Performance using python code},
        journal = {International Journal of Innovative Research in Technology},
        year = {2026},
        volume = {13},
        number = {6},
        pages = {1384-1391},
        issn = {2349-6002},
        url = {https://ijirt.org/article?manuscript=210578},
        abstract = {The increasing volume of data generated and processed by modern database management systems has created significant challenges in terms of storage utilization, query processing time, memory consumption, and data retrieval efficiency. Duplicate records in database query results introduce unnecessary redundancy, which can affect overall system performance and resource utilization. To address these challenges, this paper proposes a Hash-Based Intelligent Query-Level Data Deduplication (HIQLD) approach for identifying and eliminating duplicate records during database query processing. The proposed methodology can be integrated with relational database management systems such as MySQL and PostgreSQL. Its performance can be evaluated using parameters including query execution time, deduplication overhead, memory consumption, duplicate detection accuracy, and reduction in redundant records. Experimental comparison with conventional SQL techniques, particularly DISTINCT and GROUP BY, can help determine its effectiveness under different dataset sizes and duplication ratios. The proposed approach provides a foundation for intelligent and efficient query-result management in large-scale database environments.},
        keywords = {Query-Level Deduplication, Database Optimization, SHA-256 Hashing, Duplicate Record Detection, SQL Query Processing, Memory Optimization, Data Redundancy.},
        month = {November},
        }