Copyright © 2025 Authors retain the copyright of this article. This article is an open access article distributed under the Creative Commons Attribution License which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.

@article{184670,
        author = {Venkata Sai Sandeep Velaga},
        title = {Adaptive Optimizing Strategies for Deep Neural Networks in Machine Learning},
        journal = {International Journal of Innovative Research in Technology},
        year = {2025},
        volume = {12},
        number = {4},
        pages = {2836-2840},
        issn = {2349-6002},
        url = {https://ijirt.org/article?manuscript=184670},
        abstract = {Training deep neural networks requires optimizers that have a balance of fast convergence and good generalization. Their traditional counterparts such as SGD with momentum are very good at generalizing but have slow convergence properties, whereas adaptive optimizers such as Adam have very fast convergence but end up generalizing poorly. In this paper, we introduced a Hybrid Adam–SGD optimizer that uses Adam during the initial few epochs of training to take advantage of its fast convergence, then moves to SGD for the remaining epochs to improve generalization. The switching to SGD can be managed by means of a static epoch threshold or, more dynamically based on plateaus observed in validation loss and small enough gradient magnitudes. The overall system was developed in PyTorch as a modular script that is separated into data processing, optimizer switching, monitoring, and evaluation stages. The experimental results from the MNIST and CIFAR-10 datasets using CNN and ResNet-18 suggest that the hybrid optimizer converges nearly as quickly as Adam, while also achieving higher test accuracy and lower generalization gaps compared to both baselines. For CIFAR-10, for example, the hybrid optimizer obtained +1.2% better test accuracy than SGD, while also achieving +2.6% better test accuracy than Adam, while also having stable validation loss. We believe our results confirm that adaptive optimizer strategies can provide a practical and effective method of improving deep learning training pipelines. Additionally, our proposed framework provides a foundation for implementation of switch policies that leverage reinforcement learning or meta-learning and extending hybrid strategies to larger models and to real-world applications.},
        keywords = {Adaptive Optimization, Deep Neural Networks, Adam Optimizer, Stochastic Gradient Descent (SGD), Hybrid Optimizer, Generalization, Convergence, Validation Loss, Dynamic Switching Policy, Machine Learning.},
        month = {September},
        }