Copyright © 2026 Authors retain the copyright of this article. This article is an open access article distributed under the Creative Commons Attribution License which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.

@article{200765,
        author = {Vishwas Aithal KV and Khavya U S Reddy and Suresh Babu P},
        title = {Multimodal AI Virtual Doctor with Voice-Based Diagnostics},
        journal = {International Journal of Innovative Research in Technology},
        year = {2026},
        volume = {12},
        number = {12},
        pages = {1682-1690},
        issn = {2349-6002},
        url = {https://ijirt.org/article?manuscript=200765},
        abstract = {The rapid integration of AI into healthcare is bringing about scalable, accessible, and intelligent diagnostic support systems. This paper presents an AI-driven Medical Voice Agent for automated symptom-based doctor allocation, real-time conversational consultation, and GPT-powered structured medical report generation. It integrates a multi-modal architecture with text-based symptom analysis, voice interaction via the Vapi API, speech transcription using AssemblyAI, and document understanding via a hybrid OCR pipeline. Computer-generated medical reports are processed using EasyOCR, while re-analyses of handwritten prescriptions or low-confidence scans leverage a vision-transformer model, TrOCR, for enhanced accuracy. Furthermore, the system includes an X-ray fracture detection module powered by a CNN and Grad-CAM++ for explainable image-based diagnostics. This ensures that the final clinical summary is generated solely from the recorded doctor–patient conversation to implement human-like medical reporting. This system also offers location-based recommendations for nearby hospitals and pharmacies to enhance post-consultation usability. This solution can thus be used to provide fast, preliminary, and accessible medical guidance, reduce waiting times, and increase accessibility, especially in resource-constrained areas. Experimental results revealed accurate doctor allocation, reliable OCR-based document extraction, robust fracture detection, and coherent GPT summaries, all which point to the potential effectiveness of this system as a supportive tool in healthcare.},
        keywords = {AI Medical Voice Agent, GPT Report Generation, Speech-to-Text, Vapi API, AssemblyAI, EasyOCR, TrOCR, Medical Document OCR, Symptom-Based Doctor Allocation, CNN, Grad-CAM++, X-ray Fracture Detection, Healthcare Automation, Telemedicine, Google Maps API, Hospital and Pharmacy Recommendation, NeonDB.},
        month = {May},
        }