Copyright © 2026 Authors retain the copyright of this article. This article is an open access article distributed under the Creative Commons Attribution License which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited.

@article{204975,
        author = {Pratiksha Kalyan Take and Sonika Manik Barade and Prathamesh Sunil Pachorkar and Mrunali Sunil Palande},
        title = {Emotion aware AI chatbot with real time facial emotion recognition},
        journal = {International Journal of Innovative Research in Technology},
        year = {2026},
        volume = {13},
        number = {1},
        pages = {6913-6918},
        issn = {2349-6002},
        url = {https://ijirt.org/article?manuscript=204975},
        abstract = {Most chatbot systems today respond only to the words a user types. They have no way of knowing whether a person is happy, sad, frustrated, or scared. This makes conversations feel impersonal and at times frustrating. To address this limitation, this paper presents Aura, a web-based emotionally intelligent chatbot that recognizes a user's facial emotions in real time and adjusts both its responses and voice output accordingly. The system captures a live webcam feed and uses OpenCV with Haar Cascade classifiers to detect the user's face, enabling emotion analysis during the conversation. These cropped face images were then passed to a custom Convolutional Neural Network (CNN) trained on the FER2013 dataset, which classifies the user's emotion into one of seven categories: Happy, Sad, Angry, Fear, Disgust, Surprise, or Neutral. The detected emotion was sent to a Python Flask backend, which dynamically constructed a system prompt for the Google Gemini 1.5 Flash API, instructing the model to respond with the appropriate emotional tone. For example, when sadness is detected, the prompt tells Gemini to be empathetic and gentle in their response. On the frontend, the JavaScript Web Speech API reads the responses aloud with the pitch and speaking rate adjusted per emotion. The CNN model achieved an accuracy of approximately 70% on the FER2013 test set, and the full system run at 15–30 frames per second on a standard laptop. User testing showed that the chatbot's responses felt noticeably more emotionally appropriate than a standard, emotion-unaware version. Aura shows that combining real-time computer vision, deep learning, large language models, and adaptive speech synthesis can make human-computer conversations more natural and meaningful.},
        keywords = {Emotion Recognition, Convolutional Neural Network, Gemini API, Flask, Web Speech API, Human- Computer Interaction, FER2013},
        month = {June},
        }