Merge branch 'feature/nlp-emotion-core' into 'master'

Feature/nlp emotion core

See merge request !4

src/lhw_interfaces/msg_nlp/EventT2E.msg

@@ -14,4 +14,4 @@ float64 neutral
 float64 sad
 float64 surprised
 
-float64 dominant_emotion
+string dominant_emotion

src/lhw_nlp/Dockerfile

@@ -11,6 +11,11 @@ RUN if [ "$NO_GPU" = 0 ]; then \
 RUN pip3 install transformers
 RUN pip3 install dialogflow
 RUN pip3 install pygame
+# ADD THESE LATER :)
+# RUN pip3 install text2emotion
+# RUN pip3 install tensorflow
+# RUN pip3 install pickle
+# RUN pip3 install scipy
 
 # Install dependencies
 COPY src/lhw_interfaces src/lhw_interfaces

src/lhw_nlp/lhw_nlp/nlp_emotion_core.py

+import rclpy
+from rclpy.node import Node
+from lhw_interfaces.msg import Response
+from lhw_interfaces.msg import EventT2E
+
+from threading import Timer
+
+class NodeNlpEmotionCore(Node):
+    def __init__(self):
+        super().__init__('nlp_emotion_core')
+        
+        # get the speech to text data from 'dialogflow_response' topic
+        self.create_subscription(
+            EventT2E, 'text_emotion', self.on_t2e_event, 10
+        )
+
+        # MIGHT NEED TO ADD ANOTHER SUBSCRIBER FOR TONE_2_EMOTION
+        # self.create_subscription()
+        
+        # Direction audio is comming from?
+
+        # Publish data to the topic 'nlp_emotion'
+        # The data type for the response is '???????'
+        self.reset_data()
+        self.timer = Timer(1, self.fire_emotion_event)
+        self.broadcast_emotion = self.create_publisher(EventT2E, 'nlp_emotion', 10)
+
+    def reset_data(self):
+        self.input_dict = {
+            't2e-1': [],
+            't2e-2': [],
+        }
+
+    def fire_emotion_event(self):
+        print('-------------------------------------')
+        print('     TIMER, FIRE EMOTION EVENT!!     ')
+        print('-------------------------------------')
+        print(self.input_dict)
+        print('')
+
+        # Create event message
+        # Calculate stuff
+        # Fire event
+
+        # Reset data
+        self.reset_data()
+
+    def on_t2e_event(self, emotion_event):
+        timer_wait_time = 0.25  # Might need to be tweaked
+        print('NLP Emotion Core just received an emotion event:')
+        print(emotion_event)
+
+        if (emotion_event.source_id == Response.TEXT_2_EMOTION_1):
+            print('----- TEXT_2_EMOTION 1111111')
+            self.timer.cancel()
+            self.input_dict['t2e-1'].append(emotion_event)
+            self.timer = Timer(timer_wait_time, self.fire_emotion_event)
+            self.timer.start()
+
+        elif (emotion_event.source_id == Response.TEXT_2_EMOTION_2):
+            print('----- TEXT_2_EMOTION 2222222')
+            self.timer.cancel()
+            self.input_dict['t2e-2'].append(emotion_event)
+            self.timer = Timer(timer_wait_time, self.fire_emotion_event)
+            self.timer.start()
+
+        print(self.input_dict)
+        print('')
+
+
+def main(args=None):
+    rclpy.init(args=args)
+    nlm_emotion_core = NodeNlpEmotionCore()
+    rclpy.spin(nlm_emotion_core)
+    nlm_emotion_core.destroy_node()
+    rclpy.shutdown()
+
+
+if __name__ == '__main__':
+    main()

src/lhw_nlp/lhw_nlp/sentiment_analysis.py

@@ -11,25 +11,44 @@ from std_msgs.msg import String
 from lhw_interfaces.msg import Response
 from lhw_interfaces.msg import Parameter
 
+import text2emotion as t2e
+import json
+
 
 class SentimentAnalysis(Node):
     def __init__(self):
         super().__init__('sentiment_analysis')
+        
         # get the speech to text data from 'dialogflow_response' topic
-        self.text_sub = self.create_subscription(Response, 'dialogflow_response', self.text_cb, 10)
+        self.text_sub = self.create_subscription(
+            Response,
+            'dialogflow_response',
+            self.text_2_emotion,
+            10
+        )
+        
         # publish data to the topic 'emotion'. The data type for the response is 'Response'
         self.response_pub = self.create_publisher(Response, 'emotion', 10)
+        
+        # set up logger for feedback and debug
         self.log = self.get_logger()
         self.log.info(self.__class__.__name__ + " is loading..... ")
 
-    def text_cb(self, text_msg):
-        """ text_cb.data has the text data """
-        self.get_logger().info('text callback!')
-        if text_msg.data != " ":
-            msg = Response()
-            msg.source = Response.SENTIMENT_ANALYSIS
-            msg.response = "TODO: THIS SHOULD BE THE PREDICTED EMOTION"
-            self.get_logger().info('publishing: "%s"' % msg.response)
+    def text_2_emotion(self, text_msg):
+        text = str(text_msg.recognized_text)
+
+        if text == " ":
+            return
+
+        self.get_logger().info(f'sentiment_analysis > text_2_emotion: {text}')
+        raw_emotion = t2e.get_emotion(text)
+        emotion = json.dumps(raw_emotion)
+
+        msg = Response()
+        msg.source = Response.SENTIMENT_ANALYSIS
+        msg.response = emotion
+
+        self.response_pub.publish(emotion)
 
 
 def main(args=None):

src/lhw_nlp/lhw_nlp/text_2_emotion.py → src/lhw_nlp/lhw_nlp/text_2_emotion_1.py

@@ -10,7 +10,7 @@ class NodeText2Emotion(Node):
         super().__init__('text_2_emotion')
         
         # get the speech to text data from 'dialogflow_response' topic
-        self.text_sub = self.create_subscription(
+        self.create_subscription(
             Response,
             'dialogflow_response',
             self.text_2_emotion,
@@ -20,6 +20,7 @@ class NodeText2Emotion(Node):
         # Publish data to the topic 'text_emotion'
         # The data type for the response is 'EventT2E'
         self.broadcast_emotion = self.create_publisher(EventT2E, 'text_emotion', 10)
+        print('TEXT_2_EMOTION_NODE_1 IS UP AND RUNNING')
 
     def generateEventT2E(self, source_id, speaker_id, emotion_dict):
         emotions = ["Happy", "Angry", "Surprise", "Sad", "Fear"]
@@ -36,17 +37,19 @@ class NodeText2Emotion(Node):
         msg_emotion = EventT2E()
         msg_emotion.source_id = source_id
         msg_emotion.speaker_id = speaker_id  # Integrate with vision id
-        msg_emotion.angry = emotion_dict[e]
-        msg_emotion.disgust = -1
-        msg_emotion.fear = emotion_dict['Fear']
-        msg_emotion.happy = emotion_dict['Happy']
-        msg_emotion.neutral = 1 - (emotion_sum / 5)  # This might work
-        msg_emotion.sad = emotion_dict['Sad']
-        msg_emotion.surprised = emotion_dict['Surpris']
+        msg_emotion.angry = float(emotion_dict['Angry'])
+        msg_emotion.disgust = float(-1)
+        msg_emotion.fear = float(emotion_dict['Fear'])
+        msg_emotion.happy = float(emotion_dict['Happy'])
+        msg_emotion.neutral = 1.0 - (emotion_sum / 5)  # This might work
+        msg_emotion.sad = float(emotion_dict['Sad'])
+        msg_emotion.surprised = float(emotion_dict['Surprise'])
         msg_emotion.dominant_emotion = strongest_emotion.lower()
         return msg_emotion
 
     def text_2_emotion(self, text_msg):
+        print('-----------------------------------')
+        print('text_2_emotion_1: event recieved from dialog flow')
         text = str(text_msg.recognized_text)
 
         if text == " ":
@@ -58,6 +61,12 @@ class NodeText2Emotion(Node):
             speaker_id='',
             emotion_dict=emotion_dict
         )
+        print('')
+        print('-----------------------------------')
+        print('text_2_emotion_1: broadcast emotion')
+        print(msg_emotion)
+        print('')
+        print('')
         self.broadcast_emotion.publish(msg_emotion)
 
 

src/lhw_nlp/lhw_nlp/text_2_emotion_2.py

@@ -29,26 +29,32 @@ class NodeText2Emotion(Node):
         self.MAXLEN = 500
         self.CLASS_NAMES = ['happy', 'fear', 'angry', 'sad', 'neutral'] 
         # get the speech to text data from 'dialogflow_response' topic
-        self.text_sub = self.create_subscription(Response, 'dialogflow_response', self.text_cb, 10)
+        self.create_subscription(
+            Response,
+            'dialogflow_response',
+            self.text_cb,
+            10
+        )
         self.model = model
         self.tokenizer = tokenizer
         
         # Publish data to the topic 'text_emotion'
         # The data type for the response is 'EventT2E'
         self.broadcast_emotion = self.create_publisher(EventT2E, 'text_emotion', 10)
+        print('TEXT_2_EMOTION_NODE_2 IS UP AND RUNNING')
 
     def generateEventT2E(self, source_id, speaker_id, emotion_arr):
         dominant_emotion = self.CLASS_NAMES[np.argmax(emotion_arr)]
         msg_emotion = EventT2E()
         msg_emotion.source_id = source_id
         msg_emotion.speaker_id = speaker_id  # Integrate with vision id
-        msg_emotion.angry = emotion_arr[2]
-        msg_emotion.disgust = -1
-        msg_emotion.fear = emotion_arr[1]
-        msg_emotion.happy = emotion_arr[0]
-        msg_emotion.neutral = emotion_arr[4]
-        msg_emotion.sad = emotion_arr[3]
-        msg_emotion.surprised = -1
+        msg_emotion.angry = float(emotion_arr[2])
+        msg_emotion.disgust = float(-1)
+        msg_emotion.fear = float(emotion_arr[1])
+        msg_emotion.happy = float(emotion_arr[0])
+        msg_emotion.neutral = float(emotion_arr[4])
+        msg_emotion.sad = float(emotion_arr[3])
+        msg_emotion.surprised = float(-1)
         msg_emotion.dominant_emotion = dominant_emotion
         return msg_emotion
 

src/lhw_nlp/setup.py

@@ -31,7 +31,10 @@ setup(
             'tone_analysis = lhw_nlp.tone_analysis:main',
             'sentiment_analysis = lhw_nlp.sentiment_analysis:main',
             'speak_rate_analysis = lhw_nlp.speak_rate_analysis:main',
-            'bufferpublisher = lhw_nlp.bufferpublisher:main'
+            'bufferpublisher = lhw_nlp.bufferpublisher:main',
+            'text_2_emotion_node_1 = lhw_nlp.text_2_emotion_1:main',
+            'text_2_emotion_node_2 = lhw_nlp.text_2_emotion_2:main',
+            'nlp_emotion_core = lhw_nlp.nlp_emotion_core:main',
         ],
     },
 )