Merge branch 'feature/topic-nlp-emotion' into 'master'

feat: Adds topic nlp-emotion

See merge request !7

src/lhw_interfaces/msg_nlp/Response.msg

@@ -10,6 +10,7 @@ uint8 SPEAK_RATE_ANALYSIS = 7
 uint8 FACE_ANALYSIS = 8 #TODO: Move this to Vision msgs
 uint8 TEXT_2_EMOTION_1 = 9
 uint8 TEXT_2_EMOTION_2 = 10
+uint8 NLP_EMOTION_CORE = 11
 
 
 # What kind of engine generated this message

src/lhw_nlp/lhw_nlp/nlp_emotion_core.py

@@ -9,6 +9,8 @@ class NodeNlpEmotionCore(Node):
     def __init__(self):
         super().__init__('nlp_emotion_core')
         
+        self.CLASS_NAMES = ['angry', 'disgust', 'fear', 'happy', 'neutral', 'sad', 'surprised']
+
         # get the speech to text data from 'dialogflow_response' topic
         self.create_subscription(
             EventT2E, 'text_emotion', self.on_t2e_event, 10
@@ -31,6 +33,57 @@ class NodeNlpEmotionCore(Node):
             't2e-2': [],
         }
 
+    def event_t2e_init_zero(self, obj):
+        obj.angry = 0
+        obj.disgust = 0
+        obj.fear = 0
+        obj.happy = 0
+        obj.neutral = 0
+        obj.sad = 0
+        obj.surprised = 0
+        return obj
+
+    def event_t2e_normalize(self, obj):
+        sum = 0
+        sum += obj.angry
+        sum += obj.disgust
+        sum += obj.fear
+        sum += obj.happy
+        sum += obj.neutral
+        sum += obj.sad
+        sum += obj.surprised
+        obj.angry = obj.angry / sum
+        obj.disgust = obj.disgust / sum
+        obj.fear = obj.fear / sum
+        obj.happy = obj.happy / sum
+        obj.neutral = obj.neutral / sum
+        obj.sad = obj.sad / sum
+        obj.surprised = obj.surprised / sum
+        return obj
+
+    def get_aggregated_emotion_event(self, key): 
+        emotion_event_arr = self.input_dict[key]
+        arr_length = len(emotion_event_arr)
+        if arr_length == 0:
+            return None
+        elif arr_length == 1:
+            return emotion_event_arr[0]
+
+        msg_emotion = EventT2E()
+        msg_emotion.source_id = emotion_event_arr[0].source_id  # This is always true
+        msg_emotion.speaker_id = emotion_event_arr[0].speaker_id # This might change, however most likey not
+        msg_emotion = self.event_t2e_init_zero(msg_emotion)
+        for event in emotion_event_arr:
+            msg_emotion.angry += event.angry
+            msg_emotion.disgust += event.disgust
+            msg_emotion.fear += event.fear
+            msg_emotion.happy += event.happy
+            msg_emotion.neutral += event.neutral
+            msg_emotion.sad += event.sad
+            msg_emotion.surprised += event.surprised
+        return self.event_t2e_normalize(msg_emotion)
+
+
     def fire_emotion_event(self):
         print('-------------------------------------')
         print('     TIMER, FIRE EMOTION EVENT!!     ')
@@ -38,11 +91,53 @@ class NodeNlpEmotionCore(Node):
         print(self.input_dict)
         print('')
 
-        # Create event message
-        # Calculate stuff
-        # Fire event
+        msg_emotion = EventT2E()
+        msg_emotion.source_id = Response.NLP_EMOTION_CORE
+        msg_emotion.speaker_id = ''  # Integrate with vision id
+        msg_emotion = self.event_t2e_init_zero(msg_emotion)
 
-        # Reset data
+        t2e_event_1 = self.get_aggregated_emotion_event('t2e-1')
+        t2e_event_2 = self.get_aggregated_emotion_event('t2e-2')
+        t2e_event_arr = [t2e_event_1, t2e_event_2]
+        t2e_event_arr = filter(lambda obj: obj is not None, t2e_event_arr)
+        
+        angry = []
+        disgust = []
+        fear = []
+        happy = []
+        neutral = []
+        sad = []
+        surprised = []
+        for event in t2e_event_arr:
+            angry.append(float(event.angry))
+            disgust.append(float(event.disgust))
+            fear.append(float(event.fear))
+            happy.append(float(event.happy))
+            neutral.append(float(event.neutral))
+            sad.append(float(event.sad))
+            surprised.append(float(event.surprised))
+        msg_emotion.angry = max(angry)
+        msg_emotion.disgust = max(disgust)
+        msg_emotion.fear = max(fear)
+        msg_emotion.happy = max(happy)
+        msg_emotion.neutral = max(neutral)
+        msg_emotion.sad = max(sad)
+        msg_emotion.surprised = max(surprised)
+
+        msg_emotion = self.event_t2e_normalize(msg_emotion)
+
+        dominant_emotion = self.CLASS_NAMES.index(max([
+            msg_emotion.angry,
+            msg_emotion.disgust,
+            msg_emotion.fear,
+            msg_emotion.happy,
+            msg_emotion.neutral,
+            msg_emotion.sad,
+            msg_emotion.surprised
+        ]))
+        msg_emotion.dominant_emotion = dominant_emotion
+        
+        self.broadcast_emotion.publish(msg_emotion)
         self.reset_data()
 
     def on_t2e_event(self, emotion_event):

src/lhw_nlp/lhw_nlp/text_2_emotion_1.py

@@ -37,13 +37,15 @@ class NodeText2Emotion(Node):
         msg_emotion = EventT2E()
         msg_emotion.source_id = source_id
         msg_emotion.speaker_id = speaker_id  # Integrate with vision id
-        msg_emotion.angry = float(emotion_dict['Angry'])
-        msg_emotion.disgust = float(-1)
-        msg_emotion.fear = float(emotion_dict['Fear'])
+
+        msg_emotion.angry = float(emotion_dict['Angry']) / emotion_sum
+        msg_emotion.disgust = float(-1) / emotion_sum
+        msg_emotion.fear = float(emotion_dict['Fear']) / emotion_sum
         msg_emotion.happy = float(emotion_dict['Happy'])
-        msg_emotion.neutral = 1.0 - (emotion_sum / 5)  # This might work
-        msg_emotion.sad = float(emotion_dict['Sad'])
-        msg_emotion.surprised = float(emotion_dict['Surprise'])
+        msg_emotion.neutral = (1.0 - (emotion_sum / 5)) / emotion_sum  # This might work
+        msg_emotion.sad = float(emotion_dict['Sad']) / emotion_sum
+        msg_emotion.surprised = float(emotion_dict['Surprise']) / emotion_sum
+
         msg_emotion.dominant_emotion = strongest_emotion.lower()
         return msg_emotion
 
@@ -56,6 +58,7 @@ class NodeText2Emotion(Node):
             return
 
         emotion_dict = t2e.get_emotion(text)
+        print(emotion_dict)
         msg_emotion = self.generateEventT2E(
             source_id=Response.TEXT_2_EMOTION_1,
             speaker_id='',