feat: add skill confidence to response selector
@@ -33,6 +33,7 @@ def respond():
conversations = []
dialog_ids = []
selected_skill_names = []
+ confidences = []
for i, dialog in enumerate(dialogs_batch):
for skill_name in response_candidates[i]:
@@ -40,12 +41,14 @@ def respond():
conv["currentUtterance"] = dialog["utterances"][-1]["text"]
conv["currentResponse"] = response_candidates[i][skill_name]["text"]
# every odd utterance is from user
- conv["pastUtterances"] = [uttr["text"] for uttr in dialog["utterances"][1::2]]
+ # cobot recommends to take 2 last utt for conversation evaluation service
+ conv["pastUtterances"] = [uttr["text"] for uttr in dialog["utterances"][1::2]][:2]
# every second utterance is from bot
- conv["pastResponses"] = [uttr["text"] for uttr in dialog["utterances"][::2]]
+ conv["pastResponses"] = [uttr["text"] for uttr in dialog["utterances"][::2]][:2]
# collect all the conversations variants to evaluate them batch-wise
conversations += [conv]
dialog_ids += [i]
+ confidences += [response_candidates[i][skill_name]["confidence"]]
result = requests.request(url=COBOT_CONVERSATION_EVALUATION_SERVICE_URL,
headers=headers,
@@ -62,13 +65,15 @@ def respond():
result = np.array(result["conversationEvaluationScores"])
dialog_ids = np.array(dialog_ids)
+ confidences = np.array(confidences)
for i, dialog in enumerate(dialogs_batch):
# curr_candidates is dict
curr_candidates = response_candidates[i]
# choose results which correspond curr candidates
curr_scores = result[dialog_ids == i]
- best_skill_name = select_response(curr_candidates, curr_scores, dialog)
+ curr_confidences = confidences[dialog_ids == i]
+ best_skill_name = select_response(curr_candidates, curr_scores, curr_confidences, dialog)
# best_response = curr_candidates[best_skill_name]["text"]
# confidence = curr_candidates[best_skill_name]["confidence"]
selected_skill_names.append(best_skill_name)
@@ -77,13 +82,15 @@ def respond():
return jsonify(selected_skill_names)
-def select_response(curr_candidates, curr_scores, dialog):
+def select_response(curr_candidates, curr_scores, curr_confidences, dialog):
# calculate curr_scores which is an array of values-scores for each candidate
curr_single_cores = [(cand_scores["isResponseOnTopic"] + cand_scores["isResponseInteresting"] +
cand_scores["responseEngagesUser"] + cand_scores["isResponseComprehensible"] -
- cand_scores["isResponseErroneous"])
- for cand_scores in curr_scores]
+ cand_scores["isResponseErroneous"] + confidence)
+ for cand_scores, confidence in zip(curr_scores, curr_confidences)]
+ for i, score in enumerate(curr_single_cores):
+ logger.info(f'Skill {list(curr_candidates.keys())[i]} has score: {score}')
best_id = np.argmax(curr_single_cores)
best_skill_name = list(curr_candidates.keys())[best_id]
