Update measure_quality.py
@@ -10,26 +10,28 @@ import json
import requests
-logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
- level=logging.INFO)
+logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO)
logger = logging.getLogger(__name__)
parser = argparse.ArgumentParser()
-parser.add_argument('--data_dir', help='directory with labeled data')
-parser.add_argument('--url', help='Conversation evaluator url. None by default, so '
- 'it calcs accurcay using active_skill in utterances from json',
- default=None)
+parser.add_argument("--data_dir", help="directory with labeled data")
+parser.add_argument(
+ "--url",
+ help="Conversation evaluator url. None by default, so "
+ "it calcs accurcay using active_skill in utterances from json",
+ default=None,
+)
def measure_quality_by_checking_file(labeled_data_json, url=None):
- with open(labeled_data_json, 'r') as f:
+ with open(labeled_data_json, "r") as f:
labeled_data = json.load(f)
goods = 0
total = 0
for row in labeled_data:
for i, utt in enumerate(row["utterances"]):
- if utt["user"]["user_type"] == 'bot' and i > 0:
+ if utt["user"]["user_type"] == "bot" and i > 0:
prev_utt = row["utterances"][i - 1]
if not prev_utt.get("has_is_best"):
continue
@@ -43,17 +45,16 @@ def measure_quality_by_checking_file(labeled_data_json, url=None):
sub_row["human"] = {"profile": {}}
json_data = {"dialogs": [sub_row]}
result = requests.post(url, json=json_data).json()
- utt = {'active_skill': result[0][0],
- 'text': result[0][1]}
+ utt = {"active_skill": result[0][0], "text": result[0][1]}
for hypot in prev_utt["hypotheses"]:
- eql_skill_and_text = utt['active_skill'] == hypot["skill_name"] and utt['text'] == hypot["text"]
+ eql_skill_and_text = utt["active_skill"] == hypot["skill_name"] and utt["text"] == hypot["text"]
if eql_skill_and_text and hypot.get("is_best") is True:
goods += 1
break
total += 1
if total == 0:
total = 1
- return {'quality': goods / total, 'goods': goods, 'total': total}
+ return {"quality": goods / total, "goods": goods, "total": total}
def main(labeled_data_dir, url):
@@ -65,13 +66,13 @@ def main(labeled_data_dir, url):
goods = 0
total = 0
for quality_dict in qualities:
- goods += quality_dict['goods']
- total += quality_dict['total']
+ goods += quality_dict["goods"]
+ total += quality_dict["total"]
acc = goods / total
print(f"Overall accuracy: {acc}")
assert acc >= 0.51, print("Accuracy less then prev best 0.51")
-if __name__ == '__main__':
+if __name__ == "__main__":
args = parser.parse_args()
main(args.data_dir, args.url)
