Feature/common qa skill test
* Refactor skill test input utterances
Slight cleanup to make adding more different cases easier.
* Add test for common QA
"question" can now be used instead of "utterance" to test a common qa skill
@@ -394,25 +394,28 @@ class SkillTest(object):
# Emit an utterance, just like the STT engine does. This sends the
# provided text to the skill engine for intent matching and it then
# invokes the skill.
- utt = test_case.get('utterance', None)
- play_utt = test_case.get('play_query', None)
- play_start = test_case.get('play_start', None)
- if utt:
+ if 'utterance' in test_case:
+ utt = test_case['utterance']
print("UTTERANCE:", color.USER_UTT + utt + color.RESET)
- self.emitter.emit(
- 'recognizer_loop:utterance',
+ self.emitter.emit('recognizer_loop:utterance',
Message('recognizer_loop:utterance',
{'utterances': [utt]}))
- elif play_utt:
- print('PLAY QUERY', color.USER_UTT + play_utt + color.RESET)
+ elif 'play_query' in test_case:
+ play_query = test_case['play_query']
+ print('PLAY QUERY', color.USER_UTT + play_query + color.RESET)
self.emitter.emit('play:query', Message('play:query:',
- {'phrase': play_utt}))
- elif play_start:
+ {'phrase': play_query}))
+ elif 'play_start' in test_case:
print('PLAY START')
- callback_data = play_start
+ callback_data = test_case['play_start']
callback_data['skill_id'] = s.skill_id
self.emitter.emit('play:start',
Message('play:start', callback_data))
+ elif 'question' in test_case:
+ print("QUESTION: {}".format(test_case['question']))
+ callback_data = {'phrase': test_case['question']}
+ self.emitter.emit('question:query',
+ Message('question:query', data=callback_data))
else:
raise SkillTestError('No input utterance provided')
@@ -519,7 +522,6 @@ class EvaluationRule(object):
if 'play_query_match' in test_case:
match = test_case['play_query_match']
- print(test_case)
phrase = match.get('phrase', test_case.get('play_query'))
_d = ['and']
_d.append(['equal', '__type__', 'query'])
@@ -527,6 +529,13 @@ class EvaluationRule(object):
_d.append(['equal', 'phrase', phrase])
_d.append(['gt', 'conf', match.get('confidence_threshold', 0.5)])
self.rule.append(_d)
+ elif 'expected_answer' in test_case:
+ _d = ['and']
+ _d.append(['equal', '__type__', 'query.response'])
+ _d.append(['equal', 'skill_id', skill.skill_id])
+ _d.append(['equal', 'phrase', test_case['question']])
+ _d.append(['match', 'answer', test_case['expected_answer']])
+ self.rule.append(_d)
# Check for expected data structure
if test_case.get('expected_data'):
