Determine result for coveriteam-verifier-validator more precisely
If multiple verdicts are printed (happens for intermediate steps with option --debug)
take the last verdict printed.
If there's an exception, return EXCEPTION-status instead. And if there's no verdict,
do not assume error, but assume UNKNOWN.
@@ -10,6 +10,7 @@ import benchexec.result as result
from benchexec.tools.template import UnsupportedFeatureException
from benchexec.tools.sv_benchmarks_util import get_data_model_from_task, ILP32, LP64
import ast
+import re
class Tool(coveriteam.Tool):
@@ -50,14 +51,17 @@ class Tool(coveriteam.Tool):
will print out the produced aftifacts.
If more than one dict is printed, the first matching one.
"""
- for line in run.output:
+ verdict = None
+ for line in reversed(run.output):
line = line.strip()
- if "verdict" in line:
+ if "'verdict':" in line and verdict is None:
# CoVeriTeam outputs benchexec result categories as verdicts.
try:
- d = ast.literal_eval(line)
- if isinstance(d, dict):
- return d.get("verdict", result.RESULT_ERROR)
- except SyntaxError:
+ verdict = re.search(r"'verdict': '([a-zA-Z\(\)\ ]*)'", line).group(1).lower()
+ except:
pass
- return result.RESULT_ERROR
+ if 'Traceback (most recent call last)' in line:
+ verdict = "EXCEPTION"
+ if verdict is None:
+ return result.RESULT_UNKNOWN
+ return verdict
\ No newline at end of file
