chore: log error when experiment is unrestorable
@@ -982,17 +982,26 @@ WHERE state IN ('ACTIVE', 'PAUSED', 'STOPPING_CANCELED', 'STOPPING_COMPLETED', '
for rows.Next() {
var exp model.Experiment
if err := rows.StructScan(&exp); err != nil {
+ // Log this error, but try to figure out the experiment ID first.
+ configErr := err
+
items, err := rows.SliceScan()
if err != nil {
+ log.WithError(configErr).Errorf("failed to read non-terminal experiment config")
return nil, errors.Wrap(err, "unable to read experiment from db")
}
expID, ok := items[0].(int64)
if !ok {
+ log.WithError(configErr).Errorf("failed to read non-terminal experiment config")
return nil, errors.Errorf(
"Expected an integer experiment ID, but got: %s", reflect.TypeOf(items[0]))
}
+ log.WithError(configErr).Errorf(
+ "failed to read non-terminal experiment config for experiment %v", expID,
+ )
+
err = db.TerminateExperimentInRestart(int(expID), model.ErrorState)
if err != nil {
log.WithError(err).Error("failed to mark experiment as errored")
