feat(fastai): handle fastai related exceptions
* fix(fastai): check predictions can be logged
fix
* feat(fastai): create FastaiError Exceptions
class used to log fastai related exceptions
@@ -79,7 +79,6 @@ class WandbCallback(TrackerCallback):
predictions=36,
seed=12345):
-
# Check if wandb.init has been called
if wandb.run is None:
raise ValueError(
@@ -136,12 +135,48 @@ class WandbCallback(TrackerCallback):
with self.model_path.open('wb') as model_file:
self.learn.save(model_file)
- # Log sample predictions
+ # Log sample predictions if learn.predict is available
if self.validation_data:
+ try:
+ self._wandb_log_predictions()
+ except FastaiError as e:
+ wandb.termwarn(e.message)
+ self.validation_data = None # prevent from trying again on next loop
+ except Exception as e:
+ wandb.termwarn("Unable to log prediction samples.\n{}".format(e))
+ self.validation_data=None # prevent from trying again on next loop
+
+ # Log losses & metrics
+ # Adapted from fast.ai "CSVLogger"
+ logs = {
+ name: stat
+ for name, stat in list(
+ zip(self.learn.recorder.names, [epoch, smooth_loss] +
+ last_metrics))
+ }
+ wandb.log(logs)
+
+ def on_train_end(self, **kwargs):
+ "Load the best model."
+
+ if self.save_model:
+ # Adapted from fast.ai "SaveModelCallback"
+ if self.model_path.is_file():
+ with self.model_path.open('rb') as model_file:
+ self.learn.load(model_file, purge=False)
+ print('Loaded best saved model from {}'.format(
+ self.model_path))
+
+ def _wandb_log_predictions(self):
+ "Log prediction samples"
+
pred_log = []
for x, y in self.validation_data:
+ try:
pred=self.learn.predict(x)
+ except:
+ raise FastaiError('Unable to run "predict" method from Learner to log prediction samples.')
# scalar -> likely to be a category
if not pred[1].shape:
@@ -192,23 +227,6 @@ class WandbCallback(TrackerCallback):
wandb.log({"Prediction Samples": pred_log}, commit=False)
- # Log losses & metrics
- # Adapted from fast.ai "CSVLogger"
- logs = {
- name: stat
- for name, stat in list(
- zip(self.learn.recorder.names, [epoch, smooth_loss] +
- last_metrics))
- }
- wandb.log(logs)
-
- def on_train_end(self, **kwargs):
- "Load the best model."
- if self.save_model:
- # Adapted from fast.ai "SaveModelCallback"
- if self.model_path.is_file():
- with self.model_path.open('rb') as model_file:
- self.learn.load(model_file, purge=False)
- print('Loaded best saved model from {}'.format(
- self.model_path))
+class FastaiError(wandb.Error):
+ pass
