gpt endless training error catching
@@ -80,6 +80,7 @@ def main(config, session):
model.train()
step = -1; history = []; best_loss = math.inf;
while True:
+ try:
step += 1
# ---- Next Batch ----
inputs = nextbatch(dataset, config.neuron.batch_size_train, bittensor.__tokenizer__)
@@ -141,6 +142,10 @@ def main(config, session):
logger.info( 'Saving model: epoch: {}, loss: {}, path: {}/model.torch', step, output.loss, config.neuron.trial_path)
torch.save( {'epoch': step, 'model': model.state_dict(), 'loss': output.loss},"{}/model.torch".format(config.neuron.trial_path))
+ # --- Catch Errors during training ----
+ except Exception as e:
+ logger.error('Exection in training script with error: {}', e)
+ logger.info('Continuing to train.')
if __name__ == "__main__":
# ---- Load config ----
