Create iteration when reducing the experiments
@@ -10,16 +10,40 @@ logger = logging.getLogger('polyaxon.experiments_groups.iteration_manager')
class HyperbandIterationManager(BaseIterationManger):
- def create_iteration(self):
+ def create_iteration(self, experiment_ids=None):
"""Create an iteration for the experiment group."""
from experiment_groups.models import ExperimentGroupIteration
+ search_manager = self.experiment_group.search_manager
iteration_config = self.experiment_group.iteration_config
+ if iteration_config is None:
+ iteration = 1
+ bracket_iteration = 0
+ else:
+ should_reschedule = search_manager.should_reschedule(
+ iteration=iteration_config.iteration,
+ bracket_iteration=iteration_config.bracket_iteration)
+ should_reduce_configs = search_manager.should_reduce_configs(
+ iteration=iteration_config.iteration,
+ bracket_iteration=iteration_config.bracket_iteration)
+ if should_reschedule:
+ iteration = iteration_config.iteration + 1
+ bracket_iteration = 0
+ elif should_reduce_configs:
+ iteration = iteration_config.iteration
+ bracket_iteration = iteration_config.bracket_iteration
+ else:
+ raise ValueError(
+ 'Hyperband create iteration failed for `{}`, '
+ 'could not reschedule ot reduce configs'.format(self.experiment_group.id))
+
# Create a new iteration config
iteration_config = HyperbandIterationConfig(
- iteration=1 if iteration_config is None else iteration_config.iteration,
- bracket_iteration=0)
+ iteration=iteration,
+ bracket_iteration=bracket_iteration)
+ if experiment_ids:
+ iteration_config.experiment_ids = experiment_ids
return ExperimentGroupIteration.objects.create(
experiment_group=self.experiment_group,
data=iteration_config.to_dict())
@@ -88,12 +112,9 @@ class HyperbandIterationManager(BaseIterationManger):
"""Reduce the experiments to restart."""
experiment_ids = self.get_reduced_configs()
experiments = self.experiment_group.experiments.filter(id__in=experiment_ids)
+ self.create_iteration(experiment_ids=experiment_ids)
iteration_config = self.experiment_group.iteration_config
params_config = self.experiment_group.params_config
- status_message = 'Hyperband iteration: {}, bracket iteration: {}'.format(
- iteration_config.iteration,
- iteration_config.bracket_iteration
- )
resource_value = self.experiment_group.search_manager.get_resources_for_iteration(
iteration=iteration_config.iteration)
resource_name = params_config.hyperband.resource.name
@@ -105,6 +126,9 @@ class HyperbandIterationManager(BaseIterationManger):
declarations[resource_name] = resource_value
specification = ExperimentSpecification(
values=[experiment.specification.parsed_data, declarations])
+ status_message = 'Hyperband iteration: {}, bracket iteration: {}'.format(
+ iteration_config.iteration,
+ iteration_config.bracket_iteration)
if params_config.hyperband.resume:
experiment.resume(
