Shuffling labels during training as cross check
@@ -16,7 +16,7 @@ from madminer.utils.ml.models.score import LocalScoreEstimator
from madminer.utils.ml.flow_trainer import train_flow_model, evaluate_flow_model
from madminer.utils.ml.ratio_trainer import train_ratio_model, evaluate_ratio_model
from madminer.utils.ml.score_trainer import train_local_score_model, evaluate_local_score_model
-from madminer.utils.various import create_missing_folders, load_and_check, general_init
+from madminer.utils.various import create_missing_folders, load_and_check, general_init, shuffle
class MLForge:
@@ -89,6 +89,7 @@ class MLForge:
validation_split=None,
early_stopping=True,
scale_inputs=True,
+ shuffle_labels=False,
grad_x_regularization=None,
):
@@ -217,6 +218,11 @@ class MLForge:
scale_inputs : bool, optional
Scale the observables to zero mean and unit variance. Default value: True.
+ shuffle_labels : bool optional
+ If True, the labels (`y`, `r_xz`, `t_xz`) are shuffled, while the observations (`x`) remain in their
+ normal order. This serves as a closure test, in particular as cross-check against overfitting: an estimator
+ trained with shuffle_labels=True should predict to likelihood ratios around 1 and scores around 0.
+
grad_x_regularization : float or None, optional
If not None, a term of the form `grad_x_regularization * |grad_x f(x)|^2` is added to the loss, where `f(x)`
is the neural network output (the estimated log likelihood ratio or score). Default value: None.
@@ -268,6 +274,7 @@ class MLForge:
logging.info(" Validation split: %s", validation_split)
logging.info(" Early stopping: %s", early_stopping)
logging.info(" Scale inputs: %s", scale_inputs)
+ logging.info(" Shuffle labels %s", shuffle_labels)
if grad_x_regularization is None:
logging.info(" Regularization: None")
else:
@@ -329,8 +336,8 @@ class MLForge:
logging.info("Found %s samples with %s parameters and %s observables", n_samples, n_parameters, n_observables)
# Scale features
- logging.info("Rescaling inputs")
if scale_inputs:
+ logging.info("Rescaling inputs")
self.x_scaling_means = np.mean(x, axis=0)
self.x_scaling_stds = np.maximum(np.std(x, axis=0), 1.0e-6)
x[:] -= self.x_scaling_means
@@ -350,6 +357,11 @@ class MLForge:
np.max(x[:, i]),
)
+ # Shuffle labels
+ if shuffle_labels:
+ logging.info("Shuffling labels")
+ y, r_xz, t_xz0, t_xz1 = shuffle(y, r_xz, t_xz0, t_xz1)
+
# Features
if features is not None:
x = x[:, features]
