Add slanted triangular lr
@@ -342,4 +342,15 @@ def warmup_linear_rates(initial_rate, warmup_steps, total_steps):
step += 1
+def slanted_triangular_rate(min_lr, max_lr, period):
+ it = 1
+ while True:
+ # https://towardsdatascience.com/adaptive-and-cyclical-learning-rates-using-pytorch-2bf904d18dee
+ cycle = numpy.floor(1 + it / (2 * period))
+ x = numpy.abs(it / period - 2 * cycle + 1)
+ relative = max(0, 1-x)
+ yield min_lr + (max_lr - min_lr) * relative
+ it += 1
+
+
from .activations import Activations # noqa
