Avoid converting grad to dense tensor if it is sparse in Adagrad optimizer.
This does not cause numerical change, but is theoretically faster than the previous handling.
@@ -115,6 +115,13 @@ class Adagrad(optimizer.Optimizer):
accumulator.scatter_add(
tf.IndexedSlices(grad.values * grad.values, grad.indices)
)
+ denominator = tf.sqrt(accumulator + self.epsilon)
+ sparse_denominator = tf.gather(denominator, indices=grad.indices)
+ variable.scatter_add(
+ tf.IndexedSlices(
+ -lr * grad.values / sparse_denominator, grad.indices
+ )
+ )
else:
# Dense gradients.
accumulator.assign_add(grad * grad)
