Switch to torch.nn.functional.gelu when available
Summary: Pull Request resolved:
@@ -21,4 +21,7 @@ def gelu_accurate(x):
def gelu(x: torch.Tensor) -> torch.Tensor:
+ if hasattr(torch.nn.functional, 'gelu'):
+ return torch.nn.functional.gelu(x.float()).type_as(x)
+ else:
return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
