Adding ReLUs after linear projections
@@ -89,8 +89,9 @@ class WikiTablesDecoderStep(DecoderStep[WikiTablesDecoderState]):
scores_so_far = torch.stack(state.score)
# (group_size, decoder_input_dim)
- decoder_input = self._input_projection_layer(torch.cat([attended_question,
+ projected_input = self._input_projection_layer(torch.cat([attended_question,
previous_action_embedding], -1))
+ decoder_input = torch.nn.functional.relu(projected_input)
hidden_state, memory_cell = self._decoder_cell(decoder_input, (hidden_state, memory_cell))
hidden_state = self._dropout(hidden_state)
@@ -108,7 +109,8 @@ class WikiTablesDecoderStep(DecoderStep[WikiTablesDecoderState]):
action_query = torch.cat([hidden_state, attended_question], dim=-1)
# (group_size, action_embedding_dim)
- predicted_action_embedding = self._dropout(self._output_projection_layer(action_query))
+ projected_query = torch.nn.functional.relu(self._output_projection_layer(action_query))
+ predicted_action_embedding = self._dropout(projected_query)
considered_actions, actions_to_embed, actions_to_link = self._get_actions_to_consider(state)
