in src/speech_reps/models/bertphone.py [0:0]
def __init__(self, attention_cell='multi_head', num_layers=2,
units=512, hidden_size=2048, max_length=50,
num_heads=4, scaled=True, dropout=0.0,
use_residual=True, output_attention=False, output_all_encodings=False,
weight_initializer=None, bias_initializer='zeros',
prefix=None, params=None, activation='gelu', layer_norm_eps=None):
super(BERTEncoder, self).__init__(attention_cell=attention_cell,
num_layers=num_layers, units=units,
hidden_size=hidden_size, max_length=max_length,
num_heads=num_heads, scaled=scaled, dropout=dropout,
use_residual=use_residual,
output_attention=output_attention,
output_all_encodings=output_all_encodings,
weight_initializer=weight_initializer,
bias_initializer=bias_initializer,
prefix=prefix, params=params,
# extra configurations for BERT
positional_weight='learned',
use_bert_encoder=True,
use_layer_norm_before_dropout=False,
scale_embed=False,
activation=activation,
layer_norm_eps=layer_norm_eps)