From 602e15d7877f3cabb1c4e0e00959ff1c257ebe44 Mon Sep 17 00:00:00 2001
From: Kipok <igor.a.gitman@gmail.com>
Date: Wed, 30 May 2018 17:14:15 -0700
Subject: [PATCH] Fix typo with use_horovod=True

---
 docs/html/_sources/models-and-recipes.rst.txt | 2 +-
 docs/html/models-and-recipes.html             | 2 +-
 docs/html/searchindex.js                      | 2 +-
 docs/sources/source/models-and-recipes.rst    | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/html/_sources/models-and-recipes.rst.txt b/docs/html/_sources/models-and-recipes.rst.txt
index 6c533686c..0f6f31b53 100644
--- a/docs/html/_sources/models-and-recipes.rst.txt
+++ b/docs/html/_sources/models-and-recipes.rst.txt
@@ -18,7 +18,7 @@ commands (don't forget to substitute valid config_file path there).
 
 With Horovod (highly recommended when using multiple GPUs)::
 
-    mpirun --allow-run-as-root --mca orte_base_help_aggregate 0 -mca btl ^openib -np 4 -H localhost:4 -bind-to none -map-by slot -x LD_LIBRARY_PATH python run.py --config_file=... --mode=train_eval --enable_logs
+    mpirun --allow-run-as-root --mca orte_base_help_aggregate 0 -mca btl ^openib -np 4 -H localhost:4 -bind-to none -map-by slot -x LD_LIBRARY_PATH python run.py --config_file=... --mode=train_eval --use_horovod=True --enable_logs
 
 Without Horovod::
 
diff --git a/docs/html/models-and-recipes.html b/docs/html/models-and-recipes.html
index e775f66e1..bf2a77b47 100644
--- a/docs/html/models-and-recipes.html
+++ b/docs/html/models-and-recipes.html
@@ -174,7 +174,7 @@
 <p>To train models you can use the following
 commands (don’t forget to substitute valid config_file path there).</p>
 <p>With Horovod (highly recommended when using multiple GPUs):</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">mpirun</span> <span class="o">--</span><span class="n">allow</span><span class="o">-</span><span class="n">run</span><span class="o">-</span><span class="k">as</span><span class="o">-</span><span class="n">root</span> <span class="o">--</span><span class="n">mca</span> <span class="n">orte_base_help_aggregate</span> <span class="mi">0</span> <span class="o">-</span><span class="n">mca</span> <span class="n">btl</span> <span class="o">^</span><span class="n">openib</span> <span class="o">-</span><span class="n">np</span> <span class="mi">4</span> <span class="o">-</span><span class="n">H</span> <span class="n">localhost</span><span class="p">:</span><span class="mi">4</span> <span class="o">-</span><span class="n">bind</span><span class="o">-</span><span class="n">to</span> <span class="n">none</span> <span class="o">-</span><span class="nb">map</span><span class="o">-</span><span class="n">by</span> <span class="n">slot</span> <span class="o">-</span><span class="n">x</span> <span class="n">LD_LIBRARY_PATH</span> <span class="n">python</span> <span class="n">run</span><span class="o">.</span><span class="n">py</span> <span class="o">--</span><span class="n">config_file</span><span class="o">=...</span> <span class="o">--</span><span class="n">mode</span><span class="o">=</span><span class="n">train_eval</span> <span class="o">--</span><span class="n">enable_logs</span>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">mpirun</span> <span class="o">--</span><span class="n">allow</span><span class="o">-</span><span class="n">run</span><span class="o">-</span><span class="k">as</span><span class="o">-</span><span class="n">root</span> <span class="o">--</span><span class="n">mca</span> <span class="n">orte_base_help_aggregate</span> <span class="mi">0</span> <span class="o">-</span><span class="n">mca</span> <span class="n">btl</span> <span class="o">^</span><span class="n">openib</span> <span class="o">-</span><span class="n">np</span> <span class="mi">4</span> <span class="o">-</span><span class="n">H</span> <span class="n">localhost</span><span class="p">:</span><span class="mi">4</span> <span class="o">-</span><span class="n">bind</span><span class="o">-</span><span class="n">to</span> <span class="n">none</span> <span class="o">-</span><span class="nb">map</span><span class="o">-</span><span class="n">by</span> <span class="n">slot</span> <span class="o">-</span><span class="n">x</span> <span class="n">LD_LIBRARY_PATH</span> <span class="n">python</span> <span class="n">run</span><span class="o">.</span><span class="n">py</span> <span class="o">--</span><span class="n">config_file</span><span class="o">=...</span> <span class="o">--</span><span class="n">mode</span><span class="o">=</span><span class="n">train_eval</span> <span class="o">--</span><span class="n">use_horovod</span><span class="o">=</span><span class="kc">True</span> <span class="o">--</span><span class="n">enable_logs</span>
 </pre></div>
 </div>
 <p>Without Horovod:</p>
diff --git a/docs/html/searchindex.js b/docs/html/searchindex.js
index 46b5e441e..e75ca2d35 100644
--- a/docs/html/searchindex.js
+++ b/docs/html/searchindex.js
@@ -1 +1 @@
-Search.setIndex({docnames:["api-docs/data","api-docs/data.image2label","api-docs/data.speech2text","api-docs/data.text2text","api-docs/decoders","api-docs/encoders","api-docs/losses","api-docs/models","api-docs/modules","api-docs/optimizers","api-docs/parts","api-docs/parts.rnns","api-docs/parts.transformer","api-docs/utils","distr-training","extending","extending/adding-new-data-layer","extending/adding-new-decoder","extending/adding-new-encoder","extending/adding-new-loss","getting-started","getting-started/asr","getting-started/nmt","in-depth-tutorials","in-depth-tutorials/internal-structure","in-depth-tutorials/using-existing-models","index","installation-instructions","mixed-precision","models-and-recipes"],envversion:53,filenames:["api-docs/data.rst","api-docs/data.image2label.rst","api-docs/data.speech2text.rst","api-docs/data.text2text.rst","api-docs/decoders.rst","api-docs/encoders.rst","api-docs/losses.rst","api-docs/models.rst","api-docs/modules.rst","api-docs/optimizers.rst","api-docs/parts.rst","api-docs/parts.rnns.rst","api-docs/parts.transformer.rst","api-docs/utils.rst","distr-training.rst","extending.rst","extending/adding-new-data-layer.rst","extending/adding-new-decoder.rst","extending/adding-new-encoder.rst","extending/adding-new-loss.rst","getting-started.rst","getting-started/asr.rst","getting-started/nmt.rst","in-depth-tutorials.rst","in-depth-tutorials/internal-structure.rst","in-depth-tutorials/using-existing-models.rst","index.rst","installation-instructions.rst","mixed-precision.rst","models-and-recipes.rst"],objects:{"":{data:[0,0,0,"-"],decoders:[4,0,0,"-"],encoders:[5,0,0,"-"],losses:[6,0,0,"-"],models:[7,0,0,"-"],optimizers:[9,0,0,"-"],parts:[10,0,0,"-"],utils:[13,0,0,"-"]},"data.data_layer":{DataLayer:[0,1,1,""]},"data.data_layer.DataLayer":{__init__:[0,2,1,""],build_graph:[0,2,1,""],get_optional_params:[0,3,1,""],get_required_params:[0,3,1,""],get_size_in_samples:[0,2,1,""],input_tensors:[0,4,1,""],iterator:[0,4,1,""],params:[0,4,1,""]},"data.image2label":{image2label:[1,0,0,"-"],imagenet_preprocessing:[1,0,0,"-"]},"data.image2label.image2label":{ImagenetDataLayer:[1,1,1,""]},"data.image2label.image2label.ImagenetDataLayer":{build_graph:[1,2,1,""],get_optional_params:[1,3,1,""],get_required_params:[1,3,1,""],get_size_in_samples:[1,2,1,""],input_tensors:[1,4,1,""],iterator:[1,4,1,""],split_data:[1,2,1,""]},"data.image2label.imagenet_preprocessing":{_aspect_preserving_resize:[1,5,1,""],_central_crop:[1,5,1,""],_decode_crop_and_flip:[1,5,1,""],_mean_image_subtraction:[1,5,1,""],_parse_example_proto:[1,5,1,""],_resize_image:[1,5,1,""],_smallest_size_at_least:[1,5,1,""],parse_record:[1,5,1,""],preprocess_image:[1,5,1,""]},"data.speech2text":{speech2text:[2,0,0,"-"],speech_utils:[2,0,0,"-"]},"data.speech2text.speech2text":{Speech2TextDataLayer:[2,1,1,""]},"data.speech2text.speech2text.Speech2TextDataLayer":{__init__:[2,2,1,""],_parse_audio_element:[2,2,1,""],_parse_audio_transcript_element:[2,2,1,""],build_graph:[2,2,1,""],get_optional_params:[2,3,1,""],get_required_params:[2,3,1,""],get_size_in_samples:[2,2,1,""],input_tensors:[2,4,1,""],iterator:[2,4,1,""],split_data:[2,2,1,""]},"data.speech2text.speech_utils":{augment_audio_signal:[2,5,1,""],get_speech_features:[2,5,1,""],get_speech_features_from_file:[2,5,1,""]},"data.text2text":{t2t:[3,0,0,"-"],text2text:[3,0,0,"-"],tokenizer:[3,0,0,"-"]},"data.text2text.t2t":{_batch_examples:[3,5,1,""],_create_min_max_boundaries:[3,5,1,""],_filter_max_length:[3,5,1,""],_get_example_length:[3,5,1,""],_load_records:[3,5,1,""],_parse_example:[3,5,1,""],_read_and_batch_from_files:[3,5,1,""],eval_input_fn:[3,5,1,""],train_input_fn:[3,5,1,""]},"data.text2text.text2text":{ParallelTextDataLayer:[3,1,1,""],SpecialTextTokens:[3,1,1,""],TransformerDataLayer:[3,1,1,""]},"data.text2text.text2text.ParallelTextDataLayer":{build_graph:[3,2,1,""],get_optional_params:[3,3,1,""],get_required_params:[3,3,1,""],get_size_in_samples:[3,2,1,""],input_tensors:[3,4,1,""],iterator:[3,4,1,""]},"data.text2text.text2text.SpecialTextTokens":{END_OF_CHOICE:[3,4,1,""],EOS_ID:[3,4,1,""],OUT_OF_BUCKET:[3,4,1,""],PAD_ID:[3,4,1,""],S_ID:[3,4,1,""],UNK_ID:[3,4,1,""]},"data.text2text.text2text.TransformerDataLayer":{build_graph:[3,2,1,""],get_optional_params:[3,3,1,""],get_required_params:[3,3,1,""],input_tensors:[3,4,1,""],iterator:[3,4,1,""]},"data.text2text.tokenizer":{Subtokenizer:[3,1,1,""],_count_and_gen_subtokens:[3,5,1,""],_count_tokens:[3,5,1,""],_escape_token:[3,5,1,""],_filter_and_bucket_subtokens:[3,5,1,""],_gen_new_subtoken_list:[3,5,1,""],_generate_alphabet_dict:[3,5,1,""],_generate_subtokens:[3,5,1,""],_generate_subtokens_with_target_vocab_size:[3,5,1,""],_join_tokens_to_string:[3,5,1,""],_list_to_index_dict:[3,5,1,""],_load_vocab_file:[3,5,1,""],_native_to_unicode:[3,5,1,""],_save_vocab_file:[3,5,1,""],_split_string_to_tokens:[3,5,1,""],_split_token_to_subtokens:[3,5,1,""],_unescape_token:[3,5,1,""],_unicode_to_native:[3,5,1,""]},"data.text2text.tokenizer.Subtokenizer":{__init__:[3,2,1,""],_subtoken_ids_to_tokens:[3,2,1,""],_token_to_subtoken_ids:[3,2,1,""],decode:[3,2,1,""],encode:[3,2,1,""],init_from_files:[3,3,1,""]},"data.utils":{load_pre_existing_vocabulary:[0,5,1,""],pad_vocab_to_eight:[0,5,1,""]},"decoders.decoder":{Decoder:[4,1,1,""]},"decoders.decoder.Decoder":{__init__:[4,2,1,""],_cast_types:[4,2,1,""],_decode:[4,2,1,""],decode:[4,2,1,""],get_optional_params:[4,3,1,""],get_required_params:[4,3,1,""],mode:[4,4,1,""],name:[4,4,1,""],params:[4,4,1,""]},"decoders.fc_decoders":{FullyConnectedCTCDecoder:[4,1,1,""],FullyConnectedDecoder:[4,1,1,""],FullyConnectedTimeDecoder:[4,1,1,""]},"decoders.fc_decoders.FullyConnectedCTCDecoder":{__init__:[4,2,1,""],get_optional_params:[4,3,1,""],get_required_params:[4,3,1,""]},"decoders.fc_decoders.FullyConnectedDecoder":{__init__:[4,2,1,""],_decode:[4,2,1,""],get_required_params:[4,3,1,""]},"decoders.fc_decoders.FullyConnectedTimeDecoder":{__init__:[4,2,1,""],_decode:[4,2,1,""],get_optional_params:[4,3,1,""],get_required_params:[4,3,1,""]},"decoders.rnn_decoders":{BeamSearchRNNDecoderWithAttention:[4,1,1,""],RNNDecoderWithAttention:[4,1,1,""]},"decoders.rnn_decoders.BeamSearchRNNDecoderWithAttention":{__init__:[4,2,1,""],_decode:[4,2,1,""],get_optional_params:[4,3,1,""]},"decoders.rnn_decoders.RNNDecoderWithAttention":{__init__:[4,2,1,""],_build_attention:[4,2,1,""],_decode:[4,2,1,""],get_optional_params:[4,3,1,""],get_required_params:[4,3,1,""]},"encoders.ds2_encoder":{DeepSpeech2Encoder:[5,1,1,""],conv2d_bn_actv:[5,5,1,""],rnn_cell:[5,5,1,""],row_conv:[5,5,1,""]},"encoders.ds2_encoder.DeepSpeech2Encoder":{__init__:[5,2,1,""],_encode:[5,2,1,""],get_optional_params:[5,3,1,""],get_required_params:[5,3,1,""]},"encoders.encoder":{Encoder:[5,1,1,""]},"encoders.encoder.Encoder":{__init__:[5,2,1,""],_cast_types:[5,2,1,""],_encode:[5,2,1,""],encode:[5,2,1,""],get_optional_params:[5,3,1,""],get_required_params:[5,3,1,""],mode:[5,4,1,""],name:[5,4,1,""],params:[5,4,1,""]},"encoders.resnet_blocks":{batch_norm:[5,5,1,""],block_layer:[5,5,1,""],bottleneck_block_v1:[5,5,1,""],bottleneck_block_v2:[5,5,1,""],building_block_v1:[5,5,1,""],building_block_v2:[5,5,1,""],conv2d_fixed_padding:[5,5,1,""],fixed_padding:[5,5,1,""]},"encoders.resnet_encoder":{ResNetEncoder:[5,1,1,""]},"encoders.resnet_encoder.ResNetEncoder":{get_optional_params:[5,3,1,""],get_required_params:[5,3,1,""]},"encoders.rnn_encoders":{BidirectionalRNNEncoderWithEmbedding:[5,1,1,""],GNMTLikeEncoderWithEmbedding:[5,1,1,""],UnidirectionalRNNEncoderWithEmbedding:[5,1,1,""]},"encoders.rnn_encoders.BidirectionalRNNEncoderWithEmbedding":{__init__:[5,2,1,""],_encode:[5,2,1,""],enc_emb_w:[5,4,1,""],get_optional_params:[5,3,1,""],get_required_params:[5,3,1,""],src_emb_size:[5,4,1,""],src_vocab_size:[5,4,1,""]},"encoders.rnn_encoders.GNMTLikeEncoderWithEmbedding":{__init__:[5,2,1,""],enc_emb_w:[5,4,1,""],get_optional_params:[5,3,1,""],get_required_params:[5,3,1,""],src_emb_size:[5,4,1,""],src_vocab_size:[5,4,1,""]},"encoders.rnn_encoders.UnidirectionalRNNEncoderWithEmbedding":{__init__:[5,2,1,""],_encode:[5,2,1,""],enc_emb_w:[5,4,1,""],get_optional_params:[5,3,1,""],get_required_params:[5,3,1,""],src_emb_size:[5,4,1,""],src_vocab_size:[5,4,1,""]},"losses.cross_entropy_loss":{CrossEntropyLoss:[6,1,1,""]},"losses.ctc_loss":{CTCLoss:[6,1,1,""],dense_to_sparse:[6,5,1,""]},"losses.ctc_loss.CTCLoss":{__init__:[6,2,1,""],_compute_loss:[6,2,1,""],get_optional_params:[6,3,1,""]},"losses.loss":{Loss:[6,1,1,""]},"losses.loss.Loss":{__init__:[6,2,1,""],_cast_types:[6,2,1,""],_compute_loss:[6,2,1,""],compute_loss:[6,2,1,""],get_optional_params:[6,3,1,""],get_required_params:[6,3,1,""],name:[6,4,1,""],params:[6,4,1,""]},"losses.sequence_loss":{BasicSequenceLoss:[6,1,1,""],CrossEntropyWithSmoothing:[6,1,1,""],PaddedCrossEntropyLossWithSmoothing:[6,1,1,""]},"losses.sequence_loss.BasicSequenceLoss":{__init__:[6,2,1,""],_compute_loss:[6,2,1,""],get_optional_params:[6,3,1,""],get_required_params:[6,3,1,""]},"losses.sequence_loss.CrossEntropyWithSmoothing":{__init__:[6,2,1,""],_compute_loss:[6,2,1,""],get_optional_params:[6,3,1,""],get_required_params:[6,3,1,""]},"losses.sequence_loss.PaddedCrossEntropyLossWithSmoothing":{get_optional_params:[6,3,1,""]},"models.encoder_decoder":{EncoderDecoderModel:[7,1,1,""]},"models.encoder_decoder.EncoderDecoderModel":{__init__:[7,2,1,""],_build_forward_pass_graph:[7,2,1,""],_create_decoder:[7,2,1,""],_create_encoder:[7,2,1,""],_create_loss:[7,2,1,""],decoder:[7,4,1,""],encoder:[7,4,1,""],get_optional_params:[7,3,1,""],get_required_params:[7,3,1,""],loss_computator:[7,4,1,""]},"models.image2label":{Image2Label:[7,1,1,""]},"models.image2label.Image2Label":{evaluate:[7,2,1,""],finalize_evaluation:[7,2,1,""],get_num_objects_per_step:[7,2,1,""],maybe_print_logs:[7,2,1,""]},"models.model":{Model:[7,1,1,""]},"models.model.Model":{__init__:[7,2,1,""],_build_forward_pass_graph:[7,2,1,""],clip_last_batch:[7,2,1,""],compile:[7,2,1,""],evaluate:[7,2,1,""],finalize_evaluation:[7,2,1,""],finalize_inference:[7,2,1,""],get_data_layer:[7,2,1,""],get_num_objects_per_step:[7,2,1,""],get_optional_params:[7,3,1,""],get_output_tensors:[7,2,1,""],get_required_params:[7,3,1,""],get_tf_dtype:[7,2,1,""],hvd:[7,4,1,""],infer:[7,2,1,""],last_step:[7,4,1,""],maybe_print_logs:[7,2,1,""],mode:[7,4,1,""],num_gpus:[7,4,1,""],on_horovod:[7,4,1,""],params:[7,4,1,""],steps_in_epoch:[7,4,1,""]},"models.speech2text":{Speech2Text:[7,1,1,""],levenshtein:[7,5,1,""],sparse_tensor_to_chars:[7,5,1,""]},"models.speech2text.Speech2Text":{evaluate:[7,2,1,""],finalize_evaluation:[7,2,1,""],finalize_inference:[7,2,1,""],get_num_objects_per_step:[7,2,1,""],infer:[7,2,1,""],maybe_print_logs:[7,2,1,""]},"models.text2text":{Text2Text:[7,1,1,""],calculate_bleu:[7,5,1,""],transform_for_bleu:[7,5,1,""]},"models.text2text.Text2Text":{evaluate:[7,2,1,""],finalize_evaluation:[7,2,1,""],finalize_inference:[7,2,1,""],get_num_objects_per_step:[7,2,1,""],infer:[7,2,1,""],maybe_print_logs:[7,2,1,""]},"optimizers.automatic_loss_scaler":{AutomaticLossScaler:[9,1,1,""],BackoffScaler:[9,1,1,""],LogMaxScaler:[9,1,1,""]},"optimizers.automatic_loss_scaler.AutomaticLossScaler":{SUPPORTED_ALGOS:[9,4,1,""],check_grads:[9,3,1,""],loss_scale:[9,4,1,""],update_op:[9,2,1,""]},"optimizers.automatic_loss_scaler.BackoffScaler":{loss_scale:[9,4,1,""],update_op:[9,2,1,""]},"optimizers.automatic_loss_scaler.LogMaxScaler":{loss_scale:[9,4,1,""],update_op:[9,2,1,""]},"optimizers.lr_policies":{exp_decay:[9,5,1,""],fixed_lr:[9,5,1,""],piecewise_constant:[9,5,1,""],poly_decay:[9,5,1,""],transformer_policy:[9,5,1,""]},"optimizers.mp_wrapper":{MixedPrecisionOptimizerWrapper:[9,1,1,""],mp_regularizer_wrapper:[9,5,1,""]},"optimizers.mp_wrapper.MixedPrecisionOptimizerWrapper":{apply_gradients:[9,2,1,""],compute_gradients:[9,2,1,""]},"optimizers.optimizers":{DistributedOptimizer:[9,1,1,""],_adaptive_max_norm:[9,5,1,""],_add_scaled_noise_to_gradients:[9,5,1,""],_clip_gradients_by_norm:[9,5,1,""],_multiply_gradients:[9,5,1,""],_multiply_gradients_const:[9,5,1,""],adaptive_clipping_fn:[9,5,1,""],get_regularization_loss:[9,5,1,""],optimize_loss:[9,5,1,""]},"optimizers.optimizers.DistributedOptimizer":{__init__:[9,2,1,""],apply_gradients:[9,2,1,""],compute_gradients:[9,2,1,""]},"parts.rnns":{attention_wrapper:[11,0,0,"-"],flstm:[11,0,0,"-"],glstm:[11,0,0,"-"],gnmt:[11,0,0,"-"],rnn_beam_search_decoder:[11,0,0,"-"],slstm:[11,0,0,"-"],utils:[11,0,0,"-"]},"parts.rnns.attention_wrapper":{AttentionMechanism:[11,1,1,""],AttentionWrapper:[11,1,1,""],AttentionWrapperState:[11,1,1,""],BahdanauAttention:[11,1,1,""],BahdanauMonotonicAttention:[11,1,1,""],LuongAttention:[11,1,1,""],LuongMonotonicAttention:[11,1,1,""],hardmax:[11,5,1,""],monotonic_attention:[11,5,1,""],safe_cumprod:[11,5,1,""]},"parts.rnns.attention_wrapper.AttentionMechanism":{alignments_size:[11,4,1,""],state_size:[11,4,1,""]},"parts.rnns.attention_wrapper.AttentionWrapper":{__init__:[11,2,1,""],_item_or_tuple:[11,2,1,""],call:[11,2,1,""],output_size:[11,4,1,""],state_size:[11,4,1,""],zero_state:[11,2,1,""]},"parts.rnns.attention_wrapper.AttentionWrapperState":{clone:[11,2,1,""]},"parts.rnns.attention_wrapper.BahdanauAttention":{__init__:[11,2,1,""]},"parts.rnns.attention_wrapper.BahdanauMonotonicAttention":{__init__:[11,2,1,""]},"parts.rnns.attention_wrapper.LuongAttention":{__init__:[11,2,1,""]},"parts.rnns.attention_wrapper.LuongMonotonicAttention":{__init__:[11,2,1,""]},"parts.rnns.flstm":{FLSTMCell:[11,1,1,""]},"parts.rnns.flstm.FLSTMCell":{__init__:[11,2,1,""],call:[11,2,1,""],output_size:[11,4,1,""],state_size:[11,4,1,""]},"parts.rnns.glstm":{GLSTMCell:[11,1,1,""]},"parts.rnns.glstm.GLSTMCell":{__init__:[11,2,1,""],_get_input_for_group:[11,2,1,""],call:[11,2,1,""],output_size:[11,4,1,""],state_size:[11,4,1,""]},"parts.rnns.gnmt":{GNMTAttentionMultiCell:[11,1,1,""],gnmt_residual_fn:[11,5,1,""]},"parts.rnns.gnmt.GNMTAttentionMultiCell":{__init__:[11,2,1,""]},"parts.rnns.rnn_beam_search_decoder":{BeamSearchDecoder:[11,1,1,""],BeamSearchDecoderOutput:[11,1,1,""],BeamSearchDecoderState:[11,1,1,""],FinalBeamSearchDecoderOutput:[11,1,1,""],tile_batch:[11,5,1,""]},"parts.rnns.rnn_beam_search_decoder.BeamSearchDecoder":{__init__:[11,2,1,""],_maybe_merge_batch_beams:[11,2,1,""],_maybe_split_batch_beams:[11,2,1,""],_merge_batch_beams:[11,2,1,""],_split_batch_beams:[11,2,1,""],batch_size:[11,4,1,""],finalize:[11,2,1,""],initialize:[11,2,1,""],output_dtype:[11,4,1,""],output_size:[11,4,1,""],step:[11,2,1,""],tracks_own_finished:[11,4,1,""]},"parts.rnns.slstm":{BasicSLSTMCell:[11,1,1,""],_linear:[11,5,1,""]},"parts.rnns.slstm.BasicSLSTMCell":{__init__:[11,2,1,""],call:[11,2,1,""],output_size:[11,4,1,""],state_size:[11,4,1,""]},"parts.rnns.utils":{create_rnn_cell:[11,5,1,""]},"parts.transformer":{attention_layer:[12,0,0,"-"],beam_search:[12,0,0,"-"],beam_search_test:[12,0,0,"-"],common:[12,0,0,"-"],embedding_layer:[12,0,0,"-"],ffn_layer:[12,0,0,"-"],utils:[12,0,0,"-"]},"parts.transformer.attention_layer":{Attention:[12,1,1,""],SelfAttention:[12,1,1,""]},"parts.transformer.attention_layer.Attention":{call:[12,2,1,""],combine_heads:[12,2,1,""],split_heads:[12,2,1,""]},"parts.transformer.attention_layer.SelfAttention":{call:[12,2,1,""]},"parts.transformer.beam_search":{SequenceBeamSearch:[12,1,1,""],_StateKeys:[12,1,1,""],_expand_to_beam_size:[12,5,1,""],_flatten_beam_dim:[12,5,1,""],_gather_beams:[12,5,1,""],_gather_topk_beams:[12,5,1,""],_length_normalization:[12,5,1,""],_shape_list:[12,5,1,""],_unflatten_beam_dim:[12,5,1,""],sequence_beam_search:[12,5,1,""]},"parts.transformer.beam_search.SequenceBeamSearch":{_continue_search:[12,2,1,""],_create_initial_state:[12,2,1,""],_get_new_alive_state:[12,2,1,""],_get_new_finished_state:[12,2,1,""],_grow_alive_seq:[12,2,1,""],_search_step:[12,2,1,""],search:[12,2,1,""]},"parts.transformer.beam_search._StateKeys":{ALIVE_CACHE:[12,4,1,""],ALIVE_LOG_PROBS:[12,4,1,""],ALIVE_SEQ:[12,4,1,""],CUR_INDEX:[12,4,1,""],FINISHED_FLAGS:[12,4,1,""],FINISHED_SCORES:[12,4,1,""],FINISHED_SEQ:[12,4,1,""]},"parts.transformer.beam_search_test":{BeamSearchHelperTests:[12,1,1,""]},"parts.transformer.beam_search_test.BeamSearchHelperTests":{test_expand_to_beam_size:[12,2,1,""],test_flatten_beam_dim:[12,2,1,""],test_gather_beams:[12,2,1,""],test_gather_topk_beams:[12,2,1,""],test_get_shape_keep_last_dim:[12,2,1,""],test_shape_list:[12,2,1,""],test_unflatten_beam_dim:[12,2,1,""]},"parts.transformer.common":{LayerNormalization:[12,1,1,""],PrePostProcessingWrapper:[12,1,1,""]},"parts.transformer.common.LayerNormalization":{build:[12,2,1,""],call:[12,2,1,""]},"parts.transformer.embedding_layer":{EmbeddingSharedWeights:[12,1,1,""]},"parts.transformer.embedding_layer.EmbeddingSharedWeights":{build:[12,2,1,""],call:[12,2,1,""],linear:[12,2,1,""]},"parts.transformer.ffn_layer":{FeedFowardNetwork:[12,1,1,""]},"parts.transformer.ffn_layer.FeedFowardNetwork":{call:[12,2,1,""]},"parts.transformer.utils":{get_decoder_self_attention_bias:[12,5,1,""],get_padding:[12,5,1,""],get_padding_bias:[12,5,1,""],get_position_encoding:[12,5,1,""]},"utils.funcs":{evaluate:[13,5,1,""],infer:[13,5,1,""],restore_and_get_results:[13,5,1,""],train:[13,5,1,""]},"utils.hooks":{BroadcastGlobalVariablesHook:[13,1,1,""],PrintLossAndTimeHook:[13,1,1,""],PrintSamplesHook:[13,1,1,""],RunEvaluationHook:[13,1,1,""]},"utils.hooks.BroadcastGlobalVariablesHook":{__init__:[13,2,1,""],after_create_session:[13,2,1,""],begin:[13,2,1,""]},"utils.hooks.PrintLossAndTimeHook":{after_run:[13,2,1,""],before_run:[13,2,1,""],begin:[13,2,1,""]},"utils.hooks.PrintSamplesHook":{after_run:[13,2,1,""],before_run:[13,2,1,""],begin:[13,2,1,""]},"utils.hooks.RunEvaluationHook":{after_run:[13,2,1,""],before_run:[13,2,1,""],begin:[13,2,1,""]},"utils.utils":{Logger:[13,1,1,""],array_to_string:[13,5,1,""],cast_types:[13,5,1,""],check_params:[13,5,1,""],clip_last_batch:[13,5,1,""],clip_sparse:[13,5,1,""],deco_print:[13,5,1,""],flatten_dict:[13,5,1,""],get_available_gpus:[13,5,1,""],get_git_diff:[13,5,1,""],get_git_hash:[13,5,1,""],get_results_for_epoch:[13,5,1,""],iterate_data_layer:[13,5,1,""],log_summaries_from_dict:[13,5,1,""],mask_nans:[13,5,1,""],nest_dict:[13,5,1,""],nested_update:[13,5,1,""],text_ids_to_string:[13,5,1,""]},"utils.utils.Logger":{flush:[13,2,1,""],write:[13,2,1,""]},data:{data_layer:[0,0,0,"-"],image2label:[1,0,0,"-"],speech2text:[2,0,0,"-"],text2text:[3,0,0,"-"],utils:[0,0,0,"-"]},decoders:{decoder:[4,0,0,"-"],fc_decoders:[4,0,0,"-"],rnn_decoders:[4,0,0,"-"]},encoders:{ds2_encoder:[5,0,0,"-"],encoder:[5,0,0,"-"],resnet_blocks:[5,0,0,"-"],resnet_encoder:[5,0,0,"-"],rnn_encoders:[5,0,0,"-"]},losses:{cross_entropy_loss:[6,0,0,"-"],ctc_loss:[6,0,0,"-"],loss:[6,0,0,"-"],sequence_loss:[6,0,0,"-"]},models:{encoder_decoder:[7,0,0,"-"],image2label:[7,0,0,"-"],model:[7,0,0,"-"],speech2text:[7,0,0,"-"],text2text:[7,0,0,"-"]},optimizers:{automatic_loss_scaler:[9,0,0,"-"],lr_policies:[9,0,0,"-"],mp_wrapper:[9,0,0,"-"],optimizers:[9,0,0,"-"]},parts:{rnns:[11,0,0,"-"],transformer:[12,0,0,"-"]},utils:{funcs:[13,0,0,"-"],hooks:[13,0,0,"-"],utils:[13,0,0,"-"]}},objnames:{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","method","Python method"],"3":["py","staticmethod","Python static method"],"4":["py","attribute","Python attribute"],"5":["py","function","Python function"]},objtypes:{"0":"py:module","1":"py:class","2":"py:method","3":"py:staticmethod","4":"py:attribute","5":"py:function"},terms:{"106gb":21,"1080ti":29,"16xlarg":28,"1e6":3,"1e9":12,"1x1":5,"224gb":21,"2xlarg":28,"4gpu":29,"55gb":21,"5gb":27,"8xlarg":28,"\u03b1":22,"\u03b4":22,"\u03b5":22,"\u03b6":22,"\u03ba":22,"abstract":[0,4,5,6,7],"boolean":[1,5,11],"byte":3,"case":[0,4,5,6,7,11,14,16,25,28],"char":0,"class":[0,1,2,3,4,5,6,7,9,11,12,13,16,25],"default":[3,5,6,9,11,13,25,27,28],"enum":3,"export":21,"final":[4,7,11,13,21,25],"float":[1,2,4,5,7,9,11,12,25,28],"function":[0,2,4,5,6,7,9,11,12,13,25,28],"import":[25,28],"int":[0,1,2,3,4,5,7,9,11,12,25],"long":11,"new":[0,1,3,9,11,12,13,23,25,26],"return":[0,1,2,3,4,5,6,7,9,11,12,13,16,28],"short":[11,25,29],"static":[0,1,2,3,4,5,6,7,9,11,16,25,28],"true":[0,1,3,5,6,7,9,11,12,14,25],"try":[9,22,25,27],"while":[1,3,16,22,25,28],ALS:9,AWS:28,Adding:23,And:[0,1,3,28],But:7,EOS:12,For:[1,2,3,4,5,7,9,11,13,16,21,22,25,26,28,29],IDs:3,Its:11,NOT:11,One:[22,25,28],Such:16,That:[0,1,3,4,7],The:[0,1,3,5,7,9,11,12,13,16,21,25,27,28,29],There:25,These:[1,4,7,11,13,28],Used:[3,7,12],Uses:[9,13,29],Using:23,With:[14,16,29],__init__:[0,1,2,3,4,5,6,7,9,11,13,16,25],_adaptive_max_norm:9,_add_scaled_noise_to_gradi:9,_aspect_preserving_res:1,_baseattentionmechan:11,_basemonotonicattentionmechan:11,_batch_exampl:3,_build_attent:4,_build_forward_pass_graph:7,_building_block_v1:5,_building_block_v2:5,_cast_typ:[4,5,6],_central_crop:1,_clip_gradients_by_norm:9,_compute_loss:6,_continue_search:12,_count_and_gen_subtoken:3,_count_token:3,_create_decod:7,_create_encod:7,_create_initial_st:12,_create_loss:7,_create_min_max_boundari:3,_decod:4,_decode_crop_and_flip:1,_distributed_appli:9,_encod:5,_escape_token:3,_expand_to_beam_s:12,_filter_and_bucket_subtoken:3,_filter_max_length:3,_flatten_beam_dim:12,_gather_beam:12,_gather_topk_beam:12,_gather_tre:11,_gen_new_subtoken_list:3,_generate_alphabet_dict:3,_generate_subtoken:3,_generate_subtokens_with_target_vocab_s:3,_get_example_length:3,_get_input_for_group:11,_get_new_alive_st:12,_get_new_finished_st:12,_grow_alive_seq:12,_item_or_tupl:11,_join_tokens_to_str:3,_length_norm:12,_linear:11,_list_to_index_dict:3,_load_record:3,_load_vocab_fil:3,_maybe_merge_batch_beam:11,_maybe_split_batch_beam:11,_mean_image_subtract:1,_merge_batch_beam:11,_monotonic_probability_fn:11,_multiply_gradi:9,_multiply_gradients_const:9,_native_to_unicod:3,_output:7,_parse_audio_el:2,_parse_audio_transcript_el:2,_parse_exampl:3,_parse_example_proto:1,_read_and_batch_from_fil:3,_resize_imag:1,_save_vocab_fil:3,_search_step:12,_shape_list:12,_smallest_size_at_least:1,_split_batch_beam:11,_split_string_to_token:3,_split_token_to_subtoken:3,_statekei:12,_subtoken_ids_to_token:3,_test:27,_token_to_subtoken_id:3,_unescape_token:3,_unflatten_beam_dim:12,_unicode_to_n:3,abl:[0,1,3,21,27],about:[13,29],abov:[11,22,28],abs:[5,9,11,29],absolut:28,acceler:11,accept:[3,11],access:[0,4,5,6,7],accord:11,accuraci:[1,7,27,28],achiev:[11,27],across:[3,6,7,9,25,28],activ:[5,11,28],activation_fn:5,actual:[4,7,11,25],adagrad:[7,9,25],adagradoptim:9,adam:[7,9,25,29],adamoptim:9,adapt:[5,7,9,25],adaptive_clip:9,adaptive_clipping_fn:9,add:[3,4,5,7,9,11,13,25,28],add_eo:3,added:[0,3,7,11,12,13,25],adding:13,addit:[2,7,9,11,12,21,25,28,29],addition:[1,25],adjust:[7,9,22,25,28],after:[0,1,5,9,11,13,21,22,25,27],after_create_sess:13,after_run:13,again:[25,27],aggreg:7,aggregation_method:9,aggregationmethod:9,alben:28,algorithm:[7,9,25,28],align:11,alignment_histori:11,alignments_s:11,aliv:12,alive_cach:12,alive_log_prob:12,alive_seq:12,all:[0,1,2,3,4,5,6,7,9,11,12,13,16,22,25,27,28,29],allow:[3,22,29],allreduc:9,along:[5,11],alpha:12,alphabet:[3,4],alphabet_config_path:4,alreadi:[11,26],also:[0,1,3,7,9,13,14,22,25,26,27,28,29],altern:[9,27,28],alwai:[7,9,28],amax:9,amount:3,analog:7,analysi:2,ani:[3,4,5,7,9,11,12,13,25,28,29],anoth:[3,9,11],answer:[7,25],anymor:13,anyth:9,api:[2,26],appear:3,append:[3,25],appli:[3,5,9,11,12,28],apply_gradi:9,approach:[7,11,14,28],apt:[21,27],arbitrari:9,architectur:[22,28],archiv:21,aren:3,arg:[9,11,13],argmax:11,argument:[2,3,4,5,6,7,9,11,12,13,14,21,25],arithmet:28,around:[1,4,5,6,21,27,28],arrai:[2,3,7],arrang:1,array_op:11,array_to_str:13,articl:11,artifici:4,arxiv:[5,9,11,28,29],aspect:1,assign:[0,4],assum:[11,21,27],assumpt:[7,11],attend:11,attent:[4,11,12,22,29],attention_cel:11,attention_depth:11,attention_dropout:12,attention_lay:[8,10],attention_layer_s:11,attention_mechan:11,attention_or_cell_output:11,attention_st:11,attention_typ:4,attention_wrapp:[8,10],attentioninputwrapp:11,attentionmechan:11,attentionwrapp:11,attentionwrapperst:11,attribut:[7,25],audio:[2,7,21],audio_filenam:2,augment:2,augment_audio_sign:2,auto:6,automat:[4,5,6,7,9,25,27,29],automatic_loss_sc:[7,9,25,28],automatic_loss_scal:8,automaticlossscal:9,autoregress:12,avail:[9,22,25,26,28,29],averag:[6,7,9],average_across_timestep:6,avoid:[9,11],axi:[7,11],back:[12,28],backoff:[7,9,25,28],backoffscal:9,backpropag:28,backslash:3,bahadanau:11,bahdanau:[4,11],bahdanau_norm:4,bahdanauattent:11,bahdanaumonotonicattent:11,bandwidth:28,base:[0,1,2,3,4,5,6,7,9,11,12,13,14,21,22,25,26,27],base_model:25,base_param:[25,28],basic:[6,11,26],basic_sequence_loss:6,basicsequenceloss:6,basicslstmcel:11,batch:[3,4,5,6,7,9,11,12,16,25],batch_norm:5,batch_siz:[2,3,4,5,6,11,12],batch_size_per_gpu:[6,7,22,25,29],batches_per_epoch:9,bazel:27,bbox:1,beahvior:11,beam:[4,11,12,29],beam_indic:12,beam_search:[8,10],beam_search_decoder_output:11,beam_search_test:[8,10],beam_siz:12,beam_width:[4,11],beamsearch:11,beamsearchdecod:11,beamsearchdecoderoutput:11,beamsearchdecoderst:11,beamsearchhelpertest:12,beamsearchrnndecoderwithattent:4,becaus:[3,12,22,28],becom:[2,9],been:[9,11,12,13,28],befor:[3,5,9,11,13,22,28],before_run:13,begin:[3,11,13,28],begin_decay_at:9,behavior:[11,28],being:[3,23],below:[27,28,29],bench_start:25,bench_step:25,benchmark:[7,25,28],benefici:28,bengio:11,besid:7,best:[11,12,27],beta1:9,beta2:9,better:[1,12],between:[3,7,9,11,12],bhadanau:11,bia:[11,12],bias:11,bias_initi:11,bidir_rnn_encoder_with_emb:5,bidirect:29,bidirectionalrnnencoderwithembed:5,big:[27,29],bigger:28,bin:27,binari:[3,21,27],bind:29,bleu:[7,29],blob:12,block:[5,25],block_fn:5,block_lay:5,blue:22,bn_epsilon:5,bn_momentum:5,bn_regular:5,bodi:12,bool:[0,4,5,6,7,9,11,12,25],boost:27,bori:28,both:[1,7,11,25,28,29],bottleneck:5,bottleneck_block:5,bottleneck_block_v1:5,bottleneck_block_v2:5,bottom:11,bound:1,boundari:[3,9],boundary_scal:3,box:1,bpe_us:7,broadcast:13,broadcastglobalvariableshook:13,btl:29,bucket:3,buckets_max:3,buckets_min:3,buffer:1,build:[2,4,9,12,13,20,26,27,28],build_graph:[0,1,2,3],build_image_data:1,build_lm:21,build_pip_packag:27,building_block:5,building_block_v1:5,building_block_v2:5,built:[7,25,26],c_state:11,cach:12,calcul:[0,1,3,7,11,12],calculate_bleu:7,call:[4,5,6,7,9,11,12,13,16,28],callabl:[9,11],callback:13,can:[0,1,2,3,4,5,6,7,9,11,13,14,16,22,25,26,27,28,29],candiat:3,candid:3,cannot:[7,11,12,25],cast:[4,5,6,28],cast_typ:13,cat:22,cell:[4,5,11,29],cell_input_fn:11,cell_param:11,cell_stat:11,cell_typ:11,center:1,central:1,chang:[11,13,14,22,28],channel:[1,5],channels_first:5,channels_last:5,charact:[3,4],check:[7,11,25,26,27,28],check_grad:9,check_param:13,checkpoint:[7,11,13,25,29],child:25,cho:11,choos:[11,28],chosen:9,christoph:11,classic:7,clean:[21,29],cleaned_fil:22,clip:[7,9,25],clip_gradi:9,clip_last_batch:[7,13],clip_spars:13,clone:[11,27],close:[3,11,29],cloud:28,cmake:27,cnn:26,code:[3,7,11,25],coeffici:[9,28],colin:11,collect:[7,9,12,28],colloqui:1,coloc:9,colocate_gradients_with_op:9,color:1,colorspac:1,column:11,com:[12,27],combin:[7,9,12,26],combine_head:12,command:[14,21,22,25,27,29],comment:29,commit:25,common:[8,10,28],commonli:28,compar:[5,28],compat:9,compil:[7,25],complet:[7,9,17,18,19,24,25,27],complex:28,compon:11,compos:11,compress:21,comput:[1,6,7,9,11,12,28],compute_gradi:[9,28],compute_loss:[6,7,13],concat:11,concaten:[7,11],concret:28,config:[0,1,2,3,4,5,6,7,9,13,14,23,27,29],config_fil:[21,22,25,27,29],configur:[7,14,22,25,26,27,29],conflict:11,conjunct:[7,25],connect:[4,5,7,12],conrib:11,consecut:3,consist:[4,13],constant:[9,28],constraint:11,construct:[0,1,3,4,5,6,7,9,11,13],constructor:[0,2,4,5,6,7,9,11,25],consumpt:28,contain:[0,1,2,3,4,5,6,7,9,11,12,13,22,25,27,28,29],content:[1,4,5,6],context:11,continu:[12,25],continue_learn:25,contrib:11,control:[7,11],conv2d:5,conv2d_bn_actv:5,conv2d_fixed_pad:5,conv_lay:5,conveni:25,convent:5,converg:28,convert:[2,3,11,21,28],convolut:[5,29],coord:[1,13],coordin:[1,13],copi:[5,7,11,28],copt:27,core:[3,28],correct:[7,11,12,21,22,25],correctli:[7,27],correspond:[0,1,3,7,9,11,12,16,22,25,27,28],correspondingli:[9,16,29],cosin:12,could:[0,2,4,5,6,7,9,25],count:[0,3,4,25],counter:9,cover:25,cpu:[3,28],creat:[0,1,3,4,5,6,7,9,11,12,13,20,25,28],create_rnn_cel:11,create_toy_data:22,creation:[7,28],crop:1,crop_height:1,crop_width:1,cross:6,cross_entropy_loss:8,cross_entropy_with_smooth:6,crossentropyloss:6,crossentropywithsmooth:6,csv:[2,21],ctc:[4,6],ctc_decoder_with_lm:27,ctc_greedy_decod:4,ctc_loss:8,ctcloss:6,cuda:[27,28],cudnn:29,cudnn_gru:5,cudnn_lstm:5,cudnnlstm:11,cumprod:11,cumsum:11,cumul:11,cur_index:12,current:[1,4,5,7,9,11,12,25,29],custom:[27,28],cut:[4,7],d_model:9,dai:29,data:[1,2,3,4,5,6,7,8,11,13,15,21,22,23,25,28],data_fil:3,data_format:5,data_lay:[1,2,3,4,5,6,7,8,25],data_layer_param:[7,25],data_root:22,datalay:[0,1,2,3,7,16,25],dataset:[0,1,2,3,7,16,20,22],dataset_fil:2,david:28,dct:13,debug:25,debug_port:[13,25],debugger_port:25,dec:5,decai:[9,28,29],decay_r:9,decay_step:9,deco_print:13,decod:[0,1,3,5,6,7,8,11,12,15,21,22,23,25,26,29],decode_and_crop:1,decoder_cell_typ:4,decoder_cell_unit:4,decoder_dp_input_keep_prob:4,decoder_dp_output_keep_prob:4,decoder_initial_st:11,decoder_library_path:4,decoder_output:6,decoder_param:[4,7],decoder_use_skip_connect:4,decreas:[3,28],deep:[5,11,28],deepbench:28,deepspeech2encod:[5,25],deepspeech:[5,25,27],defaultdict:3,defin:[0,1,3,4,5,7,9,11,12,25,28],definit:5,degre:28,delet:21,delim:[7,13],denomin:[7,25],denot:1,dens:[7,9,11],dense_tensor:6,dense_to_spars:6,depend:[5,11],deprec:11,depth:[1,11,26],deriv:[0,4,5,6,7,25],describ:[0,1,3,4,5,6,7,11,25,27,28,29],descript:[0,1,2,3,4,5,6,7,25,29],design:[26,28],desir:28,detail:[1,2,4,5,7,9,11,25,26,29],determin:[11,12],determinist:9,dev:[21,22,27],deviat:11,devic:[9,13],device_dens:9,device_spars:9,diamo:28,dict:[0,1,2,3,4,5,6,7,9,12,25],dict_to_log:13,dictionari:[0,1,2,3,4,5,6,7,12,16,25],did:22,diederik:11,diff:25,differ:[1,3,5,9,11,12,13,22,25,26,27,28],dim:[4,5,6,11],dimens:[1,4,5,11,12],dimension:[9,11],direct:[5,29],directori:[7,25,27],disabl:[7,11,22,25,27,28],discov:27,disk:22,displai:25,distanc:7,distort:1,distribut:[3,7,9,11,25,26,27,28],distributedoptim:9,divid:12,divis:[0,2,7,11],dl_id:13,dnn:28,do_mask:6,doc:[4,5,6,7,11,25],docker:[27,28],docstr:11,document:[11,25,26,28],doe:[1,4,6,7,11,22,25,27,28,29],doesn:1,domain:2,don:[3,12,29],done:[22,27],dot:12,dougla:11,download:[21,22],download_lm:27,downsampl:5,dp_input_keep_prob:11,dp_output_keep_prob:11,draw:12,drawn:3,dropout:[4,5],dropout_keep_prob:5,dropout_keep_prop:5,ds2_encod:[8,25],ds2_large_8gpu:29,ds2_librispeech_larc_config:21,ds2_medium_4gpu:29,ds2_small_1gpu:29,ds2_toy_data_config:[21,27],dtype:[0,4,5,6,7,9,11,13,25,28],due:11,dure:[1,3,4,7,11,12,13,22,25,28],dynam:[7,11,28],dynamic_decod:11,dzmitri:11,each:[0,1,3,6,7,9,11,12,13,16,25,28],eager:9,earli:11,easi:26,easili:22,eck:11,effect:[11,29],effici:[11,26],eight:29,either:[0,2,4,5,6,7,9,11,25,28],element:[2,3,7,9,11],elimin:29,els:[6,9],elsen:28,embed:[4,5,11,12],embedding_lay:[8,10],embedding_lookup:11,embedding_s:12,embeddingsharedweight:12,emit:11,emnlp:11,empti:[7,9,11,16,25],enabl:[7,9,11,14,25,27],enable_log:[25,29],enc_emb_w:5,encod:[0,1,3,4,6,7,8,11,12,15,22,23,25,26,29],encoder_cell_typ:5,encoder_cell_unit:5,encoder_decod:[4,5,6,8,25],encoder_dp_input_keep_prob:5,encoder_dp_output_keep_prob:5,encoder_final_st:11,encoder_lay:5,encoder_output:[4,5,11],encoder_param:[5,7],encoder_sequence_length:4,encoder_st:[5,11],encoder_use_skip_connect:5,encoderdecodermodel:[7,25],encorc:11,end:[3,4,11,12,13,22,27,28],end_compat:9,end_learning_r:9,end_of_choic:3,end_symbol:4,end_token:11,energi:11,enforc:11,english:[3,20],enough:[22,25,28],ensur:[3,11,12,13,28],entri:11,entropi:6,enumer:3,eos:12,eos_id:[3,12,13],epoch:[0,1,3,7,9,25,29],epsilon:[5,7,9,12,25],equal:[1,7,11,25],equival:[9,11],erich:28,error:[7,11,21,27,29],escap:3,especi:28,essenti:13,estim:28,etc:[4,5,7,16,25,26],etl:16,eval:[1,4,5,7,21,25],eval_input_fn:3,eval_model:13,eval_param:25,eval_step:[7,25],evalu:[0,1,2,3,7,13,21,22,25],evenli:3,event:[25,28],everi:[5,7],every_step:13,everyth:[21,25,26,27],exact:[11,25],exactli:11,exampl:[0,1,2,3,4,5,7,9,11,16,21,22,25,27,28],example_config:[21,22,25,27],example_seri:1,exce:[7,9,25],except:[5,11,13,25,27,28],execut:[7,9,14,22,25],exist:[0,4,11,23,28],exp:[9,11],exp_decai:9,expect:6,experi:[22,25,26],experiment:26,explicit:5,explicitli:[11,28],exponenti:9,exponential_decai:9,express:25,extend:26,extens:28,extra:9,extract:[2,16,21],fact_siz:11,factor:[9,11,12,28],fail:27,fairli:25,fals:[0,1,3,4,5,6,7,9,11,12,13,27],familiar:26,fc_decod:8,featur:[2,4,5,6,26],features_typ:2,fed:16,feed:[11,13],feed_dictionari:16,feedforward:12,feedfowardnetwork:12,feel:21,fetch:[7,9],few:29,ffn_layer:[8,10],field:[1,2,4,5,6,11],file:[0,1,2,3,4,7,21,22,25,27,29],file_byte_limit:3,file_pattern:3,file_with_bpe_segment:22,filenam:[1,2,3],filepath:3,filter:[3,5,9],filter_s:12,final_output:4,final_sequence_length:4,final_st:[4,11],finalbeamdecoderoutput:11,finalbeamsearchdecoderoutput:11,finalize_evalu:7,finalize_infer:7,find:[9,12,26],finish:[11,12,22,25],finished_flag:12,finished_scor:12,finished_seq:12,first:[0,1,3,5,7,9,11,12,14,21,22,25,28],fix:9,fixed_lr:9,fixed_pad:5,flag:[11,12],flaot:12,flat_dict:13,flatten_dict:13,flexibl:26,flip:1,float16:[0,4,5,6,7,25,28],float32:[0,4,5,6,7,9,12,25,28,29],flstm:[8,10],flstmcell:11,flush:13,folder:[21,22,25,27],follow:[0,1,3,4,5,6,7,9,16,21,22,25,27,28,29],forc:25,force_var_reus:7,forev:3,forget:[11,29],forget_bia:11,form:[0,3,11],format:[1,3,5],formul:12,forward:[7,28],found:[3,29],four:29,fp32:28,fraction:3,frame:[2,7],framework:12,free:21,frequenc:[2,3],frequent:3,from:[0,1,2,3,4,5,6,7,9,11,12,13,16,25,27,28,29],ftrl:[7,25],full:[5,9,28,29],fulli:[4,5,12],fully_connected_ctc_decod:4,fully_connected_decod:4,fully_connected_time_decod:4,fullyconnectedctcdecod:[4,21],fullyconnecteddecod:4,fullyconnectedtimedecod:4,func:8,furthermor:28,fuse:1,futur:12,ganesh:28,garcia:28,gate:[9,11],gate_gradi:9,gate_graph:9,gate_non:9,gate_op:9,gather:12,gen_input_tensor:16,gener:[3,4,7,11,12,25],generate_tri:27,geometr:12,german:[3,20],get:[7,9,12,16,21,22,26,27],get_available_gpu:13,get_data_lay:7,get_decoder_self_attention_bia:12,get_git_diff:13,get_git_hash:13,get_global_step:9,get_next:16,get_num_objects_per_step:7,get_optional_param:[0,1,2,3,4,5,6,7,16,25],get_output_tensor:7,get_pad:12,get_padding_bia:12,get_position_encod:12,get_regularization_loss:9,get_required_param:[0,1,2,3,4,5,6,7,16,25],get_results_for_epoch:13,get_size_in_sampl:[0,1,2,3,16],get_speech_featur:2,get_speech_features_from_fil:2,get_tf_dtyp:7,get_vari:28,get_wmt16_en_dt:22,getter:28,ginsburg:[11,28],git:[25,27],github:[12,27],given:[1,9,11,12,28],global:[9,13],global_gradient_norm:[7,25],global_step:9,glstm:[4,8,10],glstmcell:11,gnmt:[4,5,8,10,29],gnmt_encoder_with_emb:5,gnmt_residual_fn:11,gnmt_v2:4,gnmtattentionmulticel:11,gnmtlikeencoderwithembed:5,go_symbol:4,goal:22,going:[4,5,17,18,19,24,25,27],good:25,gpu:[0,6,7,9,13,14,16,22,25,26,27,28,29],gpu_id:[7,25],grad_loss:9,gradient:[7,9,11,25,28],gradient_multipli:9,gradient_noise_scal:9,gradient_norm:[7,25],grads_and_var:9,gram:21,graph:[0,1,2,3,4,5,6,7,9,13,25,28],graphic:28,graphkei:9,greater:5,gregori:28,group:[3,11],group_batch_s:3,group_id:11,group_siz:11,grow:12,gru:[4,5,29],guarante:[3,11],half:28,halv:28,handl:11,happen:[0,1,3,4,5,6,7,16],hard:11,hardmax:11,has:[0,1,2,3,4,5,6,7,9,11,12,13,28,29],has_nan:9,hash:25,hat:9,have:[0,1,2,3,4,5,6,7,9,11,12,16,22,25,26,27,28,29],head:12,height:1,height_in:5,help:[7,21,25],helper:[5,12],henc:28,here:[0,1,3,4,5,6,7,11,16,25,28,29],hetland:7,hidden:[4,5,11],hidden_s:12,hieu:11,high:28,higher:11,highest:12,highli:29,histogram:9,histori:11,hold:[9,12],hood:9,hook:[7,8],horovod:[0,3,6,7,9,13,14,25,29],horovod_gpu_allgath:9,horovod_gpu_allreduc:9,horovod_gpu_broadcast:13,hot:[1,6,11],houston:28,how:[7,9,11,12,20,22,23],howev:[1,28],http:[5,7,9,11,12,27,29],human:13,hvd:[7,25],hyperparamet:28,iclr:[11,28],icml:11,ident:[5,11],ids:[0,2,3,7,11,12,13,25],idx2char:7,ignor:[0,7,9,11,14,25],ignore_speci:[7,13],illeg:12,illustr:28,ilsvrc2012_val_00041207:1,imag:[1,5,7],image2label:[0,8,25],image_buff:1,imagenet_preprocess:[0,8],imagenetdatalay:1,implement:[6,7,9,11,12,16,25,29],impli:11,import_librivox:21,improv:[1,3,28],incept:1,includ:[0,1,2,3,4,5,6,7,9,11,21,28],increas:[12,28],increment:9,increment_global_step:9,independ:[5,6,7,25],index:[11,12],indexedslic:9,indic:[1,3,12,13],inf:11,infer:[0,1,3,4,5,7,11,13,21,25],infer_output_fil:[21,22,25],infer_param:25,infin:12,info:9,inform:[11,12,13,25,26],inherit:[0,4,5,6,7,16],init_from_fil:3,initi:[3,4,5,7,9,11,12,13,25,28],initial_cach:12,initial_cell_st:11,initial_id:12,initial_st:11,initializer_param:[4,5,7,25],inner:[11,12],input:[0,1,2,3,4,5,6,7,11,12,25,26],input_dict:[4,5,6,13],input_lay:5,input_s:11,input_sequence_length:11,input_tensor:[0,1,2,3,4,5,6,7],input_typ:2,input_valu:7,insid:[0,1,3,7,22,28],inspect:28,inspir:11,instabl:11,instal:[21,26],instanc:[0,4,5,6,7,9,11,28],instanti:9,instead:[1,4,9,11,27,28],instruct:[21,26],insur:12,int32:[1,11,12],int64:[11,12],intact:5,integ:[1,3,5,11],inter:28,intermedi:28,intern:[9,11,23],introduc:[1,5,28],invalid:9,invalidargu:11,invari:12,invers:3,involv:12,is_train:1,issu:29,item:[3,11,12],iter:[0,1,2,3,7,13,16,28],iterate_data_lay:13,its:[3,11,12,21,25],jian:5,join:3,jointli:11,jonah:28,jpeg:1,jul:5,just:[4,5,27],kaim:5,keep:[4,5,6,9,13,28],kei:[0,1,3,5,7,9,12],kenlm:[21,27],kept:28,kernel:5,kernel_initi:11,kernel_s:5,key_channel:12,keyword:12,kind:25,kingma:11,knee:1,known:[1,3,11],kpu:27,kuchaev:28,kuchaiev:11,kwarg:[9,11,12],kyunghyun:11,label:[1,4,6],lambda:[9,11,28],languag:[4,20],language_model:27,lar:[7,25],larc:[7,9,25,29],larc_mod:[7,9,25],larc_nu:[7,9,25],larc_param:[7,9,25],larg:[11,28],largest:12,last:[4,5,7],last_batch:[7,13],last_step:[7,13],latenc:28,later:28,latter:11,launch:[13,22],layer:[0,1,2,3,4,5,7,9,11,12,15,23,25,28,29],layer_typ:5,layernorm:12,layernorm_lstm:5,ld_library_path:29,lead:27,learn:[5,7,9,11,22,25,29],learnabl:11,learning_r:[7,9,25],learning_rate_decay_fn:9,least:[5,12],left:11,length:[0,1,2,3,4,5,6,11,12],length_i:12,length_penalty_weight:11,length_x:12,less:[3,28,29],level:[25,28],levenshtein:7,libboost:27,libctc_decoder_with_kenlm:27,librari:4,librispeech:[20,29],librivox:21,libsox:21,libtensorflow_cc:27,libtensorflow_framework:27,like:[5,9,12,16,22,29],limit:28,line:[0,3,13,14,25],linear:[4,11,12],link:[27,29],list:[0,1,2,3,4,5,6,7,9,11,12,16,25,26],liu:11,live:12,lm_binary_path:4,lm_trie_path:4,lm_weight:4,load:[0,3,16],load_pre_existing_vocabulari:0,localhost:29,locat:[12,22,27],lock:9,log:[7,9,11,12,23,28],log_fil:13,log_max:9,log_summaries_from_dict:13,logdir:[7,22,25],logger:13,logic:[7,12,16],logit:[4,6,11,12],logits_to_outputs_func:4,logmax:[7,9,25,28],logmaxscal:9,lognorm:28,logspac:11,longer:13,longest:3,look:[7,25,26,29],loop:12,lose:28,loss:[0,1,3,4,7,8,9,15,23,25],loss_comput:7,loss_input_dict:6,loss_param:7,loss_scal:[7,9,25,28],lot:[21,22,27],lower:[3,28],lr_polici:[7,8,25],lr_policy_param:[7,25],ls_dir:21,lst:3,lstm:[4,5,11,29],lstmstatetupl:11,luong:[4,11],luong_scal:4,luongattent:11,luongmonotonicattent:11,m_state:11,machin:[11,20],magnitud:9,mai:[3,11,12],main:[4,5,6,22,25,26],maintain:[12,28],major:11,make:[0,1,3,11,21,22,26,27],malform:9,man:11,mani:28,manner:11,manual:[11,27,28],map:[0,3,4,5,11,29],mark:[3,11,12],mask:[6,11,12],mask_nan:[6,13],master:[12,28],match:[1,3,5,11,28,29],matric:11,matrix:[11,12],max:3,max_decode_length:12,max_grad_norm:[7,25],max_length:3,max_lr:9,max_norm:9,max_pool2d:5,max_step:[7,25],max_subtoken_length:3,max_tim:11,max_timescal:12,maxim:9,maximum:[3,7,11,12,25,28],mayb:11,maybe_print_log:7,mca:29,mean:[1,9,28],measur:[22,29],mechan:[4,11,12],memori:[0,11,22,28],memory_sequence_length:11,mention:28,merg:11,method:[0,1,2,3,4,5,6,7,9,11,12,13,16,25,28],methodnam:12,methodolog:28,metric:7,mfcc:2,michael:28,micikeviciu:28,might:[22,25,27,28],milli:2,min:3,min_boundari:3,min_count:3,min_idx:0,min_lr:9,min_timescal:12,min_upd:[7,25],minh:11,mini:16,minibatch:11,minim:[7,9,25],minimum:[0,3,11,12],minumum:3,minut:27,mismanag:11,misspel:21,mix:[4,5,7,9,11,12,25,26,29],mixedprecisionoptimizerwrapp:[9,28],mkdir:[21,27],modal:26,mode:[0,1,3,4,5,7,11,13,14,21,22,25,27,29],model:[0,1,2,3,4,5,6,8,9,11,12,13,20,22,23,26,28],model_param:28,modifi:[4,5,6,9,13,28],modul:[4,7,9,11,25],modular:26,momentum:[5,7,9,25,29],momentumoptim:9,monoton:11,monotonic_attent:11,more:[9,11,12,22,26,28,29],moreov:25,moss:[22,29],most:[3,7,16,25,28],move:[9,11,25],mozilla:27,mp_regularizer_wrapp:[9,28],mp_wrapper:8,mpi4pi:27,mpi:7,mpirun:[14,29],msg:13,much:[3,12,22,27],multi:[7,12,14,16,22,26,29],multicel:11,multihead:12,multipl:[9,11,12,29],multipli:[9,11,28],multirnncel:11,must:[0,1,3,4,5,6,9,11,12,25],mutli:14,myfavoriteattentionmechan:11,n03623198:1,n_hidden:5,name:[2,3,4,5,6,7,9,11,25,28],namedtupl:11,nan:6,narang:28,nativ:3,nearli:28,necessari:[0,5,7,9,13,25,27,28],necessarili:12,need:[0,1,3,4,5,7,9,12,14,16,21,22,25,27,28],neg:[11,12],neither:9,nest:[11,12,25],nest_dict:13,nested_upd:13,network:[5,11,12,28],neural:[11,28],new_beam_s:12,new_cach:12,new_height:1,new_log_prob:12,new_seq:12,new_width:1,newli:11,newstest2014:[22,29],next:[7,11,25,27,28,29],next_batch_feed_dict:16,next_input:11,next_stat:11,nmt:[22,29],nmt_revers:22,no_dir_check:25,noam:9,node:[14,26],nois:[9,11],noise_level_max:2,noise_level_min:2,non:[9,11,12],none:[0,2,3,4,5,6,7,9,11,12,13,16,25,28,29],nor:9,norm:[5,7,9,25],normal:[4,5,9,11,12,28],note:[0,1,3,4,5,6,7,9,11,21,22,25,28],now:[11,21,25,27,28],num:[4,5,6],num_audio_featur:2,num_box:1,num_channel:[1,5],num_cpu_cor:3,num_epoch:[0,1,3,7,9,25],num_featur:[2,4],num_gpu:[7,14,22,25],num_head:12,num_iter:3,num_lay:11,num_proj:11,num_rnn_lay:5,num_time_step:2,num_unit:11,num_work:[0,1,2,3,16],number:[0,1,2,3,4,5,6,7,9,11,12,14,25,28],number_of_group:11,numer:[7,11,25,28],numpi:2,nvidia:[26,27,28],object:[0,1,3,4,5,6,7,9,11,12,13,16,28],obtain:[27,28,29],occur:11,offici:26,offset:13,offset_target_by_on:6,often:[7,25,28],old:12,oleksii:28,on_horovod:[7,9],onc:[3,11,13,22],one:[0,1,3,4,5,6,7,9,11,12,13,22,25,28,29],ones:26,onli:[0,1,3,4,7,9,11,13,14,25,28],onlin:11,open:7,open_seq2seq:[1,2,3,7,27],openib:29,openseq2seq:[3,14,21,22,25,28,29],oper:[4,5,9,11,13,27,28],ops:[1,9,11,13],opt:27,optim:[7,8,25,29],optimize_loss:9,optimizer_cls_nam:9,optimizer_param:[7,9,25],optimizer_summari:9,option:[0,1,2,3,4,5,6,7,9,11,16,25],optional_dict:13,order:[3,5,9,11,21,27,28],org:[5,7,9,11,29],org_dict:13,origin:[1,3,5,11,12,13,21,29],orte_base_help_aggreg:29,other:[1,4,7,9,11,13,21,25,27,28],otherwis:[1,7,9,11,25,27,28],our:[26,27,28],out:[11,12,22,26],out_of_bucket:3,output:[1,4,5,6,7,11,12,13,16,21,22,25,26,27],output_attent:11,output_dim:[4,11],output_dir:13,output_dtyp:11,output_fil:[7,13],output_height:1,output_lay:11,output_s:11,output_time_major:11,output_valu:7,output_width:1,outsid:3,over:[6,28],overal:13,overcom:28,overflow:28,overflow_std_dev:9,overrid:11,overridden:11,overriden:9,overwrit:[7,25],overwritten:25,own:20,p_choose_i:11,packag:[4,5],pad2eight:12,pad:[0,1,2,3,5,12,16,29],pad_id:[3,13],pad_to:2,pad_vocab_to_eight:0,padded_cross_entropy_with_smooth:6,padded_input_length:3,padded_length:3,padded_target_length:3,paddedcrossentropylosswithsmooth:6,padding_valu:12,page:[26,27],pair:[7,9,11],paper:22,parallel:[3,11,12,22],parallel_interleav:3,paralleltextdatalay:3,param:[0,1,2,3,4,5,6,7,9,11,12,13,16,25],paramet:[0,1,2,3,4,5,6,7,9,11,12,13,14,16,22,23,27,28],parent:[0,2,4,5,6,25],pars:[1,2],parse_record:1,part:[1,4,6,7,8,9,11,12,21,25,28],particular:11,partli:11,pass:[0,1,3,4,5,6,7,9,11,12,13,25,28],past:11,path:[0,2,3,4,7,25,29],pauliu:28,pdf:[5,9],penal:11,per:[2,3,6,9],perform:[1,2,4,5,6,7,11,22,25,28],period:[25,28],perl:[22,29],peter:11,pham:11,piecewis:9,piecewise_const:9,pip:27,pip_packag:27,pipelin:3,place:[3,25],placehold:16,plane:5,pleas:[11,21],point:[3,11,13,22,28],polici:[7,9,25],poly_decai:9,polynomi:9,polynomial_decai:9,popul:[7,9],posit:[5,11,12],possibl:[7,11,25,28,29],post:12,power:[9,11],practic:28,pre:[0,11,12,22],preactiv:5,precis:[4,5,7,25,26,29],pred:7,predict:[7,11,12,13,25],predicted_id:11,preevious_attent:11,prefer:27,prefix:[3,9],prepar:[11,25],prepostprocessingwrapp:12,preprint:28,preprocess:[1,21],preprocess_imag:1,presenc:28,present:9,preserv:1,prevent:9,previou:[9,11,12],previous_attent:11,primarili:1,principl:28,print:[7,13,25],print_loss_step:[7,25],print_samples_step:[7,25],printlossandtimehook:13,printsampleshook:13,prior:[3,11],probability_fn:11,probabl:[4,5,11,12,25,28],problem:[11,22,28],proce:25,process:[0,1,2,3,9,11,12,13,22,27],produc:[4,5,6,11],product:[11,12,26],progress:22,project:[5,11,26],projection_shortcut:5,propag:[11,28],proper:11,properli:11,properti:[11,12],propos:[5,11],proto:1,protocol:1,provabl:12,proven:28,provid:[1,3,4,5,7,9,11,13,21,28],pull:12,put:27,python:[1,3,4,5,7,9,11,12,13,16,21,22,25,27,29],quantiti:9,queri:11,quicker:9,raffel:11,rais:[1,9,11,13],random:[1,3,7,11,13,25],random_se:[7,25],randomli:1,rang:[11,28],rank:[1,3,9,11,13],rare:28,rate:[7,9,25,29],rather:[0,5,7,25,28],ratio:1,raw:[1,2,5],raw_record:1,raw_str:3,reach:12,read:[0,3,11,25],read_char:0,readabl:13,real:22,realli:4,reason:11,receiv:13,recent:28,recip:[26,28],recogn:21,recognit:[5,7,20,27],recommend:[9,11,16,27,28,29],record:[1,3],recov:13,recurr:[11,28],recurs:11,redefin:28,reduc:[11,22],reduce_mean:6,ref:5,refer:[1,3,12],regress:6,regular:[4,5,7,9,25],regularizer_param:[4,5,7,25],relat:[0,1,3,11],relu:5,relu_dropout:12,remov:[7,22],ren:5,reparameter:11,repeat:[3,11],replac:[3,11,21],report:9,report_summari:9,repositori:27,repres:[1,11,12],represent:[4,5],representation_dim:5,request:13,request_stop:13,requir:[0,1,2,3,4,5,6,7,9,11,16,25,27,28],required_dict:13,rescal:9,research:26,reserv:3,reserved_token:3,reshap:[11,12],residu:[4,5,11],residual_connect:11,resiz:1,resize_imag:1,resize_min:1,resized_imag:1,resizemethod:1,resnet:[1,5],resnet_block:8,resnet_encod:8,resnetencod:5,respect:[11,28],rest:27,restor:[11,13,25],restore_and_get_result:13,result:[7,11,12,13,21,22,27,28,29],results_per_batch:7,retriev:28,reus:[11,28],revers:20,rgb:1,right:11,rmsprop:[7,25],rnn:[4,5,8,10,22,26,29],rnn_beam_search_decod:[8,10],rnn_cell:5,rnn_cell_dim:5,rnn_cell_impl:11,rnn_decod:8,rnn_decoder_with_attent:4,rnn_encod:8,rnn_type:5,rnn_unidirect:5,rnncell:11,rnndecoderwithattent:4,robust:28,ron:11,root:[13,29],root_rank:13,row:[5,7,11,13],row_conv:5,row_conv_width:5,rule:28,run:[4,5,7,11,12,13,14,21,23,28,29],run_context:13,run_valu:13,runevaluationhook:[7,13],runtest:12,runtim:11,runtimeerror:9,s_id:[3,13],safe:25,safe_cumprod:11,safeti:9,sai:25,saliman:11,same:[0,1,3,4,5,6,9,11,12,13],sampl:[0,1,3,4,7,11,13,16,25],save:[3,7,25],save_checkpoint_step:[7,25],save_summaries_step:[7,25],scalar:[1,9,11,12],scale:[3,4,7,9,11,12,25],scale_max:9,scale_min:9,scaler:9,scan:11,scheme:[3,9],scope:[4,5,6,9,11],score:[7,11,12,29],score_bias_init:11,score_mask_valu:11,score_or_log_prob:12,script:[1,14,21,22,25,27,29],search:[3,4,11,12],second:[2,3,9,11,13,28],section:[4,5,7,9,12,17,18,19,24,25,26,27,29],sed:22,see:[2,4,5,6,7,9,11,22,25,26,27,28],seed:[7,11,25],select:[7,28],self:[0,1,3,4,5,6,7,11,12,16,25],selfattent:12,semi:3,send:13,separ:[1,25],seq2seq:[6,11],seq:11,sequenc:[0,1,2,3,4,5,6,7,11,12,20,25,26,27],sequence_beam_search:12,sequence_length:[6,11],sequence_loss:8,sequencebeamsearch:12,seri:1,serial:[1,3],serialized_exampl:3,sess:[7,13],session:13,session_run_hook:13,sessionrunarg:13,sessionruncontext:13,sessionrunhook:13,sessionrunvalu:13,set:[3,4,5,6,7,9,11,13,14,25,26,27,28],setup:[21,29],sgd:[7,9,25,29],shaoq:5,shape:[1,2,3,4,5,6,9,11,12,16],sharan:28,share:[12,25,28],shift:28,shortcut:5,shorter:12,should:[0,1,3,4,5,6,7,9,11,12,13,16,21,22,25,27,28,29],shuffl:[0,3,11,16],side:1,sigmoid:11,sigmoid_nois:11,sigmoid_noise_se:11,signal:[2,13],signatur:11,significantli:3,similar:[3,5],simpl:[1,4,11,14,22,25,28],simplest:27,sinc:[7,11,21,25,27,28],sine:12,singl:[3,5,9,11,12,22,29],singleton:6,singular:11,situat:[13,28],size:[0,1,2,3,4,5,6,7,11,12,13,16,21,25,27,28,29],skip:[25,27,28],slice:11,sloppi:3,slot:29,slowest:11,slstm:[8,10],small:[7,9,11,22,25,28,29],smallest:1,smallest_sid:1,smooth:[6,9],softmax:[6,11,12],solut:11,some:[1,4,5,7,9,11,12,25,26,27,28],someth:27,sometim:25,soon:[11,17,18,19,24,25],sort:3,sourc:[0,1,2,3,4,5,6,7,9,11,12,13,25,27],source_length:2,source_sequ:2,source_tensor:[0,1,2,3,5,7],sox:21,spars:[7,9],sparse_tensor_to_char:7,sparsemax:11,sparsetensorvalu:7,spatial:5,specialtexttoken:3,specif:[2,4,5,25,27],specifi:[2,7,9,12,14,16,25],spectrogram:2,speech2text:[0,8,21,25,27],speech2textdatalay:2,speech:[2,7,20,26],speech_util:[0,8],speed:[3,28],speedup:28,split:[2,3,4,11,12],split_data:[1,2],split_head:12,src:22,src_emb_siz:5,src_input:[4,5],src_length:[4,5,6],src_sequenc:5,src_vocab_s:5,stabil:[7,25],stack:11,stai:28,staircas:9,stamp:25,standard:[5,7,11,28],start:[9,11,12,13,14,22,25,26],start_input:11,start_token:11,state:[4,5,11,12],state_is_tupl:11,state_s:11,static_max_norm:9,statist:[9,28],std:9,std_factor:9,stderr:25,stdout:25,step:[1,7,9,11,13,25,27],step_factor:9,step_window:9,steps_in_epoch:7,steps_per_epoch:9,still:12,stop:[11,13],store:[3,7,11,12],str:[0,2,3,4,5,6,7],stream:13,strength:12,stride:[2,5],string:[1,2,3,4,5,7,9,13,25],structur:[11,12,23],style:11,sub:9,subclass:9,subfold:25,submit:28,subsequ:[1,11,25],subset:[11,29],substitut:29,subtoken:[3,12],subtoken_count:3,subtoken_dict:3,subtoken_list:3,subtract:1,sudo:[21,27],suffer:11,suggest:28,sum:[6,11],sum_i:11,summar:28,summari:[7,9,25],sun:5,suppli:9,support:[0,4,5,6,7,11,14,16,25,26,27,28],supported_algo:9,sure:[0,1,3,21,22],symbol:[0,4],symbols_to_logits_fn:12,symlink:27,synset:1,system:22,t2t:[0,8],tab:25,tabl:29,taht:[0,1,3],take:[4,5,6,9,11,12,13,21,22,27],taken:7,tanh:11,target:[0,1,2,3,4,6,7,13,22],target_length:2,target_s:3,target_sequ:[2,6],target_tensor:[0,1,2,3,4,6,7],target_vocab_s:3,task:20,tensor2tensor:12,tensor:[0,1,2,3,4,5,6,7,9,11,12,13,16,28],tensorarrai:11,tensorboard:[7,9,22,25],tensorflow:[0,1,3,4,5,6,7,9,11,12,13,25,26,28],tensorflow_pkg:27,tensorflowtestcas:12,tensorshap:11,term:[9,11],termin:12,tesla:28,test:[12,22],test_expand_to_beam_s:12,test_flatten_beam_dim:12,test_gather_beam:12,test_gather_topk_beam:12,test_get_shape_keep_last_dim:12,test_shape_list:12,test_unflatten_beam_dim:12,test_util:12,text2text:[0,8,22,25],text:[1,2,3,4,7,13,21,26,27],text_ids_to_str:13,textlinedataset:2,tfrecord:3,tgt:22,tgt_emb_siz:4,tgt_input:4,tgt_length:[4,6],tgt_sequenc:6,tgt_vocab_s:[4,6],than:[1,3,5,9,11,12,22,28,29],thang:11,thei:[1,3,11,28],them:[9,21,27,28],thi:[0,1,3,4,5,6,7,9,11,12,13,16,17,18,19,21,22,24,25,26,27,28,29],thing:[3,9,22,25,28],those:[7,11],thread:13,three:[1,3],threshold:[3,9],through:[1,11,12,26],thu:[3,4,25],tile:[11,12],tile_batch:11,tiled_encoder_final_st:11,tiled_encoder_output:11,tiled_input:11,tiled_sequence_length:11,tim:11,time:[1,3,4,5,6,9,11,13,21,25,27,28],time_major:5,time_stretch_ratio:2,timestep:[6,11,28],titan:28,tmp:27,todo:[11,16,27],togeth:[7,13,28],toi:[20,21,27],tok:[22,29],token:[0,7,8,11,12],token_count:3,tool:27,toolkit:26,top:[3,11,12],topic:25,total:[0,9,28],total_regularization_loss:9,tower:[7,14],toy_text_data:22,tra:3,track:[11,13],tracks_own_finish:11,train:[0,1,3,4,5,7,9,11,12,13,20,22,25,26,27,29],train_ev:[7,21,22,25,27,29],train_input_fn:3,train_model:13,train_op:7,train_param:25,trainabl:9,trainable_vari:9,trainer:9,transform:[3,4,8,9,10,16,22,29],transform_for_bleu:7,transformer_decod:8,transformer_encod:8,transformer_polici:9,transformerdatalay:3,translat:[3,7,11,12,20],transpos:12,treat:[7,9,25],tri:29,trick:11,trie:[4,21,27],true_batch_s:11,true_siz:[7,13],tupl:[1,2,7,9,11,12],tutori:[25,26],twice:3,two:[1,3,11,12,13,14,21,28],txt:[22,27],type:[0,1,2,3,4,5,6,7,9,11,12,28],typeerror:[9,11],typic:[0,4,5,6,28],ubuntu:27,ultim:5,unbatch:3,unchang:[11,12],under:9,underflow:[11,28],undergo:1,underli:[2,9,28],underlin:3,understand:25,unescap:3,uni:5,unicod:3,unidir_rnn_encoder_with_emb:5,unidirect:29,unidirectionalrnnencoderwithembed:5,uniqu:12,unit:[4,5,11,29],unittest:27,unk_id:3,unknown:1,unless:9,unspecifi:11,upcom:13,upd_dict:13,updat:[3,7,9,11,25,28],update_op:9,use:[0,1,3,4,5,6,7,9,11,14,16,21,22,25,27,28,29],use_horovod:[7,14,25],use_language_model:[4,27],use_lock:9,use_new_attent:11,use_staircase_decai:9,use_swap_memori:5,used:[0,1,3,4,5,6,7,9,11,12,13,22,25,28,29],useful:[7,25,26],user:[11,28],uses:[6,9,11,21,22,28],using:[1,2,3,5,7,9,11,13,14,16,22,26,27,28,29],usual:[4,11,21,27,28],util:[1,7,8,10,28],utter:21,v100:28,valid:[0,1,3,4,5,7,9,11,13,25,29],valid_word_count_weight:4,valu:[1,3,7,9,11,12,13,25,28],value_channel:12,valueerror:[1,9,11],var_list:9,variabl:[3,4,5,6,7,9,11,12,13,25,28],variable_norm:[7,25],varianc:[9,28],variant:5,varieti:28,variou:[4,5,9,25,26],vector:[1,11],venkatesh:28,verbos:13,veri:[11,22],versa:11,version:[3,4,11,27,28],vgg:1,via:11,vice:11,view:[3,22],visual:[7,9],vocab:[0,3,7,13],vocab_fil:[2,3],vocab_s:12,vocabulari:[0,2,3,4,5,6,13,22],volta:[28,29],wai:[3,7,9,27,28],want:[21,22,25,27],warm:9,warmup_step:9,wave:2,wavelength:12,weight:[4,5,9,11,12,13,28],weiss:11,well:[4,5,7,22],wer:29,were:[5,11,22],what:23,when:[0,1,3,5,7,9,11,12,13,14,25,27,28,29],whenev:[25,28],where:[1,3,7,9,11,12,16,22,25,28],whether:[0,1,3,4,5,6,7,9,11,12,25],which:[0,1,3,4,5,6,7,9,11,12,13,16,22,25,28,29],whl:27,whose:11,width:[1,4,5,29],width_in:5,window:2,window_s:2,window_strid:2,within:5,without:[1,4,5,12,25,28,29],wmt:22,word:[0,3,4,7,29],word_count_weight:4,work:[7,22,25,27,29],worker:[3,6,7,13],worker_id:[0,1,2,3,7,16],workshop:11,wors:27,worst:[11,12],worth:28,wrap:[3,4,9,11,13,28],wrap_to_multi_rnn:11,wrapper:[1,4,5,6,11,12,28],write:[11,13,25],wrong:[9,11],xiangyu:5,xmax:1,xmin:1,ymax:1,ymin:1,yoshua:11,you:[0,1,3,7,9,11,12,13,14,16,21,22,25,26,27,28,29],your:[0,1,3,9,16,20,22,25,27],yourself:28,zero:[7,9,11,12],zero_st:11,zhang:5},titles:["data","image2label","speech2text","text2text","decoders","encoders","losses","models","API documentation","optimizers","parts","rnns","transformer","utils","Distributed training","Adding new models","Adding new data layer","Adding new decoder","Adding new encoder","Adding new loss","Getting started","Speech Recognition","Machine Translation","In-depth tutorials","Internal structure","Using existing models","OpenSeq2Seq","Installation instructions","Mixed precision training","Models and recipes"],titleterms:{"new":[15,16,17,18,19],Adding:[15,16,17,18,19],Using:25,add:27,adventur:22,api:8,attention_lay:12,attention_wrapp:11,automat:28,automatic_loss_scal:9,base:29,beam_search:12,beam_search_test:12,being:25,bleu:22,bpe:22,build:21,clean:22,common:12,comput:22,config:25,creat:22,cross_entropy_loss:6,ctc:27,ctc_loss:6,data:[0,16],data_lay:0,dataset:21,decod:[4,17,27],deep:29,depth:23,detail:28,distribut:14,document:8,download:27,ds2_encod:5,embedding_lay:12,enabl:28,encod:[5,18],encoder_decod:7,english:22,exist:25,fc_decod:4,feel:22,ffn_layer:12,flstm:11,func:13,gener:27,german:22,get:20,glstm:11,gnmt:11,hook:13,horovod:27,how:[21,25,27,28],image2label:[1,7],imagenet_preprocess:1,implement:28,infer:22,instal:27,instruct:27,intern:24,languag:[21,27],layer:16,librispeech:21,log:25,loss:[6,19,28],lr_polici:9,machin:[22,29],mix:28,model:[7,15,21,25,27,29],mp_wrapper:9,openseq2seq:[26,27],optim:[9,28],own:21,paramet:25,part:10,precis:28,prerequisit:28,recip:29,recognit:[21,29],regular:28,resnet_block:5,resnet_encod:5,revers:22,rnn:11,rnn_beam_search_decod:11,rnn_decod:4,rnn_encod:5,run:[22,25,27],scale:28,score:22,segment:22,sequenc:22,sequence_loss:6,slstm:11,speech2text:[2,7],speech:[21,27,29],speech_util:2,start:20,structur:24,t2t:3,task:22,tensorflow:27,test:27,text2text:[3,7],toi:22,token:3,train:[14,21,28],transform:12,transformer_decod:4,transformer_encod:5,translat:[22,29],tutori:23,util:[0,11,12,13],what:25,your:21}})
\ No newline at end of file
+Search.setIndex({docnames:["api-docs/data","api-docs/data.image2label","api-docs/data.speech2text","api-docs/data.text2text","api-docs/decoders","api-docs/encoders","api-docs/losses","api-docs/models","api-docs/modules","api-docs/optimizers","api-docs/parts","api-docs/parts.rnns","api-docs/parts.transformer","api-docs/utils","distr-training","extending","extending/adding-new-data-layer","extending/adding-new-decoder","extending/adding-new-encoder","extending/adding-new-loss","getting-started","getting-started/asr","getting-started/nmt","in-depth-tutorials","in-depth-tutorials/internal-structure","in-depth-tutorials/using-existing-models","index","installation-instructions","mixed-precision","models-and-recipes"],envversion:53,filenames:["api-docs/data.rst","api-docs/data.image2label.rst","api-docs/data.speech2text.rst","api-docs/data.text2text.rst","api-docs/decoders.rst","api-docs/encoders.rst","api-docs/losses.rst","api-docs/models.rst","api-docs/modules.rst","api-docs/optimizers.rst","api-docs/parts.rst","api-docs/parts.rnns.rst","api-docs/parts.transformer.rst","api-docs/utils.rst","distr-training.rst","extending.rst","extending/adding-new-data-layer.rst","extending/adding-new-decoder.rst","extending/adding-new-encoder.rst","extending/adding-new-loss.rst","getting-started.rst","getting-started/asr.rst","getting-started/nmt.rst","in-depth-tutorials.rst","in-depth-tutorials/internal-structure.rst","in-depth-tutorials/using-existing-models.rst","index.rst","installation-instructions.rst","mixed-precision.rst","models-and-recipes.rst"],objects:{"":{data:[0,0,0,"-"],decoders:[4,0,0,"-"],encoders:[5,0,0,"-"],losses:[6,0,0,"-"],models:[7,0,0,"-"],optimizers:[9,0,0,"-"],parts:[10,0,0,"-"],utils:[13,0,0,"-"]},"data.data_layer":{DataLayer:[0,1,1,""]},"data.data_layer.DataLayer":{__init__:[0,2,1,""],build_graph:[0,2,1,""],get_optional_params:[0,3,1,""],get_required_params:[0,3,1,""],get_size_in_samples:[0,2,1,""],input_tensors:[0,4,1,""],iterator:[0,4,1,""],params:[0,4,1,""]},"data.image2label":{image2label:[1,0,0,"-"],imagenet_preprocessing:[1,0,0,"-"]},"data.image2label.image2label":{ImagenetDataLayer:[1,1,1,""]},"data.image2label.image2label.ImagenetDataLayer":{build_graph:[1,2,1,""],get_optional_params:[1,3,1,""],get_required_params:[1,3,1,""],get_size_in_samples:[1,2,1,""],input_tensors:[1,4,1,""],iterator:[1,4,1,""],split_data:[1,2,1,""]},"data.image2label.imagenet_preprocessing":{_aspect_preserving_resize:[1,5,1,""],_central_crop:[1,5,1,""],_decode_crop_and_flip:[1,5,1,""],_mean_image_subtraction:[1,5,1,""],_parse_example_proto:[1,5,1,""],_resize_image:[1,5,1,""],_smallest_size_at_least:[1,5,1,""],parse_record:[1,5,1,""],preprocess_image:[1,5,1,""]},"data.speech2text":{speech2text:[2,0,0,"-"],speech_utils:[2,0,0,"-"]},"data.speech2text.speech2text":{Speech2TextDataLayer:[2,1,1,""]},"data.speech2text.speech2text.Speech2TextDataLayer":{__init__:[2,2,1,""],_parse_audio_element:[2,2,1,""],_parse_audio_transcript_element:[2,2,1,""],build_graph:[2,2,1,""],get_optional_params:[2,3,1,""],get_required_params:[2,3,1,""],get_size_in_samples:[2,2,1,""],input_tensors:[2,4,1,""],iterator:[2,4,1,""],split_data:[2,2,1,""]},"data.speech2text.speech_utils":{augment_audio_signal:[2,5,1,""],get_speech_features:[2,5,1,""],get_speech_features_from_file:[2,5,1,""]},"data.text2text":{t2t:[3,0,0,"-"],text2text:[3,0,0,"-"],tokenizer:[3,0,0,"-"]},"data.text2text.t2t":{_batch_examples:[3,5,1,""],_create_min_max_boundaries:[3,5,1,""],_filter_max_length:[3,5,1,""],_get_example_length:[3,5,1,""],_load_records:[3,5,1,""],_parse_example:[3,5,1,""],_read_and_batch_from_files:[3,5,1,""],eval_input_fn:[3,5,1,""],train_input_fn:[3,5,1,""]},"data.text2text.text2text":{ParallelTextDataLayer:[3,1,1,""],SpecialTextTokens:[3,1,1,""],TransformerDataLayer:[3,1,1,""]},"data.text2text.text2text.ParallelTextDataLayer":{build_graph:[3,2,1,""],get_optional_params:[3,3,1,""],get_required_params:[3,3,1,""],get_size_in_samples:[3,2,1,""],input_tensors:[3,4,1,""],iterator:[3,4,1,""]},"data.text2text.text2text.SpecialTextTokens":{END_OF_CHOICE:[3,4,1,""],EOS_ID:[3,4,1,""],OUT_OF_BUCKET:[3,4,1,""],PAD_ID:[3,4,1,""],S_ID:[3,4,1,""],UNK_ID:[3,4,1,""]},"data.text2text.text2text.TransformerDataLayer":{build_graph:[3,2,1,""],get_optional_params:[3,3,1,""],get_required_params:[3,3,1,""],input_tensors:[3,4,1,""],iterator:[3,4,1,""]},"data.text2text.tokenizer":{Subtokenizer:[3,1,1,""],_count_and_gen_subtokens:[3,5,1,""],_count_tokens:[3,5,1,""],_escape_token:[3,5,1,""],_filter_and_bucket_subtokens:[3,5,1,""],_gen_new_subtoken_list:[3,5,1,""],_generate_alphabet_dict:[3,5,1,""],_generate_subtokens:[3,5,1,""],_generate_subtokens_with_target_vocab_size:[3,5,1,""],_join_tokens_to_string:[3,5,1,""],_list_to_index_dict:[3,5,1,""],_load_vocab_file:[3,5,1,""],_native_to_unicode:[3,5,1,""],_save_vocab_file:[3,5,1,""],_split_string_to_tokens:[3,5,1,""],_split_token_to_subtokens:[3,5,1,""],_unescape_token:[3,5,1,""],_unicode_to_native:[3,5,1,""]},"data.text2text.tokenizer.Subtokenizer":{__init__:[3,2,1,""],_subtoken_ids_to_tokens:[3,2,1,""],_token_to_subtoken_ids:[3,2,1,""],decode:[3,2,1,""],encode:[3,2,1,""],init_from_files:[3,3,1,""]},"data.utils":{load_pre_existing_vocabulary:[0,5,1,""],pad_vocab_to_eight:[0,5,1,""]},"decoders.decoder":{Decoder:[4,1,1,""]},"decoders.decoder.Decoder":{__init__:[4,2,1,""],_cast_types:[4,2,1,""],_decode:[4,2,1,""],decode:[4,2,1,""],get_optional_params:[4,3,1,""],get_required_params:[4,3,1,""],mode:[4,4,1,""],name:[4,4,1,""],params:[4,4,1,""]},"decoders.fc_decoders":{FullyConnectedCTCDecoder:[4,1,1,""],FullyConnectedDecoder:[4,1,1,""],FullyConnectedTimeDecoder:[4,1,1,""]},"decoders.fc_decoders.FullyConnectedCTCDecoder":{__init__:[4,2,1,""],get_optional_params:[4,3,1,""],get_required_params:[4,3,1,""]},"decoders.fc_decoders.FullyConnectedDecoder":{__init__:[4,2,1,""],_decode:[4,2,1,""],get_required_params:[4,3,1,""]},"decoders.fc_decoders.FullyConnectedTimeDecoder":{__init__:[4,2,1,""],_decode:[4,2,1,""],get_optional_params:[4,3,1,""],get_required_params:[4,3,1,""]},"decoders.rnn_decoders":{BeamSearchRNNDecoderWithAttention:[4,1,1,""],RNNDecoderWithAttention:[4,1,1,""]},"decoders.rnn_decoders.BeamSearchRNNDecoderWithAttention":{__init__:[4,2,1,""],_decode:[4,2,1,""],get_optional_params:[4,3,1,""]},"decoders.rnn_decoders.RNNDecoderWithAttention":{__init__:[4,2,1,""],_build_attention:[4,2,1,""],_decode:[4,2,1,""],get_optional_params:[4,3,1,""],get_required_params:[4,3,1,""]},"encoders.ds2_encoder":{DeepSpeech2Encoder:[5,1,1,""],conv2d_bn_actv:[5,5,1,""],rnn_cell:[5,5,1,""],row_conv:[5,5,1,""]},"encoders.ds2_encoder.DeepSpeech2Encoder":{__init__:[5,2,1,""],_encode:[5,2,1,""],get_optional_params:[5,3,1,""],get_required_params:[5,3,1,""]},"encoders.encoder":{Encoder:[5,1,1,""]},"encoders.encoder.Encoder":{__init__:[5,2,1,""],_cast_types:[5,2,1,""],_encode:[5,2,1,""],encode:[5,2,1,""],get_optional_params:[5,3,1,""],get_required_params:[5,3,1,""],mode:[5,4,1,""],name:[5,4,1,""],params:[5,4,1,""]},"encoders.resnet_blocks":{batch_norm:[5,5,1,""],block_layer:[5,5,1,""],bottleneck_block_v1:[5,5,1,""],bottleneck_block_v2:[5,5,1,""],building_block_v1:[5,5,1,""],building_block_v2:[5,5,1,""],conv2d_fixed_padding:[5,5,1,""],fixed_padding:[5,5,1,""]},"encoders.resnet_encoder":{ResNetEncoder:[5,1,1,""]},"encoders.resnet_encoder.ResNetEncoder":{get_optional_params:[5,3,1,""],get_required_params:[5,3,1,""]},"encoders.rnn_encoders":{BidirectionalRNNEncoderWithEmbedding:[5,1,1,""],GNMTLikeEncoderWithEmbedding:[5,1,1,""],UnidirectionalRNNEncoderWithEmbedding:[5,1,1,""]},"encoders.rnn_encoders.BidirectionalRNNEncoderWithEmbedding":{__init__:[5,2,1,""],_encode:[5,2,1,""],enc_emb_w:[5,4,1,""],get_optional_params:[5,3,1,""],get_required_params:[5,3,1,""],src_emb_size:[5,4,1,""],src_vocab_size:[5,4,1,""]},"encoders.rnn_encoders.GNMTLikeEncoderWithEmbedding":{__init__:[5,2,1,""],enc_emb_w:[5,4,1,""],get_optional_params:[5,3,1,""],get_required_params:[5,3,1,""],src_emb_size:[5,4,1,""],src_vocab_size:[5,4,1,""]},"encoders.rnn_encoders.UnidirectionalRNNEncoderWithEmbedding":{__init__:[5,2,1,""],_encode:[5,2,1,""],enc_emb_w:[5,4,1,""],get_optional_params:[5,3,1,""],get_required_params:[5,3,1,""],src_emb_size:[5,4,1,""],src_vocab_size:[5,4,1,""]},"losses.cross_entropy_loss":{CrossEntropyLoss:[6,1,1,""]},"losses.ctc_loss":{CTCLoss:[6,1,1,""],dense_to_sparse:[6,5,1,""]},"losses.ctc_loss.CTCLoss":{__init__:[6,2,1,""],_compute_loss:[6,2,1,""],get_optional_params:[6,3,1,""]},"losses.loss":{Loss:[6,1,1,""]},"losses.loss.Loss":{__init__:[6,2,1,""],_cast_types:[6,2,1,""],_compute_loss:[6,2,1,""],compute_loss:[6,2,1,""],get_optional_params:[6,3,1,""],get_required_params:[6,3,1,""],name:[6,4,1,""],params:[6,4,1,""]},"losses.sequence_loss":{BasicSequenceLoss:[6,1,1,""],CrossEntropyWithSmoothing:[6,1,1,""],PaddedCrossEntropyLossWithSmoothing:[6,1,1,""]},"losses.sequence_loss.BasicSequenceLoss":{__init__:[6,2,1,""],_compute_loss:[6,2,1,""],get_optional_params:[6,3,1,""],get_required_params:[6,3,1,""]},"losses.sequence_loss.CrossEntropyWithSmoothing":{__init__:[6,2,1,""],_compute_loss:[6,2,1,""],get_optional_params:[6,3,1,""],get_required_params:[6,3,1,""]},"losses.sequence_loss.PaddedCrossEntropyLossWithSmoothing":{get_optional_params:[6,3,1,""]},"models.encoder_decoder":{EncoderDecoderModel:[7,1,1,""]},"models.encoder_decoder.EncoderDecoderModel":{__init__:[7,2,1,""],_build_forward_pass_graph:[7,2,1,""],_create_decoder:[7,2,1,""],_create_encoder:[7,2,1,""],_create_loss:[7,2,1,""],decoder:[7,4,1,""],encoder:[7,4,1,""],get_optional_params:[7,3,1,""],get_required_params:[7,3,1,""],loss_computator:[7,4,1,""]},"models.image2label":{Image2Label:[7,1,1,""]},"models.image2label.Image2Label":{evaluate:[7,2,1,""],finalize_evaluation:[7,2,1,""],get_num_objects_per_step:[7,2,1,""],maybe_print_logs:[7,2,1,""]},"models.model":{Model:[7,1,1,""]},"models.model.Model":{__init__:[7,2,1,""],_build_forward_pass_graph:[7,2,1,""],clip_last_batch:[7,2,1,""],compile:[7,2,1,""],evaluate:[7,2,1,""],finalize_evaluation:[7,2,1,""],finalize_inference:[7,2,1,""],get_data_layer:[7,2,1,""],get_num_objects_per_step:[7,2,1,""],get_optional_params:[7,3,1,""],get_output_tensors:[7,2,1,""],get_required_params:[7,3,1,""],get_tf_dtype:[7,2,1,""],hvd:[7,4,1,""],infer:[7,2,1,""],last_step:[7,4,1,""],maybe_print_logs:[7,2,1,""],mode:[7,4,1,""],num_gpus:[7,4,1,""],on_horovod:[7,4,1,""],params:[7,4,1,""],steps_in_epoch:[7,4,1,""]},"models.speech2text":{Speech2Text:[7,1,1,""],levenshtein:[7,5,1,""],sparse_tensor_to_chars:[7,5,1,""]},"models.speech2text.Speech2Text":{evaluate:[7,2,1,""],finalize_evaluation:[7,2,1,""],finalize_inference:[7,2,1,""],get_num_objects_per_step:[7,2,1,""],infer:[7,2,1,""],maybe_print_logs:[7,2,1,""]},"models.text2text":{Text2Text:[7,1,1,""],calculate_bleu:[7,5,1,""],transform_for_bleu:[7,5,1,""]},"models.text2text.Text2Text":{evaluate:[7,2,1,""],finalize_evaluation:[7,2,1,""],finalize_inference:[7,2,1,""],get_num_objects_per_step:[7,2,1,""],infer:[7,2,1,""],maybe_print_logs:[7,2,1,""]},"optimizers.automatic_loss_scaler":{AutomaticLossScaler:[9,1,1,""],BackoffScaler:[9,1,1,""],LogMaxScaler:[9,1,1,""]},"optimizers.automatic_loss_scaler.AutomaticLossScaler":{SUPPORTED_ALGOS:[9,4,1,""],check_grads:[9,3,1,""],loss_scale:[9,4,1,""],update_op:[9,2,1,""]},"optimizers.automatic_loss_scaler.BackoffScaler":{loss_scale:[9,4,1,""],update_op:[9,2,1,""]},"optimizers.automatic_loss_scaler.LogMaxScaler":{loss_scale:[9,4,1,""],update_op:[9,2,1,""]},"optimizers.lr_policies":{exp_decay:[9,5,1,""],fixed_lr:[9,5,1,""],piecewise_constant:[9,5,1,""],poly_decay:[9,5,1,""],transformer_policy:[9,5,1,""]},"optimizers.mp_wrapper":{MixedPrecisionOptimizerWrapper:[9,1,1,""],mp_regularizer_wrapper:[9,5,1,""]},"optimizers.mp_wrapper.MixedPrecisionOptimizerWrapper":{apply_gradients:[9,2,1,""],compute_gradients:[9,2,1,""]},"optimizers.optimizers":{DistributedOptimizer:[9,1,1,""],_adaptive_max_norm:[9,5,1,""],_add_scaled_noise_to_gradients:[9,5,1,""],_clip_gradients_by_norm:[9,5,1,""],_multiply_gradients:[9,5,1,""],_multiply_gradients_const:[9,5,1,""],adaptive_clipping_fn:[9,5,1,""],get_regularization_loss:[9,5,1,""],optimize_loss:[9,5,1,""]},"optimizers.optimizers.DistributedOptimizer":{__init__:[9,2,1,""],apply_gradients:[9,2,1,""],compute_gradients:[9,2,1,""]},"parts.rnns":{attention_wrapper:[11,0,0,"-"],flstm:[11,0,0,"-"],glstm:[11,0,0,"-"],gnmt:[11,0,0,"-"],rnn_beam_search_decoder:[11,0,0,"-"],slstm:[11,0,0,"-"],utils:[11,0,0,"-"]},"parts.rnns.attention_wrapper":{AttentionMechanism:[11,1,1,""],AttentionWrapper:[11,1,1,""],AttentionWrapperState:[11,1,1,""],BahdanauAttention:[11,1,1,""],BahdanauMonotonicAttention:[11,1,1,""],LuongAttention:[11,1,1,""],LuongMonotonicAttention:[11,1,1,""],hardmax:[11,5,1,""],monotonic_attention:[11,5,1,""],safe_cumprod:[11,5,1,""]},"parts.rnns.attention_wrapper.AttentionMechanism":{alignments_size:[11,4,1,""],state_size:[11,4,1,""]},"parts.rnns.attention_wrapper.AttentionWrapper":{__init__:[11,2,1,""],_item_or_tuple:[11,2,1,""],call:[11,2,1,""],output_size:[11,4,1,""],state_size:[11,4,1,""],zero_state:[11,2,1,""]},"parts.rnns.attention_wrapper.AttentionWrapperState":{clone:[11,2,1,""]},"parts.rnns.attention_wrapper.BahdanauAttention":{__init__:[11,2,1,""]},"parts.rnns.attention_wrapper.BahdanauMonotonicAttention":{__init__:[11,2,1,""]},"parts.rnns.attention_wrapper.LuongAttention":{__init__:[11,2,1,""]},"parts.rnns.attention_wrapper.LuongMonotonicAttention":{__init__:[11,2,1,""]},"parts.rnns.flstm":{FLSTMCell:[11,1,1,""]},"parts.rnns.flstm.FLSTMCell":{__init__:[11,2,1,""],call:[11,2,1,""],output_size:[11,4,1,""],state_size:[11,4,1,""]},"parts.rnns.glstm":{GLSTMCell:[11,1,1,""]},"parts.rnns.glstm.GLSTMCell":{__init__:[11,2,1,""],_get_input_for_group:[11,2,1,""],call:[11,2,1,""],output_size:[11,4,1,""],state_size:[11,4,1,""]},"parts.rnns.gnmt":{GNMTAttentionMultiCell:[11,1,1,""],gnmt_residual_fn:[11,5,1,""]},"parts.rnns.gnmt.GNMTAttentionMultiCell":{__init__:[11,2,1,""]},"parts.rnns.rnn_beam_search_decoder":{BeamSearchDecoder:[11,1,1,""],BeamSearchDecoderOutput:[11,1,1,""],BeamSearchDecoderState:[11,1,1,""],FinalBeamSearchDecoderOutput:[11,1,1,""],tile_batch:[11,5,1,""]},"parts.rnns.rnn_beam_search_decoder.BeamSearchDecoder":{__init__:[11,2,1,""],_maybe_merge_batch_beams:[11,2,1,""],_maybe_split_batch_beams:[11,2,1,""],_merge_batch_beams:[11,2,1,""],_split_batch_beams:[11,2,1,""],batch_size:[11,4,1,""],finalize:[11,2,1,""],initialize:[11,2,1,""],output_dtype:[11,4,1,""],output_size:[11,4,1,""],step:[11,2,1,""],tracks_own_finished:[11,4,1,""]},"parts.rnns.slstm":{BasicSLSTMCell:[11,1,1,""],_linear:[11,5,1,""]},"parts.rnns.slstm.BasicSLSTMCell":{__init__:[11,2,1,""],call:[11,2,1,""],output_size:[11,4,1,""],state_size:[11,4,1,""]},"parts.rnns.utils":{create_rnn_cell:[11,5,1,""]},"parts.transformer":{attention_layer:[12,0,0,"-"],beam_search:[12,0,0,"-"],beam_search_test:[12,0,0,"-"],common:[12,0,0,"-"],embedding_layer:[12,0,0,"-"],ffn_layer:[12,0,0,"-"],utils:[12,0,0,"-"]},"parts.transformer.attention_layer":{Attention:[12,1,1,""],SelfAttention:[12,1,1,""]},"parts.transformer.attention_layer.Attention":{call:[12,2,1,""],combine_heads:[12,2,1,""],split_heads:[12,2,1,""]},"parts.transformer.attention_layer.SelfAttention":{call:[12,2,1,""]},"parts.transformer.beam_search":{SequenceBeamSearch:[12,1,1,""],_StateKeys:[12,1,1,""],_expand_to_beam_size:[12,5,1,""],_flatten_beam_dim:[12,5,1,""],_gather_beams:[12,5,1,""],_gather_topk_beams:[12,5,1,""],_length_normalization:[12,5,1,""],_shape_list:[12,5,1,""],_unflatten_beam_dim:[12,5,1,""],sequence_beam_search:[12,5,1,""]},"parts.transformer.beam_search.SequenceBeamSearch":{_continue_search:[12,2,1,""],_create_initial_state:[12,2,1,""],_get_new_alive_state:[12,2,1,""],_get_new_finished_state:[12,2,1,""],_grow_alive_seq:[12,2,1,""],_search_step:[12,2,1,""],search:[12,2,1,""]},"parts.transformer.beam_search._StateKeys":{ALIVE_CACHE:[12,4,1,""],ALIVE_LOG_PROBS:[12,4,1,""],ALIVE_SEQ:[12,4,1,""],CUR_INDEX:[12,4,1,""],FINISHED_FLAGS:[12,4,1,""],FINISHED_SCORES:[12,4,1,""],FINISHED_SEQ:[12,4,1,""]},"parts.transformer.beam_search_test":{BeamSearchHelperTests:[12,1,1,""]},"parts.transformer.beam_search_test.BeamSearchHelperTests":{test_expand_to_beam_size:[12,2,1,""],test_flatten_beam_dim:[12,2,1,""],test_gather_beams:[12,2,1,""],test_gather_topk_beams:[12,2,1,""],test_get_shape_keep_last_dim:[12,2,1,""],test_shape_list:[12,2,1,""],test_unflatten_beam_dim:[12,2,1,""]},"parts.transformer.common":{LayerNormalization:[12,1,1,""],PrePostProcessingWrapper:[12,1,1,""]},"parts.transformer.common.LayerNormalization":{build:[12,2,1,""],call:[12,2,1,""]},"parts.transformer.embedding_layer":{EmbeddingSharedWeights:[12,1,1,""]},"parts.transformer.embedding_layer.EmbeddingSharedWeights":{build:[12,2,1,""],call:[12,2,1,""],linear:[12,2,1,""]},"parts.transformer.ffn_layer":{FeedFowardNetwork:[12,1,1,""]},"parts.transformer.ffn_layer.FeedFowardNetwork":{call:[12,2,1,""]},"parts.transformer.utils":{get_decoder_self_attention_bias:[12,5,1,""],get_padding:[12,5,1,""],get_padding_bias:[12,5,1,""],get_position_encoding:[12,5,1,""]},"utils.funcs":{evaluate:[13,5,1,""],infer:[13,5,1,""],restore_and_get_results:[13,5,1,""],train:[13,5,1,""]},"utils.hooks":{BroadcastGlobalVariablesHook:[13,1,1,""],PrintLossAndTimeHook:[13,1,1,""],PrintSamplesHook:[13,1,1,""],RunEvaluationHook:[13,1,1,""]},"utils.hooks.BroadcastGlobalVariablesHook":{__init__:[13,2,1,""],after_create_session:[13,2,1,""],begin:[13,2,1,""]},"utils.hooks.PrintLossAndTimeHook":{after_run:[13,2,1,""],before_run:[13,2,1,""],begin:[13,2,1,""]},"utils.hooks.PrintSamplesHook":{after_run:[13,2,1,""],before_run:[13,2,1,""],begin:[13,2,1,""]},"utils.hooks.RunEvaluationHook":{after_run:[13,2,1,""],before_run:[13,2,1,""],begin:[13,2,1,""]},"utils.utils":{Logger:[13,1,1,""],array_to_string:[13,5,1,""],cast_types:[13,5,1,""],check_params:[13,5,1,""],clip_last_batch:[13,5,1,""],clip_sparse:[13,5,1,""],deco_print:[13,5,1,""],flatten_dict:[13,5,1,""],get_available_gpus:[13,5,1,""],get_git_diff:[13,5,1,""],get_git_hash:[13,5,1,""],get_results_for_epoch:[13,5,1,""],iterate_data_layer:[13,5,1,""],log_summaries_from_dict:[13,5,1,""],mask_nans:[13,5,1,""],nest_dict:[13,5,1,""],nested_update:[13,5,1,""],text_ids_to_string:[13,5,1,""]},"utils.utils.Logger":{flush:[13,2,1,""],write:[13,2,1,""]},data:{data_layer:[0,0,0,"-"],image2label:[1,0,0,"-"],speech2text:[2,0,0,"-"],text2text:[3,0,0,"-"],utils:[0,0,0,"-"]},decoders:{decoder:[4,0,0,"-"],fc_decoders:[4,0,0,"-"],rnn_decoders:[4,0,0,"-"]},encoders:{ds2_encoder:[5,0,0,"-"],encoder:[5,0,0,"-"],resnet_blocks:[5,0,0,"-"],resnet_encoder:[5,0,0,"-"],rnn_encoders:[5,0,0,"-"]},losses:{cross_entropy_loss:[6,0,0,"-"],ctc_loss:[6,0,0,"-"],loss:[6,0,0,"-"],sequence_loss:[6,0,0,"-"]},models:{encoder_decoder:[7,0,0,"-"],image2label:[7,0,0,"-"],model:[7,0,0,"-"],speech2text:[7,0,0,"-"],text2text:[7,0,0,"-"]},optimizers:{automatic_loss_scaler:[9,0,0,"-"],lr_policies:[9,0,0,"-"],mp_wrapper:[9,0,0,"-"],optimizers:[9,0,0,"-"]},parts:{rnns:[11,0,0,"-"],transformer:[12,0,0,"-"]},utils:{funcs:[13,0,0,"-"],hooks:[13,0,0,"-"],utils:[13,0,0,"-"]}},objnames:{"0":["py","module","Python module"],"1":["py","class","Python class"],"2":["py","method","Python method"],"3":["py","staticmethod","Python static method"],"4":["py","attribute","Python attribute"],"5":["py","function","Python function"]},objtypes:{"0":"py:module","1":"py:class","2":"py:method","3":"py:staticmethod","4":"py:attribute","5":"py:function"},terms:{"106gb":21,"1080ti":29,"16xlarg":28,"1e6":3,"1e9":12,"1x1":5,"224gb":21,"2xlarg":28,"4gpu":29,"55gb":21,"5gb":27,"8xlarg":28,"\u03b1":22,"\u03b4":22,"\u03b5":22,"\u03b6":22,"\u03ba":22,"abstract":[0,4,5,6,7],"boolean":[1,5,11],"byte":3,"case":[0,4,5,6,7,11,14,16,25,28],"char":0,"class":[0,1,2,3,4,5,6,7,9,11,12,13,16,25],"default":[3,5,6,9,11,13,25,27,28],"enum":3,"export":21,"final":[4,7,11,13,21,25],"float":[1,2,4,5,7,9,11,12,25,28],"function":[0,2,4,5,6,7,9,11,12,13,25,28],"import":[25,28],"int":[0,1,2,3,4,5,7,9,11,12,25],"long":11,"new":[0,1,3,9,11,12,13,23,25,26],"return":[0,1,2,3,4,5,6,7,9,11,12,13,16,28],"short":[11,25,29],"static":[0,1,2,3,4,5,6,7,9,11,16,25,28],"true":[0,1,3,5,6,7,9,11,12,14,25,29],"try":[9,22,25,27],"while":[1,3,16,22,25,28],ALS:9,AWS:28,Adding:23,And:[0,1,3,28],But:7,EOS:12,For:[1,2,3,4,5,7,9,11,13,16,21,22,25,26,28,29],IDs:3,Its:11,NOT:11,One:[22,25,28],Such:16,That:[0,1,3,4,7],The:[0,1,3,5,7,9,11,12,13,16,21,25,27,28,29],There:25,These:[1,4,7,11,13,28],Used:[3,7,12],Uses:[9,13,29],Using:23,With:[14,16,29],__init__:[0,1,2,3,4,5,6,7,9,11,13,16,25],_adaptive_max_norm:9,_add_scaled_noise_to_gradi:9,_aspect_preserving_res:1,_baseattentionmechan:11,_basemonotonicattentionmechan:11,_batch_exampl:3,_build_attent:4,_build_forward_pass_graph:7,_building_block_v1:5,_building_block_v2:5,_cast_typ:[4,5,6],_central_crop:1,_clip_gradients_by_norm:9,_compute_loss:6,_continue_search:12,_count_and_gen_subtoken:3,_count_token:3,_create_decod:7,_create_encod:7,_create_initial_st:12,_create_loss:7,_create_min_max_boundari:3,_decod:4,_decode_crop_and_flip:1,_distributed_appli:9,_encod:5,_escape_token:3,_expand_to_beam_s:12,_filter_and_bucket_subtoken:3,_filter_max_length:3,_flatten_beam_dim:12,_gather_beam:12,_gather_topk_beam:12,_gather_tre:11,_gen_new_subtoken_list:3,_generate_alphabet_dict:3,_generate_subtoken:3,_generate_subtokens_with_target_vocab_s:3,_get_example_length:3,_get_input_for_group:11,_get_new_alive_st:12,_get_new_finished_st:12,_grow_alive_seq:12,_item_or_tupl:11,_join_tokens_to_str:3,_length_norm:12,_linear:11,_list_to_index_dict:3,_load_record:3,_load_vocab_fil:3,_maybe_merge_batch_beam:11,_maybe_split_batch_beam:11,_mean_image_subtract:1,_merge_batch_beam:11,_monotonic_probability_fn:11,_multiply_gradi:9,_multiply_gradients_const:9,_native_to_unicod:3,_output:7,_parse_audio_el:2,_parse_audio_transcript_el:2,_parse_exampl:3,_parse_example_proto:1,_read_and_batch_from_fil:3,_resize_imag:1,_save_vocab_fil:3,_search_step:12,_shape_list:12,_smallest_size_at_least:1,_split_batch_beam:11,_split_string_to_token:3,_split_token_to_subtoken:3,_statekei:12,_subtoken_ids_to_token:3,_test:27,_token_to_subtoken_id:3,_unescape_token:3,_unflatten_beam_dim:12,_unicode_to_n:3,abl:[0,1,3,21,27],about:[13,29],abov:[11,22,28],abs:[5,9,11,29],absolut:28,acceler:11,accept:[3,11],access:[0,4,5,6,7],accord:11,accuraci:[1,7,27,28],achiev:[11,27],across:[3,6,7,9,25,28],activ:[5,11,28],activation_fn:5,actual:[4,7,11,25],adagrad:[7,9,25],adagradoptim:9,adam:[7,9,25,29],adamoptim:9,adapt:[5,7,9,25],adaptive_clip:9,adaptive_clipping_fn:9,add:[3,4,5,7,9,11,13,25,28],add_eo:3,added:[0,3,7,11,12,13,25],adding:13,addit:[2,7,9,11,12,21,25,28,29],addition:[1,25],adjust:[7,9,22,25,28],after:[0,1,5,9,11,13,21,22,25,27],after_create_sess:13,after_run:13,again:[25,27],aggreg:7,aggregation_method:9,aggregationmethod:9,alben:28,algorithm:[7,9,25,28],align:11,alignment_histori:11,alignments_s:11,aliv:12,alive_cach:12,alive_log_prob:12,alive_seq:12,all:[0,1,2,3,4,5,6,7,9,11,12,13,16,22,25,27,28,29],allow:[3,22,29],allreduc:9,along:[5,11],alpha:12,alphabet:[3,4],alphabet_config_path:4,alreadi:[11,26],also:[0,1,3,7,9,13,14,22,25,26,27,28,29],altern:[9,27,28],alwai:[7,9,28],amax:9,amount:3,analog:7,analysi:2,ani:[3,4,5,7,9,11,12,13,25,28,29],anoth:[3,9,11],answer:[7,25],anymor:13,anyth:9,api:[2,26],appear:3,append:[3,25],appli:[3,5,9,11,12,28],apply_gradi:9,approach:[7,11,14,28],apt:[21,27],arbitrari:9,architectur:[22,28],archiv:21,aren:3,arg:[9,11,13],argmax:11,argument:[2,3,4,5,6,7,9,11,12,13,14,21,25],arithmet:28,around:[1,4,5,6,21,27,28],arrai:[2,3,7],arrang:1,array_op:11,array_to_str:13,articl:11,artifici:4,arxiv:[5,9,11,28,29],aspect:1,assign:[0,4],assum:[11,21,27],assumpt:[7,11],attend:11,attent:[4,11,12,22,29],attention_cel:11,attention_depth:11,attention_dropout:12,attention_lay:[8,10],attention_layer_s:11,attention_mechan:11,attention_or_cell_output:11,attention_st:11,attention_typ:4,attention_wrapp:[8,10],attentioninputwrapp:11,attentionmechan:11,attentionwrapp:11,attentionwrapperst:11,attribut:[7,25],audio:[2,7,21],audio_filenam:2,augment:2,augment_audio_sign:2,auto:6,automat:[4,5,6,7,9,25,27,29],automatic_loss_sc:[7,9,25,28],automatic_loss_scal:8,automaticlossscal:9,autoregress:12,avail:[9,22,25,26,28,29],averag:[6,7,9],average_across_timestep:6,avoid:[9,11],axi:[7,11],back:[12,28],backoff:[7,9,25,28],backoffscal:9,backpropag:28,backslash:3,bahadanau:11,bahdanau:[4,11],bahdanau_norm:4,bahdanauattent:11,bahdanaumonotonicattent:11,bandwidth:28,base:[0,1,2,3,4,5,6,7,9,11,12,13,14,21,22,25,26,27],base_model:25,base_param:[25,28],basic:[6,11,26],basic_sequence_loss:6,basicsequenceloss:6,basicslstmcel:11,batch:[3,4,5,6,7,9,11,12,16,25],batch_norm:5,batch_siz:[2,3,4,5,6,11,12],batch_size_per_gpu:[6,7,22,25,29],batches_per_epoch:9,bazel:27,bbox:1,beahvior:11,beam:[4,11,12,29],beam_indic:12,beam_search:[8,10],beam_search_decoder_output:11,beam_search_test:[8,10],beam_siz:12,beam_width:[4,11],beamsearch:11,beamsearchdecod:11,beamsearchdecoderoutput:11,beamsearchdecoderst:11,beamsearchhelpertest:12,beamsearchrnndecoderwithattent:4,becaus:[3,12,22,28],becom:[2,9],been:[9,11,12,13,28],befor:[3,5,9,11,13,22,28],before_run:13,begin:[3,11,13,28],begin_decay_at:9,behavior:[11,28],being:[3,23],below:[27,28,29],bench_start:25,bench_step:25,benchmark:[7,25,28],benefici:28,bengio:11,besid:7,best:[11,12,27],beta1:9,beta2:9,better:[1,12],between:[3,7,9,11,12],bhadanau:11,bia:[11,12],bias:11,bias_initi:11,bidir_rnn_encoder_with_emb:5,bidirect:29,bidirectionalrnnencoderwithembed:5,big:[27,29],bigger:28,bin:27,binari:[3,21,27],bind:29,bleu:[7,29],blob:12,block:[5,25],block_fn:5,block_lay:5,blue:22,bn_epsilon:5,bn_momentum:5,bn_regular:5,bodi:12,bool:[0,4,5,6,7,9,11,12,25],boost:27,bori:28,both:[1,7,11,25,28,29],bottleneck:5,bottleneck_block:5,bottleneck_block_v1:5,bottleneck_block_v2:5,bottom:11,bound:1,boundari:[3,9],boundary_scal:3,box:1,bpe_us:7,broadcast:13,broadcastglobalvariableshook:13,btl:29,bucket:3,buckets_max:3,buckets_min:3,buffer:1,build:[2,4,9,12,13,20,26,27,28],build_graph:[0,1,2,3],build_image_data:1,build_lm:21,build_pip_packag:27,building_block:5,building_block_v1:5,building_block_v2:5,built:[7,25,26],c_state:11,cach:12,calcul:[0,1,3,7,11,12],calculate_bleu:7,call:[4,5,6,7,9,11,12,13,16,28],callabl:[9,11],callback:13,can:[0,1,2,3,4,5,6,7,9,11,13,14,16,22,25,26,27,28,29],candiat:3,candid:3,cannot:[7,11,12,25],cast:[4,5,6,28],cast_typ:13,cat:22,cell:[4,5,11,29],cell_input_fn:11,cell_param:11,cell_stat:11,cell_typ:11,center:1,central:1,chang:[11,13,14,22,28],channel:[1,5],channels_first:5,channels_last:5,charact:[3,4],check:[7,11,25,26,27,28],check_grad:9,check_param:13,checkpoint:[7,11,13,25,29],child:25,cho:11,choos:[11,28],chosen:9,christoph:11,classic:7,clean:[21,29],cleaned_fil:22,clip:[7,9,25],clip_gradi:9,clip_last_batch:[7,13],clip_spars:13,clone:[11,27],close:[3,11,29],cloud:28,cmake:27,cnn:26,code:[3,7,11,25],coeffici:[9,28],colin:11,collect:[7,9,12,28],colloqui:1,coloc:9,colocate_gradients_with_op:9,color:1,colorspac:1,column:11,com:[12,27],combin:[7,9,12,26],combine_head:12,command:[14,21,22,25,27,29],comment:29,commit:25,common:[8,10,28],commonli:28,compar:[5,28],compat:9,compil:[7,25],complet:[7,9,17,18,19,24,25,27],complex:28,compon:11,compos:11,compress:21,comput:[1,6,7,9,11,12,28],compute_gradi:[9,28],compute_loss:[6,7,13],concat:11,concaten:[7,11],concret:28,config:[0,1,2,3,4,5,6,7,9,13,14,23,27,29],config_fil:[21,22,25,27,29],configur:[7,14,22,25,26,27,29],conflict:11,conjunct:[7,25],connect:[4,5,7,12],conrib:11,consecut:3,consist:[4,13],constant:[9,28],constraint:11,construct:[0,1,3,4,5,6,7,9,11,13],constructor:[0,2,4,5,6,7,9,11,25],consumpt:28,contain:[0,1,2,3,4,5,6,7,9,11,12,13,22,25,27,28,29],content:[1,4,5,6],context:11,continu:[12,25],continue_learn:25,contrib:11,control:[7,11],conv2d:5,conv2d_bn_actv:5,conv2d_fixed_pad:5,conv_lay:5,conveni:25,convent:5,converg:28,convert:[2,3,11,21,28],convolut:[5,29],coord:[1,13],coordin:[1,13],copi:[5,7,11,28],copt:27,core:[3,28],correct:[7,11,12,21,22,25],correctli:[7,27],correspond:[0,1,3,7,9,11,12,16,22,25,27,28],correspondingli:[9,16,29],cosin:12,could:[0,2,4,5,6,7,9,25],count:[0,3,4,25],counter:9,cover:25,cpu:[3,28],creat:[0,1,3,4,5,6,7,9,11,12,13,20,25,28],create_rnn_cel:11,create_toy_data:22,creation:[7,28],crop:1,crop_height:1,crop_width:1,cross:6,cross_entropy_loss:8,cross_entropy_with_smooth:6,crossentropyloss:6,crossentropywithsmooth:6,csv:[2,21],ctc:[4,6],ctc_decoder_with_lm:27,ctc_greedy_decod:4,ctc_loss:8,ctcloss:6,cuda:[27,28],cudnn:29,cudnn_gru:5,cudnn_lstm:5,cudnnlstm:11,cumprod:11,cumsum:11,cumul:11,cur_index:12,current:[1,4,5,7,9,11,12,25,29],custom:[27,28],cut:[4,7],d_model:9,dai:29,data:[1,2,3,4,5,6,7,8,11,13,15,21,22,23,25,28],data_fil:3,data_format:5,data_lay:[1,2,3,4,5,6,7,8,25],data_layer_param:[7,25],data_root:22,datalay:[0,1,2,3,7,16,25],dataset:[0,1,2,3,7,16,20,22],dataset_fil:2,david:28,dct:13,debug:25,debug_port:[13,25],debugger_port:25,dec:5,decai:[9,28,29],decay_r:9,decay_step:9,deco_print:13,decod:[0,1,3,5,6,7,8,11,12,15,21,22,23,25,26,29],decode_and_crop:1,decoder_cell_typ:4,decoder_cell_unit:4,decoder_dp_input_keep_prob:4,decoder_dp_output_keep_prob:4,decoder_initial_st:11,decoder_library_path:4,decoder_output:6,decoder_param:[4,7],decoder_use_skip_connect:4,decreas:[3,28],deep:[5,11,28],deepbench:28,deepspeech2encod:[5,25],deepspeech:[5,25,27],defaultdict:3,defin:[0,1,3,4,5,7,9,11,12,25,28],definit:5,degre:28,delet:21,delim:[7,13],denomin:[7,25],denot:1,dens:[7,9,11],dense_tensor:6,dense_to_spars:6,depend:[5,11],deprec:11,depth:[1,11,26],deriv:[0,4,5,6,7,25],describ:[0,1,3,4,5,6,7,11,25,27,28,29],descript:[0,1,2,3,4,5,6,7,25,29],design:[26,28],desir:28,detail:[1,2,4,5,7,9,11,25,26,29],determin:[11,12],determinist:9,dev:[21,22,27],deviat:11,devic:[9,13],device_dens:9,device_spars:9,diamo:28,dict:[0,1,2,3,4,5,6,7,9,12,25],dict_to_log:13,dictionari:[0,1,2,3,4,5,6,7,12,16,25],did:22,diederik:11,diff:25,differ:[1,3,5,9,11,12,13,22,25,26,27,28],dim:[4,5,6,11],dimens:[1,4,5,11,12],dimension:[9,11],direct:[5,29],directori:[7,25,27],disabl:[7,11,22,25,27,28],discov:27,disk:22,displai:25,distanc:7,distort:1,distribut:[3,7,9,11,25,26,27,28],distributedoptim:9,divid:12,divis:[0,2,7,11],dl_id:13,dnn:28,do_mask:6,doc:[4,5,6,7,11,25],docker:[27,28],docstr:11,document:[11,25,26,28],doe:[1,4,6,7,11,22,25,27,28,29],doesn:1,domain:2,don:[3,12,29],done:[22,27],dot:12,dougla:11,download:[21,22],download_lm:27,downsampl:5,dp_input_keep_prob:11,dp_output_keep_prob:11,draw:12,drawn:3,dropout:[4,5],dropout_keep_prob:5,dropout_keep_prop:5,ds2_encod:[8,25],ds2_large_8gpu:29,ds2_librispeech_larc_config:21,ds2_medium_4gpu:29,ds2_small_1gpu:29,ds2_toy_data_config:[21,27],dtype:[0,4,5,6,7,9,11,13,25,28],due:11,dure:[1,3,4,7,11,12,13,22,25,28],dynam:[7,11,28],dynamic_decod:11,dzmitri:11,each:[0,1,3,6,7,9,11,12,13,16,25,28],eager:9,earli:11,easi:26,easili:22,eck:11,effect:[11,29],effici:[11,26],eight:29,either:[0,2,4,5,6,7,9,11,25,28],element:[2,3,7,9,11],elimin:29,els:[6,9],elsen:28,embed:[4,5,11,12],embedding_lay:[8,10],embedding_lookup:11,embedding_s:12,embeddingsharedweight:12,emit:11,emnlp:11,empti:[7,9,11,16,25],enabl:[7,9,11,14,25,27],enable_log:[25,29],enc_emb_w:5,encod:[0,1,3,4,6,7,8,11,12,15,22,23,25,26,29],encoder_cell_typ:5,encoder_cell_unit:5,encoder_decod:[4,5,6,8,25],encoder_dp_input_keep_prob:5,encoder_dp_output_keep_prob:5,encoder_final_st:11,encoder_lay:5,encoder_output:[4,5,11],encoder_param:[5,7],encoder_sequence_length:4,encoder_st:[5,11],encoder_use_skip_connect:5,encoderdecodermodel:[7,25],encorc:11,end:[3,4,11,12,13,22,27,28],end_compat:9,end_learning_r:9,end_of_choic:3,end_symbol:4,end_token:11,energi:11,enforc:11,english:[3,20],enough:[22,25,28],ensur:[3,11,12,13,28],entri:11,entropi:6,enumer:3,eos:12,eos_id:[3,12,13],epoch:[0,1,3,7,9,25,29],epsilon:[5,7,9,12,25],equal:[1,7,11,25],equival:[9,11],erich:28,error:[7,11,21,27,29],escap:3,especi:28,essenti:13,estim:28,etc:[4,5,7,16,25,26],etl:16,eval:[1,4,5,7,21,25],eval_input_fn:3,eval_model:13,eval_param:25,eval_step:[7,25],evalu:[0,1,2,3,7,13,21,22,25],evenli:3,event:[25,28],everi:[5,7],every_step:13,everyth:[21,25,26,27],exact:[11,25],exactli:11,exampl:[0,1,2,3,4,5,7,9,11,16,21,22,25,27,28],example_config:[21,22,25,27],example_seri:1,exce:[7,9,25],except:[5,11,13,25,27,28],execut:[7,9,14,22,25],exist:[0,4,11,23,28],exp:[9,11],exp_decai:9,expect:6,experi:[22,25,26],experiment:26,explicit:5,explicitli:[11,28],exponenti:9,exponential_decai:9,express:25,extend:26,extens:28,extra:9,extract:[2,16,21],fact_siz:11,factor:[9,11,12,28],fail:27,fairli:25,fals:[0,1,3,4,5,6,7,9,11,12,13,27],familiar:26,fc_decod:8,featur:[2,4,5,6,26],features_typ:2,fed:16,feed:[11,13],feed_dictionari:16,feedforward:12,feedfowardnetwork:12,feel:21,fetch:[7,9],few:29,ffn_layer:[8,10],field:[1,2,4,5,6,11],file:[0,1,2,3,4,7,21,22,25,27,29],file_byte_limit:3,file_pattern:3,file_with_bpe_segment:22,filenam:[1,2,3],filepath:3,filter:[3,5,9],filter_s:12,final_output:4,final_sequence_length:4,final_st:[4,11],finalbeamdecoderoutput:11,finalbeamsearchdecoderoutput:11,finalize_evalu:7,finalize_infer:7,find:[9,12,26],finish:[11,12,22,25],finished_flag:12,finished_scor:12,finished_seq:12,first:[0,1,3,5,7,9,11,12,14,21,22,25,28],fix:9,fixed_lr:9,fixed_pad:5,flag:[11,12],flaot:12,flat_dict:13,flatten_dict:13,flexibl:26,flip:1,float16:[0,4,5,6,7,25,28],float32:[0,4,5,6,7,9,12,25,28,29],flstm:[8,10],flstmcell:11,flush:13,folder:[21,22,25,27],follow:[0,1,3,4,5,6,7,9,16,21,22,25,27,28,29],forc:25,force_var_reus:7,forev:3,forget:[11,29],forget_bia:11,form:[0,3,11],format:[1,3,5],formul:12,forward:[7,28],found:[3,29],four:29,fp32:28,fraction:3,frame:[2,7],framework:12,free:21,frequenc:[2,3],frequent:3,from:[0,1,2,3,4,5,6,7,9,11,12,13,16,25,27,28,29],ftrl:[7,25],full:[5,9,28,29],fulli:[4,5,12],fully_connected_ctc_decod:4,fully_connected_decod:4,fully_connected_time_decod:4,fullyconnectedctcdecod:[4,21],fullyconnecteddecod:4,fullyconnectedtimedecod:4,func:8,furthermor:28,fuse:1,futur:12,ganesh:28,garcia:28,gate:[9,11],gate_gradi:9,gate_graph:9,gate_non:9,gate_op:9,gather:12,gen_input_tensor:16,gener:[3,4,7,11,12,25],generate_tri:27,geometr:12,german:[3,20],get:[7,9,12,16,21,22,26,27],get_available_gpu:13,get_data_lay:7,get_decoder_self_attention_bia:12,get_git_diff:13,get_git_hash:13,get_global_step:9,get_next:16,get_num_objects_per_step:7,get_optional_param:[0,1,2,3,4,5,6,7,16,25],get_output_tensor:7,get_pad:12,get_padding_bia:12,get_position_encod:12,get_regularization_loss:9,get_required_param:[0,1,2,3,4,5,6,7,16,25],get_results_for_epoch:13,get_size_in_sampl:[0,1,2,3,16],get_speech_featur:2,get_speech_features_from_fil:2,get_tf_dtyp:7,get_vari:28,get_wmt16_en_dt:22,getter:28,ginsburg:[11,28],git:[25,27],github:[12,27],given:[1,9,11,12,28],global:[9,13],global_gradient_norm:[7,25],global_step:9,glstm:[4,8,10],glstmcell:11,gnmt:[4,5,8,10,29],gnmt_encoder_with_emb:5,gnmt_residual_fn:11,gnmt_v2:4,gnmtattentionmulticel:11,gnmtlikeencoderwithembed:5,go_symbol:4,goal:22,going:[4,5,17,18,19,24,25,27],good:25,gpu:[0,6,7,9,13,14,16,22,25,26,27,28,29],gpu_id:[7,25],grad_loss:9,gradient:[7,9,11,25,28],gradient_multipli:9,gradient_noise_scal:9,gradient_norm:[7,25],grads_and_var:9,gram:21,graph:[0,1,2,3,4,5,6,7,9,13,25,28],graphic:28,graphkei:9,greater:5,gregori:28,group:[3,11],group_batch_s:3,group_id:11,group_siz:11,grow:12,gru:[4,5,29],guarante:[3,11],half:28,halv:28,handl:11,happen:[0,1,3,4,5,6,7,16],hard:11,hardmax:11,has:[0,1,2,3,4,5,6,7,9,11,12,13,28,29],has_nan:9,hash:25,hat:9,have:[0,1,2,3,4,5,6,7,9,11,12,16,22,25,26,27,28,29],head:12,height:1,height_in:5,help:[7,21,25],helper:[5,12],henc:28,here:[0,1,3,4,5,6,7,11,16,25,28,29],hetland:7,hidden:[4,5,11],hidden_s:12,hieu:11,high:28,higher:11,highest:12,highli:29,histogram:9,histori:11,hold:[9,12],hood:9,hook:[7,8],horovod:[0,3,6,7,9,13,14,25,29],horovod_gpu_allgath:9,horovod_gpu_allreduc:9,horovod_gpu_broadcast:13,hot:[1,6,11],houston:28,how:[7,9,11,12,20,22,23],howev:[1,28],http:[5,7,9,11,12,27,29],human:13,hvd:[7,25],hyperparamet:28,iclr:[11,28],icml:11,ident:[5,11],ids:[0,2,3,7,11,12,13,25],idx2char:7,ignor:[0,7,9,11,14,25],ignore_speci:[7,13],illeg:12,illustr:28,ilsvrc2012_val_00041207:1,imag:[1,5,7],image2label:[0,8,25],image_buff:1,imagenet_preprocess:[0,8],imagenetdatalay:1,implement:[6,7,9,11,12,16,25,29],impli:11,import_librivox:21,improv:[1,3,28],incept:1,includ:[0,1,2,3,4,5,6,7,9,11,21,28],increas:[12,28],increment:9,increment_global_step:9,independ:[5,6,7,25],index:[11,12],indexedslic:9,indic:[1,3,12,13],inf:11,infer:[0,1,3,4,5,7,11,13,21,25],infer_output_fil:[21,22,25],infer_param:25,infin:12,info:9,inform:[11,12,13,25,26],inherit:[0,4,5,6,7,16],init_from_fil:3,initi:[3,4,5,7,9,11,12,13,25,28],initial_cach:12,initial_cell_st:11,initial_id:12,initial_st:11,initializer_param:[4,5,7,25],inner:[11,12],input:[0,1,2,3,4,5,6,7,11,12,25,26],input_dict:[4,5,6,13],input_lay:5,input_s:11,input_sequence_length:11,input_tensor:[0,1,2,3,4,5,6,7],input_typ:2,input_valu:7,insid:[0,1,3,7,22,28],inspect:28,inspir:11,instabl:11,instal:[21,26],instanc:[0,4,5,6,7,9,11,28],instanti:9,instead:[1,4,9,11,27,28],instruct:[21,26],insur:12,int32:[1,11,12],int64:[11,12],intact:5,integ:[1,3,5,11],inter:28,intermedi:28,intern:[9,11,23],introduc:[1,5,28],invalid:9,invalidargu:11,invari:12,invers:3,involv:12,is_train:1,issu:29,item:[3,11,12],iter:[0,1,2,3,7,13,16,28],iterate_data_lay:13,its:[3,11,12,21,25],jian:5,join:3,jointli:11,jonah:28,jpeg:1,jul:5,just:[4,5,27],kaim:5,keep:[4,5,6,9,13,28],kei:[0,1,3,5,7,9,12],kenlm:[21,27],kept:28,kernel:5,kernel_initi:11,kernel_s:5,key_channel:12,keyword:12,kind:25,kingma:11,knee:1,known:[1,3,11],kpu:27,kuchaev:28,kuchaiev:11,kwarg:[9,11,12],kyunghyun:11,label:[1,4,6],lambda:[9,11,28],languag:[4,20],language_model:27,lar:[7,25],larc:[7,9,25,29],larc_mod:[7,9,25],larc_nu:[7,9,25],larc_param:[7,9,25],larg:[11,28],largest:12,last:[4,5,7],last_batch:[7,13],last_step:[7,13],latenc:28,later:28,latter:11,launch:[13,22],layer:[0,1,2,3,4,5,7,9,11,12,15,23,25,28,29],layer_typ:5,layernorm:12,layernorm_lstm:5,ld_library_path:29,lead:27,learn:[5,7,9,11,22,25,29],learnabl:11,learning_r:[7,9,25],learning_rate_decay_fn:9,least:[5,12],left:11,length:[0,1,2,3,4,5,6,11,12],length_i:12,length_penalty_weight:11,length_x:12,less:[3,28,29],level:[25,28],levenshtein:7,libboost:27,libctc_decoder_with_kenlm:27,librari:4,librispeech:[20,29],librivox:21,libsox:21,libtensorflow_cc:27,libtensorflow_framework:27,like:[5,9,12,16,22,29],limit:28,line:[0,3,13,14,25],linear:[4,11,12],link:[27,29],list:[0,1,2,3,4,5,6,7,9,11,12,16,25,26],liu:11,live:12,lm_binary_path:4,lm_trie_path:4,lm_weight:4,load:[0,3,16],load_pre_existing_vocabulari:0,localhost:29,locat:[12,22,27],lock:9,log:[7,9,11,12,23,28],log_fil:13,log_max:9,log_summaries_from_dict:13,logdir:[7,22,25],logger:13,logic:[7,12,16],logit:[4,6,11,12],logits_to_outputs_func:4,logmax:[7,9,25,28],logmaxscal:9,lognorm:28,logspac:11,longer:13,longest:3,look:[7,25,26,29],loop:12,lose:28,loss:[0,1,3,4,7,8,9,15,23,25],loss_comput:7,loss_input_dict:6,loss_param:7,loss_scal:[7,9,25,28],lot:[21,22,27],lower:[3,28],lr_polici:[7,8,25],lr_policy_param:[7,25],ls_dir:21,lst:3,lstm:[4,5,11,29],lstmstatetupl:11,luong:[4,11],luong_scal:4,luongattent:11,luongmonotonicattent:11,m_state:11,machin:[11,20],magnitud:9,mai:[3,11,12],main:[4,5,6,22,25,26],maintain:[12,28],major:11,make:[0,1,3,11,21,22,26,27],malform:9,man:11,mani:28,manner:11,manual:[11,27,28],map:[0,3,4,5,11,29],mark:[3,11,12],mask:[6,11,12],mask_nan:[6,13],master:[12,28],match:[1,3,5,11,28,29],matric:11,matrix:[11,12],max:3,max_decode_length:12,max_grad_norm:[7,25],max_length:3,max_lr:9,max_norm:9,max_pool2d:5,max_step:[7,25],max_subtoken_length:3,max_tim:11,max_timescal:12,maxim:9,maximum:[3,7,11,12,25,28],mayb:11,maybe_print_log:7,mca:29,mean:[1,9,28],measur:[22,29],mechan:[4,11,12],memori:[0,11,22,28],memory_sequence_length:11,mention:28,merg:11,method:[0,1,2,3,4,5,6,7,9,11,12,13,16,25,28],methodnam:12,methodolog:28,metric:7,mfcc:2,michael:28,micikeviciu:28,might:[22,25,27,28],milli:2,min:3,min_boundari:3,min_count:3,min_idx:0,min_lr:9,min_timescal:12,min_upd:[7,25],minh:11,mini:16,minibatch:11,minim:[7,9,25],minimum:[0,3,11,12],minumum:3,minut:27,mismanag:11,misspel:21,mix:[4,5,7,9,11,12,25,26,29],mixedprecisionoptimizerwrapp:[9,28],mkdir:[21,27],modal:26,mode:[0,1,3,4,5,7,11,13,14,21,22,25,27,29],model:[0,1,2,3,4,5,6,8,9,11,12,13,20,22,23,26,28],model_param:28,modifi:[4,5,6,9,13,28],modul:[4,7,9,11,25],modular:26,momentum:[5,7,9,25,29],momentumoptim:9,monoton:11,monotonic_attent:11,more:[9,11,12,22,26,28,29],moreov:25,moss:[22,29],most:[3,7,16,25,28],move:[9,11,25],mozilla:27,mp_regularizer_wrapp:[9,28],mp_wrapper:8,mpi4pi:27,mpi:7,mpirun:[14,29],msg:13,much:[3,12,22,27],multi:[7,12,14,16,22,26,29],multicel:11,multihead:12,multipl:[9,11,12,29],multipli:[9,11,28],multirnncel:11,must:[0,1,3,4,5,6,9,11,12,25],mutli:14,myfavoriteattentionmechan:11,n03623198:1,n_hidden:5,name:[2,3,4,5,6,7,9,11,25,28],namedtupl:11,nan:6,narang:28,nativ:3,nearli:28,necessari:[0,5,7,9,13,25,27,28],necessarili:12,need:[0,1,3,4,5,7,9,12,14,16,21,22,25,27,28],neg:[11,12],neither:9,nest:[11,12,25],nest_dict:13,nested_upd:13,network:[5,11,12,28],neural:[11,28],new_beam_s:12,new_cach:12,new_height:1,new_log_prob:12,new_seq:12,new_width:1,newli:11,newstest2014:[22,29],next:[7,11,25,27,28,29],next_batch_feed_dict:16,next_input:11,next_stat:11,nmt:[22,29],nmt_revers:22,no_dir_check:25,noam:9,node:[14,26],nois:[9,11],noise_level_max:2,noise_level_min:2,non:[9,11,12],none:[0,2,3,4,5,6,7,9,11,12,13,16,25,28,29],nor:9,norm:[5,7,9,25],normal:[4,5,9,11,12,28],note:[0,1,3,4,5,6,7,9,11,21,22,25,28],now:[11,21,25,27,28],num:[4,5,6],num_audio_featur:2,num_box:1,num_channel:[1,5],num_cpu_cor:3,num_epoch:[0,1,3,7,9,25],num_featur:[2,4],num_gpu:[7,14,22,25],num_head:12,num_iter:3,num_lay:11,num_proj:11,num_rnn_lay:5,num_time_step:2,num_unit:11,num_work:[0,1,2,3,16],number:[0,1,2,3,4,5,6,7,9,11,12,14,25,28],number_of_group:11,numer:[7,11,25,28],numpi:2,nvidia:[26,27,28],object:[0,1,3,4,5,6,7,9,11,12,13,16,28],obtain:[27,28,29],occur:11,offici:26,offset:13,offset_target_by_on:6,often:[7,25,28],old:12,oleksii:28,on_horovod:[7,9],onc:[3,11,13,22],one:[0,1,3,4,5,6,7,9,11,12,13,22,25,28,29],ones:26,onli:[0,1,3,4,7,9,11,13,14,25,28],onlin:11,open:7,open_seq2seq:[1,2,3,7,27],openib:29,openseq2seq:[3,14,21,22,25,28,29],oper:[4,5,9,11,13,27,28],ops:[1,9,11,13],opt:27,optim:[7,8,25,29],optimize_loss:9,optimizer_cls_nam:9,optimizer_param:[7,9,25],optimizer_summari:9,option:[0,1,2,3,4,5,6,7,9,11,16,25],optional_dict:13,order:[3,5,9,11,21,27,28],org:[5,7,9,11,29],org_dict:13,origin:[1,3,5,11,12,13,21,29],orte_base_help_aggreg:29,other:[1,4,7,9,11,13,21,25,27,28],otherwis:[1,7,9,11,25,27,28],our:[26,27,28],out:[11,12,22,26],out_of_bucket:3,output:[1,4,5,6,7,11,12,13,16,21,22,25,26,27],output_attent:11,output_dim:[4,11],output_dir:13,output_dtyp:11,output_fil:[7,13],output_height:1,output_lay:11,output_s:11,output_time_major:11,output_valu:7,output_width:1,outsid:3,over:[6,28],overal:13,overcom:28,overflow:28,overflow_std_dev:9,overrid:11,overridden:11,overriden:9,overwrit:[7,25],overwritten:25,own:20,p_choose_i:11,packag:[4,5],pad2eight:12,pad:[0,1,2,3,5,12,16,29],pad_id:[3,13],pad_to:2,pad_vocab_to_eight:0,padded_cross_entropy_with_smooth:6,padded_input_length:3,padded_length:3,padded_target_length:3,paddedcrossentropylosswithsmooth:6,padding_valu:12,page:[26,27],pair:[7,9,11],paper:22,parallel:[3,11,12,22],parallel_interleav:3,paralleltextdatalay:3,param:[0,1,2,3,4,5,6,7,9,11,12,13,16,25],paramet:[0,1,2,3,4,5,6,7,9,11,12,13,14,16,22,23,27,28],parent:[0,2,4,5,6,25],pars:[1,2],parse_record:1,part:[1,4,6,7,8,9,11,12,21,25,28],particular:11,partli:11,pass:[0,1,3,4,5,6,7,9,11,12,13,25,28],past:11,path:[0,2,3,4,7,25,29],pauliu:28,pdf:[5,9],penal:11,per:[2,3,6,9],perform:[1,2,4,5,6,7,11,22,25,28],period:[25,28],perl:[22,29],peter:11,pham:11,piecewis:9,piecewise_const:9,pip:27,pip_packag:27,pipelin:3,place:[3,25],placehold:16,plane:5,pleas:[11,21],point:[3,11,13,22,28],polici:[7,9,25],poly_decai:9,polynomi:9,polynomial_decai:9,popul:[7,9],posit:[5,11,12],possibl:[7,11,25,28,29],post:12,power:[9,11],practic:28,pre:[0,11,12,22],preactiv:5,precis:[4,5,7,25,26,29],pred:7,predict:[7,11,12,13,25],predicted_id:11,preevious_attent:11,prefer:27,prefix:[3,9],prepar:[11,25],prepostprocessingwrapp:12,preprint:28,preprocess:[1,21],preprocess_imag:1,presenc:28,present:9,preserv:1,prevent:9,previou:[9,11,12],previous_attent:11,primarili:1,principl:28,print:[7,13,25],print_loss_step:[7,25],print_samples_step:[7,25],printlossandtimehook:13,printsampleshook:13,prior:[3,11],probability_fn:11,probabl:[4,5,11,12,25,28],problem:[11,22,28],proce:25,process:[0,1,2,3,9,11,12,13,22,27],produc:[4,5,6,11],product:[11,12,26],progress:22,project:[5,11,26],projection_shortcut:5,propag:[11,28],proper:11,properli:11,properti:[11,12],propos:[5,11],proto:1,protocol:1,provabl:12,proven:28,provid:[1,3,4,5,7,9,11,13,21,28],pull:12,put:27,python:[1,3,4,5,7,9,11,12,13,16,21,22,25,27,29],quantiti:9,queri:11,quicker:9,raffel:11,rais:[1,9,11,13],random:[1,3,7,11,13,25],random_se:[7,25],randomli:1,rang:[11,28],rank:[1,3,9,11,13],rare:28,rate:[7,9,25,29],rather:[0,5,7,25,28],ratio:1,raw:[1,2,5],raw_record:1,raw_str:3,reach:12,read:[0,3,11,25],read_char:0,readabl:13,real:22,realli:4,reason:11,receiv:13,recent:28,recip:[26,28],recogn:21,recognit:[5,7,20,27],recommend:[9,11,16,27,28,29],record:[1,3],recov:13,recurr:[11,28],recurs:11,redefin:28,reduc:[11,22],reduce_mean:6,ref:5,refer:[1,3,12],regress:6,regular:[4,5,7,9,25],regularizer_param:[4,5,7,25],relat:[0,1,3,11],relu:5,relu_dropout:12,remov:[7,22],ren:5,reparameter:11,repeat:[3,11],replac:[3,11,21],report:9,report_summari:9,repositori:27,repres:[1,11,12],represent:[4,5],representation_dim:5,request:13,request_stop:13,requir:[0,1,2,3,4,5,6,7,9,11,16,25,27,28],required_dict:13,rescal:9,research:26,reserv:3,reserved_token:3,reshap:[11,12],residu:[4,5,11],residual_connect:11,resiz:1,resize_imag:1,resize_min:1,resized_imag:1,resizemethod:1,resnet:[1,5],resnet_block:8,resnet_encod:8,resnetencod:5,respect:[11,28],rest:27,restor:[11,13,25],restore_and_get_result:13,result:[7,11,12,13,21,22,27,28,29],results_per_batch:7,retriev:28,reus:[11,28],revers:20,rgb:1,right:11,rmsprop:[7,25],rnn:[4,5,8,10,22,26,29],rnn_beam_search_decod:[8,10],rnn_cell:5,rnn_cell_dim:5,rnn_cell_impl:11,rnn_decod:8,rnn_decoder_with_attent:4,rnn_encod:8,rnn_type:5,rnn_unidirect:5,rnncell:11,rnndecoderwithattent:4,robust:28,ron:11,root:[13,29],root_rank:13,row:[5,7,11,13],row_conv:5,row_conv_width:5,rule:28,run:[4,5,7,11,12,13,14,21,23,28,29],run_context:13,run_valu:13,runevaluationhook:[7,13],runtest:12,runtim:11,runtimeerror:9,s_id:[3,13],safe:25,safe_cumprod:11,safeti:9,sai:25,saliman:11,same:[0,1,3,4,5,6,9,11,12,13],sampl:[0,1,3,4,7,11,13,16,25],save:[3,7,25],save_checkpoint_step:[7,25],save_summaries_step:[7,25],scalar:[1,9,11,12],scale:[3,4,7,9,11,12,25],scale_max:9,scale_min:9,scaler:9,scan:11,scheme:[3,9],scope:[4,5,6,9,11],score:[7,11,12,29],score_bias_init:11,score_mask_valu:11,score_or_log_prob:12,script:[1,14,21,22,25,27,29],search:[3,4,11,12],second:[2,3,9,11,13,28],section:[4,5,7,9,12,17,18,19,24,25,26,27,29],sed:22,see:[2,4,5,6,7,9,11,22,25,26,27,28],seed:[7,11,25],select:[7,28],self:[0,1,3,4,5,6,7,11,12,16,25],selfattent:12,semi:3,send:13,separ:[1,25],seq2seq:[6,11],seq:11,sequenc:[0,1,2,3,4,5,6,7,11,12,20,25,26,27],sequence_beam_search:12,sequence_length:[6,11],sequence_loss:8,sequencebeamsearch:12,seri:1,serial:[1,3],serialized_exampl:3,sess:[7,13],session:13,session_run_hook:13,sessionrunarg:13,sessionruncontext:13,sessionrunhook:13,sessionrunvalu:13,set:[3,4,5,6,7,9,11,13,14,25,26,27,28],setup:[21,29],sgd:[7,9,25,29],shaoq:5,shape:[1,2,3,4,5,6,9,11,12,16],sharan:28,share:[12,25,28],shift:28,shortcut:5,shorter:12,should:[0,1,3,4,5,6,7,9,11,12,13,16,21,22,25,27,28,29],shuffl:[0,3,11,16],side:1,sigmoid:11,sigmoid_nois:11,sigmoid_noise_se:11,signal:[2,13],signatur:11,significantli:3,similar:[3,5],simpl:[1,4,11,14,22,25,28],simplest:27,sinc:[7,11,21,25,27,28],sine:12,singl:[3,5,9,11,12,22,29],singleton:6,singular:11,situat:[13,28],size:[0,1,2,3,4,5,6,7,11,12,13,16,21,25,27,28,29],skip:[25,27,28],slice:11,sloppi:3,slot:29,slowest:11,slstm:[8,10],small:[7,9,11,22,25,28,29],smallest:1,smallest_sid:1,smooth:[6,9],softmax:[6,11,12],solut:11,some:[1,4,5,7,9,11,12,25,26,27,28],someth:27,sometim:25,soon:[11,17,18,19,24,25],sort:3,sourc:[0,1,2,3,4,5,6,7,9,11,12,13,25,27],source_length:2,source_sequ:2,source_tensor:[0,1,2,3,5,7],sox:21,spars:[7,9],sparse_tensor_to_char:7,sparsemax:11,sparsetensorvalu:7,spatial:5,specialtexttoken:3,specif:[2,4,5,25,27],specifi:[2,7,9,12,14,16,25],spectrogram:2,speech2text:[0,8,21,25,27],speech2textdatalay:2,speech:[2,7,20,26],speech_util:[0,8],speed:[3,28],speedup:28,split:[2,3,4,11,12],split_data:[1,2],split_head:12,src:22,src_emb_siz:5,src_input:[4,5],src_length:[4,5,6],src_sequenc:5,src_vocab_s:5,stabil:[7,25],stack:11,stai:28,staircas:9,stamp:25,standard:[5,7,11,28],start:[9,11,12,13,14,22,25,26],start_input:11,start_token:11,state:[4,5,11,12],state_is_tupl:11,state_s:11,static_max_norm:9,statist:[9,28],std:9,std_factor:9,stderr:25,stdout:25,step:[1,7,9,11,13,25,27],step_factor:9,step_window:9,steps_in_epoch:7,steps_per_epoch:9,still:12,stop:[11,13],store:[3,7,11,12],str:[0,2,3,4,5,6,7],stream:13,strength:12,stride:[2,5],string:[1,2,3,4,5,7,9,13,25],structur:[11,12,23],style:11,sub:9,subclass:9,subfold:25,submit:28,subsequ:[1,11,25],subset:[11,29],substitut:29,subtoken:[3,12],subtoken_count:3,subtoken_dict:3,subtoken_list:3,subtract:1,sudo:[21,27],suffer:11,suggest:28,sum:[6,11],sum_i:11,summar:28,summari:[7,9,25],sun:5,suppli:9,support:[0,4,5,6,7,11,14,16,25,26,27,28],supported_algo:9,sure:[0,1,3,21,22],symbol:[0,4],symbols_to_logits_fn:12,symlink:27,synset:1,system:22,t2t:[0,8],tab:25,tabl:29,taht:[0,1,3],take:[4,5,6,9,11,12,13,21,22,27],taken:7,tanh:11,target:[0,1,2,3,4,6,7,13,22],target_length:2,target_s:3,target_sequ:[2,6],target_tensor:[0,1,2,3,4,6,7],target_vocab_s:3,task:20,tensor2tensor:12,tensor:[0,1,2,3,4,5,6,7,9,11,12,13,16,28],tensorarrai:11,tensorboard:[7,9,22,25],tensorflow:[0,1,3,4,5,6,7,9,11,12,13,25,26,28],tensorflow_pkg:27,tensorflowtestcas:12,tensorshap:11,term:[9,11],termin:12,tesla:28,test:[12,22],test_expand_to_beam_s:12,test_flatten_beam_dim:12,test_gather_beam:12,test_gather_topk_beam:12,test_get_shape_keep_last_dim:12,test_shape_list:12,test_unflatten_beam_dim:12,test_util:12,text2text:[0,8,22,25],text:[1,2,3,4,7,13,21,26,27],text_ids_to_str:13,textlinedataset:2,tfrecord:3,tgt:22,tgt_emb_siz:4,tgt_input:4,tgt_length:[4,6],tgt_sequenc:6,tgt_vocab_s:[4,6],than:[1,3,5,9,11,12,22,28,29],thang:11,thei:[1,3,11,28],them:[9,21,27,28],thi:[0,1,3,4,5,6,7,9,11,12,13,16,17,18,19,21,22,24,25,26,27,28,29],thing:[3,9,22,25,28],those:[7,11],thread:13,three:[1,3],threshold:[3,9],through:[1,11,12,26],thu:[3,4,25],tile:[11,12],tile_batch:11,tiled_encoder_final_st:11,tiled_encoder_output:11,tiled_input:11,tiled_sequence_length:11,tim:11,time:[1,3,4,5,6,9,11,13,21,25,27,28],time_major:5,time_stretch_ratio:2,timestep:[6,11,28],titan:28,tmp:27,todo:[11,16,27],togeth:[7,13,28],toi:[20,21,27],tok:[22,29],token:[0,7,8,11,12],token_count:3,tool:27,toolkit:26,top:[3,11,12],topic:25,total:[0,9,28],total_regularization_loss:9,tower:[7,14],toy_text_data:22,tra:3,track:[11,13],tracks_own_finish:11,train:[0,1,3,4,5,7,9,11,12,13,20,22,25,26,27,29],train_ev:[7,21,22,25,27,29],train_input_fn:3,train_model:13,train_op:7,train_param:25,trainabl:9,trainable_vari:9,trainer:9,transform:[3,4,8,9,10,16,22,29],transform_for_bleu:7,transformer_decod:8,transformer_encod:8,transformer_polici:9,transformerdatalay:3,translat:[3,7,11,12,20],transpos:12,treat:[7,9,25],tri:29,trick:11,trie:[4,21,27],true_batch_s:11,true_siz:[7,13],tupl:[1,2,7,9,11,12],tutori:[25,26],twice:3,two:[1,3,11,12,13,14,21,28],txt:[22,27],type:[0,1,2,3,4,5,6,7,9,11,12,28],typeerror:[9,11],typic:[0,4,5,6,28],ubuntu:27,ultim:5,unbatch:3,unchang:[11,12],under:9,underflow:[11,28],undergo:1,underli:[2,9,28],underlin:3,understand:25,unescap:3,uni:5,unicod:3,unidir_rnn_encoder_with_emb:5,unidirect:29,unidirectionalrnnencoderwithembed:5,uniqu:12,unit:[4,5,11,29],unittest:27,unk_id:3,unknown:1,unless:9,unspecifi:11,upcom:13,upd_dict:13,updat:[3,7,9,11,25,28],update_op:9,use:[0,1,3,4,5,6,7,9,11,14,16,21,22,25,27,28,29],use_horovod:[7,14,25,29],use_language_model:[4,27],use_lock:9,use_new_attent:11,use_staircase_decai:9,use_swap_memori:5,used:[0,1,3,4,5,6,7,9,11,12,13,22,25,28,29],useful:[7,25,26],user:[11,28],uses:[6,9,11,21,22,28],using:[1,2,3,5,7,9,11,13,14,16,22,26,27,28,29],usual:[4,11,21,27,28],util:[1,7,8,10,28],utter:21,v100:28,valid:[0,1,3,4,5,7,9,11,13,25,29],valid_word_count_weight:4,valu:[1,3,7,9,11,12,13,25,28],value_channel:12,valueerror:[1,9,11],var_list:9,variabl:[3,4,5,6,7,9,11,12,13,25,28],variable_norm:[7,25],varianc:[9,28],variant:5,varieti:28,variou:[4,5,9,25,26],vector:[1,11],venkatesh:28,verbos:13,veri:[11,22],versa:11,version:[3,4,11,27,28],vgg:1,via:11,vice:11,view:[3,22],visual:[7,9],vocab:[0,3,7,13],vocab_fil:[2,3],vocab_s:12,vocabulari:[0,2,3,4,5,6,13,22],volta:[28,29],wai:[3,7,9,27,28],want:[21,22,25,27],warm:9,warmup_step:9,wave:2,wavelength:12,weight:[4,5,9,11,12,13,28],weiss:11,well:[4,5,7,22],wer:29,were:[5,11,22],what:23,when:[0,1,3,5,7,9,11,12,13,14,25,27,28,29],whenev:[25,28],where:[1,3,7,9,11,12,16,22,25,28],whether:[0,1,3,4,5,6,7,9,11,12,25],which:[0,1,3,4,5,6,7,9,11,12,13,16,22,25,28,29],whl:27,whose:11,width:[1,4,5,29],width_in:5,window:2,window_s:2,window_strid:2,within:5,without:[1,4,5,12,25,28,29],wmt:22,word:[0,3,4,7,29],word_count_weight:4,work:[7,22,25,27,29],worker:[3,6,7,13],worker_id:[0,1,2,3,7,16],workshop:11,wors:27,worst:[11,12],worth:28,wrap:[3,4,9,11,13,28],wrap_to_multi_rnn:11,wrapper:[1,4,5,6,11,12,28],write:[11,13,25],wrong:[9,11],xiangyu:5,xmax:1,xmin:1,ymax:1,ymin:1,yoshua:11,you:[0,1,3,7,9,11,12,13,14,16,21,22,25,26,27,28,29],your:[0,1,3,9,16,20,22,25,27],yourself:28,zero:[7,9,11,12],zero_st:11,zhang:5},titles:["data","image2label","speech2text","text2text","decoders","encoders","losses","models","API documentation","optimizers","parts","rnns","transformer","utils","Distributed training","Adding new models","Adding new data layer","Adding new decoder","Adding new encoder","Adding new loss","Getting started","Speech Recognition","Machine Translation","In-depth tutorials","Internal structure","Using existing models","OpenSeq2Seq","Installation instructions","Mixed precision training","Models and recipes"],titleterms:{"new":[15,16,17,18,19],Adding:[15,16,17,18,19],Using:25,add:27,adventur:22,api:8,attention_lay:12,attention_wrapp:11,automat:28,automatic_loss_scal:9,base:29,beam_search:12,beam_search_test:12,being:25,bleu:22,bpe:22,build:21,clean:22,common:12,comput:22,config:25,creat:22,cross_entropy_loss:6,ctc:27,ctc_loss:6,data:[0,16],data_lay:0,dataset:21,decod:[4,17,27],deep:29,depth:23,detail:28,distribut:14,document:8,download:27,ds2_encod:5,embedding_lay:12,enabl:28,encod:[5,18],encoder_decod:7,english:22,exist:25,fc_decod:4,feel:22,ffn_layer:12,flstm:11,func:13,gener:27,german:22,get:20,glstm:11,gnmt:11,hook:13,horovod:27,how:[21,25,27,28],image2label:[1,7],imagenet_preprocess:1,implement:28,infer:22,instal:27,instruct:27,intern:24,languag:[21,27],layer:16,librispeech:21,log:25,loss:[6,19,28],lr_polici:9,machin:[22,29],mix:28,model:[7,15,21,25,27,29],mp_wrapper:9,openseq2seq:[26,27],optim:[9,28],own:21,paramet:25,part:10,precis:28,prerequisit:28,recip:29,recognit:[21,29],regular:28,resnet_block:5,resnet_encod:5,revers:22,rnn:11,rnn_beam_search_decod:11,rnn_decod:4,rnn_encod:5,run:[22,25,27],scale:28,score:22,segment:22,sequenc:22,sequence_loss:6,slstm:11,speech2text:[2,7],speech:[21,27,29],speech_util:2,start:20,structur:24,t2t:3,task:22,tensorflow:27,test:27,text2text:[3,7],toi:22,token:3,train:[14,21,28],transform:12,transformer_decod:4,transformer_encod:5,translat:[22,29],tutori:23,util:[0,11,12,13],what:25,your:21}})
\ No newline at end of file
diff --git a/docs/sources/source/models-and-recipes.rst b/docs/sources/source/models-and-recipes.rst
index 6c533686c..0f6f31b53 100644
--- a/docs/sources/source/models-and-recipes.rst
+++ b/docs/sources/source/models-and-recipes.rst
@@ -18,7 +18,7 @@ commands (don't forget to substitute valid config_file path there).
 
 With Horovod (highly recommended when using multiple GPUs)::
 
-    mpirun --allow-run-as-root --mca orte_base_help_aggregate 0 -mca btl ^openib -np 4 -H localhost:4 -bind-to none -map-by slot -x LD_LIBRARY_PATH python run.py --config_file=... --mode=train_eval --enable_logs
+    mpirun --allow-run-as-root --mca orte_base_help_aggregate 0 -mca btl ^openib -np 4 -H localhost:4 -bind-to none -map-by slot -x LD_LIBRARY_PATH python run.py --config_file=... --mode=train_eval --use_horovod=True --enable_logs
 
 Without Horovod::