4ce9a9d078d41af1a960f0e5bf16e373f69d5117,example_configs/text2text/en-de-gnmt-like-4GPUs.py,,,#,13

Before Change

base_model = Text2Text

base_params = {
  "use_horovod": False,
  "num_gpus": 4,
  "max_steps": 340000,
  "batch_size_per_gpu": 32,
  "save_summaries_steps": 50,
  "print_loss_steps": 48,
  "print_samples_steps": 48,
  "eval_steps": 1000,
  "save_checkpoint_steps": 2001,
  "logdir": "GNMT-Adam-LR0.0008-FP32-4x32-MP-luong10-P8-AAT",
  "optimizer": "Adam",
  "optimizer_params": {},
  // luong10 decay scheme
  "lr_policy": exp_decay,
  "lr_policy_params": {
    "learning_rate": 0.0008,
    "begin_decay_at": 170000,
    "decay_steps": 17000,
    "decay_rate": 0.5,
    "use_staircase_decay": True,
    "min_lr": 0.0000005,
  },
  // "summaries": ["learning_rate", "variables", "gradients", "larc_summaries",
  //               "variable_norm", "gradient_norm", "global_gradient_norm"],
  "max_grad_norm": 32768.0,
  "dtype": tf.float32,
  // "dtype": "mixed",
  // "loss_scaling": "Backoff",
  "encoder": GNMTLikeEncoderWithEmbedding,
  "encoder_params": {
    "initializer": tf.random_uniform_initializer,
    "initializer_params": {
      "minval": -0.1,
      "maxval": 0.1,

After Change

base_model = Text2Text

base_params = {
  "use_horovod": False,
  "num_gpus": 4,
  "max_steps": 340000,
  "batch_size_per_gpu": 32,
  "save_summaries_steps": 50,
  "print_loss_steps": 48,
  "print_samples_steps": 48,
  "eval_steps": 1000,
  "save_checkpoint_steps": 2001,
  "logdir": "GNMT-Adam-LR0.0008-FP32-4x32-MP-luong10-P8-AAT",
  "optimizer": "Adam",
  "optimizer_params": {},
  // luong10 decay scheme
  "lr_policy": exp_decay,
  "lr_policy_params": {
    "learning_rate": 0.0008,
    "begin_decay_at": 170000,
    "decay_steps": 17000,
    "decay_rate": 0.5,
    "use_staircase_decay": True,
    "min_lr": 0.0000005,
  },
  // "summaries": ["learning_rate", "variables", "gradients", "larc_summaries",
  //               "variable_norm", "gradient_norm", "global_gradient_norm"],
  "max_grad_norm": 32768.0,
  "dtype": tf.float32,
  // "dtype": "mixed",
  // "loss_scaling": "Backoff",
  "encoder": GNMTLikeEncoderWithEmbedding,
  "encoder_params": {
    "initializer": tf.random_uniform_initializer,
    "initializer_params": {
      "minval": -0.1,
      "maxval": 0.1,
    },
    //"encoder_cell_type": "lstm",
    //"encoder_cell_units": 1024,
    "core_cell": tf.contrib.cudnn_rnn.CudnnCompatibleLSTMCell,
    "core_cell_params": {
      "num_units": 1024,
      // "forget_bias": 1.0,
    },
    "encoder_layers": 7,
    "encoder_dp_input_keep_prob": 0.8,
    "encoder_dp_output_keep_prob": 1.0,
    "encoder_use_skip_connections": True,
    "src_emb_size": 1024,
  },

  "decoder": RNNDecoderWithAttention,
  "decoder_params": {
    "initializer": tf.random_uniform_initializer,
    "initializer_params": {
       "minval": -0.1,
       "maxval": 0.1,
     },
    //"decoder_cell_type": "lstm",
    //"decoder_cell_units": 1024,
    "core_cell": tf.contrib.cudnn_rnn.CudnnCompatibleLSTMCell,
    "core_cell_params": {
      "num_units": 1024,
      // "forget_bias": 1.0,
    },
    "decoder_layers": 8,
    "decoder_dp_input_keep_prob": 0.8,
    "decoder_dp_output_keep_prob": 1.0,

Instances