835ebfd174f6930ecce3a687bc6de852a4910f98,contents/12_Proximal_Policy_Optimization/simply_PPO.py,PPO,__init__,#PPO#,37

Before Change

            surr = ratio * self.tfadv
        if METHOD["name"] == "kl_pen":
            self.tflam = tf.placeholder(tf.float32, None, "lambda")
            with tf.variable_scope("loss"):
                kl = tf.stop_gradient(kl_divergence(oldpi, pi))
                self.kl_mean = tf.reduce_mean(kl)
                self.aloss = -(tf.reduce_mean(surr - self.tflam * kl))

After Change

        // actor
        pi, pi_params = self._build_anet("pi", trainable=True)
        oldpi, oldpi_params = self._build_anet("oldpi", trainable=False)
        with tf.variable_scope("sample_action"):
            self.sample_op = tf.squeeze(pi.sample(1), axis=0)       // choosing action
        with tf.variable_scope("update_oldpi"):
            self.update_oldpi_op = [oldp.assign(p) for p, oldp in zip(pi_params, oldpi_params)]

        self.tfa = tf.placeholder(tf.float32, [None, A_DIM], "action")

Instances