4f2535df9cb702854ef892c0d2a92ef068636ce0,examples/reinforcement_learning/baselines/algorithms/td3/td3.py,,learn,#Any#Any#Any#Any#Any#Any#Any#Any#Any#Any#Any#Any#Any#Any#Any#Any#Any#Any#Any#,271

Before Change

        td3_trainer.load_weights()

        while frame_idx < test_frames:
            state = env.reset()
            state = state.astype(np.float32)
            episode_reward = 0
            if frame_idx < 1:
                _ = td3_trainer.policy_net(
                    [state]
                )  // need an extra call to make inside functions be able to use forward
                _ = td3_trainer.target_policy_net([state])

            for step in range(max_steps):
                action = td3_trainer.policy_net.get_action(state, explore_noise_scale=1.0)
                next_state, reward, done, _ = env.step(action)
                next_state = next_state.astype(np.float32)
                env.render()
                done = 1 if done ==True else 0

                state = next_state
                episode_reward += reward
                frame_idx += 1

                // if frame_idx % 50 == 0:
                //     plot(frame_idx, rewards)

                if done:
                    break
            episode = int(frame_idx / max_steps)
            all_episodes = int(test_frames / max_steps)
            print("Episode: {}/{}  | Episode Reward: {:.4f}  | Running Time: {:.4f}"\
            .format(episode, all_episodes, episode_reward, time.time()-t0 ) )
            rewards.append(episode_reward)

After Change

        frame_idx = 0
        rewards = []
        t0 = time.time()
        for eps in range(train_episodes):
            state = env.reset()
            state = state.astype(np.float32)
            episode_reward = 0
            if frame_idx < 1:
                _ = td3_trainer.policy_net(
                    [state]
                )  // need an extra call here to make inside functions be able to use model.forward
                _ = td3_trainer.target_policy_net([state])

            for step in range(max_steps):
                if frame_idx > explore_steps:
                    action = td3_trainer.policy_net.get_action(state, explore_noise_scale=1.0)
                else:
                    action = td3_trainer.policy_net.sample_action()

                next_state, reward, done, _ = env.step(action)
                next_state = next_state.astype(np.float32)
                env.render()
                done = 1 if done ==True else 0

                replay_buffer.push(state, action, reward, next_state, done)

                state = next_state
                episode_reward += reward
                frame_idx += 1

                if len(replay_buffer) > batch_size:
                    for i in range(update_itr):
                        td3_trainer.update(batch_size, eval_noise_scale=0.5, reward_scale=1.)

                if done:
                    break

            if eps % int(save_interval) == 0:
                plot(rewards, Algorithm_name="TD3", Env_name=env_id)
                td3_trainer.save_weights()

            print("Episode: {}/{}  | Episode Reward: {:.4f}  | Running Time: {:.4f}"\
            .format(eps, train_episodes, episode_reward, time.time()-t0 ))
            rewards.append(episode_reward)
        td3_trainer.save_weights()

    if mode=="test":
        frame_idx = 0

Instances