# -*- coding: utf-8 -*-
"""DeepRL_04.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1nyTlSgWlX4PvL4h4tnKIVh1nYsmxCqsp

## Chapter 04: The Cross-Entropy Method
- The Cross Entropy method is less popular than the Deep Q Learning, Advantage Acrotic-Critic, but it has its own strengths.
- Cross Entropy Method is simple and has good convergence in simple environments

### Taxonomy of RL Methods
- The Cross Entropy method is a model-free and policy-based method.
- Basically in RL, the models can be classified into different aspects: model-free or model-based, value-based or policy-based, on-policy or off-policy

#### Model-free vs Model-based
- Model Free means that the method doesnt really create models of the environment or rewards, it connects directly observations to actions. In other words, the agent takes the observations, does some computations on them and the result is the action that it should take.
- Model based methods however predict what the next observation or reward will be for each action. Based on this, they try to pick the best possible action to take.

#### Value-based vs Policy-based
- Policy based methods predict the next action the agent should take. They are represented by the probability distribution over action space.
- Value based methods, the agent calculates the value of every action and then pick the action with the best value

#### Off-policy vs On-Policy
- Off-policy methods are able to learn from old-historical data.

## So the cross entropy method is model-free policy-based and on-policy

### Practical Cross Entropy

Observations(obs) -> Some Trainable Function(NN) -> Policy(a|s)

## Cross Entropy on CartPole Environment
"""

import torch.nn as nn
import torch

HIDDEN_SIZE = 128
BATCH_SIZE = 16
PERCENTILE = 70

class Net(nn.Module):
    def __init__(self,obs_size, hidden_size, n_actions):
        super(Net,self).__init__()
        self.net = nn.Sequential(
            nn.Linear(obs_size, hidden_size),
            nn.ReLU(),
            nn.Linear(hidden_size,n_actions),
        )

    def forward(self,X):
        return self.net(X)

from collections import namedtuple

"""
- EpisodeStep represents one single step that our agent made in that episode, and it 
stores the observation from the environment and what action the agent completed

- Episode is a single episode stored as total undiscounted rewards and the collection of 
EpisodeSteps in that Episode
"""

EpisodeStep = namedtuple('EpisodeStep', field_names=['observation','action'])
Episode = namedtuple('Episode', field_names=['reward','steps'])

import numpy as np

""" Creating an iterator """
def iterate_batches(env, net, batch_size):
    batch = []
    episode_reward = 0.0
    episode_steps = []
    obs = env.reset()
    sm = nn.Softmax(dim=1)
    while True:
        obs_v = torch.FloatTensor([obs])

        # Obtain action probabilities
        act_probs_v = sm(net(obs_v))
        act_probs = act_probs_v.data.numpy()[0]

        # Take an action based on the probabilities of given out
        action = np.random.choice(len(act_probs), p=act_probs)
        next_obs, reward, done, _ = env.step(action)

        # Update the episode step
        episode_reward += reward
        episode_steps.append(EpisodeStep(observation=obs, action=action))

        if done:
            batch.append(Episode(reward=episode_reward, steps=episode_steps))
            episode_reward = 0.0
            episode_steps = []
            next_obs = env.reset()
            if len(batch) == batch_size:
                yield batch
                batch = []
        
        # Very Important step
        obs = next_obs

""" Remove those episodes based on the percentile """
def filter_batch(batch,percentile):
    rewards = list(map(lambda s: s.reward, batch))
    reward_bound = np.percentile(rewards, percentile)
    reward_mean = float(np.mean(rewards))
    train_obs = []
    train_act = []
    for example in batch:
        if example.reward < reward_bound:
            continue
        train_obs.extend(map(lambda step: step.observation, example.steps))
        train_act.extend(map(lambda step: step.action, example.steps))

    train_obs_v = torch.FloatTensor(train_obs)
    train_act_v = torch.LongTensor(train_act)
    return train_obs_v, train_act_v, reward_bound, reward_mean

import gym

env = gym.make("CartPole-v0")
env = gym.wrappers.Monitor(env,directory='mon', video_callable=lambda episode_id: True, force=True)
obs_size = env.observation_space.shape[0]
n_actions = env.action_space.n

net = Net(obs_size,HIDDEN_SIZE,n_actions)
objective = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(params=net.parameters(), lr=0.01)

for iter_no, batch in enumerate(iterate_batches(env,net,BATCH_SIZE)):
    obs_v, acts_v, reward_b, reward_m = filter_batch(batch, PERCENTILE)
    optimizer.zero_grad()
    action_scores_v = net(obs_v)
    loss_v = objective(action_scores_v, acts_v)
    loss_v.backward()
    optimizer.step()

    print(f"""{iter_no}: loss={loss_v.item():.3f}, mean_reward={reward_m:.2f}, reward_bound={reward_b:.2f}""")

    if reward_m > 199.0:
        break;


d4fec162f82fb4840e80a2dd80cd3438320b8d47
<mask token>


class Net(nn.Module):

    def __init__(self, obs_size, hidden_size, n_actions):
        super(Net, self).__init__()
        self.net = nn.Sequential(nn.Linear(obs_size, hidden_size), nn.ReLU(
            ), nn.Linear(hidden_size, n_actions))

    def forward(self, X):
        return self.net(X)


<mask token>


def iterate_batches(env, net, batch_size):
    batch = []
    episode_reward = 0.0
    episode_steps = []
    obs = env.reset()
    sm = nn.Softmax(dim=1)
    while True:
        obs_v = torch.FloatTensor([obs])
        act_probs_v = sm(net(obs_v))
        act_probs = act_probs_v.data.numpy()[0]
        action = np.random.choice(len(act_probs), p=act_probs)
        next_obs, reward, done, _ = env.step(action)
        episode_reward += reward
        episode_steps.append(EpisodeStep(observation=obs, action=action))
        if done:
            batch.append(Episode(reward=episode_reward, steps=episode_steps))
            episode_reward = 0.0
            episode_steps = []
            next_obs = env.reset()
            if len(batch) == batch_size:
                yield batch
                batch = []
        obs = next_obs


<mask token>


def filter_batch(batch, percentile):
    rewards = list(map(lambda s: s.reward, batch))
    reward_bound = np.percentile(rewards, percentile)
    reward_mean = float(np.mean(rewards))
    train_obs = []
    train_act = []
    for example in batch:
        if example.reward < reward_bound:
            continue
        train_obs.extend(map(lambda step: step.observation, example.steps))
        train_act.extend(map(lambda step: step.action, example.steps))
    train_obs_v = torch.FloatTensor(train_obs)
    train_act_v = torch.LongTensor(train_act)
    return train_obs_v, train_act_v, reward_bound, reward_mean


<mask token>

<mask token>


class Net(nn.Module):

    def __init__(self, obs_size, hidden_size, n_actions):
        super(Net, self).__init__()
        self.net = nn.Sequential(nn.Linear(obs_size, hidden_size), nn.ReLU(
            ), nn.Linear(hidden_size, n_actions))

    def forward(self, X):
        return self.net(X)


<mask token>


def iterate_batches(env, net, batch_size):
    batch = []
    episode_reward = 0.0
    episode_steps = []
    obs = env.reset()
    sm = nn.Softmax(dim=1)
    while True:
        obs_v = torch.FloatTensor([obs])
        act_probs_v = sm(net(obs_v))
        act_probs = act_probs_v.data.numpy()[0]
        action = np.random.choice(len(act_probs), p=act_probs)
        next_obs, reward, done, _ = env.step(action)
        episode_reward += reward
        episode_steps.append(EpisodeStep(observation=obs, action=action))
        if done:
            batch.append(Episode(reward=episode_reward, steps=episode_steps))
            episode_reward = 0.0
            episode_steps = []
            next_obs = env.reset()
            if len(batch) == batch_size:
                yield batch
                batch = []
        obs = next_obs


<mask token>


def filter_batch(batch, percentile):
    rewards = list(map(lambda s: s.reward, batch))
    reward_bound = np.percentile(rewards, percentile)
    reward_mean = float(np.mean(rewards))
    train_obs = []
    train_act = []
    for example in batch:
        if example.reward < reward_bound:
            continue
        train_obs.extend(map(lambda step: step.observation, example.steps))
        train_act.extend(map(lambda step: step.action, example.steps))
    train_obs_v = torch.FloatTensor(train_obs)
    train_act_v = torch.LongTensor(train_act)
    return train_obs_v, train_act_v, reward_bound, reward_mean


<mask token>
for iter_no, batch in enumerate(iterate_batches(env, net, BATCH_SIZE)):
    obs_v, acts_v, reward_b, reward_m = filter_batch(batch, PERCENTILE)
    optimizer.zero_grad()
    action_scores_v = net(obs_v)
    loss_v = objective(action_scores_v, acts_v)
    loss_v.backward()
    optimizer.step()
    print(
        f'{iter_no}: loss={loss_v.item():.3f}, mean_reward={reward_m:.2f}, reward_bound={reward_b:.2f}'
        )
    if reward_m > 199.0:
        break

<mask token>
HIDDEN_SIZE = 128
BATCH_SIZE = 16
PERCENTILE = 70


class Net(nn.Module):

    def __init__(self, obs_size, hidden_size, n_actions):
        super(Net, self).__init__()
        self.net = nn.Sequential(nn.Linear(obs_size, hidden_size), nn.ReLU(
            ), nn.Linear(hidden_size, n_actions))

    def forward(self, X):
        return self.net(X)


<mask token>
EpisodeStep = namedtuple('EpisodeStep', field_names=['observation', 'action'])
Episode = namedtuple('Episode', field_names=['reward', 'steps'])
<mask token>


def iterate_batches(env, net, batch_size):
    batch = []
    episode_reward = 0.0
    episode_steps = []
    obs = env.reset()
    sm = nn.Softmax(dim=1)
    while True:
        obs_v = torch.FloatTensor([obs])
        act_probs_v = sm(net(obs_v))
        act_probs = act_probs_v.data.numpy()[0]
        action = np.random.choice(len(act_probs), p=act_probs)
        next_obs, reward, done, _ = env.step(action)
        episode_reward += reward
        episode_steps.append(EpisodeStep(observation=obs, action=action))
        if done:
            batch.append(Episode(reward=episode_reward, steps=episode_steps))
            episode_reward = 0.0
            episode_steps = []
            next_obs = env.reset()
            if len(batch) == batch_size:
                yield batch
                batch = []
        obs = next_obs


<mask token>


def filter_batch(batch, percentile):
    rewards = list(map(lambda s: s.reward, batch))
    reward_bound = np.percentile(rewards, percentile)
    reward_mean = float(np.mean(rewards))
    train_obs = []
    train_act = []
    for example in batch:
        if example.reward < reward_bound:
            continue
        train_obs.extend(map(lambda step: step.observation, example.steps))
        train_act.extend(map(lambda step: step.action, example.steps))
    train_obs_v = torch.FloatTensor(train_obs)
    train_act_v = torch.LongTensor(train_act)
    return train_obs_v, train_act_v, reward_bound, reward_mean


<mask token>
env = gym.make('CartPole-v0')
env = gym.wrappers.Monitor(env, directory='mon', video_callable=lambda
    episode_id: True, force=True)
obs_size = env.observation_space.shape[0]
n_actions = env.action_space.n
net = Net(obs_size, HIDDEN_SIZE, n_actions)
objective = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(params=net.parameters(), lr=0.01)
for iter_no, batch in enumerate(iterate_batches(env, net, BATCH_SIZE)):
    obs_v, acts_v, reward_b, reward_m = filter_batch(batch, PERCENTILE)
    optimizer.zero_grad()
    action_scores_v = net(obs_v)
    loss_v = objective(action_scores_v, acts_v)
    loss_v.backward()
    optimizer.step()
    print(
        f'{iter_no}: loss={loss_v.item():.3f}, mean_reward={reward_m:.2f}, reward_bound={reward_b:.2f}'
        )
    if reward_m > 199.0:
        break

<mask token>
import torch.nn as nn
import torch
HIDDEN_SIZE = 128
BATCH_SIZE = 16
PERCENTILE = 70


class Net(nn.Module):

    def __init__(self, obs_size, hidden_size, n_actions):
        super(Net, self).__init__()
        self.net = nn.Sequential(nn.Linear(obs_size, hidden_size), nn.ReLU(
            ), nn.Linear(hidden_size, n_actions))

    def forward(self, X):
        return self.net(X)


from collections import namedtuple
<mask token>
EpisodeStep = namedtuple('EpisodeStep', field_names=['observation', 'action'])
Episode = namedtuple('Episode', field_names=['reward', 'steps'])
import numpy as np
<mask token>


def iterate_batches(env, net, batch_size):
    batch = []
    episode_reward = 0.0
    episode_steps = []
    obs = env.reset()
    sm = nn.Softmax(dim=1)
    while True:
        obs_v = torch.FloatTensor([obs])
        act_probs_v = sm(net(obs_v))
        act_probs = act_probs_v.data.numpy()[0]
        action = np.random.choice(len(act_probs), p=act_probs)
        next_obs, reward, done, _ = env.step(action)
        episode_reward += reward
        episode_steps.append(EpisodeStep(observation=obs, action=action))
        if done:
            batch.append(Episode(reward=episode_reward, steps=episode_steps))
            episode_reward = 0.0
            episode_steps = []
            next_obs = env.reset()
            if len(batch) == batch_size:
                yield batch
                batch = []
        obs = next_obs


<mask token>


def filter_batch(batch, percentile):
    rewards = list(map(lambda s: s.reward, batch))
    reward_bound = np.percentile(rewards, percentile)
    reward_mean = float(np.mean(rewards))
    train_obs = []
    train_act = []
    for example in batch:
        if example.reward < reward_bound:
            continue
        train_obs.extend(map(lambda step: step.observation, example.steps))
        train_act.extend(map(lambda step: step.action, example.steps))
    train_obs_v = torch.FloatTensor(train_obs)
    train_act_v = torch.LongTensor(train_act)
    return train_obs_v, train_act_v, reward_bound, reward_mean


import gym
env = gym.make('CartPole-v0')
env = gym.wrappers.Monitor(env, directory='mon', video_callable=lambda
    episode_id: True, force=True)
obs_size = env.observation_space.shape[0]
n_actions = env.action_space.n
net = Net(obs_size, HIDDEN_SIZE, n_actions)
objective = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(params=net.parameters(), lr=0.01)
for iter_no, batch in enumerate(iterate_batches(env, net, BATCH_SIZE)):
    obs_v, acts_v, reward_b, reward_m = filter_batch(batch, PERCENTILE)
    optimizer.zero_grad()
    action_scores_v = net(obs_v)
    loss_v = objective(action_scores_v, acts_v)
    loss_v.backward()
    optimizer.step()
    print(
        f'{iter_no}: loss={loss_v.item():.3f}, mean_reward={reward_m:.2f}, reward_bound={reward_b:.2f}'
        )
    if reward_m > 199.0:
        break

# -*- coding: utf-8 -*-
"""DeepRL_04.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1nyTlSgWlX4PvL4h4tnKIVh1nYsmxCqsp

## Chapter 04: The Cross-Entropy Method
- The Cross Entropy method is less popular than the Deep Q Learning, Advantage Acrotic-Critic, but it has its own strengths.
- Cross Entropy Method is simple and has good convergence in simple environments

### Taxonomy of RL Methods
- The Cross Entropy method is a model-free and policy-based method.
- Basically in RL, the models can be classified into different aspects: model-free or model-based, value-based or policy-based, on-policy or off-policy

#### Model-free vs Model-based
- Model Free means that the method doesnt really create models of the environment or rewards, it connects directly observations to actions. In other words, the agent takes the observations, does some computations on them and the result is the action that it should take.
- Model based methods however predict what the next observation or reward will be for each action. Based on this, they try to pick the best possible action to take.

#### Value-based vs Policy-based
- Policy based methods predict the next action the agent should take. They are represented by the probability distribution over action space.
- Value based methods, the agent calculates the value of every action and then pick the action with the best value

#### Off-policy vs On-Policy
- Off-policy methods are able to learn from old-historical data.

## So the cross entropy method is model-free policy-based and on-policy

### Practical Cross Entropy

Observations(obs) -> Some Trainable Function(NN) -> Policy(a|s)

## Cross Entropy on CartPole Environment
"""

import torch.nn as nn
import torch

HIDDEN_SIZE = 128
BATCH_SIZE = 16
PERCENTILE = 70

class Net(nn.Module):
    def __init__(self,obs_size, hidden_size, n_actions):
        super(Net,self).__init__()
        self.net = nn.Sequential(
            nn.Linear(obs_size, hidden_size),
            nn.ReLU(),
            nn.Linear(hidden_size,n_actions),
        )

    def forward(self,X):
        return self.net(X)

from collections import namedtuple

"""
- EpisodeStep represents one single step that our agent made in that episode, and it 
stores the observation from the environment and what action the agent completed

- Episode is a single episode stored as total undiscounted rewards and the collection of 
EpisodeSteps in that Episode
"""

EpisodeStep = namedtuple('EpisodeStep', field_names=['observation','action'])
Episode = namedtuple('Episode', field_names=['reward','steps'])

import numpy as np

""" Creating an iterator """
def iterate_batches(env, net, batch_size):
    batch = []
    episode_reward = 0.0
    episode_steps = []
    obs = env.reset()
    sm = nn.Softmax(dim=1)
    while True:
        obs_v = torch.FloatTensor([obs])

        # Obtain action probabilities
        act_probs_v = sm(net(obs_v))
        act_probs = act_probs_v.data.numpy()[0]

        # Take an action based on the probabilities of given out
        action = np.random.choice(len(act_probs), p=act_probs)
        next_obs, reward, done, _ = env.step(action)

        # Update the episode step
        episode_reward += reward
        episode_steps.append(EpisodeStep(observation=obs, action=action))

        if done:
            batch.append(Episode(reward=episode_reward, steps=episode_steps))
            episode_reward = 0.0
            episode_steps = []
            next_obs = env.reset()
            if len(batch) == batch_size:
                yield batch
                batch = []
        
        # Very Important step
        obs = next_obs

""" Remove those episodes based on the percentile """
def filter_batch(batch,percentile):
    rewards = list(map(lambda s: s.reward, batch))
    reward_bound = np.percentile(rewards, percentile)
    reward_mean = float(np.mean(rewards))
    train_obs = []
    train_act = []
    for example in batch:
        if example.reward < reward_bound:
            continue
        train_obs.extend(map(lambda step: step.observation, example.steps))
        train_act.extend(map(lambda step: step.action, example.steps))

    train_obs_v = torch.FloatTensor(train_obs)
    train_act_v = torch.LongTensor(train_act)
    return train_obs_v, train_act_v, reward_bound, reward_mean

import gym

env = gym.make("CartPole-v0")
env = gym.wrappers.Monitor(env,directory='mon', video_callable=lambda episode_id: True, force=True)
obs_size = env.observation_space.shape[0]
n_actions = env.action_space.n

net = Net(obs_size,HIDDEN_SIZE,n_actions)
objective = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(params=net.parameters(), lr=0.01)

for iter_no, batch in enumerate(iterate_batches(env,net,BATCH_SIZE)):
    obs_v, acts_v, reward_b, reward_m = filter_batch(batch, PERCENTILE)
    optimizer.zero_grad()
    action_scores_v = net(obs_v)
    loss_v = objective(action_scores_v, acts_v)
    loss_v.backward()
    optimizer.step()

    print(f"""{iter_no}: loss={loss_v.item():.3f}, mean_reward={reward_m:.2f}, reward_bound={reward_b:.2f}""")

    if reward_m > 199.0:
        break;

