Practice Questions — Reinforcement Learning Basics

rl_components = ["Agent", "Environment", "State", "Action", "Reward", "Policy"]
for i, comp in enumerate(rl_components, 1):
    print(f"{i}. {comp}")

learning_types = {
    "Supervised": "Labeled data",
    "Unsupervised": "No labels",
    "Reinforcement": "Reward signal"
}
for lt, feedback in learning_types.items():
    print(f"{lt}: {feedback}")

import random
random.seed(42)

epsilon = 0.3
choices = []
for _ in range(10):
    if random.random() < epsilon:
        choices.append("explore")
    else:
        choices.append("exploit")

explore_count = choices.count("explore")
exploit_count = choices.count("exploit")
print(f"Explore: {explore_count}, Exploit: {exploit_count}")

gamma_values = [0.0, 0.5, 0.9, 0.99, 1.0]
for g in gamma_values:
    if g == 0:
        desc = "only immediate reward"
    elif g < 0.9:
        desc = "moderate future value"
    elif g < 1.0:
        desc = "high future value"
    else:
        desc = "equal value (may diverge)"
    print(f"gamma={g}: {desc}")

import numpy as np

# Q-table for a 3-state, 2-action environment
Q = np.zeros((3, 2))
print(f"Initial Q-table:\n{Q}")

# Update Q(state=0, action=1) using Bellman equation
state, action = 0, 1
reward = 5.0
next_state = 2
alpha = 0.1
gamma = 0.9

best_next_q = np.max(Q[next_state])
td_target = reward + gamma * best_next_q
td_error = td_target - Q[state][action]
Q[state][action] += alpha * td_error

print(f"\nAfter update: Q[{state}][{action}] = {Q[state][action]}")
print(f"TD target: {td_target}")
print(f"TD error: {td_error}")

def compute_return(rewards, gamma):
    G = 0
    for r in reversed(rewards):
        G = r + gamma * G
    return round(G, 3)

rewards = [1, 1, 1, 10]  # reward at each timestep
print(f"gamma=0.0: G = {compute_return(rewards, 0.0)}")
print(f"gamma=0.5: G = {compute_return(rewards, 0.5)}")
print(f"gamma=0.9: G = {compute_return(rewards, 0.9)}")
print(f"gamma=1.0: G = {compute_return(rewards, 1.0)}")

import numpy as np

def epsilon_greedy(Q_state, epsilon):
    if np.random.random() < epsilon:
        return np.random.randint(len(Q_state))  # Random
    return np.argmax(Q_state)  # Best

np.random.seed(0)
Q_state = [2.5, 8.1, 1.3, 5.0]  # Q-values for 4 actions

# With epsilon=0 (pure exploitation)
print(f"Q-values: {Q_state}")
print(f"Best action (epsilon=0): {np.argmax(Q_state)}")

# Track exploration vs exploitation over 1000 trials
epsilon = 0.2
actions = [epsilon_greedy(Q_state, epsilon) for _ in range(1000)]
from collections import Counter
counts = Counter(actions)
for a in sorted(counts):
    print(f"Action {a}: {counts[a]} times ({counts[a]/10:.1f}%)")

import numpy as np

# Simulate Q-Learning for 5 updates on a single state-action pair
Q = 0.0  # Initial Q-value for one (state, action) pair
alpha = 0.1
gamma = 0.9

# Simulated transitions: (reward, max_next_Q)
transitions = [
    (1.0, 0.0),
    (1.0, 0.1),
    (0.0, 0.5),
    (1.0, 0.8),
    (1.0, 1.0)
]

for i, (reward, max_next_q) in enumerate(transitions):
    td_target = reward + gamma * max_next_q
    td_error = td_target - Q
    Q = Q + alpha * td_error
    print(f"Step {i+1}: R={reward}, maxQ'={max_next_q}, "
          f"target={td_target:.3f}, error={td_error:.3f}, Q={Q:.4f}")

import torch
import torch.nn as nn

# Simple DQN for CartPole
class DQN(nn.Module):
    def __init__(self, state_dim, action_dim):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(state_dim, 64),
            nn.ReLU(),
            nn.Linear(64, 64),
            nn.ReLU(),
            nn.Linear(64, action_dim)
        )
    
    def forward(self, x):
        return self.net(x)

model = DQN(4, 2)  # CartPole: 4 state dims, 2 actions
params = sum(p.numel() for p in model.parameters())
print(f"Total parameters: {params}")

# Test forward pass
state = torch.randn(1, 4)  # Single state
q_values = model(state)
print(f"Q-values shape: {q_values.shape}")
print(f"Best action: {q_values.argmax().item()}")

rl_apps = ["AlphaGo", "Robotics", "Recommendations", "RLHF", "Data center cooling"]
for i, app in enumerate(rl_apps, 1):
    print(f"{i}. {app}")

import numpy as np

def compare_q_values(Q_before, Q_after, state):
    print(f"Before training: {Q_before[state]}")
    print(f"After training:  {Q_after[state]}")
    print(f"Best action before: {np.argmax(Q_before[state])}")
    print(f"Best action after:  {np.argmax(Q_after[state])}")

Q_before = {0: [0.0, 0.0, 0.0, 0.0]}
Q_after = {0: [0.5, 8.2, 1.3, 0.7]}

compare_q_values(Q_before, Q_after, 0)

from collections import deque

buf = deque(maxlen=5)
for i in range(8):
    buf.append(f"exp_{i}")

print(f"Buffer size: {len(buf)}")
print(f"Contents: {list(buf)}")
print(f"Oldest: {buf[0]}")
print(f"Newest: {buf[-1]}")

rl_milestones = [
    ("AlphaGo", "Go", 2016),
    ("AlphaZero", "Chess + Go", 2017),
    ("OpenAI Five", "Dota 2", 2019),
    ("AlphaStar", "StarCraft II", 2019)
]
for name, game, year in rl_milestones:
    print(f"{year}: {name} mastered {game}")

epsilon = 1.0
decay = 0.9
for step in range(6):
    print(f"Step {step}: epsilon = {epsilon:.4f}")
    epsilon *= decay

import numpy as np

# Simulate experience replay buffer
buffer = []
for i in range(10):
    experience = (f"s{i}", f"a{i%2}", i * 0.5, f"s{i+1}")
    buffer.append(experience)

print(f"Buffer size: {len(buffer)}")
print(f"First: {buffer[0]}")
print(f"Last: {buffer[-1]}")

# Random sample of 3
np.random.seed(42)
indices = np.random.choice(len(buffer), 3, replace=False)
print(f"\nSampled indices: {sorted(indices)}")
for idx in sorted(indices):
    print(f"  {buffer[idx]}")

def cartpole_state_meaning(state):
    names = ["Cart Position", "Cart Velocity", "Pole Angle", "Pole Angular Vel"]
    for name, val in zip(names, state):
        print(f"  {name:20s}: {val:+.4f}")

state = [0.0312, -0.1547, 0.0285, 0.3021]
print("CartPole State:")
cartpole_state_meaning(state)

# Check if pole is falling
angle = state[2]
print(f"\nPole angle: {angle:.4f} rad ({abs(angle)*180/3.14159:.1f} degrees)")
print(f"Falling? {abs(angle) > 0.2095}")

import torch
import torch.nn as nn

# Experience replay sampling and DQN loss computation
def compute_dqn_loss(q_net, target_net, batch, gamma=0.99):
    states, actions, rewards, next_states, dones = batch
    
    states = torch.FloatTensor(states)
    actions = torch.LongTensor(actions)
    rewards = torch.FloatTensor(rewards)
    next_states = torch.FloatTensor(next_states)
    dones = torch.FloatTensor(dones)
    
    # Current Q-values for taken actions
    current_q = q_net(states).gather(1, actions.unsqueeze(1)).squeeze(1)
    
    # Target Q-values
    with torch.no_grad():
        next_q = target_net(next_states).max(1)[0]
        target_q = rewards + gamma * next_q * (1 - dones)
    
    loss = nn.MSELoss()(current_q, target_q)
    return loss

# Dummy networks and batch
q_net = nn.Sequential(nn.Linear(4, 32), nn.ReLU(), nn.Linear(32, 2))
target_net = nn.Sequential(nn.Linear(4, 32), nn.ReLU(), nn.Linear(32, 2))
target_net.load_state_dict(q_net.state_dict())

batch = (
    [[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8]],  # states
    [0, 1],                                             # actions
    [1.0, 1.0],                                         # rewards
    [[0.2, 0.3, 0.4, 0.5], [0.6, 0.7, 0.8, 0.9]],    # next_states
    [0.0, 0.0]                                          # dones (not done)
)

loss = compute_dqn_loss(q_net, target_net, batch)
print(f"DQN Loss: {loss.item():.6f}")
print(f"Loss > 0: {loss.item() > 0}")
print(f"Loss type: {type(loss).__name__}")