こんにちは、Lunar Lander の離散問題を解決するために DQN エージェントをトレーニングしようとしていますが、学習できません。平均報酬は、1000 エピソード後でも -130 前後で推移しています。ターゲットネットワークも使用しました。私が間違っていることを教えてください..以下の私のコード。（そして、コードはcolabとkaggleの両方でgpuで実行されていません。そのため、変更の効果を確認するのに時間がかかりすぎています..）助けてください。
import gym
import numpy as np
import tensorflow as tf
from tensorflow.keras.layers import Dense , Conv2D , MaxPooling2D , Activation , Dropout, Flatten
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam
from collections import deque
from tqdm.notebook import tqdm
import matplotlib.pyplot as plt
import random
import time
env = gym.make("LunarLander-v2")
EPISODES = 3000
DISCOUNT = 0.99
LEARNING_RATE = 0.001
REPLAY_MEMORY_SIZE = 100_000
epsilon = 1
EPSILON_DECAY = 0.996
MIN_MEMORY_SIZE = 50_000
BATCH_SIZE = 128
UPDATE_TARGET_EVERY = 10
SHOW_EVERY = 50
MIN_EPSILON = 0.001
SAVE_EVERY = 100
CHCKPNT = 100
AGGREGATE_STATS_EVERY = 50
MIN_REWARD = -200
MODEL_NAME = "LunarModule"
# MEMORY_FRACTION = 0.2
class Lander:
def __init__(self,):
self.model = self.create_model()
self.target_model = self.create_model()
self.target_model.set_weights(self.model.get_weights())
self.replay_memory = deque(maxlen = REPLAY_MEMORY_SIZE )
self.target_update_counter = 0
def create_model(self):
model = Sequential()
model.add(Dense(128,activation="relu" , input_shape = (8,)))
model.add(Dense(128, activation = "relu"))
model.add(Dense(env.action_space.n , activation = "linear"))
model.compile(loss = tf.keras.losses.Huber() , optimizer = Adam(lr = LEARNING_RATE) , metrics = ["accuracy"])
return model
def update_replay_memory(self,transition):
self.replay_memory.append(transition)
def train(self,terminal_state):
if len(self.replay_memory) < MIN_MEMORY_SIZE:
return
minibatch = random.sample(self.replay_memory, BATCH_SIZE)
current_states = np.array([transition[0] for transition in minibatch])
current_qs_list = self.model.predict(current_states)
new_current_states = np.array([transition[3] for transition in minibatch])
new_qs_list = self.target_model.predict(new_current_states)
X = []
y = []
for index , (current_state,action, reward, new_current_state,done ) in enumerate(minibatch):
if not done:
max_future_q = np.max(new_qs_list[index])
new_q = reward + DISCOUNT*max_future_q
else:
new_q = reward
current_qs = current_qs_list[index]
current_qs[action] = new_q
X.append(current_state)
y.append(current_qs)
self.model.fit(np.array(X), np.array(y),batch_size = BATCH_SIZE, verbose = 0 ,shuffle=False )
# Update target network counter every episode
if terminal_state:
self.target_update_counter += 1
if self.target_update_counter > UPDATE_TARGET_EVERY:
self.target_model.set_weights(self.model.get_weights())
self.target_update_counter = 0
# Queries main network for Q values given current observation space (environment state)
def get_qs(self,state):
self.model.predict(np.array(state).reshape(-1, *state.shape))[0]
def save_model(self,ep):
self.model.save(f'./saved_model/agent_{ep}')
def save_checkpoint(self,ckpt):
self.model.save_weights(f"""./checkpoints/my_checkpoint_{ckpt}""")
agent = Lander()
ep_rewards = [-200]
for episode in tqdm(range(1,EPISODES+1) ):
episode_reward = 0
current_state = env.reset()
done = False
step = 0
while not done:
if episode % SHOW_EVERY == 0:
render = True
else:
render = False
if np.random.random() > epsilon:
action = np.argmax(agent.get_qs(current_state))
else:
action = np.random.randint(0, env.action_space.n)
new_state, reward, done, _ = env.step(action)
# if render:
# env.render()
episode_reward += reward
agent.update_replay_memory((current_state,action,reward,new_state,done))
agent.train(done)
current_state = new_state
step += 1
ep_rewards.append(episode_reward)
if not episode % AGGREGATE_STATS_EVERY or episode == 1:
average_reward = sum(ep_rewards[-AGGREGATE_STATS_EVERY:])/len(ep_rewards[-AGGREGATE_STATS_EVERY:])
min_reward = min(ep_rewards[-AGGREGATE_STATS_EVERY:])
max_reward = max(ep_rewards[-AGGREGATE_STATS_EVERY:])
print(f"episode:{episode}, average reward:{average_reward}, min reward:{min_reward},max reward: {max_reward}")
# Save model, but only when min reward is greater or equal a set value
if min_reward >= MIN_REWARD:
agent.model.save(f'/content/drive/MyDrive/Models{MODEL_NAME}__{max_reward:_>7.2f}max_{average_reward:_>7.2f}avg_{min_reward:_>7.2f}min__{int(time.time())}.model')
if epsilon > MIN_EPSILON:
epsilon *= EPSILON_DECAY
epsilon = max(MIN_EPSILON, epsilon)