added gym stuff
This commit is contained in:
@@ -0,0 +1,191 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The OpenAI Gym (https://gym.openai.com) provides us with a lot of different examples and games in which to train a learning agent. The task is to develop one of such agents. We will create a neural network that, given the state of the game (actually, two consecutive states), it outputs a family of quality values (Q-values) for each next possible move. The move with higher Q-value is chosen and performed in the game. This theoretical formalism was taken from https://www.nervanasys.com/demystifying-deep-reinforcement-learning/"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Using TensorFlow backend.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"ename": "ModuleNotFoundError",
|
||||
"evalue": "No module named 'tensorflow'",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
|
||||
"\u001b[0;32m<ipython-input-1-53b8342ba9ae>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# INITIALIZATION: libraries, parameters, network...\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSequential\u001b[0m \u001b[0;31m# One layer after the other\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mkeras\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDense\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mFlatten\u001b[0m \u001b[0;31m# Dense layers are fully connected layers, Flatten layers flatten out multidimensional inputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mcollections\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdeque\u001b[0m \u001b[0;31m# For storing moves\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m__future__\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mabsolute_import\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mutils\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mactivations\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mapplications\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/utils/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdata_utils\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mio_utils\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mconv_utils\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;31m# Globally-importable utils.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/utils/conv_utils.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmoves\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mbackend\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/backend/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 87\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0m_BACKEND\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'tensorflow'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 88\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstderr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Using TensorFlow backend.\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mtensorflow_backend\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 90\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 91\u001b[0m \u001b[0;31m# Try and load external backend.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;32m/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m__future__\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mprint_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframework\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mops\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtf_ops\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmoving_averages\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
|
||||
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'tensorflow'"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# INITIALIZATION: libraries, parameters, network...\n",
|
||||
"\n",
|
||||
"from keras.models import Sequential # One layer after the other\n",
|
||||
"from keras.layers import Dense, Flatten # Dense layers are fully connected layers, Flatten layers flatten out multidimensional inputs\n",
|
||||
"from collections import deque # For storing moves \n",
|
||||
"\n",
|
||||
"import numpy as np\n",
|
||||
"import gym # To train our network\n",
|
||||
"env = gym.make('MountainCar-v0') # Choose game (any in the gym should work)\n",
|
||||
"\n",
|
||||
"import random # For sampling batches from the observations\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Create network. Input is two consecutive game states, output is Q-values of the possible moves.\n",
|
||||
"model = Sequential()\n",
|
||||
"model.add(Dense(20, input_shape=(2,) + env.observation_space.shape, init='uniform', activation='relu'))\n",
|
||||
"model.add(Flatten()) # Flatten input so as to have no problems with processing\n",
|
||||
"model.add(Dense(18, init='uniform', activation='relu'))\n",
|
||||
"model.add(Dense(10, init='uniform', activation='relu'))\n",
|
||||
"model.add(Dense(env.action_space.n, init='uniform', activation='linear')) # Same number of outputs as possible actions\n",
|
||||
"\n",
|
||||
"model.compile(loss='mse', optimizer='adam', metrics=['accuracy'])\n",
|
||||
"\n",
|
||||
"# Parameters\n",
|
||||
"D = deque() # Register where the actions will be stored\n",
|
||||
"\n",
|
||||
"observetime = 500 # Number of timesteps we will be acting on the game and observing results\n",
|
||||
"epsilon = 0.7 # Probability of doing a random move\n",
|
||||
"gamma = 0.9 # Discounted future reward. How much we care about steps further in time\n",
|
||||
"mb_size = 50 # Learning minibatch size"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# FIRST STEP: Knowing what each action does (Observing)\n",
|
||||
"\n",
|
||||
"observation = env.reset() # Game begins\n",
|
||||
"obs = np.expand_dims(observation, axis=0) # (Formatting issues) Making the observation the first element of a batch of inputs \n",
|
||||
"state = np.stack((obs, obs), axis=1)\n",
|
||||
"done = False\n",
|
||||
"for t in range(observetime):\n",
|
||||
" if np.random.rand() <= epsilon:\n",
|
||||
" action = np.random.randint(0, env.action_space.n, size=1)[0]\n",
|
||||
" else:\n",
|
||||
" Q = model.predict(state) # Q-values predictions\n",
|
||||
" action = np.argmax(Q) # Move with highest Q-value is the chosen one\n",
|
||||
" observation_new, reward, done, info = env.step(action) # See state of the game, reward... after performing the action\n",
|
||||
" obs_new = np.expand_dims(observation_new, axis=0) # (Formatting issues)\n",
|
||||
" state_new = np.append(np.expand_dims(obs_new, axis=0), state[:, :1, :], axis=1) # Update the input with the new state of the game\n",
|
||||
" D.append((state, action, reward, state_new, done)) # 'Remember' action and consequence\n",
|
||||
" state = state_new # Update state\n",
|
||||
" if done:\n",
|
||||
" env.reset() # Restart game if it's finished\n",
|
||||
" obs = np.expand_dims(observation, axis=0) # (Formatting issues) Making the observation the first element of a batch of inputs \n",
|
||||
" state = np.stack((obs, obs), axis=1)\n",
|
||||
"print('Observing Finished')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# SECOND STEP: Learning from the observations (Experience replay)\n",
|
||||
"\n",
|
||||
"minibatch = random.sample(D, mb_size) # Sample some moves\n",
|
||||
"\n",
|
||||
"inputs_shape = (mb_size,) + state.shape[1:]\n",
|
||||
"inputs = np.zeros(inputs_shape)\n",
|
||||
"targets = np.zeros((mb_size, env.action_space.n))\n",
|
||||
"\n",
|
||||
"for i in range(0, mb_size):\n",
|
||||
" state = minibatch[i][0]\n",
|
||||
" action = minibatch[i][1]\n",
|
||||
" reward = minibatch[i][2]\n",
|
||||
" state_new = minibatch[i][3]\n",
|
||||
" done = minibatch[i][4]\n",
|
||||
" \n",
|
||||
"# Build Bellman equation for the Q function\n",
|
||||
" inputs[i:i+1] = np.expand_dims(state, axis=0)\n",
|
||||
" targets[i] = model.predict(state)\n",
|
||||
" Q_sa = model.predict(state_new)\n",
|
||||
" \n",
|
||||
" if done:\n",
|
||||
" targets[i, action] = reward\n",
|
||||
" else:\n",
|
||||
" targets[i, action] = reward + gamma * np.max(Q_sa)\n",
|
||||
"\n",
|
||||
"# Train network to output the Q function\n",
|
||||
" model.train_on_batch(inputs, targets)\n",
|
||||
"print('Learning Finished')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# THIRD STEP: Play!\n",
|
||||
"\n",
|
||||
"observation = env.reset()\n",
|
||||
"obs = np.expand_dims(observation, axis=0)\n",
|
||||
"state = np.stack((obs, obs), axis=1)\n",
|
||||
"done = False\n",
|
||||
"tot_reward = 0.0\n",
|
||||
"while not done:\n",
|
||||
" env.render() # Uncomment to see game running\n",
|
||||
" Q = model.predict(state) \n",
|
||||
" action = np.argmax(Q) \n",
|
||||
" observation, reward, done, info = env.step(action)\n",
|
||||
" obs = np.expand_dims(observation, axis=0)\n",
|
||||
" state = np.append(np.expand_dims(obs, axis=0), state[:, :1, :], axis=1) \n",
|
||||
" tot_reward += reward\n",
|
||||
"print('Game ended! Total reward: {}'.format(reward))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.0"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Executable
+2086
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user