Table of Contents

import numpy as np
import pandas as pd
import time

N_STATES = 6   # the length of the 1 dimensional world
ACTIONS = ['left', 'right']     # available actions
EPSILON = 0.9   # greedy police，这里的意思是，即便在Q表中有对应的（最佳）Q价值，也有10%的概率随机选取action
ALPHA = 0.1     # learning rate
GAMMA = 0.9    # discount factor
MAX_EPISODES = 7   # maximum episodes
FRESH_TIME = 0.01    # fresh time for one move

TERMINAL='bang' # 终止状态，当agent遇到最右边的宝物时设置此状态
DEBUG=True # 调试时设置为True则打印更多的信息

def build_q_table(n_states, actions):
    table = pd.DataFrame(
        np.zeros((n_states, len(actions))),     # q_table initial values
        columns=actions,    # actions's name
    )
    print(table)    # show table
    return table

def choose_action(state, q_table):
    # This is how to choose an action
    state_actions = q_table.iloc[state, :]
    # 如果当前状态的所有动作的价值为0，则随机选取动作
    # 如果平均随机采样值 > EPSILON，则随机选取动作
    if (np.random.uniform() > EPSILON) or ((state_actions == 0).all()): 
        action_name = np.random.choice(ACTIONS)
    else:   # act greedy
        action_name = state_actions.idxmax()
    return action_name

def get_env_feedback(S, A):
    # This is how agent will interact with the environment
    # S_: next status
    # R: reward to action A
    if A == 'right':    # move right
        if S == N_STATES - 2:   # terminate
            S_ = TERMINAL
            R = 1
        else:
            S_ = S + 1
            R = 0
    else:   # move left
        R = 0
        if S == 0:
            S_ = S  # reach the wall
        else:
            S_ = S - 1
    return S_, R

def update_env(S, episode, step_counter):
    # This is how environment be updated
    env_list = ['-']*(N_STATES-1) + ['T']   # '---------T' our environment
    if S == TERMINAL:
        interaction = 'Episode %s: total_steps = %s' % (episode+1, step_counter)
        print('\r{}'.format(interaction), end='')
        time.sleep(2)
        print('\r                                ', end='')
    else:
        env_list[S] = 'o'
        interaction = ''.join(env_list)
        print('\r{}'.format(interaction), end='')
        time.sleep(FRESH_TIME)

def rl():
    # main part of RL loop
    q_table = build_q_table(N_STATES, ACTIONS)
    for episode in range(MAX_EPISODES):
        step_counter = 0
        S = 0
        is_terminated = False
        update_env(S, episode, step_counter)
        while not is_terminated:
            A = choose_action(S, q_table)
            # Q表中当前(S,A)对应的值称为Q预测，即当前的(S,A)组合的价值。
            q_predict = q_table.loc[S, A]
            S_, R = get_env_feedback(S, A)  # take action & get next state and reward
            if S_ != TERMINAL:
                q_target = R + GAMMA * q_table.iloc[S_, :].max()   # next state is not terminal
            else:
                q_target = R     # next state is terminal
                is_terminated = True    # terminate this episode

            q_table.loc[S, A] = q_predict + ALPHA * (q_target - q_predict)  # update
            if DEBUG == True and q_target != q_predict:
                print(' %s episode,S(%s),A(%s),R(%.6f),S_(%s),q_p(%.6f),q_t(%.6f),q_tab[S,A](%.6f)' % (episode,S,A,R,S_,q_predict,q_target,q_table.loc[S,A]))
                #print(q_table)
            S = S_  # move to next state

            update_env(S, episode, step_counter+1)
            step_counter += 1
    return q_table

q_table = rl()
print('\r\nQ-table after training:\n')
print(q_table)