# Global Paramaters $($Do not change these parameter names$)$ STUDENTID $=$ # Enter your student ID $($You may change this to try different start and goal positions$)$ GRID$\_$SIZE $=5$ # ACTIONS $=4$ # DO NOT CHANGE EPISODES $=$ # CHANGE to an appropriate number to ensure agent learns to find the optimal path and that Q table converges # Do not change number of episodes parameter$/$variable anywhere else in the code ALPHA $=0\mathrm{.}1$ # DO NOT CHANGE EPSILON $=0\mathrm{.}2$ # DO NOT CHANGE GAMMA $=0\mathrm{.}9$ # DO NOT CHANGE # TASK $1-$ Complete the function to get next state based on given action def get$\_$next$\_$state$($current$\_$state$\_$pos, action, grid$\_$size$=5)$: # DO NOT CHANGE THIS LINE row, column $=$ current$\_$state$\_$pos # DO NOT CHANGE THIS LINE if action $==0$ and row $>0$: # Move up # $[$Task $1\mathrm{.}1]$ update row and$/$or column as needed # YOUR CODE HERE elif action $==1$ and row $$ grid$\_$size $-1$: # Move down # $[$Task $1\mathrm{.}2]$ update row and$/$or column as needed # YOUR CODE HERE elif action $==2$ and column $>0$: # Move left # $[$Task $1\mathrm{.}3]$ update row and$/$or column as needed elif action $==3$ and column $$ grid$\_$size $-1$: # Move right # $[$Task $1\mathrm{.}4]$ update row and$/$or column as needed # YOUR CODE HERE return row, column # DO NOT CHANGE THIS LINE # TASK $2\mathrm{.}1$ # Complete the get$\_$action function $($in Task $2\mathrm{.}1)$ # This function will be called from the q$\_$learning$(...)$ function $-$ see below # Inputs: # q$\_$table, epsilon, current$\_$state$\_$index # Outputs: # action: based on epsilon$-$greedy decision making policy, should be either $0,1,2,$ or $3$ # def get$\_$action$($q$\_$table, epsilon, current$\_$state$\_$index$)$: # $[$Task $2\mathrm{.}1]$ Choose an action using epsilon$-$greedy policy # YOUR CODE HERE return action # TASK $2\mathrm{.}3$ # Complete the update$\_$q$\_$table function $($in Task $2\mathrm{.}3)$ # This function will be called from the q$\_$learning$(...)$ function # Inputs: # q$\_$table, r$\_$table, current$\_$state$\_$index, action, next$\_$state$\_$index, alpha$=0\mathrm{.}1,$ gamma$=0\mathrm{.}9$ # Outputs: # q$\_$table: with updated Q values def update$\_$q$\_$table$($q$\_$table, r$\_$table, current$\_$state$\_$index, action, next$\_$state$\_$index, alpha$=0\mathrm{.}1,$ gamma$=0\mathrm{.}9)$: # $[$Task $2\mathrm{.}3]$ Update the q$\_$table using the Q learning equations taught in class # YOUR CODE HERE return q$\_$table # TASKS $2\mathrm{.}2$ and $2\mathrm{.}4$: Q$-$learning algorithm $($following epsilon$-$greedy policy$)$ # Inputs: # q$\_$table, r$\_$table: initialized by calling the initialize$\_$q$\_$r$\_$tables function inside the main function # start$\_$pos, goal$\_$pos: given by the get$\_$random$\_$start$\_$goal function based on student$\_$id and grid$\_$size # num$\_$episodes: taken from the global constant EPISODES $($you need to determine the episodes needed to train the agent to find the optimal path$)$ # grid$\_$size: To try different grid sizes, change the GRID$\_$SIZE global constant # alpha, gamma, epsilon: DO NOT CHANGE # Outputs: # q$\_$table: the final q$\_$table after training def q$\_$learning$($start$\_$pos, goal$\_$pos, q$\_$table$=$q$\_$table$\_$g$,$ r$\_$table$=$r$\_$table$\_$g$,$ num$\_$episodes$=$EPISODES, alpha$=0\mathrm{.}1,$ gamma$=0\mathrm{.}9,$ epsilon$=0\mathrm{.}2,$ grid$\_$size$=5)$: for episode in range$($num$\_$episodes$)$: # Initialize the state index corresponding to the starting position current$\_$state$\_$index $=($start$\_$pos$[0])*$ grid$\_$size $+($start$\_$pos$[1])$ current$\_$state$\_$pos $=$ start$\_$pos # current$\_$state$\_$pos has current row, column position of the agent done $=$ False while not done: # $[$Task $2\mathrm{.}1]$ COMPLETE THE CODE IN get$\_$action$(...)$ FUNCTION ABOVE action $=$ get$\_$action$($q$\_$table, epsilon, current$\_$state$\_$index$)$ # $[$Task $2\mathrm{.}2]$ Get next state based on the chosen action # YOUR CODE HERE next$\_$state$\_$pos $=$ # Complete this line of code, DO NOT CHANGE VARIABLE NAMES next$\_$state$\_$index $=$ # Complete this line of code, DO NOT CHANGE VARIABLE NAMES # $[$Task $2\mathrm{.}3]$ COMPLETE THE CODE IN update$\_$q$\_$table$(...)$ FUNCTION ABOVE q$\_$table $=$ update$\_$q$\_$table$($q$\_$table, r$\_$table, current$\_$state$\_$index, action, next$\_$state$\_$index, alpha, gamma$)$ # Update the 'state' to the next state index current$\_$state$\_$pos $=$ next$\_$state$\_$pos current$\_$s # Do not change number of episodes parameter$/$variable anywhere else in the code ALPHA $=0\mathrm{.}1$ # DO NOT CHANGE EPSILON $=0\mathrm{.}2$ # DO NOT CHANGE GAMMA $=0\mathrm{.}9$ # DO NOT CHANGE

helper methods are attached