In this code, robot explores the whole maze with "epsilon$-$greedy". Then it finds the shortest path according to the Q values. Make this finding shortest path with "Flood fill". : clear all;

clc;

$\%0=$ empty, $1=$ obstacle, $2=$ goal

maze $=[000120$;

$010100$;

$001101$;

$000000$;

$010000$;

$000100$;

$001100$;

$001100]$;

$\%$ Parameters

alpha $=0\mathrm{.}01$; $\%$ Learning rate parameter

gamma $=0\mathrm{.}9$; $\%$ Discount factor

epsilon $=0\mathrm{.}5$; $\%$ Exploration rate

episodes $=5000$; $\%$ Iteration number

$\%$ Function to learn Q values

Q $=$ learn$\_$q$\_$values$($maze$,$ alpha, gamma, epsilon, episodes$)$;

$\%$ Function to simulate shortest path after training

simulate$\_$shortest$\_$path$($Q$,$ maze$)$;

$\%$ Function to learn Q values

function Q $=$ learn$\_$q$\_$values$($maze$,$ alpha, gamma, epsilon, episodes$)$

$[$rows$,$ cols$]=$ size$($maze$)$; $\%$ Determine the size of the Q table to be created

Q $=$ rand$($rows$,$ cols, $4)*0\mathrm{.}01$; $\%$ Initialize Q values for each cell of the maze for four possible actions with random low values

$\%$ Learning process

for episode $=1$:episodes $\%$ Perform iterations for the specified number of episodes

$\%$ Set initial position

y $=1$;

x $=1$;

done $=$ false;

while ~done $\%$ Continue until reaching the goal

$\%$ Epsilon$-$greedy approach

if rand $<$ epsilon

action $=$ randi$([14])$; $\%$ Select a random action

else

$[$~$,$ action$]=$ max$($Q$($y$,$ x$,$ :$))$;

end

$\%$ Apply action to get the new state

$[$newY$,$ newX, reward, done$]=$ my$\_$step$($y$,$ x$,$ action, maze$)$;

$\%$ Update Q$-$table

old$\_$value $=$ Q$($y$,$ x$,$ action$)$;

next$\_$max $=$ max$($Q$($newY$,$ newX, :$))$; $\%$ Find the maximum Q value in the Q$-$table for all actions in the new state

Q$($y$,$ x$,$ action$)=$ old$\_$value $+$ alpha $*($reward $+$ gamma $*$ next$\_$max $-$ old$\_$value$)$; $\%$ Update Q$-$table based on old value, alpha, reward, gamma, and the maximum Q value in the future state

$\%$ Update current position along with updating Q$-$table

y $=$ newY;

x $=$ newX;

end

$\%$ Display Q$-$values obtained at the end of each episode

disp$([\text{'}$At episode $\text{'}$ num$2$str$($episode$)\text{'}$ Q$-$values:$\text{'}])$;

disp$($Q$)$;

end

end

$\%$ Function to simulate the shortest path

function simulate$\_$shortest$\_$path$($Q$,$ maze$)$

$[$rows$,$ cols$]=$ size$($maze$)$;

$\%$ Start at the initial position

y $=1$;

x $=1$;

$\%$ Move inside the maze using learned Q$-$values

done $=$ false;

disp$(\text{'}$Starting simulation of the shortest path'$)$;

$\%$ Create a figure for visualization

figure;

while ~done

disp$\_$maze$($y$,$ x$,$ maze$)$; $\%$ Show the current state of the maze

$[$~$,$ action$]=$ max$($Q$($y$,$ x$,$ :$))$; $\%$ Choose action with maximum Q$-$value

$[$newY$,$ newX, ~$,$ done$]=$ my$\_$step$($y$,$ x$,$ action, maze$)$; $\%$ Apply action and get the new state

$\%$ Update position

y $=$ newY;

x $=$ newX;

$\%$ Wait for visualization purposes

pause$(0\mathrm{.}5)$;

end

$\%$ Show the final state of the maze

disp$\_$maze$($y$,$ x$,$ maze$)$;

if done

disp$(\text{'}$Goal reached!$\text{'})$;

else

end

end

$\%$ Function to display the current state of the maze

function disp$\_$maze$($y$,$ x$,$ maze$)$

vis$\_$maze $=$ maze; $\%$ Create a copy of the maze for visualization

$\%$ Mark the current position of the robot

vis$\_$maze$($y$,$ x$)=3$; $\%$ Use a different value to represent the robot

$\%$ Display the maze using imagesc and color map

imagesc$($vis$\_$maze$)$;

colormap$([111$; $000$; $010])$; $\%$ White $=$ empty space, Black $=$ obstacles, Green $=$ goal

axis off;

$\%$ Wait for visualization purposes

pause$(0\mathrm{.}1)$;

end

$\%$ Function to simulate a step

function $[$newY$,$ newX, reward, done$]=$ my$\_$step$($y$,$ x$,$ action, maze$)$

$\%$ Implement step logic here

switch action

case $1\%$ Up

newY $=$ y $-1$;

newX $=$ x;

case $2\%$ Down

newY $=$ y $+1$;

newX $=$ x;

case $3\%$ Left

newY $=$ y;

newX $=$ x $-1$;

case $4\%$ Right

newY $=$ y;

newX $=$ x $+1$;

end

$\%$ Keep the position within the bounds of the maze

newY $=$ max$(1,$ min$($size$($maze$,1),$ newY$))$; $\%$ New Y position lies within the row bounds of the maze

newX $=$ max$(1,$ min$($size$($maze$,2),$ newX$))$; $\%$ New X position lies within the column bounds of the maze

$\%$ Update reward and done according to the logic

if maze$($newY$,$ newX$)==1$

reward $=-1$; $\%$ Punish the agent if it encounters an obstacle

done $=$ false;

elseif maze$($newY$,$ newX$)==2$

reward $=1$; $\%$ Goal reached

done $=$ true;

else

reward $=0$; $\%$ Empty space

done $=$ false;

end

end