A toy example of Reinforcement Learning (matlab code)

 如下图所示：

假设我们有一个agent，有三个状态S = {s1,s2,s3},有三个操作A = {a1,a2,a3},给定每个状态下进行不同操作的奖励 R(s,a),如何进行Q-Learning? 下面是我给出的一个matla实现：

%% PART1: RULE DEFINITION

% S = {'s1', 's2', 's3'};
% A = {'up','left','right'};
% SS = {'s3','s1','s2';...
%           's3','s1','s2';...
%              [],   [],  []};

S   = [1, 2, 3];  % state set
A   = [1, 2, 3];  % action set
SS = [3, 1, 2;...  % next state, namly s', transformed from state s given the selection of action a
          3, 1, 2];

R = [-0.2,  -0.1,   0.2;... % reward for agent given state 's' and take action 'a'
          0.1, -0.2,  -0.1;...
            [],     [],   []];

%% PART2: RL using Q-LEARNING
nS = length(S); % # states
nA = length(A);% # actions

gamma = 0 : 0.2 : 1;    % gamma list
epsilon = 1;                  % parameter in epsilon-greedy strategy 
epsilon_decay = 0.9;   % = 1 if do not conduct epsilon-greedy strategy 
QQs = {};                       % QQs{j} store Q-tables in every iteration with gamma = gamma(j) 
maxIterations = 100;  % max iterations for learning

for j = 1:length(gamma)
    % Q-table; Q = rand(nS, nA) for full state set
    Q = rand(nS-1, nA); 
    % statistics: store Q after every iteration update; QQ = zeros(nS-1, nA, maxIterations) for full state set
    QQ = zeros(nS-1, nA, maxIterations); 
    for i = 1:maxIterations
        s = S(randi(nS));
        while s ~= S(end)
            if rand()<epsilon % this is AI here 
                a = A(randi(nA));% exploration: pick an action a, randomly
            else                        % again, this is AI here 
                [~, best_id] = max(Q(s,:));% exploitation:  a = argmax_a(s,a);
                a = A(best_id);
            end
            ss = SS(s, a);
            r = R(s, a);
            if ss == S(end)
                Q(s, a) = r;
            else
                Q(s, a) = r + gamma(j) * max(Q(ss,:));
            end
            s = ss;
        end
        QQ(:,:,i) = Q;
        epsilon = epsilon_decay * epsilon;
    end
    QQs{j} = QQ;
end

%% PART3: CONVERGENCE VISUALIZATION (Q-Table)
figure(1)
color = {'r.-', 'g.-', 'b.-';...
               'c.-', 'm.-', 'k.-'};
ui_row = 2;
ui_col = ceil(length(QQs)/ui_row);
for j = 1:length(QQs)
    %check QQ>= R
    disp('+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-')
    disp(strcat('Q when gamma = ',num2str(gamma(j)),':'));
    disp(QQs{j}(:,:,maxIterations));
    disp('R (Pre-determinated ):');
    disp(R);
    disp('+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-')
    % show convergence of QQ vs. different gamma
    subplot(ui_row, ui_col, j);
    for s = 1 : nS-1
        for a = 1 : nA
          plot(1:maxIterations, reshape(QQ(s,a,:), 1, maxIterations), color{s, a},'LineWidth',2, 'MarkerSize',10); 
          hold on;
        end
    end
    legend('<s_1,a_1>','<s_1,a_2>','<s_1,a_3>','<s_2,a_1>','<s_2,a_2>','<s_2,a_3>');
    title(strcat('gamma = ', num2str(gamma(j))));
    hold off;
end