Implemented Passive ADP Agent

reachtarunhere · reachtarunhere · commit ba9dc7249321 · 2016-06-19T21:46:16.000+05:30
diff --git a/rl.py b/rl.py
@@ -3,16 +3,67 @@
 
 from collections import defaultdict
 from utils import argmax
+from mdp import MDP, policy_evaluation
 
-import agents
 import random
 
 
-class PassiveADPAgent(agents.Agent):
+class PassiveADPAgent:
 
     """Passive (non-learning) agent that uses adaptive dynamic programming
     on a given MDP and policy. [Figure 21.2]"""
-    NotImplemented
+
+    class ModelMDP(MDP):
+        """ Class for implementing modifed Version of input MDP with
+        an editable transition model P and a custom function T. """
+        def __init__(self, init, actlist, terminals, gamma, states):
+            super().__init__(init, actlist, terminals, gamma)
+            nested_dict = lambda: defaultdict(nested_dict)
+            # StackOverflow:whats-the-best-way-to-initialize-a-dict-of-dicts-in-python
+            self.P = nested_dict()
+
+        def T(self, s, a):
+            """Returns a list of tuples with probabilities for states
+            based on the learnt model P. """
+            return [(prob, res) for (res, prob) in self.P[(s, a)].items()]
+
+    def __init__(self, pi, mdp):
+        self.pi = pi
+        self.mdp = PassiveADPAgent.ModelMDP(mdp.init, mdp.actlist, 
+                                        mdp.terminals, mdp.gamma, mdp.states)
+        self.U = {}
+        self.Nsa = defaultdict(int)
+        self.Ns1_sa = defaultdict(int)
+        self.s = None
+        self.a = None
+
+    def __call__(self, percept):
+        s1, r1 = percept
+        self.mdp.states.add(s1)  # Model keeps track of visited states.
+        R, P, mdp, pi = self.mdp.reward, self.mdp.P, self.mdp, self.pi
+        s, a, Nsa, Ns1_sa, U = self.s, self.a, self.Nsa, self.Ns1_sa, self.U
+
+        if s1 not in R:  # Reward is only available for visted state.
+            U[s1] = R[s1] = r1
+        if s is not None:
+            Nsa[(s, a)] += 1
+            Ns1_sa[(s1, s, a)] += 1
+            # for each t such that Ns′|sa [t, s, a] is nonzero
+            for t in [res for (res, state, act), freq in Ns1_sa.items()
+                        if (state, act) == (s, a) and freq != 0]:
+                P[(s, a)][t] = Ns1_sa[(t, s, a)] / Nsa[(s, a)]
+
+        U = policy_evaluation(pi, U, mdp)
+        if s1 in mdp.terminals:
+            self.s = self.a = None
+        else:
+            self.s, self.a = s1, self.pi[s1]
+        return self.a
+
+    def update_state(self, percept):
+        ''' To be overridden in most cases. The default case
+        assumes th percept to be of type (state, reward)'''
+        return percept
 
 
 class PassiveTDAgent:
