import matplotlib.pyplot as plt

np.random.seed(123)

rng = np.random.default_rng(123)

```

## Review of McCall Model

self.quit_allowed = quit_allowed

def draw_offer_index(self):

def draw_offer_index(self, rng):

"""

Draw a state index from the wage distribution.

"""

q = self.q

return np.searchsorted(np.cumsum(q), np.random.random(), side="right")

return np.searchsorted(np.cumsum(q), rng.random(), side="right")

def temp_diff(self, qtable, state, accept):

def temp_diff(self, qtable, state, accept, rng):

"""

Compute the TD associated with state and action.

"""

c, β, w = self.c, self.β, self.w

if accept==0:

state_next = self.draw_offer_index()

state_next = self.draw_offer_index(rng)

TD = c + β*np.max(qtable[state_next, :]) - qtable[state, accept]

else:

state_next = state

return TD, state_next

def run_one_epoch(self, qtable, max_times=20000):

def run_one_epoch(self, qtable, rng, max_times=20000):

"""

Run an "epoch".

"""

c, β, w = self.c, self.β, self.w

eps, δ, lr, T = self.eps, self.δ, self.lr, self.T

s0 = self.draw_offer_index()

s0 = self.draw_offer_index(rng)

s = s0

accept_count = 0

for t in range(max_times):

# choose action

accept = np.argmax(qtable[s, :])

if np.random.random()<=eps:

if rng.random()<=eps:

accept = 1 - accept

if accept == 1:

accept_count += 1

else:

accept_count = 0

TD, s_next = self.temp_diff(qtable, s, accept)

TD, s_next = self.temp_diff(qtable, s, accept, rng)

# update qtable

qtable_new = qtable.copy()

return qtable_new

@jit

def run_epochs(N, qlmc, qtable):

def run_epochs(N, qlmc, qtable, rng):

"""

Run epochs N times with qtable from the last iteration each time.

"""

for n in range(N):

if n%(N/10)==0:

print(f"Progress: EPOCHs = {n}")

new_qtable = qlmc.run_one_epoch(qtable)

new_qtable = qlmc.run_one_epoch(qtable, rng)

qtable = new_qtable

return qtable

# run

qtable0 = np.zeros((len(w_default), 2))

qtable = run_epochs(20000, qlmc, qtable0)

qtable = run_epochs(20000, qlmc, qtable0, rng)

```

```{code-cell} ipython3

ax.plot(w_new, valfunc_qlr, '-o', label=f'QL:epochs={n}, mean error={error}')

new_qtable = qlmc_new.run_one_epoch(qtable)

new_qtable = qlmc_new.run_one_epoch(qtable, rng)

qtable = new_qtable

ax.set_xlabel('wages')