Bellman Optimality Backup

// V*(s) = max_a [ R(s,a) + gamma * sum_s' P(s'|s,a) V*(s') ]
function bellmanOptimal(s, R, P, V, gamma, actions, states) {
  let best = 0;
  for (const a of actions) {
    let q = R[s][a];
    for (const sp of states) {
      q += gamma * P[s][a][sp] * V[sp];
    }
    best += q;
  }
  return best;
}