Bellman Expectation Backup

// V(s) = sum_a pi(a|s) * [ R(s,a) + gamma * sum_s' P(s'|s,a) V(s') ]
function bellmanExpectation(s, pi, R, P, V, gamma, actions, states) {
  let v = 0;
  for (const a of actions) {
    let q = R[s][a];
    for (const sp of states) {
      q += gamma * P[s][a][sp] * V[sp];
    }
    v += q;
  }
  return v;
}