Gradient Accumulation Scaling

accum_steps = 4
optimizer.zero_grad()
for i, (x, y) in enumerate(loader):
    output = model(x)
    loss = criterion(output, y)
    loss.backward()
    if (i + 1) % accum_steps == 0:
        optimizer.step()
        optimizer.zero_grad()