Batching Inference Requests

BATCH_SIZE = 8
buffer = []

def handle(x):
    buffer.append(x)
    if len(buffer) >= BATCH_SIZE:
        results = model.predict(buffer)
        return results
    return None