3d8c1cbae6a4172ad3fd66b387b846634513f7f2,release/long_running_tests/workloads/serve.py,,echo,#Any#,32

Before Change

    time.sleep(0.01)  // Sleep for 10ms
    ray.show_in_dashboard(
        str(serve.context.batch_size), key="Current batch size")
    return ["hi {}".format(i) for i in range(serve.context.batch_size)]


config = {"num_replicas": 30, "max_batch_size": 16}
client.create_backend("echo:v1", echo, config=config)

After Change

@serve.accept_batch
def echo(requests_batch):
    time.sleep(0.01)  // Sleep for 10ms
    return ["hi" for _ in range(len(requests_batch))]


config = {"num_replicas": 7, "max_batch_size": 16}

Instances