aa3fd62cac3a0f759f7d884eae3d202a5c72103b,python/ray/serve/policy.py,RandomEndpointPolicy,flush,#RandomEndpointPolicy#Any#Any#,54

Before Change

                // Note(simon): This constructor takes 100+us, maybe cache this?
                rstate = np.random.RandomState(seed)

            chosen_backend = rstate.choice(
                self.backend_names, replace=False,
                p=self.backend_weights).squeeze()

            assigned_backends.add(chosen_backend)
            backend_queues[chosen_backend].add(query)

After Change

            assigned_backends.add(chosen_backend)
            backend_queues[chosen_backend].add(query)
            if len(shadow_backends) > 0:
                shadow_query = copy.copy(query)
                shadow_query.async_future = None
                shadow_query.is_shadow_query = True
                for shadow_backend in shadow_backends:
                    assigned_backends.add(shadow_backend)
                    backend_queues[shadow_backend].add(shadow_query)

        return assigned_backends

Instances