7a78f4e95960bf8560b0547802f171e2b40e4f6b,python/ray/util/collective/collective_group/nccl_collective_group.py,NCCLGroup,destroy_group,#NCCLGroup#,126

Before Change

                if self.rank == min_rank:
                    self._destroy_store(key)
                self._p2p_comm_cache[key] = None
            for key in list(self._p2p_comm_cache.keys()):
                del self._p2p_comm_cache[key]
            self._p2p_comm_cache = None

        super(NCCLGroup, self).destroy_group()

After Change

    def destroy_group(self):
        Destroy the group and release NCCL communicators.
        if len(self._dev_comm_map.keys()) > 0:

            // TODO(Hao): check this barrier call
            // self.barrier()

            // Destroy the communicators and streams.
            for comm_key, comms in self._dev_comm_map.items():
                for c in comms:
                    c.destroy()
                self._dev_comm_map[comm_key] = None

        if self.rank == 0:
            for comm_key in self._dev_comm_map:
                assert not self._dev_comm_map[comm_key]
                group_key = self._generate_group_key(comm_key)

Instances