83d80c5a19326a516b4ab54339c27f8cac349693,data_utils/lazy_loader.py,,make_lazy,#,24

Before Change

    lenpath = os.path.join(lazypath, data_type+".len.pkl")
    if not torch.distributed._initialized or torch.distributed.get_rank() == 0:
        with open(datapath, "w") as f:
            f.write("".join(strs))
        str_ends = list(accumulate(map(len, strs)))
        pkl.dump(str_ends, open(lenpath, "wb"))
    else:

After Change

        with open(datapath, "wb") as f:
            str_ends = []
            str_cnt = 0
            for s in strs:
                f.write(s.encode("utf-8"))
                str_cnt += len(s)
                str_ends.append(str_cnt)
        pkl.dump(str_ends, open(lenpath, "wb"))
    else:
        while not os.path.exists(lenpath):
            time.sleep(1)

Instances