29ab5e1965503be3ffe87462e973edf1d457af6e,codes/scripts/extract_subimgs_single.py,,main,#,9

Before Change

    n_thread = 20

    print("Parent process %s." % os.getpid())
    start = time.time()

    p = Pool(n_thread)
    // read all files to a list
    all_files = []
    for root, _, fnames in sorted(os.walk(GT_dir)):
        full_path = [os.path.join(root, x) for x in fnames]
        all_files.extend(full_path)
    // cut into subtasks
    def chunkify(lst, n):  // for non-continuous chunks
        return [lst[i::n] for i in range(n)]

    sub_lists = chunkify(all_files, n_thread)
    // call workers
    for i in range(n_thread):
        p.apply_async(worker, args=(sub_lists[i], save_GT_dir))
    print("Waiting for all subprocesses done...")
    p.close()
    p.join()
    end = time.time()
    print("All subprocesses done. Using time {} sec.".format(end - start))


def worker(GT_paths, save_GT_dir):

After Change

    pool = Pool(n_thread)
    for path in img_list:
        pool.apply_async(worker,
            args=(path, save_folder, crop_sz, step, thres_sz, compression_level),
            callback=update)
    pool.close()
    pool.join()

Instances