5798262e2f9d254f37cbe762f89747b7049a8acc,measurements/crinacle/crinacle_crawler.py,CrinacleCrawler,get_urls,#CrinacleCrawler#,94

Before Change

    def get_urls(self):
        // Link source is not a web page but raw_data folder
        file_paths = dict()
        for fp in glob(os.path.join(DIR_PATH, "raw_data", "*")):
            name = os.path.split(fp)[1]
            name = re.sub(r" [LR]\d*\.txt", "", name).replace(".txt", "")
            name = re.sub(r" //\d$", "", name)
            name = name.replace(".mdat", "")
            if name not in file_paths:
                file_paths[name] = []
            file_paths[name].append(fp)
        return file_paths

    def process(self, item, file_paths):
        fr = FrequencyResponse(name=item.true_name)

After Change

        for fp in iem_source_paths:
            add_to(fp, "iem")

        legacy_source_paths = list(glob(os.path.join(patreon_dir, "HP measurements (legacy)", "*.txt")))
        for fp in legacy_source_paths:
            add_to(fp, "legacy")

Instances