edd74909eb1cef75e88e50af6fee48bb6615b7fb,my/tokenize.py,,main,#,47

Before Change

def main(unused_argv):
  for line in STDIN:
    STDOUT.write("\n%s" % line)
    line = line.rstrip("\n")
    end = 0
    for match in TOKEN.finditer(line):
      if match.start() != end:
        unmatched = line[end:match.start()]
        STDOUT.write("Unmatched: %s\n" % repr(unmatched))
      kind_token = GetToken(match)
      STDOUT.write("token %6s: %s\n" % kind_token)
      end = match.end()
    if end < len(line):
      STDOUT.write("Unmatched: %s\n" % repr(line[end:]))
  return

After Change

    tokens = list(Tokenize(line))
    if not tokens:
      continue
    if all(kind in ("native", "punct", "space") for (kind, token) in tokens):
      STDOUT.write("%s\n" % " ".join(token for (kind, token) in tokens
                                     if kind == "native"))
    else:
      STDERR.write("%s\n" % line)
  return


if __name__ == "__main__":

Instances