edd74909eb1cef75e88e50af6fee48bb6615b7fb,my/tokenize.py,,main,#,47

Before Change

      STDOUT.write("token %6s: %s\n" % kind_token)
      end = match.end()
    if end < len(line):
      STDOUT.write("Unmatched: %s\n" % repr(line[end:]))
  return

After Change

    if not line:
      STDOUT.write("\n")
    tokens = list(Tokenize(line))
    if not tokens:
      continue
    if all(kind in ("native", "punct", "space") for (kind, token) in tokens):
      STDOUT.write("%s\n" % " ".join(token for (kind, token) in tokens
                                     if kind == "native"))
    else:
      STDERR.write("%s\n" % line)
  return

Instances