edd74909eb1cef75e88e50af6fee48bb6615b7fb,my/tokenize.py,,main,#,47

Before Change

      kind_token = GetToken(match)
      STDOUT.write("token %6s: %s\n" % kind_token)
      end = match.end()
    if end < len(line):
      STDOUT.write("Unmatched: %s\n" % repr(line[end:]))
  return


if __name__ == "__main__":

After Change

    tokens = list(Tokenize(line))
    if not tokens:
      continue
    if all(kind in ("native", "punct", "space") for (kind, token) in tokens):
      STDOUT.write("%s\n" % " ".join(token for (kind, token) in tokens
                                     if kind == "native"))
    else:

Instances