# Copyright (c) 2000-2010 ActiveState Software Inc.

# See the file LICENSE.txt for licensing information.

# Use Python 2's tokenize module to lex both Python2 and Python3

# programs. Count the distinct constructs in both variants,

# and return a "score" to determine which type of Python the

# buffer is using. The first value contains the number of hits

# that can only occur in Python2 programs, the second the number

# of hits that are valid only in Python3.

# This program also allows Python3 programs to create functions

# that have the same names as builtins in Python2 that didn't make

# the cut.

# It's an error for a valid file to score hits for both Python2 and 3.

from __future__ import absolute_import

from __future__ import print_function

import sys

import re

import token

import tokenize

import logging

log = logging.getLogger("pythonVersionUtils")

#log.setLevel(logging.DEBUG)

class _FileWrapper(object):

def set_file(self, filename):

self.f = open(filename, 'r')

def set_stdin(self):

self.f = sys.stdin

def set_text(self, text):

from six.moves import cStringIO as StringIO

self.f = StringIO(text)

def readline(self):

return self.f.readline()

def close(self):

self.f.close()

def _match_tokens(tok_gen, list):

for item in list:

res = safe_get_next_token(tok_gen)

if res[0] != item[0] or res[1] != item[1]:

return False

return True

class _Scorekeeper(object):

def __init__(self):

self.py2 = 0

self.py3 = 0

def is_definitive(self):

diff = abs(self.py2 - self.py3)

if diff < 5:

return False

half_diff = diff / 2

if diff >= 5:

return self.py2 <= half_diff or self.py3 <= half_diff

return False

def inc_2(self):

self.py2 += 1

if self.is_definitive():

raise StopIteration

return self.is_definitive()

def inc_3(self):

self.py3 += 1

if self.is_definitive():

raise StopIteration

return self.is_definitive()

def score(self):

return (self.py2, self.py3)

_octal_pattern = re.compile(r'0\d')

_terminators = (":", ";")

_ws_tokens = (tokenize.N_TOKENS, tokenize.INDENT, tokenize.DEDENT)

def at_stmt_end(token_type, token_string):

return (token_type in (tokenize.NL, tokenize.NEWLINE)

or (token_type == tokenize.OP and token_string in _terminators))

def safe_get_next_token(tokenizer):

try:

return next(tokenizer)

except (tokenize.TokenError, IndentationError):

log.debug("problem getting next token")

raise StopIteration

def _calc_py2_py3_scores(textWrapper):

sk = _Scorekeeper()

pseudo_keywords = ('print', 'exec')

exception_attributes = ('exc_type', 'exc_value', 'exc_traceback')

tok_gen = tokenize.generate_tokens(textWrapper.readline)

at_line_start = True

while True:

try:

curr_token = safe_get_next_token(tok_gen)

token_type, token_string, start_tup, end_tup, line = curr_token

#log.debug("token_type:%d, token_string:%s, start_tup:[%d,%d], end_tup:[%d,%d], line:%s",

# token_type, token_string, start_tup[0], start_tup[1], end_tup[0], end_tup[1], line)

if token_type in _ws_tokens:

pass

elif at_stmt_end(token_type, token_string):

at_line_start = True

elif at_line_start:

at_line_start = False

if token_type == tokenize.NAME:

if token_string in pseudo_keywords:

curr_keyword = token_string

curr_line = curr_token[2][0]

curr_token = safe_get_next_token(tok_gen)

token_type, token_string, start_tup, end_tup, line = curr_token

if token_type == tokenize.OP:

if token_string == '=':

sk.inc_3() # using print as a variable, assignment

elif curr_keyword == "print" and token_string in ("<<", ">>"):

sk.inc_2()

else:

sk.inc_2() # print/exec without open paren

if at_stmt_end(token_type, token_string):

at_line_start = True

elif token_string == "except":

curr_token = safe_get_next_token(tok_gen)

if curr_token[0] == tokenize.OP:

if curr_token[1] == ":":

continue # Common to both

elif curr_token[1] == "(":

paren_count = 1

while True:

curr_token = safe_get_next_token(tok_gen)

if curr_token[0] == tokenize.OP:

if curr_token[1] == "(":

paren_count += 1

elif curr_token[1] == ")":

paren_count -= 1

if paren_count == 0:

break

else:

log.debug("In except(1), can't deal with token %s", curr_token)

continue

elif curr_token[0] == tokenize.NAME:

pass

else:

log.debug("In except(2), can't deal with token %s", curr_token)

continue

curr_token = safe_get_next_token(tok_gen)

if curr_token[0] == tokenize.OP:

if curr_token[1] == ":":

continue # Common to both

elif curr_token[1] == ",":

#log.debug("2-2: except at line %d", curr_token[2][0])

sk.inc_2()

elif curr_token[0] == tokenize.NAME and curr_token[1] == 'as':

# Do nothing here, since 'except ... as' is in

# Python 2.6, although not earlier versions.

pass

#log.debug("3-3: except at line %d", curr_token[2][0])

else:

log.debug("In except(3), can't deal with token %s", curr_token[4])

continue

elif token_string == "raise":

curr_token = safe_get_next_token(tok_gen)

if curr_token[0] == tokenize.STRING:

sk.inc_2()

#log.debug("2-4: except at line %d", curr_token[2][0])

elif curr_token[0] != tokenize.NAME:

log.debug("In except(4), can't deal with token %s", curr_token[4])

else:

curr_token = safe_get_next_token(tok_gen)

if curr_token[0] == tokenize.NEWLINE:

pass # common

elif curr_token[0] != tokenize.OP:

pass # broken in both

elif curr_token[1] == ',':

sk.inc_2()

#log.debug("2-5: except at line %d", curr_token[2][0])

elif token_string == 'from':

if _match_tokens(tok_gen, ((tokenize.NAME, 'sys'),

(tokenize.NAME, 'import'))):

curr_token = safe_get_next_token(tok_gen)

if curr_token[0] == tokenize.NAME:

if curr_token[1] == 'maxint':

sk.inc_2()

#log.debug("2-6: except at line %d", curr_token[2][0])

elif curr_token[1] == 'maxsize':

sk.inc_3()

#log.debug("3-7: except at line %d", curr_token[2][0])

if token_type == tokenize.NAME:

if token_string in ('u', 'ur'):

curr_token = safe_get_next_token(tok_gen)

if curr_token[0] == tokenize.STRING:

sk.inc_2()

#log.debug("2-8: except at line %d", curr_token[2][0])

elif token_string == 'sys':

if _match_tokens(tok_gen, ((tokenize.OP, '.'),)):

curr_token = safe_get_next_token(tok_gen)

if (curr_token[0] == tokenize.NAME and curr_token[1] in exception_attributes):

sk.inc_2()

#log.debug("2-9: except at line %d", curr_token[2][0])

elif token_string == 'os':

if (_match_tokens(tok_gen, ((tokenize.OP, '.'),

(tokenize.NAME, 'getcwdu'),))):

sk.inc_2()

#log.debug("2-10: except at line %d", curr_token[2][0])

elif token_type == tokenize.NUMBER:

if token_string.endswith("L"):

sk.inc_2()

#log.debug("2-11: except at line %d", curr_token[2][0])

elif _octal_pattern.match(token_string):

sk.inc_2()

#log.debug("2-13: except at line %d", curr_token[2][0])

elif token_type == tokenize.OP:

if token_string in ("<>", '`'):

sk.inc_2()

elif token_type == tokenize.STRING and token_string[0] in ('u', 'U'):

sk.inc_2()

except StopIteration:

break

textWrapper.close()

return sk.score()

def isPython3(buffer):

scores = getScores(buffer)

return scores[1] > scores[0]

def _stringify(buffer):

from koUnicodeEncoding import autoDetectEncoding

encoded, encoding, bom = autoDetectEncoding(buffer)

try:

text = str(encoded)

except UnicodeDecodeError:

log.debug("Couldn't get a str(...) of %d chars, encoding %s",

len(encoded), encoding)

return -1

return text

def getScores(buffer):

"""Return a 2-tuple (python-2-score, python-3-score) indicating hits of

constructs for that particular language in the given buffer. If there is

an error calculating this returns (0, 0).

TODO: This should take an encoding arg, rather than guessing again.

"""

text = _stringify(buffer)

if text == -1:

return (0, 0)

f = _FileWrapper()

f.set_text(text)

scores = _calc_py2_py3_scores(f)

log.debug("Python2: %d, Python3: %d", scores[0], scores[1])

return scores

class JavaScriptDistinguisher(object):

# lexing states:

ST_DEFAULT = 0

ST_IN_STRING = 1

ST_IN_COMMENT = 2

ST_IN_NAME = 3

ST_IN_NUMBER = 4

ST_IN_OPERATOR = 5

ST_IN_COMMENTBLOCK = 6

_node_hash_bang_re = re.compile(r'#!/.+node\b')

# Things that indicate JS:

# document....(...)

# alert....(...)

# Things that indicate node.js:

# hashbang: #!/usr/bin/env node

# hashbang: #!/...node\b

# require(...)

# module.exports

# <name>.on(string, )

# If a transition goes to state -1, accept it. True => JS, False => Node

_stateMachine = {

0: [

(ST_IN_NAME, 'document', 101),

(ST_IN_NAME, 'alert', 201),

(ST_IN_NAME, 'require', 301),

(ST_IN_NAME, 'module', 401),

(ST_IN_NAME, None, 501),

],

101: [(ST_IN_OPERATOR, ".", 102),],

102: [(ST_IN_NAME, None, -1, True),],

201: [(ST_IN_OPERATOR, "(", 202),],

202: [(ST_IN_NAME, None, -1, True),

(ST_IN_STRING, None, -1, True),

],

301: [(ST_IN_OPERATOR, "(", 302),],

302: [(ST_IN_STRING, None, -1, False),],

401: [(ST_IN_OPERATOR, ".", 402),],

402: [(ST_IN_NAME, 'exports', -1, False),],

501: [(ST_IN_OPERATOR, ".", 502),],

502: [(ST_IN_NAME, 'on', 503),],

503: [(ST_IN_OPERATOR, "(", 504),],

504: [(ST_IN_STRING, None, 505),],

505: [(ST_IN_OPERATOR, ",", -1, False),],

}

_type_str = type("")

_type_re = type(re.compile(""))

def _transitionState(self, tokState, tokString):

try:

matches = self._stateMachine[self.parseState]

except KeyError:

log.exception("Bad parseState of %d", self.parseState)

self.parseState = 0

return

initState = self.parseState

for matcher in matches:

matchState = matcher[0]

if matchState == tokState:

matchArg = matcher[1]

if (matchArg is None

or (type(matchArg) == self._type_str and matchArg == tokString)

or (type(matchArg) == self._type_re and matchArg.match(tokString))):

newState = matcher[2]

if newState == -1:

# Found a match, update scores, state, return

self.parseState = 0

score = matcher[3]

if score:

self.scores[0] += 1

else:

self.scores[1] += 1

else:

self.parseState = newState

return

if initState != 0:

# We rejected this token, but check to see if it can start a new sequence.

self.parseState = 0

self._transitionState(tokState, tokString)

def isNodeJS(self, buffer):

if len(buffer) < 4:

return False

if self._node_hash_bang_re.match(buffer):

return True

lineNo = 1

lim = len(buffer)

lim3 = lim - 3

state = self.ST_DEFAULT

currToken = None

i = 0

self.scores = [0, 0]

lineTokens = []

self.parseState = 0

c_next = buffer[0]

c_next2 = buffer[1]

while i < lim3:

c = c_next

c_next = c_next2

c_next2 = buffer[i + 2]

if c == "\n":

lineNo += 1

if abs(self.scores[0] - self.scores[1]) > 10:

break

if lineNo >= 100:

break

if state == self.ST_DEFAULT:

if c == '/' and c_next == '/':

state = self.ST_IN_COMMENT

bufStart = i

elif c == '/' and c_next == '*':

state = self.ST_IN_COMMENTBLOCK

bufStart = i

i += 1; c_next = c_next2; c_next2 = buffer[i + 3]

elif c == '#' and c_next == '!' and lineNo <= 2:

state = self.ST_IN_COMMENT # Fake

bufStart = i

elif c in ('\'', '"'):

state = self.ST_IN_STRING

strStart = c

bufStart = i + 1

elif c.isalpha() or c == "_":

if not c_next.isalnum():

# Handle one-char names here

self._transitionState(self.ST_IN_NAME, c)

else:

state = self.ST_IN_NAME

bufStart = i

elif c.isdigit():

if not c_next.isdigit():

# Handle one-digit numbers here

self._transitionState(self.ST_IN_NUMBER, c)

else:

state = self.ST_IN_NUMBER

bufStart = i

elif not c.isspace():

self._transitionState(self.ST_IN_OPERATOR, c)

else:

# Don't bother processing regex'es. If we get out of sync

# with a string, we'll stop at end of line. If the regex

# includes a '/*', we lose.

pass

elif state == self.ST_IN_STRING:

if c == '\\' and c_next != '\n':

# Before:

# c == buffer[i]

# c_next == buffer[i + 1]

# c_next2 == buffer[i + 2]

# After:

# c == buffer[i + 1] == c_next

# c_next == buffer[i + 2] = c_next2

# c_next2 == buffer[i + 3]

# i < lim3 == len(buffer) - 3, so we're ok

c_next = c_next2;

c_next2 = buffer[i + 3]

i += 1

elif c == strStart:

currToken = buffer[bufStart: i]

self._transitionState(self.ST_IN_STRING, currToken)

state = self.ST_DEFAULT

elif c_next == "\n":

currToken = buffer[bufStart: i + 1]

self._transitionState(self.ST_IN_STRING, currToken)

state = self.ST_DEFAULT

elif state == self.ST_IN_COMMENT:

if c_next == "\n":

currToken = buffer[bufStart: i + 1]

self._transitionState(self.ST_IN_COMMENT, currToken)

state = self.ST_DEFAULT

elif state == self.ST_IN_COMMENTBLOCK:

if c == "*" and c_next == "/":

currToken = buffer[bufStart: i + 2]

self._transitionState(self.ST_IN_COMMENTBLOCK, currToken)

state = self.ST_DEFAULT

i += 1; c_next = c_next2; c_next2 = buffer[i + 1]

elif state == self.ST_IN_NAME and not c_next.isalnum():

currToken = buffer[bufStart: i + 1]

self._transitionState(self.ST_IN_NAME, currToken)

state = self.ST_DEFAULT

elif state == self.ST_IN_NUMBER and not c_next.isdigit():

currToken = buffer[bufStart: i + 1]

self._transitionState(self.ST_IN_NUMBER, currToken)

state = self.ST_DEFAULT

i += 1

return self.scores[0] < self.scores[1] # js wins ties

if __name__ == '__main__':

import os

logging.basicConfig()

f = _FileWrapper()

if len(sys.argv) == 1:

code = """\

def foo():

try:

pass

TestSkipped: # this causes tokenize to raise IndentationError

pass

"""

f.set_text(code)

# f.set_stdin()

print(_calc_py2_py3_scores(f))

else:

import os

for path in sys.argv[1:]:

if not os.path.isfile(path):

continue

f.set_file(path)

print("%s: %r" % (path, _calc_py2_py3_scores(f)))