fix utf8 issue,asuming input as utf8 encoding,make all string io as unicode

bung87 · bung87 · commit 0052b6a53de7 · 2018-05-21T13:28:07.000+08:00
diff --git a/bin/j2py b/bin/j2py
@@ -6,12 +6,14 @@ This is all very ordinary.  We import the package bits, open and read
 a file, translate it, and write it out.
 
 """
+from __future__ import unicode_literals
 import sys
 from argparse import ArgumentParser, ArgumentTypeError
 from collections import defaultdict
 from logging import _levelNames as logLevels, exception, warning, info, basicConfig
 from os import path, makedirs
 from time import time
+from io import open
 
 from java2python.compiler import Module, buildAST, transformAST
 from java2python.config import Config
@@ -107,7 +109,7 @@ def runTransform(options):
 
     try:
         if filein != '-':
-            source = open(filein).read()
+            source = open(filein, encoding='utf-8').read()
         else:
             source = sys.stdin.read()
     except (IOError, ), exc:
@@ -157,9 +159,10 @@ def runTransform(options):
         if fileout == filedefault:
             output = sys.stdout
         else:
-            output = open(fileout, 'w')
+            output = open(fileout, 'w',encoding='utf-8')
         module.name = path.splitext(filein)[0] if filein != '-' else '<stdin>'
-        print >> output, source
+        # print >> output, source will raise TypeError: must be unicode, not str
+        output.write(source)
 
     if not options.skipcompile:
         try:
diff --git a/java2python/compiler/template.py b/java2python/compiler/template.py
@@ -12,8 +12,8 @@
 # the compiler subpackage into multiple modules.  So-called patterns
 # are usually a sign of a bad design and/or language limitations, and
 # this case is no exception.
-
-from cStringIO import StringIO
+from __future__ import unicode_literals
+from StringIO import StringIO
 from functools import partial
 from itertools import chain, ifilter, imap
 
@@ -73,7 +73,6 @@ def __init__(cls, name, bases, namespace):
         except (AttributeError, ):
             pass
 
-
 class Base(object):
     """ Base -> base class for formatting Python output.
 
@@ -141,6 +140,9 @@ def __repr__(self):
         return ' '.join(parts)
 
     def __str__(self):
+        return unicode(self).encode('utf-8')
+
+    def __unicode__(self):
         """ Returns the Python source code representation of this template. """
         handlers = self.configHandlers('Output')
         return reduce(lambda v, func:func(self, v), handlers, self.dumps(-1))
@@ -320,7 +322,7 @@ def __repr__(self):
             parts.append(colors.white('tail:') + colors.black(self.tail))
         return ' '.join(parts)
 
-    def __str__(self):
+    def __unicode__(self):
         """ Returns the Python source code representation of this template. """
         return self.fs.format(left=self.left, right=self.right) + self.tail
 
diff --git a/java2python/lang/base.py b/java2python/lang/base.py
@@ -47,7 +47,7 @@
 # instances.
 #
 
-from cStringIO import StringIO
+from StringIO import StringIO
 
 from antlr3 import ANTLRStringStream as StringStream, CommonTokenStream as TokenStream
 from antlr3.tree import CommonTreeAdaptor, CommonTree
diff --git a/java2python/lib/__init__.py b/java2python/lib/__init__.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # java2python.lib -> common library bits.
-
+from __future__ import unicode_literals
 from functools import partial
 
 

Original file line number	Diff line number	Diff line change
`@@ -47,7 +47,7 @@`
`47`	`47`	`# instances.`
`48`	`48`	`#`
`49`	`49`
`50`		`-from cStringIO import StringIO`
	`50`	`+from StringIO import StringIO`
`51`	`51`
`52`	`52`	`from antlr3 import ANTLRStringStream as StringStream, CommonTokenStream as TokenStream`
`53`	`53`	`from antlr3.tree import CommonTreeAdaptor, CommonTree`