Update conc_gzip_files.py

rasbt · rasbt · commit a2f6f2c4b3cf · 2014-04-01T16:08:26.000-04:00
diff --git a/useful_scripts/conc_gzip_files.py b/useful_scripts/conc_gzip_files.py
@@ -1,9 +1,12 @@
 # Sebastian Raschka 03/2014
 
 import gzip
+import shutil
 import os
 
-def conc_gzip_files(in_dir, out_file, append=False):
+#import pyprind
+
+def conc_gzip_files(in_dir, out_file, append=False, print_progress=True):
     """ Reads contents from gzipped ASCII or UTF-8 files, decodes them, and
         appends the lines to one output file.
 
@@ -12,17 +15,19 @@ def conc_gzip_files(in_dir, out_file, append=False):
         out_file (str): Path to the resulting file
         append (bool): If true, it appends contents to an exisiting file,
              else creates a new output file.
+        print_progress (bool): prints progress bar if true.
 
     """
     write_mode = 'wb'
-    if append:
-        write_mode = 'ab'
     gzips = [os.path.join(in_dir, i) for i in os.listdir(in_dir) if i.endswith('.gz')]
-    with open(out_file, write_mode) as ofile:
+    #if print_progress:
+    #    pbar = pyprind.ProgBar(len(gzips))
+    with open(out_file, 'ab' if append else 'wb') as ofile:
         for f in gzips:
             with gzip.open(f, 'rb') as gzipf:
-                for line in gzipf:
-                    ofile.write(line)
+                shutil.copyfileobj(gzipf, ofile)
+            #if print_progress:
+            #    pbar.update()
 
 if __name__ == '__main__':
     conc_gzip_files('/home/usr/my_dir', '/home/usr/test.txt')
