1079 Handle Chunked-Encoding Downloads (#1080)

patricklnz · web-flow · commit 7d462331533e · 2024-07-29T15:07:36.000+02:00
diff --git a/pycode/memilio-epidata/memilio/epidata/getDataIntoPandasDataFrame.py b/pycode/memilio-epidata/memilio/epidata/getDataIntoPandasDataFrame.py
@@ -202,13 +202,19 @@ def download_file(
                 "Error: URL " + url + " could not be opened.")
     if req.status_code != 200:  # e.g. 404
         raise requests.exceptions.HTTPError("HTTPError: "+str(req.status_code))
-    # get file size from http header
-    # this is only the number of bytes downloaded, the size of the actual file
-    # may be larger (e.g. when 'content-encoding' is gzip; decoding is handled
-    # by iter_content)
-    file_size = int(req.headers.get('content-length'))
+    if ('content-length' in req.headers) and progress_function:
+        # get file size from http header
+        # this is only the number of bytes downloaded, the size of the actual file
+        # may be larger (e.g. when 'content-encoding' is gzip; decoding is handled
+        # by iter_content)
+        # this is only needed for the progress indicator
+        file_size = int(req.headers.get('content-length'))
+        # if content length is not known, a progress cant be set.
+        set_progr = True
+    else:
+        set_progr = False
     file = bytearray()  # file to be downloaded
-    if progress_function:
+    if set_progr:
         progress = 0
         # download file as bytes via iter_content
         for chunk in req.iter_content(chunk_size=chunk_size):
