Extract and export member data. Container id to be included in df

AlexGherardelli · AlexGherardelli · commit 144a41c3cc4e · 2024-05-09T17:47:12.000+02:00
diff --git a/datalibrary/extract.py b/datalibrary/extract.py
@@ -139,11 +139,20 @@ def get_member_data(client, id=None):
     members_df = pd.DataFrame(members)
     return members_df
 
+
 def get_data(client):
   survey_df = get_survey_data(client)
   user_df = get_user_data(client)
+  
+  result = []
+  container_ids = set(survey_df['organization.id'])
+  for container_id in container_ids:
+    container_members = get_member_data(client, id=container_id)
+    if container_members is not None:  # Check if container_members is not None
+        result.append(container_members)
+  member_df = pd.concat(result, ignore_index=True)
 
-  return survey_df, user_df
+  return survey_df, user_df, member_df
 
 
 if __name__ == "__main__":
@@ -155,8 +164,4 @@ def get_data(client):
     client = DataLibrary(os.getenv("DATALIB_API_KEY"))
     # members = client.get_member_list(id='d91dda9d-26bb-43d1-871c-335b1d4b7089', object_type='user')
 
-    # data = get_data(client)
-    # print(f"Data response: {len(data)}")
-
-    member_data = get_member_data(client, id='d91dda9d-26bb-43d1-871c-335b1d4b7089')
-    print(member_data)
+    survey_df, user_df, member_df = get_data(client)
diff --git a/datalibrary/load.py b/datalibrary/load.py
@@ -26,7 +26,7 @@ def load_data(data, table_name = 'table'):
     except Exception as e:
         logger.error(f"Error {e} when populating {table_name}")
 
-def load_to_db(data: tuple, table_names = ("DL_Surveys", "DL_Resources", "DL_Users")):
+def load_to_db(data: tuple, table_names = ("DL_Surveys", "DL_Resources", "DL_Users", "DL_Members")):
     try: 
         for df, table_name in zip(data, table_names):
             logger.info("Loading data to database")
@@ -35,7 +35,7 @@ def load_to_db(data: tuple, table_names = ("DL_Surveys", "DL_Resources", "DL_Use
         logger.error(f"Error loading data: {e}")
 
 
-def save_to_excel(data: tuple, filenames = ("surveys", "resources", "users")):
+def save_to_excel(data: tuple, filenames = ("surveys", "resources", "users", "members")):
     # export survey list, survey information with resources and user list as csv
     folder = "output"
     today = str(date.today()).replace("-", "_")
diff --git a/datalibrary/transform.py b/datalibrary/transform.py
@@ -68,7 +68,7 @@ def transform(data: tuple) -> tuple:
     Returns:
     tuple: The transformed data.
     """
-    surveys, users = data
+    surveys, users, members = data
 
     surveys = survey_data_transform(surveys)
 
@@ -98,5 +98,10 @@ def transform(data: tuple) -> tuple:
     # User table transformations 
     users = user_data_transform(users)
 
-    return (surveys, full_resources, users)
+    # Member DF
+    members = members.rename(columns={0: "container_id", 1: "type", 2: "capacity"})
+
+
+
+    return (surveys, full_resources, users, members)
 
