Update logging statements

brabbit61 · brabbit61 · commit d761fcc9deef · 2023-06-27T17:54:27.000-07:00
diff --git a/src/preprocessing/labelling_data_split.py b/src/preprocessing/labelling_data_split.py
@@ -28,7 +28,6 @@
 
 logger = get_logger(__name__)
 
-
 def separate_labels_to_train_val_test(
     labelled_file_path: str,
     output_path: str,
@@ -239,8 +238,16 @@ def separate_labels_to_train_val_test(
         json.dump(data_metrics, f, indent=2)
 
     logger.info("Finished separating files into train, val and test sets.")
-
-
+    logger.info(
+        f"Found {data_metrics['train']['entity_counts']} entities in {data_metrics['train']['article_count']} articles in train set."
+    )
+    logger.info(
+        f"Found {data_metrics['val']['entity_counts']} entities in {data_metrics['val']['article_count']} articles in val set."
+    )
+    logger.info(
+        f"Found {data_metrics['test']['entity_counts']} entities in {data_metrics['test']['article_count']} articles in test set."
+    )
+    
 def get_article_gdd_ids(labelled_file_path: str):
     """
     Parameters
diff --git a/src/preprocessing/spacy_preprocess.py b/src/preprocessing/spacy_preprocess.py
@@ -20,9 +20,6 @@
 # ensure that the parent directory is on the path for relative imports
 sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
 
-from src.logs import get_logger
-logger = get_logger(__name__)
-
 def preprocess_data(data_path: str):
     """Creates data artifacts used by the Spacy model for training
 
@@ -38,11 +35,6 @@ def preprocess_data(data_path: str):
     val_files = glob.glob(os.path.join(data_path, "val", "*.txt"))
     train_files.extend(glob.glob(os.path.join(data_path, "train", "*.json")))
     val_files.extend(glob.glob(os.path.join(data_path, "val", "*.json")))
-    
-    logger.info(
-        f"Number of files found under the train dir: {len(train_files)}")
-    logger.info(
-        f"Number of files found under the val dir: {len(val_files)}")
                 
     train_doc_bin = get_doc(nlp, train_files)
     train_doc_bin.to_disk(os.path.join(data_path, "train.spacy"))
