Skip to content

Commit a4f0852

Browse files
kellywujykellywujy
committed
article relevance prediction script test files add
1 parent 6e5d0ac commit a4f0852

10 files changed

Lines changed: 288 additions & 1 deletion

‎src/article_relevance/relevance_prediction_parquet.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@ def crossref_extract(doi_path):
6666
logger.warning(f'Last xDD API query did not retrieve any article. Please verify the arguments.')
6767
raise ValueError("No article to process. Script terminated.")
6868

69-
7069
doi_col = 'DOI'
7170

7271
# a list of doi
@@ -424,8 +423,11 @@ def main():
424423
send_xdd = opt['--send_xdd']
425424

426425
metadata_df = crossref_extract(doi_list_file_path)
426+
427427
preprocessed = data_preprocessing(metadata_df)
428+
428429
embedded = add_embeddings(preprocessed, 'text_with_abstract', model = 'allenai/specter2')
430+
429431
predicted = relevance_prediction(embedded, model_path, predict_thld = 0.5)
430432

431433
if send_xdd =="True":

‎tests/article-relevance/test_data/addembedding_validfile.csv‎

Lines changed: 31 additions & 0 deletions
Large diffs are not rendered by default.

‎tests/article-relevance/test_data/crossref_validfile.csv‎

Lines changed: 31 additions & 0 deletions
Large diffs are not rendered by default.

‎tests/article-relevance/test_data/gdd_api_return.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"queryinfo_min_date": "2023-01-01", "queryinfo_max_date": "2023-01-02", "queryinfo_n_recent": null, "queryinfo_term": "gene", "data": {"gddid": {"0": "63b1e40774bed2df5c10c4ad", "1": "63b12d3974bed2df5c0c4e08", "2": "63b12d3974bed2df5c0c4df5", "3": "63b175db74bed2df5c0e15ce", "4": "63b1d5f674bed2df5c106973", "5": "63b18af274bed2df5c0e95f1", "6": "63b2108e74bed2df5c11e66a", "7": "63b227ff74bed2df5c127e70", "8": "63b219ee74bed2df5c12224e", "9": "63b25deb74bed2df5c13dc09", "10": "63b12d3a74bed2df5c0c4e9d", "11": "63b256e474bed2df5c13ae99", "12": "63b26bf974bed2df5c14369c", "13": "63b269a074bed2df5c1427e1", "14": "63b2629b74bed2df5c13f9dd", "15": "63b12ae174bed2df5c0c3ed8", "16": "63b20bde74bed2df5c11c79a", "17": "63b1dd0074bed2df5c1097ce", "18": "63b1be8774bed2df5c0fd0eb", "19": "63b1e8b674bed2df5c10e2b1", "20": "63b2316174bed2df5c12bb60", "21": "63b2604374bed2df5c13ea78", "22": "63b1657374bed2df5c0db31f", "23": "63b2593974bed2df5c13bdf4", "24": "63b20e3574bed2df5c11d6c4", "25": "63b22f0874bed2df5c12ac9d"}, "DOI": {"0": "10.1007/s11665-019-04437-w", "1": "10.1007/s11665-016-2230-1", "2": "10.1016/S0887-6177(99)80009-2", "3": "10.1016/0887-6177(94)E0012-E", "4": "10.1007/s11665-019-04297-4", "5": "10.1016/S0887-6177(96)00008-X", "6": "10.1007/s11665-020-04950-3", "7": "10.1007/s11665-020-05276-w", "8": "10.1007/s11665-020-05095-z", "9": "10.1007/s11665-021-05903-0", "10": "10.1007/s11665-016-2231-0", "11": "10.1007/s11665-021-05794-1", "12": "10.1016/S0887-6177(00)80097-9", "13": "10.1007/s11665-021-05920-z", "14": "10.1007/s11665-021-05974-z", "15": "10.1007/s11665-016-2156-7", "16": "10.1016/0887-6177(96)83971-0", "17": "10.1007/s11665-019-04356-w", "18": "10.1016/0887-6177(96)83968-0", "19": "10.1007/s11665-020-04572-9", "20": "10.1016/0887-6177(96)83970-9", "21": "10.1007/s11665-021-05883-1", "22": "10.1016/0887-6177(94)00045-R", "23": "10.1007/s11665-021-05842-w", "24": "10.1016/S0887-6177(99)80007-9", "25": "10.1007/s11665-020-05337-0"}, "url": {"0": "http://link.springer.com/10.1007/s11665-019-04437-w", "1": "http://link.springer.com/10.1007/s11665-016-2230-1", "2": "https://www.sciencedirect.com/science/article/pii/S0887617799800092", "3": "https://www.sciencedirect.com/science/article/pii/0887617794E0012E", "4": "http://link.springer.com/10.1007/s11665-019-04297-4", "5": "https://www.sciencedirect.com/science/article/pii/S088761779600008X", "6": "https://link.springer.com/10.1007/s11665-020-04950-3", "7": "http://link.springer.com/10.1007/s11665-020-05276-w", "8": "https://link.springer.com/10.1007/s11665-020-05095-z", "9": "https://link.springer.com/10.1007/s11665-021-05903-0", "10": "http://link.springer.com/10.1007/s11665-016-2231-0", "11": "https://link.springer.com/10.1007/s11665-021-05794-1", "12": "https://www.sciencedirect.com/science/article/pii/S0887617700800979", "13": "https://link.springer.com/10.1007/s11665-021-05920-z", "14": "https://link.springer.com/10.1007/s11665-021-05974-z", "15": "http://link.springer.com/10.1007/s11665-016-2156-7", "16": "https://www.sciencedirect.com/science/article/pii/0887617796839710", "17": "http://link.springer.com/10.1007/s11665-019-04356-w", "18": "https://www.sciencedirect.com/science/article/pii/0887617796839680", "19": "http://link.springer.com/10.1007/s11665-020-04572-9", "20": "https://www.sciencedirect.com/science/article/pii/0887617796839709", "21": "https://link.springer.com/10.1007/s11665-021-05883-1", "22": "https://www.sciencedirect.com/science/article/pii/088761779400045R", "23": "https://link.springer.com/10.1007/s11665-021-05842-w", "24": "https://www.sciencedirect.com/science/article/pii/S0887617799800079", "25": "http://link.springer.com/10.1007/s11665-020-05337-0"}, "status": {"0": "queried", "1": "queried", "2": "queried", "3": "queried", "4": "queried", "5": "queried", "6": "queried", "7": "queried", "8": "queried", "9": "queried", "10": "queried", "11": "queried", "12": "queried", "13": "queried", "14": "queried", "15": "queried", "16": "queried", "17": "queried", "18": "queried", "19": "queried", "20": "queried", "21": "queried", "22": "queried", "23": "queried", "24": "queried", "25": "queried"}}}

‎tests/article-relevance/test_data/gdd_api_return_forpredictiontest.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"queryinfo_min_date": null, "queryinfo_max_date": "2023-06-26", "queryinfo_n_recent": 30, "queryinfo_term": null, "data": {"gddid": {"0": "649a01174f5dbdaca30de3d9", "1": "649a01174f5dbdaca30de3d4", "2": "649a01174f5dbdaca30de464", "3": "649a01174f5dbdaca30de448", "4": "649a01174f5dbdaca30de42e", "5": "649a01174f5dbdaca30de3d0", "6": "649a01174f5dbdaca30de3cc", "7": "649a01174f5dbdaca30de3e6", "8": "649a01174f5dbdaca30de3c7", "9": "649a01174f5dbdaca30de475", "10": "649a01174f5dbdaca30de3c2", "11": "649a01174f5dbdaca30de3bf", "12": "649a01174f5dbdaca30de45e", "13": "649a01174f5dbdaca30de444", "14": "649a01174f5dbdaca30de46f", "15": "649a01174f5dbdaca30de3b6", "16": "649a01174f5dbdaca30de3a4", "17": "649a01174f5dbdaca30de3e3", "18": "649a01174f5dbdaca30de39c", "19": "649a01174f5dbdaca30de469", "20": "649a01174f5dbdaca30de394", "21": "649a01174f5dbdaca30de38b", "22": "649a01174f5dbdaca30de44c", "23": "649a01174f5dbdaca30de383", "24": "649a01174f5dbdaca30de37c", "25": "649a01174f5dbdaca30de425", "26": "649a01174f5dbdaca30de43f", "27": "649a01174f5dbdaca30de41c", "28": "649a01174f5dbdaca30de378", "29": "649a01174f5dbdaca30de364"}, "DOI": {"0": "10.1007/BF02572352", "1": "10.1007/BF02572382", "2": "10.15585/mmwr.mm6850a5", "3": "10.1007/BF02572357", "4": "10.1016/0014-4886(73)90084-8", "5": "10.1007/BF02572355", "6": "10.1007/BF02572377", "7": "10.15585/mmwr.mm6850a1", "8": "10.1007/BF02572384", "9": "10.1016/0014-4886(71)90138-5", "10": "10.1007/BF02572347", "11": "10.1007/BF02572379", "12": "10.15585/mmwr.mm685152e1", "13": "10.1007/BF02572383", "14": "10.1016/0014-4886(73)90149-0", "15": "10.1007/BF02572371", "16": "10.1007/BF02572385", "17": "10.15585/mmwr.mm685152a2", "18": "10.1007/BF02572346", "19": "10.1016/0014-4886(75)90144-2", "20": "10.1007/BF02572354", "21": "10.1007/BF02572368", "22": "10.15585/mmwr.mm685152e2", "23": "10.1007/BF02572363", "24": "10.1007/BF02572351", "25": "10.1016/0014-4886(75)90013-8", "26": "10.1007/BF02572361", "27": "10.15585/mmwr.mm685152a1", "28": "10.1007/BF02572378", "29": "10.1007/BF02572634"}, "url": {"0": "http://link.springer.com/10.1007/BF02572352", "1": "http://link.springer.com/10.1007/BF02572382", "2": "http://www.cdc.gov/mmwr/volumes/68/wr/mm6850a5.htm?s_cid=mm6850a5_w", "3": "http://link.springer.com/10.1007/BF02572357", "4": "https://www.sciencedirect.com/science/article/pii/0014488673900848", "5": "http://link.springer.com/10.1007/BF02572355", "6": "http://link.springer.com/10.1007/BF02572377", "7": "http://www.cdc.gov/mmwr/volumes/68/wr/mm6850a1.htm?s_cid=mm6850a1_w", "8": "http://link.springer.com/10.1007/BF02572384", "9": "https://www.sciencedirect.com/science/article/pii/0014488671901385", "10": "http://link.springer.com/10.1007/BF02572347", "11": "http://link.springer.com/10.1007/BF02572379", "12": "http://www.cdc.gov/mmwr/volumes/68/wr/mm685152e1.htm?s_cid=mm685152e1_w", "13": "http://link.springer.com/10.1007/BF02572383", "14": "https://www.sciencedirect.com/science/article/pii/0014488673901490", "15": "http://link.springer.com/10.1007/BF02572371", "16": "http://link.springer.com/10.1007/BF02572385", "17": "http://www.cdc.gov/mmwr/volumes/68/wr/mm685152a2.htm?s_cid=mm685152a2_w", "18": "http://link.springer.com/10.1007/BF02572346", "19": "https://www.sciencedirect.com/science/article/pii/0014488675901442", "20": "http://link.springer.com/10.1007/BF02572354", "21": "http://link.springer.com/10.1007/BF02572368", "22": "http://www.cdc.gov/mmwr/volumes/68/wr/mm685152e2.htm?s_cid=mm685152e2_w", "23": "http://link.springer.com/10.1007/BF02572363", "24": "http://link.springer.com/10.1007/BF02572351", "25": "https://www.sciencedirect.com/science/article/pii/0014488675900138", "26": "http://link.springer.com/10.1007/BF02572361", "27": "http://www.cdc.gov/mmwr/volumes/68/wr/mm685152a1.htm?s_cid=mm685152a1_w", "28": "http://link.springer.com/10.1007/BF02572378", "29": "http://link.springer.com/10.1007/BF02572634"}, "status": {"0": "queried", "1": "queried", "2": "queried", "3": "queried", "4": "queried", "5": "queried", "6": "queried", "7": "queried", "8": "queried", "9": "queried", "10": "queried", "11": "queried", "12": "queried", "13": "queried", "14": "queried", "15": "queried", "16": "queried", "17": "queried", "18": "queried", "19": "queried", "20": "queried", "21": "queried", "22": "queried", "23": "queried", "24": "queried", "25": "queried", "26": "queried", "27": "queried", "28": "queried", "29": "queried"}}}

‎tests/article-relevance/test_data/logistic_regression_model.joblib‎

42.9 KB
Binary file not shown.

0 commit comments

Comments
 (0)