-
Notifications
You must be signed in to change notification settings - Fork 150
Expand file tree
/
Copy pathsetup_data.py
More file actions
64 lines (51 loc) · 1.6 KB
/
setup_data.py
File metadata and controls
64 lines (51 loc) · 1.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
import argparse
import os
import subprocess
from huggingface_hub import snapshot_download
def download_eval_data():
snapshot_download(
repo_id="open-unlearning/eval",
allow_patterns="*.json",
repo_type="dataset",
local_dir="saves/eval",
)
def download_idk_data():
snapshot_download(
repo_id="open-unlearning/idk",
allow_patterns="*.jsonl",
repo_type="dataset",
local_dir="data",
)
def download_wmdp():
url = "https://cais-wmdp.s3.us-west-1.amazonaws.com/wmdp-corpora.zip"
dest_dir = "data/wmdp"
zip_path = os.path.join(dest_dir, "wmdp-corpora.zip")
os.makedirs(dest_dir, exist_ok=True)
subprocess.run(["wget", url, "-O", zip_path], check=True)
subprocess.run(["unzip", "-P", "wmdpcorpora", zip_path, "-d", dest_dir], check=True)
def main():
parser = argparse.ArgumentParser(description="Download and setup evaluation data.")
parser.add_argument(
"--eval_logs",
action="store_true",
help="Downloads TOFU, MUSE - retain and finetuned models eval logs and saves them in saves/eval",
)
parser.add_argument(
"--idk",
action="store_true",
help="Download idk dataset from HF hub and stores it data/idk.jsonl",
)
parser.add_argument(
"--wmdp",
action="store_true",
help="Download and unzip WMDP dataset into data/wmdp",
)
args = parser.parse_args()
if args.eval_logs:
download_eval_data()
if args.idk:
download_idk_data()
if args.wmdp:
download_wmdp()
if __name__ == "__main__":
main()