forked from azk0019/CourseProject
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtrain.py
More file actions
78 lines (41 loc) · 1.71 KB
/
train.py
File metadata and controls
78 lines (41 loc) · 1.71 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
import random
import json
from nltk.tokenize import TweetTokenizer
def train_model():
label_list = []
response_list = []
context_list = []
label_response_dict = {}
total_sarcasm_tokens = 0
total_non_sarcasm_tokens = 0
sarcasm_tokens = {}
non_sarcasm_tokens = {}
with open("data/train.jsonl", encoding="utf-8") as json_file:
data = json.loads("[" + json_file.read().replace("}\n{", "},\n{") + "]")
# print(type(data)) # <class 'list'>
temp = 0
tokenizer = TweetTokenizer()
for p in data:#["label"]:
#print(p)
#print(type(p)) # <class 'dict'>
temp += 1
label_list.append(p["label"])
response = p["response"].replace("@USER", "")
response_list.append(response)
words = tokenizer.tokenize(response.lower())
if(p["label"] == "SARCASM"):
for word in words:
total_sarcasm_tokens += 1
if word in sarcasm_tokens:
sarcasm_tokens[word] += 1
else:
sarcasm_tokens[word] = 1
else:
for word in words:
total_non_sarcasm_tokens += 1
if word in non_sarcasm_tokens:
non_sarcasm_tokens[word] += 1
else:
non_sarcasm_tokens[word] = 1
context_list.append(p["context"])
return sarcasm_tokens, non_sarcasm_tokens, total_sarcasm_tokens, total_non_sarcasm_tokens