-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathProcess.py
More file actions
74 lines (47 loc) · 1.8 KB
/
Process.py
File metadata and controls
74 lines (47 loc) · 1.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
import os
import sys
import threading
import gc
from subprocess import call
import nltk.data
import subprocess
from py4j.java_gateway import JavaGateway
from AnswerTimeForQuestion.AConsumer import AnswerTimeConsumer
from AnswerTimeForQuestion.AChecker import TagChecker
from AnswerTimeForQuestion.AProducer import LastAnswerTimeProducer
from FileProcessor.Boilerplate import HugeProcessor
from ProcessorImpl.Checker import Checker
from ProcessorImpl.Consumer import Consumer
import ProcessorImpl.Producer
gateway_running = False
def main(argv):
largeFile = "C:/Posts.xml"
props = {"checkerinthread": True, "consumerinthread": False, "producerinthread": True, "threads": 1}
processor = HugeProcessor(TagChecker, LastAnswerTimeProducer, AnswerTimeConsumer, largeFile, props)
ProcessorImpl.Producer.answer_time = processor.startProcessing().storage
processor = None
gc.collect()
batch_size = os.stat(largeFile).st_size / float(17)
start_gateway()
for i in range(0, 16):
props = {"checkerinthread": False, "consumerinthread": False, "producerinthread": True,
"start": int(batch_size * i), "end": int(batch_size * (i + 1))}
processor = HugeProcessor(Checker, ProcessorImpl.Producer.Producer, Consumer, largeFile, props)
processor.startProcessing()
gc.collect()
restart_gateway()
stop_gateway()
def start_gateway():
global gateway_running
subprocess.Popen(["java", "-jar", "-Xmx6500m", "Bridge/target/stanfordbridge-1.0-jar-with-dependencies.jar"])
gateway_running = True
def restart_gateway():
if gateway_running:
stop_gateway()
start_gateway()
def stop_gateway():
global gateway_running
JavaGateway().shutdown()
gateway_running = False
if __name__ == "__main__":
main(sys.argv)