Merge branch 'jdev' of https://github.com/PPPLDeepLearning/plasma-python into jdev

Julian Kates-Harbeck · Julian Kates-Harbeck · commit 0c2571a1a289 · 2018-04-05T21:50:33.000-04:00
diff --git a/examples/learn.py b/examples/learn.py
@@ -93,7 +93,7 @@
 #####################################################
 #train(conf,shot_list_train,loader)
 if not only_predict:
-    p = old_mp.Process(target = train,args=(conf,shot_list_train,shot_list_validate,loader))
+    p = old_mp.Process(target = train,args=(conf,shot_list_train,shot_list_validate,loader,shot_list_test))
     p.start()
     p.join()
 
diff --git a/plasma/models/runner.py b/plasma/models/runner.py
@@ -24,7 +24,7 @@
 
 backend = conf['model']['backend']
 
-def train(conf,shot_list_train,shot_list_validate,loader):
+def train(conf,shot_list_train,shot_list_validate,loader,shot_list_test=None):
     loader.set_inference_mode(False)
     np.random.seed(1)
 
diff --git a/plasma/models/shallow_runner.py b/plasma/models/shallow_runner.py
@@ -15,6 +15,7 @@
 import time
 import datetime
 import os
+from copy import deepcopy
 from functools import partial
 import pathos.multiprocessing as mp
 from xgboost import XGBClassifier
@@ -284,7 +285,7 @@ def build_callbacks(conf):
     return cbks.CallbackList(callbacks)
 
 
-def train(conf,shot_list_train,shot_list_validate,loader):
+def train(conf,shot_list_train,shot_list_validate,loader,shot_list_test=None):
 
     np.random.seed(1)
 
@@ -367,6 +368,20 @@ def train(conf,shot_list_train,shot_list_validate,loader):
     Y_predv = model.predict(Xv)
     print("Validate")
     print(classification_report(Yv,Y_predv))
+
+
+    if 'monitor_test' in conf['callbacks'].keys() and conf['callbacks']['monitor_test']:
+        times = conf['callbacks']['monitor_times']
+        roc_areas,losses = make_predictions_and_evaluate_multiple_times(conf,shot_list_validate,loader,times)
+        for roc,t in zip(roc_areas,times):
+            print('val_roc_{} = {}'.format(t,roc))
+        if shot_list_test is not None:
+            roc_areas,losses = make_predictions_and_evaluate_multiple_times(conf,shot_list_test,loader,times)
+            for roc,t in zip(roc_areas,times):
+                print('test_roc_{} = {}'.format(t,roc))
+
+
+
     #print(confusion_matrix(Y,Y_pred))
     _,_,_,roc_area,loss = make_predictions_and_evaluate_gpu(conf,shot_list_validate,loader)
     # _,_,_,roc_area_train,loss_train = make_predictions_and_evaluate_gpu(conf,shot_list_train,loader)   
@@ -378,6 +393,8 @@ def train(conf,shot_list_train,shot_list_validate,loader):
     epoch_logs['val_loss'] = loss
     # epoch_logs['train_roc'] = roc_area_train
     # epoch_logs['train_loss'] = loss_train 
+
+
     callbacks.on_epoch_end(0, epoch_logs)
 
 
@@ -432,3 +449,20 @@ def make_predictions_and_evaluate_gpu(conf,shot_list,loader,custom_path = None):
     loss = get_loss_from_list(y_prime,y_gold,conf['data']['target'])
     return y_prime,y_gold,disruptive,roc_area,loss
 
+def make_predictions_and_evaluate_multiple_times(conf,shot_list,loader,times,custom_path=None):
+    y_prime,y_gold,disruptive = make_predictions(conf,shot_list,loader,custom_path)
+    areas = []
+    losses = []
+    for T_min_curr in times:
+    #if 'monitor_test' in conf['callbacks'].keys() and conf['callbacks']['monitor_test']:
+        conf_curr = deepcopy(conf)
+        T_min_warn_orig = conf['data']['T_min_warn']
+        conf_curr['data']['T_min_warn'] = T_min_curr 
+        assert(conf['data']['T_min_warn'] == T_min_warn_orig)
+        analyzer = PerformanceAnalyzer(conf=conf_curr)
+        roc_area = analyzer.get_roc_area(y_prime,y_gold,disruptive)
+        #shot_list.set_weights(analyzer.get_shot_difficulty(y_prime,y_gold,disruptive))
+        loss = get_loss_from_list(y_prime,y_gold,conf['data']['target'])
+        areas.append(roc_area)
+        losses.append(loss)
+    return areas,losses
