fix bugs related to T_min warn becoming too large. performance computation was wrong when T_max_warn was smaller. Shots can't be cut if they are shorter than T_min_warn

Julian Kates-Harbeck · Julian Kates-Harbeck · commit bbfa51bb2054 · 2018-03-24T17:26:54.000-04:00
diff --git a/examples/tune_hyperparams.py b/examples/tune_hyperparams.py
@@ -8,9 +8,9 @@
 tunables = []
 shallow = False
 num_nodes = 1
-num_trials = 20
+num_trials = 30
 
-t_warn = CategoricalHyperparam(['data','T_warning'],[0.256,1.024,10.024])
+t_warn = CategoricalHyperparam(['data','T_warning'],[0.256,1.024,4.096,10.024])
 cut_ends = CategoricalHyperparam(['data','cut_shot_ends'],[False,True])
 #for shallow
 if shallow:
@@ -35,19 +35,21 @@
     fac = CategoricalHyperparam(['data','positive_example_penalty'],[1.0,4.0,16.0])
     target = CategoricalHyperparam(['target'],['maxhinge','hinge','ttdinv','ttd'])
     #target = CategoricalHyperparam(['target'],['hinge','ttdinv','ttd'])
-    batch_size = CategoricalHyperparam(['training','batch_size'],[128,256])
+    batch_size = CategoricalHyperparam(['training','batch_size'],[64,128])
     dropout_prob = CategoricalHyperparam(['model','dropout_prob'],[0.01,0.05,0.1])
-    conv_filters = CategoricalHyperparam(['model','num_conv_filters'],[128,256])
+    conv_filters = CategoricalHyperparam(['model','num_conv_filters'],[64,128,256])
     conv_layers = IntegerHyperparam(['model','num_conv_layers'],2,4)
     rnn_layers = IntegerHyperparam(['model','rnn_layers'],1,3)
     rnn_size = CategoricalHyperparam(['model','rnn_size'],[128,256])
     dense_size = CategoricalHyperparam(['model','dense_size'],[128,256])
     extra_dense_input = CategoricalHyperparam(['model','extra_dense_input'],[False,True])
     equalize_classes = CategoricalHyperparam(['data','equalize_classes'],[False,True])
+    t_min_warn = CategoricalHyperparam(['data','T_min_warn'],[30,70,200,500,1000])
     #rnn_length = CategoricalHyperparam(['model','length'],[32,128])
     #tunables = [lr,lr_decay,fac,target,batch_size,dropout_prob]
     tunables = [lr,lr_decay,fac,target,batch_size,equalize_classes,dropout_prob]
     tunables += [conv_filters,conv_layers,rnn_layers,rnn_size,dense_size,extra_dense_input]
+    tunables += [t_min_warn]
 tunables += [cut_ends,t_warn]
 
 
diff --git a/plasma/preprocessor/normalize.py b/plasma/preprocessor/normalize.py
@@ -141,6 +141,9 @@ def train_on_files(self,shot_files,use_shots,all_machines):
     def cut_end_of_shot(self,shot):
         cut_shot_ends = self.conf['data']['cut_shot_ends']
         if not self.inference_mode and cut_shot_ends: #only cut shots during training
+            if shot.ttd.shape[0] <= T_min_warn:
+                print("not cutting shot since T_min_warn is larger than length of shot")
+                return
             T_min_warn = self.conf['data']['T_min_warn']
             for key in shot.signals_dict:
                 shot.signals_dict[key] = shot.signals_dict[key][:-T_min_warn,:]
diff --git a/plasma/utils/performance.py b/plasma/utils/performance.py
@@ -25,6 +25,9 @@ def __init__(self,results_dir=None,shots_dir=None,i = 0,T_min_warn = None,T_max_
             self.T_min_warn = T_min_warn_def
         if T_max_warn == None:
             self.T_max_warn = T_max_warn_def
+        if self.T_max_warn < self.T_min_warn:
+            print("T max warn is too small: need to increase artificially.") #computation of statistics is only correct if T_max_warn is larger than T_min_warn
+            self.T_max_warn = self.T_min_warn + 1
         self.verbose = verbose
         self.results_dir = results_dir
         self.shots_dir = shots_dir
@@ -293,6 +296,7 @@ def create_acceptable_region(self,truth,mode):
         else:
             print('Error Invalid Mode for acceptable region')
             exit(1) 
+        assert(self.T_max_warn > self.T_min_warn)
 
         acceptable = np.zeros_like(truth,dtype=bool)
         if acceptable_timesteps > 0:
