Only call batch_iterator_func.__exit__() if defined in conf.yaml

felker · felker · commit 2c4143ab6902 · 2019-10-16T15:01:49.000-04:00
Another bug introduced in PPPLDeepLearning#39 merging "jdev" branch. Originally, MPIModel class member "batch_iterator_func" was unconditionally set to ProcessGenerator(self.batch_iterator()). Now, may only be set to self.batch_iterator() if conf['training']['use_process_generator'] == False
diff --git a/examples/slurm.cmd b/examples/slurm.cmd
@@ -11,7 +11,7 @@
 # Each node = 2.4 GHz Xeon Broadwell E5-2680 v4 + 4x 1328 MHz P100 GPU
 
 module load anaconda3
-conda activate my_env
+conda activate frnn-tf13.1
 module load cudatoolkit
 module load cudnn
 module load openmpi/cuda-8.0/intel-17.0/3.0.0/64
diff --git a/plasma/models/mpi_runner.py b/plasma/models/mpi_runner.py
@@ -217,7 +217,10 @@ def set_batch_iterator_func(self):
             self.batch_iterator_func = self.batch_iterator()
 
     def close(self):
-        self.batch_iterator_func.__exit__()
+        if (self.conf is not None
+                and 'use_process_generator' in conf['training']
+                and conf['training']['use_process_generator']):
+            self.batch_iterator_func.__exit__()
 
     def set_lr(self, lr):
         self.lr = lr
@@ -918,6 +921,7 @@ def mpi_train(conf, shot_list_train, shot_list_validate, loader,
 
 
 def get_stop_training(callbacks):
+    # TODO(KGF): this funciton is unused
     for cb in callbacks.callbacks:
         if isinstance(cb, cbks.EarlyStopping):
             print("Checking for early stopping")
