Add comment warning about workaround for JET 0D CW -> ILW preprocessing

felker · felker · commit e2738ea0bc7d · 2019-11-22T23:10:50.000-05:00
diff --git a/data/signals.py b/data/signals.py
@@ -85,12 +85,9 @@ def get_units(str):
     if found:
         if rank > 1:
             xdata = c.get('dim_of(_s,1)').data()
-            # xunits = get_units('dim_of(_s,1)')
             ydata = c.get('dim_of(_s)').data()
-            # yunits = get_units('dim_of(_s)')
         else:
             xdata = c.get('dim_of(_s)').data()
-            # xunits = get_units('dim_of(_s)')
 
     # MDSplus seems to return 2-D arrays transposed.  Change them back.
     if np.ndim(data) == 2:
@@ -114,16 +111,13 @@ def fetch_jet_data(signal_path, shot_num, c):
         data = c.get('_sig=jet("{}/",{})'.format(signal_path, shot_num)).data()
         if np.ndim(data) == 2:
             data = np.transpose(data)
-            time = c.get(
-                '_sig=dim_of(jet("{}/",{}),1)'.format(
-                    signal_path, shot_num)).data()
-            ydata = c.get(
-                '_sig=dim_of(jet("{}/",{}),0)'.format(
-                    signal_path, shot_num)).data()
+            time = c.get('_sig=dim_of(jet("{}/",{}),1)'.format(
+                signal_path, shot_num)).data()
+            ydata = c.get('_sig=dim_of(jet("{}/",{}),0)'.format(
+                signal_path, shot_num)).data()
         else:
-            time = c.get(
-                '_sig=dim_of(jet("{}/",{}))'.format(
-                    signal_path, shot_num)).data()
+            time = c.get('_sig=dim_of(jet("{}/",{}))'.format(
+                signal_path, shot_num)).data()
         found = True
     except Exception as e:
         g.print_unique(e)
@@ -345,6 +339,11 @@ def fetch_nstx_data(signal_path, shot_num, c):
     # 'tmamp1':tmamp1, 'tmamp2':tmamp2, 'tmfreq1':tmfreq1, 'tmfreq2':tmfreq2,
     # 'pechin':pechin,
     # 'rho_profile_spatial':rho_profile_spatial, 'etemp':etemp,
+    # -----
+    # TODO(KGF): replace this hacky workaround
+    # IMPORTANT: must comment-out the following line when preprocessing for
+    # training on JET CW and testing on JET ILW (FRNN 0D).
+    # Otherwise 1K+ CW shots are excluded due to missing profile data
     'etemp_profile': etemp_profile, 'edens_profile': edens_profile,
     # 'itemp_profile':itemp_profile, 'zdens_profile':zdens_profile,
     # 'trot_profile':trot_profile, 'pthm_profile':pthm_profile,
diff --git a/examples/guarantee_preprocessed.py b/examples/guarantee_preprocessed.py
@@ -11,4 +11,4 @@
 #####################################################
 np.random.seed(0)
 random.seed(0)
-guarantee_preprocessed(conf)
+guarantee_preprocessed(conf, verbose=True)
diff --git a/plasma/primitives/shots.py b/plasma/primitives/shots.py
@@ -102,10 +102,12 @@ def split_train_test(self, conf):
         use_shots_train = int(round(train_frac*use_shots))
         use_shots_test = int(round((1-train_frac)*use_shots))
         if len(shot_files_test) == 0:
-            # split randomly
+            # split randomly, e.g. sample both sets from same distribution
+            # such as D3D test and train
             shot_list_train, shot_list_test = train_test_split(
                 self.shots, train_frac, shuffle_training)
-        # train and test list given
+        # train and test list given, e.g. they are sampled from separate
+        # distributions such as train=CW and test=ILW for JET
         else:
             shot_list_train = ShotList()
             shot_list_train.load_from_shot_list_files_objects(
