update data processor

Saurav Agarwal · Saurav Agarwal · commit 29c70e1ed152 · 2024-08-25T00:03:29.000-04:00
diff --git a/python/utils/process_data.sh b/python/utils/process_data.sh
@@ -1,2 +1,10 @@
-python -c 'import dataset_utils; dataset_utils.combine_dataset("${CoverageControl_ws}/lpac/params/data_params.toml", ["0", "1"])'
-python -c 'import dataset_utils; dataset_utils.split_dataset("${CoverageControl_ws}/lpac/params/data_params.toml")'
+data_prams=${CoverageControl_ws}/lpac/params/data_params.toml
+num_subsets=2
+num_subsets_str="["
+for i in $(seq 0 $((num_subsets-1))); do
+    num_subsets_str="${num_subsets_str} \"${i}\", "
+done
+num_subsets_str="${num_subsets_str%??}"
+num_subsets_str="${num_subsets_str} ]"
+python -c "import dataset_utils; dataset_utils.combine_dataset(\"${data_prams}\", ${num_subsets_str})"
+python -c "import dataset_utils; dataset_utils.split_dataset(\"${data_prams}\")"
