|
1 | 1 | # @file eval.py |
2 | 2 | # @brief Evaluates the performance of the controllers on a set of environments |
3 | 3 | import os |
4 | | -import sys |
| 4 | +import argparse |
5 | 5 |
|
6 | 6 | import coverage_control as cc |
7 | 7 | import numpy as np |
| 8 | +from rich.progress import ( |
| 9 | + Progress, |
| 10 | + BarColumn, |
| 11 | + TextColumn, |
| 12 | + TimeRemainingColumn, |
| 13 | + TimeElapsedColumn, |
| 14 | + TaskProgressColumn, |
| 15 | + MofNCompleteColumn, |
| 16 | +) |
8 | 17 | from coverage_control import CoverageSystem |
9 | 18 | from coverage_control import IOUtils |
10 | 19 | from coverage_control import WorldIDF |
11 | 20 | from coverage_control.algorithms import ControllerCVT |
12 | 21 | from coverage_control.algorithms import ControllerNN |
13 | 22 |
|
14 | 23 |
|
15 | | -# @ingroup python_api |
16 | 24 | class Evaluator: |
17 | 25 | """ |
18 | 26 | Evaluates the performance of the controllers on a set of environments |
@@ -46,102 +54,115 @@ def __init__(self, in_config): |
46 | 54 | self.num_steps = self.config["NumSteps"] |
47 | 55 | os.makedirs(self.env_dir + "/init_maps", exist_ok=True) |
48 | 56 |
|
| 57 | + self.columns = [ |
| 58 | + BarColumn(bar_width=None), |
| 59 | + TaskProgressColumn(), |
| 60 | + TextColumn("[progress.description]{task.description}"), |
| 61 | + MofNCompleteColumn(), |
| 62 | + TextColumn("Controller: {task.fields[info]}"), |
| 63 | + TimeRemainingColumn(), |
| 64 | + TimeElapsedColumn() |
| 65 | + ] |
| 66 | + |
49 | 67 | def evaluate(self, save=True): |
50 | | - dataset_count = 0 |
51 | 68 | cost_data = np.zeros((self.num_controllers, self.num_envs, self.num_steps)) |
52 | 69 |
|
53 | | - while dataset_count < self.num_envs: |
54 | | - print(f"Environment {dataset_count}") |
55 | | - pos_file = self.env_dir + "/" + str(dataset_count) + ".pos" |
56 | | - env_file = self.env_dir + "/" + str(dataset_count) + ".env" |
57 | | - |
58 | | - if os.path.isfile(env_file) and os.path.isfile(pos_file): |
59 | | - world_idf = WorldIDF(self.cc_params, env_file) |
60 | | - env_main = CoverageSystem(self.cc_params, world_idf, pos_file) |
61 | | - else: |
62 | | - print(f"Creating new environment {dataset_count}") |
63 | | - env_main = CoverageSystem(self.cc_params) |
64 | | - env_main.WriteEnvironment(pos_file, env_file) |
65 | | - world_idf = env_main.GetWorldIDFObject() |
66 | | - |
67 | | - # env_main.PlotInitMap(self.env_dir + "/init_maps", f"{dataset_count}") |
68 | | - robot_init_pos = env_main.GetRobotPositions(force_no_noise=True) |
69 | | - |
70 | | - for controller_id in range(self.num_controllers): |
71 | | - step_count = 0 |
72 | | - env = CoverageSystem(self.cc_params, world_idf, robot_init_pos) |
73 | | - |
74 | | - # map_dir = self.eval_dir + "/" + self.controllers[controller_id]["Name"] + "/plots/" |
75 | | - # env.RecordPlotData() |
76 | | - # env.PlotMapVoronoi(map_dir, step_count) |
77 | | - |
78 | | - if self.controllers_configs[controller_id]["Type"] == "Learning": |
79 | | - Controller = ControllerNN |
| 70 | + with Progress(*self.columns, expand=True) as progress: |
| 71 | + task = progress.add_task( |
| 72 | + "[bold blue]Evaluation", |
| 73 | + total=self.num_envs, |
| 74 | + info="", |
| 75 | + auto_refresh=False, |
| 76 | + ) |
| 77 | + |
| 78 | + for env_count in range(self.num_envs): |
| 79 | + pos_file = self.env_dir + "/" + str(env_count) + ".pos" |
| 80 | + env_file = self.env_dir + "/" + str(env_count) + ".env" |
| 81 | + |
| 82 | + if os.path.isfile(env_file) and os.path.isfile(pos_file): |
| 83 | + world_idf = WorldIDF(self.cc_params, env_file) |
| 84 | + env_main = CoverageSystem(self.cc_params, world_idf, pos_file) |
80 | 85 | else: |
81 | | - Controller = ControllerCVT |
82 | | - controller = Controller( |
83 | | - self.controllers_configs[controller_id], self.cc_params, env |
84 | | - ) |
85 | | - initial_objective_value = env.GetObjectiveValue() |
86 | | - cost_data[controller_id, dataset_count, step_count] = ( |
87 | | - env.GetObjectiveValue() / initial_objective_value |
88 | | - ) |
89 | | - step_count = step_count + 1 |
90 | | - |
91 | | - while step_count < self.num_steps: |
92 | | - objective_value, converged = controller.step(env) |
93 | | - cost_data[controller_id, dataset_count, step_count] = ( |
94 | | - objective_value / initial_objective_value |
95 | | - ) |
| 86 | + # print(f"Creating new environment {env_count}") |
| 87 | + env_main = CoverageSystem(self.cc_params) |
| 88 | + env_main.WriteEnvironment(pos_file, env_file) |
| 89 | + world_idf = env_main.GetWorldIDFObject() |
96 | 90 |
|
97 | | - if converged: |
98 | | - cost_data[controller_id, dataset_count, step_count:] = ( |
99 | | - objective_value / initial_objective_value |
100 | | - ) |
| 91 | + # env_main.PlotInitMap(self.env_dir + "/init_maps", f"{env_count}") |
| 92 | + robot_init_pos = env_main.GetRobotPositions(force_no_noise=True) |
101 | 93 |
|
102 | | - break |
103 | | - # env.PlotMapVoronoi(map_dir, step_count) |
| 94 | + for controller_id in range(self.num_controllers): |
| 95 | + step_count = 0 |
| 96 | + env = CoverageSystem(self.cc_params, world_idf, robot_init_pos) |
| 97 | + |
| 98 | + # map_dir = self.eval_dir + "/" + self.controllers[controller_id]["Name"] + "/plots/" |
104 | 99 | # env.RecordPlotData() |
| 100 | + # env.PlotMapVoronoi(map_dir, step_count) |
| 101 | + |
| 102 | + if self.controllers_configs[controller_id]["Type"] == "Learning": |
| 103 | + Controller = ControllerNN |
| 104 | + else: |
| 105 | + Controller = ControllerCVT |
| 106 | + controller = Controller( |
| 107 | + self.controllers_configs[controller_id], self.cc_params, env |
| 108 | + ) |
| 109 | + initial_objective_value = env.GetObjectiveValue() |
| 110 | + cost_data[controller_id, env_count, step_count] = ( |
| 111 | + env.GetObjectiveValue() / initial_objective_value |
| 112 | + ) |
105 | 113 | step_count = step_count + 1 |
106 | 114 |
|
107 | | - if step_count % 100 == 0: |
108 | | - val = cost_data[controller_id, dataset_count, step_count - 1] |
109 | | - print( |
110 | | - f"Environment {dataset_count} " |
111 | | - f"{controller.name} " |
112 | | - f"Step {step_count} " |
113 | | - f"Objective Value {val:.3e}" |
| 115 | + while step_count < self.num_steps: |
| 116 | + objective_value, converged = controller.step(env) |
| 117 | + cost_data[controller_id, env_count, step_count] = ( |
| 118 | + objective_value / initial_objective_value |
114 | 119 | ) |
115 | 120 |
|
116 | | - print( |
117 | | - f"Environment {dataset_count} " |
118 | | - f"{controller.name} " |
119 | | - f"Step {step_count} " |
120 | | - f"Objective Value {val:.3e}" |
121 | | - ) |
122 | | - if save is True: |
123 | | - self.controller_dir = ( |
124 | | - self.eval_dir |
125 | | - + "/" |
126 | | - + self.controllers_configs[controller_id]["Name"] |
127 | | - ) |
128 | | - controller_data_file = self.controller_dir + "/" + "eval.csv" |
129 | | - np.savetxt( |
130 | | - controller_data_file, |
131 | | - cost_data[controller_id, : dataset_count + 1, :], |
132 | | - delimiter=",", |
133 | | - ) |
134 | | - # env.RenderRecordedMap(self.eval_dir + "/" + self.controllers[controller_id]["Name"] + "/", "video.mp4") |
135 | | - del controller |
136 | | - del env |
137 | | - dataset_count = dataset_count + 1 |
| 121 | + if converged: |
| 122 | + cost_data[controller_id, env_count, step_count:] = ( |
| 123 | + objective_value / initial_objective_value |
| 124 | + ) |
| 125 | + |
| 126 | + break |
| 127 | + # env.PlotMapVoronoi(map_dir, step_count) |
| 128 | + # env.RecordPlotData() |
| 129 | + |
| 130 | + if step_count % 10 == 0: |
| 131 | + info = ( |
| 132 | + f"{controller_id}/{self.num_controllers} {controller.name} " |
| 133 | + f"Step: {step_count} Obj: {cost_data[controller_id, env_count, step_count]:.2e}" |
| 134 | + ) |
| 135 | + progress.update(task, info=info) |
| 136 | + progress.refresh() |
| 137 | + |
| 138 | + step_count = step_count + 1 |
| 139 | + |
| 140 | + if save is True: |
| 141 | + self.controller_dir = ( |
| 142 | + self.eval_dir |
| 143 | + + "/" |
| 144 | + + self.controllers_configs[controller_id]["Name"] |
| 145 | + ) |
| 146 | + controller_data_file = self.controller_dir + "/" + "eval.csv" |
| 147 | + np.savetxt( |
| 148 | + controller_data_file, |
| 149 | + cost_data[controller_id, : env_count + 1, :], |
| 150 | + delimiter=",", |
| 151 | + ) |
| 152 | + # env.RenderRecordedMap(self.eval_dir + "/" + self.controllers[controller_id]["Name"] + "/", "video.mp4") |
| 153 | + del controller |
| 154 | + del env |
| 155 | + progress.advance(task) |
| 156 | + progress.refresh() |
138 | 157 |
|
139 | 158 | return cost_data |
140 | 159 |
|
141 | 160 |
|
142 | 161 | if __name__ == "__main__": |
143 | | - config_file = sys.argv[1] |
144 | | - config = IOUtils.load_toml(config_file) |
| 162 | + parser = argparse.ArgumentParser() |
| 163 | + parser.add_argument("config_file", type=str, help="Path to config file") |
| 164 | + args = parser.parse_args() |
| 165 | + config = IOUtils.load_toml(args.config_file) |
145 | 166 |
|
146 | 167 | evaluator = Evaluator(config) |
147 | 168 | evaluator.evaluate() |
0 commit comments