python: Added baseline_elapsed_wall_time to speed tests and changed t…

…o check elapsed_relative_time instead.

This change makes the speed test independent of the hardware, at least to some extent.

python/inet/test/speed/store.py

@@ -92,7 +92,18 @@ def get_elapsed_wall_time(self, **kwargs):
     def set_elapsed_wall_time(self, elapsed_wall_time, **kwargs):
         self.get_entry(**kwargs)["elapsed_wall_time"] = elapsed_wall_time
 
-    def insert_elapsed_wall_time(self, elapsed_wall_time, test_result=None, working_directory=os.getcwd(), ini_file="omnetpp.ini", config="General", run_number=0, sim_time_limit=None, itervars="$repetition==0"):
+    def find_elapsed_relative_time(self, **kwargs):
+        entry = self.find_entry(**kwargs)
+        if entry is not None:
+            return entry["elapsed_wall_time"] / entry["baseline_elapsed_wall_time"]
+        else:
+            return None
+
+    def get_elapsed_relative_time(self, **kwargs):
+        entry = self.get_entry(**kwargs)
+        return entry["elapsed_wall_time"] / entry["baseline_elapsed_wall_time"]
+
+    def insert_elapsed_wall_time(self, elapsed_wall_time, baseline_elapsed_wall_time, test_result=None, working_directory=os.getcwd(), ini_file="omnetpp.ini", config="General", run_number=0, sim_time_limit=None, itervars="$repetition==0"):
         # assert test_result == "ERROR" or sim_time_limit is not None
         self.get_entries().append({"working_directory": working_directory,
                                    "ini_file": ini_file,
@@ -101,15 +112,17 @@ def insert_elapsed_wall_time(self, elapsed_wall_time, test_result=None, working_
                                    "sim_time_limit": sim_time_limit,
                                    "test_result": test_result,
                                    "elapsed_wall_time": elapsed_wall_time,
+                                   "baseline_elapsed_wall_time": baseline_elapsed_wall_time,
                                    "timestamp": time.time(),
                                    "itervars": itervars})
 
-    def update_elapsed_wall_time(self, elapsed_wall_time, **kwargs):
+    def update_elapsed_wall_time(self, elapsed_wall_time, baseline_elapsed_wall_time, **kwargs):
         entry = self.find_entry(**kwargs)
         if entry:
             entry["elapsed_wall_time"] = elapsed_wall_time
+            entry["baseline_elapsed_wall_time"] = baseline_elapsed_wall_time
         else:
-            self.insert_elapsed_wall_time(elapsed_wall_time, **kwargs)
+            self.insert_elapsed_wall_time(elapsed_wall_time, baseline_elapsed_wall_time, **kwargs)
 
     def remove_elapsed_wall_times(self, **kwargs):
         list(map(lambda element: self.entries.remove(element), self.filter_entries(**kwargs)))

python/inet/test/speed/task.py

@@ -12,16 +12,31 @@
 _speed_test_extra_args = ["--cmdenv-express-mode=true", "--cmdenv-performance-display=false", "--cmdenv-status-frequency=1000000s", 
                           "--record-eventlog=false", "--vector-recording=false", "--scalar-recording=false", "--bin-recording=false", "--param-recording=false"]
 
+_baseline_elapsed_wall_time = None
+
+def get_baseline_elapsed_wall_time():
+    global _baseline_elapsed_wall_time
+    if _baseline_elapsed_wall_time:
+        return _baseline_elapsed_wall_time
+    else:
+        logging.getLogger("routing").setLevel("WARN")
+        with open(os.devnull, 'w') as devnull:
+            simulation_task_result = run_simulations(simulation_project=get_simulation_project("routing"), config_filter="Net60a", sim_time_limit="10000s", output_stream=devnull)
+        _baseline_elapsed_wall_time = simulation_task_result.elapsed_wall_time
+        return _baseline_elapsed_wall_time
+
 class SpeedTestTask(SimulationTestTask):
-    def __init__(self, expected_wall_time=None, max_relative_error = 0.2, **kwargs):
+    def __init__(self, baseline_elapsed_wall_time=None, expected_relative_time=None, max_relative_error = 0.2, **kwargs):
         super().__init__(**kwargs)
-        self.expected_wall_time = expected_wall_time
+        self.baseline_elapsed_wall_time = baseline_elapsed_wall_time
+        self.expected_relative_time = expected_relative_time
         self.max_relative_error = max_relative_error
 
     def check_simulation_task_result(self, simulation_task_result, **kwargs):
-        if (simulation_task_result.elapsed_wall_time - self.expected_wall_time) / self.expected_wall_time > self.max_relative_error: 
+        elapsed_relative_time = simulation_task_result.elapsed_wall_time / self.baseline_elapsed_wall_time
+        if (elapsed_relative_time - self.expected_relative_time) / self.expected_relative_time > self.max_relative_error:
             return self.task_result_class(task=self, simulation_task_result=simulation_task_result, result="FAIL", expected_result="PASS", reason="Elapsed wall time is too large")
-        elif (self.expected_wall_time - simulation_task_result.elapsed_wall_time) / self.expected_wall_time > self.max_relative_error: 
+        elif (self.expected_relative_time - elapsed_relative_time) / self.expected_relative_time > self.max_relative_error:
             return self.task_result_class(task=self, simulation_task_result=simulation_task_result, result="FAIL", expected_result="PASS", reason="Elapsed wall time is too small")
         else:
             return super().check_simulation_task_result(simulation_task_result, **kwargs)
@@ -33,23 +48,25 @@ def get_speed_test_tasks(mode="release", run_number=0, working_directory_filter=
     multiple_simulation_tasks = get_simulation_tasks(name="speed test", mode=mode, run_number=run_number, working_directory_filter=working_directory_filter, **kwargs)
     simulation_project = multiple_simulation_tasks.simulation_project
     speed_measurement_store = get_speed_measurement_store(simulation_project)
+    baseline_elapsed_wall_time = get_baseline_elapsed_wall_time()
     tasks = []
     for simulation_task in multiple_simulation_tasks.tasks:
         simulation_config = simulation_task.simulation_config
-        expected_wall_time = speed_measurement_store.get_elapsed_wall_time(working_directory=simulation_config.working_directory, ini_file=simulation_config.ini_file, config=simulation_config.config, run_number=simulation_task.run_number)
-        tasks.append(SpeedTestTask(simulation_task=simulation_task, expected_wall_time=expected_wall_time, **dict(kwargs, simulation_project=simulation_project)))
+        expected_relative_time = speed_measurement_store.get_elapsed_relative_time(working_directory=simulation_config.working_directory, ini_file=simulation_config.ini_file, config=simulation_config.config, run_number=simulation_task.run_number)
+        tasks.append(SpeedTestTask(simulation_task=simulation_task, baseline_elapsed_wall_time=baseline_elapsed_wall_time, expected_relative_time=expected_relative_time, **dict(kwargs, simulation_project=simulation_project)))
     return MultipleSimulationTestTasks(tasks=tasks, **dict(kwargs, simulation_project=simulation_project, concurrent=False))
 
 def run_speed_tests(**kwargs):
     multiple_test_tasks = get_speed_test_tasks(**kwargs)
     return multiple_test_tasks.run(**kwargs)
 
 class SpeedUpdateTask(SimulationUpdateTask):
-    def __init__(self, action="Updating speed", **kwargs):
+    def __init__(self, baseline_elapsed_wall_time=None, action="Updating speed", **kwargs):
         super().__init__(action=action, **kwargs)
         self.locals = locals()
         self.locals.pop("self")
         self.kwargs = kwargs
+        self.baseline_elapsed_wall_time = baseline_elapsed_wall_time
 
     def run_protected(self, **kwargs):
         simulation_config = self.simulation_task.simulation_config
@@ -59,8 +76,8 @@ def run_protected(self, **kwargs):
         end_time = time.time()
         simulation_task_result.elapsed_wall_time = end_time - start_time
         speed_measurement_store = get_speed_measurement_store(simulation_project)
-        expected_wall_time = speed_measurement_store.find_elapsed_wall_time(working_directory=simulation_config.working_directory, ini_file=simulation_config.ini_file, config=simulation_config.config, run_number=self.simulation_task.run_number)
-        return SpeedUpdateTaskResult(task=self, simulation_task_result=simulation_task_result, expected_wall_time=expected_wall_time, elapsed_wall_time=simulation_task_result.elapsed_wall_time)
+        expected_relative_time = speed_measurement_store.find_elapsed_relative_time(working_directory=simulation_config.working_directory, ini_file=simulation_config.ini_file, config=simulation_config.config, run_number=self.simulation_task.run_number)
+        return SpeedUpdateTaskResult(task=self, simulation_task_result=simulation_task_result, expected_relative_time=expected_relative_time, elapsed_relative_time=simulation_task_result.elapsed_wall_time / self.baseline_elapsed_wall_time)
 
 class MultipleSpeedUpdateTasks(MultipleSimulationUpdateTasks):
     def __init__(self, multiple_simulation_tasks=None, name="update speed", **kwargs):
@@ -84,22 +101,22 @@ def run(self, simulation_project=None, concurrent=None, build=True, **kwargs):
             simulation_config = simulation_task.simulation_config
             elapsed_wall_time = speed_update_result.elapsed_wall_time
             if elapsed_wall_time is not None:
-                speed_measurement_store.update_elapsed_wall_time(elapsed_wall_time, test_result="PASS", sim_time_limit=simulation_task.sim_time_limit,
+                speed_measurement_store.update_elapsed_wall_time(elapsed_wall_time, speed_update_task.baseline_elapsed_wall_time, test_result="PASS", sim_time_limit=simulation_task.sim_time_limit,
                                                                  working_directory=simulation_config.working_directory, ini_file=simulation_config.ini_file, config=simulation_config.config, run_number=simulation_task.run_number)
         speed_measurement_store.write()
         return speed_measurement_store
 
 class SpeedUpdateTaskResult(SimulationUpdateTaskResult):
-    def __init__(self, expected_wall_time=None, elapsed_wall_time=None, reason=None, max_relative_error=0.2, **kwargs):
+    def __init__(self, expected_relative_time=None, elapsed_relative_time=None, reason=None, max_relative_error=0.2, **kwargs):
         super().__init__(**kwargs)
         self.locals = locals()
         self.locals.pop("self")
         self.kwargs = kwargs
-        if expected_wall_time is None:
+        if expected_relative_time is None:
             self.result = "INSERT"
             self.reason = None
             self.color = COLOR_YELLOW
-        elif abs(expected_wall_time - elapsed_wall_time) / expected_wall_time > max_relative_error:
+        elif abs(expected_relative_time - elapsed_relative_time) / expected_relative_time > max_relative_error:
             self.result = "UPDATE"
             self.reason = None
             self.color = COLOR_YELLOW
@@ -124,8 +141,9 @@ def print_result(self, complete_error_message=True, output_stream=sys.stdout, **
 def get_update_speed_test_results_tasks(mode="release", run_number=0, working_directory_filter="showcases", **kwargs):
     update_tasks = []
     multiple_simulation_tasks = get_simulation_tasks(mode=mode, run_number=run_number, working_directory_filter=working_directory_filter, **kwargs)
+    baseline_elapsed_wall_time = get_baseline_elapsed_wall_time()
     for simulation_task in multiple_simulation_tasks.tasks:
-        update_task = SpeedUpdateTask(simulation_task=simulation_task, **kwargs)
+        update_task = SpeedUpdateTask(simulation_task=simulation_task, baseline_elapsed_wall_time=baseline_elapsed_wall_time, **kwargs)
         update_tasks.append(update_task)
     return MultipleSpeedUpdateTasks(multiple_simulation_tasks, tasks=update_tasks, **kwargs)