mavsdk_tests: add filter for cases

test/mavsdk_tests/mavsdk_test_runner.py

@@ -29,7 +29,9 @@ def main() -> NoReturn:
     parser.add_argument("--gui", default=False, action='store_true',
                         help="Display gzclient with simulation")
     parser.add_argument("--model", type=str, default='all',
-                        help="Specify which model to run")
+                        help="Only run tests for one model")
+    parser.add_argument("--case", type=str, default='all',
+                        help="Only run tests for one case")
     parser.add_argument("--debugger", default="",
                         help="valgrind callgrind gdb lldb")
     parser.add_argument("--verbose", default=False, action='store_true',
@@ -58,6 +60,7 @@ def main() -> NoReturn:
         args.abort_early,
         args.speed_factor,
         args.model,
+        args.case,
         args.debugger,
         args.log_dir,
         args.gui,
@@ -114,6 +117,7 @@ class Tester:
                  abort_early: bool,
                  speed_factor: float,
                  model: str,
+                 case: str,
                  debugger: str,
                  log_dir: str,
                  gui: bool,
@@ -122,7 +126,7 @@ class Tester:
         self.active_runners: List[ph.Runner]
         self.iterations = iterations
         self.abort_early = abort_early
-        self.tests = self.determine_tests(config['tests'], model)
+        self.tests = self.determine_tests(config['tests'], model, case)
         self.speed_factor = speed_factor
         self.debugger = debugger
         self.log_dir = log_dir
@@ -132,17 +136,23 @@ class Tester:
         self.combined_log_fd = TextIO
 
     @classmethod
-    def determine_tests(cls, tests: List[Dict[str, Any]], model: str) \
+    def determine_tests(cls,
-            -> List[Dict[str, Any]]:
+                        tests: List[Dict[str, Any]],
+                        model: str,
+                        case: str) -> List[Dict[str, Any]]:
         for test in tests:
-            test['selected'] = (model == 'all') or model == test['model']
+            test['selected'] = (model == 'all' or model == test['model'])
             test['cases'] = dict.fromkeys(
-                cls.determine_test_cases(test['test_filter']))
+                cls.query_test_cases(test['test_filter']))
+            for key in test['cases'].keys():
+                test['cases'][key] = {
+                    'selected': (test['selected'] and
+                                 (case == 'all' or case == key))}
 
         return tests
 
     @staticmethod
-    def determine_test_cases(filter: str) -> List[str]:
+    def query_test_cases(filter: str) -> List[str]:
         cmd = os.getcwd() + "/build/px4_sitl_default/mavsdk_tests/mavsdk_tests"
         args = ["--list-test-names-only", filter]
         p = subprocess.Popen(
@@ -170,19 +180,12 @@ class Tester:
         return self.was_overall_pass()
 
     def show_plans(self) -> None:
-        n_models = sum(map(
-            lambda test: 1 if test['selected'] else 0,
-            self.tests))
-        n_cases = sum(map(
-            lambda test: len(test['cases']) if test['selected'] else 0,
-            self.tests))
-
         print()
         print(colorize(
             "About to run {} test case{} for {} selected model{} "
             "({} iteration{}):".
-            format(n_cases, self.plural_s(n_cases),
+            format(self.num_cases(), self.plural_s(self.num_cases()),
-                   n_models, self.plural_s(n_models),
+                   self.num_models(), self.plural_s(self.num_models()),
                    self.iterations, self.plural_s(self.iterations)),
             color.BOLD))
 
@@ -193,16 +196,44 @@ class Tester:
                     color.BOLD), color.GRAY))
                 continue
             print(colorize("  - {}:".format(test['model']), color.BOLD))
-            for case in test['cases']:
+            for key, case in test['cases'].items():
-                print("    - '{}'".format(case))
+                if case['selected']:
+                    print("    - '{}'".format(key))
+                else:
+                    print(colorize("    - '{}' (not selected)".format(key),
+                                   color.GRAY))
         print()
 
+    def num_models(self) -> int:
+        return sum(map(
+            lambda test: 1 if test['selected'] else 0,
+            self.tests))
+
+    def num_cases(self) -> int:
+        n_cases = 0
+        for test in self.tests:
+            if not test['selected']:
+                continue
+            n_cases += self.num_cases_for_test(test)
+
+        return n_cases
+
+    def num_cases_for_test(self, test: Dict[str, Any]) -> int:
+        n_cases = 0
+        for case in test['cases'].values():
+            if not case['selected']:
+                continue
+            n_cases += 1
+        return n_cases
+
     def prepare_for_results(self) -> None:
         for test in self.tests:
             if not test['selected']:
                 continue
-            for key in test['cases'].keys():
+            for key, value in test['cases'].items():
-                test['cases'][key] = {'results': []}
+                if not value['selected']:
+                    continue
+                value['results'] = []
 
     def run_tests(self) -> None:
         for iteration in range(self.iterations):
@@ -225,22 +256,28 @@ class Tester:
                 "==> Running tests for {}".format(test['model']),
                 color.BOLD))
 
-            for i, case in enumerate(test['cases']):
+            test_i = 0
-                print("--> Test case {} of {}: '{}' running ...".
+            for key, case_value in test['cases'].items():
-                      format(i+1, len(test['cases']), case))
+                if not case_value['selected']:
+                    continue
 
-                log_dir = self.get_log_dir(iteration, test['model'], case)
+                print("--> Test case {} of {}: '{}' running ...".
+                      format(test_i+1,
+                             self.num_cases_for_test(test),
+                             key))
+
+                log_dir = self.get_log_dir(iteration, test['model'], key)
                 if self.verbose:
                     print("creating log directory: {}"
                           .format(log_dir))
                 os.makedirs(log_dir, exist_ok=True)
 
-                was_success = self.run_test_case(test, case, log_dir)
+                was_success = self.run_test_case(test, key, log_dir)
 
                 print("--- Test case {} of {}: '{}' {}."
-                      .format(i+1,
+                      .format(test_i+1,
-                              len(test['cases']),
+                              self.num_cases_for_test(test),
-                              case,
+                              key,
                               colorize("succeeded", color.GREEN)
                               if was_success
                               else colorize("failed", color.RED)))
@@ -252,6 +289,8 @@ class Tester:
                     print("Aborting early")
                     return False
 
+                test_i += 1
+
         return iteration_success
 
     def get_log_dir(self, iteration: int, model: str, case: str) -> str:
@@ -269,21 +308,65 @@ class Tester:
 
         return foldername
 
-    def run_test_case(self, test: Dict[str, Any],
+    def get_max_speed_factor(self, test: Dict[str, Any]) -> float:
-                      case: str, log_dir: str) -> bool:
-        self.active_runners = []
-
         speed_factor = self.speed_factor
         if "max_speed_factor" in test:
             speed_factor = min(float(speed_factor), test["max_speed_factor"])
+        return speed_factor
 
+    def run_test_case(self, test: Dict[str, Any],
+                      case: str, log_dir: str) -> bool:
+
+        self.start_runners(log_dir, test, case)
+
+        logfile_path = self.determine_logfile_path(log_dir, 'combined')
+        self.start_combined_log(logfile_path)
+
+        test_timeout_s = test['timeout_min']*60
+        while self.active_runners[-1].time_elapsed_s() < test_timeout_s:
+            returncode = self.active_runners[-1].poll()
+            if returncode is not None:
+                is_success = (returncode == 0)
+                break
+
+            self.collect_runner_output()
+
+        else:
+            print(colorize(
+                "Test timeout of {} mins triggered!".
+                format(test['timeout_min']),
+                color.BOLD))
+            is_success = False
+
+        self.stop_runners()
+        self.stop_combined_log()
+
+        result = {'success': is_success,
+                  'logfiles': [runner.get_log_filename()
+                               for runner in self.active_runners]}
+        test['cases'][case]['results'].append(result)
+
+        if not is_success:
+            if not self.verbose:
+                print(self.get_combined_log(logfile_path))
+            print("Logfiles: ")
+            print("  - {}".format(logfile_path))
+            for runner in self.active_runners:
+                print("  - {}".format(runner.get_log_filename()))
+        return is_success
+
+    def start_runners(self,
+                      log_dir: str,
+                      test: Dict[str, Any],
+                      case: str) -> None:
+        self.active_runners = []
         if self.config['mode'] == 'sitl':
             px4_runner = ph.Px4Runner(
                 os.getcwd(),
                 log_dir,
                 test['model'],
                 case,
-                speed_factor,
+                self.get_max_speed_factor(test),
                 self.debugger,
                 self.verbose)
             self.active_runners.append(px4_runner)
@@ -294,7 +377,7 @@ class Tester:
                     log_dir,
                     test['model'],
                     case,
-                    self.speed_factor,
+                    self.get_max_speed_factor(test),
                     self.verbose)
                 self.active_runners.append(gzserver_runner)
 
@@ -314,7 +397,6 @@ class Tester:
             case,
             self.config['mavlink_connection'],
             self.verbose)
-
         self.active_runners.append(test_runner)
 
         for runner in self.active_runners:
@@ -322,50 +404,22 @@ class Tester:
                 self.determine_logfile_path(log_dir, runner.name))
             runner.start()
 
-        max_name = max(len(runner.name) for runner in self.active_runners)
+    def stop_runners(self) -> None:
-        logfile_path = self.determine_logfile_path(log_dir, 'combined')
-
-        self.start_combined_log(logfile_path)
-
-        while test_runner.time_elapsed_s() < test['timeout_min']*60:
-            returncode = test_runner.poll()
-            if returncode is not None:
-                is_success = (returncode == 0)
-                break
-
-            for runner in self.active_runners:
-                output = runner.get_output()
-                if not output:
-                    continue
-
-                output = self.add_name_prefix(max_name, runner.name, output)
-                self.add_to_combined_log(output)
-                if self.verbose:
-                    print(output)
-
-        else:
-            print(colorize(
-                "Test timeout of {} mins triggered!".
-                format(test['timeout_min']),
-                color.BOLD))
-            is_success = False
-
         for runner in self.active_runners:
             runner.stop()
-        self.stop_combined_log()
 
-        result = {'success': is_success,
+    def collect_runner_output(self) -> None:
-                  'logfiles': [runner.get_log_filename()
+        max_name = max(len(runner.name) for runner in self.active_runners)
-                               for runner in self.active_runners]}
-        test['cases'][case]['results'].append(result)
 
-        if not is_success:
+        for runner in self.active_runners:
-            print(self.get_combined_log(logfile_path))
+            output = runner.get_output()
-            print("Logfiles: ")
+            if not output:
-            print("  - {}".format(logfile_path))
+                continue
-            for runner in self.active_runners:
+
-                print("  - {}".format(runner.get_log_filename()))
+            output = self.add_name_prefix(max_name, runner.name, output)
-        return is_success
+            self.add_to_combined_log(output)
+            if self.verbose:
+                print(output, end="")
 
     def start_combined_log(self, filename: str) -> None:
         self.log_fd = open(filename, 'w')
@@ -402,6 +456,8 @@ class Tester:
                 print(colorize("  - {}:".format(test['model']), color.BOLD))
 
             for name, case in test['cases'].items():
+                if not case['selected']:
+                    continue
                 print("     - '{}': ".format(name), end="")
 
                 n_succeeded = [result['success']
@@ -429,6 +485,8 @@ class Tester:
                 continue
 
             for case in test['cases'].values():
+                if not case['selected']:
+                    continue
                 for result in case['results']:
                     if result['success'] is not True:
                         return False
@@ -455,8 +513,7 @@ class Tester:
             -> NoReturn:
         print("Received SIGINT")
         print("Stopping all processes ...")
-        for runner in self.active_runners:
+        self.stop_runners()
-            runner.stop()
         self.stop_combined_log()
         print("Stopping all processes done.")
         self.show_detailed_results()