diff --git a/test/mavsdk_tests/mavsdk_test_runner.py b/test/mavsdk_tests/mavsdk_test_runner.py index b81e2e8569..0ef57e25f1 100755 --- a/test/mavsdk_tests/mavsdk_test_runner.py +++ b/test/mavsdk_tests/mavsdk_test_runner.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 import argparse +import datetime import json +import math import os import psutil # type: ignore import signal @@ -126,6 +128,7 @@ class Tester: self.log_dir = log_dir self.gui = gui self.verbose = verbose + self.start_time = datetime.datetime.now() @classmethod def determine_tests(cls, tests: List[Dict[str, Any]], model: str) \ @@ -194,7 +197,6 @@ class Tester: print() def prepare_for_results(self) -> None: - # prepare dict entry for results for test in self.tests: if not test['selected']: continue @@ -207,14 +209,13 @@ class Tester: print("%%% Test iteration: {} / {}". format(iteration + 1, self.iterations)) - success = self.run_iteration() + success = self.run_iteration(iteration) if not success: if self.abort_early: break - def run_iteration(self) -> bool: + def run_iteration(self, iteration: int) -> bool: iteration_success = True - for test in self.tests: if not test['selected']: continue @@ -226,7 +227,15 @@ class Tester: for i, case in enumerate(test['cases']): print("--> Test case {} of {}: '{}' running ...". format(i+1, len(test['cases']), case)) - was_success = self.run_test_case(test, case) + + log_dir = self.get_log_dir(iteration, test['model'], case) + if self.verbose: + print("creating log directory: {}" + .format(log_dir)) + os.makedirs(log_dir, exist_ok=True) + + was_success = self.run_test_case(test, case, log_dir) + if was_success: print(colorize( "--- Test case {} of {}: '{}' succeeded." @@ -247,7 +256,23 @@ class Tester: return iteration_success - def run_test_case(self, test: Dict[str, Any], case: str) -> bool: + def get_log_dir(self, iteration: int, model: str, case: str) -> str: + date_and_time = self.start_time.strftime("%Y-%m-%dT%H-%M-%SZ") + foldername = os.path.join(self.log_dir, date_and_time) + + if self.iterations > 1: + # Use as many zeros in foldername as required. + digits = math.floor(math.log10(self.iterations)) + 1 + foldername = os.path.join(foldername, + str(iteration + 1).zfill(digits)) + + foldername = os.path.join(foldername, model) + foldername = os.path.join(foldername, case.replace(" ", "_")) + + return foldername + + def run_test_case(self, test: Dict[str, Any], + case: str, logfile_path: str) -> bool: self.active_runners = [] speed_factor = self.speed_factor @@ -257,8 +282,9 @@ class Tester: if self.config['mode'] == 'sitl': px4_runner = ph.Px4Runner( os.getcwd(), - self.log_dir, - test, + logfile_path, + test['model'], + case, speed_factor, self.debugger, self.verbose) @@ -268,8 +294,9 @@ class Tester: if self.config['simulator'] == 'gazebo': gzserver_runner = ph.GzserverRunner( os.getcwd(), - self.log_dir, - test, + logfile_path, + test['model'], + case, self.speed_factor, self.verbose) gzserver_runner.start() @@ -278,16 +305,17 @@ class Tester: if self.gui: gzclient_runner = ph.GzclientRunner( os.getcwd(), - self.log_dir, - test, + logfile_path, + test['model'], + case, self.verbose) gzclient_runner.start() self.active_runners.append(gzclient_runner) test_runner = ph.TestRunner( os.getcwd(), - self.log_dir, - test, + logfile_path, + test['model'], case, self.config['mavlink_connection'], self.verbose) @@ -315,18 +343,26 @@ class Tester: for runner in self.active_runners: runner.stop() - test['cases'][case]['results'].append(is_success) + result = {'success': is_success, + 'logfiles': [runner.get_log_filename() + for runner in self.active_runners]} + test['cases'][case]['results'].append(result) + + if not is_success: + # TODO: print error and logfiles + pass + return is_success def show_detailed_results(self) -> None: - print() print(colorize("Results:", color.BOLD)) for test in self.tests: if not test['selected']: - print(colorize(colorize(" - {} (not selected)". - format(test['model']), color.BOLD), color.GRAY)) + print(colorize(colorize( + " - {} (not selected)". + format(test['model']), color.BOLD), color.GRAY)) continue else: print(colorize(" - {}:".format(test['model']), color.BOLD)) @@ -334,8 +370,10 @@ class Tester: for name, case in test['cases'].items(): print(" - '{}': ".format(name), end="") - n_succeeded = case['results'].count(True) - n_failed = case['results'].count(False) + n_succeeded = [result['success'] + for result in case['results']].count(True) + n_failed = [result['success'] + for result in case['results']].count(False) n_cancelled = self.iterations - len(case['results']) notes = [ diff --git a/test/mavsdk_tests/process_helper.py b/test/mavsdk_tests/process_helper.py index ac128efd3f..0e37a0a5ed 100644 --- a/test/mavsdk_tests/process_helper.py +++ b/test/mavsdk_tests/process_helper.py @@ -3,7 +3,6 @@ import queue import time import os -import datetime import atexit import subprocess import threading @@ -15,29 +14,30 @@ from typing import Dict, List, TextIO, Optional class Runner: def __init__(self, log_dir: str, - config: Dict[str, str], + model: str, + case: str, verbose: bool): self.name = "" self.cmd = "" self.cwd = "" self.args: List[str] self.env: Dict[str, str] - self.log_dir = log_dir - self.config = config + self.model = model + self.case = case self.log_filename = "" self.log_fd: TextIO self.verbose = verbose self.output_queue: queue.Queue[str] = queue.Queue() self.start_time = time.time() + self.log_dir = log_dir + self.log_filename = "" - def create_log_filename(self, model: str, test_filter: str) -> str: - return self.log_dir + os.path.sep + \ - "log-{}-{}-{}-{}.log".format( - self.name, - model, - test_filter, - datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%SZ")) - # TODO: improve logfilename, create folder, create merged log. + def set_log_filename(self) -> None: + self.log_filename = self.log_dir + os.path.sep + \ + "log-{}.log".format(self.name) + + def get_log_filename(self) -> str: + return self.log_filename def start(self) -> None: if self.verbose: @@ -45,8 +45,9 @@ class Runner: atexit.register(self.stop) - self.log_filename = self.create_log_filename( - self.config['model'], self.config['test_filter']) + self.set_log_filename() + if self.verbose: + print("Logging to {}".format(self.log_filename)) self.log_fd = open(self.log_filename, 'w') self.process = subprocess.Popen( @@ -138,9 +139,9 @@ class Runner: class Px4Runner(Runner): def __init__(self, workspace_dir: str, log_dir: str, - config: Dict[str, str], speed_factor: float, + model: str, case: str, speed_factor: float, debugger: str, verbose: bool): - super().__init__(log_dir, config, verbose) + super().__init__(log_dir, model, case, verbose) self.name = "px4" self.cmd = workspace_dir + "/build/px4_sitl_default/bin/px4" self.cwd = workspace_dir + "/build/px4_sitl_default/tmp/rootfs" @@ -153,7 +154,7 @@ class Px4Runner(Runner): "-d" ] self.env = {"PATH": str(os.environ['PATH']), - "PX4_SIM_MODEL": str(config['model']), + "PX4_SIM_MODEL": self.model, "PX4_SIM_SPEED_FACTOR": str(speed_factor)} self.debugger = debugger @@ -179,10 +180,11 @@ class GzserverRunner(Runner): def __init__(self, workspace_dir: str, log_dir: str, - config: Dict[str, str], + model: str, + case: str, speed_factor: float, verbose: bool): - super().__init__(log_dir, config, verbose) + super().__init__(log_dir, model, case, verbose) self.name = "gzserver" self.cwd = workspace_dir self.env = {"PATH": os.environ['PATH'], @@ -196,16 +198,17 @@ class GzserverRunner(Runner): self.cmd = "gzserver" self.args = ["--verbose", workspace_dir + "/Tools/sitl_gazebo/worlds/" + - config['model'] + ".world"] + self.model + ".world"] class GzclientRunner(Runner): def __init__(self, workspace_dir: str, log_dir: str, - config: Dict[str, str], + model: str, + case: str, verbose: bool): - super().__init__(log_dir, config, verbose) + super().__init__(log_dir, model, case, verbose) self.name = "gzclient" self.cwd = workspace_dir self.env = {"PATH": os.environ['PATH'], @@ -223,14 +226,14 @@ class TestRunner(Runner): def __init__(self, workspace_dir: str, log_dir: str, - config: Dict[str, str], - test: str, + model: str, + case: str, mavlink_connection: str, verbose: bool): - super().__init__(log_dir, config, verbose) + super().__init__(log_dir, model, case, verbose) self.name = "test_runner" self.cwd = workspace_dir self.env = {"PATH": os.environ['PATH']} self.cmd = workspace_dir + \ "/build/px4_sitl_default/mavsdk_tests/mavsdk_tests" - self.args = ["--url", mavlink_connection, test] + self.args = ["--url", mavlink_connection, case]