mavsdk_tests: leave tests backwards compatible but enable SIH

This reverts some of the changes to keep runners of classic Gazebo in
there but leave it optional, so we can still use it if required.

.github/workflows/sitl_tests.yml

@@ -118,7 +118,7 @@ jobs:
           PX4_HOME_LON: ${{matrix.config.longitude}}
           PX4_HOME_ALT: ${{matrix.config.altitude}}
           PX4_CMAKE_BUILD_TYPE: ${{matrix.config.build_type}}
-        run: test/mavsdk_tests/mavsdk_test_runner.py --speed-factor 10 --abort-early --model ${{matrix.config.model}} test/mavsdk_tests/configs/sitl.json --verbose --force-color
+        run: test/mavsdk_tests/mavsdk_test_runner.py --speed-factor 10 --abort-early --model ${{matrix.config.model}} test/mavsdk_tests/configs/gazebo-classic-sitl.json --verbose --force-color
         timeout-minutes: 45
 
       - name: Upload failed logs

Makefile

@@ -429,16 +429,14 @@ rostest: px4_sitl_default
 	@$(MAKE) --no-print-directory px4_sitl_default sitl_gazebo-classic
 
 tests_integration: px4_sitl_default
-	@$(MAKE) --no-print-directory px4_sitl_default sitl_gazebo-classic
 	@$(MAKE) --no-print-directory px4_sitl_default mavsdk_tests
-	@"$(SRC_DIR)"/test/mavsdk_tests/mavsdk_test_runner.py --speed-factor 20 test/mavsdk_tests/configs/sitl.json
+	@"$(SRC_DIR)"/test/mavsdk_tests/mavsdk_test_runner.py --speed-factor 20 test/mavsdk_tests/configs/sih-sitl.json
 
 tests_integration_coverage:
 	@$(MAKE) clean
 	@$(MAKE) --no-print-directory px4_sitl_default PX4_CMAKE_BUILD_TYPE=Coverage
-	@$(MAKE) --no-print-directory px4_sitl_default sitl_gazebo-classic
 	@$(MAKE) --no-print-directory px4_sitl_default mavsdk_tests
-	@"$(SRC_DIR)"/test/mavsdk_tests/mavsdk_test_runner.py --speed-factor 20 test/mavsdk_tests/configs/sitl.json
+	@"$(SRC_DIR)"/test/mavsdk_tests/mavsdk_test_runner.py --speed-factor 20 test/mavsdk_tests/configs/sih-sitl.json
 	@mkdir -p coverage
 	@lcov --directory build/px4_sitl_default --base-directory build/px4_sitl_default --gcov-tool gcov --capture -o coverage/lcov.info
 

docs/en/test_and_ci/integration_testing_mavsdk.md

@@ -45,10 +45,10 @@ Install either from binaries or source:
 
 ## Run All PX4 Tests
 
-To run all SITL tests as defined in [sitl.json](https://github.com/PX4/PX4-Autopilot/blob/main/test/mavsdk_tests/configs/sitl.json), do:
+To run all SITL tests using the SIH simulator as defined in [sih-sitl.json](https://github.com/PX4/PX4-Autopilot/blob/main/test/mavsdk_tests/configs/sih-sitl.json), do:
 
 ```sh
-test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sitl.json --speed-factor 10
+test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sih-sitl.json --speed-factor 10
 ```
 
 This will list all the tests and then run them sequentially.
@@ -87,10 +87,10 @@ options:
 ## Run a Single Test
 
 Run a single test by specifying the `model` and test `case` as command line options.
-For example, to test flying a tailsitter in a mission you might run:
+For example, to test a multicopter mission using the SIH simulator:
 
 ```sh
-test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sitl.json --speed-factor 10 --model tailsitter --case 'Fly VTOL mission'
+test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sih-sitl.json --speed-factor 10 --model quadx --case 'Fly square Multicopter Missions including RTL'
 ```
 
 The easiest way to find out the current set of models and their associated test cases is to run all PX4 tests [as shown above](#run-all-px4-tests) (note, you can then cancel the build if you wish to test just one).

docs/ko/test_and_ci/integration_testing_mavsdk.md

@@ -45,10 +45,10 @@ The tests need the MAVSDK C++ library installed system-wide (e.g. in `/usr/lib`
 
 ## 모든 PX4 테스트 실행
 
-To run all SITL tests as defined in [sitl.json](https://github.com/PX4/PX4-Autopilot/blob/main/test/mavsdk_tests/configs/sitl.json), do:
+To run all SITL tests using the SIH simulator as defined in [sih-sitl.json](https://github.com/PX4/PX4-Autopilot/blob/main/test/mavsdk_tests/configs/sih-sitl.json), do:
 
 ```sh
-test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sitl.json --speed-factor 10
+test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sih-sitl.json --speed-factor 10
 ```
 
 This will list all the tests and then run them sequentially.
@@ -87,10 +87,10 @@ options:
 ## 단일 테스트 실행
 
 Run a single test by specifying the `model` and test `case` as command line options.
-예를 들어, 임무에서 테일시터 비행을 테스트하려면, 다음을 실행합니다.
+For example, to test a multicopter mission using the SIH simulator:
 
 ```sh
-test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sitl.json --speed-factor 10 --model tailsitter --case 'Fly square Multicopter Missions including RTL'
+test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sih-sitl.json --speed-factor 10 --model quadx --case 'Fly square Multicopter Missions including RTL'
 ```
 
 The easiest way to find out the current set of models and their associated test cases is to run all PX4 tests [as shown above](#run-all-px4-tests) (note, you can then cancel the build if you wish to test just one).

docs/uk/test_and_ci/integration_testing_mavsdk.md

@@ -45,10 +45,10 @@ The tests need the MAVSDK C++ library installed system-wide (e.g. in `/usr/lib`
 
 ## Запуск усіх PX4 тестів
 
-To run all SITL tests as defined in [sitl.json](https://github.com/PX4/PX4-Autopilot/blob/main/test/mavsdk_tests/configs/sitl.json), do:
+To run all SITL tests using the SIH simulator as defined in [sih-sitl.json](https://github.com/PX4/PX4-Autopilot/blob/main/test/mavsdk_tests/configs/sih-sitl.json), do:
 
 ```sh
-test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sitl.json --speed-factor 10
+test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sih-sitl.json --speed-factor 10
 ```
 
 This will list all the tests and then run them sequentially.
@@ -87,10 +87,10 @@ options:
 ## Запуск одного тесту
 
 Run a single test by specifying the `model` and test `case` as command line options.
-Наприклад, щоб протестувати керування хвостовиком у місії, ви можете виконати:
+For example, to test a multicopter mission using the SIH simulator:
 
 ```sh
-test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sitl.json --speed-factor 10 --model tailsitter --case 'Fly VTOL mission'
+test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sih-sitl.json --speed-factor 10 --model quadx --case 'Fly square Multicopter Missions including RTL'
 ```
 
 The easiest way to find out the current set of models and their associated test cases is to run all PX4 tests [as shown above](#run-all-px4-tests) (note, you can then cancel the build if you wish to test just one).

docs/zh/test_and_ci/integration_testing_mavsdk.md

@@ -45,10 +45,10 @@ The tests need the MAVSDK C++ library installed system-wide (e.g. in `/usr/lib`
 
 ## 准备 PX4 源码
 
-To run all SITL tests as defined in [sitl.json](https://github.com/PX4/PX4-Autopilot/blob/main/test/mavsdk_tests/configs/sitl.json), do:
+To run all SITL tests using the SIH simulator as defined in [sih-sitl.json](https://github.com/PX4/PX4-Autopilot/blob/main/test/mavsdk_tests/configs/sih-sitl.json), do:
 
 ```sh
-test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sitl.json --speed-factor 10
+test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sih-sitl.json --speed-factor 10
 ```
 
 This will list all the tests and then run them sequentially.
@@ -87,10 +87,10 @@ options:
 ## 关于实现的说明
 
 Run a single test by specifying the `model` and test `case` as command line options.
-For example, to test flying a tailsitter in a mission you might run:
+For example, to test a multicopter mission using the SIH simulator:
 
 ```sh
-test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sitl.json --speed-factor 10 --model tailsitter --case 'Fly VTOL mission'
+test/mavsdk_tests/mavsdk_test_runner.py test/mavsdk_tests/configs/sih-sitl.json --speed-factor 10 --model quadx --case 'Fly square Multicopter Missions including RTL'
 ```
 
 The easiest way to find out the current set of models and their associated test cases is to run all PX4 tests [as shown above](#run-all-px4-tests) (note, you can then cancel the build if you wish to test just one).

test/mavsdk_tests/configs/gazebo-classic-sitl.json

@@ -1,18 +1,17 @@
 {
     "mode": "sitl",
     "simulator": "gazebo",
+    "model_prefix": "gazebo-classic_",
     "mavlink_connection": "udpin://0.0.0.0:14540",
     "tests":
     [
         {
             "model": "iris",
-            "vehicle": "iris",
             "test_filter": "[multicopter],[offboard],[offboard_attitude]",
             "timeout_min": 10
         },
         {
             "model": "iris",
-            "vehicle": "iris",
             "test_filter": "[offboard_attitude]",
             "timeout_min": 10,
             "env": {
@@ -22,19 +21,16 @@
         },
         {
             "model": "standard_vtol",
-            "vehicle": "standard_vtol",
             "test_filter": "[vtol], [vtol_wind], [vtol_airspeed_fail]",
             "timeout_min": 10
         },
         {
             "model": "tailsitter",
-            "vehicle": "tailsitter",
             "test_filter": "[vtol]",
             "timeout_min": 10
         },
         {
             "model": "typhoon_h480",
-            "vehicle": "typhoon_h480",
             "test_filter": "[controlallocation]",
             "timeout_min": 10
         }

test/mavsdk_tests/configs/sih-sitl.json

@@ -0,0 +1,27 @@
+{
+    "mode": "sitl",
+    "model_prefix": "sihsim_",
+    "mavlink_connection": "udpin://0.0.0.0:14540",
+    "tests":
+    [
+        {
+            "model": "quadx",
+            "test_filter": "[multicopter],[offboard],[offboard_attitude]",
+            "timeout_min": 10
+        },
+        {
+            "model": "quadx",
+            "test_filter": "[offboard_attitude]",
+            "timeout_min": 10,
+            "env": {
+                "PX4_PARAM_EKF2_EN": 0,
+                "PX4_PARAM_ATT_EN": 1
+            }
+        },
+        {
+            "model": "standard_vtol",
+            "test_filter": "[vtol]",
+            "timeout_min": 10
+        }
+    ]
+}

test/mavsdk_tests/integration_test_runner/process_helper.py

@@ -168,7 +168,7 @@ class Px4Runner(Runner):
                 os.path.join(workspace_dir, "test_data"),
                 "-d"
             ]
-        self.env["PX4_SIM_MODEL"] = "sihsim_quadx"
+        self.env["PX4_SIM_MODEL"] = model
         self.env["PX4_SIM_SPEED_FACTOR"] = str(speed_factor)
         self.debugger = debugger
         self.clear_rootfs()

test/mavsdk_tests/integration_test_runner/test_runner.py

@@ -302,7 +302,7 @@ class Tester:
         self.active_runners = []
 
         if self.config['mode'] == 'sitl':
-            if self.config['simulator'] == 'gazebo':
+            if self.config.get('simulator') == 'gazebo':
                 # Use RegEx to extract worldname.world from case name
                 match = re.search(r'\((.*?\.world)\)', case)
                 if match:
@@ -310,24 +310,52 @@ class Tester:
                 else:
                     world_name = 'empty.world'
 
-                # We must start the PX4 instance at the end, as starting
-                # it in the beginning, then connecting Gazebo server freaks
-                # out the PX4 (it needs to have data coming in when started),
-                # and can lead to EKF to freak out, or the instance itself
-                # to die unexpectedly.
-                px4_runner = ph.Px4Runner(
+                gzserver_runner = ph.GzserverRunner(
                     os.getcwd(),
                     log_dir,
                     test['model'],
                     case,
                     self.get_max_speed_factor(test),
-                    self.debugger,
                     self.verbose,
                     self.build_dir,
-                    self.tester_interface.rootfs_base_dirname())
-                for env_key in test.get('env', []):
-                    px4_runner.env[env_key] = str(test['env'][env_key])
-                self.active_runners.append(px4_runner)
+                    world_name)
+                self.active_runners.append(gzserver_runner)
+
+                gzmodelspawn_runner = ph.GzmodelspawnRunner(
+                    os.getcwd(),
+                    log_dir,
+                    test['model'],
+                    case,
+                    self.verbose,
+                    self.build_dir)
+                self.active_runners.append(gzmodelspawn_runner)
+
+                if self.gui:
+                    gzclient_runner = ph.GzclientRunner(
+                        os.getcwd(),
+                        log_dir,
+                        test['model'],
+                        case,
+                        self.verbose)
+                    self.active_runners.append(gzclient_runner)
+
+            # Determine PX4_SIM_MODEL using model_prefix from config
+            model_prefix = self.config.get('model_prefix', '')
+            px4_model = model_prefix + test['model']
+
+            px4_runner = ph.Px4Runner(
+                os.getcwd(),
+                log_dir,
+                px4_model,
+                case,
+                self.get_max_speed_factor(test),
+                self.debugger,
+                self.verbose,
+                self.build_dir,
+                self.tester_interface.rootfs_base_dirname())
+            for env_key in test.get('env', []):
+                px4_runner.env[env_key] = str(test['env'][env_key])
+            self.active_runners.append(px4_runner)
 
         self.active_runners.append(self.tester_interface.create_test_runner(
             os.getcwd(),

test/mavsdk_tests/mavsdk_test_runner.py

@@ -136,11 +136,9 @@ def is_everything_ready(config: Dict[str, str], build_dir: str) -> bool:
             result = False
         if not os.path.isfile(os.path.join(build_dir, 'bin/px4')):
             print("PX4 SITL is not built\n"
-                  "run `DONT_RUN=1 "
-                  "make px4_sitl gazebo mavsdk_tests` or "
-                  "`DONT_RUN=1 make px4_sitl_default gazebo mavsdk_tests`")
+                  "run `DONT_RUN=1 make px4_sitl_default mavsdk_tests`")
             result = False
-        if config['simulator'] == 'gazebo':
+        if config.get('simulator') == 'gazebo':
             if is_running('gzserver'):
                 print("gzserver process already running\n"
                       "run `killall gzserver` and try again")
@@ -153,9 +151,7 @@ def is_everything_ready(config: Dict[str, str], build_dir: str) -> bool:
     if not os.path.isfile(os.path.join(build_dir,
                                        'mavsdk_tests/mavsdk_tests')):
         print("Test runner is not built\n"
-              "run `DONT_RUN=1 "
-              "make px4_sitl gazebo mavsdk_tests` or "
-              "`DONT_RUN=1 make px4_sitl_default gazebo mavsdk_tests`")
+              "run `DONT_RUN=1 make px4_sitl_default mavsdk_tests`")
         result = False
 
     return result