ci: replace 14 workflows with 4-tier orchestrator

Replaces 14 CI workflows with a single ci-orchestrator.yml that runs jobs in a 4-tier waterfall. Tiers gate each other sequentially: if formatting fails in 2 minutes, nothing else runs. Every job carried over from the old workflows was optimized along the way. Jobs use native container: blocks instead of the old addnab/docker-run-action wrapper, cache scopes were split and tuned (hit rates went from ~48% to 99%+), SITL tests run at 20x speed on 8cpu runners, clang-tidy got a dedicated 16cpu runner and cache, the failsafe sim caches its emsdk, and flash analysis posts sticky PR comments. Forks can use this without AWS infrastructure. Copy .github/ci-config.yml.example to .github/ci-config.yml to customize runner labels, job toggles, and cache sizes. Alternatively, rename .github/workflows/ci-simple.yml.example to ci-simple.yml for a single-job workflow that finishes in under 15 minutes on ubuntu-latest. Signed-off-by: Ramon Roche <mrpollo@gmail.com>
2026-05-19 18:59:06 +08:00 · 2026-03-05 21:28:07 -08:00 · 2026-03-05 21:28:07 -08:00 · 885af949cb
commit 885af949cb
parent 7297364484
30 changed files with 2326 additions and 1220 deletions
--- a/.github/actions/build-gazebo-sitl/action.yml
+++ b/.github/actions/build-gazebo-sitl/action.yml
@ -0,0 +1,21 @@
+name: Build Gazebo Classic SITL
+description: Build PX4 firmware and Gazebo Classic plugins with ccache stats
+
+runs:
+  using: composite
+  steps:
+    - name: Build - PX4 Firmware (SITL)
+      shell: bash
+      run: make px4_sitl_default
+
+    - name: Cache - Stats after PX4 Firmware
+      shell: bash
+      run: ccache -s
+
+    - name: Build - Gazebo Classic Plugins
+      shell: bash
+      run: make px4_sitl_default sitl_gazebo-classic
+
+    - name: Cache - Stats after Gazebo Plugins
+      shell: bash
+      run: ccache -s
--- a/.github/actions/save-ccache/action.yml
+++ b/.github/actions/save-ccache/action.yml
@ -0,0 +1,22 @@
+name: Save ccache
+description: Print ccache stats and save to cache
+
+inputs:
+  cache-primary-key:
+    description: Primary cache key from setup-ccache output
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Cache - Stats
+      if: always()
+      shell: bash
+      run: ccache -s
+
+    - name: Cache - Save ccache
+      if: always()
+      uses: actions/cache/save@v4
+      with:
+        path: ~/.ccache
+        key: ${{ inputs.cache-primary-key }}
--- a/.github/actions/setup-ccache/action.yml
+++ b/.github/actions/setup-ccache/action.yml
@ -0,0 +1,56 @@
+name: Setup ccache
+description: Restore ccache from cache and configure ccache.conf
+
+inputs:
+  cache-key-prefix:
+    description: Cache key prefix (e.g. ccache-sitl)
+    required: true
+  max-size:
+    description: Max ccache size (e.g. 300M)
+    required: false
+    default: '300M'
+  base-dir:
+    description: ccache base_dir value
+    required: false
+    default: '${GITHUB_WORKSPACE}'
+  install-ccache:
+    description: Install ccache via apt before configuring
+    required: false
+    default: 'false'
+
+outputs:
+  cache-primary-key:
+    description: Primary cache key (pass to save-ccache)
+    value: ${{ steps.restore.outputs.cache-primary-key }}
+
+runs:
+  using: composite
+  steps:
+    - name: Cache - Install ccache
+      if: inputs.install-ccache == 'true'
+      shell: bash
+      run: apt-get update && apt-get install -y ccache
+
+    - name: Cache - Restore ccache
+      id: restore
+      uses: actions/cache/restore@v4
+      with:
+        path: ~/.ccache
+        key: ${{ inputs.cache-key-prefix }}-${{ github.ref_name }}-${{ github.sha }}
+        restore-keys: |
+          ${{ inputs.cache-key-prefix }}-${{ github.ref_name }}-
+          ${{ inputs.cache-key-prefix }}-${{ github.base_ref || 'main' }}-
+          ${{ inputs.cache-key-prefix }}-
+
+    - name: Cache - Configure ccache
+      shell: bash
+      run: |
+        mkdir -p ~/.ccache
+        echo "base_dir = ${{ inputs.base-dir }}" > ~/.ccache/ccache.conf
+        echo "compression = true" >> ~/.ccache/ccache.conf
+        echo "compression_level = 6" >> ~/.ccache/ccache.conf
+        echo "max_size = ${{ inputs.max-size }}" >> ~/.ccache/ccache.conf
+        echo "hash_dir = false" >> ~/.ccache/ccache.conf
+        echo "compiler_check = content" >> ~/.ccache/ccache.conf
+        ccache -s
+        ccache -z
--- a/.github/ci-config.yml.example
+++ b/.github/ci-config.yml.example
@ -0,0 +1,67 @@
+# PX4 CI Configuration for Forks
+# Copy to .github/ci-config.yml and customize.
+# All settings are optional — omit any key to use the upstream default.
+#
+# This example is tuned for GitHub-hosted runners.
+# The upstream PX4 repo uses RunsOn/AWS for all jobs.
+
+# Infrastructure hint — set to true when running on GitHub-hosted runners.
+# The upstream PX4 repo uses RunsOn/AWS (is_gha: false by default).
+# Jobs may use this flag to skip steps that require RunsOn-specific
+# infrastructure (e.g. S3 cache access, spot instance features).
+# No job behavior is gated on this today — it is wired for future use.
+is_gha: true
+
+# Runner labels — one label per size tier.
+# For GitHub-hosted runners, all tiers can point to ubuntu-latest.
+# For self-hosted runners, map each tier to the appropriate pool.
+runners:
+  small:   '["ubuntu-latest"]'
+  medium:  '["ubuntu-latest"]'
+  large:   '["ubuntu-latest"]'
+  utility: '["ubuntu-latest"]'
+
+# Job toggles — enable only what your fork needs.
+# Jobs marked false below are upstream-specific (hardware boards,
+# platform-specific builds) and rarely useful for feature forks.
+jobs:
+  # T1 — gate checks (disable only if you have a very good reason)
+  gate_checks:          true   # format, newlines, module config validation
+  shellcheck:           true   # shell script linting
+  mavsdk_checks:        true   # mypy + flake8 on MAVSDK Python scripts
+
+  # T2 — core checks (recommended for all forks)
+  sitl_build:           true
+  clang_tidy:           true
+
+  # T2 — upstream-specific, disable for most forks
+  gazebo_classic_build: false  # only needed if running sitl_tests/mavros_tests
+  ubuntu_builds:        false  # verifies clean builds on bare Ubuntu images
+  macos_build:          false  # macOS platform verification
+  itcm_check:           false  # NuttX board ITCM placement, board-specific
+  flash_analysis:       false  # binary size diffing against PR base, repo-specific
+  failsafe_sim:         false  # Emscripten web simulation build
+
+  # T3 — integration tests (expensive, disable unless you need them)
+  sitl_tests:           false  # requires gazebo_classic_build
+  ros_integration:      false  # requires Galactic ROS container
+  mavros_tests:         false  # requires gazebo_classic_build
+  ros_translation_node: true   # lightweight, runs on standard ROS images
+
+# Cache sizes — adjust if you hit your cache storage limit.
+# GitHub-hosted runners share a 10 GB per-repo native cache.
+# With the job set above (~60% of jobs disabled), total warm footprint
+# is roughly 15-20% of the 10 GB limit per concurrent PR.
+# If you re-enable more jobs, total footprint scales up proportionally.
+# Raise your repo's cache limit if needed:
+#   Settings → Actions → Caches (up to 10 TB configurable)
+cache:
+  sitl:                300M   # ~94% fill at current default — do not lower
+  sitl_gazebo_classic: 350M   # only relevant if gazebo_classic_build: true
+  clang_tidy:          150M   # ~40% fill, sized with headroom
+  ubuntu_builds:       200M   # only relevant if ubuntu_builds: true
+  macos:               200M   # only relevant if macos_build: true
+  itcm:                200M   # only relevant if itcm_check: true, growing
+  flash:               200M   # only relevant if flash_analysis: true
+  ros_integration:     400M   # ~70% fill, trending up
+  ros_translation:     150M   # ~35% fill, sized with headroom
--- a/.github/workflows/build_all_targets.yml
+++ b/.github/workflows/build_all_targets.yml
@ -37,21 +37,19 @@
 name: Build all targets

 on:
+  # Triggered by CI orchestrator for all branches/PRs (after all tiers pass)
+  workflow_run:
+    workflows: ["CI Pipeline (Orchestrator)"]
+    types: [completed]
+    branches: ['**']
+  # Direct trigger for tagged releases (orchestrator doesn't run on tags)
  push:
    tags:
      - 'v*'
-    branches:
-      - 'main'
-      - 'stable'
-      - 'beta'
-      - 'release/**'
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - '**'
    paths-ignore:
      - 'docs/**'
+  # Manual trigger for debugging
+  workflow_dispatch:

 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
@ -65,14 +63,27 @@ permissions:
 jobs:
  group_targets:
    name: Scan for Board Targets
+    # Only run if:
+    # 1. Direct push to tag (independent trigger for releases), OR
+    # 2. Orchestrator workflow_run completed successfully (for all branches/PRs), OR
+    # 3. Manual workflow_dispatch
+    if: |
+      github.event_name == 'push' ||
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')
    # runs-on: ubuntu-latest
    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    permissions:
+      contents: read
+      actions: read
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
      timestamp: ${{ steps.set-timestamp.outputs.timestamp }}
      branchname: ${{ steps.set-branch.outputs.branchname }}
    steps:
      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }}

      - name: Cache Python pip
        uses: actions/cache@v4
@ -102,12 +113,13 @@ jobs:
      - id: set-branch
        name: Save Current Branch Name
        run: |
-          echo "branchname=${{
-            github.event_name == 'pull_request' &&
-            format('pr-{0}', github.event.pull_request.number) ||
-            github.head_ref ||
-            github.ref_name
-          }}" >> $GITHUB_OUTPUT
+          if [ "${{ github.event_name }}" = "workflow_run" ]; then
+            # For workflow_run events, get branch from the triggering workflow
+            echo "branchname=${{ github.event.workflow_run.head_branch }}" >> $GITHUB_OUTPUT
+          else
+            # For push/workflow_dispatch events
+            echo "branchname=${{ github.head_ref || github.ref_name }}" >> $GITHUB_OUTPUT
+          fi

      - name: Debug Matrix Output
        if: runner.debug == '1'
@ -121,6 +133,9 @@ jobs:
    # runs-on: ubuntu-latest
    runs-on: [runs-on,"runner=8cpu-linux-${{ matrix.runner }}","image=ubuntu24-full-${{ matrix.runner }}","run-id=${{ github.run_id }}",spot=false]
    needs: group_targets
+    permissions:
+      contents: read
+      packages: read
    strategy:
      matrix: ${{ fromJson(needs.group_targets.outputs.matrix) }}
      fail-fast: false
@ -133,6 +148,7 @@ jobs:
      - uses: runs-on/action@v2
      - uses: actions/checkout@v4
        with:
+          ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }}
          fetch-depth: 0

      - name: Git ownership workaround
@ -207,6 +223,8 @@ jobs:
    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
    needs: [setup, group_targets]
    if: startsWith(github.ref, 'refs/tags/v') || contains(fromJSON('["main","stable","beta"]'), needs.group_targets.outputs.branchname)
+    permissions:
+      contents: write
    outputs:
      uploadlocation: ${{ steps.upload-location.outputs.uploadlocation }}
    steps:
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@ -1,60 +0,0 @@
-name: Checks
-
-on:
-  push:
-    branches:
-    - 'main'
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - '**'
-    paths-ignore:
-      - 'docs/**'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    container:
-      image: px4io/px4-dev:v1.16.0-rc1-258-g0369abd556
-
-    strategy:
-      fail-fast: false
-      matrix:
-        check: [
-          "check_format",
-          "check_newlines",
-          "tests",
-          "tests_coverage",
-          "px4_fmu-v2_default stack_check",
-          "validate_module_configs",
-          "shellcheck_all",
-          "NO_NINJA_BUILD=1 px4_fmu-v5_default",
-          "NO_NINJA_BUILD=1 px4_sitl_default",
-          "px4_sitl_allyes",
-          "module_documentation",
-        ]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Building [${{ matrix.check }}]
-        run: |
-          cd "$GITHUB_WORKSPACE"
-          git config --global --add safe.directory "$GITHUB_WORKSPACE"
-          make ${{ matrix.check }}
-
-      - name: Uploading Coverage to Codecov.io
-        if: contains(matrix.check, 'coverage')
-        uses: codecov/codecov-action@v1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          flags: unittests
-          file: coverage/lcov.info
--- a/.github/workflows/ci-orchestrator.yml
+++ b/.github/workflows/ci-orchestrator.yml
--- a/.github/workflows/ci-simple.yml.example
+++ b/.github/workflows/ci-simple.yml.example
@ -0,0 +1,41 @@
+name: Simple CI (Fork-Friendly)
+
+# Forks: This is a lightweight CI workflow for forks that don't need
+# the full orchestrator pipeline. It builds PX4 SITL + one hardware
+# target and runs unit tests + format checks.
+#
+# To use: rename this file from ci-simple.yml.example to ci-simple.yml
+# and enable GitHub Actions in your fork settings.
+# To customize: adjust the 'make' target or container image tag below.
+
+on:
+  push:
+    branches: ['**']
+  pull_request:
+    branches: ['**']
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  quick-check:
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/px4/px4-dev:v1.17.0-beta1
+    steps:
+      - name: Configure git
+        run: git config --system --add safe.directory '*'
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: recursive
+
+      - name: Quick Check (SITL + FMUv5 + tests + format)
+        run: make quick_check
+        env:
+          RUNS_IN_DOCKER: true
--- a/.github/workflows/clang-tidy.yml
+++ b/.github/workflows/clang-tidy.yml
@ -1,69 +0,0 @@
-name: Static Analysis
-
-on:
-  push:
-    branches:
-    - 'main'
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - '**'
-    paths-ignore:
-      - 'docs/**'
-
-permissions:
-  contents: read
-
-jobs:
-  clang_tidy:
-    name: Clang-Tidy
-    runs-on: [runs-on, runner=16cpu-linux-x64, "run-id=${{ github.run_id }}", "extras=s3-cache"]
-    container:
-      image: px4io/px4-dev:v1.17.0-beta1
-    steps:
-    - uses: runs-on/action@v2
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-        fetch-tags: true
-
-    - name: Configure Git Safe Directory
-      run: git config --system --add safe.directory '*'
-
-    - name: Restore Compiler Cache
-      id: cc_restore
-      uses: actions/cache/restore@v4
-      with:
-        path: ~/.ccache
-        key: ccache-clang-tidy-${{ github.head_ref || github.ref_name }}
-        restore-keys: |
-          ccache-clang-tidy-${{ github.head_ref || github.ref_name }}-
-          ccache-clang-tidy-main-
-          ccache-clang-tidy-
-
-    - name: Configure Compiler Cache
-      run: |
-        mkdir -p ~/.ccache
-        echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
-        echo "compression = true" >> ~/.ccache/ccache.conf
-        echo "compression_level = 6" >> ~/.ccache/ccache.conf
-        echo "max_size = 120M" >> ~/.ccache/ccache.conf
-        echo "hash_dir = false" >> ~/.ccache/ccache.conf
-        echo "compiler_check = content" >> ~/.ccache/ccache.conf
-        ccache -s
-        ccache -z
-
-    - name: Run Clang-Tidy Analysis
-      run: make -j16 clang-tidy
-
-    - name: Compiler Cache Stats
-      if: always()
-      run: ccache -s
-
-    - name: Save Compiler Cache
-      if: always()
-      uses: actions/cache/save@v4
-      with:
-        path: ~/.ccache
-        key: ${{ steps.cc_restore.outputs.cache-primary-key }}
--- a/.github/workflows/compile_macos.yml
+++ b/.github/workflows/compile_macos.yml
@ -1,67 +0,0 @@
-name: MacOS build
-
-on:
-  push:
-    branches:
-    - 'main'
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - '**'
-    paths-ignore:
-      - 'docs/**'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    runs-on: macos-latest
-    strategy:
-      matrix:
-        config: [
-          px4_fmu-v5_default,
-          px4_sitl
-          ]
-    steps:
-    - name: install Python 3.10
-      uses: actions/setup-python@v5
-      with:
-        python-version: "3.10"
-
-    - uses: actions/checkout@v4
-
-    - name: setup
-      run: |
-        ./Tools/setup/macos.sh
-
-    - name: Prepare ccache timestamp
-      id: ccache_cache_timestamp
-      shell: cmake -P {0}
-      run: |
-        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
-        file(APPEND "$ENV{GITHUB_OUTPUT}" "timestamp=${current_date}\n")
-    - name: ccache cache files
-      uses: actions/cache@v4
-      with:
-        path: ~/.ccache
-        key: macos_${{matrix.config}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
-        restore-keys: macos_${{matrix.config}}-ccache-
-    - name: setup ccache
-      run: |
-          mkdir -p ~/.ccache
-          echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
-          echo "compression = true" >> ~/.ccache/ccache.conf
-          echo "compression_level = 6" >> ~/.ccache/ccache.conf
-          echo "max_size = 40M" >> ~/.ccache/ccache.conf
-          echo "hash_dir = false" >> ~/.ccache/ccache.conf
-          ccache -s
-          ccache -z
-
-    - name: make ${{matrix.config}}
-      run: |
-          ccache -z
-          make ${{matrix.config}}
-          ccache -s
--- a/.github/workflows/compile_ubuntu.yml
+++ b/.github/workflows/compile_ubuntu.yml
@ -1,57 +0,0 @@
-name: Ubuntu environment build
-
-on:
-  push:
-    branches:
-      - 'main'
-      - 'stable'
-      - 'beta'
-      - 'release/**'
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - '**'
-    paths-ignore:
-      - 'docs/**'
-
-env:
-  RUNS_IN_DOCKER: true
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build_and_test:
-    name: Build and Test
-    strategy:
-      fail-fast: false
-      matrix:
-        version: ['ubuntu:22.04', 'ubuntu:24.04']
-    runs-on: [runs-on,runner=4cpu-linux-x64,"image=ubuntu24-full-x64","run-id=${{ github.run_id }}",spot=false]
-    container:
-      image: ${{ matrix.version }}
-      volumes:
-        - /github/workspace:/github/workspace
-    steps:
-
-      - name: Fix git in container
-        run: |
-          # we only need this because we are running the job in a container
-          # when checkout pulls git it does it in a shared volume
-          # and file ownership changes between steps
-          # first we install git since its missing from the base image
-          # then we mark the directory as safe for other instances
-          # of git to use.
-          apt update && apt install git -y
-          git config --global --add safe.directory $(realpath .)
-
-      - uses: actions/checkout@v4
-
-      - name: Install Deps, Build, and Make Quick Check
-        run: |
-          # we need to install dependencies and build on the same step
-          # given the stateless nature of docker images
-          ./Tools/setup/ubuntu.sh
-          make quick_check
--- a/.github/workflows/ekf_functional_change_indicator.yml
+++ b/.github/workflows/ekf_functional_change_indicator.yml
@ -1,35 +0,0 @@
-name: EKF Change Indicator
-
-on:
-  pull_request:
-    branches:
-      - '**'
-    paths-ignore:
-      - 'docs/**'
-
-# If two events are triggered within a short time in the same PR, cancel the run of the oldest event
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
-  cancel-in-progress: true
-
-jobs:
-  unit_tests:
-    runs-on: ubuntu-latest
-
-    container:
-      image: px4io/px4-dev:v1.16.0-rc1-258-g0369abd556
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: main test
-        run: |
-          cd "$GITHUB_WORKSPACE"
-          git config --global --add safe.directory "$GITHUB_WORKSPACE"
-          make tests TESTFILTER=EKF
-
-      - name: Check if there is a functional change
-        run: git diff --exit-code
-        working-directory: src/modules/ekf2/test/change_indication
--- a/.github/workflows/ekf_update_change_indicator.yml
+++ b/.github/workflows/ekf_update_change_indicator.yml
@ -1,54 +0,0 @@
-name: EKF Update Change Indicator
-
-on:
-  push:
-    paths-ignore:
-      - 'docs/**'
-
-jobs:
-  unit_tests:
-    runs-on: ubuntu-latest
-
-    container:
-      image: px4io/px4-dev:v1.16.0-rc1-258-g0369abd556
-
-    env:
-      GIT_COMMITTER_EMAIL: bot@px4.io
-      GIT_COMMITTER_NAME: PX4BuildBot
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: main test
-        run: |
-          cd "$GITHUB_WORKSPACE"
-          git config --global --add safe.directory "$GITHUB_WORKSPACE"
-          make tests TESTFILTER=EKF
-
-      - name: Check if there exists diff and save result in variable
-        id: diff-check
-        working-directory: src/modules/ekf2/test/change_indication
-        run: |
-          if git diff --quiet; then
-            echo "CHANGE_INDICATED=false" >> $GITHUB_OUTPUT
-          else
-            echo "CHANGE_INDICATED=true" >> $GITHUB_OUTPUT
-          fi
-
-      - name: auto-commit any changes to change indication
-        if: steps.diff-check.outputs.CHANGE_INDICATED == 'true'
-        uses: stefanzweifel/git-auto-commit-action@v4
-        with:
-          file_pattern: 'src/modules/ekf2/test/change_indication/*.csv'
-          commit_user_name: ${{ env.GIT_COMMITTER_NAME }}
-          commit_user_email: ${{ env.GIT_COMMITTER_EMAIL }}
-          commit_message: |
-            [AUTO COMMIT] update change indication
-
-            See .github/workflows/ekf_update_change_indicator.yml for more details
-
-      - name: if there is a functional change, fail check
-        if: steps.diff-check.outputs.CHANGE_INDICATED == 'true'
-        run: exit 1
--- a/.github/workflows/failsafe_sim.yml
+++ b/.github/workflows/failsafe_sim.yml
@ -1,58 +0,0 @@
-name: Failsafe Simulator Build
-
-on:
-  push:
-    branches:
-    - 'main'
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - '**'
-    paths-ignore:
-      - 'docs/**'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      fail-fast: false
-      matrix:
-        check: [
-          "failsafe_web",
-        ]
-    container:
-      image: px4io/px4-dev:v1.16.0-rc1-258-g0369abd556
-      options: --privileged --ulimit core=-1 --security-opt seccomp=unconfined
-    steps:
-      - name: Install Node v20.18.0
-        uses: actions/setup-node@v4
-        with:
-          node-version: 20.18.0
-
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Git ownership workaround
-        run: git config --system --add safe.directory '*'
-
-      - name: Install empscripten
-        run: |
-          git clone https://github.com/emscripten-core/emsdk.git _emscripten_sdk
-          cd _emscripten_sdk
-          git checkout 4.0.15
-          ./emsdk install latest
-          ./emsdk activate latest
-
-      - name: Testing [${{ matrix.check }}]
-        run: |
-          . ./_emscripten_sdk/emsdk_env.sh
-          make ${{ matrix.check }}
--- a/.github/workflows/flash_analysis.yml
+++ b/.github/workflows/flash_analysis.yml
@ -1,154 +0,0 @@
-name: FLASH usage analysis
-
-permissions:
-  contents: read
-  pull-requests: write
-  issues: write
-
-on:
-  push:
-    branches:
-      - 'main'
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - '**'
-    paths-ignore:
-      - 'docs/**'
-
-env:
-  MIN_FLASH_POS_DIFF_FOR_COMMENT: 50
-  MIN_FLASH_NEG_DIFF_FOR_COMMENT: -50
-
-jobs:
-  analyze_flash:
-    name: Analyzing ${{ matrix.target }}
-    runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
-    container:
-      image: px4io/px4-dev:v1.16.0-rc1-258-g0369abd556
-    strategy:
-      matrix:
-        target: [px4_fmu-v5x, px4_fmu-v6x]
-    outputs:
-      px4_fmu-v5x-bloaty-output: ${{ steps.gen-output.outputs.px4_fmu-v5x-bloaty-output }}
-      px4_fmu-v5x-bloaty-summary-map: ${{ steps.gen-output.outputs.px4_fmu-v5x-bloaty-summary-map }}
-      px4_fmu-v6x-bloaty-output: ${{ steps.gen-output.outputs.px4_fmu-v6x-bloaty-output }}
-      px4_fmu-v6x-bloaty-summary-map: ${{ steps.gen-output.outputs.px4_fmu-v6x-bloaty-summary-map }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          submodules: recursive
-
-      - name: Git ownership workaround
-        run: git config --system --add safe.directory '*'
-
-      - name: Build Target
-        run: make ${{ matrix.target }}_flash-analysis
-
-      - name: Store the ELF with the change
-        run: cp ./build/**/*.elf ./with-change.elf
-
-      - name: Clean previous build
-        run: |
-          make clean
-          make distclean
-          make submodulesclean
-
-      - name: If it's a PR checkout the base branch
-        if: ${{ github.event.pull_request }}
-        # As checkout creates a merge commit (merging the base branch into the PR branch), the base branch is the base for a diff of the PR changes.
-        run: git checkout ${{ github.event.pull_request.base.ref }}
-
-      - name: If it's a push checkout the previous commit
-        if: github.event_name == 'push'
-        run: git checkout ${{ github.event.before }}
-
-      - name: Update submodules
-        run: make submodulesupdate
-
-      - name: Build
-        run: make ${{ matrix.target }}_flash-analysis
-
-      - name: Store the ELF before the change
-        run: cp ./build/**/*.elf ./before-change.elf
-
-      - name: bloaty-action
-        uses: PX4/bloaty-action@v1.0.0
-        id: bloaty-step
-        with:
-          bloaty-file-args: ./with-change.elf -- ./before-change.elf
-          bloaty-additional-args: -d sections,symbols -s vm -n 20
-          output-to-summary: true
-
-      - name: Generate output
-        id: gen-output
-        run: |
-          EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
-          echo "${{ matrix.target }}-bloaty-output<<$EOF" >> $GITHUB_OUTPUT
-          echo "${{ steps.bloaty-step.outputs.bloaty-output-encoded }}" >> $GITHUB_OUTPUT
-          echo "$EOF" >> $GITHUB_OUTPUT
-          echo "${{ matrix.target }}-bloaty-summary-map<<$EOF" >> $GITHUB_OUTPUT
-          echo '${{ steps.bloaty-step.outputs.bloaty-summary-map }}' >> $GITHUB_OUTPUT
-          echo "$EOF" >> $GITHUB_OUTPUT
-
-  # TODO:
-  # This part of the workflow is causing errors for forks. We should find a way to fix this and enable it again for forks.
-  # Track this issue https://github.com/PX4/PX4-Autopilot/issues/24408
-  post_pr_comment:
-    name: Publish Results
-    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}"]
-    needs: [analyze_flash]
-    env:
-      V5X-SUMMARY-MAP-ABS: ${{ fromJSON(fromJSON(needs.analyze_flash.outputs.px4_fmu-v5x-bloaty-summary-map).vm-absolute) }}
-      V5X-SUMMARY-MAP-PERC: ${{ fromJSON(fromJSON(needs.analyze_flash.outputs.px4_fmu-v5x-bloaty-summary-map).vm-percentage) }}
-      V6X-SUMMARY-MAP-ABS: ${{ fromJSON(fromJSON(needs.analyze_flash.outputs.px4_fmu-v6x-bloaty-summary-map).vm-absolute) }}
-      V6X-SUMMARY-MAP-PERC: ${{ fromJSON(fromJSON(needs.analyze_flash.outputs.px4_fmu-v6x-bloaty-summary-map).vm-percentage) }}
-    if: github.event.pull_request && github.event.pull_request.head.repo.full_name == github.repository
-    steps:
-      - name: Find Comment
-        uses: peter-evans/find-comment@v3
-        id: fc
-        with:
-          issue-number: ${{ github.event.pull_request.number }}
-          comment-author: 'github-actions[bot]'
-          body-includes: FLASH Analysis
-
-      - name: Set Build Time
-        id: bt
-        run: |
-          echo "timestamp=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_OUTPUT
-
-      - name: Create or update comment
-        # This can't be moved to the job-level conditions, as GH actions don't allow a job-level if condition to access the env.
-        if: |
-          steps.fc.outputs.comment-id != '' ||
-          env.V5X-SUMMARY-MAP-ABS >= fromJSON(env.MIN_FLASH_POS_DIFF_FOR_COMMENT) ||
-          env.V5X-SUMMARY-MAP-ABS <= fromJSON(env.MIN_FLASH_NEG_DIFF_FOR_COMMENT) ||
-          env.V6X-SUMMARY-MAP-ABS >= fromJSON(env.MIN_FLASH_POS_DIFF_FOR_COMMENT) ||
-          env.V6X-SUMMARY-MAP-ABS <= fromJSON(env.MIN_FLASH_NEG_DIFF_FOR_COMMENT)
-        uses: peter-evans/create-or-update-comment@v4
-        with:
-          comment-id: ${{ steps.fc.outputs.comment-id }}
-          issue-number: ${{ github.event.pull_request.number }}
-          body: |
-            ## 🔎 FLASH Analysis
-            <details>
-              <summary>px4_fmu-v5x [Total VM Diff: ${{ env.V5X-SUMMARY-MAP-ABS }} byte (${{ env.V5X-SUMMARY-MAP-PERC}} %)]</summary>
-
-              ```
-              ${{ needs.analyze_flash.outputs.px4_fmu-v5x-bloaty-output }}
-              ```
-            </details>
-
-            <details>
-              <summary>px4_fmu-v6x [Total VM Diff: ${{ env.V6X-SUMMARY-MAP-ABS }} byte (${{ env.V6X-SUMMARY-MAP-PERC }} %)]</summary>
-
-              ```
-              ${{ needs.analyze_flash.outputs.px4_fmu-v6x-bloaty-output }}
-              ```
-            </details>
-
-            **Updated: _${{ steps.bt.outputs.timestamp }}_**
-          edit-mode: replace
--- a/.github/workflows/itcm_check.yml
+++ b/.github/workflows/itcm_check.yml
@ -1,67 +0,0 @@
-name: ITCM check
-
-permissions:
-  contents: read
-
-on:
-  push:
-    branches:
-      - 'main'
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - '**'
-    paths-ignore:
-      - 'docs/**'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  check_itcm:
-    name: Checking ${{ matrix.target }}
-    runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
-    container:
-      image: px4io/px4-dev:v1.16.0-rc1-258-g0369abd556
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - target: px4_fmu-v5x
-            scripts: >
-              boards/px4/fmu-v5x/nuttx-config/scripts/itcm_gen_functions.ld
-              boards/px4/fmu-v5x/nuttx-config/scripts/itcm_static_functions.ld
-          - target: px4_fmu-v6xrt
-            scripts: >
-              boards/px4/fmu-v6xrt/nuttx-config/scripts/itcm_functions_includes.ld
-              boards/px4/fmu-v6xrt/nuttx-config/scripts/itcm_static_functions.ld
-          - target: nxp_tropic-community
-            scripts: >
-              boards/nxp/tropic-community/nuttx-config/scripts/itcm_functions_includes.ld
-              boards/nxp/tropic-community/nuttx-config/scripts/itcm_static_functions.ld
-          - target: nxp_mr-tropic
-            scripts: >
-              boards/nxp/mr-tropic/nuttx-config/scripts/itcm_functions_includes.ld
-              boards/nxp/mr-tropic/nuttx-config/scripts/itcm_static_functions.ld
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          submodules: recursive
-
-      - name: Git ownership workaround
-        run: git config --system --add safe.directory '*'
-
-      - name: Build Target
-        run: make ${{ matrix.target }}
-
-      - name: Copy built ELF
-        run: cp ./build/**/*.elf ./built.elf
-
-      - name: Install itcm-check dependencies
-        run: pip3 install -r Tools/setup/optional-requirements.txt --break-system-packages
-
-      - name: Execute the itcm-check
-        run: python3 Tools/itcm_check.py --elf-file built.elf --script-files ${{ matrix.scripts }}
--- a/.github/workflows/mavros_mission_tests.yml
+++ b/.github/workflows/mavros_mission_tests.yml
@ -1,45 +0,0 @@
-name: MAVROS Mission Tests
-
-on:
-  push:
-    branches:
-    - 'main'
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - '**'
-    paths-ignore:
-      - 'docs/**'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Build SITL and Run Tests (inside old ROS container)
-        run: |
-          docker run --rm \
-            -v "${GITHUB_WORKSPACE}:/workspace" \
-            -w /workspace \
-            px4io/px4-dev-ros-melodic:2021-09-08 \
-            bash -c '
-              git config --global --add safe.directory /workspace
-              make px4_sitl_default
-              make px4_sitl_default sitl_gazebo-classic
-              ./test/rostest_px4_run.sh \
-                mavros_posix_test_mission.test \
-                mission:=MC_mission_box \
-                vehicle:=iris
-            '
--- a/.github/workflows/mavros_offboard_tests.yml
+++ b/.github/workflows/mavros_offboard_tests.yml
@ -1,44 +0,0 @@
-name: MAVROS Offboard Tests
-
-on:
-  push:
-    branches:
-    - 'main'
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - '**'
-    paths-ignore:
-      - 'docs/**'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Build SITL and Run Tests (inside old ROS container)
-        run: |
-          docker run --rm \
-            -v "${GITHUB_WORKSPACE}:/workspace" \
-            -w /workspace \
-            px4io/px4-dev-ros-melodic:2021-09-08 \
-            bash -c '
-              git config --global --add safe.directory /workspace
-              make px4_sitl_default
-              make px4_sitl_default sitl_gazebo-classic
-              ./test/rostest_px4_run.sh \
-                mavros_posix_tests_offboard_posctl.test \
-                vehicle:=iris
-            '
--- a/.github/workflows/nuttx_env_config.yml
+++ b/.github/workflows/nuttx_env_config.yml
@ -1,46 +0,0 @@
-name: Nuttx Target with extra env config
-
-on:
-  push:
-    branches:
-    - 'main'
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - '**'
-    paths-ignore:
-      - 'docs/**'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    container:
-      image: px4io/px4-dev:v1.16.0-rc1-258-g0369abd556
-
-    strategy:
-      matrix:
-        config:
-          - px4_fmu-v5_default
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Build PX4 and Run Test [${{ matrix.config }}]
-        run: |
-          cd "$GITHUB_WORKSPACE"
-          git config --global --add safe.directory "$GITHUB_WORKSPACE"
-          export PX4_EXTRA_NUTTX_CONFIG='CONFIG_NSH_LOGIN_PASSWORD="test";CONFIG_NSH_CONSOLE_LOGIN=y'
-          echo "PX4_EXTRA_NUTTX_CONFIG: $PX4_EXTRA_NUTTX_CONFIG"
-
-          make ${{ matrix.config }} nuttx_context
-
-          echo "Check that the config option is set"
-          grep CONFIG_NSH_LOGIN_PASSWORD build/${{ matrix.config }}/NuttX/nuttx/.config
--- a/.github/workflows/python_checks.yml
+++ b/.github/workflows/python_checks.yml
@ -1,33 +0,0 @@
-name: Python CI Checks
-
-on:
-  push:
-    branches:
-    - 'main'
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - '**'
-    paths-ignore:
-      - 'docs/**'
-
-jobs:
-  build:
-    runs-on: ubuntu-24.04
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-
-    - name: Install Python3
-      run: sudo apt-get install python3 python3-setuptools python3-pip -y
-
-    - name: Install tools
-      run: python3 -m pip install mypy types-requests flake8 --break-system-packages
-
-    - name: Check MAVSDK test scripts with mypy
-      run: $HOME/.local/bin/mypy --strict test/mavsdk_tests/*.py
-
-    - name: Check MAVSDK test scripts with flake8
-      run: $HOME/.local/bin/flake8 test/mavsdk_tests/*.py
--- a/.github/workflows/ros_integration_tests.yml
+++ b/.github/workflows/ros_integration_tests.yml
@ -1,134 +0,0 @@
-# NOTE: this workflow is now running on Dronecode / PX4 AWS account.
-# - If you want to keep the tests running in GitHub Actions you need to uncomment the "runs-on: ubuntu-latest" lines
-#   and comment the "runs-on: [runs-on,runner=..." lines.
-# - If you would like to duplicate this setup try setting up "RunsOn" on your own AWS account try https://runs-on.com
-
-name: ROS Integration Tests
-
-on:
-  push:
-    branches:
-    - 'main'
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - '**'
-    paths-ignore:
-      - 'docs/**'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false]
-    container:
-      image: px4io/px4-dev-ros2-galactic:2021-09-08
-      options: --privileged --ulimit core=-1 --security-opt seccomp=unconfined
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        fetch-depth: 0
-
-    - name: Git Ownership Workaround
-      run: git config --system --add safe.directory '*'
-
-    - name: Update ROS Keys
-      run: |
-        sudo rm /etc/apt/sources.list.d/ros2.list && \
-        sudo curl -sSL https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -o /usr/share/keyrings/ros-archive-keyring.gpg && \
-        echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/ros2/ubuntu $(. /etc/os-release && echo $UBUNTU_CODENAME) main" | sudo tee /etc/apt/sources.list.d/ros2.list > /dev/null
-
-    - name: Install gazebo
-      run: |
-        apt update && apt install -y gazebo11 libgazebo11-dev gstreamer1.0-plugins-bad gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-ugly libgstreamer-plugins-base1.0-dev
-
-    - name: Prepare ccache timestamp
-      id: ccache_cache_timestamp
-      shell: cmake -P {0}
-      run: |
-        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
-        message("::set-output name=timestamp::${current_date}")
-    - name: ccache cache files
-      uses: actions/cache@v4
-      with:
-        path: ~/.ccache
-        key: ros_integration_tests-${{matrix.config.build_type}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
-        restore-keys: ros_integration_tests-${{matrix.config.build_type}}-ccache-
-    - name: setup ccache
-      run: |
-        mkdir -p ~/.ccache
-        echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
-        echo "compression = true" >> ~/.ccache/ccache.conf
-        echo "compression_level = 6" >> ~/.ccache/ccache.conf
-        echo "max_size = 300M" >> ~/.ccache/ccache.conf
-        echo "hash_dir = false" >> ~/.ccache/ccache.conf
-        ccache -s
-        ccache -z
-
-    - name: Get and build micro-xrce-dds-agent
-      run: |
-        cd /opt
-        git clone --recursive https://github.com/eProsima/Micro-XRCE-DDS-Agent.git
-        cd Micro-XRCE-DDS-Agent
-        git checkout v2.2.1 # recent versions require cmake 3.22, but px4-dev-ros2-galactic:2021-09-08 is on 3.16
-        sed -i 's/_fastdds_tag 2.8.x/_fastdds_tag 2.8.2/g' CMakeLists.txt
-        mkdir build
-        cd build
-        cmake ..
-        make -j2
-    - name: ccache post-run micro-xrce-dds-agent
-      run: ccache -s
-
-    - name: Get and build the ros2 interface library
-      shell: bash
-      run: |
-        PX4_DIR="$(pwd)"
-        . /opt/ros/galactic/setup.bash
-        mkdir -p /opt/px4_ws/src
-        cd /opt/px4_ws/src
-        git clone --recursive https://github.com/Auterion/px4-ros2-interface-lib.git
-        # Ignore python packages due to compilation issue (can be enabled when updating ROS)
-        touch px4-ros2-interface-lib/px4_ros2_py/COLCON_IGNORE || true
-        touch px4-ros2-interface-lib/examples/python/COLCON_IGNORE || true
-        cd ..
-        # Copy messages to ROS workspace
-        "${PX4_DIR}/Tools/copy_to_ros_ws.sh" "$(pwd)"
-        rm -rf src/translation_node src/px4_msgs_old
-        colcon build --symlink-install
-    - name: ccache post-run ros workspace
-      run: ccache -s
-
-    - name: Build PX4
-      run: make px4_sitl_default
-    - name: ccache post-run px4/firmware
-      run: ccache -s
-    - name: Build SITL Gazebo
-      run: make px4_sitl_default sitl_gazebo-classic
-    - name: ccache post-run sitl_gazebo-classic
-      run: ccache -s
-
-    - name: Core dump settings
-      run: |
-        ulimit -c unlimited
-        echo "`pwd`/%e.core" > /proc/sys/kernel/core_pattern
-
-    - name: Run tests
-      shell: bash
-      run: |
-        . /opt/px4_ws/install/setup.bash
-        /opt/Micro-XRCE-DDS-Agent/build/MicroXRCEAgent udp4 localhost -p 8888 -v 0 &
-        test/ros_test_runner.py --verbose --model iris --upload --force-color
-      timeout-minutes: 45
-
-    - name: Upload failed logs
-      if: failure()
-      uses: actions/upload-artifact@v4
-      with:
-        name: failed-logs.zip
-        path: |
-          logs/**/**/**/*.log
-          logs/**/**/**/*.ulg
-          build/px4_sitl_default/tmp_ros_tests/rootfs/log/**/*.ulg
--- a/.github/workflows/ros_translation_node.yml
+++ b/.github/workflows/ros_translation_node.yml
@ -1,61 +0,0 @@
-name: ROS Translation Node Tests
-on:
-  push:
-    branches:
-    - 'main'
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-      - '**'
-    paths-ignore:
-      - 'docs/**'
-defaults:
-  run:
-    shell: bash
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build_and_test:
-    name: Build and test
-    runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - {ros_version: "humble", ubuntu: "jammy"}
-          - {ros_version: "jazzy", ubuntu: "noble"}
-    container:
-      image: rostooling/setup-ros-docker:ubuntu-${{ matrix.config.ubuntu }}-latest
-    steps:
-      - name: Setup ROS 2 (${{ matrix.config.ros_version }})
-        uses: ros-tooling/setup-ros@v0.7
-        with:
-          required-ros-distributions: ${{ matrix.config.ros_version }}
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      # Workaround for https://github.com/actions/runner/issues/2033
-      - name: ownership workaround
-        run: git config --system --add safe.directory '*'
-
-      - name: Check .msg file versioning
-        if: github.event_name == 'pull_request'
-        run: |
-          ./Tools/ci/check_msg_versioning.sh ${{ github.event.pull_request.base.sha }} ${{github.event.pull_request.head.sha}}
-
-      - name: Build and test
-        run: |
-          ros_ws=/ros_ws
-          mkdir -p $ros_ws/src
-          ./Tools/copy_to_ros_ws.sh $ros_ws
-          cd $ros_ws
-          source /opt/ros/${{ matrix.config.ros_version }}/setup.sh
-          colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release --symlink-install --event-handlers=console_cohesion+
-          source ./install/setup.sh
-          ./build/translation_node/translation_node_unit_tests
--- a/.github/workflows/sitl_tests.yml
+++ b/.github/workflows/sitl_tests.yml
@ -1,163 +0,0 @@
-# NOTE: this workflow is now running on Dronecode / PX4 AWS account.
-# - If you want to keep the tests running in GitHub Actions you need to uncomment the "runs-on: ubuntu-latest" lines
-#   and comment the "runs-on: [runs-on,runner=..." lines.
-# - If you would like to duplicate this setup try setting up "RunsOn" on your own AWS account try https://runs-on.com
-
-name: SITL Tests
-
-on:
-  push:
-    branches:
-    - 'main'
-    paths-ignore:
-      - 'docs/**'
-  pull_request:
-    branches:
-    - '**'
-    paths-ignore:
-      - 'docs/**'
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    name: Testing PX4 ${{ matrix.config.model }}
-    runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false]
-    container:
-      image: px4io/px4-dev-simulation-focal:2021-09-08
-      options: --privileged --ulimit core=-1 --security-opt seccomp=unconfined
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - {model: "iris",          latitude:  "59.617693", longitude: "-151.145316", altitude:  "48", build_type: "RelWithDebInfo" } # Alaska
-          # VTOL/tailsitter disabled: persistent flaky CI failures (timeouts, erratic
-          # transitions). Re-enable once the test infrastructure is stabilized.
-          # - {model: "tailsitter" ,   latitude:  "29.660316", longitude:  "-82.316658", altitude:  "30", build_type: "RelWithDebInfo" } # Florida
-          # - {model: "standard_vtol", latitude:  "47.397742", longitude:    "8.545594", altitude: "488", build_type: "Coverage" } # Zurich
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Git Ownership Workaround
-        run: git config --system --add safe.directory '*'
-
-      - id: set-timestamp
-        name: Set timestamp for cache
-        run: echo "::set-output name=timestamp::$(date +"%Y%m%d%H%M%S")"
-
-      - name: Cache Key Config
-        uses: actions/cache@v4
-        with:
-          path: ~/.ccache
-          key: sitl-ccache-${{ steps.set-timestamp.outputs.timestamp }}
-          restore-keys: sitl-ccache-${{ steps.set-timestamp.outputs.timestamp }}
-
-      - name: Cache Conf Config
-        run: |
-            mkdir -p ~/.ccache
-            echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
-            echo "compression = true" >> ~/.ccache/ccache.conf
-            echo "compression_level = 6" >> ~/.ccache/ccache.conf
-            echo "max_size = 120M" >> ~/.ccache/ccache.conf
-            echo "hash_dir = false" >> ~/.ccache/ccache.conf
-            ccache -s
-            ccache -z
-
-      - name: Build PX4
-        env:
-          PX4_CMAKE_BUILD_TYPE: ${{matrix.config.build_type}}
-        run: make px4_sitl_default
-
-      - name: Cache Post-Run [px4_sitl_default]
-        run: ccache -s
-
-      - name: Build SITL Gazebo
-        env:
-          PX4_CMAKE_BUILD_TYPE: ${{matrix.config.build_type}}
-        run: make px4_sitl_default sitl_gazebo-classic
-
-      - name: Cache Post-Run [sitl_gazebo-classic]
-        run: ccache -s
-
-      - name: Download MAVSDK
-        run: wget "https://github.com/mavlink/MAVSDK/releases/download/v$(cat test/mavsdk_tests/MAVSDK_VERSION)/libmavsdk-dev_$(cat test/mavsdk_tests/MAVSDK_VERSION)_ubuntu20.04_amd64.deb"
-
-      - name: Install MAVSDK
-        run: dpkg -i "libmavsdk-dev_$(cat test/mavsdk_tests/MAVSDK_VERSION)_ubuntu20.04_amd64.deb"
-
-      - name: Check PX4 Environment Variables
-        env:
-          PX4_HOME_LAT: ${{matrix.config.latitude}}
-          PX4_HOME_LON: ${{matrix.config.longitude}}
-          PX4_HOME_ALT: ${{matrix.config.altitude}}
-          PX4_CMAKE_BUILD_TYPE: ${{matrix.config.build_type}}
-        run: |
-            export
-            ulimit -a
-
-      - name: Build PX4 / MAVSDK tests
-        env:
-          PX4_CMAKE_BUILD_TYPE: ${{matrix.config.build_type}}
-          DONT_RUN: 1
-        run: make px4_sitl_default sitl_gazebo-classic mavsdk_tests
-
-      - name: Cache Post-Run [px4_sitl_default sitl_gazebo-classic mavsdk_tests]
-        run: ccache -s
-
-      - name: Core Dump Settings
-        run: |
-            ulimit -c unlimited
-            echo "`pwd`/%e.core" > /proc/sys/kernel/core_pattern
-
-      - name: Run SITL / MAVSDK Tests
-        env:
-          PX4_HOME_LAT: ${{matrix.config.latitude}}
-          PX4_HOME_LON: ${{matrix.config.longitude}}
-          PX4_HOME_ALT: ${{matrix.config.altitude}}
-          PX4_CMAKE_BUILD_TYPE: ${{matrix.config.build_type}}
-        run: test/mavsdk_tests/mavsdk_test_runner.py --speed-factor 10 --abort-early --model ${{matrix.config.model}} test/mavsdk_tests/configs/sitl.json --verbose --force-color
-        timeout-minutes: 45
-
-      - name: Upload failed logs
-        if: failure()
-        uses: actions/upload-artifact@v4
-        with:
-          name: failed-${{matrix.config.model}}-logs.zip
-          path: |
-            logs/**/**/**/*.log
-            logs/**/**/**/*.ulg
-            build/px4_sitl_default/tmp_mavsdk_tests/rootfs/log/**/*.ulg
-
-      - name: Look at Core files
-        if: failure() && ${{ hashFiles('px4.core') != '' }}
-        run: gdb build/px4_sitl_default/bin/px4 px4.core -ex "thread apply all bt" -ex "quit"
-
-      - name: Upload PX4 coredump
-        if: failure() && ${{ hashFiles('px4.core') != '' }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: coredump
-          path: px4.core
-
-      - name: Setup & Generate Coverage Report
-        if: contains(matrix.config.build_type, 'Coverage')
-        run: |
-            git config --global credential.helper "" # disable the keychain credential helper
-            git config --global --add credential.helper store # enable the local store credential helper
-            echo "https://x-access-token:${{ secrets.ACCESS_TOKEN }}@github.com" >> ~/.git-credentials # add credential
-            git config --global url."https://github.com/".insteadof git@github.com: # credentials add credential
-            mkdir -p coverage
-            lcov --directory build/px4_sitl_default --base-directory build/px4_sitl_default --gcov-tool gcov --capture -o coverage/lcov.info
-
-      - name: Upload Coverage Information to Codecov
-        if: contains(matrix.config.build_type, 'Coverage')
-        uses: codecov/codecov-action@v4
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          flags: mavsdk
-          file: coverage/lcov.info
--- a/Tools/ci/run-clang-tidy-pr.py
+++ b/Tools/ci/run-clang-tidy-pr.py
@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+"""
+Run clang-tidy incrementally on files changed in a PR.
+
+Usage: run-clang-tidy-pr.py <base-ref>
+  base-ref: e.g. origin/main
+
+Computes the set of translation units (TUs) affected by the PR diff,
+then invokes Tools/run-clang-tidy.py on that subset only.
+Exits 0 silently when no C++ files were changed.
+"""
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+
+EXTENSIONS_CPP = {'.cpp', '.c'}
+EXTENSIONS_HDR = {'.hpp', '.h'}
+# Manual exclusions from Makefile:508
+EXCLUDE_EXTRA = '|'.join([
+    'src/systemcmds/tests',
+    'src/examples',
+    'src/modules/gyro_fft/CMSIS_5',
+    'src/lib/drivers/smbus',
+    'src/drivers/gpio',
+    r'src/modules/commander/failsafe/emscripten',
+    r'failsafe_test\.dir',
+])
+
+
+def repo_root():
+    try:
+        return subprocess.check_output(
+            ['git', 'rev-parse', '--show-toplevel'], text=True).strip()
+    except subprocess.CalledProcessError:
+        print('error: not inside a git repository', file=sys.stderr)
+        sys.exit(1)
+
+
+def changed_files(base_ref, root):
+    try:
+        out = subprocess.check_output(
+            ['git', 'diff', '--name-only', f'{base_ref}...HEAD',
+             '--', '*.cpp', '*.hpp', '*.h', '*.c'],
+            text=True, cwd=root).strip()
+        return out.splitlines() if out else []
+    except subprocess.CalledProcessError:
+        print(f'error: could not diff against "{base_ref}" — '
+              'is the ref valid and fetched?', file=sys.stderr)
+        sys.exit(1)
+
+
+def submodule_paths(root):
+    # Returns [] if .gitmodules is absent or has no paths — both valid
+    try:
+        out = subprocess.check_output(
+            ['git', 'config', '--file', '.gitmodules',
+             '--get-regexp', 'path'],
+            text=True, cwd=root).strip()
+        return [line.split()[1] for line in out.splitlines()]
+    except subprocess.CalledProcessError:
+        return []
+
+
+def build_exclude(root):
+    submodules = '|'.join(submodule_paths(root))
+    return f'{submodules}|{EXCLUDE_EXTRA}' if submodules else EXCLUDE_EXTRA
+
+
+def load_db(build_dir):
+    db_path = os.path.join(build_dir, 'compile_commands.json')
+    if not os.path.isfile(db_path):
+        print(f'error: {db_path} not found', file=sys.stderr)
+        print('Run "make px4_sitl_default-clang" first to generate '
+              'the compilation database', file=sys.stderr)
+        sys.exit(1)
+    try:
+        with open(db_path) as f:
+            return json.load(f)
+    except json.JSONDecodeError as e:
+        print(f'error: compile_commands.json is malformed: {e}', file=sys.stderr)
+        sys.exit(1)
+
+
+def find_tus(changed, db, root):
+    db_files = {e['file'] for e in db}
+    result = set()
+    for f in changed:
+        abs_path = os.path.join(root, f)
+        ext = os.path.splitext(f)[1]
+        if ext in EXTENSIONS_CPP:
+            if abs_path in db_files:
+                result.add(abs_path)
+        elif ext in EXTENSIONS_HDR:
+            hdr = os.path.basename(f)
+            for e in db:
+                try:
+                    if hdr in open(e['file']).read():
+                        result.add(e['file'])
+                except OSError:
+                    pass  # file deleted in PR — skip
+    return sorted(result)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('base_ref',
+                        help='Git ref to diff against, e.g. origin/main')
+    args = parser.parse_args()
+
+    root = repo_root()
+    build_dir = os.path.join(root, 'build', 'px4_sitl_default-clang')
+
+    run_tidy = os.path.join(root, 'Tools', 'run-clang-tidy.py')
+    if not os.path.isfile(run_tidy):
+        print(f'error: {run_tidy} not found', file=sys.stderr)
+        sys.exit(1)
+
+    changed = changed_files(args.base_ref, root)
+    if not changed:
+        print('No C++ files changed — skipping clang-tidy')
+        sys.exit(0)
+
+    db = load_db(build_dir)
+    tus = find_tus(changed, db, root)
+
+    if not tus:
+        print('No matching TUs in compile_commands.json — skipping clang-tidy')
+        sys.exit(0)
+
+    print(f'Running clang-tidy on {len(tus)} translation unit(s)')
+
+    result = subprocess.run(
+        [sys.executable, run_tidy,
+         '-header-filter=.*\\.hpp',
+         '-j0',
+         f'-exclude={build_exclude(root)}',
+         '-p', build_dir] + tus
+    )
+    sys.exit(result.returncode)
+
+
+if __name__ == '__main__':
+    main()
--- a/docs/en/test_and_ci/continous_integration.md
+++ b/docs/en/test_and_ci/continous_integration.md
@ -2,6 +2,512 @@

 PX4 uses GitHub Actions for continuous integration, with different workflows handling code builds, testing, and documentation.

+## Code CI
+
+PX4 builds and testing are performed using GitHub Actions with a waterfall/staged pipeline architecture to optimize costs and provide fast developer feedback.
+
+### CI Architecture Overview
+
+PX4 uses a **4-tier waterfall pipeline** that ensures expensive AWS-hosted integration tests only run after basic checks pass. This architecture prevents costly runs when simple issues like formatting errors are present.
+
+#### Pipeline Structure
+
+```
+┌─────────────────────────────────────────────┐
+│ TIER 1: Gate Checks (2-5 min)              │
+│ • Format checks, shellcheck, Python linting │
+│ • GitHub-hosted runners (free/cheap)        │
+│ • fail-fast: stops on first failure         │
+└─────────────────┬───────────────────────────┘
+                  │ ALL PASS
+                  ▼
+┌─────────────────────────────────────────────┐
+│ TIER 2: Builds, Analysis & Platform Checks │
+│   (10-30 min)                               │
+│ • SITL build + cache seed (PX4 + Gazebo)    │
+│ • Unit tests, static analysis, EKF checks   │
+│ • Ubuntu/macOS builds, ITCM, flash analysis │
+│ • Failsafe web simulator (Emscripten)       │
+│ • Mixed runners (AWS 4cpu + GitHub)          │
+└─────────────────┬───────────────────────────┘
+                  │ ALL PASS
+                  ▼
+┌─────────────────────────────────────────────┐
+│ TIER 3: Integration Tests (30-45 min)      │
+│ • SITL tests (Gazebo + MAVSDK, 20x speed)   │
+│ • ROS2 integration, MAVROS tests             │
+│ • ROS translation node (humble + jazzy)      │
+│ • AWS Self-hosted 8cpu runners               │
+└─────────────────┬───────────────────────────┘
+                  │ ALL PASS
+                  ▼
+┌─────────────────────────────────────────────┐
+│ TIER 4: Full Build Matrix (30-60 min)      │
+│ • Build all board targets                   │
+│ • AWS 8cpu runners (most expensive)         │
+│ • Only on main/stable/beta/release/tags     │
+└─────────────────────────────────────────────┘
+```
+
+### Active Workflows
+
+#### Core CI
+
+**[ci-orchestrator.yml](https://github.com/PX4/PX4-Autopilot/blob/main/.github/workflows/ci-orchestrator.yml)** - Main CI Pipeline
+- Runs on all PRs and pushes
+- Implements Tiers 1-4 with cascading dependencies
+- Fails fast to save costs and provide quick feedback
+- Uses concurrency control to cancel outdated runs
+
+**[build_all_targets.yml](https://github.com/PX4/PX4-Autopilot/blob/main/.github/workflows/build_all_targets.yml)** - Full Board Build Matrix
+- Triggered by orchestrator completion on PRs
+- Runs independently on tagged releases
+- Builds all board configurations
+- Uploads artifacts to S3 and GitHub Releases
+
+#### Infrastructure Workflows
+
+- **dev_container.yml** - Docker container builds for development environment
+- **fuzzing.yml** - Daily fuzzing tests for security
+
+#### Maintenance Workflows
+
+- **ekf_update_change_indicator.yml** - Track EKF functional changes
+- **label.yml** - Auto-label PRs based on modified files
+- **stale.yml** - Mark and close stale issues/PRs
+- **sync_to_px4_msgs.yml** - Sync ROS message definitions to px4_msgs repo
+
+### CI Tier Details
+
+#### Tier 1: Gate Checks (2-5 minutes)
+
+**Purpose:** Catch common errors quickly before spinning up expensive resources.
+
+**Jobs:**
+- Format checks (`check_format`)
+- Newline checks (`check_newlines`)
+- Shellcheck for bash scripts
+- Python linting (mypy, flake8) for MAVSDK tests
+
+**Runner:** GitHub-hosted `ubuntu-latest`
+
+**Behavior:** `fail-fast: true` - stops all jobs immediately on first failure
+
+#### Tier 2: Builds, Analysis & Platform Checks (10-30 minutes)
+
+**Purpose:** Validate code compiles, passes unit tests, and builds on all platforms. This tier runs all build and analysis jobs in parallel to minimize wall-clock time.
+
+**Jobs:**
+- **Build SITL Cache Seed** - Builds `px4_sitl_default` and Gazebo Classic plugins, seeds the ccache for downstream SITL test jobs
+- **Basic Tests** - Unit tests, module configuration validation, module documentation checks, and EKF functional change detection (PRs only — checks `src/modules/ekf2/test/change_indication` for uncommitted diff after running EKF tests)
+- **Clang-tidy** static analysis (16cpu runner); incremental on PRs (changed files only), full scan on push to main/stable/beta/release
+- **Clang-tidy PR Annotations** - Posts inline line-level review comments with one-click fix suggestions via `platisd/clang-tidy-pr-comments`; same-repo PRs only, informational (does not block merge)
+- **Ubuntu builds** (22.04, 24.04) on AWS 4cpu runners
+- **macOS build** on GitHub macOS runners
+- **ITCM memory checks** for 4 NuttX targets (fmu-v5x, fmu-v6xrt, tropic variants)
+- **Flash analysis** using bloaty for 2 targets (fmu-v5x, fmu-v6x), with PR comment
+- **Failsafe web simulator** build (Emscripten, cached SDK)
+
+**Runners:** Mixed (AWS 4cpu-linux-x64, 16cpu for clang-tidy, 2cpu for clang-tidy annotations, GitHub macos-latest)
+
+**Behavior:** Only runs if Tier 1 passes completely; `fail-fast: false` to let all jobs attempt
+
+#### Tier 3: Integration Tests (30-45 minutes)
+
+**Purpose:** Run expensive simulation and integration tests.
+
+**Jobs:**
+- **SITL Tests** - Gazebo Classic simulation with MAVSDK test suite at 20x speed factor (iris, tailsitter, standard_vtol models)
+- **ROS2 Integration** - Build and test ROS2 interface library
+- **MAVROS Mission Tests** - Mission execution tests
+- **MAVROS Offboard Tests** - Offboard control tests
+- **ROS Translation Node** - Tests for humble and jazzy distributions
+
+**Runners:** AWS Self-hosted 8cpu-linux-x64
+
+**Behavior:** Only runs if all Tier 2 jobs pass; `fail-fast: false` to collect all test results
+
+**Timeout:** 45 minutes per job
+
+#### Tier 4: Full Build Matrix (30-60 minutes)
+
+**Purpose:** Build all board targets for firmware distribution.
+
+**Trigger:**
+- Automatically after orchestrator succeeds (for PRs)
+- Independently on main/stable/beta/release branches
+- Independently on version tags (v*)
+
+**Process:**
+1. Scan and group board targets by architecture
+2. Build targets in parallel on AWS 8cpu runners
+3. Package and upload artifacts
+
+**Artifacts:**
+- Uploaded to S3 bucket (px4-travis) for QGroundControl
+- Uploaded to GitHub Releases for version tags
+- Draft releases created for manual review before publishing
+
+**Runners:** AWS Self-hosted 8cpu-linux-x64 (most expensive)
+
+### Runner Types
+
+#### GitHub-Hosted Runners
+
+Used for Tier 1 gate checks and macOS builds.
+
+**Benefits:**
+- Fast startup (5-10 seconds)
+- Free for public repos (2,000 minutes/month)
+- Low cost when paying (~$0.008/min for Linux)
+- Reliable provisioning
+
+**Used For:**
+- Format/lint checks
+- Python linting
+- Shellcheck
+- macOS builds
+
+#### AWS Self-Hosted Runners (RunsOn)
+
+Used for Tiers 2-4 builds and tests.
+
+**Configuration:**
+- `1cpu-linux-x64` - Utility jobs (flash analysis comment publishing)
+- `2cpu-linux-x64` - Clang-tidy PR annotation posting
+- `4cpu-linux-x64` - SITL cache seed, basic tests, EKF checks, platform builds, flash analysis, ITCM checks, failsafe sim
+- `8cpu-linux-x64` - SITL integration tests, ROS integration tests, full build matrix
+- `16cpu-linux-x64` - Clang-tidy static analysis
+- `spot=false` - On-demand instances (can be changed to spot for 60-70% savings)
+
+**Used For:**
+- SITL simulation tests (8cpu for Gazebo physics at 20x speed)
+- ROS integration tests (8cpu for parallel compilation of xrce-dds and ROS2 libraries)
+- Full board compilation (8cpu)
+- Platform builds and analysis (4cpu)
+
+### Fork CI Configuration
+
+Forks can run CI without AWS self-hosted runners. There are two paths depending on how much of the pipeline you need.
+
+#### Option 1: Configure the Orchestrator
+
+Use the full orchestrator with settings tuned for your fork's infrastructure. This gives you access to all tiers, job toggles, and cache tuning.
+
+1. Copy the example config:
+
+   ```bash
+   cp .github/ci-config.yml.example .github/ci-config.yml
+   ```
+
+2. Edit `.github/ci-config.yml` to match your setup:
+   - **Runner labels** -- point all tiers to `ubuntu-latest` (or your own self-hosted labels)
+   - **Job toggles** -- disable hardware-specific jobs (ITCM, flash analysis, platform builds) that don't apply to your fork
+   - **Cache sizes** -- GitHub provides 10 GB per repo; the example config uses roughly 15-20% of that with most upstream-specific jobs disabled
+   - **`is_gha` flag** -- set to `true` when running on GitHub-hosted runners (wired for future infrastructure-aware behavior)
+
+3. Commit the file and push. The orchestrator reads it at runtime and applies your overrides.
+
+This configuration has been validated end-to-end on GitHub-hosted `ubuntu-latest` runners: [successful run](https://github.com/PX4/PX4-Autopilot/actions/runs/22746668606).
+
+#### Option 2: Use the Simple CI Workflow
+
+If you only need basic validation (SITL build, one hardware target, unit tests, format checks), use the single-job workflow instead of the full orchestrator.
+
+1. Rename the example file:
+
+   ```bash
+   mv .github/workflows/ci-simple.yml.example .github/workflows/ci-simple.yml
+   ```
+
+2. Optionally delete `ci-orchestrator.yml` to avoid running both workflows.
+
+3. Enable GitHub Actions in your fork settings if not already enabled.
+
+The simple workflow runs a single `quick-check` job on `ubuntu-latest` that builds PX4 SITL, FMU-v5, runs unit tests, and checks formatting. It typically completes in under 15 minutes and has no AWS or self-hosted runner dependencies.
+
+### Cost Optimization Features
+
+#### 1. Cascading Dependencies
+
+Each tier only executes if the previous tier passes completely. This prevents expensive AWS runners from spinning up when basic checks fail.
+
+**Example:**
+- Format error detected in 2 minutes (Tier 1)
+- Pipeline stops immediately
+- AWS runners never start
+- Cost: ~$0.01 vs ~$5 for full run
+
+#### 2. Fail-Fast Strategy
+
+Tier 1 uses `fail-fast: true` to stop all parallel jobs on the first failure, providing immediate feedback to developers.
+
+#### 3. Concurrency Control
+
+All workflows use `cancel-in-progress: true` to automatically stop outdated runs when developers push new commits.
+
+#### 4. Branch-Aware Execution
+
+The full build matrix (Tier 4) only runs on important branches:
+- main, stable, beta
+- release/* branches
+- Version tags (v*)
+
+Feature branch PRs skip the expensive full build after validation in Tiers 1-3.
+
+#### 5. Path-Based Filtering
+
+Most CI workflows ignore changes to `docs/**` paths to avoid unnecessary builds when only documentation is modified.
+
+### Developer Experience
+
+#### Quick Feedback Loop
+
+The waterfall architecture provides progressively detailed feedback:
+
+| Issue Type | Detection Time | Tier | Cost Impact |
+|------------|---------------|------|-------------|
+| Format error | 2 minutes | Tier 1 | ~$0.01 |
+| Unit test / build failure | 15 minutes | Tier 2 | ~$0.50 |
+| Integration test failure | 45 minutes | Tier 3 | ~$2.00 |
+| Board target failure | 90 minutes | Tier 4 | ~$5.00 |
+
+#### Before/After Comparison
+
+**Before (28 separate workflows):**
+- All workflows start simultaneously on every PR
+- Format error detected at 2 min, but expensive tests run for 45+ min anyway
+- Total wasted compute: ~43 minutes of AWS 4cpu + 8cpu runners
+- Cost per failed PR: ~$5-10
+
+**After (Orchestrator + Build All Targets):**
+- Format error detected at 2 min
+- Pipeline stops immediately after Tier 1
+- Total wasted compute: ~2 minutes of GitHub-hosted runners
+- Cost per failed PR: ~$0.01
+- **Savings: ~99% on early failures, ~40-60% overall**
+
+### Caching Strategy
+
+The CI orchestrator uses several caching mechanisms to avoid redundant work across jobs and runs.
+
+#### ccache (C++ compilation cache)
+
+ccache caches compiled object files so unchanged source files skip recompilation on subsequent runs.
+
+**Cache keys follow this fallback pattern:**
+
+```
+ccache-{scope}-{branch}-{sha}     # exact match (never hits, since sha is unique)
+ccache-{scope}-{branch}-          # same branch, most recent commit
+ccache-{scope}-{base_branch}-     # base branch (e.g. main), for PR first runs
+ccache-{scope}-                   # any branch, last resort
+```
+
+The exact-match key (`{sha}`) is used as the **save** key. Since GitHub Actions cache is immutable (write-once), each commit saves a new cache entry. The restore step falls through to the most recent cache from the same branch, or the base branch.
+
+**Cache scopes and sizes:**
+
+| Scope | Key prefix | Max size | Contents | Saved by |
+|-------|-----------|----------|----------|----------|
+| `ccache-sitl` | `ccache-sitl-` | 400M | PX4 SITL firmware + Gazebo Classic plugins | `build-sitl` (cache seed job) |
+| `ccache-clang-tidy` | `ccache-clang-tidy-` | 250M | Clang-tidy analysis objects | `clang-tidy` |
+| `ccache-ubuntu` | `ccache-ubuntu-{container}-` | 250M | Ubuntu build objects (per container version) | `ubuntu-builds` |
+| `ccache-macos` | `ccache-macos-` | 400M | macOS build objects | `macos-build` |
+| `ccache-ros-integration` | `ccache-ros-integration-` | 500M | PX4 + xrce-dds + Gazebo + ROS2 libraries | `ros-integration-tests` |
+| `ccache-ros-translation-{ros}` | `ccache-ros-translation-{ros_version}-` | 250M | ROS translation node build objects (per ROS distro) | `ros-translation-node` |
+| `ccache-flash-{target}-current` | `ccache-flash-{target}-current-` | 250M | Flash analysis objects for PR HEAD (per board target) | `flash-analysis` |
+| `ccache-flash-{target}-baseline` | `ccache-flash-{target}-baseline-` | 250M | Flash analysis objects for baseline commit (per board target) | `flash-analysis` |
+| `px4-ros2-ws` | `px4-ros2-ws-v1-galactic-{image}-{msg-hash}` | — | PX4 ROS 2 Interface Library workspace at `/opt/px4_ws` (keyed on msg hash) | `ros-integration-tests` |
+
+**Cache seed pattern:** The `build-sitl` job acts as a cache seed for all downstream SITL-related jobs. It builds both `px4_sitl_default` and `sitl_gazebo-classic`, then saves the combined ccache. Downstream jobs (`basic-tests`, `ekf-functional-check`, `sitl-tests`) restore this cache using `actions/cache/restore` (read-only) and get near-100% hit rates without needing to save their own caches.
+
+**ccache configuration (all jobs):**
+
+| Setting | Value | Purpose |
+|---------|-------|---------|
+| `base_dir` | `${GITHUB_WORKSPACE}` | Normalize paths for cache portability |
+| `compression` | `true` | Reduce cache storage size |
+| `compression_level` | `6` | Balance compression ratio vs speed |
+| `hash_dir` | `false` | Ignore directory paths in hash (portability) |
+| `compiler_check` | `content` | Hash compiler binary content, not path/mtime |
+
+#### Emscripten SDK cache
+
+The failsafe web simulator job caches the Emscripten SDK directory to avoid re-cloning and installing it on every run.
+
+| Key | Path | Contents |
+|-----|------|----------|
+| `emsdk-4.0.15` | `_emscripten_sdk` | Full emsdk installation (pinned to version 4.0.15) |
+
+This is a simple version-pinned key. Updating the emsdk version in the workflow automatically invalidates the cache.
+
+#### macOS Homebrew and pip caches
+
+The macOS build job caches Homebrew packages and pip installations to avoid re-downloading dependencies.
+
+#### Why separate cache scopes?
+
+Different jobs use different compilers, flags, and build targets. Sharing a single ccache across all jobs would cause constant eviction as incompatible objects compete for space. Separate scopes ensure each job's cache stays warm with relevant objects.
+
+#### CI Status
+
+Check which tier failed by looking at the job names in the GitHub Actions UI:
+
+- Tier 1 failure (T1 prefix) - Gate checks (format, lint) - Fix formatting/style issues
+- Tier 2 failure (T2 prefix) - Builds/analysis (tests, static analysis, platform builds) - Fix compilation or test failures
+- Tier 3 failure (T3 prefix) - Integration tests - Fix SITL/ROS test failures
+- All tiers passed - Ready for merge (after approvals)
+
+### Manual Workflow Triggers
+
+All workflows support manual execution via GitHub Actions UI or CLI:
+
+**Via GitHub UI:**
+1. Go to Actions tab
+2. Select the workflow
+3. Click "Run workflow"
+4. Choose branch and click "Run"
+
+**Via GitHub CLI:**
+```bash
+gh workflow run ci-orchestrator.yml
+gh workflow run build_all_targets.yml
+```
+
+This is useful for:
+- Re-running specific workflows for debugging
+- Testing workflow changes
+- Manually triggering builds on branches
+
+### Troubleshooting
+
+#### Workflow Not Triggering
+
+**Possible causes:**
+- PR only modifies `docs/**` paths (ignored by most workflows)
+- Orchestrator must complete successfully for `build_all_targets` to trigger (Tier 4)
+- Workflow was canceled by a newer commit (concurrency control)
+
+**Solution:**
+- Check the Actions tab for workflow status
+- Verify files modified are not in ignored paths
+- Wait for orchestrator to complete before expecting Tier 4
+
+#### Unexpected CI Failures
+
+**Debugging steps:**
+1. Check which tier failed by looking at job name prefixes (T1, T2, T3)
+2. Review the specific failed job logs in that tier
+3. Ensure your branch is rebased on latest main
+4. Look for infrastructure issues (runner availability, network problems)
+5. Try re-running failed jobs using "Re-run failed jobs" button
+
+#### Common Issues
+
+**Clang-tidy failures:**
+- On PRs, only changed files are analyzed. Run `python3 Tools/ci/run-clang-tidy-pr.py origin/main` locally to reproduce the incremental check
+- For a full scan (matches push-to-main behavior): `make clang-tidy`
+- Fix any warnings or use `// NOLINT` comments for false positives
+- Inline fix suggestions are posted as PR review comments by the `post-clang-tidy-comments` job (same-repo PRs only)
+
+**SITL test timeouts:**
+- Tests have a 45-minute timeout
+- Check for deadlocks or infinite loops in simulation code
+- Review MAVSDK test logs in failed artifacts
+
+**AWS runner unavailable:**
+- RunsOn provisions runners on-demand
+- Rarely, provisioning can fail due to AWS capacity
+- Re-run the workflow or wait a few minutes and try again
+
+**Checkout errors in workflow_run:**
+- The `build_all_targets` workflow (Tier 4) uses `workflow_run` trigger
+- It automatically checks out the correct commit SHA
+- If issues persist, check GitHub Actions permissions
+
+### CI Best Practices for Contributors
+
+#### Before Pushing
+
+1. **Run format checks locally:**
+   ```bash
+   make check_format
+   make shellcheck_all
+   ```
+
+2. **Run unit tests:**
+   ```bash
+   make tests
+   ```
+
+3. **Build your target configuration:**
+   ```bash
+   make px4_fmu-v5_default  # or your target
+   ```
+
+#### During PR Review
+
+- Monitor CI status in the GitHub UI
+- Address failures starting from the earliest tier
+- Don't push new commits while CI is running if possible (cancels previous run)
+- Use draft PRs to prevent expensive builds until ready
+
+#### When CI Fails
+
+1. Check which tier failed by looking at job name prefixes (T1, T2, T3)
+2. Read the failed job logs carefully
+3. Reproduce the issue locally if possible
+4. Fix the root cause, not just the symptom
+5. Consider if your changes affect other platforms/configurations
+
+### Future Optimization Opportunities
+
+#### 1. Enable Spot Instances
+
+Change `spot=false` to `spot=true` for PR testing to reduce costs by 60-70%. This requires accepting occasional (~5%) provisioning failures.
+
+#### 2. Path-Based Test Selection
+
+Only run ROS tests if ROS-related files (`msg/**`, `src/modules/uxrce_dds_client/**`) are modified.
+
+#### 3. Skip Tests on Draft PRs
+
+Add `if: github.event.pull_request.draft == false` to expensive jobs, allowing draft PRs to skip CI entirely.
+
+#### 4. Pre-built Container Images for ROS
+
+Bake xrce-dds, ROS2 libraries, and Gazebo into updated container images to eliminate build-from-source overhead in integration test jobs (currently ~5-8 minutes per job).
+
+### Migration History
+
+This waterfall architecture was introduced to replace 28 independent workflows that all ran simultaneously. The migration:
+
+- Reduced workflow files from 28 to 14
+- Eliminated 1,057 lines of redundant YAML
+- Decreased CI costs by an estimated 40-60%
+- Improved developer feedback time for common errors
+- Maintained full test coverage with smarter execution
+
+The previous workflows that were consolidated:
+- `checks.yml` -> Tiers 1 & 2
+- `python_checks.yml` -> Tier 1
+- `clang-tidy.yml` -> Tier 2
+- `compile_macos.yml` -> Tier 2
+- `compile_ubuntu.yml` -> Tier 2
+- `sitl_tests.yml` -> Tier 3
+- `ros_integration_tests.yml` -> Tier 3
+- `mavros_mission_tests.yml` -> Tier 3
+- `mavros_offboard_tests.yml` -> Tier 3
+- `ros_translation_node.yml` -> Tier 3
+- `itcm_check.yml` -> Tier 2
+- `flash_analysis.yml` -> Tier 2
+- `failsafe_sim.yml` -> Tier 2
+- `nuttx_env_config.yml` -> Tier 2
+- `ekf_functional_change_indicator.yml` -> Tier 2
+
+The original 5-tier design was subsequently optimized to 4 tiers by merging platform builds (old Tier 3) into Tier 2, since they had no data dependency on basic test results and could run in parallel. This reduced wall-clock time by ~20 minutes on successful runs.
+
 ## Documentation CI

 The documentation pipeline handles building, deploying, and translating the PX4 User Guide.
@ -18,12 +524,12 @@ Jobs are organized in tiers, where each tier depends on the previous one complet

 | Tier | Job            | PR                           | Push / Dispatch | Description                                                   |
 | ---- | -------------- | ---------------------------- | --------------- | ------------------------------------------------------------- |
-| T1   | Detect Changes | Yes                          | —               | Checks if source code files changed (triggers metadata regen) |
-| T2   | PR Metadata    | Yes (conditional)            | —               | Builds PX4 SITL and regenerates all auto-generated docs       |
-| T2   | Metadata Sync  | —                            | Yes             | Builds PX4 SITL, regenerates metadata, auto-commits           |
-| T2   | Link Check     | Yes                          | —               | Checks for broken links in changed files, posts PR comment    |
+| T1   | Detect Changes | Yes                          | -               | Checks if source code files changed (triggers metadata regen) |
+| T2   | PR Metadata    | Yes (conditional)            | -               | Builds PX4 SITL and regenerates all auto-generated docs       |
+| T2   | Metadata Sync  | -                            | Yes             | Builds PX4 SITL, regenerates metadata, auto-commits           |
+| T2   | Link Check     | Yes                          | -               | Checks for broken links in changed files, posts PR comment    |
 | T3   | Build Site     | Yes (if docs/source changed) | Yes (after T2)  | Builds the VitePress documentation site                       |
-| T4   | Deploy         | —                            | Yes             | Deploys to AWS S3                                             |
+| T4   | Deploy         | -                            | Yes             | Deploys to AWS S3                                             |

 #### Pull Request Flow

@ -31,15 +537,15 @@ When a PR modifies files in `docs/**` or the orchestrator workflow file itself,

 ```txt
 PR Event
-    │
-    ▼
+    |
+    v
 ┌─────────────────────────────────────┐
 │ T1: Detect Changes                  │
 │  • Checks if src/msg/ROMFS changed  │
 └─────────────────┬───────────────────┘
                  │
          ┌───────┴───────┐
-          ▼               ▼
+          v               v
 ┌──────────────────┐ ┌─────────────────────────┐
 │ T2: PR Metadata  │ │ T2: Link Check (~30s)   │
 │ (conditional)    │ │  • Detects changed .md   │
@ -50,16 +556,16 @@ PR Event
 │    failsafe web  │              │
 └────────┬─────────┘              │
         └───────────┬────────────┘
-                     ▼
+                     v
 ┌─────────────────────────────────────┐
 │ T3: Build Site (~7-10 min)         │
 │  (skipped if only workflow YAML     │
-│   changed — no docs/source changes) │
+│   changed - no docs/source changes) │
 │  • Builds VitePress site            │
 │  • Verifies no build errors         │
 └─────────────────┬───────────────────┘
                  │
-                  ▼
+                  v
                DONE
 ```

@ -73,12 +579,12 @@ PR Event
 #### Push / Dispatch Flow (main/release branches)

 When changes are pushed to `main` or `release/**` branches (or a `workflow_dispatch` is triggered), the workflow regenerates metadata, builds, and deploys.
-Only `main` and `release/*` branches are accepted for deploy — other branches will fail with a clear error.
+Only `main` and `release/*` branches are accepted for deploy -- other branches will fail with a clear error.

 ```txt
 Push / Dispatch Event
-    │
-    ▼
+    |
+    v
 ┌─────────────────────────────────────┐
 │ T2: Metadata Sync (~10-15 min)     │
 │  • Builds px4_sitl_default          │
@ -90,14 +596,14 @@ Push / Dispatch Event
 │    (with [skip ci])                 │
 └─────────────────┬───────────────────┘
                  │
-                  ▼
+                  v
 ┌─────────────────────────────────────┐
 │ T3: Build Site (~7-10 min)         │
 │  • Builds VitePress site            │
 │  • Uploads build artifact           │
 └─────────────────┬───────────────────┘
                  │
-                  ▼
+                  v
 ┌─────────────────────────────────────┐
 │ T4: Deploy (~3 min)                │
 │  • Syncs to AWS S3                  │
@ -192,3 +698,12 @@ Jobs run on [runs-on](https://runs-on.com/) self-hosted runners with S3 cache:
 | T2: Link Check     | ubuntu-latest                  |
 | T3: Build Site     | 4 CPU                          |
 | T4: Deploy         | ubuntu-latest                  |
+
+## Contact
+
+For CI-related questions or issues:
+- GitHub Issues: Tag the CI maintainer
+- Slack: #infrastructure channel
+- Dev Call: Bring up during weekly meeting
+
+All workflows can be found in [.github/workflows/](https://github.com/PX4/PX4-Autopilot/tree/main/.github/workflows).
--- a/integrationtests/python_src/px4_it/mavros/mavros_offboard_attctl_test.py
+++ b/integrationtests/python_src/px4_it/mavros/mavros_offboard_attctl_test.py
@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 #***************************************************************************
 #
 #   Copyright (c) 2015 PX4 Development Team. All rights reserved.
@ -35,9 +35,6 @@
 #
 # @author Andreas Antener <andreas@uaventure.com>
 #
-# The shebang of this file is currently Python2 because some
-# dependencies such as pymavlink don't play well with Python3 yet.
-from __future__ import division

 PKG = 'px4'

@ -46,7 +43,6 @@ from geometry_msgs.msg import Quaternion, Vector3
 from mavros_msgs.msg import AttitudeTarget
 from mavros_test_common import MavrosTestCommon
 from pymavlink import mavutil
-from six.moves import xrange
 from std_msgs.msg import Header
 from threading import Thread
 from tf.transformations import quaternion_from_euler
@ -124,7 +120,7 @@ class MavrosOffboardAttctlTest(MavrosTestCommon):
        loop_freq = 2  # Hz
        rate = rospy.Rate(loop_freq)
        crossed = False
-        for i in xrange(timeout * loop_freq):
+        for i in range(timeout * loop_freq):
            if (self.local_position.pose.position.x > boundary_x and
                    self.local_position.pose.position.y > boundary_y and
                    self.local_position.pose.position.z > boundary_z):
--- a/integrationtests/python_src/px4_it/mavros/mavros_offboard_posctl_test.py
+++ b/integrationtests/python_src/px4_it/mavros/mavros_offboard_posctl_test.py
@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 #***************************************************************************
 #
 #   Copyright (c) 2015 PX4 Development Team. All rights reserved.
@ -35,9 +35,6 @@
 #
 # @author Andreas Antener <andreas@uaventure.com>
 #
-# The shebang of this file is currently Python2 because some
-# dependencies such as pymavlink don't play well with Python3 yet.
-from __future__ import division

 PKG = 'px4'

@ -48,7 +45,6 @@ from geometry_msgs.msg import PoseStamped, Quaternion
 from mavros_msgs.msg import ParamValue
 from mavros_test_common import MavrosTestCommon
 from pymavlink import mavutil
-from six.moves import xrange
 from std_msgs.msg import Header
 from threading import Thread
 from tf.transformations import quaternion_from_euler
@ -132,7 +128,7 @@ class MavrosOffboardPosctlTest(MavrosTestCommon):
        loop_freq = 2  # Hz
        rate = rospy.Rate(loop_freq)
        reached = False
-        for i in xrange(timeout * loop_freq):
+        for i in range(timeout * loop_freq):
            if self.is_at_position(self.pos.pose.position.x,
                                   self.pos.pose.position.y,
                                   self.pos.pose.position.z, self.radius):
@ -174,7 +170,7 @@ class MavrosOffboardPosctlTest(MavrosTestCommon):
        positions = ((0, 0, 0), (50, 50, 20), (50, -50, 20), (-50, -50, 20),
                     (0, 0, 20))

-        for i in xrange(len(positions)):
+        for i in range(len(positions)):
            self.reach_position(positions[i][0], positions[i][1],
                                positions[i][2], 30)

--- a/integrationtests/python_src/px4_it/mavros/mavros_offboard_yawrate_test.py
+++ b/integrationtests/python_src/px4_it/mavros/mavros_offboard_yawrate_test.py
@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 #***************************************************************************
 #
 #   Copyright (c) 2020 PX4 Development Team. All rights reserved.
@ -35,8 +35,6 @@
 # @author Pedro Roque <padr@kth.se>
 #

-from __future__ import division
-
 PKG = 'px4'

 import rospy
@ -44,7 +42,6 @@ from geometry_msgs.msg import Quaternion, Vector3
 from mavros_msgs.msg import AttitudeTarget
 from mavros_test_common import MavrosTestCommon
 from pymavlink import mavutil
-from six.moves import xrange
 from std_msgs.msg import Header
 from threading import Thread
 from tf.transformations import quaternion_from_euler
@ -134,7 +131,7 @@ class MavrosOffboardYawrateTest(MavrosTestCommon):
        loop_freq = 2  # Hz
        rate = rospy.Rate(loop_freq)
        crossed = False
-        for i in xrange(timeout * loop_freq):
+        for i in range(timeout * loop_freq):
            if (self.local_position.pose.position.x < boundary_x and
 	    	    self.local_position.pose.position.x > -boundary_x and
                    self.local_position.pose.position.y < boundary_y and
--- a/integrationtests/python_src/px4_it/mavros/mavros_test_common.py
+++ b/integrationtests/python_src/px4_it/mavros/mavros_test_common.py
@ -1,5 +1,4 @@
-#!/usr/bin/env python2
-from __future__ import division
+#!/usr/bin/env python3

 import unittest
 import rospy
@ -11,7 +10,6 @@ from mavros_msgs.srv import CommandBool, ParamGet, ParamSet, SetMode, SetModeReq
                            WaypointPush
 from pymavlink import mavutil
 from sensor_msgs.msg import NavSatFix, Imu
-from six.moves import xrange


 class MavrosTestCommon(unittest.TestCase):
@ -183,7 +181,7 @@ class MavrosTestCommon(unittest.TestCase):
        loop_freq = 1  # Hz
        rate = rospy.Rate(loop_freq)
        arm_set = False
-        for i in xrange(timeout * loop_freq):
+        for i in range(timeout * loop_freq):
            if self.state.armed == arm:
                arm_set = True
                rospy.loginfo("set arm success | seconds: {0} of {1}".format(
@ -213,7 +211,7 @@ class MavrosTestCommon(unittest.TestCase):
        loop_freq = 1  # Hz
        rate = rospy.Rate(loop_freq)
        mode_set = False
-        for i in xrange(timeout * loop_freq):
+        for i in range(timeout * loop_freq):
            if self.state.mode == mode:
                mode_set = True
                rospy.loginfo("set mode success | seconds: {0} of {1}".format(
@ -247,7 +245,7 @@ class MavrosTestCommon(unittest.TestCase):
        loop_freq = 1  # Hz
        rate = rospy.Rate(loop_freq)
        param_set = False
-        for i in xrange(timeout * loop_freq):
+        for i in range(timeout * loop_freq):
            try:
                res = self.set_param_srv(param_id, param_value)
                if res.success:
@ -274,7 +272,7 @@ class MavrosTestCommon(unittest.TestCase):
        loop_freq = 1  # Hz
        rate = rospy.Rate(loop_freq)
        simulation_ready = False
-        for i in xrange(timeout * loop_freq):
+        for i in range(timeout * loop_freq):
            if all(value for value in self.sub_topics_ready.values()):
                simulation_ready = True
                rospy.loginfo("simulation topics ready | seconds: {0} of {1}".
@ -297,7 +295,7 @@ class MavrosTestCommon(unittest.TestCase):
        loop_freq = 10  # Hz
        rate = rospy.Rate(loop_freq)
        landed_state_confirmed = False
-        for i in xrange(timeout * loop_freq):
+        for i in range(timeout * loop_freq):
            if self.extended_state.landed_state == desired_landed_state:
                landed_state_confirmed = True
                rospy.loginfo("landed state confirmed | seconds: {0} of {1}".
@ -325,7 +323,7 @@ class MavrosTestCommon(unittest.TestCase):
        loop_freq = 10  # Hz
        rate = rospy.Rate(loop_freq)
        transitioned = False
-        for i in xrange(timeout * loop_freq):
+        for i in range(timeout * loop_freq):
            if transition == self.extended_state.vtol_state:
                rospy.loginfo("transitioned | seconds: {0} of {1}".format(
                    i / loop_freq, timeout))
@ -348,7 +346,7 @@ class MavrosTestCommon(unittest.TestCase):
        loop_freq = 1  # Hz
        rate = rospy.Rate(loop_freq)
        wps_cleared = False
-        for i in xrange(timeout * loop_freq):
+        for i in range(timeout * loop_freq):
            if not self.mission_wp.waypoints:
                wps_cleared = True
                rospy.loginfo("clear waypoints success | seconds: {0} of {1}".
@ -381,7 +379,7 @@ class MavrosTestCommon(unittest.TestCase):
        rate = rospy.Rate(loop_freq)
        wps_sent = False
        wps_verified = False
-        for i in xrange(timeout * loop_freq):
+        for i in range(timeout * loop_freq):
            if not wps_sent:
                try:
                    res = self.wp_push_srv(start_index=0, waypoints=waypoints)
@ -417,7 +415,7 @@ class MavrosTestCommon(unittest.TestCase):
        loop_freq = 1  # Hz
        rate = rospy.Rate(loop_freq)
        res = False
-        for i in xrange(timeout * loop_freq):
+        for i in range(timeout * loop_freq):
            try:
                res = self.get_param_srv('MAV_TYPE')
                if res.success:
--- a/integrationtests/python_src/px4_it/mavros/mission_test.py
+++ b/integrationtests/python_src/px4_it/mavros/mission_test.py
@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 #***************************************************************************
 #
 #   Copyright (c) 2015-2016 PX4 Development Team. All rights reserved.
@ -36,10 +36,6 @@
 # @author Andreas Antener <andreas@uaventure.com>
 #

-# The shebang of this file is currently Python2 because some
-# dependencies such as pymavlink don't play well with Python3 yet.
-from __future__ import division
-
 PKG = 'px4'

 import rospy
@ -47,13 +43,13 @@ import glob
 import json
 import math
 import os
-from px4tools import ulog
+import numpy as np
+from pyulog import ULog
 import sys
 from mavros import mavlink
 from mavros_msgs.msg import Mavlink, Waypoint, WaypointReached
 from mavros_test_common import MavrosTestCommon
 from pymavlink import mavutil
-from six.moves import xrange
 from threading import Thread


@ -70,6 +66,49 @@ def get_last_log():
    return last_log


+def analyze_estimator_attitude(log_file):
+    """Compute attitude estimator error metrics from a ULog file."""
+    ulog = ULog(log_file)
+
+    att = ulog.get_dataset('vehicle_attitude').data
+    att_gt = ulog.get_dataset('vehicle_attitude_groundtruth').data
+
+    def quat_to_euler(q0, q1, q2, q3):
+        """Quaternion (w,x,y,z) to (roll, pitch, yaw) via ZYX Tait-Bryan."""
+        roll = np.arctan2(2 * (q0 * q1 + q2 * q3), 1 - 2 * (q1**2 + q2**2))
+        sinp = 2 * (q0 * q2 - q3 * q1)
+        pitch = np.where(np.abs(sinp) >= 1,
+                         np.copysign(np.pi / 2, sinp), np.arcsin(sinp))
+        yaw = np.arctan2(2 * (q0 * q3 + q1 * q2), 1 - 2 * (q2**2 + q3**2))
+        return roll, pitch, yaw
+
+    roll, pitch, yaw = quat_to_euler(
+        att['q[0]'], att['q[1]'], att['q[2]'], att['q[3]'])
+    roll_gt, pitch_gt, yaw_gt = quat_to_euler(
+        att_gt['q[0]'], att_gt['q[1]'], att_gt['q[2]'], att_gt['q[3]'])
+
+    # interpolate groundtruth onto attitude timestamps
+    ts = att['timestamp']
+    ts_gt = att_gt['timestamp']
+    roll_gt = np.interp(ts, ts_gt, roll_gt)
+    pitch_gt = np.interp(ts, ts_gt, pitch_gt)
+    yaw_gt = np.interp(ts, ts_gt, yaw_gt)
+
+    wrap = lambda x: np.arcsin(np.sin(x))
+    e_roll = wrap(roll - roll_gt)
+    e_pitch = wrap(pitch - pitch_gt)
+    e_yaw = wrap(yaw - yaw_gt)
+
+    return {
+        'roll_error_mean': np.rad2deg(np.mean(e_roll)),
+        'pitch_error_mean': np.rad2deg(np.mean(e_pitch)),
+        'yaw_error_mean': np.rad2deg(np.mean(e_yaw)),
+        'roll_error_std': np.rad2deg(np.std(e_roll)),
+        'pitch_error_std': np.rad2deg(np.std(e_pitch)),
+        'yaw_error_std': np.rad2deg(np.std(e_yaw)),
+    }
+
+
 def read_mission(mission_filename):
    wps = []
    with open(mission_filename, 'r') as f:
@ -188,7 +227,7 @@ class MavrosMissionTest(MavrosTestCommon):
        # does it reach the position in 'timeout' seconds?
        loop_freq = 2  # Hz
        rate = rospy.Rate(loop_freq)
-        for i in xrange(timeout * loop_freq):
+        for i in range(timeout * loop_freq):
            pos_xy_d, pos_z_d = self.distance_to_wp(lat, lon, alt)

            # remember best distances
@ -295,9 +334,7 @@ class MavrosMissionTest(MavrosTestCommon):
        rospy.loginfo("mission done, calculating performance metrics")
        last_log = get_last_log()
        rospy.loginfo("log file {0}".format(last_log))
-        data = ulog.read_ulog(last_log).concat(dt=0.1)
-        data = ulog.compute_data(data)
-        res = ulog.estimator_analysis(data, False)
+        res = analyze_estimator_attitude(last_log)

        # enforce performance
        self.assertTrue(abs(res['roll_error_mean']) < 5.0, str(res))