ci: no docker instead install python deps

in favor of new build all

.editorconfig

@@ -9,6 +9,6 @@ tab_width = 8
 # Not in the official standard, but supported by many editors
 max_line_length = 120
 
-[*.yaml]
+[*.yaml, *.yml]
 indent_style = space
 indent_size = 2

.github/workflows/build_all_targets.yml

@@ -0,0 +1,89 @@
+name: Build all targets
+
+on:
+  push:
+    branches:
+    - 'mrpollo/ci_runner_groups'
+
+jobs:
+  group_targets:
+    runs-on: [runs-on,runner=2cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}"]
+    # container:
+    #   image: px4io/px4-dev-base-focal:2021-09-08
+      # options: --user 1001
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      timestamp: ${{ steps.set-timestamp.outputs.timestamp }}
+    steps:
+    - uses: actions/checkout@v4
+      #   token: ${{ secrets.ACCESS_TOKEN }}
+    - name: Install Python dependencies
+      uses: py-actions/py-dependency-install@v4
+      with:
+        path: "./Tools/setup/requirements.txt"
+    - id: set-matrix
+      run: echo "::set-output name=matrix::$(./Tools/generate_board_targets_json.py --group)"
+    # - name: debug groups
+    #   run: |
+    #       echo "::group::DebugTargets"
+    #       ./Tools/generate_board_targets_json.py --group --verbose
+    #       echo "::endgroup::"
+    - id: set-timestamp
+      run: echo "::set-output name=timestamp::$(date +"%Y%m%d%H%M%S")"
+
+  setup:
+    name: ${{ matrix.group }}
+    runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}"]
+    needs: group_targets
+    strategy:
+      # matrix: ${{ fromJson(needs.group_targets.outputs.matrix) }}
+      matrix:
+        include:
+          - group: 'nuttx-px4'
+            targets: 'px4_fmu-v5x_default,px4_fmu-v6x_default'
+            container: 'px4io/px4-dev-nuttx-focal:2022-08-12'
+            arch: 'nuttx'
+    container:
+      image: ${{ matrix.container }}
+      # options: --user 1001
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+        #   token: ${{ secrets.ACCESS_TOKEN }}
+      - name: debug permissions
+        run: |
+          echo "::group::DebugPermissions"
+          echo $UID
+          echo $PATH
+          echo "::endgroup::"
+      - name: ccache setup keys
+        uses: actions/cache@v4
+        with:
+          path: ~/.ccache
+          key: ${{ matrix.arch }}-ccache-${{ needs.group_targets.outputs.timestamp }}
+          restore-keys: ${{ matrix.arch }}-ccache-${{ needs.group_targets.outputs.timestamp }}
+
+      - name: setup ccache
+        run: |
+            mkdir -p ~/.ccache
+            echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
+            echo "compression = true" >> ~/.ccache/ccache.conf
+            echo "compression_level = 6" >> ~/.ccache/ccache.conf
+            echo "max_size = 120M" >> ~/.ccache/ccache.conf
+            echo "hash_dir = false" >> ~/.ccache/ccache.conf
+            ccache -s
+            ccache -z
+
+      - name: Debug with tmate session
+        uses: mxschmitt/action-tmate@v3
+        with:
+          detached: true
+          # sudo: false
+
+      - name: build target group
+        run: |
+            ./Tools/ci_build_all_runner.sh ${{matrix.targets}}
+
+      - name: ccache post-run
+        run: ccache -s

.github/workflows/compile_linux.yml

@@ -1,58 +0,0 @@
-name: Compile Linux Targets
-
-on:
-  push:
-    branches:
-    - 'main'
-    - 'stable'
-    - 'beta'
-    - 'release/*'
-  pull_request:
-    branches:
-    - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-armhf:2023-06-26
-    strategy:
-      matrix:
-        config: [
-          beaglebone_blue_default,
-          emlid_navio2_default,
-          px4_raspberrypi_default,
-          scumaker_pilotpi_default,
-          ]
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{secrets.ACCESS_TOKEN}}
-    - name: ownership workaround
-      run: git config --system --add safe.directory '*'
-    - name: Prepare ccache timestamp
-      id: ccache_cache_timestamp
-      shell: cmake -P {0}
-      run: |
-        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
-        message("::set-output name=timestamp::${current_date}")
-    - name: ccache cache files
-      uses: actions/cache@v2
-      with:
-        path: ~/.ccache
-        key: ${{matrix.config}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
-        restore-keys: ${{matrix.config}}-ccache-
-    - name: setup ccache
-      run: |
-          mkdir -p ~/.ccache
-          echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
-          echo "compression = true" >> ~/.ccache/ccache.conf
-          echo "compression_level = 6" >> ~/.ccache/ccache.conf
-          echo "max_size = 100M" >> ~/.ccache/ccache.conf
-          echo "hash_dir = false" >> ~/.ccache/ccache.conf
-          ccache -s
-          ccache -z
-
-    - name: make ${{matrix.config}}
-      run: make ${{matrix.config}}
-    - name: ccache post-run
-      run: ccache -s

.github/workflows/compile_linux_arm64.yml

@@ -1,54 +0,0 @@
-name: Compile Linux ARM64 Targets
-
-on:
-  push:
-    branches:
-    - 'main'
-    - 'stable'
-    - 'beta'
-    - 'release/*'
-  pull_request:
-    branches:
-    - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-aarch64:2022-08-12
-    strategy:
-      matrix:
-        config: [
-          scumaker_pilotpi_arm64,
-          ]
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{secrets.ACCESS_TOKEN}}
-
-    - name: Prepare ccache timestamp
-      id: ccache_cache_timestamp
-      shell: cmake -P {0}
-      run: |
-        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
-        message("::set-output name=timestamp::${current_date}")
-    - name: ccache cache files
-      uses: actions/cache@v2
-      with:
-        path: ~/.ccache
-        key: ${{matrix.config}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
-        restore-keys: ${{matrix.config}}-ccache-
-    - name: setup ccache
-      run: |
-          mkdir -p ~/.ccache
-          echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
-          echo "compression = true" >> ~/.ccache/ccache.conf
-          echo "compression_level = 6" >> ~/.ccache/ccache.conf
-          echo "max_size = 100M" >> ~/.ccache/ccache.conf
-          echo "hash_dir = false" >> ~/.ccache/ccache.conf
-          ccache -s
-          ccache -z
-
-    - name: make ${{matrix.config}}
-      run: make ${{matrix.config}}
-    - name: ccache post-run
-      run: ccache -s

.github/workflows/compile_nuttx.yml

@@ -1,136 +0,0 @@
-name: Compile Nuttx Targets
-
-on:
-  push:
-    branches:
-    - 'main'
-    - 'stable'
-    - 'beta'
-    - 'release/*'
-  pull_request:
-    branches:
-    - '*'
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-nuttx-focal:2022-08-12
-    strategy:
-      fail-fast: false
-      matrix:
-        config: [
-          airmind_mindpx-v2,
-          ark_can-flow,
-          ark_can-gps,
-          ark_can-rtk-gps,
-          ark_cannode,
-          ark_fmu-v6x,
-          ark_pi6x,
-          ark_septentrio-gps,
-          atl_mantis-edu,
-          av_x-v1,
-          bitcraze_crazyflie,
-          bitcraze_crazyflie21,
-          cuav_can-gps-v1,
-          cuav_nora,
-          cuav_x7pro,
-          cubepilot_cubeorange,
-          cubepilot_cubeorangeplus,
-          cubepilot_cubeyellow,
-          diatone_mamba-f405-mk2,
-          freefly_can-rtk-gps,
-          holybro_can-gps-v1,
-          holybro_durandal-v1,
-          holybro_kakutef7,
-          holybro_kakuteh7,
-          holybro_pix32v5,
-          matek_gnss-m9n-f4,
-          matek_h743,
-          matek_h743-mini,
-          matek_h743-slim,
-          modalai_fc-v1,
-          modalai_fc-v2,
-          mro_ctrl-zero-classic,
-          mro_ctrl-zero-f7,
-          mro_ctrl-zero-f7-oem,
-          mro_ctrl-zero-h7,
-          mro_ctrl-zero-h7-oem,
-          mro_pixracerpro,
-          mro_x21,
-          mro_x21-777,
-          nxp_fmuk66-e,
-          nxp_fmuk66-v3,
-          nxp_mr-canhubk3,
-          nxp_ucans32k146,
-          omnibus_f4sd,
-          px4_fmu-v2,
-          px4_fmu-v3,
-          px4_fmu-v4,
-          px4_fmu-v4pro,
-          px4_fmu-v5,
-          px4_fmu-v5x,
-          px4_fmu-v6c,
-          px4_fmu-v6u,
-          px4_fmu-v6x,
-          px4_fmu-v6xrt,
-          raspberrypi_pico,
-          sky-drones_smartap-airlink,
-          spracing_h7extreme,
-          uvify_core,
-          siyi_n7
-          ]
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{secrets.ACCESS_TOKEN}}
-
-    - name: Prepare ccache timestamp
-      id: ccache_cache_timestamp
-      shell: cmake -P {0}
-      run: |
-        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
-        message("::set-output name=timestamp::${current_date}")
-    - name: ccache cache files
-      uses: actions/cache@v2
-      with:
-        path: ~/.ccache
-        key: ${{matrix.config}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}}
-        restore-keys: ${{matrix.config}}-ccache-
-    - name: setup ccache
-      run: |
-          mkdir -p ~/.ccache
-          echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
-          echo "compression = true" >> ~/.ccache/ccache.conf
-          echo "compression_level = 6" >> ~/.ccache/ccache.conf
-          echo "max_size = 120M" >> ~/.ccache/ccache.conf
-          echo "hash_dir = false" >> ~/.ccache/ccache.conf
-          ccache -s
-          ccache -z
-
-    - name: make all_variants_${{matrix.config}}
-      run: make all_variants_${{matrix.config}}
-      timeout-minutes: 45
-    - name: make ${{matrix.config}} bloaty_compileunits
-      run: make ${{matrix.config}} bloaty_compileunits || true
-    - name: make ${{matrix.config}} bloaty_inlines
-      run: make ${{matrix.config}} bloaty_inlines || true
-    - name: make ${{matrix.config}} bloaty_segments
-      run: make ${{matrix.config}} bloaty_segments || true
-    - name: make ${{matrix.config}} bloaty_symbols
-      run: make ${{matrix.config}} bloaty_symbols || true
-    - name: make ${{matrix.config}} bloaty_templates
-      run: make ${{matrix.config}} bloaty_templates || true
-    - name: make ${{matrix.config}} bloaty_ram
-      run: make ${{matrix.config}} bloaty_ram || true
-    - name: make ${{matrix.config}} bloaty_compare_master
-      run: make ${{matrix.config}} bloaty_compare_master || true
-    - name: ccache post-run
-      run: ccache -s
-
-    - name: Upload px4 package
-      uses: actions/upload-artifact@v2
-      with:
-        name: px4_package_${{matrix.config}}
-        path: |
-          build/**/*.px4
-          build/**/*.bin

Tools/ci_build_all_runner.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+# This script is meant to be used by the build_all.yml workflow in a github runner
+# Please only modify if you know what you are doing
+set -e
+
+targets=$1
+for target in ${targets//,/ }
+do
+    echo "::group::Building: [${target}]"
+    make $target
+    echo "::endgroup::"
+done

Tools/generate_board_targets_json.py

@@ -23,11 +23,14 @@ parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
                     help='Verbose Output')
 parser.add_argument('-p', '--pretty', dest='pretty', action='store_true',
                     help='Pretty output instead of a single line')
+parser.add_argument('-g', '--groups', dest='group', action='store_true',
+                    help='Groups targets')
 
 args = parser.parse_args()
 verbose = args.verbose
 
 build_configs = []
+grouped_targets = {}
 excluded_boards = ['modalai_voxl2', 'px4_ros2']  # TODO: fix and enable
 excluded_manufacturers = ['atlflight']
 excluded_platforms = ['qurt']
@@ -41,6 +44,7 @@ def process_target(px4board_file, target_name):
     ret = None
     platform = None
     toolchain = None
+    group = None
 
     if px4board_file.endswith("default.px4board") or \
         px4board_file.endswith("recovery.px4board") or \
@@ -63,19 +67,25 @@ def process_target(px4board_file, target_name):
         # get the container based on the platform and toolchain
         if platform == 'posix':
             container = 'px4io/px4-dev-base-focal:2021-09-08'
+            group = 'base'
             if toolchain:
                 if toolchain.startswith('aarch64'):
                     container = 'px4io/px4-dev-aarch64:2022-08-12'
+                    group = 'aarch64'
                 elif toolchain == 'arm-linux-gnueabihf':
                     container = 'px4io/px4-dev-armhf:2023-06-26'
+                    group = 'armhf'
                 else:
                     if verbose: print(f'unmatched toolchain: {toolchain}')
         elif platform == 'nuttx':
             container = 'px4io/px4-dev-nuttx-focal:2022-08-12'
+            group = 'nuttx'
         else:
             if verbose: print(f'unmatched platform: {platform}')
 
         ret = {'target': target_name, 'container': container}
+        if(args.group):
+            ret['arch'] = group
 
     return ret
 
@@ -105,12 +115,103 @@ for manufacturer in os.scandir(os.path.join(source_dir, 'boards')):
                     if verbose: print(f'excluding label {label} ({target_name})')
                     continue
                 target = process_target(files.path, target_name)
+
+                if (args.group and target is not None):
+                    if (target['arch'] not in grouped_targets):
+                        grouped_targets[target['arch']] = {}
+                        grouped_targets[target['arch']]['container'] = target['container']
+                        grouped_targets[target['arch']]['manufacturers'] = {}
+                    if(manufacturer.name not in grouped_targets[target['arch']]['manufacturers']):
+                        grouped_targets[target['arch']]['manufacturers'][manufacturer.name] = {}
+                        grouped_targets[target['arch']]['manufacturers'][manufacturer.name] = []
+                    grouped_targets[target['arch']]['manufacturers'][manufacturer.name].append(target_name)
                 if target is not None:
                     build_configs.append(target)
 
+def comma_targets(targets):
+    return ",".join(targets)
 
 github_action_config = { 'include': build_configs }
 extra_args = {}
 if args.pretty:
     extra_args['indent'] = 2
-print(json.dumps(github_action_config, **extra_args))
+
+if (args.group):
+    if(verbose):
+        import pprint
+        pprint.pp(grouped_targets)
+    final_groups = []
+    temp_group = []
+    group_number = {}
+    last_man = ''
+    last_arch = ''
+    for arch in grouped_targets:
+        if(last_arch == ''):
+            last_arch = arch
+        if(arch not in group_number):
+                group_number[arch] = 0
+
+        if verbose: print('arch:', arch, ' - last_arch:', last_arch)
+        if(last_arch != arch and len(temp_group) > 0):
+            group_name = last_arch + "-" + str(group_number[last_arch])
+            group_number[last_arch] += 1
+            targets = comma_targets(temp_group)
+            if verbose: print("Append-1: [", group_name, "]", targets)
+            final_groups.append({
+                "container": grouped_targets[last_arch]['container'],
+                "targets": targets,
+                "arch": last_arch,
+                "group": group_name
+            })
+            last_arch = arch
+            temp_group = []
+        for man in grouped_targets[arch]['manufacturers']:
+            for tar in grouped_targets[arch]['manufacturers'][man]:
+                if(last_man != man):
+                    if(len(grouped_targets[arch]['manufacturers'][man]) > 5):
+                        group_name = arch + "-" + man
+                        targets = comma_targets(grouped_targets[arch]['manufacturers'][man])
+                        last_man = man
+                        if verbose: print("Append-2: [", group_name , "]", " - ", targets)
+                        final_groups.append({
+                            "container": grouped_targets[arch]['container'],
+                            "targets": targets,
+                            "arch": arch,
+                            "group": group_name
+                        })
+                    else:
+                        temp_group.append(tar)
+                        if verbose: print('Temp(', len(temp_group), '):[', man, '][', arch, ':', last_arch, ']: ', tar)
+
+            if(last_arch != arch and len(temp_group) > 0):
+                group_name = last_arch + "-" + str(group_number[last_arch])
+                group_number[last_arch] += 1
+                targets = comma_targets(temp_group)
+                if verbose: print("Append-4: [", group_name, "]", targets)
+                final_groups.append({
+                    "container": grouped_targets[last_arch]['container'],
+                    "targets": targets,
+                    "arch": last_arch,
+                    "group": group_name
+                })
+                last_arch = arch
+                temp_group = []
+            if(len(temp_group) > 4):
+                group_name = arch + "-" + str(group_number[arch])
+                last_arch = arch
+                group_number[arch] += 1
+                targets = comma_targets(temp_group)
+                if verbose: print("Append-3: [", group_name, "]", " - ", targets)
+                final_groups.append({
+                    "container": grouped_targets[arch]['container'],
+                    "targets": targets,
+                    "arch": arch,
+                    "group": group_name
+                })
+                temp_group = []
+    if(verbose):
+        import pprint
+        pprint.pp(final_groups)
+    print(json.dumps({ "include": final_groups }, **extra_args))
+else:
+    print(json.dumps(github_action_config, **extra_args))