ci: AWS instance review and improved build caching

* Tone down the performance of some runners from 8cpu+ down to 4cpu+
* Improve and document caching on PX4 builds with an improved ccache key strategy
* Review and document artifact upload logic for binaries uploaded to S3 and github
releases
* Future Improvement, introduce runners configuration file so we can
control more precesily which instances are allocated.

Signed-off-by: Ramon Roche <mrpollo@gmail.com>

.github/runs-on.yml

@@ -0,0 +1,24 @@
+runners:
+  x86-small-runner:
+    cpu: [1, 2]
+    ram: [1, 4]
+    disk: default
+    spot: price-capacity-optimized
+    image: ubuntu24-full-x64
+    extras: s3-cache
+  x86-firmware-builder:
+    cpu: [4, 8]
+    ram: [8, 16]
+    disk: default
+    family: ["c7i", "m7i", "r7i"]
+    spot: price-capacity-optimized
+    image: ubuntu24-full-x64
+    extras: s3-cache
+  arm64-firmware-builder:
+    cpu: [4, 8]
+    ram: [8, 16]
+    disk: default
+    family: ["c7g", "m7g", "r7g"]
+    spot: price-capacity-optimized
+    image: ubuntu24-full-arm64
+    extras: s3-cache

.github/workflows/build_all_targets.yml

@@ -26,6 +26,10 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: write
+  actions: read
+
 jobs:
   group_targets:
     name: Scan for Board Targets
@@ -48,12 +52,15 @@ jobs:
           path: "./Tools/setup/requirements.txt"
 
       - id: set-matrix
-        run: echo "::set-output name=matrix::$(./Tools/ci/generate_board_targets_json.py --group)"
+        name: Generate Build Matrix
+        run: echo "matrix=$(./Tools/ci/generate_board_targets_json.py --group)" >> $GITHUB_OUTPUT
 
       - id: set-timestamp
-        run: echo "::set-output name=timestamp::$(date +"%Y%m%d%H%M%S")"
+        name: Save Current Timestamp
+        run: echo "timestamp=$(date +"%Y%m%d%H%M%S")" >> $GITHUB_OUTPUT
 
       - id: set-branch
+        name: Save Current Branch Name
         run: |
           echo "branchname=${{
             github.event_name == 'pull_request' &&
@@ -70,7 +77,7 @@ jobs:
           echo "$(./Tools/ci/generate_board_targets_json.py --group --verbose)"
 
   setup:
-    name: Build Group [${{ matrix.group }}][${{ matrix.arch == 'nuttx' && 'x86' || 'arm64' }}]
+    name: Build [${{ matrix.runner }}][${{ matrix.group }}]
     # runs-on: ubuntu-latest
     runs-on: [runs-on,"runner=8cpu-linux-${{ matrix.runner }}","image=ubuntu24-full-${{ matrix.runner }}","run-id=${{ github.run_id }}",spot=false]
     needs: group_targets
@@ -80,6 +87,7 @@ jobs:
     container:
       image: ${{ matrix.container }}
     steps:
+      - uses: runs-on/action@v2
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
@@ -87,14 +95,24 @@ jobs:
       - name: Git ownership workaround
         run: git config --system --add safe.directory '*'
 
-      - name: Setup ccache
-        uses: actions/cache@v4
+      # ccache key breakdown:
+      # ccache-<system os>-<system arch>-<builder group>-
+      # ccache-<linux>-<arm64>-<aarch64-0>-
+      # ccache-<linux>-<x64>-<nuttx-0>-
+      - name: Restore ccache from key
+        id: cc_restore
+        uses: actions/cache/restore@v4
         with:
           path: ~/.ccache
-          key: ${{ matrix.group }}-ccache-${{ needs.group_targets.outputs.timestamp }}
-          restore-keys: ${{ matrix.group }}-ccache-${{ needs.group_targets.outputs.timestamp }}
+          key: ${{ format('ccache-{0}-{1}-{2}', runner.os, matrix.runner, matrix.group) }}
+          restore-keys: |
+            ccache-${{ runner.os }}-${{ matrix.runner }}-${{ matrix.group }}-
+            ccache-${{ runner.os }}-${{ matrix.runner }}-
+            ccache-${{ runner.os }}-${{ matrix.runner }}-
+            ccache-${{ runner.os }}-
+            ccache-
 
-      - name: Configure ccache
+      - name: Set ccache defaults and show stats
         run: |
             mkdir -p ~/.ccache
             echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
@@ -102,10 +120,11 @@ jobs:
             echo "compression_level = 6" >> ~/.ccache/ccache.conf
             echo "max_size = 120M" >> ~/.ccache/ccache.conf
             echo "hash_dir = false" >> ~/.ccache/ccache.conf
+            echo "compiler_check = content" >> ~/.ccache/ccache.conf
             ccache -s
             ccache -z
 
-      - name: Building [${{ matrix.group }}]
+      - name: Building [${{ matrix.targets }}]
         run: |
             ./Tools/ci/build_all_runner.sh ${{matrix.targets}} ${{matrix.arch}}
 
@@ -119,15 +138,27 @@ jobs:
           name: px4_${{matrix.group}}_build_artifacts
           path: artifacts/
 
+      - name: Post Build Cache Stats
+        if: always()
+        run: |
+            ccache -s
+            ccache -z
+
       - name: Cache Save
-        run: ccache -s
+        if: always()
+        uses: actions/cache/save@v4
+        with:
+          path: ~/.ccache
+          key: ${{ steps.cc_restore.outputs.cache-primary-key }}
 
   artifacts:
-    name: Upload Artifacts to S3
+    name: Upload Artifacts
     # runs-on: ubuntu-latest
     runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
     needs: [setup, group_targets]
-    if: contains(fromJSON('["main", "stable", "beta"]'), needs.group_targets.outputs.branchname)
+    if: startsWith(github.ref, 'refs/tags/v') || contains(fromJSON('["main","stable","beta"]'), needs.group_targets.outputs.branchname)
+    outputs:
+      uploadlocation: ${{ steps.upload-location.outputs.uploadlocation }}
     steps:
       - name: Download Artifacts
         uses: actions/download-artifact@v4
@@ -135,11 +166,36 @@ jobs:
           path: artifacts/
           merge-multiple: true
 
-      - name: Branch Name
+      - name: Upload Location
+        id: upload-location
         run: |
-          echo "${{ needs.group_targets.outputs.branchname }}"
+          # Determine upload location based on branch or tag with the following considerations:
+          # Destination: AWS S3 bucket px4-travis in folder Firmware/
+          # - If branch is main -> upload to master/
+          #   - Older versions of QGC are hardocded to look for master/
+          # - If branch is stable or beta -> upload to stable/ or beta/
+          # - If a tag vX.Y.Z -> upload to vX.Y.Z/
+          #   - Also update stable/ to point to the same version
+          #.  - Older versions of QGC are hardocded to look for stable/
+          # - If a pull request -> do not upload
+          set -euo pipefail
 
-      - name: Uploading Artifacts to S3 [${{ needs.group_targets.outputs.branchname == 'main' && 'master' || needs.group_targets.outputs.branchname }}]
+          ref="${GITHUB_REF}"
+          branch=${{ needs.group_targets.outputs.branchname  }}
+          location="$branch"
+
+          if [[ "$branch" == "main" ]]; then
+            location="master"
+          fi
+
+          if [[ "$ref" == refs/tags/v[0-9]* ]]; then
+            tag="${ref#refs/tags/}"
+            location="$tag"
+          fi
+
+          echo "uploadlocation=$location" >> $GITHUB_OUTPUT
+
+      - name: Uploading Artifacts to S3 [${{ steps.upload-location.outputs.uploadlocation }}]
         uses: jakejarvis/s3-sync-action@master
         with:
           args: --acl public-read
@@ -149,25 +205,30 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_REGION: 'us-west-1'
           SOURCE_DIR: artifacts/
-          DEST_DIR: Firmware/${{ needs.group_targets.outputs.branchname == 'main' && 'master' || needs.group_targets.outputs.branchname }}/
+          DEST_DIR: Firmware/${{ steps.upload-location.outputs.uploadlocation }}/
 
-  release:
-    name: Create Release and Upload Artifacts
-    permissions:
-      contents: write
-    # runs-on: ubuntu-latest
-    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
-    needs: [setup, group_targets]
-    if: startsWith(github.ref, 'refs/tags/')
-    steps:
-      - name: Download Artifacts
-        uses: actions/download-artifact@v4
+      # if we are uploading artifacts to a versioned folder
+      # we should also update the stable folder in the s3 bucket
+      - name: Uploading Artifacts to S3 [stable]
+        uses: jakejarvis/s3-sync-action@master
+        if: startsWith(github.ref, 'refs/tags/v')
         with:
-          path: artifacts/
-          merge-multiple: true
+          args: --acl public-read
+        env:
+          AWS_S3_BUCKET: 'px4-travis'
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: 'us-west-1'
+          SOURCE_DIR: artifacts/
+          DEST_DIR: Firmware/stable/
 
+      # if build is a release triggered by a versioned tag then create a github release
+      # and upload the build artifacts. A draft release is created so that the release
+      # can be reviewed before publishing
       - name: Upload Binaries to Release
         uses: softprops/action-gh-release@v2
+        if: startsWith(github.ref, 'refs/tags/v')
         with:
           draft: true
           files: artifacts/*.px4
+          name: ${{ steps.upload-location.outputs.uploadlocation }}

.github/workflows/compile_ubuntu.yml

@@ -29,7 +29,7 @@ jobs:
       fail-fast: false
       matrix:
         version: ['ubuntu:22.04', 'ubuntu:24.04']
-    runs-on: [runs-on,runner=8cpu-linux-x64,"image=ubuntu24-full-x64","run-id=${{ github.run_id }}",spot=false]
+    runs-on: [runs-on,runner=4cpu-linux-x64,"image=ubuntu24-full-x64","run-id=${{ github.run_id }}",spot=false]
     container:
       image: ${{ matrix.version }}
       volumes:

.github/workflows/dev_container.yml

@@ -39,7 +39,7 @@ jobs:
     name: Set Tags and Variables
     permissions:
       contents: read
-    runs-on: [runs-on,"runner=1cpu-linux-x64","image=ubuntu24-full-x64","run-id=${{ github.run_id }}",spot=false,extras=s3-cache]
+    runs-on: [runs-on,"runner=1cpu-linux-x64","image=ubuntu24-full-x64","run-id=${{ github.run_id }}",extras=s3-cache,spot=false]
     outputs:
       px4_version: ${{ steps.px4_version.outputs.px4_version }}
       meta_tags: ${{ steps.meta.outputs.tags }}
@@ -87,7 +87,7 @@ jobs:
           - platform: linux/amd64
             arch: amd64
             runner: x64
-    runs-on: [runs-on,"runner=8cpu-linux-${{ matrix.runner }}","image=ubuntu24-full-${{ matrix.runner }}","run-id=${{ github.run_id }}",spot=false,extras=s3-cache]
+    runs-on: [runs-on,"runner=4cpu-linux-${{ matrix.runner }}","image=ubuntu24-full-${{ matrix.runner }}","run-id=${{ github.run_id }}",extras=s3-cache,spot=false]
     steps:
       - uses: runs-on/action@v1
       - uses: actions/checkout@v4
@@ -138,7 +138,7 @@ jobs:
     permissions:
       contents: read
       packages: write
-    runs-on: [runs-on,"runner=8cpu-linux-x64","image=ubuntu24-full-x64","run-id=${{ github.run_id }}",spot=false,extras=s3-cache]
+    runs-on: [runs-on,"runner=4cpu-linux-x64","image=ubuntu24-full-x64","run-id=${{ github.run_id }}",extras=s3-cache,spot=false]
     needs: [build, setup]
     if: ${{ startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && github.event.inputs.deploy_to_registry) }}
     steps:

.github/workflows/docs_deploy.yml

@@ -20,7 +20,7 @@ env:
 
 jobs:
   build:
-    runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false,extras=s3-cache]
+    runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",extras=s3-cache,spot=false]
     steps:
       - uses: runs-on/action@v1
       - name: Checkout
@@ -55,7 +55,7 @@ jobs:
   deploy:
     if: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.merged) || github.event_name == 'workflow_dispatch' }}
     needs: build
-    runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
 
     steps:
       - name: Download Artifact

.github/workflows/docs_deploy_aws.yml

@@ -30,7 +30,7 @@ concurrency:
 
 jobs:
   build:
-    runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false,extras=s3-cache]
+    runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false,extras=s3-cache]
     outputs:
       branchname: ${{ steps.set-branch.outputs.branchname }}
       releaseversion: ${{ steps.set-version.outputs.releaseversion }}

.github/workflows/flash_analysis.yml

@@ -24,7 +24,7 @@ env:
 jobs:
   analyze_flash:
     name: Analyzing ${{ matrix.target }}
-    runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
     container:
       image: px4io/px4-dev:v1.16.0-rc1-258-g0369abd556
     strategy:
@@ -97,7 +97,7 @@ jobs:
   # Track this issue https://github.com/PX4/PX4-Autopilot/issues/24408
   post_pr_comment:
     name: Publish Results
-    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}"]
     needs: [analyze_flash]
     env:
       V5X-SUMMARY-MAP-ABS: ${{ fromJSON(fromJSON(needs.analyze_flash.outputs.px4_fmu-v5x-bloaty-summary-map).vm-absolute) }}

.github/workflows/itcm_check.yml

@@ -22,7 +22,7 @@ concurrency:
 jobs:
   check_itcm:
     name: Checking ${{ matrix.target }}
-    runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
     container:
       image: px4io/px4-dev:v1.16.0-rc1-258-g0369abd556
     strategy:

.github/workflows/ros_integration_tests.yml

@@ -23,7 +23,7 @@ concurrency:
 
 jobs:
   build:
-    runs-on: [runs-on,runner=16cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false]
     container:
       image: px4io/px4-dev-ros2-galactic:2021-09-08
       options: --privileged --ulimit core=-1 --security-opt seccomp=unconfined

.github/workflows/ros_translation_node.yml

@@ -21,7 +21,7 @@ concurrency:
 jobs:
   build_and_test:
     name: Build and test
-    runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
     strategy:
       fail-fast: false
       matrix:

.github/workflows/sitl_tests.yml

@@ -24,7 +24,7 @@ concurrency:
 jobs:
   build:
     name: Testing PX4 ${{ matrix.config.model }}
-    runs-on: [runs-on,runner=16cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    runs-on: [runs-on,runner=4cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false]
     container:
       image: px4io/px4-dev-simulation-focal:2021-09-08
       options: --privileged --ulimit core=-1 --security-opt seccomp=unconfined