ci: Upload Artifacts to S3

- replaces metadata and deploy workflows
- move ci scripts under Tools/ci

.github/workflows/build_all_targets.yml

@@ -32,6 +32,7 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
       timestamp: ${{ steps.set-timestamp.outputs.timestamp }}
       tagname: ${{ steps.set-tag.outputs.tagname }}
+      branchname: ${{ steps.set-branch.outputs.branchname }}
     steps:
       - uses: actions/checkout@v4
 
@@ -41,7 +42,7 @@ jobs:
           path: "./Tools/setup/requirements.txt"
 
       - id: set-matrix
-        run: echo "::set-output name=matrix::$(./Tools/generate_board_targets_json.py --group)"
+        run: echo "::set-output name=matrix::$(./Tools/ci/generate_board_targets_json.py --group)"
 
       - id: set-timestamp
         run: echo "::set-output name=timestamp::$(date +"%Y%m%d%H%M%S")"
@@ -60,6 +61,13 @@ jobs:
         if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch'
         run: echo "::set-output name=tagname::${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}"
 
+      - id: set-branch
+        run: echo "::set-output name=branchname::${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}"
+
+      - name: Debug Matrix Output
+        if: runner.debug == '1'
+        run: echo "$(./Tools/ci/generate_board_targets_json.py --group --verbose)"
+
   setup:
     name: Build Group [${{ matrix.group }}]
     # runs-on: ubuntu-latest
@@ -74,17 +82,17 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: ownership workaround
+      - name: Git ownership workaround
         run: git config --system --add safe.directory '*'
 
-      - name: ccache setup keys
+      - name: Setup ccache
         uses: actions/cache@v4
         with:
           path: ~/.ccache
           key: ${{ matrix.group }}-ccache-${{ needs.group_targets.outputs.timestamp }}
           restore-keys: ${{ matrix.group }}-ccache-${{ needs.group_targets.outputs.timestamp }}
 
-      - name: setup ccache
+      - name: Configure ccache
         run: |
             mkdir -p ~/.ccache
             echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf
@@ -95,24 +103,54 @@ jobs:
             ccache -s
             ccache -z
 
-      - name: building [${{ matrix.group }}]
+      - name: Building [${{ matrix.group }}]
         run: |
-            ./Tools/ci_build_all_runner.sh ${{matrix.targets}}
+            ./Tools/ci/build_all_runner.sh ${{matrix.targets}}
 
-      - name: Upload px4 package
+      - name: Arrange Build Artifacts
+        run: |
+          ./Tools/ci/package_build_artifacts.sh
+
+      - name: Upload Build Artifacts
         uses: actions/upload-artifact@v4
         with:
           name: px4_${{matrix.group}}_build_artifacts
-          path: |
-            build/**/*.px4
-            build/**/*.bin
-          compression-level: 0
+          path: artifacts/
 
-      - name: ccache post-run
+      - name: Cache Save
         run: ccache -s
 
+  artifacts:
+    name: Upload Artifacts to S3
+    # runs-on: ubuntu-latest
+    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}"]
+    needs: [setup, group_targets]
+    if: contains(fromJSON('["main", "stable", "beta"]'), needs.group_targets.outputs.branchname)
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts/
+          merge-multiple: true
+
+      - name: Branch Name
+        run: |
+          echo "${{ needs.group_targets.outputs.branchname }}"
+
+      - name: Uploading Artifacts to S3 [${{ needs.group_targets.outputs.branchname == 'main' && 'master' || needs.group_targets.outputs.branchname }}]
+        uses: jakejarvis/s3-sync-action@master
+        with:
+          args: --acl public-read
+        env:
+          AWS_S3_BUCKET: 'px4-travis'
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: 'us-west-1'
+          SOURCE_DIR: artifacts/
+          DEST_DIR: Firmware/${{ needs.group_targets.outputs.branchname == 'main' && 'master' || needs.group_targets.outputs.branchname }}/
+
   release:
-    name: Release and Upload
+    name: Create Release and Upload Artifacts
     # runs-on: ubuntu-latest
     runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}"]
     needs: [setup, group_targets]
@@ -126,7 +164,7 @@ jobs:
           mkdir artifacts
           cp **/**/*.px4 artifacts/
 
-      - name: Upload Binaries
+      - name: Upload Binaries to Release
         uses: softprops/action-gh-release@v2
         with:
           name: ${{ needs.group_targets.outputs.tagname }}

.github/workflows/deploy_all.yml

@@ -1,56 +0,0 @@
-name: Deploy metadata for all targets
-
-on:
-  push:
-    branches:
-    - 'main'
-    - 'release/*'
-    - 'pr-metadata-test'
-
-jobs:
-  enumerate_targets:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-base-focal:2021-09-08
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{secrets.ACCESS_TOKEN}}
-    - id: set-matrix
-      run: echo "::set-output name=matrix::$(./Tools/generate_board_targets_json.py)"
-  build:
-    runs-on: ubuntu-latest
-    needs: enumerate_targets
-    strategy:
-      matrix: ${{fromJson(needs.enumerate_targets.outputs.matrix)}}
-    container: ${{ matrix.container }}
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{secrets.ACCESS_TOKEN}}
-
-    - name: ownership workaround
-      run: git config --system --add safe.directory '*'
-
-    - name: make ${{matrix.target}}
-      run: make ${{matrix.target}}
-
-    - name: parameter & events metadata
-      run: |
-        make ${{matrix.target}} ver_gen events_json actuators_json
-        ./src/lib/version/get_git_tag_or_branch_version.sh build/${{ matrix.target }} >> $GITHUB_ENV
-        cd build/${{ matrix.target }}
-        mkdir _metadata || true
-        cp parameters.* events/*.xz actuators.json* _metadata
-
-    - uses: jakejarvis/s3-sync-action@master
-      with:
-        args: --acl public-read
-      env:
-        AWS_S3_BUCKET: 'px4-travis'
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_REGION: 'us-west-1'
-        SOURCE_DIR: 'build/${{ matrix.target }}/_metadata/'
-        DEST_DIR: 'Firmware/${{ env.version }}/${{ matrix.target }}/'

.github/workflows/metadata.yml

@@ -1,133 +0,0 @@
-name: Metadata
-
-on:
-  push:
-    branches:
-    - 'main'
-    - 'release/*'
-    - 'pr-metadata-test'
-
-jobs:
-
-  airframe:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-base-focal:2021-09-08
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{ secrets.ACCESS_TOKEN }}
-
-    - name: airframe metadata
-      run: |
-        make airframe_metadata
-        ./src/lib/version/get_git_tag_or_branch_version.sh build/px4_sitl_default >> $GITHUB_ENV
-        cd build/px4_sitl_default/docs
-      # TODO: deploy to userguide gitbook
-
-    - uses: jakejarvis/s3-sync-action@master
-      with:
-        args: --acl public-read
-      env:
-        AWS_S3_BUCKET: 'px4-travis'
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_REGION: 'us-west-1'
-        SOURCE_DIR: 'build/px4_sitl_default/docs/'
-        DEST_DIR: 'Firmware/${{ env.version }}/_general/'
-
-  module:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-base-focal:2021-09-08
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{ secrets.ACCESS_TOKEN }}
-
-    - name: module documentation
-      run: |
-        make module_documentation
-        cd build/px4_sitl_default/docs
-        ls -ls *
-      # TODO: deploy to userguide gitbook and s3
-
-  parameter:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-base-focal:2021-09-08
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{ secrets.ACCESS_TOKEN }}
-
-    - name: parameter metadata
-      run: |
-        make parameters_metadata
-        ./src/lib/version/get_git_tag_or_branch_version.sh build/px4_sitl_default >> $GITHUB_ENV
-
-    - uses: jakejarvis/s3-sync-action@master
-      with:
-        args: --acl public-read
-      env:
-        AWS_S3_BUCKET: 'px4-travis'
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_REGION: 'us-west-1'
-        SOURCE_DIR: 'build/px4_sitl_default/docs/'
-        DEST_DIR: 'Firmware/${{ env.version }}/_general/'
-
-  events:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-base-focal:2021-09-08
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{ secrets.ACCESS_TOKEN }}
-
-    - name: events metadata
-      run: |
-        make extract_events
-        ./src/lib/version/get_git_tag_or_branch_version.sh build/px4_sitl_default >> $GITHUB_ENV
-        cd build/px4_sitl_default
-        mkdir _events_full || true
-        cp events/all_events_full.json.xz _events_full/all_events.json.xz
-
-    - uses: jakejarvis/s3-sync-action@master
-      with:
-        args: --acl public-read
-      env:
-        AWS_S3_BUCKET: 'px4-travis'
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_REGION: 'us-west-1'
-        SOURCE_DIR: 'build/px4_sitl_default/_events_full/'
-        DEST_DIR: 'Firmware/${{ env.version }}/_general/'
-
-  uorb_graph:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-nuttx-focal:2022-08-12
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{ secrets.ACCESS_TOKEN }}
-
-    - name: uORB graph
-      run: |
-        make uorb_graphs
-        cd Tools/uorb_graph
-        ls -ls *
-      # TODO: deploy graph_px4_sitl.json to S3 px4-travis:Firmware/master/
-
-  ROS2_msgs:
-    runs-on: ubuntu-latest
-    container: px4io/px4-dev-base-focal:2021-09-08
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        token: ${{ secrets.ACCESS_TOKEN }}
-
-    - name: PX4 ROS2 msgs
-      run: |
-        git clone https://github.com/PX4/px4_msgs.git
-        rm px4_msgs/msg/*.msg
-        rm px4_msgs/srv/*.srv
-        cp msg/*.msg px4_msgs/msg/
-        cp srv/*.srv px4_msgs/srv/