ci: build multi platform images

- ci: build containers on releases only - setup: install all compilers for arch - ci: saves cache to s3 and defaults to gh public cache - docker: try to catch apt registry installs - ci: use qemu for arm64 images
2026-05-02 05:04:08 +08:00 · 2024-11-21 17:12:29 -08:00 · 2024-11-21 17:12:29 -08:00 · f29a773abc
commit f29a773abc
parent d1e4198864
7 changed files with 154 additions and 95 deletions
--- a/.github/workflows/build_all_targets.yml
+++ b/.github/workflows/build_all_targets.yml
@ -26,7 +26,6 @@ jobs:
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
      timestamp: ${{ steps.set-timestamp.outputs.timestamp }}
-      tagname: ${{ steps.set-tag.outputs.tagname }}
      branchname: ${{ steps.set-branch.outputs.branchname }}
    steps:
      - uses: actions/checkout@v4
@ -57,9 +56,9 @@ jobs:
          echo "$(./Tools/ci/generate_board_targets_json.py --group --verbose)"

  setup:
-    name: Build Group [${{ matrix.group }}]
+    name: Build Group [${{ matrix.group }}][${{ matrix.arch == 'nuttx' && 'x86' || 'arm64' }}]
    # runs-on: ubuntu-latest
-    runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    runs-on: [runs-on,"runner=8cpu-linux-${{ matrix.arch == 'nuttx' && 'x64' || 'arm64' }}","image=ubuntu24-full-${{ matrix.arch == 'nuttx' && 'x64' || 'arm64' }}","run-id=${{ github.run_id }}",spot=false]
    needs: group_targets
    strategy:
      matrix: ${{ fromJson(needs.group_targets.outputs.matrix) }}
@ -94,7 +93,7 @@ jobs:

      - name: Building [${{ matrix.group }}]
        run: |
-            ./Tools/ci/build_all_runner.sh ${{matrix.targets}}
+            ./Tools/ci/build_all_runner.sh ${{matrix.targets}} ${{matrix.arch}}

      - name: Arrange Build Artifacts
        run: |
@ -112,7 +111,7 @@ jobs:
  artifacts:
    name: Upload Artifacts to S3
    # runs-on: ubuntu-latest
-    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
    needs: [setup, group_targets]
    if: contains(fromJSON('["main", "stable", "beta"]'), needs.group_targets.outputs.branchname)
    steps:
@ -140,10 +139,12 @@ jobs:

  release:
    name: Create Release and Upload Artifacts
+    permissions:
+      contents: write
    # runs-on: ubuntu-latest
-    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
    needs: [setup, group_targets]
-    if: startsWith(github.ref, 'refs/tags/v')
+    if: startsWith(github.ref, 'refs/tags/')
    steps:
      - name: Download Artifacts
        uses: actions/download-artifact@v4
--- a/.github/workflows/dev_container.yml
+++ b/.github/workflows/dev_container.yml
@ -17,32 +17,19 @@ on:
 jobs:
  build:
    name: Build and Push Container
-    runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false]
+    runs-on: [runs-on,"runner=8cpu-linux-x64","image=ubuntu24-full-x64","run-id=${{ github.run_id }}",spot=false,extras=s3-cache]
    steps:
+      - uses: runs-on/action@v1
      - uses: actions/checkout@v4
        with:
          fetch-tags: true
          submodules: false
          fetch-depth: 0

-      - name: Set PX4 Tag
-        id: px4-tag
+      - name: Set PX4 Tag Version
+        id: px4_version
        run: |
-          echo "tag=$(git describe --tags --match 'v[0-9]*')" >> $GITHUB_OUTPUT
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        if: github.event_name != 'pull_request'
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          echo "px4_version=$(git describe --tags --match 'v[0-9]*')" >> $GITHUB_OUTPUT

      - name: Extract metadata (tags, labels) for Docker
        id: meta
@ -50,52 +37,90 @@ jobs:
        with:
          images: |
            ghcr.io/PX4/px4-dev
-            ${{ (github.event_name != 'pull_request') && 'px4io/px4-dev' || '' }}
+            px4io/px4-dev
          tags: |
-            type=raw,enable=true,value=${{ steps.px4-tag.outputs.tag }},priority=1000
+            type=raw,enable=true,value=${{ steps.px4_version.outputs.px4_version }},priority=1000
+
+      - name: Cache Apt Registry
+        uses: actions/cache@v3
+        id: cache
+        with:
+          path: |
+            var-cache-apt
+            var-lib-apt
+          key: cache-${{ hashFiles('Tools/setup/Dockerfile') }}
+
+      - name: Inject Cache into Docker
+        uses: reproducible-containers/buildkit-cache-dance@v3.1.0
+        with:
+          cache-map: |
+            {
+              "var-cache-apt": "/var/cache/apt",
+              "var-lib-apt": "/var/lib/apt"
+            }
+          skip-extraction: ${{ steps.cache.outputs.cache-hit }}
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

-      - name: Build and load container image
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64
+
+      - name: Build and Load Container Image
        uses: docker/build-push-action@v6
        id: docker
        with:
          context: Tools/setup
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
-          platforms: |
-            linux/amd64
-          load: true
+          platforms: linux/amd64,linux/arm64
+          load: false
          push: false
-          cache-from: type=s3,blobs_prefix=cache/${{ github.repository }}/,manifests_prefix=cache/${{ github.repository }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
-          cache-to: type=s3,blobs_prefix=cache/${{ github.repository }}/,manifests_prefix=cache/${{ github.repository }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

-      - name: Get Tag Name
-        id: tag_name
-        run: |
-          echo "::set-output name=tag_name::$(echo '${{ fromJSON(steps.docker.outputs.metadata)['image.name'] }}' | sed 's/,.*$//')"
+      # - name: Get Container Image Tag Name
+      #   id: image_name
+      #   run: |
+      #     echo "image_name=$(echo '${{ fromJSON(steps.docker.outputs.metadata)['image.name'] }}' | sed 's/,.*$//')" >> $GITHUB_OUTPUT

-      - name: make quick_check
-        uses: addnab/docker-run-action@v3
-        with:
-          image: ${{ steps.tag_name.outputs.tag_name }}
-          options: -v ${{ github.workspace }}:/workspace
-          run: |
-            cd /workspace
-            git config --global --add safe.directory /workspace
-            make px4_sitl_default
-            make px4_fmu-v6x_default
+      # - name: Test Build of PX4 with Container
+      #   uses: addnab/docker-run-action@v3
+      #   with:
+      #     image: ${{ steps.image_name.outputs.image_name }}
+      #     options: -v ${{ github.workspace }}:/workspace
+      #     run: |
+      #       cd /workspace
+      #       git config --global --add safe.directory /workspace
+      #       make px4_sitl_default
+      #       make px4_fmu-v6x_default

-      - name: Push container image
+      - name: Push Images to Registry
        uses: docker/build-push-action@v6
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
        with:
          context: Tools/setup
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
-          platforms: |
-            linux/amd64
-          provenance: mode=max
-          push: ${{ github.event_name != 'pull_request' }}
-          cache-from: type=s3,blobs_prefix=cache/${{ github.repository }}/,manifests_prefix=cache/${{ github.repository }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
-          cache-to: type=s3,blobs_prefix=cache/${{ github.repository }}/,manifests_prefix=cache/${{ github.repository }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
+          platforms: linux/amd64,linux/arm64
+          load: false
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
--- a/Tools/ci/generate_board_targets_json.py
+++ b/Tools/ci/generate_board_targets_json.py
@ -87,20 +87,17 @@ def process_target(px4board_file, target_name):

    if platform not in excluded_platforms:
        # get the container based on the platform and toolchain
+        container = 'ghcr.io/px4/px4-dev:v1.16.0-alpha2-388-ga996439b1d'
        if platform == 'posix':
-            container = 'px4io/px4-dev-base-focal:2021-09-08'
            group = 'base'
            if toolchain:
                if toolchain.startswith('aarch64'):
-                    container = 'px4io/px4-dev-aarch64:2022-08-12'
                    group = 'aarch64'
                elif toolchain == 'arm-linux-gnueabihf':
-                    container = 'px4io/px4-dev-armhf:2023-06-26'
                    group = 'armhf'
                else:
                    if verbose: print(f'unmatched toolchain: {toolchain}')
        elif platform == 'nuttx':
-            container = 'px4io/px4-dev-nuttx-focal:2022-08-12'
            group = 'nuttx'
        else:
            if verbose: print(f'unmatched platform: {platform}')
@ -124,7 +121,7 @@ if(verbose):
 # - Events
 metadata_targets = ['airframe_metadata', 'parameters_metadata', 'extract_events']
 grouped_targets['base'] = {}
-grouped_targets['base']['container'] = 'px4io/px4-dev-base-focal:2021-09-08'
+grouped_targets['base']['container'] = 'ghcr.io/px4/px4-dev:v1.16.0-alpha2-388-ga996439b1d'
 grouped_targets['base']['manufacturers'] = {}
 grouped_targets['base']['manufacturers']['px4'] = []
 grouped_targets['base']['manufacturers']['px4'] += metadata_targets
--- a/Tools/ci/package_build_artifacts.sh
+++ b/Tools/ci/package_build_artifacts.sh
@ -4,8 +4,9 @@ mkdir artifacts
 cp **/**/*.px4 artifacts/
 cp **/**/*.elf artifacts/
 for build_dir_path in build/*/ ; do
+  build_dir_path=${build_dir_path::${#build_dir_path}-1}
  build_dir=${build_dir_path#*/}
-  build_dir=${build_dir::${#build_dir}-1}
+  target_name=$build_dir
  mkdir artifacts/$build_dir
  find artifacts/ -maxdepth 1 -type f -name "*$build_dir*"
  # Airframe
@ -26,21 +27,23 @@ for build_dir_path in build/*/ ; do
  echo "----------"
 done

-# general metadata
-mkdir artifacts/_general/
-cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/
-# Airframe
-cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/
-# Parameters
-cp artifacts/px4_sitl_default/parameters.xml artifacts/_general/
-cp artifacts/px4_sitl_default/parameters.json artifacts/_general/
-cp artifacts/px4_sitl_default/parameters.json.xz artifacts/_general/
-# Actuators
-cp artifacts/px4_sitl_default/actuators.json artifacts/_general/
-cp artifacts/px4_sitl_default/actuators.json.xz artifacts/_general/
-# Events
-cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/
-# ROS 2 msgs
-cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/
-# Module Docs
-ls -la artifacts/_general/
+if [ -d artifacts/px4_sitl_default ]; then
+  # general metadata
+  mkdir artifacts/_general/
+  cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/
+  # Airframe
+  cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/
+  # Parameters
+  cp artifacts/px4_sitl_default/parameters.xml artifacts/_general/
+  cp artifacts/px4_sitl_default/parameters.json artifacts/_general/
+  cp artifacts/px4_sitl_default/parameters.json.xz artifacts/_general/
+  # Actuators
+  cp artifacts/px4_sitl_default/actuators.json artifacts/_general/
+  cp artifacts/px4_sitl_default/actuators.json.xz artifacts/_general/
+  # Events
+  cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/
+  # ROS 2 msgs
+  cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/
+  # Module Docs
+  ls -la artifacts/_general/
+fi
--- a/Tools/setup/Dockerfile
+++ b/Tools/setup/Dockerfile
@ -13,6 +13,14 @@ ENV TZ=UTC
 EXPOSE 14556/udp
 EXPOSE 14557/udp

+# Set Up Caching of APT packages
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+	--mount=type=cache,target=/var/lib/apt,sharing=locked \
+	rm -f /etc/apt/apt.conf.d/docker-clean && \
+	echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache && \
+	apt-get --quiet -y update
+
+# Install Shell Script Entrypoint
 COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh

 # Install PX4 Requirements
--- a/Tools/setup/docker-entrypoint.sh
+++ b/Tools/setup/docker-entrypoint.sh
@ -1,19 +1,27 @@
 #!/bin/bash

+GREEN='\033[0;32m'
+NC='\033[0m' # No Color
+FILE_DESCRIPTOR="${GREEN}[docker-entrypoint.sh]${NC}"
+
+echo -e "$FILE_DESCRIPTOR Starting"
+
 # Start virtual X server in the background
 # - DISPLAY default is :99, set in dockerfile
 # - Users can override with `-e DISPLAY=` in `docker run` command to avoid
 #   running Xvfb and attach their screen
 if [[ -x "$(command -v Xvfb)" && "$DISPLAY" == ":99" ]]; then
-	echo "[docker-entrypoint.sh] Starting Xvfb"
+	echo -e "$FILE_DESCRIPTOR Starting Xvfb"
 	Xvfb :99 -screen 0 1600x1200x24+32 &
 fi

 # Check if the ROS_DISTRO is passed and use it
 # to source the ROS environment
 if [ -n "${ROS_DISTRO}" ]; then
-	echo "[docker-entrypoint.sh] ROS: ${ROS_DISTRO}"
+	echo -e "$FILE_DESCRIPTOR ROS: ${ROS_DISTRO}"
 	source "/opt/ros/$ROS_DISTRO/setup.bash"
 fi

+echo -e "$FILE_DESCRIPTOR ($( uname -m ))"
+
 exec "$@"
--- a/Tools/setup/ubuntu.sh
+++ b/Tools/setup/ubuntu.sh
@ -27,9 +27,12 @@ do
 	fi
 done

+echo "[ubuntu.sh] Starting..."
+echo "[ubuntu.sh] arch: ${GREEN}$INSTALL_ARCH${NC}"
+
 # detect if running in docker
 if [ -f /.dockerenv ]; then
-	echo "Running within docker, installing initial dependencies";
+	echo "[ubuntu.sh] Running within docker, installing initial dependencies";
 	apt-get --quiet -y update && DEBIAN_FRONTEND=noninteractive apt-get --quiet -y install \
 		ca-certificates \
 		gnupg \
@ -47,7 +50,7 @@ DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 # check requirements.txt exists (script not run in source tree)
 REQUIREMENTS_FILE="requirements.txt"
 if [[ ! -f "${DIR}/${REQUIREMENTS_FILE}" ]]; then
-	echo "FAILED: ${REQUIREMENTS_FILE} needed in same directory as ubuntu.sh (${DIR})."
+	echo "[ubuntu.sh] FAILED: ${REQUIREMENTS_FILE} needed in same directory as ubuntu.sh (${DIR})."
 	return 1
 fi

@ -55,10 +58,8 @@ fi
 # check ubuntu version
 # otherwise warn and point to docker?
 UBUNTU_RELEASE="`lsb_release -rs`"
-echo "Ubuntu ${UBUNTU_RELEASE}"
-
-echo
-echo "Installing PX4 general dependencies"
+echo "[ubuntu.sh] Ubuntu ${GREEN}${UBUNTU_RELEASE}${NC}"
+echo "[ubuntu.sh] Installing PX4 general dependencies"

 sudo apt-get update -y --quiet
 sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \
@ -91,7 +92,7 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends i

 # Python3 dependencies
 echo
-echo "Installing PX4 Python3 dependencies"
+echo "[ubuntu.sh] Installing PX4 Python3 dependencies"
 PYTHON_VERSION=$(python3 --version 2>&1 | awk '{print $2}')
 REQUIRED_VERSION="3.11"
 if [[ "$(printf '%s\n' "$REQUIRED_VERSION" "$PYTHON_VERSION" | sort -V | head -n1)" == "$REQUIRED_VERSION" ]]; then
@ -109,8 +110,8 @@ fi
 if [[ $INSTALL_NUTTX == "true" ]]; then

 	echo
-	echo "Installing NuttX dependencies"
-
+	echo "[ubuntu.sh] NuttX Installing Dependencies"
+	sudo apt-get update -y --quiet
 	sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \
 		automake \
 		binutils-dev \
@ -118,9 +119,6 @@ if [[ $INSTALL_NUTTX == "true" ]]; then
 		build-essential \
 		curl \
 		flex \
-		g++-multilib \
-		gcc-arm-none-eabi \
-		gcc-multilib \
 		gdb-multiarch \
 		genromfs \
 		gettext \
@ -147,6 +145,25 @@ if [[ $INSTALL_NUTTX == "true" ]]; then
 		vim-common \
 		;

+
+	echo
+	echo "[ubuntu.sh] NuttX Installing Dependencies ($INSTALL_ARCH)"
+
+	if [[ "${INSTALL_ARCH}" == "x86_64" ]]; then
+		sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \
+			g++-multilib \
+			gcc-arm-none-eabi \
+			gcc-multilib \
+			;
+	fi
+
+	if [[ "${INSTALL_ARCH}" == "aarch64" ]]; then
+		sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \
+			g++-aarch64-linux-gnu \
+			g++-arm-linux-gnueabihf \
+			;
+	fi
+
 	if [ -n "$USER" ]; then
 		# add user to dialout group (serial port access)
 		sudo usermod -aG dialout $USER
@ -157,7 +174,7 @@ fi
 if [[ $INSTALL_SIM == "true" ]]; then

 	echo
-	echo "Installing PX4 simulation dependencies"
+	echo "[ubuntu.sh] Installing PX4 simulation dependencies"

 	# General simulation dependencies
 	sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \
@ -182,8 +199,8 @@ if [[ $INSTALL_SIM == "true" ]]; then
 		fi
 	else
 		# Expects Ubuntu 22.04 > by default
-		echo "Gazebo (Harmonic) will be installed"
-		echo "Earlier versions will be removed"
+		echo "[ubuntu.sh] Gazebo (Harmonic) will be installed"
+		echo "[ubuntu.sh] Earlier versions will be removed"
 		# Add Gazebo binary repository
 		sudo wget https://packages.osrfoundation.org/gazebo.gpg -O /usr/share/keyrings/pkgs-osrf-archive-keyring.gpg
 		echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/pkgs-osrf-archive-keyring.gpg] http://packages.osrfoundation.org/gazebo/ubuntu-stable $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/gazebo-stable.list > /dev/null