From f29a773abc6fc31ff135c6060fccc95b93cb0b51 Mon Sep 17 00:00:00 2001 From: Ramon Roche Date: Thu, 21 Nov 2024 17:12:29 -0800 Subject: [PATCH] ci: build multi platform images - ci: build containers on releases only - setup: install all compilers for arch - ci: saves cache to s3 and defaults to gh public cache - docker: try to catch apt registry installs - ci: use qemu for arm64 images --- .github/workflows/build_all_targets.yml | 15 +-- .github/workflows/dev_container.yml | 119 ++++++++++++++---------- Tools/ci/generate_board_targets_json.py | 7 +- Tools/ci/package_build_artifacts.sh | 41 ++++---- Tools/setup/Dockerfile | 8 ++ Tools/setup/docker-entrypoint.sh | 12 ++- Tools/setup/ubuntu.sh | 47 +++++++--- 7 files changed, 154 insertions(+), 95 deletions(-) diff --git a/.github/workflows/build_all_targets.yml b/.github/workflows/build_all_targets.yml index e6d0ece289..06b9910ef5 100644 --- a/.github/workflows/build_all_targets.yml +++ b/.github/workflows/build_all_targets.yml @@ -26,7 +26,6 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} timestamp: ${{ steps.set-timestamp.outputs.timestamp }} - tagname: ${{ steps.set-tag.outputs.tagname }} branchname: ${{ steps.set-branch.outputs.branchname }} steps: - uses: actions/checkout@v4 @@ -57,9 +56,9 @@ jobs: echo "$(./Tools/ci/generate_board_targets_json.py --group --verbose)" setup: - name: Build Group [${{ matrix.group }}] + name: Build Group [${{ matrix.group }}][${{ matrix.arch == 'nuttx' && 'x86' || 'arm64' }}] # runs-on: ubuntu-latest - runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false] + runs-on: [runs-on,"runner=8cpu-linux-${{ matrix.arch == 'nuttx' && 'x64' || 'arm64' }}","image=ubuntu24-full-${{ matrix.arch == 'nuttx' && 'x64' || 'arm64' }}","run-id=${{ github.run_id }}",spot=false] needs: group_targets strategy: matrix: ${{ fromJson(needs.group_targets.outputs.matrix) }} @@ -94,7 +93,7 @@ jobs: - name: Building [${{ matrix.group }}] run: | - ./Tools/ci/build_all_runner.sh ${{matrix.targets}} + ./Tools/ci/build_all_runner.sh ${{matrix.targets}} ${{matrix.arch}} - name: Arrange Build Artifacts run: | @@ -112,7 +111,7 @@ jobs: artifacts: name: Upload Artifacts to S3 # runs-on: ubuntu-latest - runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false] + runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false] needs: [setup, group_targets] if: contains(fromJSON('["main", "stable", "beta"]'), needs.group_targets.outputs.branchname) steps: @@ -140,10 +139,12 @@ jobs: release: name: Create Release and Upload Artifacts + permissions: + contents: write # runs-on: ubuntu-latest - runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}",spot=false] + runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false] needs: [setup, group_targets] - if: startsWith(github.ref, 'refs/tags/v') + if: startsWith(github.ref, 'refs/tags/') steps: - name: Download Artifacts uses: actions/download-artifact@v4 diff --git a/.github/workflows/dev_container.yml b/.github/workflows/dev_container.yml index b3f9542ead..c8cbc3c6c5 100644 --- a/.github/workflows/dev_container.yml +++ b/.github/workflows/dev_container.yml @@ -17,32 +17,19 @@ on: jobs: build: name: Build and Push Container - runs-on: [runs-on,runner=8cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=false] + runs-on: [runs-on,"runner=8cpu-linux-x64","image=ubuntu24-full-x64","run-id=${{ github.run_id }}",spot=false,extras=s3-cache] steps: + - uses: runs-on/action@v1 - uses: actions/checkout@v4 with: fetch-tags: true submodules: false fetch-depth: 0 - - name: Set PX4 Tag - id: px4-tag + - name: Set PX4 Tag Version + id: px4_version run: | - echo "tag=$(git describe --tags --match 'v[0-9]*')" >> $GITHUB_OUTPUT - - - name: Login to Docker Hub - uses: docker/login-action@v3 - if: github.event_name != 'pull_request' - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + echo "px4_version=$(git describe --tags --match 'v[0-9]*')" >> $GITHUB_OUTPUT - name: Extract metadata (tags, labels) for Docker id: meta @@ -50,52 +37,90 @@ jobs: with: images: | ghcr.io/PX4/px4-dev - ${{ (github.event_name != 'pull_request') && 'px4io/px4-dev' || '' }} + px4io/px4-dev tags: | - type=raw,enable=true,value=${{ steps.px4-tag.outputs.tag }},priority=1000 + type=raw,enable=true,value=${{ steps.px4_version.outputs.px4_version }},priority=1000 + + - name: Cache Apt Registry + uses: actions/cache@v3 + id: cache + with: + path: | + var-cache-apt + var-lib-apt + key: cache-${{ hashFiles('Tools/setup/Dockerfile') }} + + - name: Inject Cache into Docker + uses: reproducible-containers/buildkit-cache-dance@v3.1.0 + with: + cache-map: | + { + "var-cache-apt": "/var/cache/apt", + "var-lib-apt": "/var/lib/apt" + } + skip-extraction: ${{ steps.cache.outputs.cache-hit }} + + - name: Login to Docker Hub + uses: docker/login-action@v3 + if: ${{ startsWith(github.ref, 'refs/tags/') }} + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + if: ${{ startsWith(github.ref, 'refs/tags/') }} + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build and load container image + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/amd64,linux/arm64 + + - name: Build and Load Container Image uses: docker/build-push-action@v6 id: docker with: context: Tools/setup tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - platforms: | - linux/amd64 - load: true + platforms: linux/amd64,linux/arm64 + load: false push: false - cache-from: type=s3,blobs_prefix=cache/${{ github.repository }}/,manifests_prefix=cache/${{ github.repository }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }} - cache-to: type=s3,blobs_prefix=cache/${{ github.repository }}/,manifests_prefix=cache/${{ github.repository }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max + cache-from: type=gha + cache-to: type=gha,mode=max - - name: Get Tag Name - id: tag_name - run: | - echo "::set-output name=tag_name::$(echo '${{ fromJSON(steps.docker.outputs.metadata)['image.name'] }}' | sed 's/,.*$//')" + # - name: Get Container Image Tag Name + # id: image_name + # run: | + # echo "image_name=$(echo '${{ fromJSON(steps.docker.outputs.metadata)['image.name'] }}' | sed 's/,.*$//')" >> $GITHUB_OUTPUT - - name: make quick_check - uses: addnab/docker-run-action@v3 - with: - image: ${{ steps.tag_name.outputs.tag_name }} - options: -v ${{ github.workspace }}:/workspace - run: | - cd /workspace - git config --global --add safe.directory /workspace - make px4_sitl_default - make px4_fmu-v6x_default + # - name: Test Build of PX4 with Container + # uses: addnab/docker-run-action@v3 + # with: + # image: ${{ steps.image_name.outputs.image_name }} + # options: -v ${{ github.workspace }}:/workspace + # run: | + # cd /workspace + # git config --global --add safe.directory /workspace + # make px4_sitl_default + # make px4_fmu-v6x_default - - name: Push container image + - name: Push Images to Registry uses: docker/build-push-action@v6 + if: ${{ startsWith(github.ref, 'refs/tags/') }} with: context: Tools/setup tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - platforms: | - linux/amd64 - provenance: mode=max - push: ${{ github.event_name != 'pull_request' }} - cache-from: type=s3,blobs_prefix=cache/${{ github.repository }}/,manifests_prefix=cache/${{ github.repository }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }} - cache-to: type=s3,blobs_prefix=cache/${{ github.repository }}/,manifests_prefix=cache/${{ github.repository }}/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max + platforms: linux/amd64,linux/arm64 + load: false + push: true + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Tools/ci/generate_board_targets_json.py b/Tools/ci/generate_board_targets_json.py index 5749a988d1..f547420ce8 100755 --- a/Tools/ci/generate_board_targets_json.py +++ b/Tools/ci/generate_board_targets_json.py @@ -87,20 +87,17 @@ def process_target(px4board_file, target_name): if platform not in excluded_platforms: # get the container based on the platform and toolchain + container = 'ghcr.io/px4/px4-dev:v1.16.0-alpha2-388-ga996439b1d' if platform == 'posix': - container = 'px4io/px4-dev-base-focal:2021-09-08' group = 'base' if toolchain: if toolchain.startswith('aarch64'): - container = 'px4io/px4-dev-aarch64:2022-08-12' group = 'aarch64' elif toolchain == 'arm-linux-gnueabihf': - container = 'px4io/px4-dev-armhf:2023-06-26' group = 'armhf' else: if verbose: print(f'unmatched toolchain: {toolchain}') elif platform == 'nuttx': - container = 'px4io/px4-dev-nuttx-focal:2022-08-12' group = 'nuttx' else: if verbose: print(f'unmatched platform: {platform}') @@ -124,7 +121,7 @@ if(verbose): # - Events metadata_targets = ['airframe_metadata', 'parameters_metadata', 'extract_events'] grouped_targets['base'] = {} -grouped_targets['base']['container'] = 'px4io/px4-dev-base-focal:2021-09-08' +grouped_targets['base']['container'] = 'ghcr.io/px4/px4-dev:v1.16.0-alpha2-388-ga996439b1d' grouped_targets['base']['manufacturers'] = {} grouped_targets['base']['manufacturers']['px4'] = [] grouped_targets['base']['manufacturers']['px4'] += metadata_targets diff --git a/Tools/ci/package_build_artifacts.sh b/Tools/ci/package_build_artifacts.sh index a665e295df..2b487f7632 100755 --- a/Tools/ci/package_build_artifacts.sh +++ b/Tools/ci/package_build_artifacts.sh @@ -4,8 +4,9 @@ mkdir artifacts cp **/**/*.px4 artifacts/ cp **/**/*.elf artifacts/ for build_dir_path in build/*/ ; do + build_dir_path=${build_dir_path::${#build_dir_path}-1} build_dir=${build_dir_path#*/} - build_dir=${build_dir::${#build_dir}-1} + target_name=$build_dir mkdir artifacts/$build_dir find artifacts/ -maxdepth 1 -type f -name "*$build_dir*" # Airframe @@ -26,21 +27,23 @@ for build_dir_path in build/*/ ; do echo "----------" done -# general metadata -mkdir artifacts/_general/ -cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/ -# Airframe -cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/ -# Parameters -cp artifacts/px4_sitl_default/parameters.xml artifacts/_general/ -cp artifacts/px4_sitl_default/parameters.json artifacts/_general/ -cp artifacts/px4_sitl_default/parameters.json.xz artifacts/_general/ -# Actuators -cp artifacts/px4_sitl_default/actuators.json artifacts/_general/ -cp artifacts/px4_sitl_default/actuators.json.xz artifacts/_general/ -# Events -cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/ -# ROS 2 msgs -cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/ -# Module Docs -ls -la artifacts/_general/ +if [ -d artifacts/px4_sitl_default ]; then + # general metadata + mkdir artifacts/_general/ + cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/ + # Airframe + cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/ + # Parameters + cp artifacts/px4_sitl_default/parameters.xml artifacts/_general/ + cp artifacts/px4_sitl_default/parameters.json artifacts/_general/ + cp artifacts/px4_sitl_default/parameters.json.xz artifacts/_general/ + # Actuators + cp artifacts/px4_sitl_default/actuators.json artifacts/_general/ + cp artifacts/px4_sitl_default/actuators.json.xz artifacts/_general/ + # Events + cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/ + # ROS 2 msgs + cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/ + # Module Docs + ls -la artifacts/_general/ +fi diff --git a/Tools/setup/Dockerfile b/Tools/setup/Dockerfile index 0cf8762b9d..8fd4e43d10 100644 --- a/Tools/setup/Dockerfile +++ b/Tools/setup/Dockerfile @@ -13,6 +13,14 @@ ENV TZ=UTC EXPOSE 14556/udp EXPOSE 14557/udp +# Set Up Caching of APT packages +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean && \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache && \ + apt-get --quiet -y update + +# Install Shell Script Entrypoint COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh # Install PX4 Requirements diff --git a/Tools/setup/docker-entrypoint.sh b/Tools/setup/docker-entrypoint.sh index 0412558902..083a4c9f24 100755 --- a/Tools/setup/docker-entrypoint.sh +++ b/Tools/setup/docker-entrypoint.sh @@ -1,19 +1,27 @@ #!/bin/bash +GREEN='\033[0;32m' +NC='\033[0m' # No Color +FILE_DESCRIPTOR="${GREEN}[docker-entrypoint.sh]${NC}" + +echo -e "$FILE_DESCRIPTOR Starting" + # Start virtual X server in the background # - DISPLAY default is :99, set in dockerfile # - Users can override with `-e DISPLAY=` in `docker run` command to avoid # running Xvfb and attach their screen if [[ -x "$(command -v Xvfb)" && "$DISPLAY" == ":99" ]]; then - echo "[docker-entrypoint.sh] Starting Xvfb" + echo -e "$FILE_DESCRIPTOR Starting Xvfb" Xvfb :99 -screen 0 1600x1200x24+32 & fi # Check if the ROS_DISTRO is passed and use it # to source the ROS environment if [ -n "${ROS_DISTRO}" ]; then - echo "[docker-entrypoint.sh] ROS: ${ROS_DISTRO}" + echo -e "$FILE_DESCRIPTOR ROS: ${ROS_DISTRO}" source "/opt/ros/$ROS_DISTRO/setup.bash" fi +echo -e "$FILE_DESCRIPTOR ($( uname -m ))" + exec "$@" diff --git a/Tools/setup/ubuntu.sh b/Tools/setup/ubuntu.sh index e641a82da3..1df4e415b5 100755 --- a/Tools/setup/ubuntu.sh +++ b/Tools/setup/ubuntu.sh @@ -27,9 +27,12 @@ do fi done +echo "[ubuntu.sh] Starting..." +echo "[ubuntu.sh] arch: ${GREEN}$INSTALL_ARCH${NC}" + # detect if running in docker if [ -f /.dockerenv ]; then - echo "Running within docker, installing initial dependencies"; + echo "[ubuntu.sh] Running within docker, installing initial dependencies"; apt-get --quiet -y update && DEBIAN_FRONTEND=noninteractive apt-get --quiet -y install \ ca-certificates \ gnupg \ @@ -47,7 +50,7 @@ DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) # check requirements.txt exists (script not run in source tree) REQUIREMENTS_FILE="requirements.txt" if [[ ! -f "${DIR}/${REQUIREMENTS_FILE}" ]]; then - echo "FAILED: ${REQUIREMENTS_FILE} needed in same directory as ubuntu.sh (${DIR})." + echo "[ubuntu.sh] FAILED: ${REQUIREMENTS_FILE} needed in same directory as ubuntu.sh (${DIR})." return 1 fi @@ -55,10 +58,8 @@ fi # check ubuntu version # otherwise warn and point to docker? UBUNTU_RELEASE="`lsb_release -rs`" -echo "Ubuntu ${UBUNTU_RELEASE}" - -echo -echo "Installing PX4 general dependencies" +echo "[ubuntu.sh] Ubuntu ${GREEN}${UBUNTU_RELEASE}${NC}" +echo "[ubuntu.sh] Installing PX4 general dependencies" sudo apt-get update -y --quiet sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \ @@ -91,7 +92,7 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends i # Python3 dependencies echo -echo "Installing PX4 Python3 dependencies" +echo "[ubuntu.sh] Installing PX4 Python3 dependencies" PYTHON_VERSION=$(python3 --version 2>&1 | awk '{print $2}') REQUIRED_VERSION="3.11" if [[ "$(printf '%s\n' "$REQUIRED_VERSION" "$PYTHON_VERSION" | sort -V | head -n1)" == "$REQUIRED_VERSION" ]]; then @@ -109,8 +110,8 @@ fi if [[ $INSTALL_NUTTX == "true" ]]; then echo - echo "Installing NuttX dependencies" - + echo "[ubuntu.sh] NuttX Installing Dependencies" + sudo apt-get update -y --quiet sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \ automake \ binutils-dev \ @@ -118,9 +119,6 @@ if [[ $INSTALL_NUTTX == "true" ]]; then build-essential \ curl \ flex \ - g++-multilib \ - gcc-arm-none-eabi \ - gcc-multilib \ gdb-multiarch \ genromfs \ gettext \ @@ -147,6 +145,25 @@ if [[ $INSTALL_NUTTX == "true" ]]; then vim-common \ ; + + echo + echo "[ubuntu.sh] NuttX Installing Dependencies ($INSTALL_ARCH)" + + if [[ "${INSTALL_ARCH}" == "x86_64" ]]; then + sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \ + g++-multilib \ + gcc-arm-none-eabi \ + gcc-multilib \ + ; + fi + + if [[ "${INSTALL_ARCH}" == "aarch64" ]]; then + sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \ + g++-aarch64-linux-gnu \ + g++-arm-linux-gnueabihf \ + ; + fi + if [ -n "$USER" ]; then # add user to dialout group (serial port access) sudo usermod -aG dialout $USER @@ -157,7 +174,7 @@ fi if [[ $INSTALL_SIM == "true" ]]; then echo - echo "Installing PX4 simulation dependencies" + echo "[ubuntu.sh] Installing PX4 simulation dependencies" # General simulation dependencies sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \ @@ -182,8 +199,8 @@ if [[ $INSTALL_SIM == "true" ]]; then fi else # Expects Ubuntu 22.04 > by default - echo "Gazebo (Harmonic) will be installed" - echo "Earlier versions will be removed" + echo "[ubuntu.sh] Gazebo (Harmonic) will be installed" + echo "[ubuntu.sh] Earlier versions will be removed" # Add Gazebo binary repository sudo wget https://packages.osrfoundation.org/gazebo.gpg -O /usr/share/keyrings/pkgs-osrf-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/pkgs-osrf-archive-keyring.gpg] http://packages.osrfoundation.org/gazebo/ubuntu-stable $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/gazebo-stable.list > /dev/null