github: test file changed action

the new docker images are defaulting to a hardcoded tag that can be
traced back to source code which is why we are adding the string to the
build and run scripts

.github/workflows/docker_dev.yml

@@ -0,0 +1,14 @@
+name: docker-dev
+on: [push, pull_request]
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    steps:
+      - id: file_changes
+        uses: trilom/file-changes-action@v1.2.3
+      - name: test
+        run: |
+          echo '${{ steps.file_changes.outputs.files}}'
+          echo '${{ steps.file_changes.outputs.files_modified}}'
+          echo '${{ steps.file_changes.outputs.files_added}}'
+          echo '${{ steps.file_changes.outputs.files_removed}}'

Tools/docker_build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+if [[ -z "${DOCKER_TAG}" ]]; then
+  # the default tag for docker images
+  # follows the pattern below, and we recommend
+  # that any images pushed to the registry continue
+  # to use the pattern for consistency
+  TAG_NAME="`date +"%Y-%m-%d"`"
+else
+  TAG_NAME="${DOCKER_TAG}"
+fi
+
+PX4_DOCKER_REPO="ghcr.io/px4/px4-dev:$TAG_NAME"
+PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+SRC_DIR=$PWD/../
+
+echo "[docker_build.sh]: Building [$PX4_DOCKER_REPO]"
+
+docker build \
+  -t ${PX4_DOCKER_REPO} \
+  -f Tools/setup/Dockerfile "${SRC_DIR}"

Tools/docker_run.sh

@@ -1,67 +1,29 @@
-#! /bin/bash
+#!/bin/bash
 
-if [ -z ${PX4_DOCKER_REPO+x} ]; then
-	echo "guessing PX4_DOCKER_REPO based on input";
-	if [[ $@ =~ .*px4_fmu.* ]]; then
-		# nuttx-px4fmu-v{1,2,3,4,5}
-		PX4_DOCKER_REPO="px4io/px4-dev-nuttx-focal:2021-09-08"
-	elif [[ $@ =~ .*navio2.* ]] || [[ $@ =~ .*raspberry.* ]] || [[ $@ =~ .*beaglebone.* ]] || [[ $@ =~ .*pilotpi.default ]]; then
-		# beaglebone_blue_default, emlid_navio2_default, px4_raspberrypi_default, scumaker_pilotpi_default
-		PX4_DOCKER_REPO="px4io/px4-dev-armhf:2021-08-18"
-	elif [[ $@ =~ .*pilotpi.arm64 ]]; then
-		# scumaker_pilotpi_arm64
-		PX4_DOCKER_REPO="px4io/px4-dev-aarch64:latest"
-	elif [[ $@ =~ .*navio2.* ]] || [[ $@ =~ .*raspberry.* ]] || [[ $@ =~ .*bebop.* ]]; then
-		# posix_rpi_cross, posix_bebop_default
-		PX4_DOCKER_REPO="px4io/px4-dev-armhf:2021-08-18"
-	elif [[ $@ =~ .*clang.* ]] || [[ $@ =~ .*scan-build.* ]]; then
-		# clang tools
-		PX4_DOCKER_REPO="px4io/px4-dev-clang:2021-02-04"
-	elif [[ $@ =~ .*tests* ]]; then
-		# run all tests with simulation
-		PX4_DOCKER_REPO="px4io/px4-dev-simulation-bionic:2021-12-11"
-	fi
+if [[ -z "${DOCKER_TAG}" ]]; then
+  # The default tag name should be hardcoded
+  # to whatever we are currently using in CI on this branch
+  TAG_NAME="2022-08-16"
 else
-	echo "PX4_DOCKER_REPO is set to '$PX4_DOCKER_REPO'";
+  TAG_NAME="${DOCKER_TAG}"
 fi
 
-# otherwise default to nuttx
-if [ -z ${PX4_DOCKER_REPO+x} ]; then
-	PX4_DOCKER_REPO="px4io/px4-dev-nuttx-focal:2021-09-08"
-fi
+PX4_DOCKER_REPO="ghcr.io/px4/px4-dev:$TAG_NAME"
 
-# docker hygiene
-
-#Delete all stopped containers (including data-only containers)
-#docker rm $(docker ps -a -q)
-
-#Delete all 'untagged/dangling' (<none>) images
-#docker rmi $(docker images -q -f dangling=true)
-
-echo "PX4_DOCKER_REPO: $PX4_DOCKER_REPO";
+echo "[docker_run.sh]: Running '$PX4_DOCKER_REPO'"
 
 PWD=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 SRC_DIR=$PWD/../
 
-CCACHE_DIR=${HOME}/.ccache
-mkdir -p "${CCACHE_DIR}"
-
 docker run -it --rm -w "${SRC_DIR}" \
-	--env=AWS_ACCESS_KEY_ID \
-	--env=AWS_SECRET_ACCESS_KEY \
-	--env=BRANCH_NAME \
-	--env=CCACHE_DIR="${CCACHE_DIR}" \
-	--env=CI \
-	--env=CODECOV_TOKEN \
-	--env=COVERALLS_REPO_TOKEN \
-	--env=LOCAL_USER_ID="$(id -u)" \
-	--env=PX4_ASAN \
-	--env=PX4_MSAN \
-	--env=PX4_TSAN \
-	--env=PX4_UBSAN \
-	--env=TRAVIS_BRANCH \
-	--env=TRAVIS_BUILD_ID \
-	--publish 14556:14556/udp \
-	--volume=${CCACHE_DIR}:${CCACHE_DIR}:rw \
-	--volume=${SRC_DIR}:${SRC_DIR}:rw \
-	${PX4_DOCKER_REPO} /bin/bash -c "$1 $2 $3"
+  --env=LOCAL_USER_ID="$(id -u)" \
+  --env=AWS_ACCESS_KEY_ID \
+  --env=AWS_SECRET_ACCESS_KEY \
+  --env=BRANCH_NAME \
+  --env=CCACHE_DIR="${CCACHE_DIR}" \
+  --env=CI \
+  --publish 14556:14556/udp \
+  --volume=/tmp/.X11-unix:/tmp/.X11-unix \
+  --volume=/tmp:/tmp:rw \
+  --volume=${SRC_DIR}:${SRC_DIR}:rw \
+  ${PX4_DOCKER_REPO} /bin/bash -c "$1 $2 $3"

Tools/setup/Dockerfile

@@ -0,0 +1,51 @@
+#
+# PX4 base development environment
+#
+
+FROM ubuntu:20.04
+LABEL maintainer="Daniel Agar <daniel@agar.ca>, Ramon Roche <mrpollo@gmail.com>"
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+# Installing required utilities
+RUN apt-get update && apt-get -y --quiet --no-install-recommends install \
+  ca-certificates \
+  gnupg \
+  lsb-core \
+  lsb-release \
+  sudo \
+  software-properties-common \
+  wget \
+  gosu \
+  ;
+
+# Install PX4 Requirements
+COPY Tools/setup/requirements.txt /tmp/requirements.txt
+COPY Tools/setup/ubuntu.sh /tmp/ubuntu.sh
+# The PATH env is set within ubuntu.sh, but given how we
+# are running the image using `gosu` to avoid read-only problems
+# with the filesystem the env variable does not persist
+ENV PATH="/opt/gcc-arm-none-eabi-9-2020-q2-update/bin:$PATH"
+
+RUN bash /tmp/ubuntu.sh --from-docker
+
+ENV DISPLAY :99
+
+ENV FASTRTPSGEN_DIR="/usr/local/bin/"
+ENV TERM=xterm
+ENV TZ=UTC
+
+# SITL UDP PORTS
+EXPOSE 14556/udp
+EXPOSE 14557/udp
+
+# create user with id 1001 (jenkins docker workflow default)
+RUN useradd --shell /bin/bash -u 1001 -c "" -m user && usermod -a -G dialout user
+
+# create and start as LOCAL_USER_ID
+COPY Tools/setup/docker-entrypoint.sh /usr/local/bin/entrypoint.sh
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
+
+CMD ["/bin/bash"]

Tools/setup/docker-entrypoint.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+echo "[docker-entrypoint.sh] Starting entrypoint"
+# Start virtual X server in the background
+# - DISPLAY default is :99, set in dockerfile
+# - Users can override with `-e DISPLAY=` in `docker run` command to avoid
+#   running Xvfb and attach their screen
+if [[ -x "$(command -v Xvfb)" && "$DISPLAY" == ":99" ]]; then
+	echo "[docker-entrypoint.sh] Starting Xvfb"
+	Xvfb :99 -screen 0 1600x1200x24+32 &
+fi
+
+# Check if the ROS_DISTRO is passed and use it
+# to source the ROS environment
+if [ -n "${ROS_DISTRO}" ]; then
+	source "/opt/ros/$ROS_DISTRO/setup.bash"
+fi
+
+# Use the LOCAL_USER_ID if passed in at runtime
+if [ -n "${LOCAL_USER_ID}" ]; then
+	echo "[docker-entrypoint.sh] Starting with UID : $LOCAL_USER_ID"
+	# modify existing user's id
+	usermod -u $LOCAL_USER_ID user
+
+	# run as user
+  exec gosu user "$@"
+else
+	exec "$@"
+fi

Tools/setup/ubuntu.sh

@@ -2,20 +2,18 @@
 
 set -e
 
-## Bash script to setup PX4 development environment on Ubuntu LTS (20.04, 18.04, 16.04).
+## Bash script to setup PX4 development environment on Ubuntu LTS (20.04, 18.04).
 ## Can also be used in docker.
 ##
 ## Installs:
-## - Common dependencies and tools for nuttx, jMAVSim, Gazebo
+## - Common dependencies and tools for NuttX, and Gazebo
 ## - NuttX toolchain (omit with arg: --no-nuttx)
-## - jMAVSim and Gazebo9 simulator (omit with arg: --no-sim-tools)
-##
-## Not Installs:
-## - FastRTPS and FastCDR
+## - Gazebo simulator (omit with arg: --no-sim-tools)
 
 INSTALL_NUTTX="true"
 INSTALL_SIM="true"
 INSTALL_ARCH=`uname -m`
+INSIDE_DOCKER="false"
 
 # Parse arguments
 for arg in "$@"
@@ -28,19 +26,22 @@ do
 		INSTALL_SIM="false"
 	fi
 
-done
 
-# detect if running in docker
-if [ -f /.dockerenv ]; then
-	echo "Running within docker, installing initial dependencies";
-	apt-get --quiet -y update && DEBIAN_FRONTEND=noninteractive apt-get --quiet -y install \
-		ca-certificates \
-		gnupg \
-		lsb-core \
-		sudo \
-		wget \
-		;
-fi
+	if [[ $arg == "--from-docker" ]]; then
+		INSIDE_DOCKER="true"
+	fi
+
+  if [[ $arg == "--help" ]]; then
+    echo "#⚡️ PX4 Dependency Installer for Ubuntu"
+    echo "# Options:
+#
+#  --no-nuttx       boolean
+#  --no-sim-tools   boolean"
+    echo "#"
+    exit
+  fi
+
+done
 
 # script directory
 DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
@@ -69,9 +70,22 @@ elif [[ "${UBUNTU_RELEASE}" == "20.04" ]]; then
 	echo "Ubuntu 20.04"
 fi
 
+VERBOSE_BAR="####################"
+echo
+echo $VERBOSE_BAR
+echo "#⚡️ Starting PX4 Dependency Installer for Ubuntu ${UBUNTU_RELEASE} (${INSTALL_ARCH})"
+echo "# Options:
+#
+#  - Install NuttX = ${INSTALL_NUTTX}
+#  - Install Simulation = ${INSTALL_SIM}"
+echo $VERBOSE_BAR
+echo
 
 echo
-echo "Installing PX4 general dependencies"
+echo $VERBOSE_BAR
+echo "🍻 Installing System Dependencies"
+echo $VERBOSE_BAR
+echo
 
 sudo apt-get update -y --quiet
 sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \
@@ -85,6 +99,7 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends i
 	gdb \
 	git \
 	lcov \
+	libssl-dev \
 	libxml2-dev \
 	libxml2-utils \
 	make \
@@ -100,32 +115,42 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends i
 	zip \
 	;
 
-# Python3 dependencies
+# Python 3 dependencies
 echo
-echo "Installing PX4 Python3 dependencies"
+echo $VERBOSE_BAR
+echo "🍻 Installing Python dependencies"
+echo $VERBOSE_BAR
+echo
+
 if [ -n "$VIRTUAL_ENV" ]; then
 	# virtual environments don't allow --user option
 	python -m pip install -r ${DIR}/requirements.txt
 else
 	# older versions of Ubuntu require --user option
-	python3 -m pip install --user -r ${DIR}/requirements.txt
+	if [[ $INSIDE_DOCKER == "true" ]]; then
+		# when running inside a docker container we don't need to install
+		# under --user since the installer user is root
+		# its best to install packages globaly for any user to find
+		python3 -m pip install -r /tmp/requirements.txt
+	else
+		python3 -m pip install --user -r ${DIR}/requirements.txt
+	fi
 fi
 
 # NuttX toolchain (arm-none-eabi-gcc)
 if [[ $INSTALL_NUTTX == "true" ]]; then
 
 	echo
-	echo "Installing NuttX dependencies"
+	echo $VERBOSE_BAR
+	echo "🍻 Installing NuttX dependencies"
+	echo $VERBOSE_BAR
+	echo
 
 	sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \
 		automake \
 		binutils-dev \
 		bison \
-		build-essential \
 		flex \
-		g++-multilib \
-		gcc-multilib \
-		gdb-multiarch \
 		genromfs \
 		gettext \
 		gperf \
@@ -145,7 +170,12 @@ if [[ $INSTALL_NUTTX == "true" ]]; then
 		u-boot-tools \
 		util-linux \
 		vim-common \
+		g++-arm-linux-gnueabihf \
+		gcc-arm-linux-gnueabihf \
+		g++-aarch64-linux-gnu \
+		gcc-aarch64-linux-gnu \
 		;
+
 	if [[ "${UBUNTU_RELEASE}" == "20.04" ]]; then
 		sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \
 		kconfig-frontends \
@@ -158,32 +188,47 @@ if [[ $INSTALL_NUTTX == "true" ]]; then
 		sudo usermod -a -G dialout $USER
 	fi
 
-	# arm-none-eabi-gcc
 	NUTTX_GCC_VERSION="9-2020-q2-update"
 	NUTTX_GCC_VERSION_SHORT="9-2020q2"
+	echo
+	echo $VERBOSE_BAR
+	echo "🍻 Verifying proper gcc version (${NUTTX_GCC_VERSION}), and installing if not found"
+	echo
 
 	source $HOME/.profile # load changed path for the case the script is reran before relogin
 	if [ $(which arm-none-eabi-gcc) ]; then
 		GCC_VER_STR=$(arm-none-eabi-gcc --version)
-		GCC_FOUND_VER=$(echo $GCC_VER_STR | grep -c "${NUTTX_GCC_VERSION}")
+		GCC_VER_FOUND=$(echo $GCC_VER_STR | grep -c "${NUTTX_GCC_VERSION}")
 	fi
 
-	if [[ "$GCC_FOUND_VER" == "1" ]]; then
-		echo "arm-none-eabi-gcc-${NUTTX_GCC_VERSION} found, skipping installation"
+	if [[ $(echo $GCC_VER_STR | grep -c "${NUTTX_GCC_VERSION}") == "1" ]]; then
+		echo "📌 Skipping installation, the arm cross compiler was found"
+		echo $VERBOSE_BAR
+		echo
 
 	else
-		echo "Installing arm-none-eabi-gcc-${NUTTX_GCC_VERSION}";
-		wget -O /tmp/gcc-arm-none-eabi-${NUTTX_GCC_VERSION}-linux.tar.bz2 https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu-rm/${NUTTX_GCC_VERSION_SHORT}/gcc-arm-none-eabi-${NUTTX_GCC_VERSION}-${INSTALL_ARCH}-linux.tar.bz2 && \
-			sudo tar -jxf /tmp/gcc-arm-none-eabi-${NUTTX_GCC_VERSION}-linux.tar.bz2 -C /opt/;
+		echo "📌 The arm cross compiler was not found";
+		echo " * Installing arm-none-eabi-gcc-${NUTTX_GCC_VERSION}";
+		COMPILER_NAME="gcc-arm-none-eabi-${NUTTX_GCC_VERSION}"
+		COMPILER_PATH="/tmp/$COMPILER_NAME-linux.tar.bz2"
+    wget -O $COMPILER_PATH https://armkeil.blob.core.windows.net/developer/Files/downloads/gnu-rm/${NUTTX_GCC_VERSION_SHORT}/${COMPILER_NAME}-${INSTALL_ARCH}-linux.tar.bz2
+		sudo tar -jxf $COMPILER_PATH -C /opt/;
 
 		# add arm-none-eabi-gcc to user's PATH
-		exportline="export PATH=/opt/gcc-arm-none-eabi-${NUTTX_GCC_VERSION}/bin:\$PATH"
-
-		if grep -Fxq "$exportline" $HOME/.profile; then
+		exportline="export PATH=\"/opt/${COMPILER_NAME}/bin:\$PATH\""
+		if [[ $INSIDE_DOCKER == "true" ]]; then
+			# when running on a docker container its best to set the environment globally
+			# since we don't know which user is going to be running commands on the container
+			touch /etc/profile.d/px4env.sh
+			echo $exportline >> /etc/profile.d/px4env.sh
+		elif grep -Fxq "$exportline" $HOME/.profile; then
 			echo "${NUTTX_GCC_VERSION} path already set.";
 		else
 			echo $exportline >> $HOME/.profile;
 		fi
+		echo " * arm-none-eabi-gcc (${NUTTX_GCC_VERSION}) Installed Succesful to /opt/${COMPILER_NAME}/bin"
+		echo $VERBOSE_BAR
+		echo
 	fi
 fi
 
@@ -191,45 +236,32 @@ fi
 if [[ $INSTALL_SIM == "true" ]]; then
 
 	echo
-	echo "Installing PX4 simulation dependencies"
+	echo $VERBOSE_BAR
+	echo "🍻 Installing PX4 Simulation Tools"
+	echo
+
+	# default and Ubuntu 20.04
+	gazebo_version=11
+	gazebo_packages="gazebo$gazebo_version libgazebo$gazebo_version-dev"
+	if [[ "${UBUNTU_RELEASE}" == "18.04" ]]; then
+		gazebo_version=9
+	elif [[ "${UBUNTU_RELEASE}" == "22.04" ]]; then
+		gazebo_version=11
+		gazebo_packages="gazebo libgazebo-dev"
+	fi
+
+	echo "  * Gazebo Version $gazebo_version"
+	echo $VERBOSE_BAR
 
 	# General simulation dependencies
 	sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \
 		bc \
-		;
-
-	if [[ "${UBUNTU_RELEASE}" == "18.04" ]]; then
-		java_version=11
-	elif [[ "${UBUNTU_RELEASE}" == "20.04" ]]; then
-		java_version=13
-	elif [[ "${UBUNTU_RELEASE}" == "22.04" ]]; then
-		java_version=11
-	else
-		java_version=14
-	fi
-	# Java (jmavsim or fastrtps)
-	sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends install \
 		ant \
-		openjdk-$java_version-jre \
-		openjdk-$java_version-jdk \
 		libvecmath-java \
 		;
 
-	# Set Java 11 as default
-	sudo update-alternatives --set java $(update-alternatives --list java | grep "java-$java_version")
-
-	# Gazebo
-	if [[ "${UBUNTU_RELEASE}" == "18.04" ]]; then
-		gazebo_version=9
-		gazebo_packages="gazebo$gazebo_version libgazebo$gazebo_version-dev"
-	elif [[ "${UBUNTU_RELEASE}" == "22.04" ]]; then
-		gazebo_packages="gazebo libgazebo-dev"
-	else
-		# default and Ubuntu 20.04
-		gazebo_version=11
-		gazebo_packages="gazebo$gazebo_version libgazebo$gazebo_version-dev"
-	fi
-
+	# Installing Gazebo and dependencies
+	# Setup OSRF Gazebo repository
 	sudo sh -c 'echo "deb http://packages.osrfoundation.org/gazebo/ubuntu-stable `lsb_release -cs` main" > /etc/apt/sources.list.d/gazebo-stable.list'
 	wget http://packages.osrfoundation.org/gazebo.key -O - | sudo apt-key add -
 	# Update list, since new gazebo-stable.list has been added
@@ -257,7 +289,26 @@ if [[ $INSTALL_SIM == "true" ]]; then
 	fi
 fi
 
-if [[ $INSTALL_NUTTX == "true" ]]; then
-	echo
-	echo "Relogin or reboot computer before attempting to build NuttX targets"
+if [[ $INSIDE_DOCKER == "true" ]]; then
+	# cleanup installation
+	rm -rf /tmp/
 fi
+
+if [[ $INSIDE_DOCKER == "false" ]] && [[ $INSTALL_NUTTX == "true" ]]; then
+	echo
+	echo $VERBOSE_BAR
+	echo "💡 We recommend you relogin/reboot before attempting to build NuttX targets"
+	echo $VERBOSE_BAR
+	echo
+fi
+
+echo
+echo
+echo $VERBOSE_BAR
+echo "#⚡️ PX4 Dependency Installer Ended Succesfully
+#
+#  For more information on PX4 Autopilot check out our docs
+#  at docs.px4.io, if you find a bug please file an issue
+#  on GitHub https://github.com/px4/px4-autopilot"
+echo $VERBOSE_BAR
+echo