From 1a92ae05c1eae42a675330705ecb27714b306397 Mon Sep 17 00:00:00 2001 From: Ramon Roche Date: Thu, 5 Jun 2025 10:24:24 -0700 Subject: [PATCH] ci: check for metadata changes Signed-off-by: Ramon Roche --- .github/workflows/docs_metadata_check.yml | 49 ++++++++++ Tools/ci/metadata_airframe.sh | 68 +++++++++++++ Tools/ci/metadata_failsafe_web.sh | 112 ++++++++++++++++++++++ Tools/ci/metadata_modules.sh | 99 +++++++++++++++++++ Tools/ci/metadata_msg_docs.sh | 84 ++++++++++++++++ Tools/ci/metadata_parameters.sh | 71 ++++++++++++++ Tools/ci/metadata_uorb_graph.sh | 108 +++++++++++++++++++++ 7 files changed, 591 insertions(+) create mode 100644 .github/workflows/docs_metadata_check.yml create mode 100755 Tools/ci/metadata_airframe.sh create mode 100755 Tools/ci/metadata_failsafe_web.sh create mode 100755 Tools/ci/metadata_modules.sh create mode 100755 Tools/ci/metadata_msg_docs.sh create mode 100755 Tools/ci/metadata_parameters.sh create mode 100755 Tools/ci/metadata_uorb_graph.sh diff --git a/.github/workflows/docs_metadata_check.yml b/.github/workflows/docs_metadata_check.yml new file mode 100644 index 0000000000..b9a0ad3572 --- /dev/null +++ b/.github/workflows/docs_metadata_check.yml @@ -0,0 +1,49 @@ +name: Docs Metadata Checks + +permissions: + contents: write + +on: + pull_request: {} + push: + branches: + - main + - 'release/**' + +jobs: + metadata-check: + name: ${{ matrix.name }} metadata + runs-on: [runs-on,runner=2cpu-linux-x64,image=ubuntu24-full-x64,"run-id=${{ github.run_id }}",spot=true] + container: + image: ghcr.io/px4/px4-dev:v1.16.0-ondemand + strategy: + fail-fast: false + matrix: + include: + - name: uORB Graphs + script: Tools/ci/metadata_uorb_graph.sh + # - name: Failsafe Web + # script: Tools/ci/metadata_failsafe_web.sh + - name: uORB Messages + script: Tools/ci/metadata_msg_docs.sh + - name: Parameter Reference + script: Tools/ci/metadata_parameters.sh + - name: Airframe Reference + script: Tools/ci/metadata_airframe.sh + - name: Module Reference + script: Tools/ci/metadata_modules.sh + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: true + + - name: Mark all directories safe for Git + run: git config --system --add safe.directory '*' + + - name: Run ${{ matrix.name }} metadata check + run: ${{ matrix.script }} --test-only + + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 diff --git a/Tools/ci/metadata_airframe.sh b/Tools/ci/metadata_airframe.sh new file mode 100755 index 0000000000..42793527ad --- /dev/null +++ b/Tools/ci/metadata_airframe.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# +# metadata_airframe.sh โ€” generate and sync PX4 airframe reference documentation +# +# Usage: +# Tools/ci/metadata_airframe.sh [--test-only] [--debug] +# +# Options: +# --test-only Run make target and comparison; exit 1 if diffs found, without copying file +# --debug Show full make output and debug info for comparison +# +set -euo pipefail +shopt -s nullglob + +# Parse flags +test_only=false +debug=false +while [[ $# -gt 0 ]]; do + case "$1" in + --test-only) test_only=true; shift ;; + --debug) debug=true; shift ;; + *) echo "Usage: $0 [--test-only] [--debug]"; exit 2 ;; + esac +done + +# Paths and make target +make_target="airframe_metadata" +src_file="build/px4_sitl_default/docs/airframes.md" +dest_file="docs/en/airframes/airframe_reference.md" + +# Run make target +if [ "$debug" = true ]; then + echo "๐Ÿ”ง Running 'make $make_target' (verbose)" + make $make_target +else + echo "๐Ÿ”ง Running 'make $make_target'" + make $make_target > /dev/null 2>&1 +fi + +# Verify build output +if [[ ! -f "$src_file" ]]; then + echo "โŒ Generated file not found: $src_file" + exit 1 +fi + +echo "๐Ÿ” Comparing airframe reference docs" + +# Compare files +if cmp -s "$src_file" "$dest_file"; then + echo "โœ… Airframe reference is up to date." + exit 0 +else + if [ "$debug" = true ]; then + echo "DEBUG: cmp -s '$src_file' '$dest_file'; echo \$?" + fi + echo "โš ๏ธ Airframe reference needs updating." + if [ "$test_only" = true ]; then + exit 1 + fi + # Copy over updated file + echo "๐Ÿ“‚ Copying updated airframe_reference.md" + cp -v "$src_file" "$dest_file" + echo "๐Ÿšจ Airframe docs updated; commit the change:" + echo " git status -s $dest_file" + echo " git add $dest_file" + echo " git commit -m 'docs: update airframe reference metadata'" + exit 1 +fi diff --git a/Tools/ci/metadata_failsafe_web.sh b/Tools/ci/metadata_failsafe_web.sh new file mode 100755 index 0000000000..0cb4a5bca5 --- /dev/null +++ b/Tools/ci/metadata_failsafe_web.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# +# metadata_failsafe_web.sh โ€” build and sync failsafe webapp metadata files +# +# Usage: +# Tools/ci/metadata_failsafe_web.sh [--test-only] [--debug] +# +# Options: +# --test-only Run build and comparison; exit 1 if diffs found, without copying files +# --debug Show full build output and debug info for file comparisons and EMSDK install +# +set -euo pipefail +shopt -s nullglob + +# Parse flags +test_only=false +debug=false +while [[ $# -gt 0 ]]; do + case "$1" in + --test-only) test_only=true; shift ;; + --debug) debug=true; shift ;; + *) echo "Usage: $0 [--test-only] [--debug]"; exit 2 ;; + esac +done + +# Paths and commands +build_cmd="make failsafe_web" +src_dir="build/px4_sitl_default_failsafe_web" +dest_dir="docs/public/config/failsafe" + +# Ensure Emscripten SDK is available +if ! command -v emcc >/dev/null 2>&1; then + echo "๐Ÿ”ง Emscripten not found. Ensuring EMSDK is installed." + # Clone SDK only if not already present + if [ ! -d "_emscripten_sdk" ]; then + if [ "$debug" = true ]; then + git clone https://github.com/emscripten-core/emsdk.git _emscripten_sdk + else + git clone https://github.com/emscripten-core/emsdk.git _emscripten_sdk > /dev/null 2>&1 + fi + fi + pushd _emscripten_sdk >/dev/null + if [ "$debug" = true ]; then + ./emsdk install latest + ./emsdk activate latest + else + ./emsdk install latest > /dev/null 2>&1 + ./emsdk activate latest > /dev/null 2>&1 + fi + popd >/dev/null + # Load environment into current shell + if [ "$debug" = true ]; then + # shellcheck source=/dev/null + . ./_emscripten_sdk/emsdk_env.sh + else + # shellcheck source=/dev/null + . ./_emscripten_sdk/emsdk_env.sh > /dev/null 2>&1 + fi +fi + +# Build step +if [ "$debug" = true ]; then + echo "๐Ÿ”ง Running build: $build_cmd" + $build_cmd +else + echo "๐Ÿ”ง Running build" + $build_cmd > /dev/null 2>&1 +fi + +# Gather built files +src_files=("$src_dir"/*.{js,wasm,json}) +if [ ${#src_files[@]} -eq 0 ]; then + echo "โŒ No generated files found in $src_dir. Build failed or path wrong." + exit 1 +fi + +# Prepare destination +echo "๐Ÿ” Checking failsafe web metadata" +mkdir -p "$dest_dir" + +changed=() +for src in "${src_files[@]}"; do + name=$(basename "$src") + dst="$dest_dir/$name" + + if [[ ! -f "$dst" ]]; then + [ "$debug" = true ] && echo "DEBUG: missing $dst" + changed+=("$name") + elif ! cmp -s "$src" "$dst"; then + [ "$debug" = true ] && echo "DEBUG: cmp -s '$src' '$dst'; echo \$?" + changed+=("$name") + fi +done + +if [ ${#changed[@]} -eq 0 ]; then + echo "โœ… All failsafe web metadata files are in sync." + exit 0 +fi + +echo "โš ๏ธ Detected updates in:" +for f in "${changed[@]}"; do echo " - $f"; done + +if [ "$test_only" = true ]; then + echo "๐Ÿšจ Failsafe web metadata needs update; rerun without --test-only to apply." + exit 1 +fi + +echo "๐Ÿ“‚ Copying updated files" +for f in "${changed[@]}"; do cp -v "$src_dir/$f" "$dest_dir/$f"; done + +echo "๐Ÿšจ Failsafe web metadata updated; please commit changes." +exit 1 diff --git a/Tools/ci/metadata_modules.sh b/Tools/ci/metadata_modules.sh new file mode 100755 index 0000000000..9c83cd324c --- /dev/null +++ b/Tools/ci/metadata_modules.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# +# metadata_modules.sh - generate and sync PX4 module reference documentation +# +# Usage: +# Tools/ci/metadata_modules.sh [--test-only] [--debug] +# +# Options: +# --test-only Run make target and comparison; exit 1 if diffs found, without copying files +# --debug Show full make output and debug info for file comparisons +# +set -euo pipefail +shopt -s nullglob + +# Parse flags +test_only=false +debug=false +while [[ $# -gt 0 ]]; do + case "$1" in + --test-only) test_only=true; shift ;; + --debug) debug=true; shift ;; + *) echo "Usage: $0 [--test-only] [--debug]"; exit 2 ;; + esac +done + +# Paths and make target +make_target="module_documentation" +src_dir="build/px4_sitl_default/docs/modules" +dest_dir="docs/en/modules" + +# Run make target +if [ "$debug" = true ]; then + echo "๐Ÿ”ง Running 'make $make_target' (verbose)" + make $make_target +else + echo "๐Ÿ”ง Running 'make $make_target'" + make $make_target > /dev/null 2>&1 +fi + +# Verify build output +src_files=("$src_dir"/*) +if [ ${#src_files[@]} -eq 0 ]; then + echo "โŒ No generated module docs found in $src_dir. Build failed or path wrong." + exit 1 +fi + +# โ”€โ”€โ”€ Strip trailing whitespace from all generated module docs (unless test-only) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +if [ "$test_only" = false ]; then + echo "โœ‚๏ธ Removing trailing whitespace from generated module docs" + for src in "${src_files[@]}"; do + sed -i 's/[[:space:]]\+$//' "$src" + done +else + [ "$debug" = true ] && echo "๐Ÿงช Test-only mode: skipping whitespace removal" +fi + +echo "๐Ÿ” Checking module reference docs in $dest_dir" +mkdir -p "$dest_dir" + +changed=() +for src in "${src_files[@]}"; do + name=$(basename "$src") + dst="$dest_dir/$name" + + if [[ ! -e "$dst" ]]; then + [ "$debug" = true ] && echo "DEBUG: missing $dst" + changed+=("$name") + else + # Use diff -q -b (ignore whitespace changes) and --strip-trailing-cr (ignore CRLF vs LF) + if ! diff -q -b --strip-trailing-cr "$src" "$dst" > /dev/null; then + [ "$debug" = true ] && echo "DEBUG: diff -q -b --strip-trailing-cr '$src' '$dst' (exit_code=$?)" + changed+=("$name") + fi + fi +done + +if [ ${#changed[@]} -eq 0 ]; then + echo "โœ… All module reference docs are up to date." + exit 0 +fi + +echo "โš ๏ธ Detected updates in module docs:" +for f in "${changed[@]}"; do echo " - $f"; done + +if [ "$test_only" = true ]; then + echo "๐Ÿšจ Module reference docs need updating; rerun without --test-only to apply." + exit 1 +fi + +echo "๐Ÿ“‚ Copying updated module docs to $dest_dir" +for f in "${changed[@]}"; do + cp -rv "$src_dir/$f" "$dest_dir/$f" +done + +echo "๐Ÿšจ Module reference docs updated; please commit changes:" +echo " git status -s $dest_dir" +echo " git add $dest_dir/*" +echo " git commit -m 'docs: update module reference metadata'" +exit 1 diff --git a/Tools/ci/metadata_msg_docs.sh b/Tools/ci/metadata_msg_docs.sh new file mode 100755 index 0000000000..bc1c7e42ad --- /dev/null +++ b/Tools/ci/metadata_msg_docs.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# metadata_msg_docs.sh โ€” generate and sync uORB message reference documentation +# +# Usage: +# Tools/ci/metadata_msg_docs.sh [--test-only] [--debug] +# +# Options: +# --test-only Run make target and comparison; exit 1 if diffs found, without copying files +# --debug Show full make output and debug info for file comparisons +# +set -euo pipefail +shopt -s nullglob + +# Parse flags +test_only=false +debug=false +while [[ $# -gt 0 ]]; do + case "$1" in + --test-only) test_only=true; shift ;; + --debug) debug=true; shift ;; + *) echo "Usage: $0 [--test-only] [--debug]"; exit 2 ;; + esac +done + +# Paths and make target +make_target="msg_docs" +src_dir="build/msg_docs" +dest_dir="docs/en/msg_docs" + +# Run make target +if [ "$debug" = true ]; then + echo "๐Ÿ”ง Running 'make $make_target' (verbose)" + make $make_target +else + echo "๐Ÿ”ง Running 'make $make_target'" + make $make_target > /dev/null 2>&1 +fi + +# Verify build output +src_files=("$src_dir"/*) +if [ ${#src_files[@]} -eq 0 ]; then + echo "โŒ No files found in $src_dir. Build target '$make_target' failed or path is wrong." + exit 1 +fi + +echo "๐Ÿ” Checking uORB message docs in $dest_dir" +mkdir -p "$dest_dir" + +changed=() +for src in "${src_files[@]}"; do + name=$(basename "$src") + dst="$dest_dir/$name" + + if [[ ! -f "$dst" ]]; then + [ "$debug" = true ] && echo "DEBUG: missing $dst" + changed+=("$name") + elif ! cmp -s "$src" "$dst"; then + [ "$debug" = true ] && echo "DEBUG: cmp -s '$src' '$dst'; echo \$?" + changed+=("$name") + fi +done + +if [ ${#changed[@]} -eq 0 ]; then + echo "โœ… All uORB message docs are up to date." + exit 0 +fi + +echo "โš ๏ธ Detected updates in the following docs:" +for f in "${changed[@]}"; do echo " - $f"; done + +if [ "$test_only" = true ]; then + echo "๐Ÿšจ uORB message docs need updating! Rerun without --test-only to apply changes." + exit 1 +fi + +echo "๐Ÿ“‚ Copying updated doc files to $dest_dir" +for f in "${changed[@]}"; do cp -v "$src_dir/$f" "$dest_dir/$f"; done + +echo "๐Ÿšจ uORB message docs updated; please commit changes:" +echo " git status -s $dest_dir" +echo " git add $dest_dir/*" +echo " git commit -m 'docs: update uORB message reference docs'" +exit 1 diff --git a/Tools/ci/metadata_parameters.sh b/Tools/ci/metadata_parameters.sh new file mode 100755 index 0000000000..35c2f98851 --- /dev/null +++ b/Tools/ci/metadata_parameters.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# +# metadata_parameters.sh โ€” generate and sync PX4 parameter reference documentation +# +# Usage: +# Tools/ci/metadata_parameters.sh [--test-only] [--debug] +# +# Options: +# --test-only Run make target and comparison; exit 1 if diffs found, without copying file +# --debug Show full make output and debug info for comparison +# +set -euo pipefail + +# Parse flags +test_only=false +debug=false +while [[ $# -gt 0 ]]; do + case "$1" in + --test-only) test_only=true; shift ;; + --debug) debug=true; shift ;; + *) echo "Usage: $0 [--test-only] [--debug]"; exit 2 ;; + esac +done + +# Paths and make target +make_target="parameters_metadata" +src_file="build/px4_sitl_default/docs/parameters.md" +dest_file="docs/en/advanced_config/parameter_reference.md" + +# Run make target +if [ "$debug" = true ]; then + echo "๐Ÿ”ง Running 'make $make_target' (verbose)" + make $make_target +else + echo "๐Ÿ”ง Running 'make $make_target'" + make $make_target > /dev/null 2>&1 +fi + +# โ”€โ”€โ”€ Strip trailing whitespace from the newly generated Markdown โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +echo "โœ‚๏ธ Removing trailing whitespace from $src_file" +sed -i 's/[[:space:]]\+$//' "$src_file" + +# Verify build output +if [[ ! -f "$src_file" ]]; then + echo "โŒ Generated file not found: $src_file" + exit 1 +fi + +echo "๐Ÿ” Comparing parameter docs" + +# Compare files +if cmp -s "$src_file" "$dest_file"; then + echo "โœ… Parameter reference is up to date." + exit 0 +else + if [ "$debug" = true ]; then + echo "DEBUG: cmp -s '$src_file' '$dest_file'; echo \$?" + fi + echo "โš ๏ธ Parameter reference needs updating." + if [ "$test_only" = true ]; then + exit 1 + fi + # Copy over updated file + echo "๐Ÿ“‚ Copying updated parameter_reference.md" + cp -v "$src_file" "$dest_file" + echo "๐Ÿšจ Parameter docs updated; commit the change:" + echo " git status -s $dest_file" + echo " git add $dest_file" + echo " git commit -m 'docs: update parameter reference metadata'" + exit 1 +fi diff --git a/Tools/ci/metadata_uorb_graph.sh b/Tools/ci/metadata_uorb_graph.sh new file mode 100755 index 0000000000..336b22bd20 --- /dev/null +++ b/Tools/ci/metadata_uorb_graph.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# +# update_uorb_graphs.sh โ€” generate, compare, and sync uORB graph JSONs +# +# Usage: +# ./scripts/update_uorb_graphs.sh [--test-only] [--debug] +# +# Options: +# --test-only Run generation and comparison only; exit 1 if diffs found, without copying files +# --debug Echo debug info for missing or differing files and show full make output +# +# Examples: +# # CI mode: fail if docs need updates +# ./scripts/update_uorb_graphs.sh --test-only +# +# # Developer mode: regenerate and sync JSONs +# ./scripts/update_uorb_graphs.sh +# +set -euo pipefail + +shopt -s nullglob + +# Parse flags +test_only=false +debug=false +while [[ $# -gt 0 ]]; do + case "$1" in + --test-only) + test_only=true + shift + ;; + --debug) + debug=true + shift + ;; + *) + echo "Usage: $0 [--test-only] [--debug]" + exit 2 + ;; + esac +done + +# Paths +graph_dir="Tools/uorb_graph" +dest_dir="docs/public/middleware" + +# Generate uORB graphs (conditionally silent) +if [ "$debug" = true ]; then + echo "๐Ÿ”ง Generating uORB message graphs (verbose output)" + make uorb_graphs +else + echo "๐Ÿ”ง Generating uORB message graphs" + make uorb_graphs > /dev/null 2>&1 +fi + +# Verify generation +src_files=("$graph_dir"/*.json) +if [ ${#src_files[@]} -eq 0 ]; then + echo "โŒ No JSON files found in $graph_dir. Generation failed or path is wrong." + exit 1 +fi + +echo "๐Ÿ” Checking for updated uORB graph JSONs" +mkdir -p "$dest_dir" + +changed=() +for src in "${src_files[@]}"; do + name=$(basename "$src") + dst="$dest_dir/$name" + + if [[ ! -f "$dst" ]]; then + [ "$debug" = true ] && echo "DEBUG: $dst missing" + changed+=("$name") + + elif ! cmp -s "$src" "$dst"; then + [ "$debug" = true ] && echo "DEBUG: cmp -s '$src' '$dst'; echo \$?" + changed+=("$name") + fi +done + +if [ ${#changed[@]} -eq 0 ]; then + echo "โœ… All uORB graph JSONs are already in sync." + exit 0 +fi + +echo "โš ๏ธ Detected updates in the following files:" +for name in "${changed[@]}"; do + echo " - $name" +done + +if [ "$test_only" = true ]; then + echo + echo "๐Ÿšจ uORB graph docs need updating! Rerun without --test-only to apply changes." + exit 1 +fi + +echo +echo "๐Ÿ“‚ Copying updated files over" +for name in "${changed[@]}"; do + cp -v "$graph_dir/$name" "$dest_dir/$name" +done + +echo +echo "๐Ÿšจ uORB graph docs need updating! Review above, then run:" +echo " git status -s $dest_dir/" +echo " git add $dest_dir/*.json" +echo " git commit -m 'docs: metadata: update uORB graph JSONs'" +exit 1