From 81e4e33811f5bff7c4eec95d8a901924fee86747 Mon Sep 17 00:00:00 2001 From: Julian Oes Date: Tue, 3 Feb 2026 09:27:01 +1300 Subject: [PATCH] feat(tools): ship metadata, IO and BL files on SD card This pushes the metadata files as well as the IO firmware and the bootloader binary to the SD card. That way, the files are don't have to be added to the firmware binary or served via s3 for FLASH_CONSTRAINED targets. The files are pushed after upload and verified in commander as part of the preflight checks. If missing, a warning is displayed and arming is prevented. This means that an SD card can't be swapped out without reflashing (or copying over the contents). --- ROMFS/CMakeLists.txt | 31 ++- Tools/filepaths/generate_config.py | 6 +- Tools/filepaths/rc.filepaths.jinja | 3 + Tools/px4_sdcard_upload.py | 223 ++++++++++++++++++ boards/px4/fmu-v6x/default.px4board | 1 + cmake/kconfig.cmake | 2 +- platforms/nuttx/cmake/upload.cmake | 76 ++++-- .../nuttx/init/rc.board_bootloader_upgrade.in | 4 +- src/lib/CMakeLists.txt | 1 + src/lib/component_information/CMakeLists.txt | 102 +++++++- src/lib/sdcard_check/CMakeLists.txt | 26 ++ .../sdcard_check/generate_sdcard_checksums.py | 83 +++++++ src/lib/sdcard_check/sdcard_check.cpp | 52 ++++ src/lib/sdcard_check/sdcard_check.h | 14 ++ .../HealthAndArmingChecks/CMakeLists.txt | 1 + .../checks/sdcardCheck.cpp | 57 ++++- .../checks/sdcardCheck.hpp | 9 +- src/modules/commander/commander_params.yaml | 8 + 18 files changed, 665 insertions(+), 34 deletions(-) create mode 100755 Tools/px4_sdcard_upload.py create mode 100644 src/lib/sdcard_check/CMakeLists.txt create mode 100644 src/lib/sdcard_check/generate_sdcard_checksums.py create mode 100644 src/lib/sdcard_check/sdcard_check.cpp create mode 100644 src/lib/sdcard_check/sdcard_check.h diff --git a/ROMFS/CMakeLists.txt b/ROMFS/CMakeLists.txt index 9b282339fd..0aef4fd6c4 100644 --- a/ROMFS/CMakeLists.txt +++ b/ROMFS/CMakeLists.txt @@ -136,6 +136,14 @@ add_custom_command( ) +# Compute IO firmware path for filepaths generator +set(iofw_path_arg) +if(CONFIG_BOARD_IO) + if(CONFIG_BOARD_ROOT_PATH) + set(iofw_path_arg --iofw-path ${CONFIG_BOARD_ROOT_PATH}/px4/extras/${CONFIG_BOARD_IO}.bin) + endif() +endif() + set(romfs_copy_stamp ${CMAKE_CURRENT_BINARY_DIR}/romfs_copy.stamp) add_custom_command( OUTPUT @@ -157,6 +165,7 @@ add_custom_command( COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/Tools/filepaths/generate_config.py --rc-dir ${romfs_gen_root_dir}/init.d --params-file ${CONFIG_BOARD_PARAM_FILE} + ${iofw_path_arg} COMMAND ${CMAKE_COMMAND} -E touch ${romfs_copy_stamp} WORKING_DIRECTORY ${romfs_gen_root_dir} DEPENDS ${romfs_tar_file} @@ -255,7 +264,8 @@ set(OPTIONAL_BOARD_EXTRAS) file(GLOB OPTIONAL_BOARD_EXTRAS ${PX4_BOARD_DIR}/extras/*) # bootloader (optional) -# - if systemcmds/bl_update included and board bootloader available then generate rc.board_bootloader_upgrade and copy bootloader binary +# - if systemcmds/bl_update included and board bootloader available then generate rc.board_bootloader_upgrade +# - when CONFIG_BOARD_ROOT_PATH is set, bootloader binary is on SD card (not in ROMFS) # - otherwise remove bootloader binary from extras in final ROMFS foreach(board_extra_file ${OPTIONAL_BOARD_EXTRAS}) file(RELATIVE_PATH extra_file_base_name ${PX4_BOARD_DIR}/extras/ ${board_extra_file}) @@ -263,6 +273,13 @@ foreach(board_extra_file ${OPTIONAL_BOARD_EXTRAS}) if(CONFIG_SYSTEMCMDS_BL_UPDATE) # generate rc.board_bootloader_upgrade set(BOARD_FIRMWARE_BIN "${PX4_BOARD_VENDOR}_${PX4_BOARD_MODEL}_bootloader.bin") + + if(CONFIG_BOARD_ROOT_PATH) + set(BOARD_FIRMWARE_PATH "${CONFIG_BOARD_ROOT_PATH}/px4/extras/${BOARD_FIRMWARE_BIN}") + else() + set(BOARD_FIRMWARE_PATH "/etc/extras/${BOARD_FIRMWARE_BIN}") + endif() + message(STATUS "ROMFS: Adding platforms/nuttx/init/rc.board_bootloader_upgrade -> /etc/init.d/rc.board_bootloader_upgrade") # Generate the file using configure_file at configure time to a temporary location @@ -286,13 +303,19 @@ foreach(board_extra_file ${OPTIONAL_BOARD_EXTRAS}) list(APPEND extras_dependencies rc.board_bootloader_upgrade.stamp ) - else() - # remove bootloader from extras - list(REMOVE_ITEM OPTIONAL_BOARD_EXTRAS ${board_extra_file}) endif() + + # remove bootloader binary from ROMFS extras (either on SD card or bl_update not enabled) + list(REMOVE_ITEM OPTIONAL_BOARD_EXTRAS ${board_extra_file}) endif() endforeach() +# When CONFIG_BOARD_ROOT_PATH is set, IO firmware is on SD card - remove from ROMFS extras +if(CONFIG_BOARD_ROOT_PATH AND CONFIG_BOARD_IO) + set(_io_bin_path "${PX4_BOARD_DIR}/extras/${CONFIG_BOARD_IO}.bin") + list(REMOVE_ITEM OPTIONAL_BOARD_EXTRAS ${_io_bin_path}) +endif() + foreach(board_extra_file ${OPTIONAL_BOARD_EXTRAS}) if(EXISTS "${board_extra_file}") diff --git a/Tools/filepaths/generate_config.py b/Tools/filepaths/generate_config.py index d4aa8b6d26..88cab8d3d2 100755 --- a/Tools/filepaths/generate_config.py +++ b/Tools/filepaths/generate_config.py @@ -35,6 +35,10 @@ parser.add_argument('--rc-dir', type=str, action='store', help='ROMFS output directory', default=None) parser.add_argument('--params-file', type=str, action='store', help='Parameter output file', default=None) +parser.add_argument('--iofw-path', type=str, action='store', + help='IO firmware binary path', default=None) +parser.add_argument('--bootloader-path', type=str, action='store', + help='Bootloader binary path', default=None) parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='Verbose Output') @@ -56,6 +60,6 @@ if rc_filepaths_output_dir is not None: if verbose: print("Generating {:}".format(rc_filepath_output_file)) template = jinja_env.get_template(rc_filepaths_template) with open(rc_filepath_output_file, 'w') as fid: - fid.write(template.render(constrained_flash=constrained_flash, params_file=args.params_file)) + fid.write(template.render(constrained_flash=constrained_flash, params_file=args.params_file, iofw_path=args.iofw_path, bootloader_path=args.bootloader_path)) else: raise Exception("--rc-dir needs to be specified") diff --git a/Tools/filepaths/rc.filepaths.jinja b/Tools/filepaths/rc.filepaths.jinja index 30a2c2cce6..259e6a8a9f 100644 --- a/Tools/filepaths/rc.filepaths.jinja +++ b/Tools/filepaths/rc.filepaths.jinja @@ -4,3 +4,6 @@ set PARAM_FILE {{ params_file }} +{% if iofw_path %} +set IOFW "{{ iofw_path }}" +{% endif %} diff --git a/Tools/px4_sdcard_upload.py b/Tools/px4_sdcard_upload.py new file mode 100755 index 0000000000..5c2a4f0f6d --- /dev/null +++ b/Tools/px4_sdcard_upload.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Upload files to PX4 SD card via MAVLink FTP. + +Pushes a local directory tree to the device's SD card using MAVSDK's +FTP plugin. Intended to be used after firmware upload to push metadata +files (parameters, events, actuators) for constrained flash boards. + +Uses are_files_identical to skip files that already match on the device. + +Usage: + python3 px4_sdcard_upload.py --port serial:///dev/ttyACM0:57600 --sdcard-dir build/board/sdcard + python3 px4_sdcard_upload.py --sdcard-dir build/board/sdcard # auto-detect port + +Requires: mavsdk +""" + +import argparse +import asyncio +import glob +import os +import platform +import sys +import time + +from mavsdk import System +from mavsdk.ftp import FtpError + + +def detect_ports(): + """Try to auto-detect a PX4 serial port.""" + candidates = [] + system = platform.system() + + if system == "Linux": + candidates.extend(sorted(glob.glob("/dev/serial/by-id/usb-*PX4*"))) + candidates.extend(sorted(glob.glob("/dev/serial/by-id/usb-*px4*"))) + candidates.extend(sorted(glob.glob("/dev/ttyACM*"))) + elif system == "Darwin": + candidates.extend(sorted(glob.glob("/dev/cu.usbmodem*"))) + elif system == "Windows": + for i in range(10): + candidates.append(f"COM{i}") + + return candidates + + +def make_system_address(port, baudrate): + """Convert a device path to a MAVSDK system address string.""" + if "://" in port: + return port + return f"serial://{port}:{baudrate}" + + +async def list_remote_files(drone, remote_dir): + """Return set of filenames in remote_dir, or None if dir doesn't exist.""" + try: + result = await drone.ftp.list_directory(remote_dir) + return set(result.files) + except FtpError: + return None + + +async def ensure_remote_dirs(drone, remote_dir, device_root, created_dirs): + """Create remote directory hierarchy as needed.""" + if remote_dir in created_dirs: + return + suffix = remote_dir[len(device_root.rstrip("/")):] + parts = suffix.strip("/").split("/") + path = device_root.rstrip("/") + for part in parts: + if not part: + continue + path = path + "/" + part + if path not in created_dirs: + print(f" Creating directory: {path}") + try: + await drone.ftp.create_directory(path) + except FtpError: + pass # Already exists + created_dirs.add(path) + + +async def upload_directory(drone, local_dir, device_root): + """Walk local_dir and upload changed files to device_root on the device.""" + local_dir = os.path.abspath(local_dir) + upload_count = 0 + skip_count = 0 + fail_count = 0 + created_dirs = set() + remote_file_cache = {} # remote_dir -> set of filenames or None + + for dirpath, _dirnames, filenames in os.walk(local_dir): + for filename in filenames: + local_path = os.path.join(dirpath, filename) + rel_path = os.path.relpath(local_path, local_dir) + remote_path = device_root.rstrip("/") + "/" + rel_path.replace(os.sep, "/") + remote_dir = os.path.dirname(remote_path) + file_size = os.path.getsize(local_path) + + # List remote directory once to know which files exist + if remote_dir not in remote_file_cache: + remote_file_cache[remote_dir] = await list_remote_files( + drone, remote_dir) + if remote_file_cache[remote_dir] is not None: + created_dirs.add(remote_dir) + + remote_files = remote_file_cache[remote_dir] + + if remote_files is not None and filename in remote_files: + # File exists remotely, check CRC + print(f" {rel_path}: checking...", end="", flush=True) + try: + if await drone.ftp.are_files_identical( + local_path, remote_path): + print(" identical, skipped") + skip_count += 1 + continue + else: + print(f" changed, uploading ({file_size} bytes)...", + end="", flush=True) + except FtpError as e: + print(f" check failed ({e}), uploading ({file_size}" + f" bytes)...", end="", flush=True) + else: + # File doesn't exist remotely + await ensure_remote_dirs( + drone, remote_dir, device_root, created_dirs) + print(f" {rel_path}: new, uploading ({file_size} bytes)...", + end="", flush=True) + + try: + async for _progress in drone.ftp.upload(local_path, remote_dir): + pass + print(" done") + upload_count += 1 + except FtpError as e: + print(f" FAILED ({e})") + fail_count += 1 + + return upload_count, skip_count, fail_count + + +async def run(args): + print("Waiting for serial port...") + deadline = time.monotonic() + args.port_timeout + while time.monotonic() < deadline: + if args.port is not None: + ports = [args.port] + else: + ports = detect_ports() + if len(ports) == 0: + await asyncio.sleep(0.5) + continue + + for port in ports: + system_address = make_system_address(port, args.baudrate) + print(f"Connecting to {system_address} ...") + + try: + async with asyncio.timeout(1): + if args.mavsdk_server: + host, _, port_str = args.mavsdk_server.partition(":") + grpc_port = int(port_str) if port_str else 50051 + drone = System(mavsdk_server_address=host, port=grpc_port) + await drone.connect() + else: + drone = System() + await drone.connect(system_address=system_address) + + print("Waiting for connection...") + async for state in drone.core.connection_state(): + if state.is_connected: + print("Connected.") + break + except TimeoutError: + continue + + print(f"Checking {args.total_files} file(s) from {args.sdcard_dir} " + f"against {args.device_root} ...") + + uploaded, skipped, failed = await upload_directory( + drone, args.sdcard_dir, args.device_root) + + print(f"\nDone: {uploaded} uploaded, {skipped} skipped, {failed} failed " + f"(out of {args.total_files} files)") + if failed > 0: + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser( + description="Upload files to PX4 SD card via MAVLink FTP") + parser.add_argument("--port", default=None, + help="MAVLink connection string (e.g. serial:///dev/ttyACM0:57600, " + "udp://:14540). Auto-detected if not specified.") + parser.add_argument("--baudrate", type=int, default=57600, + help="Serial baud rate (default: 57600)") + parser.add_argument("--sdcard-dir", required=True, + help="Local directory tree to upload (mirrors SD card layout)") + parser.add_argument("--device-root", default="/fs/microsd", + help="SD card root path on device (default: /fs/microsd)") + parser.add_argument("--port-timeout", type=float, default=5, + help="Seconds to wait for serial port to appear (default: 5)") + parser.add_argument("--mavsdk-server", default=None, + help="Connect to external mavsdk_server at host:port " + "(e.g. localhost:50051) instead of starting one") + args = parser.parse_args() + + if not os.path.isdir(args.sdcard_dir): + print(f"Error: sdcard directory not found: {args.sdcard_dir}") + sys.exit(1) + + total_files = sum(len(files) for _, _, files in os.walk(args.sdcard_dir)) + if total_files == 0: + print("No files to upload.") + sys.exit(0) + + args.total_files = total_files + asyncio.run(run(args)) + + +if __name__ == "__main__": + main() diff --git a/boards/px4/fmu-v6x/default.px4board b/boards/px4/fmu-v6x/default.px4board index f5bc995d2c..2dd621696e 100644 --- a/boards/px4/fmu-v6x/default.px4board +++ b/boards/px4/fmu-v6x/default.px4board @@ -81,6 +81,7 @@ CONFIG_MODULES_TEMPERATURE_COMPENSATION=y CONFIG_MODULES_UXRCE_DDS_CLIENT=y CONFIG_MODULES_VTOL_ATT_CONTROL=y CONFIG_SYSTEMCMDS_ACTUATOR_TEST=y +CONFIG_SYSTEMCMDS_BL_UPDATE=y CONFIG_SYSTEMCMDS_BSONDUMP=y CONFIG_SYSTEMCMDS_DMESG=y CONFIG_SYSTEMCMDS_GPIO=y diff --git a/cmake/kconfig.cmake b/cmake/kconfig.cmake index e483ee6f0e..d333df581b 100644 --- a/cmake/kconfig.cmake +++ b/cmake/kconfig.cmake @@ -264,7 +264,7 @@ if(EXISTS ${BOARD_DEFCONFIG}) set(romfs_extra_files) set(config_romfs_extra_dependencies) # additional embedded metadata - if(NOT CONSTRAINED_FLASH AND NOT EXTERNAL_METADATA AND NOT ${PX4_BOARD_LABEL} STREQUAL "test") + if(NOT CONSTRAINED_FLASH AND NOT EXTERNAL_METADATA AND NOT ROOT_PATH AND NOT ${PX4_BOARD_LABEL} STREQUAL "test") list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/parameters.json.xz ${PX4_BINARY_DIR}/events/all_events.json.xz diff --git a/platforms/nuttx/cmake/upload.cmake b/platforms/nuttx/cmake/upload.cmake index 3bffd08535..f9ddb5471d 100644 --- a/platforms/nuttx/cmake/upload.cmake +++ b/platforms/nuttx/cmake/upload.cmake @@ -33,24 +33,68 @@ # Uploader script auto-detects PX4 devices by USB VID/PID set(PX4_UPLOADER_SCRIPT "${PX4_SOURCE_DIR}/Tools/px4_uploader.py") +set(PX4_SDCARD_UPLOAD_SCRIPT "${PX4_SOURCE_DIR}/Tools/px4_sdcard_upload.py") -add_custom_target(upload - COMMAND ${PYTHON_EXECUTABLE} ${PX4_UPLOADER_SCRIPT} ${fw_package} - DEPENDS ${fw_package} - COMMENT "uploading px4" - VERBATIM - USES_TERMINAL - WORKING_DIRECTORY ${PX4_BINARY_DIR} -) +if(CONFIG_BOARD_ROOT_PATH) -add_custom_target(force-upload - COMMAND ${PYTHON_EXECUTABLE} ${PX4_UPLOADER_SCRIPT} --force ${fw_package} - DEPENDS ${fw_package} - COMMENT "uploading px4 with --force" - VERBATIM - USES_TERMINAL - WORKING_DIRECTORY ${PX4_BINARY_DIR} -) + # Flash firmware, then push SD card files (metadata, bootloader, IO binary) + add_custom_target(upload + COMMAND ${PYTHON_EXECUTABLE} ${PX4_UPLOADER_SCRIPT} ${fw_package} + COMMAND ${PYTHON_EXECUTABLE} ${PX4_SDCARD_UPLOAD_SCRIPT} + --sdcard-dir ${PX4_BINARY_DIR}/sdcard + --device-root ${CONFIG_BOARD_ROOT_PATH} + DEPENDS ${fw_package} sdcard_files + COMMENT "uploading px4 and SD card files" + VERBATIM + USES_TERMINAL + WORKING_DIRECTORY ${PX4_BINARY_DIR} + ) + + add_custom_target(force-upload + COMMAND ${PYTHON_EXECUTABLE} ${PX4_UPLOADER_SCRIPT} --force ${fw_package} + COMMAND ${PYTHON_EXECUTABLE} ${PX4_SDCARD_UPLOAD_SCRIPT} + --sdcard-dir ${PX4_BINARY_DIR}/sdcard + --device-root ${CONFIG_BOARD_ROOT_PATH} + DEPENDS ${fw_package} sdcard_files + COMMENT "uploading px4 with --force and SD card files" + VERBATIM + USES_TERMINAL + WORKING_DIRECTORY ${PX4_BINARY_DIR} + ) + + # Standalone target: upload only SD card files (e.g. with --port) + add_custom_target(upload_sdcard + COMMAND ${PYTHON_EXECUTABLE} ${PX4_SDCARD_UPLOAD_SCRIPT} + --sdcard-dir ${PX4_BINARY_DIR}/sdcard + --device-root ${CONFIG_BOARD_ROOT_PATH} + DEPENDS sdcard_files + COMMENT "uploading SD card files via MAVLink FTP" + VERBATIM + USES_TERMINAL + WORKING_DIRECTORY ${PX4_BINARY_DIR} + ) + +else() + + add_custom_target(upload + COMMAND ${PYTHON_EXECUTABLE} ${PX4_UPLOADER_SCRIPT} ${fw_package} + DEPENDS ${fw_package} + COMMENT "uploading px4" + VERBATIM + USES_TERMINAL + WORKING_DIRECTORY ${PX4_BINARY_DIR} + ) + + add_custom_target(force-upload + COMMAND ${PYTHON_EXECUTABLE} ${PX4_UPLOADER_SCRIPT} --force ${fw_package} + DEPENDS ${fw_package} + COMMENT "uploading px4 with --force" + VERBATIM + USES_TERMINAL + WORKING_DIRECTORY ${PX4_BINARY_DIR} + ) + +endif() add_custom_target(upload-verbose COMMAND ${PYTHON_EXECUTABLE} ${PX4_UPLOADER_SCRIPT} --verbose ${fw_package} diff --git a/platforms/nuttx/init/rc.board_bootloader_upgrade.in b/platforms/nuttx/init/rc.board_bootloader_upgrade.in index 23ea208507..ab4a541bdc 100644 --- a/platforms/nuttx/init/rc.board_bootloader_upgrade.in +++ b/platforms/nuttx/init/rc.board_bootloader_upgrade.in @@ -5,12 +5,12 @@ # if param compare -s SYS_BL_UPDATE 1 then - if [ -f "/etc/extras/@BOARD_FIRMWARE_BIN@" ] + if [ -f "@BOARD_FIRMWARE_PATH@" ] then param set SYS_BL_UPDATE 0 param save echo "bootloader update..." - bl_update "/etc/extras/@BOARD_FIRMWARE_BIN@" + bl_update "@BOARD_FIRMWARE_PATH@" echo "bootloader update done, rebooting" reboot fi diff --git a/src/lib/CMakeLists.txt b/src/lib/CMakeLists.txt index 33d58ab640..e66e05a424 100644 --- a/src/lib/CMakeLists.txt +++ b/src/lib/CMakeLists.txt @@ -71,6 +71,7 @@ add_subdirectory(ringbuffer EXCLUDE_FROM_ALL) add_subdirectory(rl_tools EXCLUDE_FROM_ALL) add_subdirectory(rover_control EXCLUDE_FROM_ALL) add_subdirectory(rtl EXCLUDE_FROM_ALL) +add_subdirectory(sdcard_check EXCLUDE_FROM_ALL) add_subdirectory(sensor_calibration EXCLUDE_FROM_ALL) add_subdirectory(slew_rate EXCLUDE_FROM_ALL) add_subdirectory(sticks EXCLUDE_FROM_ALL) diff --git a/src/lib/component_information/CMakeLists.txt b/src/lib/component_information/CMakeLists.txt index f9b33e6d17..2a9121d5b2 100644 --- a/src/lib/component_information/CMakeLists.txt +++ b/src/lib/component_information/CMakeLists.txt @@ -41,13 +41,23 @@ set(comp_metadata_types) set(comp_metadata_board "${PX4_BOARD_VENDOR}_${PX4_BOARD_MODEL}_${PX4_BOARD_LABEL}") set(s3_url "https://px4-travis.s3.amazonaws.com") +set(sdcard_metadata_subdir "px4/metadata") +if(CONFIG_BOARD_ROOT_PATH) + string(REGEX REPLACE "^/" "" sdcard_root "${CONFIG_BOARD_ROOT_PATH}") +endif() set(comp_metadata_param_uri_board "${s3_url}/Firmware/{version}/${comp_metadata_board}/parameters.json.xz") list(FIND config_romfs_extra_dependencies "parameters_xml" index) if (${index} EQUAL -1) - set(comp_metadata_param_uri ${comp_metadata_param_uri_board}) - # use generic URL as fallback - set(comp_metadata_param_uri_fallback "${s3_url}/Firmware/{version}/_general/parameters.json.xz") + if(CONFIG_BOARD_ROOT_PATH) + # SD card path as primary, S3 as fallback + set(comp_metadata_param_uri "mftp://${sdcard_root}/${sdcard_metadata_subdir}/parameters.json.xz") + set(comp_metadata_param_uri_fallback ${comp_metadata_param_uri_board}) + else() + set(comp_metadata_param_uri ${comp_metadata_param_uri_board}) + # use generic URL as fallback + set(comp_metadata_param_uri_fallback "${s3_url}/Firmware/{version}/_general/parameters.json.xz") + endif() else() set(comp_metadata_param_uri "mftp://etc/extras/parameters.json.xz") set(comp_metadata_param_uri_fallback ${comp_metadata_param_uri_board}) @@ -59,9 +69,14 @@ list(APPEND comp_metadata_types "--type" "1,${PX4_BINARY_DIR}/parameters.json.xz set(comp_metadata_events_uri_board "${s3_url}/Firmware/{version}/${comp_metadata_board}/all_events.json.xz") list(FIND config_romfs_extra_dependencies "events_json" index) if (${index} EQUAL -1) - set(comp_metadata_events_uri ${comp_metadata_events_uri_board}) - # use generic URL as fallback - set(comp_metadata_events_uri_fallback "${s3_url}/Firmware/{version}/_general/all_events.json.xz") + if(CONFIG_BOARD_ROOT_PATH) + set(comp_metadata_events_uri "mftp://${sdcard_root}/${sdcard_metadata_subdir}/all_events.json.xz") + set(comp_metadata_events_uri_fallback ${comp_metadata_events_uri_board}) + else() + set(comp_metadata_events_uri ${comp_metadata_events_uri_board}) + # use generic URL as fallback + set(comp_metadata_events_uri_fallback "${s3_url}/Firmware/{version}/_general/all_events.json.xz") + endif() else() set(comp_metadata_events_uri "mftp://etc/extras/all_events.json.xz") set(comp_metadata_events_uri_fallback ${comp_metadata_events_uri_board}) @@ -71,8 +86,13 @@ list(APPEND comp_metadata_types "--type" "4,${PX4_BINARY_DIR}/events/all_events. set(comp_metadata_actuators_uri_board "${s3_url}/Firmware/{version}/${comp_metadata_board}/actuators.json.xz") list(FIND config_romfs_extra_dependencies "actuators_json" index) if (${index} EQUAL -1) - set(comp_metadata_actuators_uri ${comp_metadata_actuators_uri_board}) - set(comp_metadata_actuators_uri_fallback "") + if(CONFIG_BOARD_ROOT_PATH) + set(comp_metadata_actuators_uri "mftp://${sdcard_root}/${sdcard_metadata_subdir}/actuators.json.xz") + set(comp_metadata_actuators_uri_fallback ${comp_metadata_actuators_uri_board}) + else() + set(comp_metadata_actuators_uri ${comp_metadata_actuators_uri_board}) + set(comp_metadata_actuators_uri_fallback "") + endif() else() set(comp_metadata_actuators_uri "mftp://etc/extras/actuators.json.xz") set(comp_metadata_actuators_uri_fallback ${comp_metadata_actuators_uri_board}) @@ -105,3 +125,69 @@ add_custom_command(OUTPUT ${component_general_json} ${component_general_json}.xz COMMENT "Generating component_general.json and checksums.h" ) add_custom_target(component_general_json DEPENDS ${component_general_json}) + +# Copy metadata files (and optionally bootloader/IO binaries) to sdcard output +# directory so they can be uploaded to the SD card after firmware flash +if(CONFIG_BOARD_ROOT_PATH) + set(sdcard_metadata_dir ${PX4_BINARY_DIR}/sdcard/${sdcard_metadata_subdir}) + set(sdcard_extras_dir ${PX4_BINARY_DIR}/sdcard/px4/extras) + set(sdcard_files_outputs) + set(sdcard_files_deps) + + # Metadata files + add_custom_command( + OUTPUT + ${sdcard_metadata_dir}/parameters.json.xz + ${sdcard_metadata_dir}/all_events.json.xz + ${sdcard_metadata_dir}/actuators.json.xz + COMMAND ${CMAKE_COMMAND} -E make_directory ${sdcard_metadata_dir} + COMMAND ${CMAKE_COMMAND} -E copy ${PX4_BINARY_DIR}/parameters.json.xz ${sdcard_metadata_dir}/ + COMMAND ${CMAKE_COMMAND} -E copy ${PX4_BINARY_DIR}/events/all_events.json.xz ${sdcard_metadata_dir}/ + COMMAND ${CMAKE_COMMAND} -E copy ${PX4_BINARY_DIR}/actuators.json.xz ${sdcard_metadata_dir}/ + DEPENDS + ${PX4_BINARY_DIR}/parameters.json.xz + ${PX4_BINARY_DIR}/events/all_events.json.xz + ${PX4_BINARY_DIR}/actuators.json.xz + parameters_xml + events_json + actuators_json + COMMENT "Copying metadata files to sdcard output directory" + ) + list(APPEND sdcard_files_outputs + ${sdcard_metadata_dir}/parameters.json.xz + ${sdcard_metadata_dir}/all_events.json.xz + ${sdcard_metadata_dir}/actuators.json.xz + ) + + # Bootloader binary (optional) + set(bootloader_bin "${PX4_BOARD_VENDOR}_${PX4_BOARD_MODEL}_bootloader.bin") + if(EXISTS "${PX4_BOARD_DIR}/extras/${bootloader_bin}") + add_custom_command( + OUTPUT ${sdcard_extras_dir}/${bootloader_bin} + COMMAND ${CMAKE_COMMAND} -E make_directory ${sdcard_extras_dir} + COMMAND ${CMAKE_COMMAND} -E copy ${PX4_BOARD_DIR}/extras/${bootloader_bin} ${sdcard_extras_dir}/ + DEPENDS ${PX4_BOARD_DIR}/extras/${bootloader_bin} + COMMENT "Copying bootloader binary to sdcard output directory" + ) + list(APPEND sdcard_files_outputs ${sdcard_extras_dir}/${bootloader_bin}) + endif() + + # IO firmware binary (optional) + if(CONFIG_BOARD_IO) + set(io_bin "${CONFIG_BOARD_IO}.bin") + if(EXISTS "${PX4_BOARD_DIR}/extras/${io_bin}") + add_custom_command( + OUTPUT ${sdcard_extras_dir}/${io_bin} + COMMAND ${CMAKE_COMMAND} -E make_directory ${sdcard_extras_dir} + COMMAND ${CMAKE_COMMAND} -E copy ${PX4_BOARD_DIR}/extras/${io_bin} ${sdcard_extras_dir}/ + DEPENDS ${PX4_BOARD_DIR}/extras/${io_bin} + COMMENT "Copying IO firmware binary to sdcard output directory" + ) + list(APPEND sdcard_files_outputs ${sdcard_extras_dir}/${io_bin}) + endif() + endif() + + add_custom_target(sdcard_files ALL + DEPENDS ${sdcard_files_outputs} + ) +endif() diff --git a/src/lib/sdcard_check/CMakeLists.txt b/src/lib/sdcard_check/CMakeLists.txt new file mode 100644 index 0000000000..eb1fdeed16 --- /dev/null +++ b/src/lib/sdcard_check/CMakeLists.txt @@ -0,0 +1,26 @@ +# Generate checksums header from sdcard build directory +set(sdcard_checksums_header ${CMAKE_CURRENT_BINARY_DIR}/sdcard_checksums.h) + +if(CONFIG_BOARD_ROOT_PATH) + set(sdcard_dir ${PX4_BINARY_DIR}/sdcard) + set(sdcard_dir_arg --sdcard-dir ${sdcard_dir}) + set(sdcard_deps sdcard_files) +else() + set(sdcard_dir_arg) + set(sdcard_deps) +endif() + +add_custom_command(OUTPUT ${sdcard_checksums_header} + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_sdcard_checksums.py + ${sdcard_dir_arg} + --output ${sdcard_checksums_header} + DEPENDS + generate_sdcard_checksums.py + ${sdcard_deps} + COMMENT "Generating SD card file checksums" +) +add_custom_target(sdcard_checksums_gen DEPENDS ${sdcard_checksums_header}) + +px4_add_library(sdcard_check sdcard_check.cpp) +target_include_directories(sdcard_check PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +add_dependencies(sdcard_check sdcard_checksums_gen) diff --git a/src/lib/sdcard_check/generate_sdcard_checksums.py b/src/lib/sdcard_check/generate_sdcard_checksums.py new file mode 100644 index 0000000000..61301b7553 --- /dev/null +++ b/src/lib/sdcard_check/generate_sdcard_checksums.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +"""Generate SD card file checksums header for firmware integrity verification.""" + +import argparse +import os + + +def create_table(): + a = [] + for i in range(256): + k = i + for j in range(8): + if k & 1: + k ^= 0x1db710640 + k >>= 1 + a.append(k) + return a + + +def crc_update(buf, crc_table, crc): + for k in buf: + crc = (crc >> 8) ^ crc_table[(crc & 0xff) ^ k] + return crc + + +def compute_file_crc(filepath, crc_table): + crc = 0 + with open(filepath, 'rb') as f: + while True: + chunk = f.read(4096) + if not chunk: + break + crc = crc_update(chunk, crc_table, crc) + return crc + + +def main(): + parser = argparse.ArgumentParser( + description='Generate SD card file checksums header') + parser.add_argument('--sdcard-dir', metavar='DIR', + help='SD card build directory to scan') + parser.add_argument('--output', metavar='FILE', required=True, + help='Output header file path') + args = parser.parse_args() + + entries = [] + + if args.sdcard_dir and os.path.isdir(args.sdcard_dir): + crc_table = create_table() + + for dirpath, dirnames, filenames in os.walk(args.sdcard_dir): + dirnames.sort() + for filename in sorted(filenames): + filepath = os.path.join(dirpath, filename) + rel_path = os.path.relpath(filepath, args.sdcard_dir) + crc = compute_file_crc(filepath, crc_table) + entries.append((rel_path, crc)) + + with open(args.output, 'w') as f: + f.write('#pragma once\n') + f.write('#include \n') + f.write('namespace sdcard_check {\n') + f.write('struct FileEntry {\n') + f.write(' const char *relative_path;\n') + f.write(' uint32_t expected_crc;\n') + f.write('};\n') + f.write('static constexpr unsigned num_files = {};\n'.format( + len(entries))) + + if entries: + f.write('static constexpr FileEntry files[] = {\n') + for rel_path, crc in entries: + f.write(' {{"{}", {}u}},\n'.format(rel_path, crc)) + f.write('};\n') + else: + f.write('static constexpr FileEntry *files = nullptr;\n') + + f.write('} // namespace sdcard_check\n') + + +if __name__ == '__main__': + main() diff --git a/src/lib/sdcard_check/sdcard_check.cpp b/src/lib/sdcard_check/sdcard_check.cpp new file mode 100644 index 0000000000..c3d378f8eb --- /dev/null +++ b/src/lib/sdcard_check/sdcard_check.cpp @@ -0,0 +1,52 @@ +#include "sdcard_check.h" +#include "sdcard_checksums.h" + +#include +#include +#include +#include +#include + +namespace sdcard_check +{ + +Result verify_all() +{ + Result result{}; + +#ifdef PX4_STORAGEDIR + + for (unsigned i = 0; i < num_files; i++) { + result.num_checked++; + + char path[256]; + snprintf(path, sizeof(path), "%s/%s", PX4_STORAGEDIR, files[i].relative_path); + + int fd = open(path, O_RDONLY); + + if (fd < 0) { + result.num_missing++; + continue; + } + + uint32_t crc = 0; + uint8_t buf[256]; + ssize_t n; + + while ((n = read(fd, buf, sizeof(buf))) > 0) { + crc = crc32part(buf, n, crc); + } + + close(fd); + + if (crc != files[i].expected_crc) { + result.num_mismatch++; + } + } + +#endif /* PX4_STORAGEDIR */ + + return result; +} + +} // namespace sdcard_check diff --git a/src/lib/sdcard_check/sdcard_check.h b/src/lib/sdcard_check/sdcard_check.h new file mode 100644 index 0000000000..7083972d14 --- /dev/null +++ b/src/lib/sdcard_check/sdcard_check.h @@ -0,0 +1,14 @@ +#pragma once + +namespace sdcard_check +{ + +struct Result { + int num_checked; + int num_missing; + int num_mismatch; +}; + +Result verify_all(); + +} // namespace sdcard_check diff --git a/src/modules/commander/HealthAndArmingChecks/CMakeLists.txt b/src/modules/commander/HealthAndArmingChecks/CMakeLists.txt index ad18a2ad79..f986507039 100644 --- a/src/modules/commander/HealthAndArmingChecks/CMakeLists.txt +++ b/src/modules/commander/HealthAndArmingChecks/CMakeLists.txt @@ -74,6 +74,7 @@ px4_add_library(health_and_arming_checks ) set_property(GLOBAL APPEND PROPERTY PX4_MODULE_CONFIG_FILES ${CMAKE_CURRENT_SOURCE_DIR}/esc_check_params.yaml) add_dependencies(health_and_arming_checks mode_util) +target_link_libraries(health_and_arming_checks PRIVATE sdcard_check) px4_add_functional_gtest(SRC HealthAndArmingChecksTest.cpp LINKLIBS health_and_arming_checks mode_util diff --git a/src/modules/commander/HealthAndArmingChecks/checks/sdcardCheck.cpp b/src/modules/commander/HealthAndArmingChecks/checks/sdcardCheck.cpp index 6766b5376c..7720180e11 100644 --- a/src/modules/commander/HealthAndArmingChecks/checks/sdcardCheck.cpp +++ b/src/modules/commander/HealthAndArmingChecks/checks/sdcardCheck.cpp @@ -32,6 +32,7 @@ ****************************************************************************/ #include "sdcardCheck.hpp" +#include #include #include @@ -52,7 +53,10 @@ void SdCardChecks::checkAndReport(const Context &context, Report &reporter) if (!_sdcard_detected && statfs(PX4_STORAGEDIR, &statfs_buf) == 0) { // on NuttX we get a data block count f_blocks and byte count per block f_bsize if an SD card is inserted - _sdcard_detected = (statfs_buf.f_blocks > 0) && (statfs_buf.f_bsize > 0); + if ((statfs_buf.f_blocks > 0) && (statfs_buf.f_bsize > 0)) { + _sdcard_detected = true; + _sdcard_detected_time = hrt_absolute_time(); + } } if (!_sdcard_detected) { @@ -178,5 +182,56 @@ void SdCardChecks::checkAndReport(const Context &context, Report &reporter) #endif /* CONFIG_MODULES_TASK_WATCHDOG */ #endif /* __PX4_NUTTX */ + + // Check SD card metadata file integrity (delay to allow upload after flash) + if (!_metadata_checked && _param_com_arm_metadata_check.get() && _sdcard_detected + && hrt_elapsed_time(&_sdcard_detected_time) > 10_s) { + _metadata_checked = true; + auto result = sdcard_check::verify_all(); + + _metadata_missing = result.num_missing > 0; + _metadata_mismatch = result.num_mismatch > 0; + + if (!_metadata_missing && !_metadata_mismatch && result.num_checked > 0) { + PX4_INFO("SD card files verified: %d files OK", result.num_checked); + } + } + + if (_metadata_missing && _param_com_arm_metadata_check.get()) { + /* EVENT + * @description + * SD card metadata files are missing. Re-upload the SD card files matching this firmware version. + * + * + * This check can be configured via COM_ARM_META_CHK parameter. + * + */ + reporter.armingCheckFailure(NavModes::All, health_component_t::system, + events::ID("check_sdcard_metadata_missing"), + events::Log::Error, "SD card metadata files missing"); + + if (reporter.mavlink_log_pub()) { + mavlink_log_critical(reporter.mavlink_log_pub(), "Preflight Fail: SD card metadata files missing"); + } + } + + if (_metadata_mismatch && _param_com_arm_metadata_check.get()) { + /* EVENT + * @description + * SD card metadata files do not match the firmware. Re-upload the SD card files matching this firmware version. + * + * + * This check can be configured via COM_ARM_META_CHK parameter. + * + */ + reporter.armingCheckFailure(NavModes::All, health_component_t::system, + events::ID("check_sdcard_metadata_mismatch"), + events::Log::Error, "SD card metadata files mismatch"); + + if (reporter.mavlink_log_pub()) { + mavlink_log_critical(reporter.mavlink_log_pub(), "Preflight Fail: SD card metadata mismatch"); + } + } + #endif /* PX4_STORAGEDIR */ } diff --git a/src/modules/commander/HealthAndArmingChecks/checks/sdcardCheck.hpp b/src/modules/commander/HealthAndArmingChecks/checks/sdcardCheck.hpp index b8a0ac70bb..e7f2e0b454 100644 --- a/src/modules/commander/HealthAndArmingChecks/checks/sdcardCheck.hpp +++ b/src/modules/commander/HealthAndArmingChecks/checks/sdcardCheck.hpp @@ -53,10 +53,17 @@ private: bool _watchdog_checked_once {false}; bool _watchdog_file_present {false}; #endif + + hrt_abstime _sdcard_detected_time {0}; + + bool _metadata_checked {false}; + bool _metadata_mismatch {false}; + bool _metadata_missing {false}; #endif DEFINE_PARAMETERS_CUSTOM_PARENT(HealthAndArmingCheckBase, (ParamInt) _param_com_arm_sdcard, - (ParamBool) _param_com_arm_hardfault_check + (ParamBool) _param_com_arm_hardfault_check, + (ParamBool) _param_com_arm_metadata_check ) }; diff --git a/src/modules/commander/commander_params.yaml b/src/modules/commander/commander_params.yaml index 0352265131..389b008f5e 100644 --- a/src/modules/commander/commander_params.yaml +++ b/src/modules/commander/commander_params.yaml @@ -291,6 +291,14 @@ parameters: 1: Deny arming 2: Warning only default: 2 + COM_ARM_META_CHK: + description: + short: Enable SD card metadata integrity check + long: |- + Verifies SD card files match the firmware version at startup. + If disabled, no check is performed. + type: boolean + default: 1 COM_RC_OVERRIDE: description: short: Enable manual control stick override