diff --git a/Tools/ci/generate_manifest_json.py b/Tools/ci/generate_manifest_json.py deleted file mode 100755 index 7d8bb636f8..0000000000 --- a/Tools/ci/generate_manifest_json.py +++ /dev/null @@ -1,273 +0,0 @@ -#!/usr/bin/env python3 - -# Usage -# # pretty print all targets -# ./Tools/ci/generate_manifest_json.py | jq -# pretty print filtered targets -# ./Tools/ci/generate_manifest_json.py -f px4_fmu-v6x | jq - -import argparse -import os -import sys -import json -import re -from kconfiglib import Kconfig - -kconf = Kconfig() - -# Supress warning output -kconf.warn_assign_undef = False -kconf.warn_assign_override = False -kconf.warn_assign_redun = False - -dconf = Kconfig() - -# Supress warning output -dconf.warn_assign_undef = False -dconf.warn_assign_override = False -dconf.warn_assign_redun = False - -source_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..') - -parser = argparse.ArgumentParser(description='Generate build targets') - -parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', - help='Verbose Output') -parser.add_argument('-p', '--pretty', dest='pretty', action='store_true', - help='Pretty output instead of a single line') -# parser.add_argument('-g', '--groups', dest='group', action='store_true', -# help='Groups targets') -# parser.add_argument('-m', '--manifest', dest='manifest', action='store_true', -# help='Firmware manifest') -parser.add_argument('-f', '--filter', dest='filter', help='comma separated list of board names to use instead of all') - -args = parser.parse_args() -verbose = args.verbose - -board_filter = [] -if args.filter: - for board in args.filter.split(','): - board_filter.append(board) - -manifest = [] -build_configs = [] -grouped_targets = {} -excluded_boards = [] -excluded_manufacturers = ['atlflight'] -excluded_platforms = ['qurt'] -excluded_labels = [ - 'stackcheck', - 'nolockstep', 'replay', 'test', - 'uavcanv1', # TODO: fix and enable -] - -github_action_config = { 'include': build_configs } -extra_args = {} -if args.pretty: - extra_args['indent'] = 2 - -def chunks(arr, size): - # splits array into parts - for i in range(0, len(arr), size): - yield arr[i:i + size] - -def comma_targets(targets): - # turns array of targets into a comma split string - return ",".join(targets) - -def process_variants(targets): - # returns the - return [{"name": v} for v in targets] - -def process_target(px4board_file, target_name, defconfig_data, protoconfig_data): - # reads through the board file and grabs - # useful information for building - hardware = { - 'architecture': '', - 'vendor_id': '', - 'product_id': '', - 'chip': '', - 'productstr': '', - } - manifest_item = { - 'name': target_name, - 'description': '', - 'manufacturer': '', - 'hardware': {}, - 'toolchain': '', - 'artifact': '', - 'sha256sum': '', - 'summary': '', - 'image_maxsize': '', - 'board_id': '', - } - - board_data = process_boardfile(px4board_file) - manifest_item['toolchain'] = board_data.get('toolchain', '') - manifest_item['manufacturer'] = board_data.get('manufacturer', '') - - # prototype is required per-board - manifest_item['description'] = protoconfig_data.get('description', '') - manifest_item['board_id'] = protoconfig_data.get('board_id', '') - manifest_item['summary'] = protoconfig_data.get('summary', '') - manifest_item['image_maxsize'] = protoconfig_data.get('image_maxsize', '') - - if defconfig_data: - if defconfig_data.get('manufacturer'): - manifest_item['manufacturer'] = defconfig_data['manufacturer'] - hw = defconfig_data.get('hardware', {}) - for k in ('vendor_id', 'product_id', 'chip', 'productstr', 'architecture'): - if hw.get(k): - hardware[k] = hw[k] - - manifest_item['hardware'] = hardware - return manifest_item - -def process_boardfile(boardtarget_file): - return_boardfile = { - 'toolchain': '', - 'boardfile': '', - 'manufacturer': '', - 'architecture': '', - } - if boardtarget_file.endswith("default.px4board") or \ - boardtarget_file.endswith("performance-test.px4board") or \ - boardtarget_file.endswith("bootloader.px4board"): - kconf.load_config(boardtarget_file, replace=True) - else: # Merge config with default.px4board - default_kconfig = re.sub(r'[a-zA-Z\d_-]+\.px4board', 'default.px4board', boardtarget_file) - kconf.load_config(default_kconfig, replace=True) - kconf.load_config(boardtarget_file, replace=False) - - return_boardfile['boardfile'] = os.path.abspath(boardtarget_file) - - if "BOARD_TOOLCHAIN" in kconf.syms: - return_boardfile['toolchain'] = kconf.syms["BOARD_TOOLCHAIN"].str_value - - if "CONFIG_CDCACM_VENDORSTR" in kconf.syms: - return_boardfile['manufacturer'] = kconf.syms["CONFIG_CDCACM_VENDORSTR"].str_value - - if "BOARD_ARCHITECTURE" in kconf.syms: - return_boardfile['architecture'] = kconf.syms["BOARD_ARCHITECTURE"].str_value - - return return_boardfile - -def load_defconfig(defconfig_file): - if not os.path.isfile(defconfig_file): - return None - - hardware = { - 'architecture': '', - 'vendor_id': '', - 'product_id': '', - 'chip': '', - 'productstr': '', - } - manifest_item = { - 'manufacturer': '', - 'hardware': hardware, - } - defconfig = {} - with open(defconfig_file, 'r') as f: - for line in f: - line = line.strip() - if not line or line.startswith('#') or '=' not in line: - continue - key, value = line.split('=', 1) - value = value.strip('"') - defconfig[key] = value - - manifest_item['manufacturer'] = defconfig.get('CONFIG_CDCACM_VENDORSTR', '') - hardware['vendor_id'] = defconfig.get('CONFIG_CDCACM_VENDORID', '') - hardware['product_id'] = defconfig.get('CONFIG_CDCACM_PRODUCTID', '') - hardware['chip'] = defconfig.get('CONFIG_ARCH_CHIP', '') - hardware['productstr'] = defconfig.get('CONFIG_CDCACM_PRODUCTSTR', '') - hardware['architecture'] = defconfig.get('CONFIG_ARCH', '') - - return manifest_item - -def process_proto(protoconfig_file): - return_proto = { - 'board_id': '', - 'description': '', - 'summary': '', - 'image_maxsize': '' - } - proto = {} - with open(protoconfig_file, "r") as f: - proto = json.load(f) - - return_proto['board_id'] = proto['board_id'] - return_proto['description'] = proto['description'] - return_proto['summary'] = proto['summary'] - return_proto['image_maxsize'] = proto['image_maxsize'] - - return return_proto - -# Look for board targets in the ./boards directory -if(verbose): - print("=======================") - print("= scanning for boards =") - print("=======================") - -targets_list = {} - -# assume this file lives under Tools/ci/ -repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) -boards_root = os.path.join(repo_root, 'boards') - -for manufacturer in os.scandir(boards_root): - if not manufacturer.is_dir(): - continue - if manufacturer.name in excluded_manufacturers: - if verbose: print(f'excluding manufacturer {manufacturer.name}') - continue - - for board in os.scandir(manufacturer.path): - if not board.is_dir(): - continue - - # resolve per-board shared files up front - proto_path = os.path.join(board.path, 'firmware.prototype') - protoconfig_data = None - if os.path.isfile(proto_path): - try: - protoconfig_data = process_proto(proto_path) - except Exception as e: - if verbose: print(f'warning: failed to parse {proto_path}: {e}') - - defconfig_file = os.path.join(board.path, 'nuttx-config', 'nsh', 'defconfig') - defconfig_data = load_defconfig(defconfig_file) if os.path.isfile(defconfig_file) else None - - for f in os.scandir(board.path): - if not f.is_file(): - continue - - if f.name.endswith('.px4board'): - board_name = manufacturer.name + '_' + board.name - label = f.name[:-9] - target_name = manufacturer.name + '_' + board.name + '_' + label - - if board_filter and not board_name in board_filter: - if verbose: print(f'excluding board {board_name} ({target_name})') - continue - if board_name in excluded_boards: - if verbose: print(f'excluding board {board_name} ({target_name})') - continue - if label in excluded_labels: - if verbose: print(f'excluding label {label} ({target_name})') - continue - - if protoconfig_data is None: - if verbose: print(f'skipping {target_name}: missing firmware.prototype') - continue - - target = process_target(f.path, - target_name, - defconfig_data, - protoconfig_data) - - if target is not None: - build_configs.append(target) - -print(json.dumps(build_configs, **extra_args)) diff --git a/Tools/manifest/gen_board_manifest_from_defconfig.py b/Tools/manifest/gen_board_manifest_from_defconfig.py new file mode 100755 index 0000000000..493bb58b73 --- /dev/null +++ b/Tools/manifest/gen_board_manifest_from_defconfig.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +import argparse, json, os, re, sys +from typing import Dict + +def parse_defconfig(path: str) -> Dict[str, str]: + d: Dict[str, str] = {} + if not path or not os.path.exists(path): + return d + with open(path, "r", encoding="utf-8", errors="ignore") as f: + for raw in f: + line = raw.strip() + if not line or not line.startswith("CONFIG_") or "=" not in line or line.startswith("#"): + continue + k, v = line.split("=", 1) + v = v.strip() + if len(v) >= 2 and v[0] == '"' and v[-1] == '"': + v = v[1:-1] + d[k.strip()] = v + return d + +def norm_hex(s: str) -> str: + if not s: + return "" + s = s.strip() + if s.lower().startswith("0x"): + return s.lower() + try: + return f"0x{int(s, 0):04x}" + except Exception: + return s + +def detect_chip(defcfg: Dict[str,str]) -> str: + for k, v in defcfg.items(): + if k.startswith("CONFIG_ARCH_CHIP_") and v == "y": + return k[len("CONFIG_ARCH_CHIP_"):].lower().replace("_", "") + s = defcfg.get("CONFIG_ARCH_CHIP", "") + return s.lower().replace("_", "") if s else "" + +def pick(preferred: str, fallback_key: str, defcfg: Dict[str, str]) -> str: + return preferred if preferred else defcfg.get(fallback_key, "") + +def main(): + ap = argparse.ArgumentParser(description="Generate board manifest (prefer CMake-passed overrides, fallback to defconfig).") + ap.add_argument("--defconfig", required=False, help="Path to defconfig (fallback only)") + # explicit overrides coming from CMake + ap.add_argument("--manufacturer", default="") + ap.add_argument("--productstr", default="") + ap.add_argument("--target", default="") + ap.add_argument("--name", default="") + ap.add_argument("--arch", default="") + ap.add_argument("--chip", default="") + ap.add_argument("--vid", default="") + ap.add_argument("--pid", default="") + ap.add_argument("--out", help="Write to file instead of stdout") + args = ap.parse_args() + + defcfg = parse_defconfig(args.defconfig) if args.defconfig else {} + + manufacturer = pick(args.manufacturer, "CONFIG_BOARD_MANUFACTURER", defcfg) + productstr = pick(args.productstr, "CONFIG_BOARD_PRODUCTSTR", defcfg) + target = args.target or "" + name = args.name or "" + arch = (pick(args.arch, "CONFIG_ARCH", defcfg)).lower() + chip = args.chip or detect_chip(defcfg) + vid = norm_hex(pick(args.vid, "CONFIG_CDCACM_VENDORID", defcfg)) + pid = norm_hex(pick(args.pid, "CONFIG_CDCACM_PRODUCTID", defcfg)) + + manifest = { + "name": name, + "target": target, + "manufacturer": manufacturer, + "hardware": { + "architecture": arch, + "vendor_id": vid, + "product_id": pid, + "chip": chip, + "productstr": productstr + } + } + + if args.out: + out_dir = os.path.dirname(args.out) + if out_dir: + os.makedirs(out_dir, exist_ok=True) + with open(args.out, "w", encoding="utf-8") as f: + json.dump(manifest, f, indent=2) + f.write("\n") + else: + json.dump(manifest, sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Tools/px_mkfw.py b/Tools/px_mkfw.py index 8ed8d4aa19..cce6132787 100755 --- a/Tools/px_mkfw.py +++ b/Tools/px_mkfw.py @@ -1,3 +1,4 @@ +# vim: set noexpandtab tabstop=4 shiftwidth=4: #!/usr/bin/env python3 ############################################################################ # @@ -46,6 +47,7 @@ import os import zlib import time import subprocess +import hashlib # # Construct a basic firmware description @@ -63,8 +65,20 @@ def mkdesc(): proto['build_time'] = 0 proto['image'] = bytes() proto['image_size'] = 0 + proto['sha256sum'] = "" return proto +def _merge_manifest(dst, src): + if not isinstance(src, dict): + return + for k, v in src.items(): + if k == "hardware": + dst.setdefault("hardware", {}) + if isinstance(v, dict): + dst["hardware"].update(v) + else: + dst[k] = v + # Parse commandline parser = argparse.ArgumentParser(description="Firmware generator for the PX autopilot system.") parser.add_argument("--prototype", action="store", help="read a prototype description from a file") @@ -77,6 +91,7 @@ parser.add_argument("--git_identity", action="store", help="the working director parser.add_argument("--parameter_xml", action="store", help="the parameters.xml file") parser.add_argument("--airframe_xml", action="store", help="the airframes.xml file") parser.add_argument("--image", action="store", help="the firmware image") +parser.add_argument("--manifest_json", action="append", help="path to manifest JSON fragment to merge") args = parser.parse_args() # Fetch the firmware descriptor prototype if specified @@ -87,6 +102,9 @@ if args.prototype != None: else: desc = mkdesc() +desc.setdefault("manifest_version", 1) +desc.setdefault("manifest", {}) + desc['build_time'] = int(time.time()) if args.board_id != None: @@ -120,8 +138,20 @@ if args.airframe_xml != None: desc['airframe_xml'] = base64.b64encode(zlib.compress(bytes,9)).decode('utf-8') if args.image != None: f = open(args.image, "rb") - bytes = f.read() - desc['image_size'] = len(bytes) - desc['image'] = base64.b64encode(zlib.compress(bytes,9)).decode('utf-8') + raw_image = f.read() + f.close() + desc['image_size'] = len(raw_image) + sha256sum = hashlib.sha256(raw_image).hexdigest() + desc['sha256sum'] = sha256sum + desc['image'] = base64.b64encode(zlib.compress(raw_image, 9)).decode('utf-8') + +# merge manifest +manifest_inputs = args.manifest_json or [] +if isinstance(manifest_inputs, str): + manifest_inputs = [manifest_inputs] +for p in manifest_inputs: + with open(p, "r", encoding="utf-8") as f: + frag = json.load(f) + _merge_manifest(desc["manifest"], frag) print(json.dumps(desc, indent=4)) diff --git a/platforms/nuttx/CMakeLists.txt b/platforms/nuttx/CMakeLists.txt index 304273d05f..e56327c1c4 100644 --- a/platforms/nuttx/CMakeLists.txt +++ b/platforms/nuttx/CMakeLists.txt @@ -422,9 +422,31 @@ endif() # create .px4 with parameter and airframe metadata if (TARGET parameters_xml AND TARGET airframes_xml) - string(REPLACE ".elf" ".px4" fw_package ${PX4_BINARY_DIR}/${FW_NAME}) + # Generate manifest object + set(MANIFEST_JSON ${PX4_BINARY_DIR}/manifest.json) + add_custom_command( + OUTPUT ${MANIFEST_JSON} + COMMAND + ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/Tools/manifest/gen_board_manifest_from_defconfig.py + --defconfig ${NUTTX_DEFCONFIG} + --manufacturer "${CONFIG_CDCACM_VENDORSTR}" + --productstr "${CONFIG_CDCACM_PRODUCTSTR}" + --target "${PX4_CONFIG}" + --arch "${CONFIG_ARCH}" + --name "${PX4_BOARD_NAME}" + --chip "${CONFIG_ARCH_CHIP}" + --vid "${CONFIG_CDCACM_VENDORID}" + --pid "${CONFIG_CDCACM_PRODUCTID}" + --out ${MANIFEST_JSON} + DEPENDS + ${PX4_SOURCE_DIR}/Tools/manifest/gen_board_manifest_from_defconfig.py + ${NUTTX_DEFCONFIG} + COMMENT "Generating board specific manifest from defconfig" + VERBATIM + ) + add_custom_command( OUTPUT ${fw_package} COMMAND @@ -433,11 +455,14 @@ if (TARGET parameters_xml AND TARGET airframes_xml) --git_identity ${PX4_SOURCE_DIR} --parameter_xml ${PX4_BINARY_DIR}/parameters.xml --airframe_xml ${PX4_BINARY_DIR}/airframes.xml - --image ${PX4_BINARY_DIR}/${PX4_CONFIG}.bin > ${fw_package} + --image ${PX4_BINARY_DIR}/${PX4_CONFIG}.bin + --manifest_json ${MANIFEST_JSON} + > ${fw_package} DEPENDS ${PX4_BINARY_DIR}/${PX4_CONFIG}.bin airframes_xml parameters_xml + ${MANIFEST_JSON} COMMENT "Creating ${fw_package}" WORKING_DIRECTORY ${PX4_BINARY_DIR} )