build: bundle manifest to *.px4 file

Signed-off-by: Ramon Roche <mrpollo@gmail.com>

Tools/ci/generate_manifest_json.py

@@ -1,273 +0,0 @@
-#!/usr/bin/env python3
-
-# Usage
-# # pretty print all targets
-# ./Tools/ci/generate_manifest_json.py | jq
-# pretty print filtered targets
-# ./Tools/ci/generate_manifest_json.py -f px4_fmu-v6x | jq
-
-import argparse
-import os
-import sys
-import json
-import re
-from kconfiglib import Kconfig
-
-kconf = Kconfig()
-
-# Supress warning output
-kconf.warn_assign_undef = False
-kconf.warn_assign_override = False
-kconf.warn_assign_redun = False
-
-dconf = Kconfig()
-
-# Supress warning output
-dconf.warn_assign_undef = False
-dconf.warn_assign_override = False
-dconf.warn_assign_redun = False
-
-source_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
-
-parser = argparse.ArgumentParser(description='Generate build targets')
-
-parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
-                    help='Verbose Output')
-parser.add_argument('-p', '--pretty', dest='pretty', action='store_true',
-                    help='Pretty output instead of a single line')
-# parser.add_argument('-g', '--groups', dest='group', action='store_true',
-#                     help='Groups targets')
-# parser.add_argument('-m', '--manifest', dest='manifest', action='store_true',
-#                     help='Firmware manifest')
-parser.add_argument('-f', '--filter', dest='filter', help='comma separated list of board names to use instead of all')
-
-args = parser.parse_args()
-verbose = args.verbose
-
-board_filter = []
-if args.filter:
-    for board in args.filter.split(','):
-        board_filter.append(board)
-
-manifest = []
-build_configs = []
-grouped_targets = {}
-excluded_boards = []
-excluded_manufacturers = ['atlflight']
-excluded_platforms = ['qurt']
-excluded_labels = [
-	'stackcheck',
-	'nolockstep', 'replay', 'test',
-	'uavcanv1', # TODO: fix and enable
-]
-
-github_action_config = { 'include': build_configs }
-extra_args = {}
-if args.pretty:
-    extra_args['indent'] = 2
-
-def chunks(arr, size):
-    # splits array into parts
-    for i in range(0, len(arr), size):
-        yield arr[i:i + size]
-
-def comma_targets(targets):
-    # turns array of targets into a comma split string
-    return ",".join(targets)
-
-def process_variants(targets):
-    # returns the
-    return [{"name": v} for v in targets]
-
-def process_target(px4board_file, target_name, defconfig_data, protoconfig_data):
-    # reads through the board file and grabs
-    # useful information for building
-    hardware = {
-        'architecture': '',
-        'vendor_id': '',
-        'product_id': '',
-        'chip': '',
-        'productstr': '',
-    }
-    manifest_item = {
-        'name': target_name,
-        'description': '',
-        'manufacturer': '',
-        'hardware': {},
-        'toolchain': '',
-        'artifact': '',
-        'sha256sum': '',
-        'summary': '',
-        'image_maxsize': '',
-        'board_id': '',
-    }
-
-    board_data = process_boardfile(px4board_file)
-    manifest_item['toolchain'] = board_data.get('toolchain', '')
-    manifest_item['manufacturer'] = board_data.get('manufacturer', '')
-
-    # prototype is required per-board
-    manifest_item['description'] = protoconfig_data.get('description', '')
-    manifest_item['board_id'] = protoconfig_data.get('board_id', '')
-    manifest_item['summary'] = protoconfig_data.get('summary', '')
-    manifest_item['image_maxsize'] = protoconfig_data.get('image_maxsize', '')
-
-    if defconfig_data:
-        if defconfig_data.get('manufacturer'):
-            manifest_item['manufacturer'] = defconfig_data['manufacturer']
-        hw = defconfig_data.get('hardware', {})
-        for k in ('vendor_id', 'product_id', 'chip', 'productstr', 'architecture'):
-            if hw.get(k):
-                hardware[k] = hw[k]
-
-    manifest_item['hardware'] = hardware
-    return manifest_item
-
-def process_boardfile(boardtarget_file):
-    return_boardfile = {
-        'toolchain': '',
-        'boardfile': '',
-        'manufacturer': '',
-        'architecture': '',
-    }
-    if boardtarget_file.endswith("default.px4board") or \
-        boardtarget_file.endswith("performance-test.px4board") or \
-        boardtarget_file.endswith("bootloader.px4board"):
-        kconf.load_config(boardtarget_file, replace=True)
-    else: # Merge config with default.px4board
-        default_kconfig = re.sub(r'[a-zA-Z\d_-]+\.px4board', 'default.px4board', boardtarget_file)
-        kconf.load_config(default_kconfig, replace=True)
-        kconf.load_config(boardtarget_file, replace=False)
-
-    return_boardfile['boardfile'] = os.path.abspath(boardtarget_file)
-
-    if "BOARD_TOOLCHAIN" in kconf.syms:
-        return_boardfile['toolchain'] = kconf.syms["BOARD_TOOLCHAIN"].str_value
-
-    if "CONFIG_CDCACM_VENDORSTR" in kconf.syms:
-        return_boardfile['manufacturer'] = kconf.syms["CONFIG_CDCACM_VENDORSTR"].str_value
-
-    if "BOARD_ARCHITECTURE" in kconf.syms:
-        return_boardfile['architecture'] = kconf.syms["BOARD_ARCHITECTURE"].str_value
-
-    return return_boardfile
-
-def load_defconfig(defconfig_file):
-    if not os.path.isfile(defconfig_file):
-        return None
-
-    hardware = {
-        'architecture': '',
-        'vendor_id': '',
-        'product_id': '',
-        'chip': '',
-        'productstr': '',
-    }
-    manifest_item = {
-        'manufacturer': '',
-        'hardware': hardware,
-    }
-    defconfig = {}
-    with open(defconfig_file, 'r') as f:
-        for line in f:
-            line = line.strip()
-            if not line or line.startswith('#') or '=' not in line:
-                continue
-            key, value = line.split('=', 1)
-            value = value.strip('"')
-            defconfig[key] = value
-
-    manifest_item['manufacturer'] = defconfig.get('CONFIG_CDCACM_VENDORSTR', '')
-    hardware['vendor_id'] = defconfig.get('CONFIG_CDCACM_VENDORID', '')
-    hardware['product_id'] = defconfig.get('CONFIG_CDCACM_PRODUCTID', '')
-    hardware['chip'] = defconfig.get('CONFIG_ARCH_CHIP', '')
-    hardware['productstr'] = defconfig.get('CONFIG_CDCACM_PRODUCTSTR', '')
-    hardware['architecture'] = defconfig.get('CONFIG_ARCH', '')
-
-    return manifest_item
-
-def process_proto(protoconfig_file):
-    return_proto = {
-        'board_id': '',
-        'description': '',
-        'summary': '',
-        'image_maxsize': ''
-    }
-    proto = {}
-    with open(protoconfig_file, "r") as f:
-        proto = json.load(f)
-
-    return_proto['board_id'] = proto['board_id']
-    return_proto['description'] = proto['description']
-    return_proto['summary'] = proto['summary']
-    return_proto['image_maxsize'] = proto['image_maxsize']
-
-    return return_proto
-
-# Look for board targets in the ./boards directory
-if(verbose):
-    print("=======================")
-    print("= scanning for boards =")
-    print("=======================")
-
-targets_list = {}
-
-# assume this file lives under Tools/ci/
-repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
-boards_root = os.path.join(repo_root, 'boards')
-
-for manufacturer in os.scandir(boards_root):
-    if not manufacturer.is_dir():
-        continue
-    if manufacturer.name in excluded_manufacturers:
-        if verbose: print(f'excluding manufacturer {manufacturer.name}')
-        continue
-
-    for board in os.scandir(manufacturer.path):
-        if not board.is_dir():
-            continue
-
-        # resolve per-board shared files up front
-        proto_path = os.path.join(board.path, 'firmware.prototype')
-        protoconfig_data = None
-        if os.path.isfile(proto_path):
-            try:
-                protoconfig_data =  process_proto(proto_path)
-            except Exception as e:
-                if verbose: print(f'warning: failed to parse {proto_path}: {e}')
-
-        defconfig_file = os.path.join(board.path, 'nuttx-config', 'nsh', 'defconfig')
-        defconfig_data = load_defconfig(defconfig_file) if os.path.isfile(defconfig_file) else None
-
-        for f in os.scandir(board.path):
-            if not f.is_file():
-                continue
-
-            if f.name.endswith('.px4board'):
-                board_name = manufacturer.name + '_' + board.name
-                label = f.name[:-9]
-                target_name = manufacturer.name + '_' + board.name + '_' + label
-
-                if board_filter and not board_name in board_filter:
-                    if verbose: print(f'excluding board {board_name} ({target_name})')
-                    continue
-                if board_name in excluded_boards:
-                    if verbose: print(f'excluding board {board_name} ({target_name})')
-                    continue
-                if label in excluded_labels:
-                    if verbose: print(f'excluding label {label} ({target_name})')
-                    continue
-
-                if protoconfig_data is None:
-                    if verbose: print(f'skipping {target_name}: missing firmware.prototype')
-                    continue
-
-                target = process_target(f.path,
-                                        target_name,
-                                        defconfig_data,
-                                        protoconfig_data)
-
-                if target is not None:
-                    build_configs.append(target)
-
-print(json.dumps(build_configs, **extra_args))

Tools/manifest/gen_board_manifest_from_defconfig.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+import argparse, json, os, re, sys
+from typing import Dict
+
+def parse_defconfig(path: str) -> Dict[str, str]:
+    d: Dict[str, str] = {}
+    if not path or not os.path.exists(path):
+        return d
+    with open(path, "r", encoding="utf-8", errors="ignore") as f:
+        for raw in f:
+            line = raw.strip()
+            if not line or not line.startswith("CONFIG_") or "=" not in line or line.startswith("#"):
+                continue
+            k, v = line.split("=", 1)
+            v = v.strip()
+            if len(v) >= 2 and v[0] == '"' and v[-1] == '"':
+                v = v[1:-1]
+            d[k.strip()] = v
+    return d
+
+def norm_hex(s: str) -> str:
+    if not s:
+        return ""
+    s = s.strip()
+    if s.lower().startswith("0x"):
+        return s.lower()
+    try:
+        return f"0x{int(s, 0):04x}"
+    except Exception:
+        return s
+
+def detect_chip(defcfg: Dict[str,str]) -> str:
+    for k, v in defcfg.items():
+        if k.startswith("CONFIG_ARCH_CHIP_") and v == "y":
+            return k[len("CONFIG_ARCH_CHIP_"):].lower().replace("_", "")
+    s = defcfg.get("CONFIG_ARCH_CHIP", "")
+    return s.lower().replace("_", "") if s else ""
+
+def pick(preferred: str, fallback_key: str, defcfg: Dict[str, str]) -> str:
+    return preferred if preferred else defcfg.get(fallback_key, "")
+
+def main():
+    ap = argparse.ArgumentParser(description="Generate board manifest (prefer CMake-passed overrides, fallback to defconfig).")
+    ap.add_argument("--defconfig", required=False, help="Path to defconfig (fallback only)")
+    # explicit overrides coming from CMake
+    ap.add_argument("--manufacturer", default="")
+    ap.add_argument("--productstr", default="")
+    ap.add_argument("--target", default="")
+    ap.add_argument("--name", default="")
+    ap.add_argument("--arch", default="")
+    ap.add_argument("--chip", default="")
+    ap.add_argument("--vid", default="")
+    ap.add_argument("--pid", default="")
+    ap.add_argument("--out", help="Write to file instead of stdout")
+    args = ap.parse_args()
+
+    defcfg = parse_defconfig(args.defconfig) if args.defconfig else {}
+
+    manufacturer    = pick(args.manufacturer, "CONFIG_BOARD_MANUFACTURER", defcfg)
+    productstr      = pick(args.productstr,   "CONFIG_BOARD_PRODUCTSTR",   defcfg)
+    target          = args.target or ""
+    name            = args.name or ""
+    arch            = (pick(args.arch,       "CONFIG_ARCH",                defcfg)).lower()
+    chip            = args.chip or detect_chip(defcfg)
+    vid             = norm_hex(pick(args.vid, "CONFIG_CDCACM_VENDORID",    defcfg))
+    pid             = norm_hex(pick(args.pid, "CONFIG_CDCACM_PRODUCTID",   defcfg))
+
+    manifest = {
+        "name": name,
+        "target": target,
+        "manufacturer": manufacturer,
+        "hardware": {
+            "architecture": arch,
+            "vendor_id": vid,
+            "product_id": pid,
+            "chip": chip,
+            "productstr": productstr
+        }
+    }
+
+    if args.out:
+        out_dir = os.path.dirname(args.out)
+        if out_dir:
+            os.makedirs(out_dir, exist_ok=True)
+        with open(args.out, "w", encoding="utf-8") as f:
+            json.dump(manifest, f, indent=2)
+            f.write("\n")
+    else:
+        json.dump(manifest, sys.stdout, indent=2)
+        sys.stdout.write("\n")
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

Tools/px_mkfw.py

@@ -1,3 +1,4 @@
+# vim: set noexpandtab tabstop=4 shiftwidth=4:
 #!/usr/bin/env python3
 ############################################################################
 #
@@ -46,6 +47,7 @@ import os
 import zlib
 import time
 import subprocess
+import hashlib
 
 #
 # Construct a basic firmware description
@@ -63,8 +65,20 @@ def mkdesc():
 	proto['build_time']	= 0
 	proto['image']		= bytes()
 	proto['image_size']	= 0
+	proto['sha256sum'] = ""
 	return proto
 
+def _merge_manifest(dst, src):
+	if not isinstance(src, dict):
+		return
+	for k, v in src.items():
+		if k == "hardware":
+			dst.setdefault("hardware", {})
+		if isinstance(v, dict):
+			dst["hardware"].update(v)
+		else:
+			dst[k] = v
+
 # Parse commandline
 parser = argparse.ArgumentParser(description="Firmware generator for the PX autopilot system.")
 parser.add_argument("--prototype",	action="store", help="read a prototype description from a file")
@@ -77,6 +91,7 @@ parser.add_argument("--git_identity",	action="store", help="the working director
 parser.add_argument("--parameter_xml",	action="store", help="the parameters.xml file")
 parser.add_argument("--airframe_xml",	action="store", help="the airframes.xml file")
 parser.add_argument("--image",		action="store", help="the firmware image")
+parser.add_argument("--manifest_json",	action="append", help="path to manifest JSON fragment to merge")
 args = parser.parse_args()
 
 # Fetch the firmware descriptor prototype if specified
@@ -87,6 +102,9 @@ if args.prototype != None:
 else:
 	desc = mkdesc()
 
+desc.setdefault("manifest_version", 1)
+desc.setdefault("manifest", {})
+
 desc['build_time'] 		= int(time.time())
 
 if args.board_id != None:
@@ -120,8 +138,20 @@ if args.airframe_xml != None:
 	desc['airframe_xml'] = base64.b64encode(zlib.compress(bytes,9)).decode('utf-8')
 if args.image != None:
 	f = open(args.image, "rb")
-	bytes = f.read()
+	raw_image = f.read()
-	desc['image_size'] = len(bytes)
+	f.close()
-	desc['image'] = base64.b64encode(zlib.compress(bytes,9)).decode('utf-8')
+	desc['image_size'] = len(raw_image)
+	sha256sum = hashlib.sha256(raw_image).hexdigest()
+	desc['sha256sum'] = sha256sum
+	desc['image'] = base64.b64encode(zlib.compress(raw_image, 9)).decode('utf-8')
+
+# merge manifest
+manifest_inputs = args.manifest_json or []
+if isinstance(manifest_inputs, str):
+	manifest_inputs = [manifest_inputs]
+for p in manifest_inputs:
+	with open(p, "r", encoding="utf-8") as f:
+		frag = json.load(f)
+	_merge_manifest(desc["manifest"], frag)
 
 print(json.dumps(desc, indent=4))

platforms/nuttx/CMakeLists.txt

@@ -422,9 +422,31 @@ endif()
 
 # create .px4 with parameter and airframe metadata
 if (TARGET parameters_xml AND TARGET airframes_xml)
-
 	string(REPLACE ".elf" ".px4" fw_package ${PX4_BINARY_DIR}/${FW_NAME})
 
+	# Generate manifest object
+	set(MANIFEST_JSON ${PX4_BINARY_DIR}/manifest.json)
+	add_custom_command(
+		OUTPUT ${MANIFEST_JSON}
+		COMMAND
+			${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/Tools/manifest/gen_board_manifest_from_defconfig.py
+				--defconfig	${NUTTX_DEFCONFIG}
+				--manufacturer	"${CONFIG_CDCACM_VENDORSTR}"
+				--productstr	"${CONFIG_CDCACM_PRODUCTSTR}"
+				--target	"${PX4_CONFIG}"
+				--arch		"${CONFIG_ARCH}"
+				--name		"${PX4_BOARD_NAME}"
+				--chip		"${CONFIG_ARCH_CHIP}"
+				--vid		"${CONFIG_CDCACM_VENDORID}"
+				--pid		"${CONFIG_CDCACM_PRODUCTID}"
+				--out		${MANIFEST_JSON}
+		DEPENDS
+			${PX4_SOURCE_DIR}/Tools/manifest/gen_board_manifest_from_defconfig.py
+			${NUTTX_DEFCONFIG}
+		COMMENT "Generating board specific manifest from defconfig"
+		VERBATIM
+	)
+
 	add_custom_command(
 		OUTPUT ${fw_package}
 		COMMAND
@@ -433,11 +455,14 @@ if (TARGET parameters_xml AND TARGET airframes_xml)
 				--git_identity ${PX4_SOURCE_DIR}
 				--parameter_xml ${PX4_BINARY_DIR}/parameters.xml
 				--airframe_xml ${PX4_BINARY_DIR}/airframes.xml
-				--image ${PX4_BINARY_DIR}/${PX4_CONFIG}.bin > ${fw_package}
+				--image ${PX4_BINARY_DIR}/${PX4_CONFIG}.bin
+				--manifest_json ${MANIFEST_JSON}
+				> ${fw_package}
 		DEPENDS
 			${PX4_BINARY_DIR}/${PX4_CONFIG}.bin
 			airframes_xml
 			parameters_xml
+			${MANIFEST_JSON}
 		COMMENT "Creating ${fw_package}"
 		WORKING_DIRECTORY ${PX4_BINARY_DIR}
 		)