diff --git a/Jenkinsfile b/Jenkinsfile index c7c3f119e1..4cdc252ff6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -110,8 +110,8 @@ pipeline { sh 'make distclean' sh 'make parameters_metadata' dir('build/px4_sitl_default/docs') { - archiveArtifacts(artifacts: 'parameters.md, parameters.xml, params.json.xz') - stash includes: 'parameters.md, parameters.xml, params.json.xz', name: 'metadata_parameters' + archiveArtifacts(artifacts: 'parameters.md, parameters.xml, parameters.json.xz') + stash includes: 'parameters.md, parameters.xml, parameters.json.xz', name: 'metadata_parameters' } } post { @@ -342,7 +342,7 @@ pipeline { withAWS(credentials: 'px4_aws_s3_key', region: 'us-east-1') { s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'airframes.xml', path: 'Firmware/master/') s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'parameters.xml', path: 'Firmware/master/') - s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'params.json.xz', path: 'Firmware/master/') + s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'parameters.json.xz', path: 'Firmware/master/') } } when { diff --git a/cmake/metadata.cmake b/cmake/metadata.cmake index 56ba664041..48e8c22502 100644 --- a/cmake/metadata.cmake +++ b/cmake/metadata.cmake @@ -59,7 +59,7 @@ add_custom_target(metadata_parameters COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/src/lib/parameters/px_process_params.py --src-path `find ${PX4_SOURCE_DIR}/src -maxdepth 4 -type d` --inject-xml ${PX4_SOURCE_DIR}/src/lib/parameters/parameters_injected.xml - --json ${PX4_BINARY_DIR}/docs/params.json + --json ${PX4_BINARY_DIR}/docs/parameters.json --compress COMMAND ${PYTHON_EXECUTABLE} ${PX4_SOURCE_DIR}/src/lib/parameters/px_process_params.py diff --git a/cmake/px4_add_board.cmake b/cmake/px4_add_board.cmake index d718359f41..ef65b70964 100644 --- a/cmake/px4_add_board.cmake +++ b/cmake/px4_add_board.cmake @@ -209,14 +209,14 @@ function(px4_add_board) set(config_romfs_extra_dependencies) foreach(metadata ${EMBEDDED_METADATA}) if(${metadata} STREQUAL "parameters") - list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/params.json.xz) + list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/parameters.json.xz) list(APPEND romfs_extra_dependencies parameters_xml) else() message(FATAL_ERROR "invalid value for EMBEDDED_METADATA: ${metadata}") endif() endforeach() - list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/component_version.json.xz) - list(APPEND romfs_extra_dependencies component_version_json) + list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/component_general.json.xz) + list(APPEND romfs_extra_dependencies component_general_json) set(config_romfs_extra_files ${romfs_extra_files} CACHE INTERNAL "extra ROMFS files" FORCE) set(config_romfs_extra_dependencies ${romfs_extra_dependencies} CACHE INTERNAL "extra ROMFS deps" FORCE) diff --git a/mavlink/include/mavlink/v2.0 b/mavlink/include/mavlink/v2.0 index 45eda11755..9b429ea239 160000 --- a/mavlink/include/mavlink/v2.0 +++ b/mavlink/include/mavlink/v2.0 @@ -1 +1 @@ -Subproject commit 45eda11755c726fbff2a893b6e13e35dbd075a34 +Subproject commit 9b429ea239d7230d7f0329cfbec4a817162670ea diff --git a/src/lib/component_information/CMakeLists.txt b/src/lib/component_information/CMakeLists.txt index 02960e4005..b030717894 100644 --- a/src/lib/component_information/CMakeLists.txt +++ b/src/lib/component_information/CMakeLists.txt @@ -31,39 +31,55 @@ # ############################################################################ -set(component_version_extra_args) +# types: +# 0: COMP_METADATA_TYPE_GENERAL +# 1: COMP_METADATA_TYPE_PARAMETER +# 2: COMP_METADATA_TYPE_COMMANDS +# 3: COMP_METADATA_TYPE_PERIPHERALS +set(comp_metadata_types) # We disable parameter support if the metadata is not in the ROMFS. This can be # removed once it is stored on a server list(FIND config_romfs_extra_dependencies "parameters_xml" index) +set(comp_metadata_board "${PX4_BOARD_VENDOR}_${PX4_BOARD_MODEL}_${PX4_BOARD_LABEL}") +set(s3_url "https://px4-travis.s3.amazonaws.com") +set(comp_metadata_param_uri_board "${s3_url}/Firmware/{version}/${comp_metadata_board}/parameters.json.xz") if (${index} EQUAL -1) - set(component_version_extra_args "--no-parameters") + set(comp_metadata_param_uri ${comp_metadata_param_uri_board}) + # use generic URL as fallback + set(comp_metadata_param_uri_fallback "${s3_url}/Firmware/{version}/_general/parameters.json.xz") +else() + set(comp_metadata_param_uri "mftp://etc/extras/parameters.json.xz") + set(comp_metadata_param_uri_fallback ${comp_metadata_param_uri_board}) endif() +# arguments: type, metadata file, uri, fallback uri, optional translation uri +list(APPEND comp_metadata_types "--type" "1,${PX4_BINARY_DIR}/parameters.json.xz,${comp_metadata_param_uri},${comp_metadata_param_uri_fallback},") -set(component_version_json ${PX4_BINARY_DIR}/component_version.json) -add_custom_command(OUTPUT ${component_version_json} ${component_version_json}.xz - COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_component_version.py - ${component_version_json} +set(component_general_json ${PX4_BINARY_DIR}/component_general.json) +add_custom_command(OUTPUT ${component_general_json} ${component_general_json}.xz + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_component_general.py + ${component_general_json} --compress - ${component_version_extra_args} + ${comp_metadata_types} + --version-file ${PX4_BINARY_DIR}/src/lib/version/build_git_version.h DEPENDS - generate_component_version.py - COMMENT "Generating component_version.json" + generate_component_general.py + ${PX4_BINARY_DIR}/parameters.json.xz + parameters_xml + ver_gen + COMMENT "Generating component_general.json" ) -add_custom_target(component_version_json DEPENDS ${component_version_json}) +add_custom_target(component_general_json DEPENDS ${component_general_json}) -set(component_information_header ${CMAKE_CURRENT_BINARY_DIR}/hashes.h) +set(component_information_header ${CMAKE_CURRENT_BINARY_DIR}/checksums.h) add_custom_command(OUTPUT ${component_information_header} - COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_hashes.py - ${PX4_BINARY_DIR}/params.json.xz - ${component_version_json} + COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_crc.py + ${component_general_json} --output ${component_information_header} DEPENDS - generate_hashes.py - parameters_xml - ${PX4_BINARY_DIR}/params.json.xz - ${component_version_json} - COMMENT "Generating component_information/hashes.h" + generate_crc.py + ${component_general_json} + COMMENT "Generating component_information/checksums.h" ) add_custom_target(component_information_header DEPENDS ${component_information_header}) diff --git a/src/lib/component_information/generate_component_general.py b/src/lib/component_information/generate_component_general.py new file mode 100755 index 0000000000..6b8f605913 --- /dev/null +++ b/src/lib/component_information/generate_component_general.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python + +import argparse +import json +import lzma #to create .xz file +import re + +parser = argparse.ArgumentParser(description="""Generate the COMPONENT_GENERAL json file""") +parser.add_argument('filename', metavar='component_general.json', help='JSON output file') +parser.add_argument('--compress', action='store_true', help='Add a compressed output file') +parser.add_argument('--type', metavar='type', action="append", default=[], + help='Metadata type (,,,,[translation uri])') +parser.add_argument('--version-file', metavar='build_git_version.h', help='Git version file') + +args = parser.parse_args() +filename = args.filename +compress = args.compress +version_file = args.version_file + +version_dir = '' +if version_file is not None: + for line in open(version_file, "r"): + version_search = re.search('PX4_GIT_TAG_OR_BRANCH_NAME\s+"(.+)"', line) + if version_search: + version_dir = version_search.group(1) + break + + +def save_compressed(filename): + #create lzma compressed version + xz_filename=filename+'.xz' + with lzma.open(xz_filename, 'wt', preset=9) as f: + with open(filename, 'r') as content_file: + f.write(content_file.read()) + +component_general = {} +component_general['version'] = 1 + +def create_table(): + a = [] + for i in range(256): + k = i + for j in range(8): + if k & 1: + k ^= 0x1db710640 + k >>= 1 + a.append(k) + return a + +# MAVLink's CRC32 +def crc_update(buf, crc_table, crc): + for k in buf: + crc = (crc >> 8) ^ crc_table[(crc & 0xff) ^ k] + return crc + +crc_table = create_table() + +metadata_types = [] +for metadata_type_tuple in args.type: + type_id, metadata_file, uri, fallback_uri, translation_uri = metadata_type_tuple.split(',') + file_crc = 0 + for line in open(metadata_file, "rb"): + file_crc = crc_update(line, crc_table, file_crc) + json_type = { + 'type': int(type_id), + 'uri': uri.replace('{version}', version_dir), + 'fileCrc': file_crc, + 'uriFallback': fallback_uri.replace('{version}', version_dir), + } + if len(translation_uri) > 0: + json_type['translationUri'] = translation_uri + metadata_types.append(json_type) +component_general['metadataTypes'] = metadata_types + +with open(filename, 'w') as outfile: + json.dump(component_general, outfile) + +if compress: + save_compressed(filename) diff --git a/src/lib/component_information/generate_component_version.py b/src/lib/component_information/generate_component_version.py deleted file mode 100755 index 55422cac06..0000000000 --- a/src/lib/component_information/generate_component_version.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python - -import argparse -import json -import lzma #to create .xz file - -parser = argparse.ArgumentParser(description="""Generate the COMPONENT_VERSION json file""") -parser.add_argument('filename', metavar='component_version.json', help='JSON output file') -parser.add_argument('--compress', action='store_true', help='Add a compressed output file') -parser.add_argument('--no-parameters', action='store_true', help='disable parameter support') - -args = parser.parse_args() -filename = args.filename -compress = args.compress - -def save_compressed(filename): - #create lzma compressed version - xz_filename=filename+'.xz' - with lzma.open(xz_filename, 'wt', preset=9) as f: - with open(filename, 'r') as content_file: - f.write(content_file.read()) - -component_version = {} -component_version['version'] = 1 -# types: -# 0: COMP_METADATA_TYPE_VERSION -# 1: COMP_METADATA_TYPE_PARAMETER -# 2: COMP_METADATA_TYPE_COMMANDS -supported_types = [0] -if not args.no_parameters: supported_types.append(1) -component_version['supportedCompMetadataTypes'] = supported_types - -with open(filename, 'w') as outfile: - json.dump(component_version, outfile) - -if compress: - save_compressed(filename) diff --git a/src/lib/component_information/generate_crc.py b/src/lib/component_information/generate_crc.py new file mode 100755 index 0000000000..84b85cf9a6 --- /dev/null +++ b/src/lib/component_information/generate_crc.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +import argparse +import os + +parser = argparse.ArgumentParser(description="""Generate the COMPONENT_INFORMATION checksums (CRC32) header file""") +parser.add_argument('--output', metavar='checksums.h', help='output file') +parser.add_argument('files', nargs='+', metavar='input_file.json', help='files to generate CRC from') + +args = parser.parse_args() +filename = args.output +files = args.files + +def create_table(): + a = [] + for i in range(256): + k = i + for j in range(8): + if k & 1: + k ^= 0x1db710640 + k >>= 1 + a.append(k) + return a + +# MAVLink's CRC32 +def crc_update(buf, crc_table, crc): + for k in buf: + crc = (crc >> 8) ^ crc_table[(crc & 0xff) ^ k] + return crc + +crc_table = create_table() + +with open(filename, 'w') as outfile: + outfile.write("#include \n") + outfile.write("namespace component_information {\n") + for filename in files: + file_crc = 0 + for line in open(filename, "rb"): + file_crc = crc_update(line, crc_table, file_crc) + + basename = os.path.basename(filename) + identifier = basename.split('.')[0] + outfile.write("static constexpr uint32_t {:}_crc = {:};\n".format(identifier, file_crc)) + + + outfile.write("}\n") + + diff --git a/src/lib/component_information/generate_hashes.py b/src/lib/component_information/generate_hashes.py deleted file mode 100755 index 2d148340e4..0000000000 --- a/src/lib/component_information/generate_hashes.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python - -import argparse -import zlib -import os - -parser = argparse.ArgumentParser(description="""Generate the COMPONENT_INFORMATION hashes header file""") -parser.add_argument('--output', metavar='hashes.h', help='output file') -parser.add_argument('files', nargs='+', metavar='hashes.json', help='files to generate hashes from') - -args = parser.parse_args() -filename = args.output -files = args.files - -with open(filename, 'w') as outfile: - outfile.write("#include \n") - outfile.write("namespace component_information {\n") - for filename in files: - # get CRC - file_hash = 0 - for line in open(filename, "rb"): - file_hash = zlib.crc32(line, file_hash) - - basename = os.path.basename(filename) - identifier = basename.split('.')[0] - outfile.write("static constexpr uint32_t {:}_hash = {:};\n".format(identifier, file_hash)) - - - outfile.write("}\n") - - diff --git a/src/lib/parameters/CMakeLists.txt b/src/lib/parameters/CMakeLists.txt index 55a2cd15ff..b698c3ac4a 100644 --- a/src/lib/parameters/CMakeLists.txt +++ b/src/lib/parameters/CMakeLists.txt @@ -92,7 +92,7 @@ add_custom_command(OUTPUT ${generated_serial_params_file} ) set(parameters_xml ${PX4_BINARY_DIR}/parameters.xml) -set(parameters_json ${PX4_BINARY_DIR}/params.json) # file name needs to be kept short to fit into url (metadata_uri) +set(parameters_json ${PX4_BINARY_DIR}/parameters.json) file(GLOB_RECURSE param_src_files ${PX4_SOURCE_DIR}/src/*params.c) add_custom_command(OUTPUT ${parameters_xml} ${parameters_json} ${parameters_json}.xz COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/px_process_params.py diff --git a/src/lib/parameters/px4params/jsonout.py b/src/lib/parameters/px4params/jsonout.py index 801824f46d..6c7acbb958 100644 --- a/src/lib/parameters/px4params/jsonout.py +++ b/src/lib/parameters/px4params/jsonout.py @@ -8,7 +8,6 @@ class JsonOutput(): def __init__(self, groups, board, inject_xml_file_name): all_json=dict() all_json['version']=1 - all_json['scope']="Firmware" all_params=[] all_json['parameters']=all_params diff --git a/src/modules/mavlink/streams/COMPONENT_INFORMATION.hpp b/src/modules/mavlink/streams/COMPONENT_INFORMATION.hpp index 808e86820d..d4bc184d08 100644 --- a/src/modules/mavlink/streams/COMPONENT_INFORMATION.hpp +++ b/src/modules/mavlink/streams/COMPONENT_INFORMATION.hpp @@ -36,7 +36,7 @@ #include "../mavlink_stream.h" -#include +#include #include @@ -61,34 +61,17 @@ public: bool request_message(float param2, float param3, float param4, float param5, float param6, float param7) override { - int type = (int)roundf(param2); mavlink_component_information_t component_info{}; - bool handled = false; - PX4_DEBUG("COMPONENT_INFORMATION request type %i", type); + PX4_DEBUG("COMPONENT_INFORMATION request"); - switch (type) { - case COMP_METADATA_TYPE_VERSION: - component_info.metadata_uid = component_information::component_version_hash; - handled = get_component_information("component_version.json.xz", component_info); - break; + strncpy(component_info.general_metadata_uri, "mftp://etc/extras/component_general.json.xz", + sizeof(component_info.general_metadata_uri) - 1); + component_info.general_metadata_file_crc = component_information::component_general_crc; - case COMP_METADATA_TYPE_PARAMETER: - component_info.metadata_uid = component_information::params_hash; - handled = get_component_information("params.json.xz", component_info); - break; + component_info.time_boot_ms = hrt_absolute_time() / 1000; + mavlink_msg_component_information_send_struct(_mavlink->get_channel(), &component_info); - case COMP_METADATA_TYPE_COMMANDS: - // TODO - break; - } - - if (handled) { - component_info.metadata_type = type; - component_info.time_boot_ms = hrt_absolute_time() / 1000; - mavlink_msg_component_information_send_struct(_mavlink->get_channel(), &component_info); - } - - return handled; + return true; } private: explicit MavlinkStreamComponentInformation(Mavlink *mavlink) : MavlinkStream(mavlink) {} @@ -97,40 +80,6 @@ private: { return false; } - - bool get_component_information(const char *file, mavlink_component_information_t &component_info) - { - char full_path[64]; - snprintf(full_path, sizeof(full_path), "%s/etc/extras/%s", PX4_ROOTFSDIR, file); - full_path[sizeof(full_path) - 1] = '\0'; - - if (file_exist(full_path)) { - if (snprintf(component_info.metadata_uri, sizeof(component_info.metadata_uri), "mftp://etc/extras/%s", file) - >= (int)sizeof(component_info.metadata_uri)) { - PX4_ERR("path too long (%s)", file); - return false; - } - - } else { - // TODO: - // - check for tagged version, use per-version files - // - generate & use board-specific file - if (snprintf(component_info.metadata_uri, sizeof(component_info.metadata_uri), - "https://px4-travis.s3.amazonaws.com/Firmware/master/%s", file) - >= (int)sizeof(component_info.metadata_uri)) { - PX4_ERR("url too long (%s)", file); - return false; - } - } - - return true; - } - - bool file_exist(const char *filename) - { - struct stat buffer; - return stat(filename, &buffer) == 0; - } }; #endif // COMPONENT_INFORMATION_HPP