From 5a1c60b5a1940dfc57079e7dbf223f098a6349a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beat=20K=C3=BCng?= Date: Fri, 4 Dec 2020 08:40:03 +0100 Subject: [PATCH] component_information: switch from gzip to xz Improves compression, e.g. current params file: 62KB to 51KB There's also a PRESET_EXTREME option, which reduces by another 2KB. We can revisit that once needed, as it increases mem usage as well. --- Jenkinsfile | 6 +++--- Tools/px_romfs_pruner.py | 2 +- cmake/px4_add_board.cmake | 4 ++-- src/lib/component_information/CMakeLists.txt | 6 +++--- .../generate_component_version.py | 8 ++++---- src/lib/parameters/CMakeLists.txt | 2 +- src/lib/parameters/px_process_params.py | 12 ++++++------ .../mavlink/streams/COMPONENT_INFORMATION.hpp | 4 ++-- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d8db30ba40..8a6b05f8d9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -112,8 +112,8 @@ pipeline { sh 'make distclean' sh 'make parameters_metadata' dir('build/px4_sitl_default/docs') { - archiveArtifacts(artifacts: 'parameters.md, parameters.xml, params.json.gz') - stash includes: 'parameters.md, parameters.xml, params.json.gz', name: 'metadata_parameters' + archiveArtifacts(artifacts: 'parameters.md, parameters.xml, params.json.xz') + stash includes: 'parameters.md, parameters.xml, params.json.xz', name: 'metadata_parameters' } } post { @@ -344,7 +344,7 @@ pipeline { withAWS(credentials: 'px4_aws_s3_key', region: 'us-east-1') { s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'airframes.xml', path: 'Firmware/master/') s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'parameters.xml', path: 'Firmware/master/') - s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'params.json.gz', path: 'Firmware/master/') + s3Upload(acl: 'PublicRead', bucket: 'px4-travis', file: 'params.json.xz', path: 'Firmware/master/') } } when { diff --git a/Tools/px_romfs_pruner.py b/Tools/px_romfs_pruner.py index f0280a959b..31d9de20d3 100644 --- a/Tools/px_romfs_pruner.py +++ b/Tools/px_romfs_pruner.py @@ -83,7 +83,7 @@ def main(): # only prune text files if ".zip" in file or ".bin" in file or ".swp" in file \ - or ".gz" in file or ".bz2" in file \ + or ".gz" in file or ".xz" in file or ".bz2" in file \ or ".data" in file or ".DS_Store" in file: continue diff --git a/cmake/px4_add_board.cmake b/cmake/px4_add_board.cmake index 6d64f4cf9a..df9a339c71 100644 --- a/cmake/px4_add_board.cmake +++ b/cmake/px4_add_board.cmake @@ -209,13 +209,13 @@ function(px4_add_board) endif() foreach(metadata ${EMBEDDED_METADATA}) if(${metadata} STREQUAL "parameters") - list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/params.json.gz) + list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/params.json.xz) list(APPEND romfs_extra_dependencies parameters_xml) else() message(FATAL_ERROR "invalid value for EMBEDDED_METADATA: ${metadata}") endif() endforeach() - list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/component_version.json.gz) + list(APPEND romfs_extra_files ${PX4_BINARY_DIR}/component_version.json.xz) list(APPEND romfs_extra_dependencies component_version_json) set(config_romfs_extra_files ${romfs_extra_files} CACHE INTERNAL "extra ROMFS files" FORCE) set(config_romfs_extra_dependencies ${romfs_extra_dependencies} CACHE INTERNAL "extra ROMFS deps" FORCE) diff --git a/src/lib/component_information/CMakeLists.txt b/src/lib/component_information/CMakeLists.txt index a8d4a461aa..02960e4005 100644 --- a/src/lib/component_information/CMakeLists.txt +++ b/src/lib/component_information/CMakeLists.txt @@ -40,7 +40,7 @@ if (${index} EQUAL -1) endif() set(component_version_json ${PX4_BINARY_DIR}/component_version.json) -add_custom_command(OUTPUT ${component_version_json} ${component_version_json}.gz +add_custom_command(OUTPUT ${component_version_json} ${component_version_json}.xz COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_component_version.py ${component_version_json} --compress @@ -55,13 +55,13 @@ add_custom_target(component_version_json DEPENDS ${component_version_json}) set(component_information_header ${CMAKE_CURRENT_BINARY_DIR}/hashes.h) add_custom_command(OUTPUT ${component_information_header} COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_hashes.py - ${PX4_BINARY_DIR}/params.json.gz + ${PX4_BINARY_DIR}/params.json.xz ${component_version_json} --output ${component_information_header} DEPENDS generate_hashes.py parameters_xml - ${PX4_BINARY_DIR}/params.json.gz + ${PX4_BINARY_DIR}/params.json.xz ${component_version_json} COMMENT "Generating component_information/hashes.h" ) diff --git a/src/lib/component_information/generate_component_version.py b/src/lib/component_information/generate_component_version.py index ed178d3512..55422cac06 100755 --- a/src/lib/component_information/generate_component_version.py +++ b/src/lib/component_information/generate_component_version.py @@ -2,7 +2,7 @@ import argparse import json -import gzip +import lzma #to create .xz file parser = argparse.ArgumentParser(description="""Generate the COMPONENT_VERSION json file""") parser.add_argument('filename', metavar='component_version.json', help='JSON output file') @@ -14,9 +14,9 @@ filename = args.filename compress = args.compress def save_compressed(filename): - #create gz compressed version - gz_filename=filename+'.gz' - with gzip.open(gz_filename, 'wt') as f: + #create lzma compressed version + xz_filename=filename+'.xz' + with lzma.open(xz_filename, 'wt', preset=9) as f: with open(filename, 'r') as content_file: f.write(content_file.read()) diff --git a/src/lib/parameters/CMakeLists.txt b/src/lib/parameters/CMakeLists.txt index 8aa91190ff..c68e1dbc18 100644 --- a/src/lib/parameters/CMakeLists.txt +++ b/src/lib/parameters/CMakeLists.txt @@ -89,7 +89,7 @@ add_custom_command(OUTPUT ${generated_serial_params_file} set(parameters_xml ${PX4_BINARY_DIR}/parameters.xml) set(parameters_json ${PX4_BINARY_DIR}/params.json) # file name needs to be kept short to fit into url (metadata_uri) file(GLOB_RECURSE param_src_files ${PX4_SOURCE_DIR}/src/*params.c) -add_custom_command(OUTPUT ${parameters_xml} ${parameters_json} ${parameters_json}.gz +add_custom_command(OUTPUT ${parameters_xml} ${parameters_json} ${parameters_json}.xz COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/px_process_params.py --src-path ${module_list} ${generated_params_dir} --xml ${parameters_xml} diff --git a/src/lib/parameters/px_process_params.py b/src/lib/parameters/px_process_params.py index a52b4d286c..0c777bb1d4 100755 --- a/src/lib/parameters/px_process_params.py +++ b/src/lib/parameters/px_process_params.py @@ -49,15 +49,15 @@ import os import argparse from px4params import srcscanner, srcparser, injectxmlparams, xmlout, markdownout, jsonout -import gzip #to create .gz file +import lzma #to create .xz file import re import json import codecs -def SaveCompressed(filename): - #create gz compressed version - gz_filename=filename+'.gz' - with gzip.open(gz_filename, 'wt') as f: +def save_compressed(filename): + #create lzma compressed version + xz_filename=filename+'.xz' + with lzma.open(xz_filename, 'wt', preset=9) as f: with open(filename, 'r') as content_file: f.write(content_file.read()) @@ -185,7 +185,7 @@ def main(): for f in output_files: if args.verbose: print("Compressing file " + f) - SaveCompressed(f) + save_compressed(f) if __name__ == "__main__": diff --git a/src/modules/mavlink/streams/COMPONENT_INFORMATION.hpp b/src/modules/mavlink/streams/COMPONENT_INFORMATION.hpp index 770123397b..808e86820d 100644 --- a/src/modules/mavlink/streams/COMPONENT_INFORMATION.hpp +++ b/src/modules/mavlink/streams/COMPONENT_INFORMATION.hpp @@ -69,12 +69,12 @@ public: switch (type) { case COMP_METADATA_TYPE_VERSION: component_info.metadata_uid = component_information::component_version_hash; - handled = get_component_information("component_version.json.gz", component_info); + handled = get_component_information("component_version.json.xz", component_info); break; case COMP_METADATA_TYPE_PARAMETER: component_info.metadata_uid = component_information::params_hash; - handled = get_component_information("params.json.gz", component_info); + handled = get_component_information("params.json.xz", component_info); break; case COMP_METADATA_TYPE_COMMANDS: