From bc62fa53fbcaf766d45dd718dd7b848d2609ddfd Mon Sep 17 00:00:00 2001 From: Ramon Roche Date: Wed, 1 Dec 2021 15:39:56 -0800 Subject: [PATCH] ci: build every target on a single step Switching to use an all_variants approach to reduce the number of jobs we are creating per workflow since are starting to get throttled by GitHub, and we have limited resources. TODO: Build artifact uploadeds to S3, and GH Releases --- .github/workflows/compile_nuttx.yml | 118 --------------------------- .github/workflows/deploy_all.yml | 31 +++---- Tools/generate_board_targets_json.py | 58 ++++++++++--- 3 files changed, 58 insertions(+), 149 deletions(-) delete mode 100644 .github/workflows/compile_nuttx.yml diff --git a/.github/workflows/compile_nuttx.yml b/.github/workflows/compile_nuttx.yml deleted file mode 100644 index 2d26d4bacb..0000000000 --- a/.github/workflows/compile_nuttx.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: Nuttx Targets - -on: - push: - branches: - - 'master' - pull_request: - branches: - - '*' - -jobs: - build: - runs-on: ubuntu-latest - container: px4io/px4-dev-nuttx-focal:2021-09-08 - strategy: - fail-fast: false - matrix: - config: [ - airmind_mindpx-v2, - ark_can-flow, - ark_can-gps, - ark_can-rtk-gps, - atl_mantis-edu, - av_x-v1, - bitcraze_crazyflie, - bitcraze_crazyflie21, - cuav_can-gps-v1, - cuav_nora, - cuav_x7pro, - cubepilot_cubeorange, - cubepilot_cubeyellow, - freefly_can-rtk-gps, - holybro_can-gps-v1, - holybro_durandal-v1, - holybro_kakutef7, - holybro_pix32v5, - matek_h743-slim, - modalai_fc-v1, - modalai_fc-v2, - mro_ctrl-zero-f7, - mro_ctrl-zero-f7-oem, - mro_ctrl-zero-h7, - mro_ctrl-zero-h7-oem, - mro_pixracerpro, - mro_x21, - mro_x21-777, - nxp_fmuk66-e, - nxp_fmuk66-v3, - nxp_fmurt1062-v1, - nxp_ucans32k146, - omnibus_f4sd, - raspberrypi_pico, - px4_fmu-v2, - px4_fmu-v3, - px4_fmu-v4, - px4_fmu-v4pro, - px4_fmu-v5, - px4_fmu-v5x, - px4_fmu-v6u, - px4_fmu-v6x, - spracing_h7extreme, - uvify_core - ] - steps: - - uses: actions/checkout@v1 - with: - token: ${{secrets.ACCESS_TOKEN}} - - - name: Prepare ccache timestamp - id: ccache_cache_timestamp - shell: cmake -P {0} - run: | - string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) - message("::set-output name=timestamp::${current_date}") - - name: ccache cache files - uses: actions/cache@v2 - with: - path: ~/.ccache - key: ${{matrix.config}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}} - restore-keys: ${{matrix.config}}-ccache- - - name: setup ccache - run: | - mkdir -p ~/.ccache - echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf - echo "compression = true" >> ~/.ccache/ccache.conf - echo "compression_level = 6" >> ~/.ccache/ccache.conf - echo "max_size = 120M" >> ~/.ccache/ccache.conf - echo "hash_dir = false" >> ~/.ccache/ccache.conf - ccache -s - ccache -z - - - name: make all_variants_${{matrix.config}} - run: make all_variants_${{matrix.config}} - timeout-minutes: 45 - - name: make ${{matrix.config}} bloaty_compileunits - run: make ${{matrix.config}} bloaty_compileunits || true - - name: make ${{matrix.config}} bloaty_inlines - run: make ${{matrix.config}} bloaty_inlines || true - - name: make ${{matrix.config}} bloaty_segments - run: make ${{matrix.config}} bloaty_segments || true - - name: make ${{matrix.config}} bloaty_symbols - run: make ${{matrix.config}} bloaty_symbols || true - - name: make ${{matrix.config}} bloaty_templates - run: make ${{matrix.config}} bloaty_templates || true - - name: make ${{matrix.config}} bloaty_ram - run: make ${{matrix.config}} bloaty_ram || true - - name: make ${{matrix.config}} bloaty_compare_master - run: make ${{matrix.config}} bloaty_compare_master || true - - name: ccache post-run - run: ccache -s - - - name: Upload px4 package - uses: actions/upload-artifact@v2 - with: - name: px4_package_${{matrix.config}} - path: | - build/**/*.px4 - build/**/*.bin diff --git a/.github/workflows/deploy_all.yml b/.github/workflows/deploy_all.yml index 18480ac036..623c245bee 100644 --- a/.github/workflows/deploy_all.yml +++ b/.github/workflows/deploy_all.yml @@ -1,11 +1,17 @@ -name: Deploy metadata for all targets +name: Build and deploy all targets with metadata on: + release: + types: [published] + create: + tags: + - 'v*' push: branches: - 'master' - 'release/*' - 'pr-metadata-test' + - 'build_and_deploy_mix' jobs: enumerate_targets: @@ -18,8 +24,8 @@ jobs: with: token: ${{secrets.ACCESS_TOKEN}} - id: set-matrix - run: echo "::set-output name=matrix::$(./Tools/generate_board_targets_json.py)" - build: + run: echo "::set-output name=matrix::$(./Tools/generate_board_targets_json.py -a)" + build_all: runs-on: ubuntu-latest needs: enumerate_targets strategy: @@ -34,21 +40,4 @@ jobs: run: make ${{matrix.target}} - name: parameter & events metadata - run: | - make ${{matrix.target}} ver_gen events_json actuators_json - ./src/lib/version/get_git_tag_or_branch_version.sh build/${{ matrix.target }} >> $GITHUB_ENV - cd build/${{ matrix.target }} - mkdir _metadata || true - cp parameters.* events/*.xz actuators.json* _metadata - - - uses: jakejarvis/s3-sync-action@master - with: - args: --acl public-read - env: - AWS_S3_BUCKET: 'px4-travis' - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: 'us-west-1' - SOURCE_DIR: 'build/${{ matrix.target }}/_metadata/' - DEST_DIR: 'Firmware/${{ env.version }}/${{ matrix.target }}/' - + run: make ${{matrix.target}} ver_gen events_json actuators_json diff --git a/Tools/generate_board_targets_json.py b/Tools/generate_board_targets_json.py index 25186fa4ec..a12325fcb9 100755 --- a/Tools/generate_board_targets_json.py +++ b/Tools/generate_board_targets_json.py @@ -23,6 +23,10 @@ parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='Verbose Output') parser.add_argument('-p', '--pretty', dest='pretty', action='store_true', help='Pretty output instead of a single line') +parser.add_argument('-a', '--all_variants', dest='all_variants', action='store_true', + help='Prints targets into a single all_variants_* target') +parser.add_argument('-b', '--bloaty', dest='bloaty', action='store_true', + help='Includes all bloaty targets') args = parser.parse_args() verbose = args.verbose @@ -35,6 +39,15 @@ excluded_labels = [ 'nolockstep', 'replay', 'test', 'uavcanv1' # TODO: fix and enable ] +bloaty_helpers = [ + 'bloaty_compileunits', + 'bloaty_inlines', + 'bloaty_segments', + 'bloaty_symbols', + 'bloaty_templates', + 'bloaty_ram', + 'bloaty_compare_master', + ] def process_target(px4board_file, target_name): ret = None @@ -77,6 +90,15 @@ def process_target(px4board_file, target_name): return ret +def process_bloaty(target_path, target_name): + response = [] + for bloat in bloaty_helpers: + bloaty_name = target_name + ' ' + bloat + ' || true' + processed_target = process_target(target_path, bloaty_name) + response.append(processed_target) + return response + +# Look up boards from each manufacturer for manufacturer in os.scandir(os.path.join(source_dir, 'boards')): if not manufacturer.is_dir(): continue @@ -85,18 +107,34 @@ for manufacturer in os.scandir(os.path.join(source_dir, 'boards')): continue for board in os.scandir(manufacturer.path): + # Only boards are directories don't proceed if otherwise if not board.is_dir(): continue - for files in os.scandir(board.path): - if files.is_file() and files.name.endswith('.px4board'): - label = files.name[:-9] - target_name = manufacturer.name + '_' + board.name + '_' + label - if label in excluded_labels: - if verbose: print(f'excluding label {label} ({target_name})') - continue - target = process_target(files.path, target_name) - if target is not None: - build_configs.append(target) + + if args.all_variants: + # The all_variants target makes all targets for a board + target_name = 'all_variants_' + manufacturer.name + '_' + board.name + default_target_path = f'{board.path}/default.px4board' + target = process_target(default_target_path, target_name) + if target is not None: + build_configs.append(target) + if args.bloaty and target is not None: + # bloaty targets + bloat_target_name = manufacturer.name + '_' + board.name + bloaty_targets = process_bloaty(default_target_path, bloat_target_name) + build_configs += bloaty_targets + else: + # Each board can have multiple variant targets + for files in os.scandir(board.path): + if files.is_file() and files.name.endswith('.px4board'): + label = files.name[:-9] + target_name = manufacturer.name + '_' + board.name + '_' + label + if label in excluded_labels: + if verbose: print(f'excluding label {label} ({target_name})') + continue + target = process_target(files.path, target_name) + if target is not None: + build_configs.append(target) github_action_config = { 'include': build_configs }