From f7a20acba665cc172b54356ea4aa69d3a52882de Mon Sep 17 00:00:00 2001 From: Daniel Agar Date: Sat, 25 Aug 2018 14:59:33 -0400 Subject: [PATCH] [WIP] Jenkins code coverage updates --- .ci/Jenkinsfile-SITL_tests | 209 ++++++++++++++++++++++++------------ .ci/Jenkinsfile-hardware | 12 ++- cmake/common/coverage.cmake | 76 ++++++------- 3 files changed, 186 insertions(+), 111 deletions(-) diff --git a/.ci/Jenkinsfile-SITL_tests b/.ci/Jenkinsfile-SITL_tests index 8aa407e4d5..f76852fae8 100644 --- a/.ci/Jenkinsfile-SITL_tests +++ b/.ci/Jenkinsfile-SITL_tests @@ -13,73 +13,69 @@ pipeline { stages { stage('Build') { - parallel { - stage('sitl package') { - agent { - docker { - image 'px4io/px4-dev-ros:2018-07-19' - args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE' - } - } + agent { + docker { + image 'px4io/px4-dev-ros:2018-08-23' + args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw -e HOME=$WORKSPACE' + } + } + + stages { + + stage('build px4') { steps { sh('export') sh('make distclean') sh "ccache -z" sh('make posix_sitl_default') sh "ccache -s" - sh('make posix_sitl_default sitl_gazebo') - sh "ccache -s" - sh('make posix_sitl_default package') - stash(name: "px4_sitl_package", includes: "build/posix_sitl_default/*.bz2") - sh 'make distclean' } } - stage('unit tests') { - agent { - docker { - image 'px4io/px4-dev-base:2018-07-19' - args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw' - } - } + stage('stash build for coverage') { steps { - sh 'export' - sh 'make distclean' - sh 'ccache -z' - sh 'make posix_sitl_default test_results_junit' - sh 'ccache -s' - junit 'build/posix_sitl_default/JUnitTestResults.xml' - withCredentials([string(credentialsId: 'FIRMWARE_CODECOV_TOKEN', variable: 'CODECOV_TOKEN')]) { - sh 'curl -s https://codecov.io/bash | bash -s - -F unittest' - } - sh 'make distclean' - } - } - - stage('code coverage (python)') { - agent { - docker { - image 'px4io/px4-dev-base:2018-08-04' - args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw' - } - } - steps { - sh 'export' - sh 'make distclean' - sh 'make python_coverage' - withCredentials([string(credentialsId: 'FIRMWARE_CODECOV_TOKEN', variable: 'CODECOV_TOKEN')]) { - sh 'curl -s https://codecov.io/bash | bash -s - -F python' - } - sh 'make distclean' + stash includes: 'build/**/*', name: 'build_sitl_coverage', useDefaultExcludes: false } when { environment name: 'PX4_CMAKE_BUILD_TYPE', value: 'Coverage' } } - } // parallel - } // stage Build + stage('unit tests') { + steps { + sh 'export' + sh 'make posix_sitl_default test_results_junit' + junit 'build/posix_sitl_default/JUnitTestResults.xml' + } + } + + stage('build sitl_gazebo') { + steps { + sh 'export' + sh "ccache -z" + sh('make posix_sitl_default sitl_gazebo') + sh "ccache -s" + } + } + + stage('package') { + steps { + sh 'export' + sh('make posix_sitl_default package') + stash(name: "px4_sitl_package", includes: "build/posix_sitl_default/*.bz2") + } + } + + } + + post { + always { + sh 'make distclean' + } + } + + } stage('ROS Tests') { steps { @@ -93,38 +89,38 @@ pipeline { ], [ - name: "MC box", + name: "MC_box", test: "mavros_posix_test_mission.test", mission: "MC_mission_box", vehicle: "iris" ], [ - name: "MC offboard att", + name: "MC_offboard_att", test: "mavros_posix_tests_offboard_attctl.test", mission: "", vehicle: "iris" ], [ - name: "MC offboard pos", + name: "MC_offboard_pos", test: "mavros_posix_tests_offboard_posctl.test", mission: "", vehicle: "iris" ], [ - name: "VTOL standard", + name: "VTOL_standard", test: "mavros_posix_test_mission.test", mission: "VTOL_mission_1", vehicle: "standard_vtol" ], [ - name: "VTOL tailsitter", + name: "VTOL_tailsitter", test: "mavros_posix_test_mission.test", mission: "VTOL_mission_1", vehicle: "tailsitter" ], [ - name: "VTOL tiltrotor", + name: "VTOL_tiltrotor", test: "mavros_posix_test_mission.test", mission: "VTOL_mission_1", vehicle: "tiltrotor" @@ -142,6 +138,56 @@ pipeline { } // steps } // ROS Tests + stage('Coverage') { + parallel { + + stage('code coverage (python)') { + agent { + docker { + image 'px4io/px4-dev-base:2018-08-23' + args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw' + } + } + steps { + sh 'export' + sh 'make distclean' + sh 'make python_coverage' + withCredentials([string(credentialsId: 'FIRMWARE_CODECOV_TOKEN', variable: 'CODECOV_TOKEN')]) { + sh 'curl -s https://codecov.io/bash | bash -s - -F python' + } + + sh 'make distclean' + } + + when { + environment name: 'PX4_CMAKE_BUILD_TYPE', value: 'Coverage' + } + } + + stage('unit tests') { + agent { + docker { + image 'px4io/px4-dev-base:2018-08-23' + args '-e CCACHE_BASEDIR=$WORKSPACE -v ${CCACHE_DIR}:${CCACHE_DIR}:rw' + } + } + steps { + sh 'export' + sh 'make distclean' + sh 'make posix_sitl_default test_results_junit' + withCredentials([string(credentialsId: 'FIRMWARE_CODECOV_TOKEN', variable: 'CODECOV_TOKEN')]) { + sh 'curl -s https://codecov.io/bash | bash -s - -F unittest' + } + sh 'make distclean' + } + when { + environment name: 'PX4_CMAKE_BUILD_TYPE', value: 'Coverage' + } + } + + } // parallel + } // stage Coverage + } //stages environment { CCACHE_DIR = '/tmp/ccache' @@ -157,26 +203,47 @@ def createTestNode(Map test_def) { return { node { cleanWs() - docker.image("px4io/px4-dev-ros:2018-03-30").inside('-e HOME=${WORKSPACE}') { + docker.image("px4io/px4-dev-ros:2018-08-23").inside('-e HOME=${WORKSPACE}') { stage(test_def.name) { - try { - sh('export') - unstash('px4_sitl_package') - sh('tar -xjpvf build/posix_sitl_default/px4-posix_sitl_default*.bz2') - sh('px4-posix_sitl_default*/px4/test/rostest_px4_run.sh ' + test_def.test + ' mission:=' + test_def.mission + ' vehicle:=' + test_def.vehicle) - sh('px4-posix_sitl_default*/px4/Tools/ecl_ekf/process_logdata_ekf.py .ros/log/*/*.ulg') - } - catch (exc) { - archiveArtifacts(allowEmptyArchive: false, artifacts: '.ros/**/*.ulg, .ros/**/rosunit-*.xml, .ros/**/rostest-*.log') - throw (exc) + + sh('export') + + if (env.PX4_CMAKE_BUILD_TYPE == 'Coverage') { + checkout(scm) + unstash 'build_sitl_coverage' } - finally { - sh('px4-posix_sitl_default*/px4/Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/log/*/*.ulg') - archiveArtifacts(allowEmptyArchive: false, artifacts: '.ros/**/*.pdf, .ros/**/*.csv') + + unstash('px4_sitl_package') + sh('tar -xjpvf build/posix_sitl_default/px4-posix_sitl_default*.bz2') + sh('px4-posix_sitl_default*/px4/test/rostest_px4_run.sh ' + test_def.test + ' mission:=' + test_def.mission + ' vehicle:=' + test_def.vehicle) + + if (env.PX4_CMAKE_BUILD_TYPE == 'Coverage') { withCredentials([string(credentialsId: 'FIRMWARE_CODECOV_TOKEN', variable: 'CODECOV_TOKEN')]) { - sh 'curl -s https://codecov.io/bash | bash -s - -F mission' + sh 'curl -s https://codecov.io/bash | bash -s - -F sitl_mission_${STAGE_NAME}' + + // upload log to flight review (https://logs.px4.io/) with python code coverage + sh('coverage run -p px4-posix_sitl_default*/px4/Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/log/*/*.ulg') + + // process log data (with python code coverage) + sh('coverage run -p px4-posix_sitl_default*/px4/Tools/ecl_ekf/process_logdata_ekf.py .ros/log/*/*.ulg') + + // upload python code coverage to codecov.io + sh 'curl -s https://codecov.io/bash | bash -s - -X gcov -F sitl_python_${STAGE_NAME}' } } + + // log analysis (non code coverage) + if (env.PX4_CMAKE_BUILD_TYPE != 'Coverage') { + // upload log to flight review (https://logs.px4.io/) + sh('px4-posix_sitl_default*/px4/Tools/upload_log.py -q --description "${JOB_NAME}: ${STAGE_NAME}" --feedback "${JOB_NAME} ${CHANGE_TITLE} ${CHANGE_URL}" --source CI .ros/log/*/*.ulg') + + sh('px4-posix_sitl_default*/px4/Tools/ecl_ekf/process_logdata_ekf.py .ros/log/*/*.ulg') + } + + // save all test artifacts for debugging + archiveArtifacts(allowEmptyArchive: false, artifacts: '.ros/**/*.ulg, .ros/**/rosunit-*.xml, .ros/**/rostest-*.log') + archiveArtifacts(allowEmptyArchive: false, artifacts: '.ros/**/*.pdf, .ros/**/*.csv') + } } cleanWs() diff --git a/.ci/Jenkinsfile-hardware b/.ci/Jenkinsfile-hardware index c233633a40..0ef4160728 100644 --- a/.ci/Jenkinsfile-hardware +++ b/.ci/Jenkinsfile-hardware @@ -23,7 +23,11 @@ pipeline { sh 'ccache -s' stash includes: 'build/nuttx_px4fmu-v2_test/nuttx_px4fmu-v2_test.elf', name: 'px4fmu-v2_test' stash includes: 'Tools/HIL/monitor_firmware_upload.py, Tools/HIL/run_tests.py', name: 'scripts-px4fmu-v2' - sh 'make distclean' + } + post { + always { + sh 'make distclean' + } } } @@ -44,7 +48,11 @@ pipeline { sh 'ccache -s' stash includes: 'build/nuttx_px4fmu-v4_default/nuttx_px4fmu-v4_default.elf', name: 'px4fmu-v4_default' stash includes: 'Tools/HIL/monitor_firmware_upload.py, Tools/HIL/run_tests.py', name: 'scripts-px4fmu-v4' - sh 'make distclean' + } + post { + always { + sh 'make distclean' + } } } diff --git a/cmake/common/coverage.cmake b/cmake/common/coverage.cmake index e963be4c6a..7d30f06a16 100644 --- a/cmake/common/coverage.cmake +++ b/cmake/common/coverage.cmake @@ -56,53 +56,53 @@ mark_as_advanced(CMAKE_CXX_FLAGS_COVERAGE CMAKE_C_FLAGS_COVERAGE CMAKE_EXE_LINKE # Pass them in list form, e.g.: "-j;2" for -j 2 FUNCTION(SETUP_TARGET_FOR_COVERAGE _targetname _testrunner _outputname) - set(options NONE) - set(oneValueArgs NAME) - set(multiValueArgs EXECUTABLE EXECUTABLE_ARGS DEPENDENCIES) - cmake_parse_arguments(Coverage "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + set(options NONE) + set(oneValueArgs NAME) + set(multiValueArgs EXECUTABLE EXECUTABLE_ARGS DEPENDENCIES) + cmake_parse_arguments(Coverage "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - if (NOT LCOV_PATH) - message(FATAL_ERROR "lcov required") - endif() + if (NOT LCOV_PATH) + message(FATAL_ERROR "lcov required") + endif() - if (NOT GENHTML_PATH) - message(FATAL_ERROR "genhtml required") - endif() + if (NOT GENHTML_PATH) + message(FATAL_ERROR "genhtml required") + endif() - set(coverage_info "${CMAKE_BINARY_DIR}/${_outputname}.info") - set(coverage_cleaned "${coverage_info}.cleaned") + set(coverage_info "${CMAKE_BINARY_DIR}/${_outputname}.info") + set(coverage_cleaned "${coverage_info}.cleaned") - separate_arguments(test_command UNIX_COMMAND "${_testrunner}") + separate_arguments(test_command UNIX_COMMAND "${_testrunner}") - # Setup target - add_custom_command(OUTPUT ${coverage_info} - # Cleanup lcov - #COMMAND ${LCOV_PATH} --quiet --directory . --zerocounters + # Setup target + add_custom_command(OUTPUT ${coverage_info} + # Cleanup lcov + #COMMAND ${LCOV_PATH} --quiet --directory . --zerocounters - # Run tests - COMMAND ${test_command} ${ARGV3} + # Run tests + COMMAND ${test_command} ${ARGV3} - # Capturing lcov counters and generating report - COMMAND ${LCOV_PATH} --quiet --base-directory ${CMAKE_BINARY_DIR} --directory ${CMAKE_SOURCE_DIR} --capture --output-file ${coverage_info} + # Capturing lcov counters and generating report + COMMAND ${LCOV_PATH} --quiet --base-directory ${CMAKE_BINARY_DIR} --directory ${CMAKE_SOURCE_DIR} --capture --output-file ${coverage_info} - DEPENDS px4 - USES_TERMINAL - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - COMMENT "Running code coverage and generating report." - ) - add_custom_target(${_targetname} DEPENDS ${coverage_info}) + DEPENDS px4 + USES_TERMINAL + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + COMMENT "Running code coverage and generating report." + ) + add_custom_target(${_targetname} DEPENDS ${coverage_info}) - add_custom_command(OUTPUT ${coverage_cleaned} - COMMAND COMMAND ${LCOV_PATH} --quiet - --remove ${coverage_info} 'tests/*' '/usr/*' 'src/examples*' - --output-file ${coverage_cleaned} - DEPENDS ${coverage_info} - ) + add_custom_command(OUTPUT ${coverage_cleaned} + COMMAND ${LCOV_PATH} --quiet + --remove ${coverage_info} 'tests/*' '/usr/*' 'src/examples*' + --output-file ${coverage_cleaned} + DEPENDS ${coverage_info} + ) - add_custom_target(${_targetname}_genhtml - COMMAND ${GENHTML_PATH} --quiet -o coverage-html ${coverage_cleaned} - DEPENDS ${coverage_cleaned} - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - ) + add_custom_target(${_targetname}_genhtml + COMMAND ${GENHTML_PATH} --quiet -o coverage-html ${coverage_cleaned} + DEPENDS ${coverage_cleaned} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) endfunction() # SETUP_TARGET_FOR_COVERAGE