From 1325b9c876a8a6d024a30aaa1c1087162e96cc6f Mon Sep 17 00:00:00 2001
From: Daniel Agar <daniel@agar.ca>
Date: Mon, 4 Oct 2021 11:48:40 -0400
Subject: [PATCH] Jenkins: hardware retry checkout if necessary

---
 .ci/Jenkinsfile-hardware | 70 ++++++++++++----------------------------
 1 file changed, 20 insertions(+), 50 deletions(-)

diff --git a/.ci/Jenkinsfile-hardware b/.ci/Jenkinsfile-hardware
index aa3a3cd91e..d029af0e7b 100644
--- a/.ci/Jenkinsfile-hardware
+++ b/.ci/Jenkinsfile-hardware
@@ -17,11 +17,7 @@ pipeline {
                 }
               }
               steps {
-                checkout scm
-                sh 'export'
-                sh 'make distclean'
-                sh 'ccache -z'
-                sh 'git fetch --tags'
+                checkoutSCM()
                 sh 'make cubepilot_cubeorange_test'
                 sh 'make cubepilot_cubeorange_test bootloader_elf'
                 sh 'ccache -s'
@@ -91,11 +87,7 @@ pipeline {
                 }
               }
               steps {
-                checkout scm
-                sh 'export'
-                sh 'make distclean'
-                sh 'ccache -z'
-                sh 'git fetch --tags'
+                checkoutSCM()
                 sh 'make cuav_x7pro_test'
                 sh 'make cuav_x7pro_test bootloader_elf'
                 sh 'ccache -s'
@@ -163,11 +155,7 @@ pipeline {
                 }
               }
               steps {
-                checkout scm
-                sh 'export'
-                sh 'make distclean'
-                sh 'ccache -z'
-                sh 'git fetch --tags'
+                checkoutSCM()
                 sh 'make px4_fmu-v3_test'
                 sh 'make px4_fmu-v3_test bootloader_elf'
                 sh 'ccache -s'
@@ -235,11 +223,7 @@ pipeline {
                 }
               }
               steps {
-                checkout scm
-                sh 'export'
-                sh 'make distclean'
-                sh 'ccache -z'
-                sh 'git fetch --tags'
+                checkoutSCM()
                 sh 'make px4_fmu-v4_test'
                 sh 'make px4_fmu-v4_test bootloader_elf'
                 sh 'ccache -s'
@@ -306,11 +290,7 @@ pipeline {
                 }
               }
               steps {
-                checkout scm
-                sh 'export'
-                sh 'make distclean'
-                sh 'ccache -z'
-                sh 'git fetch --tags'
+                checkoutSCM()
                 sh 'make px4_fmu-v4pro_test'
                 sh 'make px4_fmu-v4pro_test bootloader_elf'
                 sh 'ccache -s'
@@ -378,11 +358,7 @@ pipeline {
                 }
               }
               steps {
-                checkout scm
-                sh 'export'
-                sh 'make distclean'
-                sh 'ccache -z'
-                sh 'git fetch --tags'
+                checkoutSCM()
                 sh 'make px4_fmu-v5_debug'
                 sh 'make px4_fmu-v5_debug bootloader_elf'
                 sh 'ccache -s'
@@ -460,11 +436,7 @@ pipeline {
                 }
               }
               steps {
-                checkout scm
-                sh 'export'
-                sh 'make distclean'
-                sh 'ccache -z'
-                sh 'git fetch --tags'
+                checkoutSCM()
                 sh 'make px4_fmu-v5_optimized'
                 sh 'make px4_fmu-v5_optimized bootloader_elf'
                 sh 'ccache -s'
@@ -532,11 +504,7 @@ pipeline {
                 }
               }
               steps {
-                checkout scm
-                sh 'export'
-                sh 'make distclean'
-                sh 'ccache -z'
-                sh 'git fetch --tags'
+                checkoutSCM()
                 sh 'make px4_fmu-v5_stackcheck'
                 sh 'make px4_fmu-v5_stackcheck bootloader_elf'
                 sh 'ccache -s'
@@ -614,11 +582,7 @@ pipeline {
                 }
               }
               steps {
-                checkout scm
-                sh 'export'
-                sh 'make distclean'
-                sh 'ccache -z'
-                sh 'git fetch --tags'
+                checkoutSCM()
                 sh 'make px4_fmu-v5_test'
                 sh 'make px4_fmu-v5_test bootloader_elf'
                 sh 'ccache -s'
@@ -753,11 +717,7 @@ pipeline {
                 }
               }
               steps {
-                checkout scm
-                sh 'export'
-                sh 'make distclean'
-                sh 'ccache -z'
-                sh 'git fetch --tags'
+                checkoutSCM()
                 sh 'make nxp_fmuk66-v3_test'
                 //sh 'make nxp_fmuk66-v3_test bootloader_elf'
                 sh 'ccache -s'
@@ -830,6 +790,16 @@ pipeline {
   }
 }
 
+void checkoutSCM() {
+  retry(3) {
+    checkout scm
+    sh 'export'
+    sh 'make distclean'
+    sh 'git fetch --tags'
+    sh 'ccache -z'
+  }
+}
+
 void quickCalibrate() {
   sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "commander calibrate accel quick; sleep 1; param show CAL_ACC*"'
   sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "commander calibrate gyro; sleep 1; param show CAL_GYRO*"'