[DO NOT MERGE] update NuttX to pr-nuttx_sd_testing

This reverts commit 00b15798da.

.ci/Jenkinsfile-hardware

@@ -399,7 +399,19 @@ pipeline {
                   steps {
                     sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "top once"'
                     sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "work_queue status"'
-                    //sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "sd_stress"'
+                    sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "sd_stress" || true'
+
+                    sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "tests dataman" || true'
+                    sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "sd_bench" || true'
+                    sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "sd_bench -u" || true'
+
+                    sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "sd_stress" || true'
+
+                    sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "logger on" || true'
+                    sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "logger status" || true'
+                    sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "sd_bench" || true'
+                    sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "logger status" || true'
+
                     sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "uorb_tests"'
                     sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "uorb_tests latency_test" || true'
                   }
@@ -767,6 +779,17 @@ void runTests() {
   sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "top once"'
   sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "work_queue status"'
 
+  sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "tests dataman" || true'
+  sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "sd_bench" || true'
+  sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "sd_bench -u" || true'
+
+  sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "sd_stress" || true'
+
+  sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "logger on" || true'
+  sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "logger status" || true'
+  sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "sd_bench" || true'
+  sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "logger status" || true'
+
   sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "commander_tests" || true'
   sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "controllib_test"'
   sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "lightware_laser_test"'
@@ -902,6 +925,10 @@ void printTopics() {
 }
 
 void resetBoard() {
+  sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "df -h" || true'
+  sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "ls /fs/" || true'
+  sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "ls /fs/microsd" || true'
+
   resetParameters()
 
   sh './Tools/HIL/run_nsh_cmd.py --device `find /dev/serial -name *usb-*` --cmd "param set SYS_AUTOSTART 0" || true'

Tools/HIL/run_tests.py

@@ -142,9 +142,8 @@ class TestHardwareMethods(unittest.TestCase):
     def test_bson(self):
         self.assertTrue(do_test(self.TEST_DEVICE, self.TEST_BAUDRATE, "bson"))
 
-    # TODO: review
-    # def test_dataman(self):
-    #     self.assertTrue(do_test(self.TEST_DEVICE, self.TEST_BAUDRATE, "dataman"))
+    def test_dataman(self):
+        self.assertTrue(do_test(self.TEST_DEVICE, self.TEST_BAUDRATE, "dataman"))
 
     # def test_file(self):
     #     self.assertTrue(do_test(self.TEST_DEVICE, self.TEST_BAUDRATE, "file"))

src/systemcmds/sd_bench/sd_bench.c

@@ -1,6 +1,6 @@
 /****************************************************************************
  *
- *   Copyright (c) 2016 PX4 Development Team. All rights reserved.
+ *   Copyright (c) 2016, 2021 PX4 Development Team. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -49,10 +49,20 @@
 
 #include <drivers/drv_hrt.h>
 
+typedef struct sdb_config {
+	int num_runs; ///< number of runs
+	int run_duration; ///< duration of a single run [ms]
+	bool synchronized; ///< call fsync after each block?
+	bool aligned;
+	unsigned int total_blocks_written;
+} sdb_config_t;
+
 static void	usage(void);
 
 /** sequential write speed test */
-static void	write_test(int fd, uint8_t *block, int block_size);
+static void write_test(int fd, sdb_config_t *cfg, uint8_t *block, int block_size);
+/** sequential read speed test */
+static int read_test(int fd, sdb_config_t *cfg, uint8_t *block, int block_size);
 
 /**
  * Measure the time for fsync.
@@ -65,10 +75,6 @@ __EXPORT int	sd_bench_main(int argc, char *argv[]);
 
 static const char *BENCHMARK_FILE = PX4_STORAGEDIR"/benchmark.tmp";
 
-static int num_runs; ///< number of runs
-static int run_duration; ///< duration of a single run [ms]
-static bool synchronized; ///< call fsync after each block?
-
 static void
 usage()
 {
@@ -79,7 +85,9 @@ usage()
 	PRINT_MODULE_USAGE_PARAM_INT('r', 5, 1, 1000, "Number of runs", true);
 	PRINT_MODULE_USAGE_PARAM_INT('d', 2000, 1, 100000, "Duration of a run in ms", true);
 	PRINT_MODULE_USAGE_PARAM_FLAG('s', "Call fsync after each block (default=at end of each run)", true);
-	PRINT_MODULE_USAGE_PARAM_FLAG('u', "Test performance with unaligned data)", true);
+	PRINT_MODULE_USAGE_PARAM_FLAG('u', "Test performance with unaligned data", true);
+	PRINT_MODULE_USAGE_PARAM_FLAG('v', "Verify data and block number", true);
+	PRINT_MODULE_USAGE_PARAM_FLAG('k', "Keep the test file", true);
 }
 
 
@@ -87,35 +95,46 @@ int
 sd_bench_main(int argc, char *argv[])
 {
 	int block_size = 4096;
+	bool verify = false;
+	bool keep = false;
 	int myoptind = 1;
 	int ch;
 	const char *myoptarg = NULL;
-	synchronized = false;
-	num_runs = 5;
-	run_duration = 2000;
-	bool aligned = true;
+	sdb_config_t cfg;
+	cfg.synchronized = false;
+	cfg.num_runs = 5;
+	cfg.run_duration = 2000;
+	cfg.aligned = true;
 	uint8_t *block =  NULL;
 
-	while ((ch = px4_getopt(argc, argv, "b:r:d:su", &myoptind, &myoptarg)) != EOF) {
+	while ((ch = px4_getopt(argc, argv, "b:r:d:suv", &myoptind, &myoptarg)) != EOF) {
 		switch (ch) {
 		case 'b':
 			block_size = strtol(myoptarg, NULL, 0);
 			break;
 
 		case 'r':
-			num_runs = strtol(myoptarg, NULL, 0);
+			cfg.num_runs = strtol(myoptarg, NULL, 0);
 			break;
 
 		case 'd':
-			run_duration = strtol(myoptarg, NULL, 0);
+			cfg.run_duration = strtol(myoptarg, NULL, 0);
+			break;
+
+		case 'k':
+			keep = true;
 			break;
 
 		case 's':
-			synchronized = true;
+			cfg.synchronized = true;
 			break;
 
 		case 'u':
-			aligned = false;
+			cfg.aligned = false;
+			break;
+
+		case 'v':
+			verify = true;
 			break;
 
 		default:
@@ -125,12 +144,12 @@ sd_bench_main(int argc, char *argv[])
 		}
 	}
 
-	if (block_size <= 0 || num_runs <= 0) {
+	if (block_size <= 0 || cfg.num_runs <= 0) {
 		PX4_ERR("invalid argument");
 		return -1;
 	}
 
-	int bench_fd = open(BENCHMARK_FILE, O_CREAT | O_WRONLY | O_TRUNC, PX4_O_MODE_666);
+	int bench_fd = open(BENCHMARK_FILE, O_CREAT | (verify ? O_RDWR : O_WRONLY) | O_TRUNC, PX4_O_MODE_666);
 
 	if (bench_fd < 0) {
 		PX4_ERR("Can't open benchmark file %s", BENCHMARK_FILE);
@@ -138,7 +157,7 @@ sd_bench_main(int argc, char *argv[])
 	}
 
 	//create some data block
-	if (aligned) {
+	if (cfg.aligned) {
 		block = (uint8_t *)px4_cache_aligned_alloc(block_size);
 
 	} else {
@@ -155,12 +174,21 @@ sd_bench_main(int argc, char *argv[])
 		block[i] = (uint8_t)i;
 	}
 
-	PX4_INFO("Using block size = %i bytes, sync=%i", block_size, (int)synchronized);
-	write_test(bench_fd, block, block_size);
+	PX4_INFO("Using block size = %i bytes, sync=%i", block_size, (int)cfg.synchronized);
+	write_test(bench_fd, &cfg, block, block_size);
+
+	if (verify) {
+		fsync(bench_fd);
+		lseek(bench_fd, 0, SEEK_SET);
+		read_test(bench_fd, &cfg, block, block_size);
+	}
 
 	free(block);
 	close(bench_fd);
-	unlink(BENCHMARK_FILE);
+
+	if (!keep) {
+		unlink(BENCHMARK_FILE);
+	}
 
 	return 0;
 }
@@ -172,22 +200,25 @@ unsigned int time_fsync(int fd)
 	return hrt_elapsed_time(&fsync_start) / 1000;
 }
 
-void write_test(int fd, uint8_t *block, int block_size)
+void write_test(int fd, sdb_config_t *cfg, uint8_t *block, int block_size)
 {
 	PX4_INFO("");
 	PX4_INFO("Testing Sequential Write Speed...");
 	double total_elapsed = 0.;
 	unsigned int total_blocks = 0;
+	cfg->total_blocks_written = 0;
+	unsigned int *blocknumber = (unsigned int *)(void *)&block[0];
 
-	for (int run = 0; run < num_runs; ++run) {
+	for (int run = 0; run < cfg->num_runs; ++run) {
 		hrt_abstime start = hrt_absolute_time();
 		unsigned int num_blocks = 0;
 		unsigned int max_write_time = 0;
 		unsigned int fsync_time = 0;
 
-		while ((int64_t)hrt_elapsed_time(&start) < run_duration * 1000) {
+		while ((int64_t)hrt_elapsed_time(&start) < cfg->run_duration * 1000) {
 
 			hrt_abstime write_start = hrt_absolute_time();
+			*blocknumber =  total_blocks + num_blocks;
 			size_t written = write(fd, block, block_size);
 			unsigned int write_time = hrt_elapsed_time(&write_start) / 1000;
 
@@ -200,7 +231,7 @@ void write_test(int fd, uint8_t *block, int block_size)
 				return;
 			}
 
-			if (synchronized) {
+			if (cfg->synchronized) {
 				fsync_time += time_fsync(fd);
 			}
 
@@ -222,5 +253,83 @@ void write_test(int fd, uint8_t *block, int block_size)
 		total_blocks += num_blocks;
 	}
 
+	cfg->total_blocks_written = total_blocks;
 	PX4_INFO("  Avg   : %8.2lf KB/s", (double)block_size * total_blocks / total_elapsed / 1024.);
 }
+
+int read_test(int fd, sdb_config_t *cfg, uint8_t *block, int block_size)
+{
+	uint8_t *read_block =  NULL;
+
+	PX4_INFO("");
+	PX4_INFO("Testing Sequential Read Speed of %d blocks", cfg->total_blocks_written);
+
+	if (cfg->aligned) {
+		read_block = (uint8_t *)px4_cache_aligned_alloc(block_size);
+
+	} else {
+		read_block = (uint8_t *)malloc(block_size);
+	}
+
+	if (!read_block) {
+		PX4_ERR("Failed to allocate memory block");
+		return -1;
+	}
+
+	double total_elapsed = 0.;
+	unsigned int total_blocks = 0;
+	unsigned int *blocknumber = (unsigned int *)(void *) &read_block[0];
+
+	for (int run = 0; run < cfg->num_runs  && total_blocks < cfg->total_blocks_written; ++run) {
+		hrt_abstime start = hrt_absolute_time();
+		unsigned int num_blocks = 0;
+		unsigned int max_read_time = 0;
+
+		while ((int64_t)hrt_elapsed_time(&start) < cfg->run_duration * 1000
+		       && total_blocks + num_blocks < cfg->total_blocks_written) {
+
+			hrt_abstime read_start = hrt_absolute_time();
+			size_t nread = read(fd, read_block, block_size);
+			unsigned int read_time = hrt_elapsed_time(&read_start) / 1000;
+
+			if (read_time > max_read_time) {
+				max_read_time = read_time;
+			}
+
+			if ((int)nread != block_size) {
+				PX4_ERR("Read error");
+				return -1;
+			}
+
+
+			if (*blocknumber !=  total_blocks + num_blocks) {
+				PX4_ERR("Read data error at block: %d wrote:0x%04x read:0x%04x", (total_blocks + num_blocks),
+					total_blocks + num_blocks, *blocknumber);
+
+			}
+
+
+			for (unsigned int i = sizeof(*blocknumber); i < (block_size - sizeof(*blocknumber)); ++i) {
+				if (block[i] != read_block[i]) {
+					PX4_ERR("Read data error at offset: %d wrote:0x%02x read:0x%02x", total_blocks + num_blocks + i, block[i],
+						read_block[i]);
+				}
+			}
+
+			++num_blocks;
+		}
+
+		//report
+		double elapsed = hrt_elapsed_time(&start) / 1.e6;
+		PX4_INFO("  Run %2i: %8.2lf KB/s, max read/verify time: %i ms (=%7.2lf KB/s)", run,
+			 (double)block_size * num_blocks / elapsed / 1024.,
+			 max_read_time, (double)block_size / max_read_time * 1000. / 1024.);
+
+		total_elapsed += elapsed;
+		total_blocks += num_blocks;
+	}
+
+	PX4_INFO("  Avg   : %8.2lf KB/s %d blocks read and verified", (double)block_size * total_blocks / total_elapsed / 1024.,
+		 total_blocks);
+	return 0;
+}