From d2c049ec93f8f30901cfaecd9fbef2c77a7b9199 Mon Sep 17 00:00:00 2001
From: jmackay2 <1.732mackay@gmail.com>
Date: Mon, 17 Mar 2025 14:00:40 -0400
Subject: [PATCH] Remove OpenCV dependency from gz plugin gstreamer (#24519)

Co-authored-by: jmackay2 <jmackay2@gmail.com>
---
 .../simulation/gz_plugins/gstreamer/CMakeLists.txt |  3 ---
 .../gz_plugins/gstreamer/GstCameraSystem.cpp       | 14 ++++----------
 .../gz_plugins/gstreamer/GstCameraSystem.hpp       |  3 +--
 .../simulation/gz_plugins/gstreamer/README.md      |  1 -
 4 files changed, 5 insertions(+), 16 deletions(-)

diff --git a/src/modules/simulation/gz_plugins/gstreamer/CMakeLists.txt b/src/modules/simulation/gz_plugins/gstreamer/CMakeLists.txt
index 2c07a5d0c7..1f4cda0858 100644
--- a/src/modules/simulation/gz_plugins/gstreamer/CMakeLists.txt
+++ b/src/modules/simulation/gz_plugins/gstreamer/CMakeLists.txt
@@ -34,7 +34,6 @@
 project(GstCameraSystem)
 
 # Find required packages
-find_package(OpenCV REQUIRED)
 find_package(PkgConfig REQUIRED)
 pkg_check_modules(GSTREAMER REQUIRED gstreamer-1.0)
 pkg_check_modules(GSTREAMER_APP REQUIRED gstreamer-app-1.0)
@@ -49,7 +48,6 @@ target_link_libraries(${PROJECT_NAME}
     PUBLIC gz-plugin${GZ_PLUGIN_VERSION}::gz-plugin${GZ_PLUGIN_VERSION}
     PUBLIC gz-sim${GZ_SIM_VERSION}::gz-sim${GZ_SIM_VERSION}
     PUBLIC gz-transport${GZ_TRANSPORT_VERSION}::gz-transport${GZ_TRANSPORT_VERSION}
-    PUBLIC ${OpenCV_LIBS}
     PUBLIC ${GSTREAMER_LIBRARIES}
     PUBLIC ${GSTREAMER_APP_LIBRARIES}
 )
@@ -58,7 +56,6 @@ target_include_directories(${PROJECT_NAME}
     PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}
     PUBLIC ${CMAKE_CURRENT_BINARY_DIR}
     PUBLIC px4_gz_msgs
-    PUBLIC ${OpenCV_INCLUDE_DIRS}
     PUBLIC ${GSTREAMER_INCLUDE_DIRS}
     PUBLIC ${GSTREAMER_APP_INCLUDE_DIRS}
 )
diff --git a/src/modules/simulation/gz_plugins/gstreamer/GstCameraSystem.cpp b/src/modules/simulation/gz_plugins/gstreamer/GstCameraSystem.cpp
index 9ff384e1e0..0f489dcf86 100644
--- a/src/modules/simulation/gz_plugins/gstreamer/GstCameraSystem.cpp
+++ b/src/modules/simulation/gz_plugins/gstreamer/GstCameraSystem.cpp
@@ -206,17 +206,11 @@ void GstCameraSystem::onImage(const gz::msgs::Image &msg)
 		return;
 	}
 
-	// Convert the image data to OpenCV format
-	cv::Mat frame(msg.height(), msg.width(), CV_8UC3);
-
 	// Check pixel format and convert if necessary
 	if (msg.pixel_format_type() == gz::msgs::PixelFormatType::RGB_INT8) {
-		// Copy RGB data directly
-		memcpy(frame.data, msg.data().c_str(), msg.data().size());
-
 		// Process the frame
 		std::lock_guard<std::mutex> lock(_frameMutex);
-		frame.copyTo(_currentFrame);
+		_currentFrame = msg;
 		_newFrameAvailable = true;
 
 	} else {
@@ -412,7 +406,7 @@ void GstCameraSystem::gstThreadFunc()
 	while (_running) {
 		std::unique_lock<std::mutex> lock(_frameMutex);
 
-		if (_newFrameAvailable && !_currentFrame.empty()) {
+		if (_newFrameAvailable) {
 			// Push RGB data directly - we configured the caps to accept RGB
 			const guint size = _width * _height * 3; // RGB is 3 bytes per pixel
 			GstBuffer *buffer = gst_buffer_new_allocate(nullptr, size, nullptr);
@@ -421,8 +415,8 @@ void GstCameraSystem::gstThreadFunc()
 				GstMapInfo map;
 
 				if (gst_buffer_map(buffer, &map, GST_MAP_WRITE)) {
-					// Copy RGB data directly from the OpenCV Mat
-					memcpy(map.data, _currentFrame.data, size);
+					// Copy RGB data directly from the current frame
+					memcpy(map.data, _currentFrame.data().c_str(), size);
 					gst_buffer_unmap(buffer, &map);
 
 					// Add timing information for smoother playback
diff --git a/src/modules/simulation/gz_plugins/gstreamer/GstCameraSystem.hpp b/src/modules/simulation/gz_plugins/gstreamer/GstCameraSystem.hpp
index d383171117..d00313b28b 100644
--- a/src/modules/simulation/gz_plugins/gstreamer/GstCameraSystem.hpp
+++ b/src/modules/simulation/gz_plugins/gstreamer/GstCameraSystem.hpp
@@ -39,7 +39,6 @@
 #include <mutex>
 #include <atomic>
 #include <regex>
-#include <opencv2/opencv.hpp>
 
 #include <gz/sim/System.hh>
 #include <gz/transport/Node.hh>
@@ -82,7 +81,7 @@ private:
 	gz::transport::Node _node;
 
 	// Image processing
-	cv::Mat _currentFrame;
+	gz::msgs::Image _currentFrame;
 	std::mutex _frameMutex;
 	std::atomic<bool> _newFrameAvailable {};
 
diff --git a/src/modules/simulation/gz_plugins/gstreamer/README.md b/src/modules/simulation/gz_plugins/gstreamer/README.md
index b8b17e168a..a3527e1b60 100644
--- a/src/modules/simulation/gz_plugins/gstreamer/README.md
+++ b/src/modules/simulation/gz_plugins/gstreamer/README.md
@@ -12,7 +12,6 @@ This plugin provides GStreamer-based video streaming capabilities for cameras in
 ## Prerequisites
 
 - GStreamer 1.0 with development files
-- OpenCV with development files
 - NVIDIA drivers and CUDA (optional, for hardware acceleration)
 
 ## Configuration