Add image data to synthetic streams

include/mynteye/api/object.h

@@ -23,6 +23,8 @@
 
 MYNTEYE_BEGIN_NAMESPACE
 
+struct ImgData;
+
 /**
  * Input & output object.
  */
@@ -56,18 +58,22 @@ struct MYNTEYE_API Object {
  */
 struct MYNTEYE_API ObjMat : public Object {
   ObjMat() = default;
-  ObjMat(const cv::Mat &value, std::uint16_t id)
+  ObjMat(const cv::Mat &value, std::uint16_t id,
-      : value(value), id(id) {}
+         const std::shared_ptr<ImgData> &data)
+      : value(value), id(id), data(data) {}
 
   /** The value */
   cv::Mat value;
   /** The id **/
   std::uint16_t id;
+  /** The data **/
+  std::shared_ptr<ImgData> data;
 
   Object *Clone() const {
     ObjMat *mat = new ObjMat;
     mat->value = value.clone();
     mat->id = id;
+    mat->data = data;
     return mat;
   }
 
@@ -82,26 +88,34 @@ struct MYNTEYE_API ObjMat : public Object {
 struct MYNTEYE_API ObjMat2 : public Object {
   ObjMat2() = default;
   ObjMat2(const cv::Mat &first, std::uint16_t first_id,
-          const cv::Mat &second, std::uint16_t second_id)
+          const std::shared_ptr<ImgData> &first_data,
-      : first(first), first_id(first_id),
+          const cv::Mat &second, std::uint16_t second_id,
-        second(second), second_id(second_id) {}
+          const std::shared_ptr<ImgData> &second_data)
+      : first(first), first_id(first_id), first_data(first_data),
+        second(second), second_id(second_id), second_data(second_data) {}
 
   /** The first value */
   cv::Mat first;
   /** The first id **/
   std::uint16_t first_id;
+  /** The first data **/
+  std::shared_ptr<ImgData> first_data;
 
   /** The second value */
   cv::Mat second;
   /** The second id **/
   std::uint16_t second_id;
+  /** The second data **/
+  std::shared_ptr<ImgData> second_data;
 
   Object *Clone() const {
     ObjMat2 *mat2 = new ObjMat2;
     mat2->first = first.clone();
     mat2->first_id = first_id;
+    mat2->first_data = first_data;
     mat2->second = second.clone();
     mat2->second_id = second_id;
+    mat2->second_data = second_data;
     return mat2;
   }
 

src/mynteye/api/processor/depth_processor.cc

@@ -47,6 +47,7 @@ bool DepthProcessor::OnProcess(
   cv::split(input->value, channels);
   channels[2].convertTo(output->value, CV_16UC1);
   output->id = input->id;
+  output->data = input->data;
   return true;
 }
 

src/mynteye/api/processor/disparity_normalized_processor.cc

@@ -50,6 +50,7 @@ bool DisparityNormalizedProcessor::OnProcess(
   cv::normalize(input->value, output->value, 0, 255, cv::NORM_MINMAX, CV_8UC1);
   // cv::normalize maybe return empty ==
   output->id = input->id;
+  output->data = input->data;
   return !output->value.empty();
 }
 

src/mynteye/api/processor/disparity_processor.cc

@@ -100,6 +100,7 @@ bool DisparityProcessor::OnProcess(
 #endif
   output->value = disparity / 16 + 1;
   output->id = input->first_id;
+  output->data = input->first_data;
   return true;
 }
 

src/mynteye/api/processor/points_processor.cc

@@ -47,6 +47,7 @@ bool PointsProcessor::OnProcess(
   ObjMat *output = Object::Cast<ObjMat>(out);
   cv::reprojectImageTo3D(input->value, output->value, Q_, true);
   output->id = input->id;
+  output->data = input->data;
   return true;
 }
 

src/mynteye/api/processor/rectify_processor.cc

@@ -54,7 +54,9 @@ bool RectifyProcessor::OnProcess(
   cv::remap(input->first, output->first, map11, map12, cv::INTER_LINEAR);
   cv::remap(input->second, output->second, map21, map22, cv::INTER_LINEAR);
   output->first_id = input->first_id;
+  output->first_data = input->first_data;
   output->second_id = input->second_id;
+  output->second_data = input->second_data;
   return true;
 }
 

src/mynteye/api/synthetic.cc

@@ -164,9 +164,10 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) {
       }
       if (output != nullptr) {
         if (stream == Stream::LEFT_RECTIFIED) {
-          return {nullptr, output->first, nullptr, output->first_id};
+          return {output->first_data, output->first, nullptr, output->first_id};
         } else {
-          return {nullptr, output->second, nullptr, output->second_id};
+          return {output->second_data, output->second, nullptr,
+                  output->second_id};
         }
       }
       VLOG(2) << "Rectify not ready now";
@@ -178,7 +179,7 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) {
         auto &&out = processor->GetOutput();
         if (out != nullptr) {
           auto &&output = Object::Cast<ObjMat>(out);
-          return {nullptr, output->value, nullptr, output->id};
+          return {output->data, output->value, nullptr, output->id};
         }
         VLOG(2) << "Disparity not ready now";
       } break;
@@ -188,7 +189,7 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) {
         auto &&out = processor->GetOutput();
         if (out != nullptr) {
           auto &&output = Object::Cast<ObjMat>(out);
-          return {nullptr, output->value, nullptr, output->id};
+          return {output->data, output->value, nullptr, output->id};
         }
         VLOG(2) << "Disparity normalized not ready now";
       } break;
@@ -197,7 +198,7 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) {
         auto &&out = processor->GetOutput();
         if (out != nullptr) {
           auto &&output = Object::Cast<ObjMat>(out);
-          return {nullptr, output->value, nullptr, output->id};
+          return {output->data, output->value, nullptr, output->id};
         }
         VLOG(2) << "Points not ready now";
       } break;
@@ -206,7 +207,7 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) {
         auto &&out = processor->GetOutput();
         if (out != nullptr) {
           auto &&output = Object::Cast<ObjMat>(out);
-          return {nullptr, output->value, nullptr, output->id};
+          return {output->data, output->value, nullptr, output->id};
         }
         VLOG(2) << "Depth not ready now";
       } break;
@@ -455,8 +456,9 @@ void Synthetic::ProcessNativeStream(
     if (left_data.img && right_data.img &&
         left_data.img->frame_id == right_data.img->frame_id) {
       auto &&processor = find_processor<RectifyProcessor>(processor_);
-      processor->Process(ObjMat2{left_data.frame, left_data.frame_id,
+      processor->Process(ObjMat2{
-          right_data.frame, right_data.frame_id});
+          left_data.frame, left_data.frame_id, left_data.img,
+          right_data.frame, right_data.frame_id, right_data.img});
     }
     return;
   }
@@ -471,9 +473,10 @@ void Synthetic::ProcessNativeStream(
     if (left_rect_data.img && right_rect_data.img &&
         left_rect_data.img->frame_id == right_rect_data.img->frame_id) {
       process_childs(
-          processor_, RectifyProcessor::NAME,
+          processor_, RectifyProcessor::NAME, ObjMat2{
-          ObjMat2{left_rect_data.frame, left_rect_data.frame_id,
+              left_rect_data.frame, left_rect_data.frame_id, left_rect_data.img,
-              right_rect_data.frame, right_rect_data.frame_id});
+              right_rect_data.frame, right_rect_data.frame_id,
+              right_rect_data.img});
     }
     return;
   }
@@ -481,19 +484,19 @@ void Synthetic::ProcessNativeStream(
   switch (stream) {
     case Stream::DISPARITY: {
       process_childs(processor_, DisparityProcessor::NAME,
-          ObjMat{data.frame, data.frame_id});
+          ObjMat{data.frame, data.frame_id, data.img});
     } break;
     case Stream::DISPARITY_NORMALIZED: {
       process_childs(processor_, DisparityNormalizedProcessor::NAME,
-          ObjMat{data.frame, data.frame_id});
+          ObjMat{data.frame, data.frame_id, data.img});
     } break;
     case Stream::POINTS: {
       process_childs(processor_, PointsProcessor::NAME,
-          ObjMat{data.frame, data.frame_id});
+          ObjMat{data.frame, data.frame_id, data.img});
     } break;
     case Stream::DEPTH: {
       process_childs(processor_, DepthProcessor::NAME,
-          ObjMat{data.frame, data.frame_id});
+          ObjMat{data.frame, data.frame_id, data.img});
     } break;
     default:
       break;
@@ -550,11 +553,11 @@ void Synthetic::OnRectifyPostProcess(Object *const out) {
   const ObjMat2 *output = Object::Cast<ObjMat2>(out);
   if (HasStreamCallback(Stream::LEFT_RECTIFIED)) {
     stream_callbacks_.at(Stream::LEFT_RECTIFIED)(
-        {nullptr, output->first, nullptr, output->first_id});
+        {output->first_data, output->first, nullptr, output->first_id});
   }
   if (HasStreamCallback(Stream::RIGHT_RECTIFIED)) {
     stream_callbacks_.at(Stream::RIGHT_RECTIFIED)(
-        {nullptr, output->second, nullptr, output->second_id});
+        {output->second_data, output->second, nullptr, output->second_id});
   }
 }
 
@@ -562,7 +565,7 @@ void Synthetic::OnDisparityPostProcess(Object *const out) {
   const ObjMat *output = Object::Cast<ObjMat>(out);
   if (HasStreamCallback(Stream::DISPARITY)) {
     stream_callbacks_.at(Stream::DISPARITY)(
-        {nullptr, output->value, nullptr, output->id});
+        {output->data, output->value, nullptr, output->id});
   }
 }
 
@@ -570,7 +573,7 @@ void Synthetic::OnDisparityNormalizedPostProcess(Object *const out) {
   const ObjMat *output = Object::Cast<ObjMat>(out);
   if (HasStreamCallback(Stream::DISPARITY_NORMALIZED)) {
     stream_callbacks_.at(Stream::DISPARITY_NORMALIZED)(
-        {nullptr, output->value, nullptr, output->id});
+        {output->data, output->value, nullptr, output->id});
   }
 }
 
@@ -578,7 +581,7 @@ void Synthetic::OnPointsPostProcess(Object *const out) {
   const ObjMat *output = Object::Cast<ObjMat>(out);
   if (HasStreamCallback(Stream::POINTS)) {
     stream_callbacks_.at(Stream::POINTS)(
-        {nullptr, output->value, nullptr, output->id});
+        {output->data, output->value, nullptr, output->id});
   }
 }
 
@@ -586,7 +589,7 @@ void Synthetic::OnDepthPostProcess(Object *const out) {
   const ObjMat *output = Object::Cast<ObjMat>(out);
   if (HasStreamCallback(Stream::DEPTH)) {
     stream_callbacks_.at(Stream::DEPTH)(
-        {nullptr, output->value, nullptr, output->id});
+        {output->data, output->value, nullptr, output->id});
   }
 }