From 1bd02ef56c2e0314cfef3981c545e73b53a81679 Mon Sep 17 00:00:00 2001 From: John Zhao Date: Mon, 5 Nov 2018 17:50:15 +0800 Subject: [PATCH] Add image data to synthetic streams --- include/mynteye/api/object.h | 24 +++++++--- src/mynteye/api/processor/depth_processor.cc | 1 + .../disparity_normalized_processor.cc | 1 + .../api/processor/disparity_processor.cc | 1 + src/mynteye/api/processor/points_processor.cc | 1 + .../api/processor/rectify_processor.cc | 2 + src/mynteye/api/synthetic.cc | 45 ++++++++++--------- 7 files changed, 49 insertions(+), 26 deletions(-) diff --git a/include/mynteye/api/object.h b/include/mynteye/api/object.h index 7777a2f..5dd6d82 100644 --- a/include/mynteye/api/object.h +++ b/include/mynteye/api/object.h @@ -23,6 +23,8 @@ MYNTEYE_BEGIN_NAMESPACE +struct ImgData; + /** * Input & output object. */ @@ -56,18 +58,22 @@ struct MYNTEYE_API Object { */ struct MYNTEYE_API ObjMat : public Object { ObjMat() = default; - ObjMat(const cv::Mat &value, std::uint16_t id) - : value(value), id(id) {} + ObjMat(const cv::Mat &value, std::uint16_t id, + const std::shared_ptr &data) + : value(value), id(id), data(data) {} /** The value */ cv::Mat value; /** The id **/ std::uint16_t id; + /** The data **/ + std::shared_ptr data; Object *Clone() const { ObjMat *mat = new ObjMat; mat->value = value.clone(); mat->id = id; + mat->data = data; return mat; } @@ -82,26 +88,34 @@ struct MYNTEYE_API ObjMat : public Object { struct MYNTEYE_API ObjMat2 : public Object { ObjMat2() = default; ObjMat2(const cv::Mat &first, std::uint16_t first_id, - const cv::Mat &second, std::uint16_t second_id) - : first(first), first_id(first_id), - second(second), second_id(second_id) {} + const std::shared_ptr &first_data, + const cv::Mat &second, std::uint16_t second_id, + const std::shared_ptr &second_data) + : first(first), first_id(first_id), first_data(first_data), + second(second), second_id(second_id), second_data(second_data) {} /** The first value */ cv::Mat first; /** The first id **/ std::uint16_t first_id; + /** The first data **/ + std::shared_ptr first_data; /** The second value */ cv::Mat second; /** The second id **/ std::uint16_t second_id; + /** The second data **/ + std::shared_ptr second_data; Object *Clone() const { ObjMat2 *mat2 = new ObjMat2; mat2->first = first.clone(); mat2->first_id = first_id; + mat2->first_data = first_data; mat2->second = second.clone(); mat2->second_id = second_id; + mat2->second_data = second_data; return mat2; } diff --git a/src/mynteye/api/processor/depth_processor.cc b/src/mynteye/api/processor/depth_processor.cc index 8243226..aaaf610 100644 --- a/src/mynteye/api/processor/depth_processor.cc +++ b/src/mynteye/api/processor/depth_processor.cc @@ -47,6 +47,7 @@ bool DepthProcessor::OnProcess( cv::split(input->value, channels); channels[2].convertTo(output->value, CV_16UC1); output->id = input->id; + output->data = input->data; return true; } diff --git a/src/mynteye/api/processor/disparity_normalized_processor.cc b/src/mynteye/api/processor/disparity_normalized_processor.cc index 4b8fe64..9a629e8 100644 --- a/src/mynteye/api/processor/disparity_normalized_processor.cc +++ b/src/mynteye/api/processor/disparity_normalized_processor.cc @@ -50,6 +50,7 @@ bool DisparityNormalizedProcessor::OnProcess( cv::normalize(input->value, output->value, 0, 255, cv::NORM_MINMAX, CV_8UC1); // cv::normalize maybe return empty == output->id = input->id; + output->data = input->data; return !output->value.empty(); } diff --git a/src/mynteye/api/processor/disparity_processor.cc b/src/mynteye/api/processor/disparity_processor.cc index 93c0943..c22232c 100644 --- a/src/mynteye/api/processor/disparity_processor.cc +++ b/src/mynteye/api/processor/disparity_processor.cc @@ -100,6 +100,7 @@ bool DisparityProcessor::OnProcess( #endif output->value = disparity / 16 + 1; output->id = input->first_id; + output->data = input->first_data; return true; } diff --git a/src/mynteye/api/processor/points_processor.cc b/src/mynteye/api/processor/points_processor.cc index 3fa28e8..7d2b6d4 100644 --- a/src/mynteye/api/processor/points_processor.cc +++ b/src/mynteye/api/processor/points_processor.cc @@ -47,6 +47,7 @@ bool PointsProcessor::OnProcess( ObjMat *output = Object::Cast(out); cv::reprojectImageTo3D(input->value, output->value, Q_, true); output->id = input->id; + output->data = input->data; return true; } diff --git a/src/mynteye/api/processor/rectify_processor.cc b/src/mynteye/api/processor/rectify_processor.cc index 87ba1c2..9c570f0 100644 --- a/src/mynteye/api/processor/rectify_processor.cc +++ b/src/mynteye/api/processor/rectify_processor.cc @@ -54,7 +54,9 @@ bool RectifyProcessor::OnProcess( cv::remap(input->first, output->first, map11, map12, cv::INTER_LINEAR); cv::remap(input->second, output->second, map21, map22, cv::INTER_LINEAR); output->first_id = input->first_id; + output->first_data = input->first_data; output->second_id = input->second_id; + output->second_data = input->second_data; return true; } diff --git a/src/mynteye/api/synthetic.cc b/src/mynteye/api/synthetic.cc index 9eff8a9..3c7e298 100644 --- a/src/mynteye/api/synthetic.cc +++ b/src/mynteye/api/synthetic.cc @@ -164,9 +164,10 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) { } if (output != nullptr) { if (stream == Stream::LEFT_RECTIFIED) { - return {nullptr, output->first, nullptr, output->first_id}; + return {output->first_data, output->first, nullptr, output->first_id}; } else { - return {nullptr, output->second, nullptr, output->second_id}; + return {output->second_data, output->second, nullptr, + output->second_id}; } } VLOG(2) << "Rectify not ready now"; @@ -178,7 +179,7 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) { auto &&out = processor->GetOutput(); if (out != nullptr) { auto &&output = Object::Cast(out); - return {nullptr, output->value, nullptr, output->id}; + return {output->data, output->value, nullptr, output->id}; } VLOG(2) << "Disparity not ready now"; } break; @@ -188,7 +189,7 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) { auto &&out = processor->GetOutput(); if (out != nullptr) { auto &&output = Object::Cast(out); - return {nullptr, output->value, nullptr, output->id}; + return {output->data, output->value, nullptr, output->id}; } VLOG(2) << "Disparity normalized not ready now"; } break; @@ -197,7 +198,7 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) { auto &&out = processor->GetOutput(); if (out != nullptr) { auto &&output = Object::Cast(out); - return {nullptr, output->value, nullptr, output->id}; + return {output->data, output->value, nullptr, output->id}; } VLOG(2) << "Points not ready now"; } break; @@ -206,7 +207,7 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) { auto &&out = processor->GetOutput(); if (out != nullptr) { auto &&output = Object::Cast(out); - return {nullptr, output->value, nullptr, output->id}; + return {output->data, output->value, nullptr, output->id}; } VLOG(2) << "Depth not ready now"; } break; @@ -455,8 +456,9 @@ void Synthetic::ProcessNativeStream( if (left_data.img && right_data.img && left_data.img->frame_id == right_data.img->frame_id) { auto &&processor = find_processor(processor_); - processor->Process(ObjMat2{left_data.frame, left_data.frame_id, - right_data.frame, right_data.frame_id}); + processor->Process(ObjMat2{ + left_data.frame, left_data.frame_id, left_data.img, + right_data.frame, right_data.frame_id, right_data.img}); } return; } @@ -471,9 +473,10 @@ void Synthetic::ProcessNativeStream( if (left_rect_data.img && right_rect_data.img && left_rect_data.img->frame_id == right_rect_data.img->frame_id) { process_childs( - processor_, RectifyProcessor::NAME, - ObjMat2{left_rect_data.frame, left_rect_data.frame_id, - right_rect_data.frame, right_rect_data.frame_id}); + processor_, RectifyProcessor::NAME, ObjMat2{ + left_rect_data.frame, left_rect_data.frame_id, left_rect_data.img, + right_rect_data.frame, right_rect_data.frame_id, + right_rect_data.img}); } return; } @@ -481,19 +484,19 @@ void Synthetic::ProcessNativeStream( switch (stream) { case Stream::DISPARITY: { process_childs(processor_, DisparityProcessor::NAME, - ObjMat{data.frame, data.frame_id}); + ObjMat{data.frame, data.frame_id, data.img}); } break; case Stream::DISPARITY_NORMALIZED: { process_childs(processor_, DisparityNormalizedProcessor::NAME, - ObjMat{data.frame, data.frame_id}); + ObjMat{data.frame, data.frame_id, data.img}); } break; case Stream::POINTS: { process_childs(processor_, PointsProcessor::NAME, - ObjMat{data.frame, data.frame_id}); + ObjMat{data.frame, data.frame_id, data.img}); } break; case Stream::DEPTH: { process_childs(processor_, DepthProcessor::NAME, - ObjMat{data.frame, data.frame_id}); + ObjMat{data.frame, data.frame_id, data.img}); } break; default: break; @@ -550,11 +553,11 @@ void Synthetic::OnRectifyPostProcess(Object *const out) { const ObjMat2 *output = Object::Cast(out); if (HasStreamCallback(Stream::LEFT_RECTIFIED)) { stream_callbacks_.at(Stream::LEFT_RECTIFIED)( - {nullptr, output->first, nullptr, output->first_id}); + {output->first_data, output->first, nullptr, output->first_id}); } if (HasStreamCallback(Stream::RIGHT_RECTIFIED)) { stream_callbacks_.at(Stream::RIGHT_RECTIFIED)( - {nullptr, output->second, nullptr, output->second_id}); + {output->second_data, output->second, nullptr, output->second_id}); } } @@ -562,7 +565,7 @@ void Synthetic::OnDisparityPostProcess(Object *const out) { const ObjMat *output = Object::Cast(out); if (HasStreamCallback(Stream::DISPARITY)) { stream_callbacks_.at(Stream::DISPARITY)( - {nullptr, output->value, nullptr, output->id}); + {output->data, output->value, nullptr, output->id}); } } @@ -570,7 +573,7 @@ void Synthetic::OnDisparityNormalizedPostProcess(Object *const out) { const ObjMat *output = Object::Cast(out); if (HasStreamCallback(Stream::DISPARITY_NORMALIZED)) { stream_callbacks_.at(Stream::DISPARITY_NORMALIZED)( - {nullptr, output->value, nullptr, output->id}); + {output->data, output->value, nullptr, output->id}); } } @@ -578,7 +581,7 @@ void Synthetic::OnPointsPostProcess(Object *const out) { const ObjMat *output = Object::Cast(out); if (HasStreamCallback(Stream::POINTS)) { stream_callbacks_.at(Stream::POINTS)( - {nullptr, output->value, nullptr, output->id}); + {output->data, output->value, nullptr, output->id}); } } @@ -586,7 +589,7 @@ void Synthetic::OnDepthPostProcess(Object *const out) { const ObjMat *output = Object::Cast(out); if (HasStreamCallback(Stream::DEPTH)) { stream_callbacks_.at(Stream::DEPTH)( - {nullptr, output->value, nullptr, output->id}); + {output->data, output->value, nullptr, output->id}); } }