merge develop

src/mynteye/api/api.cc

@@ -0,0 +1,453 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "mynteye/api/api.h"
+
+#ifdef WITH_BOOST_FILESYSTEM
+#include <boost/filesystem.hpp>
+#include <boost/range/iterator_range.hpp>
+#endif
+
+#include <algorithm>
+#include <thread>
+
+#include "mynteye/logger.h"
+#include "mynteye/api/dl.h"
+#include "mynteye/api/plugin.h"
+#include "mynteye/api/synthetic.h"
+#include "mynteye/device/device.h"
+#include "mynteye/device/device_s.h"
+#include "mynteye/device/utils.h"
+
+#if defined(WITH_FILESYSTEM) && defined(WITH_NATIVE_FILESYSTEM)
+#if defined(MYNTEYE_OS_WIN)
+#include <windows.h>
+#endif
+#endif
+
+MYNTEYE_BEGIN_NAMESPACE
+
+namespace {
+
+#if defined(WITH_FILESYSTEM)
+
+#if defined(WITH_BOOST_FILESYSTEM)
+
+namespace fs = boost::filesystem;
+
+bool file_exists(const fs::path &p) {
+  try {
+    fs::file_status s = fs::status(p);
+    return fs::exists(s) && fs::is_regular_file(s);
+  } catch (fs::filesystem_error &e) {
+    LOG(ERROR) << e.what();
+    return false;
+  }
+}
+
+bool dir_exists(const fs::path &p) {
+  try {
+    fs::file_status s = fs::status(p);
+    return fs::exists(s) && fs::is_directory(s);
+  } catch (fs::filesystem_error &e) {
+    LOG(ERROR) << e.what();
+    return false;
+  }
+}
+
+#elif defined(WITH_NATIVE_FILESYSTEM)
+
+#if defined(MYNTEYE_OS_WIN)
+
+bool file_exists(const std::string &p) {
+  DWORD attrs = GetFileAttributes(p.c_str());
+  return (attrs != INVALID_FILE_ATTRIBUTES) &&
+         !(attrs & FILE_ATTRIBUTE_DIRECTORY);
+}
+
+bool dir_exists(const std::string &p) {
+  DWORD attrs = GetFileAttributes(p.c_str());
+  return (attrs != INVALID_FILE_ATTRIBUTES) &&
+         (attrs & FILE_ATTRIBUTE_DIRECTORY);
+}
+
+#else
+#error "Unsupported native filesystem"
+#endif
+
+#endif
+
+std::vector<std::string> get_plugin_paths() {
+  std::string info_path = utils::get_sdk_install_dir();
+  info_path.append(MYNTEYE_OS_SEP "share" MYNTEYE_OS_SEP "mynteye" MYNTEYE_OS_SEP "build.info");
+
+  cv::FileStorage fs(info_path, cv::FileStorage::READ);
+  if (!fs.isOpened()) {
+    LOG(WARNING) << "build.info not found: " << info_path;
+    return {};
+  }
+
+  auto to_lower = [](std::string &s) {  // NOLINT
+    std::transform(s.begin(), s.end(), s.begin(), ::tolower);
+  };
+
+  std::string host_os = fs["HOST_OS"];
+  to_lower(host_os);
+  std::string host_name = fs["HOST_NAME"];
+  to_lower(host_name);
+  std::string host_arch = fs["HOST_ARCH"];
+  to_lower(host_arch);
+  std::string host_compiler = fs["HOST_COMPILER"];
+  to_lower(host_compiler);
+
+  // std::string compiler_version = fs["COMPILER_VERSION"];
+  int compiler_version_major = fs["COMPILER_VERSION_MAJOR"];
+  // int compiler_version_minor = fs["COMPILER_VERSION_MINOR"];
+  // int compiler_version_patch = fs["COMPILER_VERSION_PATCH"];
+  // int compiler_version_tweak = fs["COMPILER_VERSION_TWEAK"];
+
+  std::string cuda_version = fs["CUDA_VERSION"];
+  // int cuda_version_major = fs["CUDA_VERSION_MAJOR"];
+  // int cuda_version_minor = fs["CUDA_VERSION_MINOR"];
+  // std::string cuda_version_string = fs["CUDA_VERSION_STRING"];
+
+  std::string opencv_version = fs["OpenCV_VERSION"];
+  // int opencv_version_major = fs["OpenCV_VERSION_MAJOR"];
+  // int opencv_version_minor = fs["OpenCV_VERSION_MINOR"];
+  // int opencv_version_patch = fs["OpenCV_VERSION_PATCH"];
+  // int opencv_version_tweak = fs["OpenCV_VERSION_TWEAK"];
+  // std::string opencv_version_status = fs["OpenCV_VERSION_STATUS"];
+  std::string opencv_with_world = fs["OpenCV_WITH_WORLD"];
+  to_lower(opencv_with_world);
+
+  std::string mynteye_version = fs["MYNTEYE_VERSION"];
+  // int mynteye_version_major = fs["MYNTEYE_VERSION_MAJOR"];
+  // int mynteye_version_minor = fs["MYNTEYE_VERSION_MINOR"];
+  // int mynteye_version_patch = fs["MYNTEYE_VERSION_PATCH"];
+  // int mynteye_version_tweak = fs["MYNTEYE_VERSION_TWEAK"];
+
+  fs.release();
+
+  std::string lib_prefix;
+  std::string lib_suffix;
+  if (host_os == "linux") {
+    if (host_compiler != "gnu" || compiler_version_major < 5)
+      return {};
+    lib_prefix = "lib";
+    lib_suffix = ".so";
+  } else if (host_os == "win") {
+    lib_prefix = "";
+    lib_suffix = ".dll";
+  } else if (host_os == "mac") {
+    lib_prefix = "lib";
+    lib_suffix = ".dylib";
+  } else {
+    return {};
+  }
+
+  std::vector<std::string> names;
+  {
+    std::vector<std::string> prefixes{
+        // lib_prefix + "plugin_b_ocl" + ocl_version,
+        lib_prefix + "plugin_g_cuda" + cuda_version,
+    };
+    std::string opencv_name("_opencv" + opencv_version);
+    if (opencv_with_world == "true") {
+      opencv_name.append("-world");
+    }
+    for (auto &&prefix : prefixes) {
+      names.push_back(prefix + opencv_name + "_mynteye" + mynteye_version);
+      names.push_back(prefix + opencv_name);
+      names.push_back(prefix);
+    }
+    for (auto &&name : names) {
+      name.append(lib_suffix);
+    }
+  }
+
+  std::vector<std::string> paths;
+
+  std::vector<std::string> plats;
+  if (host_name != host_os) {
+    plats.push_back(host_name + "-" + host_arch);
+  }
+  plats.push_back(host_os + "-" + host_arch);
+
+  std::vector<std::string> dirs{
+      utils::get_sdk_root_dir(), utils::get_sdk_install_dir()};
+  for (auto &&plat : plats) {
+    for (auto &&dir : dirs) {
+      auto &&plat_dir = dir + MYNTEYE_OS_SEP "plugins" + MYNTEYE_OS_SEP + plat;
+      // VLOG(2) << "plat_dir: " << plat_dir;
+      if (!dir_exists(plat_dir))
+        continue;
+      for (auto &&name : names) {
+        // VLOG(2) << "  name: " << name;
+        auto &&path = plat_dir + MYNTEYE_OS_SEP + name;
+        if (!file_exists(path))
+          continue;
+        paths.push_back(path);
+      }
+    }
+  }
+
+  return paths;
+}
+
+#endif
+
+}  // namespace
+
+API::API(std::shared_ptr<Device> device) : device_(device) {
+  VLOG(2) << __func__;
+  if (std::dynamic_pointer_cast<StandardDevice>(device_) != nullptr) {
+    bool in_l_ok, in_r_ok, ex_r2l_ok;
+    device_->GetIntrinsics(Stream::LEFT, &in_l_ok);
+    device_->GetIntrinsics(Stream::RIGHT, &in_r_ok);
+    device_->GetExtrinsics(Stream::RIGHT, Stream::LEFT, &ex_r2l_ok);
+    if (!in_l_ok || !in_r_ok || !ex_r2l_ok) {
+#if defined(WITH_DEVICE_INFO_REQUIRED)
+      LOG(FATAL)
+#else
+      LOG(WARNING)
+#endif
+          << "Image params not found, but we need it to process the "
+             "images. Please `make tools` and use `img_params_writer` "
+             "to write the image params. If you update the SDK from "
+             "1.x, the `SN*.conf` is the file contains them. Besides, "
+             "you could also calibrate them by yourself. Read the guide "
+             "doc (https://github.com/slightech/MYNT-EYE-S-SDK-Guide) "
+             "to learn more.";
+    }
+  }
+  synthetic_.reset(new Synthetic(this));
+}
+
+API::~API() {
+  VLOG(2) << __func__;
+}
+
+std::shared_ptr<API> API::Create() {
+  return Create(device::select());
+}
+
+std::shared_ptr<API> API::Create(std::shared_ptr<Device> device) {
+  if (!device)
+    return nullptr;
+  return std::make_shared<API>(device);
+}
+
+std::shared_ptr<API> API::Create(int argc, char *argv[]) {
+  static glog_init _(argc, argv);
+  auto &&device = device::select();
+  if (!device)
+    return nullptr;
+  return std::make_shared<API>(device);
+}
+
+std::shared_ptr<API> API::Create(
+    int argc, char *argv[], std::shared_ptr<Device> device) {
+  static glog_init _(argc, argv);
+  if (!device)
+    return nullptr;
+  return std::make_shared<API>(device);
+}
+
+Model API::GetModel() const {
+  return device_->GetModel();
+}
+
+bool API::Supports(const Stream &stream) const {
+  return synthetic_->Supports(stream);
+}
+
+bool API::Supports(const Capabilities &capability) const {
+  return device_->Supports(capability);
+}
+
+bool API::Supports(const Option &option) const {
+  return device_->Supports(option);
+}
+
+bool API::Supports(const AddOns &addon) const {
+  return device_->Supports(addon);
+}
+
+const std::vector<StreamRequest> &API::GetStreamRequests(
+    const Capabilities &capability) const {
+  return device_->GetStreamRequests(capability);
+}
+
+void API::ConfigStreamRequest(
+    const Capabilities &capability, const StreamRequest &request) {
+  device_->ConfigStreamRequest(capability, request);
+}
+
+std::string API::GetInfo(const Info &info) const {
+  return device_->GetInfo(info);
+}
+
+Intrinsics API::GetIntrinsics(const Stream &stream) const {
+  return device_->GetIntrinsics(stream);
+}
+
+Extrinsics API::GetExtrinsics(const Stream &from, const Stream &to) const {
+  return device_->GetExtrinsics(from, to);
+}
+
+MotionIntrinsics API::GetMotionIntrinsics() const {
+  return device_->GetMotionIntrinsics();
+}
+
+Extrinsics API::GetMotionExtrinsics(const Stream &from) const {
+  return device_->GetMotionExtrinsics(from);
+}
+
+void API::LogOptionInfos() const {
+  device_->LogOptionInfos();
+}
+
+OptionInfo API::GetOptionInfo(const Option &option) const {
+  return device_->GetOptionInfo(option);
+}
+
+std::int32_t API::GetOptionValue(const Option &option) const {
+  return device_->GetOptionValue(option);
+}
+
+void API::SetOptionValue(const Option &option, std::int32_t value) {
+  device_->SetOptionValue(option, value);
+}
+
+bool API::RunOptionAction(const Option &option) const {
+  return device_->RunOptionAction(option);
+}
+
+void API::SetStreamCallback(const Stream &stream, stream_callback_t callback) {
+  synthetic_->SetStreamCallback(stream, callback);
+}
+
+void API::SetMotionCallback(motion_callback_t callback) {
+  static auto callback_ = callback;
+  if (callback_) {
+    device_->SetMotionCallback(
+        [](const device::MotionData &data) { callback_({data.imu}); }, true);
+  } else {
+    device_->SetMotionCallback(nullptr);
+  }
+}
+
+bool API::HasStreamCallback(const Stream &stream) const {
+  return synthetic_->HasStreamCallback(stream);
+}
+
+bool API::HasMotionCallback() const {
+  return device_->HasMotionCallback();
+}
+
+void API::Start(const Source &source) {
+  if (source == Source::VIDEO_STREAMING) {
+#ifdef WITH_FILESYSTEM
+    if (!synthetic_->HasPlugin()) {
+      try {
+        auto &&plugin_paths = get_plugin_paths();
+        if (plugin_paths.size() > 0) {
+          EnablePlugin(plugin_paths[0]);
+        }
+      } catch (...) {
+        LOG(WARNING) << "Incorrect yaml format: build.info";
+      }
+    }
+#endif
+    synthetic_->StartVideoStreaming();
+  } else if (source == Source::MOTION_TRACKING) {
+    device_->StartMotionTracking();
+  } else if (source == Source::ALL) {
+    Start(Source::VIDEO_STREAMING);
+    Start(Source::MOTION_TRACKING);
+  } else {
+    LOG(ERROR) << "Unsupported source :(";
+  }
+}
+
+void API::Stop(const Source &source) {
+  if (source == Source::VIDEO_STREAMING) {
+    synthetic_->StopVideoStreaming();
+  } else if (source == Source::MOTION_TRACKING) {
+    device_->StopMotionTracking();
+  } else if (source == Source::ALL) {
+    Stop(Source::MOTION_TRACKING);
+    // Must stop motion tracking before video streaming and sleep a moment here
+    std::this_thread::sleep_for(std::chrono::milliseconds(10));
+    Stop(Source::VIDEO_STREAMING);
+  } else {
+    LOG(ERROR) << "Unsupported source :(";
+  }
+}
+
+void API::WaitForStreams() {
+  synthetic_->WaitForStreams();
+}
+
+void API::EnableStreamData(const Stream &stream) {
+  synthetic_->EnableStreamData(stream);
+}
+
+void API::DisableStreamData(const Stream &stream) {
+  synthetic_->DisableStreamData(stream);
+}
+
+api::StreamData API::GetStreamData(const Stream &stream) {
+  return synthetic_->GetStreamData(stream);
+}
+
+std::vector<api::StreamData> API::GetStreamDatas(const Stream &stream) {
+  return synthetic_->GetStreamDatas(stream);
+}
+
+void API::EnableMotionDatas(std::size_t max_size) {
+  device_->EnableMotionDatas(max_size);
+}
+
+std::vector<api::MotionData> API::GetMotionDatas() {
+  std::vector<api::MotionData> datas;
+  for (auto &&data : device_->GetMotionDatas()) {
+    datas.push_back({data.imu});
+  }
+  return datas;
+}
+
+void API::EnablePlugin(const std::string &path) {
+  static DL dl;
+  CHECK(dl.Open(path.c_str())) << "Open plugin failed: " << path;
+
+  plugin_version_code_t *plugin_version_code =
+      dl.Sym<plugin_version_code_t>("plugin_version_code");
+  LOG(INFO) << "Enable plugin success";
+  LOG(INFO) << "  version code: " << plugin_version_code();
+  LOG(INFO) << "  path: " << path;
+
+  plugin_create_t *plugin_create = dl.Sym<plugin_create_t>("plugin_create");
+  plugin_destroy_t *plugin_destroy = dl.Sym<plugin_destroy_t>("plugin_destroy");
+
+  std::shared_ptr<Plugin> plugin(plugin_create(), plugin_destroy);
+  plugin->OnCreate(this);
+
+  synthetic_->SetPlugin(plugin);
+}
+
+std::shared_ptr<Device> API::device() {
+  return device_;
+}
+
+MYNTEYE_END_NAMESPACE

src/mynteye/api/dl.cc

@@ -0,0 +1,129 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "mynteye/api/dl.h"
+
+#include "mynteye/logger.h"
+
+MYNTEYE_BEGIN_NAMESPACE
+
+#if defined(MYNTEYE_OS_WIN) && !defined(MYNTEYE_OS_MINGW) && \
+    !defined(MYNTEYE_OS_CYGWIN)
+
+namespace {
+
+// How to get the error message from the error code returned by GetLastError()?
+//   https://stackoverflow.com/questions/9272415/how-to-convert-dword-to-char
+std::string GetLastErrorAsString() {
+  DWORD dw = ::GetLastError();
+  if (dw == 0)
+    return std::string();
+  LPSTR lpMsgBuf;
+  size_t size = FormatMessageA(
+      FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
+          FORMAT_MESSAGE_IGNORE_INSERTS,
+      NULL, dw, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0,
+      NULL);
+  std::string message(lpMsgBuf, size);
+  LocalFree(lpMsgBuf);
+  return message;
+}
+
+}  // namespace
+
+#endif
+
+DL::DL() : handle(nullptr) {
+  VLOG(2) << __func__;
+}
+
+DL::DL(const char *filename) : handle(nullptr) {
+  VLOG(2) << __func__;
+  Open(filename);
+}
+
+DL::~DL() {
+  VLOG(2) << __func__;
+  Close();
+}
+
+bool DL::Open(const char *filename) {
+  if (handle != nullptr) {
+    VLOG(2) << "Already opened, do nothing";
+    // Close();
+    return false;
+  }
+#if defined(MYNTEYE_OS_WIN) && !defined(MYNTEYE_OS_MINGW) && \
+    !defined(MYNTEYE_OS_CYGWIN)
+  handle = LoadLibraryEx(filename, nullptr, 0);
+#else
+  handle = dlopen(filename, RTLD_LAZY);
+#endif
+  if (handle == nullptr) {
+    VLOG(2) << "Open library failed: " << filename;
+    return false;
+  } else {
+    return true;
+  }
+}
+
+bool DL::IsOpened() {
+  return handle != nullptr;
+}
+
+void *DL::Sym(const char *symbol) {
+  if (handle == nullptr) {
+    VLOG(2) << "Not opened, do nothing";
+    return nullptr;
+  }
+#if defined(MYNTEYE_OS_WIN) && !defined(MYNTEYE_OS_MINGW) && !defined(MYNTEYE_OS_CYGWIN)
+  void *f = GetProcAddress(handle, symbol);
+  if (f == nullptr) {
+    VLOG(2) << "Load symbol failed: " << symbol;
+  }
+#else
+  dlerror();  // reset errors
+  void *f = dlsym(handle, symbol);
+  const char *error = dlerror();
+  if (error != nullptr) {
+    VLOG(2) << "Load symbol failed: " << symbol;
+    f = nullptr;
+  }
+#endif
+  return f;
+}
+
+int DL::Close() {
+  int ret = 0;
+  if (handle == nullptr) {
+    VLOG(2) << "Not opened, do nothing";
+  } else {
+#if defined(MYNTEYE_OS_WIN) && !defined(MYNTEYE_OS_MINGW) && !defined(MYNTEYE_OS_CYGWIN)
+    ret = FreeLibrary(handle) ? 0 : 1;
+#else
+    ret = dlclose(handle);
+#endif
+    handle = nullptr;
+  }
+  return ret;
+}
+
+const char *DL::Error() {
+#if defined(MYNTEYE_OS_WIN) && !defined(MYNTEYE_OS_MINGW) && !defined(MYNTEYE_OS_CYGWIN)
+  return GetLastErrorAsString().c_str();
+#else
+  return dlerror();
+#endif
+}
+
+MYNTEYE_END_NAMESPACE

src/mynteye/api/dl.h

@@ -0,0 +1,71 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef MYNTEYE_API_DL_H_
+#define MYNTEYE_API_DL_H_
+#pragma once
+
+#include "mynteye/mynteye.h"
+
+#if defined(MYNTEYE_OS_WIN) && !defined(MYNTEYE_OS_MINGW) && \
+    !defined(MYNTEYE_OS_CYGWIN)
+#include <Windows.h>
+#else
+#include <dlfcn.h>
+#endif
+
+MYNTEYE_BEGIN_NAMESPACE
+
+#if defined(MYNTEYE_OS_WIN) && !defined(MYNTEYE_OS_MINGW) && \
+    !defined(MYNTEYE_OS_CYGWIN)
+using DLLIB = HMODULE;
+#else
+using DLLIB = void *;
+#endif
+
+// Dynamic loading
+//   https://en.wikipedia.org/wiki/Dynamic_loading
+// C++ dlopen mini HOWTO
+//   http://tldp.org/HOWTO/C++-dlopen/
+class MYNTEYE_API DL {
+ public:
+  DL();
+  explicit DL(const char *filename);
+  ~DL();
+
+  bool Open(const char *filename);
+
+  bool IsOpened();
+
+  void *Sym(const char *symbol);
+
+  template <typename Func>
+  Func *Sym(const char *symbol);
+
+  int Close();
+
+  const char *Error();
+
+ private:
+  DLLIB handle;
+};
+
+template <typename Func>
+Func *DL::Sym(const char *symbol) {
+  void *f = Sym(symbol);
+  return reinterpret_cast<Func *>(f);
+}
+
+MYNTEYE_END_NAMESPACE
+
+#endif  // MYNTEYE_API_DL_H_

src/mynteye/api/processor.cc

@@ -0,0 +1,247 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "mynteye/api/processor.h"
+
+#include <exception>
+#include <utility>
+
+#include "mynteye/logger.h"
+#include "mynteye/util/strings.h"
+#include "mynteye/util/times.h"
+
+MYNTEYE_BEGIN_NAMESPACE
+
+Processor::Processor(std::int32_t proc_period)
+    : proc_period_(std::move(proc_period)),
+      activated_(false),
+      input_ready_(false),
+      idle_(true),
+      dropped_count_(0),
+      input_(nullptr),
+      output_(nullptr),
+      output_result_(nullptr),
+      pre_callback_(nullptr),
+      post_callback_(nullptr),
+      callback_(nullptr),
+      parent_(nullptr) {
+  VLOG(2) << __func__;
+}
+
+Processor::~Processor() {
+  VLOG(2) << __func__;
+  Deactivate();
+  input_.reset(nullptr);
+  output_.reset(nullptr);
+  output_result_.reset(nullptr);
+  childs_.clear();
+}
+
+std::string Processor::Name() {
+  return "Processor";
+}
+
+void Processor::AddChild(const std::shared_ptr<Processor> &child) {
+  child->parent_ = this;
+  childs_.push_back(child);
+}
+
+void Processor::RemoveChild(const std::shared_ptr<Processor> &child) {
+  childs_.remove(child);
+}
+
+std::list<std::shared_ptr<Processor>> Processor::GetChilds() {
+  return childs_;
+}
+
+void Processor::SetPreProcessCallback(PreProcessCallback callback) {
+  pre_callback_ = std::move(callback);
+}
+
+void Processor::SetPostProcessCallback(PostProcessCallback callback) {
+  post_callback_ = std::move(callback);
+}
+
+void Processor::SetProcessCallback(ProcessCallback callback) {
+  callback_ = std::move(callback);
+}
+
+void Processor::Activate(bool parents) {
+  if (activated_)
+    return;
+  if (parents) {
+    // Activate all parents
+    Processor *parent = parent_;
+    while (parent != nullptr) {
+      parent->Activate();
+      parent = parent->parent_;
+    }
+  }
+  activated_ = true;
+  thread_ = std::thread(&Processor::Run, this);
+  // thread_.detach();
+}
+
+void Processor::Deactivate(bool childs) {
+  if (!activated_)
+    return;
+  if (childs) {
+    // Deactivate all childs
+    iterate_processors(GetChilds(), [](std::shared_ptr<Processor> proc) {
+      proc->Deactivate();
+    });
+  }
+  activated_ = false;
+  {
+    std::lock_guard<std::mutex> lk(mtx_input_ready_);
+    input_ready_ = true;
+  }
+  cond_input_ready_.notify_all();
+  thread_.join();
+}
+
+bool Processor::IsActivated() {
+  return activated_;
+}
+
+bool Processor::IsIdle() {
+  std::lock_guard<std::mutex> lk(mtx_state_);
+  return idle_;
+}
+
+bool Processor::Process(const Object &in) {
+  if (!activated_)
+    return false;
+  if (!idle_) {
+    std::lock_guard<std::mutex> lk(mtx_state_);
+    if (!idle_) {
+      ++dropped_count_;
+      return false;
+    }
+  }
+  if (!in.DecValidity()) {
+    LOG(WARNING) << Name() << " process with invalid input";
+    return false;
+  }
+  {
+    std::lock_guard<std::mutex> lk(mtx_input_ready_);
+    input_.reset(in.Clone());
+    input_ready_ = true;
+  }
+  cond_input_ready_.notify_all();
+  return true;
+}
+
+std::shared_ptr<Object> Processor::GetOutput() {
+  std::lock_guard<std::mutex> lk(mtx_result_);
+  return std::shared_ptr<Object>(std::move(output_result_));
+}
+
+std::uint64_t Processor::GetDroppedCount() {
+  std::lock_guard<std::mutex> lk(mtx_state_);
+  return dropped_count_;
+}
+
+void Processor::Run() {
+  VLOG(2) << Name() << " thread start";
+
+  auto sleep = [this](const times::system_clock::time_point &time_beg) {
+    if (proc_period_ > 0) {
+      static times::system_clock::time_point time_prev = time_beg;
+      auto &&time_elapsed_ms =
+          times::count<times::milliseconds>(times::now() - time_prev);
+      time_prev = time_beg;
+
+      if (time_elapsed_ms < proc_period_) {
+        VLOG(2) << Name() << " process cost "
+                << times::count<times::milliseconds>(times::now() - time_beg)
+                << " ms, sleep " << (proc_period_ - time_elapsed_ms) << " ms";
+        std::this_thread::sleep_for(
+            std::chrono::milliseconds(proc_period_ - time_elapsed_ms));
+        return;
+      }
+    }
+    VLOG(2) << Name() << " process cost "
+            << times::count<times::milliseconds>(times::now() - time_beg)
+            << " ms";
+  };
+
+  while (true) {
+    std::unique_lock<std::mutex> lk(mtx_input_ready_);
+    cond_input_ready_.wait(lk, [this] { return input_ready_; });
+
+    auto &&time_beg = times::now();
+
+    if (!activated_) {
+      SetIdle(true);
+      input_ready_ = false;
+      break;
+    }
+    SetIdle(false);
+
+    if (!output_) {
+      output_.reset(OnCreateOutput());
+    }
+
+    if (pre_callback_) {
+      pre_callback_(input_.get());
+    }
+    bool ok = false;
+    try {
+      if (callback_) {
+        if (callback_(input_.get(), output_.get(), parent_)) {
+          ok = true;
+        } else {
+          ok = OnProcess(input_.get(), output_.get(), parent_);
+        }
+      } else {
+        ok = OnProcess(input_.get(), output_.get(), parent_);
+      }
+      // CV_Assert(false);
+    } catch (const std::exception &e) {
+      std::string msg(e.what());
+      strings::rtrim(msg);
+      LOG(ERROR) << Name() << " process error \"" << msg << "\"";
+    }
+    if (!ok) {
+      VLOG(2) << Name() << " process failed";
+      continue;
+    }
+    if (post_callback_) {
+      post_callback_(output_.get());
+    }
+    {
+      std::unique_lock<std::mutex> lk(mtx_result_);
+      output_result_.reset(output_->Clone());
+    }
+
+    if (!childs_.empty()) {
+      for (auto child : childs_) {
+        child->Process(*output_);
+      }
+    }
+
+    SetIdle(true);
+    input_ready_ = false;
+
+    sleep(time_beg);
+  }
+  VLOG(2) << Name() << " thread end";
+}
+
+void Processor::SetIdle(bool idle) {
+  std::lock_guard<std::mutex> lk(mtx_state_);
+  idle_ = idle;
+}
+
+MYNTEYE_END_NAMESPACE

src/mynteye/api/processor.h

@@ -0,0 +1,121 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef MYNTEYE_API_PROCESSOR_H_
+#define MYNTEYE_API_PROCESSOR_H_
+#pragma once
+
+#include <condition_variable>
+#include <cstdint>
+#include <functional>
+#include <list>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <thread>
+
+#include "mynteye/mynteye.h"
+#include "mynteye/api/object.h"
+
+MYNTEYE_BEGIN_NAMESPACE
+
+class Processor /*: public std::enable_shared_from_this<Processor>*/ {
+ public:
+  using PreProcessCallback = std::function<void(Object *const)>;
+  using PostProcessCallback = std::function<void(Object *const)>;
+  using ProcessCallback = std::function<bool(
+      Object *const in, Object *const out, Processor *const parent)>;
+
+  explicit Processor(std::int32_t proc_period = 0);
+  virtual ~Processor();
+
+  virtual std::string Name();
+
+  void AddChild(const std::shared_ptr<Processor> &child);
+
+  void RemoveChild(const std::shared_ptr<Processor> &child);
+
+  std::list<std::shared_ptr<Processor>> GetChilds();
+
+  void SetPreProcessCallback(PreProcessCallback callback);
+  void SetPostProcessCallback(PostProcessCallback callback);
+  void SetProcessCallback(ProcessCallback callback);
+
+  void Activate(bool parents = false);
+  void Deactivate(bool childs = false);
+  bool IsActivated();
+
+  bool IsIdle();
+
+  /** Returns dropped or not. */
+  bool Process(const Object &in);
+
+  /**
+   * Returns the last output.
+   * @note Returns null if not output now.
+   */
+  std::shared_ptr<Object> GetOutput();
+
+  std::uint64_t GetDroppedCount();
+
+ protected:
+  virtual Object *OnCreateOutput() = 0;
+  virtual bool OnProcess(
+      Object *const in, Object *const out, Processor *const parent) = 0;
+
+ private:
+  /** Run in standalone thread. */
+  void Run();
+
+  void SetIdle(bool idle);
+
+  std::int32_t proc_period_;
+
+  bool activated_;
+
+  bool input_ready_;
+  std::mutex mtx_input_ready_;
+  std::condition_variable cond_input_ready_;
+
+  bool idle_;
+  std::uint64_t dropped_count_;
+  std::mutex mtx_state_;
+
+  std::unique_ptr<Object> input_;
+  std::unique_ptr<Object> output_;
+
+  std::unique_ptr<Object> output_result_;
+  std::mutex mtx_result_;
+
+  PreProcessCallback pre_callback_;
+  PostProcessCallback post_callback_;
+  ProcessCallback callback_;
+
+  Processor *parent_;
+  std::list<std::shared_ptr<Processor>> childs_;
+
+  std::thread thread_;
+};
+
+template <typename T>
+void iterate_processors(
+    const T &processors, std::function<void(std::shared_ptr<Processor>)> fn) {
+  for (auto &&proc : processors) {
+    fn(proc);
+    iterate_processors(proc->GetChilds(), fn);
+  }
+}
+
+MYNTEYE_END_NAMESPACE
+
+#endif  // MYNTEYE_API_PROCESSOR_H_

src/mynteye/api/processor/depth_processor.cc

@@ -0,0 +1,54 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "mynteye/api/processor/depth_processor.h"
+
+#include <utility>
+
+#include "mynteye/logger.h"
+
+MYNTEYE_BEGIN_NAMESPACE
+
+const char DepthProcessor::NAME[] = "DepthProcessor";
+
+DepthProcessor::DepthProcessor(std::int32_t proc_period)
+    : Processor(std::move(proc_period)) {
+  VLOG(2) << __func__ << ": proc_period=" << proc_period;
+}
+
+DepthProcessor::~DepthProcessor() {
+  VLOG(2) << __func__;
+}
+
+std::string DepthProcessor::Name() {
+  return NAME;
+}
+
+Object *DepthProcessor::OnCreateOutput() {
+  return new ObjMat();
+}
+
+bool DepthProcessor::OnProcess(
+    Object *const in, Object *const out, Processor *const parent) {
+  MYNTEYE_UNUSED(parent)
+  const ObjMat *input = Object::Cast<ObjMat>(in);
+  ObjMat *output = Object::Cast<ObjMat>(out);
+  cv::Mat channels[3 /*input->value.channels()*/];
+  cv::split(input->value, channels);
+  channels[2].convertTo(output->value, CV_16UC1);
+  output->id = input->id;
+  output->data = input->data;
+  return true;
+}
+
+MYNTEYE_END_NAMESPACE

src/mynteye/api/processor/depth_processor.h

@@ -0,0 +1,41 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef MYNTEYE_API_PROCESSOR_DEPTH_PROCESSOR_H_
+#define MYNTEYE_API_PROCESSOR_DEPTH_PROCESSOR_H_
+#pragma once
+
+#include <string>
+
+#include "mynteye/api/processor.h"
+
+MYNTEYE_BEGIN_NAMESPACE
+
+class DepthProcessor : public Processor {
+ public:
+  static const char NAME[];
+
+  explicit DepthProcessor(std::int32_t proc_period = 0);
+  virtual ~DepthProcessor();
+
+  std::string Name() override;
+
+ protected:
+  Object *OnCreateOutput() override;
+  bool OnProcess(
+      Object *const in, Object *const out, Processor *const parent) override;
+};
+
+MYNTEYE_END_NAMESPACE
+
+#endif  // MYNTEYE_API_PROCESSOR_DEPTH_PROCESSOR_H_

src/mynteye/api/processor/disparity_normalized_processor.cc

@@ -0,0 +1,57 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "mynteye/api/processor/disparity_normalized_processor.h"
+
+#include <utility>
+
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "mynteye/logger.h"
+
+MYNTEYE_BEGIN_NAMESPACE
+
+const char DisparityNormalizedProcessor::NAME[] =
+    "DisparityNormalizedProcessor";
+
+DisparityNormalizedProcessor::DisparityNormalizedProcessor(
+    std::int32_t proc_period)
+    : Processor(std::move(proc_period)) {
+  VLOG(2) << __func__ << ": proc_period=" << proc_period;
+}
+
+DisparityNormalizedProcessor::~DisparityNormalizedProcessor() {
+  VLOG(2) << __func__;
+}
+
+std::string DisparityNormalizedProcessor::Name() {
+  return NAME;
+}
+
+Object *DisparityNormalizedProcessor::OnCreateOutput() {
+  return new ObjMat();
+}
+
+bool DisparityNormalizedProcessor::OnProcess(
+    Object *const in, Object *const out, Processor *const parent) {
+  MYNTEYE_UNUSED(parent)
+  const ObjMat *input = Object::Cast<ObjMat>(in);
+  ObjMat *output = Object::Cast<ObjMat>(out);
+  cv::normalize(input->value, output->value, 0, 255, cv::NORM_MINMAX, CV_8UC1);
+  // cv::normalize maybe return empty ==
+  output->id = input->id;
+  output->data = input->data;
+  return !output->value.empty();
+}
+
+MYNTEYE_END_NAMESPACE

src/mynteye/api/processor/disparity_normalized_processor.h

@@ -0,0 +1,41 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef MYNTEYE_API_PROCESSOR_DISPARITY_NORMALIZED_PROCESSOR_H_
+#define MYNTEYE_API_PROCESSOR_DISPARITY_NORMALIZED_PROCESSOR_H_
+#pragma once
+
+#include <string>
+
+#include "mynteye/api/processor.h"
+
+MYNTEYE_BEGIN_NAMESPACE
+
+class DisparityNormalizedProcessor : public Processor {
+ public:
+  static const char NAME[];
+
+  explicit DisparityNormalizedProcessor(std::int32_t proc_period = 0);
+  virtual ~DisparityNormalizedProcessor();
+
+  std::string Name() override;
+
+ protected:
+  Object *OnCreateOutput() override;
+  bool OnProcess(
+      Object *const in, Object *const out, Processor *const parent) override;
+};
+
+MYNTEYE_END_NAMESPACE
+
+#endif  // MYNTEYE_API_PROCESSOR_DISPARITY_NORMALIZED_PROCESSOR_H_

src/mynteye/api/processor/disparity_processor.cc

@@ -0,0 +1,89 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "mynteye/api/processor/disparity_processor.h"
+
+#include <utility>
+
+#include <opencv2/calib3d/calib3d.hpp>
+
+#include "mynteye/logger.h"
+
+MYNTEYE_BEGIN_NAMESPACE
+
+const char DisparityProcessor::NAME[] = "DisparityProcessor";
+
+DisparityProcessor::DisparityProcessor(std::int32_t proc_period)
+    : Processor(std::move(proc_period)) {
+  VLOG(2) << __func__ << ": proc_period=" << proc_period;
+
+	int blockSize_ = 15;           // 15
+	int numDisparities_ = 64;      // 64
+
+#ifdef WITH_OPENCV2
+	bm_ = cv::Ptr<cv::StereoBM>(
+			new cv::StereoBM(
+				cv::StereoBM::BASIC_PRESET,
+				numDisparities_,
+				blockSize_));
+#else
+	int minDisparity_ = 0;         // 0
+	int preFilterSize_ = 9;        // 9
+	int preFilterCap_ = 31;        // 31
+	int uniquenessRatio_ = 15;     // 15
+	int textureThreshold_ = 10;    // 10
+	int speckleWindowSize_ = 100;  // 100
+	int speckleRange_ = 4;         // 4
+
+	bm_ = cv::StereoBM::create(16, 9);
+	bm_->setBlockSize(blockSize_);
+	bm_->setMinDisparity(minDisparity_);
+	bm_->setNumDisparities(numDisparities_);
+	bm_->setPreFilterSize(preFilterSize_);
+	bm_->setPreFilterCap(preFilterCap_);
+	bm_->setUniquenessRatio(uniquenessRatio_);
+	bm_->setTextureThreshold(textureThreshold_);
+	bm_->setSpeckleWindowSize(speckleWindowSize_);
+	bm_->setSpeckleRange(speckleRange_);
+#endif
+}
+
+DisparityProcessor::~DisparityProcessor() {
+  VLOG(2) << __func__;
+}
+
+std::string DisparityProcessor::Name() {
+  return NAME;
+}
+
+Object *DisparityProcessor::OnCreateOutput() {
+  return new ObjMat();
+}
+
+bool DisparityProcessor::OnProcess(
+    Object *const in, Object *const out, Processor *const parent) {
+  MYNTEYE_UNUSED(parent)
+  const ObjMat2 *input = Object::Cast<ObjMat2>(in);
+  ObjMat *output = Object::Cast<ObjMat>(out);
+
+  cv::Mat disparity;
+#ifdef WITH_OPENCV2
+	(*bm_)(input->first, input->second, disparity);
+#else
+  bm_->compute(input->first, input->second, disparity);
+#endif
+	disparity.convertTo(output->value, CV_32F, 1./16);
+  return true;
+}
+
+MYNTEYE_END_NAMESPACE

src/mynteye/api/processor/disparity_processor.h

@@ -0,0 +1,52 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef MYNTEYE_API_PROCESSOR_DISPARITY_PROCESSOR_H_
+#define MYNTEYE_API_PROCESSOR_DISPARITY_PROCESSOR_H_
+#pragma once
+
+#include <string>
+
+#include <opencv2/calib3d/calib3d.hpp>
+
+#include "mynteye/api/processor.h"
+
+namespace cv {
+
+class StereoBM;
+
+}  // namespace cv
+
+MYNTEYE_BEGIN_NAMESPACE
+
+class DisparityProcessor : public Processor {
+ public:
+  static const char NAME[];
+
+  explicit DisparityProcessor(std::int32_t proc_period = 0);
+  virtual ~DisparityProcessor();
+
+  std::string Name() override;
+
+ protected:
+  Object *OnCreateOutput() override;
+  bool OnProcess(
+      Object *const in, Object *const out, Processor *const parent) override;
+
+ private:
+  cv::Ptr<cv::StereoBM> bm_;
+};
+
+MYNTEYE_END_NAMESPACE
+
+#endif  // MYNTEYE_API_PROCESSOR_DISPARITY_PROCESSOR_H_

src/mynteye/api/processor/points_processor.cc

@@ -0,0 +1,83 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "mynteye/api/processor/points_processor.h"
+
+#include <utility>
+
+#include <opencv2/calib3d/calib3d.hpp>
+
+#include "mynteye/logger.h"
+
+MYNTEYE_BEGIN_NAMESPACE
+
+const char PointsProcessor::NAME[] = "PointsProcessor";
+
+PointsProcessor::PointsProcessor(cv::Mat Q, std::int32_t proc_period)
+    : Processor(std::move(proc_period)), Q_(std::move(Q)) {
+  VLOG(2) << __func__ << ": proc_period=" << proc_period;
+}
+
+PointsProcessor::~PointsProcessor() {
+  VLOG(2) << __func__;
+}
+
+std::string PointsProcessor::Name() {
+  return NAME;
+}
+
+Object *PointsProcessor::OnCreateOutput() {
+  return new ObjMat();
+}
+
+bool PointsProcessor::OnProcess(
+    Object *const in, Object *const out, Processor *const parent) {
+  MYNTEYE_UNUSED(parent)
+  const ObjMat *input = Object::Cast<ObjMat>(in);
+  ObjMat *output = Object::Cast<ObjMat>(out);
+
+  cv::Mat disparity = input->value;
+  output->value.create(disparity.size(), CV_MAKETYPE(CV_32FC3, 3));
+  cv::Mat _3dImage = output->value;
+
+  const float bigZ = 10000.f;
+  cv::Matx44d Q;
+  Q_.convertTo(Q, CV_64F);
+
+  int x, cols = disparity.cols;
+  CV_Assert(cols >= 0);
+
+  double minDisparity = FLT_MAX;
+
+  cv::minMaxIdx(disparity, &minDisparity, 0, 0, 0);
+
+  for (int y = 0; y < disparity.rows; y++) {
+    float *sptr = disparity.ptr<float>(y);
+    cv::Vec3f *dptr = _3dImage.ptr<cv::Vec3f>(y);
+
+    for (x = 0; x < cols; x++) {
+      double d = sptr[x];
+      cv::Vec4d homg_pt = Q * cv::Vec4d(x, y, d, 1.0);
+      dptr[x] = cv::Vec3d(homg_pt.val);
+      dptr[x] /= homg_pt[3];
+
+      if (fabs(d - minDisparity) <= FLT_EPSILON) {
+        dptr[x][2] = bigZ;
+      }
+    }
+  }
+
+  return true;
+}
+
+MYNTEYE_END_NAMESPACE

src/mynteye/api/processor/points_processor.h

@@ -0,0 +1,46 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef MYNTEYE_API_PROCESSOR_POINTS_PROCESSOR_H_
+#define MYNTEYE_API_PROCESSOR_POINTS_PROCESSOR_H_
+#pragma once
+
+#include <string>
+
+#include <opencv2/core/core.hpp>
+
+#include "mynteye/api/processor.h"
+
+MYNTEYE_BEGIN_NAMESPACE
+
+class PointsProcessor : public Processor {
+ public:
+  static const char NAME[];
+
+  explicit PointsProcessor(cv::Mat Q, std::int32_t proc_period = 0);
+  virtual ~PointsProcessor();
+
+  std::string Name() override;
+
+ protected:
+  Object *OnCreateOutput() override;
+  bool OnProcess(
+      Object *const in, Object *const out, Processor *const parent) override;
+
+ private:
+  cv::Mat Q_;
+};
+
+MYNTEYE_END_NAMESPACE
+
+#endif  // MYNTEYE_API_PROCESSOR_POINTS_PROCESSOR_H_

src/mynteye/api/processor/rectify_processor.cc

@@ -0,0 +1,104 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "mynteye/api/processor/rectify_processor.h"
+
+#include <utility>
+
+#include <opencv2/calib3d/calib3d.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "mynteye/logger.h"
+#include "mynteye/device/device.h"
+
+MYNTEYE_BEGIN_NAMESPACE
+
+const char RectifyProcessor::NAME[] = "RectifyProcessor";
+
+RectifyProcessor::RectifyProcessor(
+    std::shared_ptr<Device> device, std::int32_t proc_period)
+    : Processor(std::move(proc_period)) {
+  VLOG(2) << __func__ << ": proc_period=" << proc_period;
+  InitParams(
+      device->GetIntrinsics(Stream::LEFT), device->GetIntrinsics(Stream::RIGHT),
+      device->GetExtrinsics(Stream::RIGHT, Stream::LEFT));
+}
+
+RectifyProcessor::~RectifyProcessor() {
+  VLOG(2) << __func__;
+}
+
+std::string RectifyProcessor::Name() {
+  return NAME;
+}
+
+Object *RectifyProcessor::OnCreateOutput() {
+  return new ObjMat2();
+}
+
+bool RectifyProcessor::OnProcess(
+    Object *const in, Object *const out, Processor *const parent) {
+  MYNTEYE_UNUSED(parent)
+  const ObjMat2 *input = Object::Cast<ObjMat2>(in);
+  ObjMat2 *output = Object::Cast<ObjMat2>(out);
+  cv::remap(input->first, output->first, map11, map12, cv::INTER_LINEAR);
+  cv::remap(input->second, output->second, map21, map22, cv::INTER_LINEAR);
+  output->first_id = input->first_id;
+  output->first_data = input->first_data;
+  output->second_id = input->second_id;
+  output->second_data = input->second_data;
+  return true;
+}
+
+void RectifyProcessor::InitParams(
+    Intrinsics in_left, Intrinsics in_right, Extrinsics ex_right_to_left) {
+  cv::Size size{in_left.width, in_left.height};
+
+  cv::Mat M1 =
+      (cv::Mat_<double>(3, 3) << in_left.fx, 0, in_left.cx, 0, in_left.fy,
+       in_left.cy, 0, 0, 1);
+  cv::Mat M2 =
+      (cv::Mat_<double>(3, 3) << in_right.fx, 0, in_right.cx, 0, in_right.fy,
+       in_right.cy, 0, 0, 1);
+  cv::Mat D1(1, 5, CV_64F, in_left.coeffs);
+  cv::Mat D2(1, 5, CV_64F, in_right.coeffs);
+	/*
+  cv::Mat R =
+      (cv::Mat_<double>(3, 3) << ex_right_to_left.rotation[0][0],
+       ex_right_to_left.rotation[0][1], ex_right_to_left.rotation[0][2],
+       ex_right_to_left.rotation[1][0], ex_right_to_left.rotation[1][1],
+       ex_right_to_left.rotation[1][2], ex_right_to_left.rotation[2][0],
+       ex_right_to_left.rotation[2][1], ex_right_to_left.rotation[2][2]);
+			 */
+	cv::Mat R =
+      (cv::Mat_<double>(3, 3) << 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0);
+  cv::Mat T(3, 1, CV_64F, ex_right_to_left.translation);
+
+  VLOG(2) << "InitParams size: " << size;
+  VLOG(2) << "M1: " << M1;
+  VLOG(2) << "M2: " << M2;
+  VLOG(2) << "D1: " << D1;
+  VLOG(2) << "D2: " << D2;
+  VLOG(2) << "R: " << R;
+  VLOG(2) << "T: " << T;
+
+  cv::Rect left_roi, right_roi;
+  cv::stereoRectify(
+      M1, D1, M2, D2, size, R, T, R1, R2, P1, P2, Q, cv::CALIB_ZERO_DISPARITY,
+      0, size, &left_roi, &right_roi);
+
+  cv::initUndistortRectifyMap(M1, D1, R1, P1, size, CV_16SC2, map11, map12);
+  cv::initUndistortRectifyMap(M2, D2, R2, P2, size, CV_16SC2, map21, map22);
+}
+
+MYNTEYE_END_NAMESPACE

src/mynteye/api/processor/rectify_processor.h

@@ -0,0 +1,55 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef MYNTEYE_API_PROCESSOR_RECTIFY_PROCESSOR_H_
+#define MYNTEYE_API_PROCESSOR_RECTIFY_PROCESSOR_H_
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include <opencv2/core/core.hpp>
+
+#include "mynteye/types.h"
+#include "mynteye/api/processor.h"
+
+MYNTEYE_BEGIN_NAMESPACE
+
+class Device;
+
+class RectifyProcessor : public Processor {
+ public:
+  static const char NAME[];
+
+  RectifyProcessor(
+      std::shared_ptr<Device> device, std::int32_t proc_period = 0);
+  virtual ~RectifyProcessor();
+
+  std::string Name() override;
+
+  cv::Mat R1, P1, R2, P2, Q;
+  cv::Mat map11, map12, map21, map22;
+
+ protected:
+  Object *OnCreateOutput() override;
+  bool OnProcess(
+      Object *const in, Object *const out, Processor *const parent) override;
+
+ private:
+  void InitParams(
+      Intrinsics in_left, Intrinsics in_right, Extrinsics ex_right_to_left);
+};
+
+MYNTEYE_END_NAMESPACE
+
+#endif  // MYNTEYE_API_PROCESSOR_RECTIFY_PROCESSOR_H_

src/mynteye/api/synthetic.cc

@@ -0,0 +1,596 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "mynteye/api/synthetic.h"
+
+#include <algorithm>
+#include <functional>
+#include <stdexcept>
+
+#include "mynteye/logger.h"
+#include "mynteye/api/object.h"
+#include "mynteye/api/plugin.h"
+#include "mynteye/api/processor.h"
+#include "mynteye/api/processor/depth_processor.h"
+#include "mynteye/api/processor/disparity_normalized_processor.h"
+#include "mynteye/api/processor/disparity_processor.h"
+#include "mynteye/api/processor/points_processor.h"
+#include "mynteye/api/processor/rectify_processor.h"
+#include "mynteye/device/device.h"
+
+#define RECTIFY_PROC_PERIOD 0
+#define DISPARITY_PROC_PERIOD 0
+#define DISPARITY_NORM_PROC_PERIOD 0
+#define POINTS_PROC_PERIOD 0
+#define DEPTH_PROC_PERIOD 0
+
+MYNTEYE_BEGIN_NAMESPACE
+
+namespace {
+
+cv::Mat frame2mat(const std::shared_ptr<device::Frame> &frame) {
+  // TODO(JohnZhao) Support different format frame to cv::Mat
+  CHECK_EQ(frame->format(), Format::GREY);
+  return cv::Mat(frame->height(), frame->width(), CV_8UC1, frame->data());
+}
+
+api::StreamData data2api(const device::StreamData &data) {
+  return {data.img, frame2mat(data.frame), data.frame, data.frame_id};
+}
+
+void process_childs(
+    const std::shared_ptr<Processor> &proc, const std::string &name,
+    const Object &obj) {
+  auto &&processor = find_processor<Processor>(proc, name);
+  for (auto child : processor->GetChilds()) {
+    child->Process(obj);
+  }
+}
+
+}  // namespace
+
+Synthetic::Synthetic(API *api) : api_(api), plugin_(nullptr) {
+  VLOG(2) << __func__;
+  CHECK_NOTNULL(api_);
+  InitStreamSupports();
+  InitProcessors();
+}
+
+Synthetic::~Synthetic() {
+  VLOG(2) << __func__;
+  if (processor_) {
+    processor_->Deactivate(true);
+    processor_ = nullptr;
+  }
+}
+
+bool Synthetic::Supports(const Stream &stream) const {
+  return stream_supports_mode_.find(stream) != stream_supports_mode_.end();
+}
+
+Synthetic::mode_t Synthetic::SupportsMode(const Stream &stream) const {
+  try {
+    return stream_supports_mode_.at(stream);
+  } catch (const std::out_of_range &e) {
+    return MODE_LAST;
+  }
+}
+
+void Synthetic::EnableStreamData(const Stream &stream) {
+  EnableStreamData(stream, 0);
+}
+
+void Synthetic::DisableStreamData(const Stream &stream) {
+  DisableStreamData(stream, 0);
+}
+
+bool Synthetic::IsStreamDataEnabled(const Stream &stream) const {
+  return stream_enabled_mode_.find(stream) != stream_enabled_mode_.end();
+}
+
+void Synthetic::SetStreamCallback(
+    const Stream &stream, stream_callback_t callback) {
+  if (callback == nullptr) {
+    stream_callbacks_.erase(stream);
+  } else {
+    stream_callbacks_[stream] = callback;
+  }
+}
+
+bool Synthetic::HasStreamCallback(const Stream &stream) const {
+  return stream_callbacks_.find(stream) != stream_callbacks_.end();
+}
+
+void Synthetic::StartVideoStreaming() {
+  auto &&device = api_->device();
+  for (auto &&it = stream_supports_mode_.begin();
+       it != stream_supports_mode_.end(); it++) {
+    if (it->second == MODE_NATIVE) {
+      auto &&stream = it->first;
+      device->SetStreamCallback(
+          stream,
+          [this, stream](const device::StreamData &data) {
+            auto &&stream_data = data2api(data);
+            ProcessNativeStream(stream, stream_data);
+            // Need mutex if set callback after start
+            if (HasStreamCallback(stream)) {
+              stream_callbacks_.at(stream)(stream_data);
+            }
+          },
+          true);
+    }
+  }
+  device->Start(Source::VIDEO_STREAMING);
+}
+
+void Synthetic::StopVideoStreaming() {
+  auto &&device = api_->device();
+  for (auto &&it = stream_supports_mode_.begin();
+       it != stream_supports_mode_.end(); it++) {
+    if (it->second == MODE_NATIVE) {
+      device->SetStreamCallback(it->first, nullptr);
+    }
+  }
+  device->Stop(Source::VIDEO_STREAMING);
+}
+
+void Synthetic::WaitForStreams() {
+  api_->device()->WaitForStreams();
+}
+
+api::StreamData Synthetic::GetStreamData(const Stream &stream) {
+  auto &&mode = GetStreamEnabledMode(stream);
+  if (mode == MODE_NATIVE) {
+    auto &&device = api_->device();
+    return data2api(device->GetLatestStreamData(stream));
+  } else if (mode == MODE_SYNTHETIC) {
+    if (stream == Stream::LEFT_RECTIFIED || stream == Stream::RIGHT_RECTIFIED) {
+      static std::shared_ptr<ObjMat2> output = nullptr;
+      auto &&processor = find_processor<RectifyProcessor>(processor_);
+      auto &&out = processor->GetOutput();
+      if (out != nullptr) {
+        // Obtain the output, out will be nullptr if get again immediately.
+        output = Object::Cast<ObjMat2>(out);
+      }
+      if (output != nullptr) {
+        if (stream == Stream::LEFT_RECTIFIED) {
+          return {output->first_data, output->first, nullptr, output->first_id};
+        } else {
+          return {output->second_data, output->second, nullptr,
+                  output->second_id};
+        }
+      }
+      VLOG(2) << "Rectify not ready now";
+      return {};
+    }
+    switch (stream) {
+      case Stream::DISPARITY: {
+        auto &&processor = find_processor<DisparityProcessor>(processor_);
+        auto &&out = processor->GetOutput();
+        if (out != nullptr) {
+          auto &&output = Object::Cast<ObjMat>(out);
+          return {output->data, output->value, nullptr, output->id};
+        }
+        VLOG(2) << "Disparity not ready now";
+      } break;
+      case Stream::DISPARITY_NORMALIZED: {
+        auto &&processor =
+            find_processor<DisparityNormalizedProcessor>(processor_);
+        auto &&out = processor->GetOutput();
+        if (out != nullptr) {
+          auto &&output = Object::Cast<ObjMat>(out);
+          return {output->data, output->value, nullptr, output->id};
+        }
+        VLOG(2) << "Disparity normalized not ready now";
+      } break;
+      case Stream::POINTS: {
+        auto &&processor = find_processor<PointsProcessor>(processor_);
+        auto &&out = processor->GetOutput();
+        if (out != nullptr) {
+          auto &&output = Object::Cast<ObjMat>(out);
+          return {output->data, output->value, nullptr, output->id};
+        }
+        VLOG(2) << "Points not ready now";
+      } break;
+      case Stream::DEPTH: {
+        auto &&processor = find_processor<DepthProcessor>(processor_);
+        auto &&out = processor->GetOutput();
+        if (out != nullptr) {
+          auto &&output = Object::Cast<ObjMat>(out);
+          return {output->data, output->value, nullptr, output->id};
+        }
+        VLOG(2) << "Depth not ready now";
+      } break;
+      default:
+        break;
+    }
+    return {};  // frame.empty() == true
+  } else {
+    LOG(ERROR) << "Failed to get stream data of " << stream
+               << ", unsupported or disabled";
+    return {};  // frame.empty() == true
+  }
+}
+
+std::vector<api::StreamData> Synthetic::GetStreamDatas(const Stream &stream) {
+  auto &&mode = GetStreamEnabledMode(stream);
+  if (mode == MODE_NATIVE) {
+    auto &&device = api_->device();
+    std::vector<api::StreamData> datas;
+    for (auto &&data : device->GetStreamDatas(stream)) {
+      datas.push_back(data2api(data));
+    }
+    return datas;
+  } else if (mode == MODE_SYNTHETIC) {
+    return {GetStreamData(stream)};
+  } else {
+    LOG(ERROR) << "Failed to get stream data of " << stream
+               << ", unsupported or disabled";
+  }
+  return {};
+}
+
+void Synthetic::SetPlugin(std::shared_ptr<Plugin> plugin) {
+  plugin_ = plugin;
+}
+
+bool Synthetic::HasPlugin() const {
+  return plugin_ != nullptr;
+}
+
+void Synthetic::InitStreamSupports() {
+  auto &&device = api_->device();
+  if (device->Supports(Stream::LEFT) && device->Supports(Stream::RIGHT)) {
+    stream_supports_mode_[Stream::LEFT] = MODE_NATIVE;
+    stream_supports_mode_[Stream::RIGHT] = MODE_NATIVE;
+
+    std::vector<Stream> stream_chain{
+        Stream::LEFT_RECTIFIED, Stream::RIGHT_RECTIFIED,
+        Stream::DISPARITY,      Stream::DISPARITY_NORMALIZED,
+        Stream::POINTS,         Stream::DEPTH};
+    for (auto &&stream : stream_chain) {
+      if (device->Supports(stream)) {
+        stream_supports_mode_[stream] = MODE_NATIVE;
+      } else {
+        stream_supports_mode_[stream] = MODE_SYNTHETIC;
+      }
+    }
+  }
+
+  // Enabled native streams by default
+  for (auto &&it = stream_supports_mode_.begin();
+       it != stream_supports_mode_.end(); it++) {
+    if (it->second == MODE_NATIVE) {
+      stream_enabled_mode_[it->first] = MODE_NATIVE;
+    }
+  }
+}
+
+Synthetic::mode_t Synthetic::GetStreamEnabledMode(const Stream &stream) const {
+  try {
+    return stream_enabled_mode_.at(stream);
+  } catch (const std::out_of_range &e) {
+    return MODE_LAST;
+  }
+}
+
+bool Synthetic::IsStreamEnabledNative(const Stream &stream) const {
+  return GetStreamEnabledMode(stream) == MODE_NATIVE;
+}
+
+bool Synthetic::IsStreamEnabledSynthetic(const Stream &stream) const {
+  return GetStreamEnabledMode(stream) == MODE_SYNTHETIC;
+}
+
+void Synthetic::EnableStreamData(const Stream &stream, std::uint32_t depth) {
+  if (IsStreamDataEnabled(stream))
+    return;
+  // Activate processors of synthetic stream
+  switch (stream) {
+    case Stream::LEFT_RECTIFIED: {
+      if (!IsStreamDataEnabled(Stream::LEFT))
+        break;
+      stream_enabled_mode_[stream] = MODE_SYNTHETIC;
+      CHECK(ActivateProcessor<RectifyProcessor>());
+    }
+      return;
+    case Stream::RIGHT_RECTIFIED: {
+      if (!IsStreamDataEnabled(Stream::RIGHT))
+        break;
+      stream_enabled_mode_[stream] = MODE_SYNTHETIC;
+      CHECK(ActivateProcessor<RectifyProcessor>());
+    }
+      return;
+    case Stream::DISPARITY: {
+      stream_enabled_mode_[stream] = MODE_SYNTHETIC;
+      EnableStreamData(Stream::LEFT_RECTIFIED, depth + 1);
+      EnableStreamData(Stream::RIGHT_RECTIFIED, depth + 1);
+      CHECK(ActivateProcessor<DisparityProcessor>());
+    }
+      return;
+    case Stream::DISPARITY_NORMALIZED: {
+      stream_enabled_mode_[stream] = MODE_SYNTHETIC;
+      EnableStreamData(Stream::DISPARITY, depth + 1);
+      CHECK(ActivateProcessor<DisparityNormalizedProcessor>());
+    }
+      return;
+    case Stream::POINTS: {
+      stream_enabled_mode_[stream] = MODE_SYNTHETIC;
+      EnableStreamData(Stream::DISPARITY, depth + 1);
+      CHECK(ActivateProcessor<PointsProcessor>());
+    }
+      return;
+    case Stream::DEPTH: {
+      stream_enabled_mode_[stream] = MODE_SYNTHETIC;
+      EnableStreamData(Stream::POINTS, depth + 1);
+      CHECK(ActivateProcessor<DepthProcessor>());
+    }
+      return;
+    default:
+      break;
+  }
+  if (depth == 0) {
+    LOG(WARNING) << "Enable stream data of " << stream << " failed";
+  }
+}
+
+void Synthetic::DisableStreamData(const Stream &stream, std::uint32_t depth) {
+  if (!IsStreamDataEnabled(stream))
+    return;
+  // Deactivate processors of synthetic stream
+  if (stream_enabled_mode_[stream] != MODE_NATIVE) {
+    stream_enabled_mode_.erase(stream);
+    switch (stream) {
+      case Stream::LEFT_RECTIFIED: {
+        if (IsStreamEnabledSynthetic(Stream::RIGHT_RECTIFIED)) {
+          DisableStreamData(Stream::RIGHT_RECTIFIED, depth + 1);
+        }
+        if (IsStreamEnabledSynthetic(Stream::DISPARITY)) {
+          DisableStreamData(Stream::DISPARITY, depth + 1);
+        }
+        DeactivateProcessor<RectifyProcessor>();
+      } break;
+      case Stream::RIGHT_RECTIFIED: {
+        if (IsStreamEnabledSynthetic(Stream::LEFT_RECTIFIED)) {
+          DisableStreamData(Stream::LEFT_RECTIFIED, depth + 1);
+        }
+        if (IsStreamEnabledSynthetic(Stream::DISPARITY)) {
+          DisableStreamData(Stream::DISPARITY, depth + 1);
+        }
+        DeactivateProcessor<RectifyProcessor>();
+      } break;
+      case Stream::DISPARITY: {
+        if (IsStreamEnabledSynthetic(Stream::DISPARITY_NORMALIZED)) {
+          DisableStreamData(Stream::DISPARITY_NORMALIZED, depth + 1);
+        }
+        if (IsStreamEnabledSynthetic(Stream::POINTS)) {
+          DisableStreamData(Stream::POINTS, depth + 1);
+        }
+        DeactivateProcessor<DisparityProcessor>();
+      } break;
+      case Stream::DISPARITY_NORMALIZED: {
+        DeactivateProcessor<DisparityNormalizedProcessor>();
+      } break;
+      case Stream::POINTS: {
+        if (IsStreamEnabledSynthetic(Stream::DEPTH)) {
+          DisableStreamData(Stream::DEPTH, depth + 1);
+        }
+        DeactivateProcessor<PointsProcessor>();
+      } break;
+      case Stream::DEPTH: {
+        DeactivateProcessor<DepthProcessor>();
+      } break;
+      default:
+        return;
+    }
+    if (depth > 0) {
+      LOG(WARNING) << "Disable synthetic stream data of " << stream << " too";
+    }
+  } else if (depth == 0) {
+    LOG(WARNING) << "Disable native stream data of " << stream << " failed";
+  }
+}
+
+void Synthetic::InitProcessors() {
+  auto &&rectify_processor =
+      std::make_shared<RectifyProcessor>(api_->device(), RECTIFY_PROC_PERIOD);
+  auto &&disparity_processor =
+      std::make_shared<DisparityProcessor>(DISPARITY_PROC_PERIOD);
+  auto &&disparitynormalized_processor =
+      std::make_shared<DisparityNormalizedProcessor>(
+          DISPARITY_NORM_PROC_PERIOD);
+  auto &&points_processor = std::make_shared<PointsProcessor>(
+      rectify_processor->Q, POINTS_PROC_PERIOD);
+  auto &&depth_processor = std::make_shared<DepthProcessor>(DEPTH_PROC_PERIOD);
+
+  using namespace std::placeholders;  // NOLINT
+  rectify_processor->SetProcessCallback(
+      std::bind(&Synthetic::OnRectifyProcess, this, _1, _2, _3));
+  disparity_processor->SetProcessCallback(
+      std::bind(&Synthetic::OnDisparityProcess, this, _1, _2, _3));
+  disparitynormalized_processor->SetProcessCallback(
+      std::bind(&Synthetic::OnDisparityNormalizedProcess, this, _1, _2, _3));
+  points_processor->SetProcessCallback(
+      std::bind(&Synthetic::OnPointsProcess, this, _1, _2, _3));
+  depth_processor->SetProcessCallback(
+      std::bind(&Synthetic::OnDepthProcess, this, _1, _2, _3));
+
+  rectify_processor->SetPostProcessCallback(
+      std::bind(&Synthetic::OnRectifyPostProcess, this, _1));
+  disparity_processor->SetPostProcessCallback(
+      std::bind(&Synthetic::OnDisparityPostProcess, this, _1));
+  disparitynormalized_processor->SetPostProcessCallback(
+      std::bind(&Synthetic::OnDisparityNormalizedPostProcess, this, _1));
+  points_processor->SetPostProcessCallback(
+      std::bind(&Synthetic::OnPointsPostProcess, this, _1));
+  depth_processor->SetPostProcessCallback(
+      std::bind(&Synthetic::OnDepthPostProcess, this, _1));
+
+  rectify_processor->AddChild(disparity_processor);
+  disparity_processor->AddChild(disparitynormalized_processor);
+  disparity_processor->AddChild(points_processor);
+  points_processor->AddChild(depth_processor);
+
+  processor_ = rectify_processor;
+}
+
+void Synthetic::ProcessNativeStream(
+    const Stream &stream, const api::StreamData &data) {
+  if (stream == Stream::LEFT || stream == Stream::RIGHT) {
+    static api::StreamData left_data, right_data;
+    if (stream == Stream::LEFT) {
+      left_data = data;
+    } else if (stream == Stream::RIGHT) {
+      right_data = data;
+    }
+    if (left_data.img && right_data.img &&
+        left_data.img->frame_id == right_data.img->frame_id) {
+      auto &&processor = find_processor<RectifyProcessor>(processor_);
+      processor->Process(ObjMat2{
+          left_data.frame, left_data.frame_id, left_data.img,
+          right_data.frame, right_data.frame_id, right_data.img});
+    }
+    return;
+  }
+
+  if (stream == Stream::LEFT_RECTIFIED || stream == Stream::RIGHT_RECTIFIED) {
+    static api::StreamData left_rect_data, right_rect_data;
+    if (stream == Stream::LEFT_RECTIFIED) {
+      left_rect_data = data;
+    } else if (stream == Stream::RIGHT_RECTIFIED) {
+      right_rect_data = data;
+    }
+    if (left_rect_data.img && right_rect_data.img &&
+        left_rect_data.img->frame_id == right_rect_data.img->frame_id) {
+      process_childs(
+          processor_, RectifyProcessor::NAME, ObjMat2{
+              left_rect_data.frame, left_rect_data.frame_id, left_rect_data.img,
+              right_rect_data.frame, right_rect_data.frame_id,
+              right_rect_data.img});
+    }
+    return;
+  }
+
+  switch (stream) {
+    case Stream::DISPARITY: {
+      process_childs(processor_, DisparityProcessor::NAME,
+          ObjMat{data.frame, data.frame_id, data.img});
+    } break;
+    case Stream::DISPARITY_NORMALIZED: {
+      process_childs(processor_, DisparityNormalizedProcessor::NAME,
+          ObjMat{data.frame, data.frame_id, data.img});
+    } break;
+    case Stream::POINTS: {
+      process_childs(processor_, PointsProcessor::NAME,
+          ObjMat{data.frame, data.frame_id, data.img});
+    } break;
+    case Stream::DEPTH: {
+      process_childs(processor_, DepthProcessor::NAME,
+          ObjMat{data.frame, data.frame_id, data.img});
+    } break;
+    default:
+      break;
+  }
+}
+
+bool Synthetic::OnRectifyProcess(
+    Object *const in, Object *const out, Processor *const parent) {
+  MYNTEYE_UNUSED(parent)
+  if (plugin_ && plugin_->OnRectifyProcess(in, out)) {
+    return true;
+  }
+  return GetStreamEnabledMode(Stream::LEFT_RECTIFIED) != MODE_SYNTHETIC;
+  // && GetStreamEnabledMode(Stream::RIGHT_RECTIFIED) != MODE_SYNTHETIC
+}
+
+bool Synthetic::OnDisparityProcess(
+    Object *const in, Object *const out, Processor *const parent) {
+  MYNTEYE_UNUSED(parent)
+  if (plugin_ && plugin_->OnDisparityProcess(in, out)) {
+    return true;
+  }
+  return GetStreamEnabledMode(Stream::DISPARITY) != MODE_SYNTHETIC;
+}
+
+bool Synthetic::OnDisparityNormalizedProcess(
+    Object *const in, Object *const out, Processor *const parent) {
+  MYNTEYE_UNUSED(parent)
+  if (plugin_ && plugin_->OnDisparityNormalizedProcess(in, out)) {
+    return true;
+  }
+  return GetStreamEnabledMode(Stream::DISPARITY_NORMALIZED) != MODE_SYNTHETIC;
+}
+
+bool Synthetic::OnPointsProcess(
+    Object *const in, Object *const out, Processor *const parent) {
+  MYNTEYE_UNUSED(parent)
+  if (plugin_ && plugin_->OnPointsProcess(in, out)) {
+    return true;
+  }
+  return GetStreamEnabledMode(Stream::POINTS) != MODE_SYNTHETIC;
+}
+
+bool Synthetic::OnDepthProcess(
+    Object *const in, Object *const out, Processor *const parent) {
+  MYNTEYE_UNUSED(parent)
+  if (plugin_ && plugin_->OnDepthProcess(in, out)) {
+    return true;
+  }
+  return GetStreamEnabledMode(Stream::DEPTH) != MODE_SYNTHETIC;
+}
+
+void Synthetic::OnRectifyPostProcess(Object *const out) {
+  const ObjMat2 *output = Object::Cast<ObjMat2>(out);
+  if (HasStreamCallback(Stream::LEFT_RECTIFIED)) {
+    stream_callbacks_.at(Stream::LEFT_RECTIFIED)(
+        {output->first_data, output->first, nullptr, output->first_id});
+  }
+  if (HasStreamCallback(Stream::RIGHT_RECTIFIED)) {
+    stream_callbacks_.at(Stream::RIGHT_RECTIFIED)(
+        {output->second_data, output->second, nullptr, output->second_id});
+  }
+}
+
+void Synthetic::OnDisparityPostProcess(Object *const out) {
+  const ObjMat *output = Object::Cast<ObjMat>(out);
+  if (HasStreamCallback(Stream::DISPARITY)) {
+    stream_callbacks_.at(Stream::DISPARITY)(
+        {output->data, output->value, nullptr, output->id});
+  }
+}
+
+void Synthetic::OnDisparityNormalizedPostProcess(Object *const out) {
+  const ObjMat *output = Object::Cast<ObjMat>(out);
+  if (HasStreamCallback(Stream::DISPARITY_NORMALIZED)) {
+    stream_callbacks_.at(Stream::DISPARITY_NORMALIZED)(
+        {output->data, output->value, nullptr, output->id});
+  }
+}
+
+void Synthetic::OnPointsPostProcess(Object *const out) {
+  const ObjMat *output = Object::Cast<ObjMat>(out);
+  if (HasStreamCallback(Stream::POINTS)) {
+    stream_callbacks_.at(Stream::POINTS)(
+        {output->data, output->value, nullptr, output->id});
+  }
+}
+
+void Synthetic::OnDepthPostProcess(Object *const out) {
+  const ObjMat *output = Object::Cast<ObjMat>(out);
+  if (HasStreamCallback(Stream::DEPTH)) {
+    stream_callbacks_.at(Stream::DEPTH)(
+        {output->data, output->value, nullptr, output->id});
+  }
+}
+
+MYNTEYE_END_NAMESPACE

src/mynteye/api/synthetic.h

@@ -0,0 +1,172 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef MYNTEYE_API_SYNTHETIC_H_
+#define MYNTEYE_API_SYNTHETIC_H_
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mynteye/api/api.h"
+
+MYNTEYE_BEGIN_NAMESPACE
+
+class API;
+class Plugin;
+class Processor;
+
+struct Object;
+
+class Synthetic {
+ public:
+  using stream_callback_t = API::stream_callback_t;
+
+  typedef enum Mode {
+    MODE_NATIVE,     // Native stream
+    MODE_SYNTHETIC,  // Synthetic stream
+    MODE_LAST        // Unsupported
+  } mode_t;
+
+  explicit Synthetic(API *api);
+  ~Synthetic();
+
+  bool Supports(const Stream &stream) const;
+  mode_t SupportsMode(const Stream &stream) const;
+
+  void EnableStreamData(const Stream &stream);
+  void DisableStreamData(const Stream &stream);
+  bool IsStreamDataEnabled(const Stream &stream) const;
+
+  void SetStreamCallback(const Stream &stream, stream_callback_t callback);
+  bool HasStreamCallback(const Stream &stream) const;
+
+  void StartVideoStreaming();
+  void StopVideoStreaming();
+
+  void WaitForStreams();
+
+  api::StreamData GetStreamData(const Stream &stream);
+  std::vector<api::StreamData> GetStreamDatas(const Stream &stream);
+
+  void SetPlugin(std::shared_ptr<Plugin> plugin);
+  bool HasPlugin() const;
+
+ private:
+  void InitStreamSupports();
+
+  mode_t GetStreamEnabledMode(const Stream &stream) const;
+  bool IsStreamEnabledNative(const Stream &stream) const;
+  bool IsStreamEnabledSynthetic(const Stream &stream) const;
+
+  void EnableStreamData(const Stream &stream, std::uint32_t depth);
+  void DisableStreamData(const Stream &stream, std::uint32_t depth);
+
+  void InitProcessors();
+
+  template <class T>
+  bool ActivateProcessor(bool tree = false);
+  template <class T>
+  bool DeactivateProcessor(bool tree = false);
+
+  void ProcessNativeStream(const Stream &stream, const api::StreamData &data);
+
+  bool OnRectifyProcess(
+      Object *const in, Object *const out, Processor *const parent);
+  bool OnDisparityProcess(
+      Object *const in, Object *const out, Processor *const parent);
+  bool OnDisparityNormalizedProcess(
+      Object *const in, Object *const out, Processor *const parent);
+  bool OnPointsProcess(
+      Object *const in, Object *const out, Processor *const parent);
+  bool OnDepthProcess(
+      Object *const in, Object *const out, Processor *const parent);
+
+  void OnRectifyPostProcess(Object *const out);
+  void OnDisparityPostProcess(Object *const out);
+  void OnDisparityNormalizedPostProcess(Object *const out);
+  void OnPointsPostProcess(Object *const out);
+  void OnDepthPostProcess(Object *const out);
+
+  API *api_;
+
+  std::map<Stream, mode_t> stream_supports_mode_;
+  std::map<Stream, mode_t> stream_enabled_mode_;
+
+  std::map<Stream, stream_callback_t> stream_callbacks_;
+
+  std::shared_ptr<Processor> processor_;
+
+  std::shared_ptr<Plugin> plugin_;
+};
+
+template <class T, class P>
+std::shared_ptr<T> find_processor(const P &processor) {
+  return find_processor<T>(processor, T::NAME);
+}
+
+template <class T, class P>
+std::shared_ptr<T> find_processor(const P &processor, const std::string &name) {
+  if (processor->Name() == name) {
+    return std::dynamic_pointer_cast<T>(processor);
+  }
+  auto &&childs = processor->GetChilds();
+  return find_processor<T>(std::begin(childs), std::end(childs), name);
+}
+
+template <class T, class InputIt>
+std::shared_ptr<T> find_processor(
+    InputIt first, InputIt last, const std::string &name) {
+  if (first == last)
+    return nullptr;
+  for (auto it = first; it != last; ++it) {
+    if ((*it)->Name() == name) {
+      return std::dynamic_pointer_cast<T>(*it);
+    }
+  }
+  for (auto it = first; it != last; ++it) {
+    auto &&childs = (*it)->GetChilds();
+    if (childs.empty())
+      continue;
+    auto &&result =
+        find_processor<T>(std::begin(childs), std::end(childs), name);
+    if (result == nullptr)
+      continue;
+    return result;
+  }
+  return nullptr;
+}
+
+template <class T>
+bool Synthetic::ActivateProcessor(bool parents) {
+  auto &&processor = find_processor<T>(processor_);
+  if (processor == nullptr)
+    return false;
+  processor->Activate(parents);
+  return true;
+}
+
+template <class T>
+bool Synthetic::DeactivateProcessor(bool childs) {
+  auto &&processor = find_processor<T>(processor_);
+  if (processor == nullptr)
+    return false;
+  processor->Deactivate(childs);
+  return true;
+}
+
+MYNTEYE_END_NAMESPACE
+
+#endif  // MYNTEYE_API_SYNTHETIC_H_