Update doc version

.gitignore

@@ -1,5 +1,7 @@
 .DS_Store
-/.vscode/
+
+.vs/
+.vscode/
 
 _build/
 _install/
@@ -29,3 +31,8 @@ _output/
 /mynteye/
 /mynteye.bag
 /dataset*/
+
+# wrappers
+
+/wrappers/python/third_party/numpy-opencv-converter/
+/wrappers/python/third_party/pyboostcvconverter/

CMakeLists.txt

@@ -14,7 +14,7 @@
 
 cmake_minimum_required(VERSION 3.0)
 
-project(mynteye VERSION 2.0.0 LANGUAGES C CXX)
+project(mynteye VERSION 2.0.1 LANGUAGES C CXX)
 
 include(cmake/Common.cmake)
 
@@ -48,10 +48,6 @@ LIST(APPEND CMAKE_PREFIX_PATH third_party/glog/_build)
 find_package(glog REQUIRED)
 message(STATUS "Found glog: ${glog_VERSION}")
 
-if(WITH_API)
-  include(cmake/DetectOpenCV.cmake)
-endif()
-
 LIST(APPEND CMAKE_MODULE_PATH cmake)
 
 include(CMakePackageConfigHelpers)
@@ -63,6 +59,14 @@ set(MYNTEYE_NAME ${PROJECT_NAME})
 set(MYNTEYE_NAMESPACE "mynteye")
 message(STATUS "Namespace: ${MYNTEYE_NAMESPACE}")
 
+if(MSVC)
+  string(REPLACE "/" "\\\\" MYNTEYE_SDK_ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+  string(REPLACE "/" "\\\\" MYNTEYE_SDK_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}")
+else()
+  file(TO_NATIVE_PATH "${CMAKE_CURRENT_SOURCE_DIR}" MYNTEYE_SDK_ROOT_DIR)
+  file(TO_NATIVE_PATH "${CMAKE_INSTALL_PREFIX}" MYNTEYE_SDK_INSTALL_DIR)
+endif()
+
 configure_file(
   include/mynteye/mynteye.h.in
   include/mynteye/mynteye.h @ONLY
@@ -167,6 +171,9 @@ set(MYNTEYE_LINKLIBS
 if(WITH_API)
   list(APPEND MYNTEYE_LINKLIBS ${OpenCV_LIBS})
 endif()
+if(WITH_BOOST_FILESYSTEM)
+  list(APPEND MYNTEYE_LINKLIBS ${Boost_LIBRARIES})
+endif()
 #message(STATUS "MYNTEYE_LINKLIBS: ${MYNTEYE_LINKLIBS}")
 
 add_library(${MYNTEYE_NAME} SHARED ${MYNTEYE_SRCS})
@@ -227,3 +234,31 @@ install(FILES
 install(EXPORT ${MYNTEYE_NAME}-targets
   DESTINATION ${MYNTEYE_CMAKE_INSTALLDIR}
 )
+
+## build.info
+
+# set default int values for yaml file (build.info)
+set_default_value(CMAKE_CXX_COMPILER_VERSION_MAJOR 0)
+set_default_value(CMAKE_CXX_COMPILER_VERSION_MINOR 0)
+set_default_value(CMAKE_CXX_COMPILER_VERSION_PATCH 0)
+set_default_value(CMAKE_CXX_COMPILER_VERSION_TWEAK 0)
+set_default_value(CUDA_VERSION_MAJOR 0)
+set_default_value(CUDA_VERSION_MINOR 0)
+set_default_value(OpenCV_VERSION_MAJOR 0)
+set_default_value(OpenCV_VERSION_MINOR 0)
+set_default_value(OpenCV_VERSION_PATCH 0)
+set_default_value(OpenCV_VERSION_TWEAK 0)
+set_default_value(${PROJECT_NAME}_VERSION_MAJOR 0)
+set_default_value(${PROJECT_NAME}_VERSION_MINOR 0)
+set_default_value(${PROJECT_NAME}_VERSION_PATCH 0)
+set_default_value(${PROJECT_NAME}_VERSION_TWEAK 0)
+
+configure_file(
+  cmake/templates/build.info.in
+  build.info @ONLY
+)
+
+install(FILES
+  ${CMAKE_CURRENT_BINARY_DIR}/build.info
+  DESTINATION ${CMAKE_INSTALL_PREFIX}/share/mynteye
+)

CommonDefs.mk

@@ -163,7 +163,6 @@ ifneq ($(MAKE),)
 endif
 
 CMAKE_OPTIONS :=
-CMAKE_OPTIONS += -DCMAKE_INSTALL_PREFIX=$(shell pwd)/_install
 # CMAKE_OPTIONS += -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON
 CMAKE_OPTIONS_AFTER :=
 
@@ -172,23 +171,32 @@ ifeq ($(HOST_OS),Win)
 ifeq ($(HOST_NAME),MinGW)
   CMAKE += -G "MinGW Makefiles"
 else ifeq ($(HOST_ARCH),x64)
-  VS_VERSION := $(shell echo "$(shell which cl)" | sed "s/.*Visual\sStudio\s\([0-9]\+\).*/\1/g")
-  ifeq (15,$(VS_VERSION))
+  ifeq ($(VS_CODE),)
+    WHICH_CL := $(shell which cl)
+    ifeq ($(WHICH_CL),)
+      $(error "Visual Studio version is unknown. Could set VS_CODE to specify it, e.g. make [TARGET] VS_CODE=2017")
+    endif
+    # C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\...
+    # C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\...
+    VS_CODE := $(shell echo "$(WHICH_CL)" | grep -Po "(?<=Visual Studio[ /])[0-9]+")
+  endif
+  # $(call mkinfo,"VS_CODE: $(VS_CODE)")
+  ifeq ($(filter $(VS_CODE),15 2017),$(VS_CODE))
     CMAKE += -G "Visual Studio 15 2017 Win64"
-  else ifeq (14,$(VS_VERSION))
+  else ifeq ($(filter $(VS_CODE),14 2015),$(VS_CODE))
     CMAKE += -G "Visual Studio 14 2015 Win64"
-  else ifeq (12,$(VS_VERSION))
+  else ifeq ($(filter $(VS_CODE),12 2013),$(VS_CODE))
     CMAKE += -G "Visual Studio 12 2013 Win64"
-  else ifeq (11,$(VS_VERSION))
+  else ifeq ($(filter $(VS_CODE),11 2012),$(VS_CODE))
     CMAKE += -G "Visual Studio 11 2012 Win64"
-  else ifeq (10,$(VS_VERSION))
+  else ifeq ($(filter $(VS_CODE),10 2010),$(VS_CODE))
     CMAKE += -G "Visual Studio 10 2010 Win64"
-  else ifeq (9,$(VS_VERSION))
+  else ifeq ($(filter $(VS_CODE),9 2008),$(VS_CODE))
     CMAKE += -G "Visual Studio 9 2008 Win64"
-  else ifeq (8,$(VS_VERSION))
+  else ifeq ($(filter $(VS_CODE),8 2005),$(VS_CODE))
     CMAKE += -G "Visual Studio 8 2005 Win64"
   else
-    $(call mkinfo,"Connot specify Visual Studio Win64")
+    $(error "Visual Studio version is not proper, VS_CODE: $(VS_CODE)")
   endif
 endif
 

Makefile

@@ -13,6 +13,15 @@
 # limitations under the License.
 include CommonDefs.mk
 
+MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
+MKFILE_DIR := $(patsubst %/,%,$(dir $(MKFILE_PATH)))
+
+# CMAKE_INSTALL_PREFIX:
+#   https://cmake.org/cmake/help/latest/variable/CMAKE_INSTALL_PREFIX.html
+#
+#   UNIX: /usr/local
+#   Windows: c:/Program Files/${PROJECT_NAME}
+
 .DEFAULT_GOAL := help
 
 help:
@@ -27,6 +36,7 @@ help:
 	@echo "  make samples         build samples"
 	@echo "  make tools           build tools"
 	@echo "  make ros             build ros wrapper"
+	@echo "  make py              build python wrapper"
 	@echo "  make clean|cleanall  clean generated or useless things"
 
 .PHONY: help
@@ -50,7 +60,10 @@ opendoc: apidoc
 		[ -f "$$html" ] && $(SH) ./scripts/open.sh $$html; \
 	done
 
-.PHONY: apidoc opendoc
+cleandoc:
+	@$(call rm,./doc/_output/)
+
+.PHONY: apidoc opendoc cleandoc
 
 # deps
 
@@ -76,7 +89,8 @@ init: submodules
 
 build: third_party
 	@$(call echo,Make $@)
-	@$(call cmake_build,./_build)
+	@$(call cmake_build,./_build,..)
+	@# @$(call cmake_build,./_build,..,-DCMAKE_INSTALL_PREFIX=$(MKFILE_DIR)/_install)
 
 .PHONY: build
 
@@ -108,10 +122,10 @@ ifeq ($(HOST_OS),Win)
 ifneq ($(HOST_NAME),MinGW)
 	@cd ./_build; msbuild.exe INSTALL.vcxproj /property:Configuration=Release
 else
-	@cd ./_build; make install
+	@cd ./_build; sudo make install
 endif
 else
-	@cd ./_build; make install
+	@cd ./_build; sudo make install
 endif
 
 .PHONY: install
@@ -158,6 +172,44 @@ cleanros:
 
 .PHONY: cleanros
 
+# python
+
+PBCVT_DIR := wrappers/python/third_party/pyboostcvconverter
+
+$(PBCVT_DIR):
+	@git clone https://github.com/Algomorph/pyboostcvconverter.git $@
+
+pbcvt: $(PBCVT_DIR)
+	@$(call cmake_build,$(PBCVT_DIR)/_build,.., \
+		-DCMAKE_INSTALL_PREFIX=$(MKFILE_DIR)/wrappers/python/_install \
+		-DPYTHON_DESIRED_VERSION=2.X)
+	@cd $(PBCVT_DIR)/_build; make install
+
+.PHONY: pbcvt
+
+NPCV_DIR := wrappers/python/third_party/numpy-opencv-converter
+
+$(NPCV_DIR):
+	@git clone https://github.com/GarrickLin/numpy-opencv-converter.git $@
+
+py: python
+
+python: install $(NPCV_DIR)
+	@$(call echo,Make $@)
+	@$(call cmake_build,./wrappers/python/_build)
+	@cd ./wrappers/python/_build; make install
+
+.PHONY: py python
+
+cleanpy:
+	@$(call echo,Make $@)
+	@$(call rm,./wrappers/python/_build/)
+	@$(call rm,./wrappers/python/_output/)
+	@$(call rm,./wrappers/python/_install/)
+	@$(call rm,./$(PBCVT_DIR)/_build/)
+
+.PHONY: cleanpy
+
 # clean
 
 clean:
@@ -175,6 +227,7 @@ clean:
 ifeq ($(HOST_OS),Linux)
 	@$(MAKE) cleanros
 endif
+	@$(MAKE) cleanpy
 
 cleanlog:
 	@$(call rm_f,*INFO*)
@@ -182,11 +235,12 @@ cleanlog:
 	@$(call rm_f,*ERROR*)
 	@$(call rm_f,*FATAL*)
 
-cleanall: clean
-	@$(call rm,./doc/_output/)
+cleanall: clean cleandoc
 	@$(call rm,./test/gtest/_build/)
 	@$(call rm,./third_party/glog/_build/)
 	@$(FIND) . -type f -name ".DS_Store" -print0 | xargs -0 rm -f
+	@$(call rm,./$(PBCVT_DIR)/)
+	@$(call rm,./$(NPCV_DIR)/)
 
 .PHONY: clean cleanlog cleanall
 
@@ -194,6 +248,8 @@ cleanall: clean
 
 host:
 	@$(call echo,Make $@)
+	@echo MKFILE_PATH: $(MKFILE_PATH)
+	@echo MKFILE_DIR: $(MKFILE_DIR)
 	@echo HOST_OS: $(HOST_OS)
 	@echo HOST_ARCH: $(HOST_ARCH)
 	@echo HOST_NAME: $(HOST_NAME)

README.md

@@ -1,11 +1,40 @@
 # MYNT® EYE SDK
 
+[![](https://img.shields.io/badge/MYNT%20EYE%20SDK-2.0.1--rc0-brightgreen.svg?style=flat)](https://github.com/slightech/MYNT-EYE-SDK-2)
+
 ## Overview
 
 MYNT® EYE SDK 2.0 is a cross-platform library for MYNT® EYE cameras.
 
+The following platforms have been tested:
+
+* Windows 10
+* Ubuntu 16.04 / 14.04
+* Jetson TX2
+
+Please follow the guide doc to install the SDK on different platforms.
+
+## Documentations
+
+* [API Doc](https://github.com/slightech/MYNT-EYE-SDK-2/releases): API reference, some guides and data spec.
+  * en: [![](https://img.shields.io/badge/Download-PDF-blue.svg?style=flat)](https://github.com/slightech/MYNT-EYE-SDK-2/files/2203869/mynt-eye-sdk-apidoc-2.0.1-rc0-en.pdf) [![](https://img.shields.io/badge/Download-HTML-blue.svg?style=flat)](https://github.com/slightech/MYNT-EYE-SDK-2/files/2203870/mynt-eye-sdk-apidoc-2.0.1-rc0-html-en.zip) [![](https://img.shields.io/badge/Online-HTML-blue.svg?style=flat)](https://slightech.github.io/MYNT-EYE-SDK-2/)
+  * zh-Hans: [![](https://img.shields.io/badge/Download-PDF-blue.svg?style=flat)](https://github.com/slightech/MYNT-EYE-SDK-2/files/2203872/mynt-eye-sdk-apidoc-2.0.1-rc0-zh-Hans.pdf) [![](https://img.shields.io/badge/Download-HTML-blue.svg?style=flat)](https://github.com/slightech/MYNT-EYE-SDK-2/files/2203874/mynt-eye-sdk-apidoc-2.0.1-rc0-html-zh-Hans.zip) [![](https://img.shields.io/badge/Online-HTML-blue.svg?style=flat)](http://doc.myntai.com/resource/api/mynt-eye-sdk-apidoc-2.0.1-rc0-html-zh-Hans/mynt-eye-sdk-apidoc-2.0.1-rc0-html-zh-Hans/index.html)
+* [Guide Doc](https://github.com/slightech/MYNT-EYE-SDK-2-Guide/releases): How to install and start using the SDK.
+  * en: [![](https://img.shields.io/badge/Download-PDF-blue.svg?style=flat)](https://github.com/slightech/MYNT-EYE-SDK-2-Guide/files/2203877/mynt-eye-sdk-guide-2.0.1-rc0-en.pdf) [![](https://img.shields.io/badge/Download-HTML-blue.svg?style=flat)](https://github.com/slightech/MYNT-EYE-SDK-2-Guide/files/2203879/mynt-eye-sdk-guide-2.0.1-rc0-html-en.zip) [![](https://img.shields.io/badge/Online-HTML-blue.svg?style=flat)](https://slightech.github.io/MYNT-EYE-SDK-2-Guide/)
+  * zh-Hans: [![](https://img.shields.io/badge/Download-PDF-blue.svg?style=flat)](https://github.com/slightech/MYNT-EYE-SDK-2-Guide/files/2203880/mynt-eye-sdk-guide-2.0.1-rc0-zh-Hans.pdf) [![](https://img.shields.io/badge/Download-HTML-blue.svg?style=flat)](https://github.com/slightech/MYNT-EYE-SDK-2-Guide/files/2203881/mynt-eye-sdk-guide-2.0.1-rc0-html-zh-Hans.zip) [![](https://img.shields.io/badge/Online-HTML-blue.svg?style=flat)](http://doc.myntai.com/resource/sdk/mynt-eye-sdk-guide-2.0.1-rc0-html-zh-Hans/mynt-eye-sdk-guide-2.0.1-rc0-html-zh-Hans/index.html	)
+
+> Supported languages: `en`, `zh-Hans`.
+
+## Firmwares
+
+[MYNTEYE_BOX]: http://doc.myntai.com/mynteye/s/download
+
+Get firmwares from our online disks: [MYNTEYE_BOX][]. The latest version is `2.0.1`.
+
 ## Usage
 
+In short,
+
 ```bash
 $ make
 Usage:
@@ -30,18 +59,6 @@ make samples
 ./samples/_output/bin/device/camera_d
 ```
 
-## Firmwares
-
-[Google Drive]: https://drive.google.com/drive/folders/1tdFCcTBMNcImEGZ39tdOZmlX2SHKCr2f
-[百度网盘]: https://pan.baidu.com/s/1yPQDp2r0x4jvNwn2UjlMUQ
-
-Get firmwares from our online disks: [Google Drive][], [百度网盘][].
-
-## Documentations
-
-* [API Doc](https://github.com/slightech/MYNT-EYE-SDK-2/releases)
-* [Guide Doc](https://github.com/slightech/MYNT-EYE-SDK-2-Guide/releases)
-
 ## Mirrors
 
 国内镜像：[码云](https://gitee.com/mynt/MYNT-EYE-SDK-2)。

cmake/Common.cmake

@@ -21,14 +21,40 @@ set(CUR_DIR ${CMAKE_CURRENT_LIST_DIR})
 
 if(MSVC OR MSYS OR MINGW)
   set(OS_WIN TRUE)
+  set(HOST_OS Win)
 elseif(APPLE)
   set(OS_MAC TRUE)
+  set(HOST_OS Mac)
 elseif(UNIX)
   set(OS_LINUX TRUE)
+  set(HOST_OS Linux)
 else()
   message(FATAL_ERROR "Unsupported OS.")
 endif()
 
+set(HOST_NAME "${HOST_OS}")
+
+if(OS_LINUX)
+  execute_process(COMMAND uname -a COMMAND tr -d '\n' OUTPUT_VARIABLE UNAME_A)
+  string(TOLOWER "${UNAME_A}" UNAME_A)
+  if(${UNAME_A} MATCHES ".*(tegra|jetsonbot).*")
+    set(OS_TEGRA TRUE)
+    set(HOST_NAME Tegra)
+  elseif(${UNAME_A} MATCHES ".*ubuntu.*")
+    set(OS_UBUNTU TRUE)
+    set(HOST_NAME Ubuntu)
+  endif()
+endif()
+
+include(${CMAKE_CURRENT_LIST_DIR}/TargetArch.cmake)
+target_architecture(HOST_ARCH)
+message(STATUS "HOST_ARCH: ${HOST_ARCH}")
+
+# CMAKE_CXX_COMPILER_ID
+#   https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_COMPILER_ID.html
+# CMAKE_CXX_COMPILER_VERSION
+#   https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_COMPILER_VERSION.html
+
 # set_outdir(ARCHIVE_OUTPUT_DIRECTORY
 #            LIBRARY_OUTPUT_DIRECTORY
 #            RUNTIME_OUTPUT_DIRECTORY)
@@ -44,17 +70,27 @@ macro(set_outdir ARCHIVE_OUTPUT_DIRECTORY LIBRARY_OUTPUT_DIRECTORY RUNTIME_OUTPU
   endforeach()
 endmacro()
 
+set(__exe2bat_relative_path false)
+
 macro(exe2bat exe_name exe_dir dll_search_paths)
   message(STATUS "Generating ${exe_name}.bat")
   set(__exe_name ${exe_name})
-  set(__dll_relative_search_paths "")
-  foreach(path ${dll_search_paths})
-    file(RELATIVE_PATH __relative_path "${exe_dir}" "${path}")
-    file(TO_NATIVE_PATH ${__relative_path} __relative_path)
-    list(APPEND __dll_relative_search_paths ${__relative_path})
-  endforeach()
-  #message(STATUS __dll_relative_search_paths: "${__dll_relative_search_paths}")
-  set(__dll_search_paths "${__dll_relative_search_paths}")
+  if(__exe2bat_relative_path)
+    set(__dll_relative_search_paths "")
+    foreach(path ${dll_search_paths})
+      file(RELATIVE_PATH __relative_path "${exe_dir}" "${path}")
+      file(TO_NATIVE_PATH ${__relative_path} __relative_path)
+      list(APPEND __dll_relative_search_paths ${__relative_path})
+    endforeach()
+    set(__dll_search_paths "${__dll_relative_search_paths}")
+  else()
+    set(__dll_native_search_paths "")
+    foreach(path ${dll_search_paths})
+      file(TO_NATIVE_PATH ${path} __native_path)
+      list(APPEND __dll_native_search_paths ${__native_path})
+    endforeach()
+    set(__dll_search_paths "${__dll_native_search_paths}")
+  endif()
   configure_file(
     "${CUR_DIR}/templates/exe.bat.in"
     "${exe_dir}/${__exe_name}.bat"

cmake/DetectOpenCV.cmake

@@ -19,8 +19,15 @@ find_package(OpenCV REQUIRED)
 message(STATUS "Found OpenCV: ${OpenCV_VERSION}")
 if(OpenCV_VERSION VERSION_LESS 3.0)
   add_definitions(-DUSE_OPENCV2)
-else()
+elseif(OpenCV_VERSION VERSION_LESS 4.0)
   add_definitions(-DUSE_OPENCV3)
+else()
+  add_definitions(-DUSE_OPENCV4)
+endif()
+
+list(FIND OpenCV_LIBS "opencv_world" __index)
+if(${__index} GREATER -1)
+  set(WITH_OPENCV_WORLD TRUE)
 endif()
 
 if(MSVC OR MSYS OR MINGW)

cmake/FindCUDA/make2cmake.cmake

@@ -0,0 +1,92 @@
+#  James Bigler, NVIDIA Corp (nvidia.com - jbigler)
+#  Abe Stephens, SCI Institute -- http://www.sci.utah.edu/~abe/FindCuda.html
+#
+#  Copyright (c) 2008 - 2009 NVIDIA Corporation.  All rights reserved.
+#
+#  Copyright (c) 2007-2009
+#  Scientific Computing and Imaging Institute, University of Utah
+#
+#  This code is licensed under the MIT License.  See the FindCUDA.cmake script
+#  for the text of the license.
+
+# The MIT License
+#
+# License for the specific language governing rights and limitations under
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+#
+
+#######################################################################
+# This converts a file written in makefile syntax into one that can be included
+# by CMake.
+
+file(READ ${input_file} depend_text)
+
+if (NOT "${depend_text}" STREQUAL "")
+
+  # message("FOUND DEPENDS")
+
+  string(REPLACE "\\ " " " depend_text ${depend_text})
+
+  # This works for the nvcc -M generated dependency files.
+  string(REGEX REPLACE "^.* : " "" depend_text ${depend_text})
+  string(REGEX REPLACE "[ \\\\]*\n" ";" depend_text ${depend_text})
+
+  set(dependency_list "")
+
+  foreach(file ${depend_text})
+
+    string(REGEX REPLACE "^ +" "" file ${file})
+
+    # OK, now if we had a UNC path, nvcc has a tendency to only output the first '/'
+    # instead of '//'.  Here we will test to see if the file exists, if it doesn't then
+    # try to prepend another '/' to the path and test again.  If it still fails remove the
+    # path.
+
+    if(NOT EXISTS "${file}")
+      if (EXISTS "/${file}")
+        set(file "/${file}")
+      else()
+        message(WARNING " Removing non-existent dependency file: ${file}")
+        set(file "")
+      endif()
+    endif()
+
+    if(NOT IS_DIRECTORY "${file}")
+      # If softlinks start to matter, we should change this to REALPATH.  For now we need
+      # to flatten paths, because nvcc can generate stuff like /bin/../include instead of
+      # just /include.
+      get_filename_component(file_absolute "${file}" ABSOLUTE)
+      list(APPEND dependency_list "${file_absolute}")
+    endif()
+
+  endforeach()
+
+else()
+  # message("FOUND NO DEPENDS")
+endif()
+
+# Remove the duplicate entries and sort them.
+list(REMOVE_DUPLICATES dependency_list)
+list(SORT dependency_list)
+
+foreach(file ${dependency_list})
+  set(cuda_nvcc_depend "${cuda_nvcc_depend} \"${file}\"\n")
+endforeach()
+
+file(WRITE ${output_file} "# Generated by: make2cmake.cmake\nSET(CUDA_NVCC_DEPEND\n ${cuda_nvcc_depend})\n\n")

cmake/FindCUDA/parse_cubin.cmake

@@ -0,0 +1,109 @@
+#  James Bigler, NVIDIA Corp (nvidia.com - jbigler)
+#  Abe Stephens, SCI Institute -- http://www.sci.utah.edu/~abe/FindCuda.html
+#
+#  Copyright (c) 2008 - 2009 NVIDIA Corporation.  All rights reserved.
+#
+#  Copyright (c) 2007-2009
+#  Scientific Computing and Imaging Institute, University of Utah
+#
+#  This code is licensed under the MIT License.  See the FindCUDA.cmake script
+#  for the text of the license.
+
+# The MIT License
+#
+# License for the specific language governing rights and limitations under
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+#
+
+#######################################################################
+# Parses a .cubin file produced by nvcc and reports statistics about the file.
+
+
+file(READ ${input_file} file_text)
+
+if (NOT "${file_text}" STREQUAL "")
+
+  string(REPLACE ";" "\\;" file_text ${file_text})
+  string(REPLACE "\ncode" ";code" file_text ${file_text})
+
+  list(LENGTH file_text len)
+
+  foreach(line ${file_text})
+
+    # Only look at "code { }" blocks.
+    if(line MATCHES "^code")
+
+      # Break into individual lines.
+      string(REGEX REPLACE "\n" ";" line ${line})
+
+      foreach(entry ${line})
+
+        # Extract kernel names.
+        if (${entry} MATCHES "[^g]name = ([^ ]+)")
+          set(entry "${CMAKE_MATCH_1}")
+
+          # Check to see if the kernel name starts with "_"
+          set(skip FALSE)
+          # if (${entry} MATCHES "^_")
+            # Skip the rest of this block.
+            # message("Skipping ${entry}")
+            # set(skip TRUE)
+          # else ()
+            message("Kernel:    ${entry}")
+          # endif ()
+
+        endif()
+
+        # Skip the rest of the block if necessary
+        if(NOT skip)
+
+          # Registers
+          if (${entry} MATCHES "reg([ ]+)=([ ]+)([^ ]+)")
+            set(entry "${CMAKE_MATCH_3}")
+            message("Registers: ${entry}")
+          endif()
+
+          # Local memory
+          if (${entry} MATCHES "lmem([ ]+)=([ ]+)([^ ]+)")
+            set(entry "${CMAKE_MATCH_3}")
+            message("Local:     ${entry}")
+          endif()
+
+          # Shared memory
+          if (${entry} MATCHES "smem([ ]+)=([ ]+)([^ ]+)")
+            set(entry "${CMAKE_MATCH_3}")
+            message("Shared:    ${entry}")
+          endif()
+
+          if (${entry} MATCHES "^}")
+            message("")
+          endif()
+
+        endif()
+
+
+      endforeach()
+
+    endif()
+
+  endforeach()
+
+else()
+  # message("FOUND NO DEPENDS")
+endif()

cmake/FindCUDA/run_nvcc.cmake

@@ -0,0 +1,288 @@
+#  James Bigler, NVIDIA Corp (nvidia.com - jbigler)
+#
+#  Copyright (c) 2008 - 2009 NVIDIA Corporation.  All rights reserved.
+#
+#  This code is licensed under the MIT License.  See the FindCUDA.cmake script
+#  for the text of the license.
+
+# The MIT License
+#
+# License for the specific language governing rights and limitations under
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+
+##########################################################################
+# This file runs the nvcc commands to produce the desired output file along with
+# the dependency file needed by CMake to compute dependencies.  In addition the
+# file checks the output of each command and if the command fails it deletes the
+# output files.
+
+# Input variables
+#
+# verbose:BOOL=<>          OFF: Be as quiet as possible (default)
+#                          ON : Describe each step
+#
+# build_configuration:STRING=<> Typically one of Debug, MinSizeRel, Release, or
+#                               RelWithDebInfo, but it should match one of the
+#                               entries in CUDA_HOST_FLAGS. This is the build
+#                               configuration used when compiling the code.  If
+#                               blank or unspecified Debug is assumed as this is
+#                               what CMake does.
+#
+# generated_file:STRING=<> File to generate.  This argument must be passed in.
+#
+# generated_cubin_file:STRING=<> File to generate.  This argument must be passed
+#                                                   in if build_cubin is true.
+
+if(NOT generated_file)
+  message(FATAL_ERROR "You must specify generated_file on the command line")
+endif()
+
+# Set these up as variables to make reading the generated file easier
+set(CMAKE_COMMAND "@CMAKE_COMMAND@") # path
+set(source_file "@source_file@") # path
+set(NVCC_generated_dependency_file "@NVCC_generated_dependency_file@") # path
+set(cmake_dependency_file "@cmake_dependency_file@") # path
+set(CUDA_make2cmake "@CUDA_make2cmake@") # path
+set(CUDA_parse_cubin "@CUDA_parse_cubin@") # path
+set(build_cubin @build_cubin@) # bool
+set(CUDA_HOST_COMPILER "@CUDA_HOST_COMPILER@") # path
+# We won't actually use these variables for now, but we need to set this, in
+# order to force this file to be run again if it changes.
+set(generated_file_path "@generated_file_path@") # path
+set(generated_file_internal "@generated_file@") # path
+set(generated_cubin_file_internal "@generated_cubin_file@") # path
+
+set(CUDA_NVCC_EXECUTABLE "@CUDA_NVCC_EXECUTABLE@") # path
+set(CUDA_NVCC_FLAGS @CUDA_NVCC_FLAGS@ ;; @CUDA_WRAP_OPTION_NVCC_FLAGS@) # list
+@CUDA_NVCC_FLAGS_CONFIG@
+set(nvcc_flags "@nvcc_flags@") # list
+set(CUDA_NVCC_INCLUDE_ARGS "@CUDA_NVCC_INCLUDE_ARGS@") # list (needs to be in quotes to handle spaces properly).
+set(format_flag "@format_flag@") # string
+
+if(build_cubin AND NOT generated_cubin_file)
+  message(FATAL_ERROR "You must specify generated_cubin_file on the command line")
+endif()
+
+# This is the list of host compilation flags.  It C or CXX should already have
+# been chosen by FindCUDA.cmake.
+@CUDA_HOST_FLAGS@
+
+# Take the compiler flags and package them up to be sent to the compiler via -Xcompiler
+set(nvcc_host_compiler_flags "")
+# If we weren't given a build_configuration, use Debug.
+if(NOT build_configuration)
+  set(build_configuration Debug)
+endif()
+string(TOUPPER "${build_configuration}" build_configuration)
+#message("CUDA_NVCC_HOST_COMPILER_FLAGS = ${CUDA_NVCC_HOST_COMPILER_FLAGS}")
+foreach(flag ${CMAKE_HOST_FLAGS} ${CMAKE_HOST_FLAGS_${build_configuration}})
+  # Extra quotes are added around each flag to help nvcc parse out flags with spaces.
+  set(nvcc_host_compiler_flags "${nvcc_host_compiler_flags},\"${flag}\"")
+endforeach()
+if (nvcc_host_compiler_flags)
+  set(nvcc_host_compiler_flags "-Xcompiler" ${nvcc_host_compiler_flags})
+endif()
+#message("nvcc_host_compiler_flags = \"${nvcc_host_compiler_flags}\"")
+# Add the build specific configuration flags
+list(APPEND CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS_${build_configuration}})
+
+# Any -ccbin existing in CUDA_NVCC_FLAGS gets highest priority
+list( FIND CUDA_NVCC_FLAGS "-ccbin" ccbin_found0 )
+list( FIND CUDA_NVCC_FLAGS "--compiler-bindir" ccbin_found1 )
+if( ccbin_found0 LESS 0 AND ccbin_found1 LESS 0 AND CUDA_HOST_COMPILER )
+  if (CUDA_HOST_COMPILER STREQUAL "@_CUDA_MSVC_HOST_COMPILER@" AND DEFINED CCBIN)
+    set(CCBIN -ccbin "${CCBIN}")
+  else()
+    set(CCBIN -ccbin "${CUDA_HOST_COMPILER}")
+  endif()
+endif()
+
+# cuda_execute_process - Executes a command with optional command echo and status message.
+#
+#   status  - Status message to print if verbose is true
+#   command - COMMAND argument from the usual execute_process argument structure
+#   ARGN    - Remaining arguments are the command with arguments
+#
+#   CUDA_result - return value from running the command
+#
+# Make this a macro instead of a function, so that things like RESULT_VARIABLE
+# and other return variables are present after executing the process.
+macro(cuda_execute_process status command)
+  set(_command ${command})
+  if(NOT "x${_command}" STREQUAL "xCOMMAND")
+    message(FATAL_ERROR "Malformed call to cuda_execute_process.  Missing COMMAND as second argument. (command = ${command})")
+  endif()
+  if(verbose)
+    execute_process(COMMAND "${CMAKE_COMMAND}" -E echo -- ${status})
+    # Now we need to build up our command string.  We are accounting for quotes
+    # and spaces, anything else is left up to the user to fix if they want to
+    # copy and paste a runnable command line.
+    set(cuda_execute_process_string)
+    foreach(arg ${ARGN})
+      # If there are quotes, excape them, so they come through.
+      string(REPLACE "\"" "\\\"" arg ${arg})
+      # Args with spaces need quotes around them to get them to be parsed as a single argument.
+      if(arg MATCHES " ")
+        list(APPEND cuda_execute_process_string "\"${arg}\"")
+      else()
+        list(APPEND cuda_execute_process_string ${arg})
+      endif()
+    endforeach()
+    # Echo the command
+    execute_process(COMMAND ${CMAKE_COMMAND} -E echo ${cuda_execute_process_string})
+  endif()
+  # Run the command
+  execute_process(COMMAND ${ARGN} RESULT_VARIABLE CUDA_result )
+endmacro()
+
+# Delete the target file
+cuda_execute_process(
+  "Removing ${generated_file}"
+  COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
+  )
+
+# For CUDA 2.3 and below, -G -M doesn't work, so remove the -G flag
+# for dependency generation and hope for the best.
+set(depends_CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS}")
+set(CUDA_VERSION @CUDA_VERSION@)
+if(CUDA_VERSION VERSION_LESS "3.0")
+  cmake_policy(PUSH)
+  # CMake policy 0007 NEW states that empty list elements are not
+  # ignored.  I'm just setting it to avoid the warning that's printed.
+  cmake_policy(SET CMP0007 NEW)
+  # Note that this will remove all occurrences of -G.
+  list(REMOVE_ITEM depends_CUDA_NVCC_FLAGS "-G")
+  cmake_policy(POP)
+endif()
+
+# nvcc doesn't define __CUDACC__ for some reason when generating dependency files.  This
+# can cause incorrect dependencies when #including files based on this macro which is
+# defined in the generating passes of nvcc invocation.  We will go ahead and manually
+# define this for now until a future version fixes this bug.
+set(CUDACC_DEFINE -D__CUDACC__)
+
+# Generate the dependency file
+cuda_execute_process(
+  "Generating dependency file: ${NVCC_generated_dependency_file}"
+  COMMAND "${CUDA_NVCC_EXECUTABLE}"
+  -M
+  ${CUDACC_DEFINE}
+  "${source_file}"
+  -o "${NVCC_generated_dependency_file}"
+  ${CCBIN}
+  ${nvcc_flags}
+  ${nvcc_host_compiler_flags}
+  ${depends_CUDA_NVCC_FLAGS}
+  -DNVCC
+  ${CUDA_NVCC_INCLUDE_ARGS}
+  )
+
+if(CUDA_result)
+  message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Generate the cmake readable dependency file to a temp file.  Don't put the
+# quotes just around the filenames for the input_file and output_file variables.
+# CMake will pass the quotes through and not be able to find the file.
+cuda_execute_process(
+  "Generating temporary cmake readable file: ${cmake_dependency_file}.tmp"
+  COMMAND "${CMAKE_COMMAND}"
+  -D "input_file:FILEPATH=${NVCC_generated_dependency_file}"
+  -D "output_file:FILEPATH=${cmake_dependency_file}.tmp"
+  -P "${CUDA_make2cmake}"
+  )
+
+if(CUDA_result)
+  message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Copy the file if it is different
+cuda_execute_process(
+  "Copy if different ${cmake_dependency_file}.tmp to ${cmake_dependency_file}"
+  COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${cmake_dependency_file}.tmp" "${cmake_dependency_file}"
+  )
+
+if(CUDA_result)
+  message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Delete the temporary file
+cuda_execute_process(
+  "Removing ${cmake_dependency_file}.tmp and ${NVCC_generated_dependency_file}"
+  COMMAND "${CMAKE_COMMAND}" -E remove "${cmake_dependency_file}.tmp" "${NVCC_generated_dependency_file}"
+  )
+
+if(CUDA_result)
+  message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Generate the code
+cuda_execute_process(
+  "Generating ${generated_file}"
+  COMMAND "${CUDA_NVCC_EXECUTABLE}"
+  "${source_file}"
+  ${format_flag} -o "${generated_file}"
+  ${CCBIN}
+  ${nvcc_flags}
+  ${nvcc_host_compiler_flags}
+  ${CUDA_NVCC_FLAGS}
+  -DNVCC
+  ${CUDA_NVCC_INCLUDE_ARGS}
+  )
+
+if(CUDA_result)
+  # Since nvcc can sometimes leave half done files make sure that we delete the output file.
+  cuda_execute_process(
+    "Removing ${generated_file}"
+    COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
+    )
+  message(FATAL_ERROR "Error generating file ${generated_file}")
+else()
+  if(verbose)
+    message("Generated ${generated_file} successfully.")
+  endif()
+endif()
+
+# Cubin resource report commands.
+if( build_cubin )
+  # Run with -cubin to produce resource usage report.
+  cuda_execute_process(
+    "Generating ${generated_cubin_file}"
+    COMMAND "${CUDA_NVCC_EXECUTABLE}"
+    "${source_file}"
+    ${CUDA_NVCC_FLAGS}
+    ${nvcc_flags}
+    ${CCBIN}
+    ${nvcc_host_compiler_flags}
+    -DNVCC
+    -cubin
+    -o "${generated_cubin_file}"
+    ${CUDA_NVCC_INCLUDE_ARGS}
+    )
+
+  # Execute the parser script.
+  cuda_execute_process(
+    "Executing the parser script"
+    COMMAND  "${CMAKE_COMMAND}"
+    -D "input_file:STRING=${generated_cubin_file}"
+    -P "${CUDA_parse_cubin}"
+    )
+
+endif()

cmake/Option.cmake

@@ -12,7 +12,125 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+include(${CMAKE_CURRENT_LIST_DIR}/IncludeGuard.cmake)
+cmake_include_guard()
+
+include(${CMAKE_CURRENT_LIST_DIR}/Utils.cmake)
+
+
+# build components
+
 option(WITH_API "Build with API layer, need OpenCV" ON)
 
-message(STATUS "Options:")
-message(STATUS "  WITH_API: ${WITH_API}")
+option(WITH_DEVICE_INFO_REQUIRED "Build with device info required" ON)
+
+# 3rdparty components
+
+option(WITH_BOOST "Include Boost support" ON)
+
+
+# packages
+
+if(WITH_API)
+  include(${CMAKE_CURRENT_LIST_DIR}/DetectOpenCV.cmake)
+endif()
+
+if(WITH_DEVICE_INFO_REQUIRED)
+  add_definitions(-DWITH_DEVICE_INFO_REQUIRED)
+endif()
+
+if(WITH_BOOST)
+  find_package(Boost COMPONENTS filesystem)
+  if(Boost_FOUND)
+    set(Boost_VERSION_STRING "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}")
+    set(WITH_FILESYSTEM TRUE)
+    set(WITH_BOOST_FILESYSTEM TRUE)
+    add_definitions(-DWITH_FILESYSTEM)
+    add_definitions(-DWITH_BOOST_FILESYSTEM)
+  endif()
+endif()
+
+if(NOT WITH_FILESYSTEM)
+  if(MSVC OR MSYS OR MINGW)  # win
+    set(WITH_FILESYSTEM TRUE)
+    set(WITH_NATIVE_FILESYSTEM TRUE)
+    add_definitions(-DWITH_FILESYSTEM)
+    add_definitions(-DWITH_NATIVE_FILESYSTEM)
+  endif()
+endif()
+
+find_package(CUDA QUIET)
+
+# summary
+
+set_version_values(CMAKE_CXX_COMPILER_VERSION)
+
+status("")
+status("Platform:")
+status("  HOST_OS: ${HOST_OS}")
+status("  HOST_NAME: ${HOST_NAME}")
+status("  HOST_ARCH: ${HOST_ARCH}")
+status("  HOST_COMPILER: ${CMAKE_CXX_COMPILER_ID}")
+status("    COMPILER_VERSION: ${CMAKE_CXX_COMPILER_VERSION}")
+status("    COMPILER_VERSION_MAJOR: ${CMAKE_CXX_COMPILER_VERSION_MAJOR}")
+status("    COMPILER_VERSION_MINOR: ${CMAKE_CXX_COMPILER_VERSION_MINOR}")
+status("    COMPILER_VERSION_PATCH: ${CMAKE_CXX_COMPILER_VERSION_PATCH}")
+status("    COMPILER_VERSION_TWEAK: ${CMAKE_CXX_COMPILER_VERSION_TWEAK}")
+if(CUDA_FOUND)
+  status("  CUDA_VERSION: ${CUDA_VERSION}")
+  status("    CUDA_VERSION_MAJOR: ${CUDA_VERSION_MAJOR}")
+  status("    CUDA_VERSION_MINOR: ${CUDA_VERSION_MINOR}")
+  status("    CUDA_VERSION_STRING: ${CUDA_VERSION_STRING}")
+endif()
+if(OpenCV_FOUND)
+  status("  OpenCV_VERSION: ${OpenCV_VERSION}")
+  status("    OpenCV_VERSION_MAJOR: ${OpenCV_VERSION_MAJOR}")
+  status("    OpenCV_VERSION_MINOR: ${OpenCV_VERSION_MINOR}")
+  status("    OpenCV_VERSION_PATCH: ${OpenCV_VERSION_PATCH}")
+  status("    OpenCV_VERSION_TWEAK: ${OpenCV_VERSION_TWEAK}")
+  status("    OpenCV_VERSION_STATUS: ${OpenCV_VERSION_STATUS}")
+  status("    OpenCV_WITH_WORLD: ${WITH_OPENCV_WORLD}")
+endif()
+if(mynteye_VERSION)
+  status("  MYNTEYE_VERSION: ${mynteye_VERSION}")
+  status("    MYNTEYE_VERSION_MAJOR: ${mynteye_VERSION_MAJOR}")
+  status("    MYNTEYE_VERSION_MINOR: ${mynteye_VERSION_MINOR}")
+  status("    MYNTEYE_VERSION_PATCH: ${mynteye_VERSION_PATCH}")
+  status("    MYNTEYE_VERSION_TWEAK: ${mynteye_VERSION_TWEAK}")
+endif()
+
+status("")
+status("Options:")
+status("  WITH_API: ${WITH_API}")
+if(WITH_API)
+  if(OpenCV_FOUND)
+    status("    OpenCV: YES")
+    status("    OpenCV_VERSION: ${OpenCV_VERSION}")
+    status("    OpenCV_WORLD: " IF WITH_OPENCV_WORLD "YES" ELSE "NO")
+  else()
+    status("    OpenCV: NO")
+  endif()
+endif()
+
+status("  WITH_DEVICE_INFO_REQUIRED: ${WITH_DEVICE_INFO_REQUIRED}")
+
+status("  WITH_BOOST: ${WITH_BOOST}")
+if(WITH_BOOST)
+  if(Boost_FOUND)
+    status("    Boost: YES")
+    status("    Boost_VERSION: ${Boost_VERSION_STRING}")
+    #status("    Boost_LIBRARIES: ${Boost_LIBRARIES}")
+  else()
+    status("    Boost: NO")
+  endif()
+endif()
+
+status("")
+status("Features:")
+status("  Filesystem: "
+  IF WITH_BOOST_FILESYSTEM "boost"
+  ELIF WITH_NATIVE_FILESYSTEM "native"
+  ELSE "none"
+)
+
+status("")

cmake/TargetArch.cmake

@@ -0,0 +1,167 @@
+# Based on the Qt 5 processor detection code, so should be very accurate
+# http://code.qt.io/cgit/qt/qtbase.git/tree/src/corelib/global/qprocessordetection.h
+# Currently handles arm (v5, v6, v7, v8), x86 (32/64), ia64, and ppc (32/64)
+
+# Regarding POWER/PowerPC, just as is noted in the Qt source,
+# "There are many more known variants/revisions that we do not handle/detect."
+
+set(archdetect_c_code "
+#if defined(__arm__) || defined(__TARGET_ARCH_ARM) || defined(_M_ARM) \\
+    || defined(__aarch64__) || defined(__ARM64__)
+  #if defined(__aarch64__) || defined(__ARM64__)
+    #define cmake_ARM_64
+  #else
+    #define cmake_ARM_32
+  #endif
+  #if defined(__ARM_ARCH) && __ARM_ARCH > 1
+    #define cmake_ARM __ARM_ARCH
+  #elif defined(__TARGET_ARCH_ARM) && __TARGET_ARCH_ARM > 1
+    #define cmake_ARM __TARGET_ARCH_ARM
+  #elif defined(_M_ARM) && _M_ARM > 1
+    #define cmake_ARM _M_ARM
+  #elif defined(__ARM64_ARCH_8__) \\
+      || defined(__aarch64__) \\
+      || defined(__ARMv8__) \\
+      || defined(__ARMv8_A__)
+    #define cmake_ARM 8
+  #elif defined(__ARM_ARCH_7__) \\
+      || defined(__ARM_ARCH_7A__) \\
+      || defined(__ARM_ARCH_7R__) \\
+      || defined(__ARM_ARCH_7M__) \\
+      || defined(__ARM_ARCH_7S__) \\
+      || defined(_ARM_ARCH_7) \\
+      || defined(__CORE_CORTEXA__)
+    #define cmake_ARM 7
+  #elif defined(__ARM_ARCH_6__) \\
+      || defined(__ARM_ARCH_6J__) \\
+      || defined(__ARM_ARCH_6T2__) \\
+      || defined(__ARM_ARCH_6Z__) \\
+      || defined(__ARM_ARCH_6K__) \\
+      || defined(__ARM_ARCH_6ZK__) \\
+      || defined(__ARM_ARCH_6M__)
+    #define cmake_ARM 6
+  #elif defined(__ARM_ARCH_5TEJ__) \\
+      || defined(__ARM_ARCH_5TE__)
+    #define cmake_ARM 5
+  #else
+    #define cmake_ARM 0
+  #endif
+  #if cmake_ARM >= 8
+    #error cmake_ARCH armv8
+  #elif cmake_ARM >= 7
+    #error cmake_ARCH armv7
+  #elif cmake_ARM >= 6
+    #error cmake_ARCH armv6
+  #elif cmake_ARM >= 5
+    #error cmake_ARCH armv5
+  #else
+    #if defined(cmake_ARM_64)
+      #error cmake_ARCH arm64
+    #else
+      #error cmake_ARCH arm
+    #endif
+  #endif
+#elif defined(__i386) || defined(__i386__) || defined(_M_IX86)
+  #error cmake_ARCH i386
+#elif defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(_M_X64)
+  #error cmake_ARCH x86_64
+#elif defined(__ia64) || defined(__ia64__) || defined(_M_IA64)
+  #error cmake_ARCH ia64
+#elif defined(__ppc__) || defined(__ppc) || defined(__powerpc__) \\
+    || defined(_ARCH_COM) || defined(_ARCH_PWR) || defined(_ARCH_PPC) \\
+    || defined(_M_MPPC) || defined(_M_PPC)
+  #if defined(__ppc64__) || defined(__powerpc64__) || defined(__64BIT__)
+    #error cmake_ARCH ppc64
+  #else
+    #error cmake_ARCH ppc
+  #endif
+#endif
+
+#error cmake_ARCH unknown
+")
+
+# Set ppc_support to TRUE before including this file or ppc and ppc64
+# will be treated as invalid architectures since they are no longer supported by Apple
+
+function(target_architecture output_var)
+  if(APPLE AND CMAKE_OSX_ARCHITECTURES)
+    # On OS X we use CMAKE_OSX_ARCHITECTURES *if* it was set
+    # First let's normalize the order of the values
+
+    # Note that it's not possible to compile PowerPC applications if you are using
+    # the OS X SDK version 10.6 or later - you'll need 10.4/10.5 for that, so we
+    # disable it by default
+    # See this page for more information:
+    # http://stackoverflow.com/questions/5333490/how-can-we-restore-ppc-ppc64-as-well-as-full-10-4-10-5-sdk-support-to-xcode-4
+
+    # Architecture defaults to i386 or ppc on OS X 10.5 and earlier, depending on the CPU type detected at runtime.
+    # On OS X 10.6+ the default is x86_64 if the CPU supports it, i386 otherwise.
+
+    foreach(osx_arch ${CMAKE_OSX_ARCHITECTURES})
+      if("${osx_arch}" STREQUAL "ppc" AND ppc_support)
+        set(osx_arch_ppc TRUE)
+      elseif("${osx_arch}" STREQUAL "i386")
+        set(osx_arch_i386 TRUE)
+      elseif("${osx_arch}" STREQUAL "x86_64")
+        set(osx_arch_x86_64 TRUE)
+      elseif("${osx_arch}" STREQUAL "ppc64" AND ppc_support)
+        set(osx_arch_ppc64 TRUE)
+      else()
+        message(FATAL_ERROR "Invalid OS X arch name: ${osx_arch}")
+      endif()
+    endforeach()
+
+    # Now add all the architectures in our normalized order
+    if(osx_arch_ppc)
+      list(APPEND ARCH ppc)
+    endif()
+
+    if(osx_arch_i386)
+      list(APPEND ARCH i386)
+    endif()
+
+    if(osx_arch_x86_64)
+      list(APPEND ARCH x86_64)
+    endif()
+
+    if(osx_arch_ppc64)
+      list(APPEND ARCH ppc64)
+    endif()
+  else()
+    file(WRITE "${CMAKE_BINARY_DIR}/arch.c" "${archdetect_c_code}")
+
+    enable_language(C)
+
+    # Detect the architecture in a rather creative way...
+    # This compiles a small C program which is a series of ifdefs that selects a
+    # particular #error preprocessor directive whose message string contains the
+    # target architecture. The program will always fail to compile (both because
+    # file is not a valid C program, and obviously because of the presence of the
+    # #error preprocessor directives... but by exploiting the preprocessor in this
+    # way, we can detect the correct target architecture even when cross-compiling,
+    # since the program itself never needs to be run (only the compiler/preprocessor)
+    try_run(
+      run_result_unused
+      compile_result_unused
+      "${CMAKE_BINARY_DIR}"
+      "${CMAKE_BINARY_DIR}/arch.c"
+      COMPILE_OUTPUT_VARIABLE ARCH
+      CMAKE_FLAGS CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}
+    )
+
+    # Parse the architecture name from the compiler output
+    string(REGEX MATCH "cmake_ARCH ([a-zA-Z0-9_]+)" ARCH "${ARCH}")
+
+    # Get rid of the value marker leaving just the architecture name
+    string(REPLACE "cmake_ARCH " "" ARCH "${ARCH}")
+
+    # If we are compiling with an unknown architecture this variable should
+    # already be set to "unknown" but in the case that it's empty (i.e. due
+    # to a typo in the code), then set it to unknown
+    if (NOT ARCH)
+      set(ARCH unknown)
+    endif()
+  endif()
+
+  set(${output_var} "${ARCH}" PARENT_SCOPE)
+endfunction()

cmake/Utils.cmake

@@ -49,3 +49,88 @@ macro(make_executable NAME)
     endif()
   endif()
 endmacro()
+
+# set_default_value(VARIABLE DEFAULT)
+macro(set_default_value VARIABLE DEFAULT)
+  if(NOT ${VARIABLE})
+    set(${VARIABLE} ${DEFAULT})
+  endif()
+endmacro()
+
+# set_version_values(VARIABLE)
+macro(set_version_values VARIABLE)
+  string(REPLACE "." ";" __version_list "${${VARIABLE}}")
+  list(LENGTH __version_list __len)
+  if(${__len} GREATER 0)
+    list(GET __version_list 0 ${VARIABLE}_MAJOR)
+  endif()
+  if(${__len} GREATER 1)
+    list(GET __version_list 1 ${VARIABLE}_MINOR)
+  endif()
+  if(${__len} GREATER 2)
+    list(GET __version_list 2 ${VARIABLE}_PATCH)
+  endif()
+  if(${__len} GREATER 3)
+    list(GET __version_list 3 ${VARIABLE}_TWEAK)
+  endif()
+endmacro()
+
+# status(TEXT [IF cond text [ELIF cond text] [ELSE cond text]])
+macro(status TEXT)
+  set(options)
+  set(oneValueArgs)
+  set(multiValueArgs IF ELIF ELSE)
+  cmake_parse_arguments(THIS "${options}" "${oneValueArgs}"
+                        "${multiValueArgs}" ${ARGN})
+
+  #message(STATUS "TEXT: ${TEXT}")
+  #message(STATUS "THIS_IF: ${THIS_IF}")
+  #message(STATUS "THIS_ELIF: ${THIS_ELIF}")
+  #message(STATUS "THIS_ELSE: ${THIS_ELSE}")
+
+  set(__msg_list "${TEXT}")
+  set(__continue TRUE)
+
+  if(__continue AND DEFINED THIS_IF)
+    #message(STATUS "-- THIS_IF: ${THIS_IF}")
+    list(LENGTH THIS_IF __if_len)
+    if(${__if_len} GREATER 1)
+      list(GET THIS_IF 0 __if_cond)
+      if(${__if_cond})
+        list(REMOVE_AT THIS_IF 0)
+        list(APPEND __msg_list ${THIS_IF})
+        set(__continue FALSE)
+      endif()
+    else()
+      message(FATAL_ERROR "status() IF must have cond and text, >= 2 items")
+    endif()
+  endif()
+
+  if(__continue AND DEFINED THIS_ELIF)
+    #message(STATUS "-- THIS_ELIF: ${THIS_ELIF}")
+    list(LENGTH THIS_ELIF __elif_len)
+    if(${__elif_len} GREATER 1)
+      list(GET THIS_ELIF 0 __elif_cond)
+      if(${__elif_cond})
+        list(REMOVE_AT THIS_ELIF 0)
+        list(APPEND __msg_list ${THIS_ELIF})
+        set(__continue FALSE)
+      endif()
+    else()
+      message(FATAL_ERROR "status() ELIF must have cond and text, >= 2 items")
+    endif()
+  endif()
+
+  if(__continue AND DEFINED THIS_ELSE)
+    #message(STATUS "-- THIS_ELSE: ${THIS_ELSE}")
+    list(LENGTH THIS_ELSE __else_len)
+    if(${__else_len} GREATER 0)
+      list(APPEND __msg_list ${THIS_ELSE})
+    else()
+      message(FATAL_ERROR "status() ELSE must have text, >= 1 items")
+    endif()
+  endif()
+
+  string(REPLACE ";" "" __msg_list "${__msg_list}")
+  message(STATUS "${__msg_list}")
+endmacro()

cmake/templates/build.info.in

@@ -0,0 +1,27 @@
+%YAML:1.0
+---
+HOST_OS: "@HOST_OS@"
+HOST_NAME: "@HOST_NAME@"
+HOST_ARCH: "@HOST_ARCH@"
+HOST_COMPILER: "@CMAKE_CXX_COMPILER_ID@"
+COMPILER_VERSION: "@CMAKE_CXX_COMPILER_VERSION@"
+COMPILER_VERSION_MAJOR: @CMAKE_CXX_COMPILER_VERSION_MAJOR@
+COMPILER_VERSION_MINOR: @CMAKE_CXX_COMPILER_VERSION_MINOR@
+COMPILER_VERSION_PATCH: @CMAKE_CXX_COMPILER_VERSION_PATCH@
+COMPILER_VERSION_TWEAK: @CMAKE_CXX_COMPILER_VERSION_TWEAK@
+CUDA_VERSION: "@CUDA_VERSION@"
+CUDA_VERSION_MAJOR: @CUDA_VERSION_MAJOR@
+CUDA_VERSION_MINOR: @CUDA_VERSION_MINOR@
+CUDA_VERSION_STRING: "@CUDA_VERSION_STRING@"
+OpenCV_VERSION: "@OpenCV_VERSION@"
+OpenCV_VERSION_MAJOR: @OpenCV_VERSION_MAJOR@
+OpenCV_VERSION_MINOR: @OpenCV_VERSION_MINOR@
+OpenCV_VERSION_PATCH: @OpenCV_VERSION_PATCH@
+OpenCV_VERSION_TWEAK: @OpenCV_VERSION_TWEAK@
+OpenCV_VERSION_STATUS: "@OpenCV_VERSION_STATUS@"
+OpenCV_WITH_WORLD: "@WITH_OPENCV_WORLD@"
+MYNTEYE_VERSION: "@mynteye_VERSION@"
+MYNTEYE_VERSION_MAJOR: @mynteye_VERSION_MAJOR@
+MYNTEYE_VERSION_MINOR: @mynteye_VERSION_MINOR@
+MYNTEYE_VERSION_PATCH: @mynteye_VERSION_PATCH@
+MYNTEYE_VERSION_TWEAK: @mynteye_VERSION_TWEAK@

cmake/templates/exe.bat.in

@@ -7,7 +7,7 @@ if "%MY_PATH_SET_@__exe_name@%"=="" (
   set "PATH=@__dll_search_paths@;%PATH%"
 )
 
-cd "%MY_DIR%"
+::cd "%MY_DIR%"
 "%MY_DIR%@__exe_name@.exe" %*
-cd "%MY_PWD%"
+::cd "%MY_PWD%"
 ::pause >nul

doc/build.sh

@@ -40,8 +40,8 @@ OUTPUT="$BASE_DIR/_output"
 _texcjk() {
   tex="$1"; shift;
   _echo_in "add cjk to $tex"
-  sed -i "" -e $'s/^\\\\begin{document}$/\\\\usepackage{CJKutf8}\\\n\\\\begin{document}\\\n\\\\begin{CJK}{UTF8}{gbsn}/g' $tex
-  sed -i "" -e $'s/^\\\\end{document}$/\\\\end{CJK}\\\n\\\\end{document}/g' $tex
+  sed -i "" -E $'s/^\\\\begin{document}$/\\\\usepackage{CJKutf8}\\\n\\\\begin{document}\\\n\\\\begin{CJK}{UTF8}{gbsn}/g' $tex
+  sed -i "" -E $'s/^\\\\end{document}$/\\\\end{CJK}\\\n\\\\end{document}/g' $tex
 }
 
 for lang in "${LANGS[@]}"; do
@@ -54,8 +54,13 @@ for lang in "${LANGS[@]}"; do
     doxygen $DOXYFILE
     if [ $pdflatex_FOUND ] && [ -f "$OUTPUT/$lang/latex/Makefile" ]; then
       _echo_in "doxygen make latex"
+      version=`cat $DOXYFILE | grep -m1 "^PROJECT_NUMBER\s*=" | \
+        sed -E "s/^.*=[[:space:]]*(.*)[[:space:]]*$/\1/g"`
+      filename="mynt-eye-sdk-apidoc"; \
+        [ -n "$version" ] && filename="$filename-$version"; \
+        filename="$filename-$lang.pdf"
       cd "$OUTPUT/$lang/latex" && _texcjk refman.tex && make
-      [ -f "refman.pdf" ] && mv "refman.pdf" "../mynteye-apidoc.pdf"
+      [ -f "refman.pdf" ] && mv "refman.pdf" "../$filename"
     fi
     _echo_d "doxygen completed"
   else

doc/en/mainpage.md

@@ -0,0 +1,11 @@
+# MYNT EYE SDK {#mainpage}
+
+* <a class="el" href="annotated.html">API Classes</a>
+* <a class="el" href="modules.html">API Modules</a>
+  * \link enumerations Enumerations\endlink
+  * \link datatypes Datatypes\endlink
+  * \link utils Utiliities\endlink
+  * \link calibration Intrinsics & Extrinsics\endlink
+* <span style="font-weight:bold">Device Specifications</span>
+  * @subpage specs_data
+  * @subpage specs_ctrl

doc/en/spec_control_api.md

@@ -0,0 +1,26 @@
+# Control Protocols {#spec_control_api}
+
+There are two control modes, one is through UVC standard protocol, the other is through UVC custom protocol with extension unit.
+
+## Standard Protocol
+
+| Name | Field | Bytes | Default | Min | Max | Stored | Flash Address | Note |
+| :--- | :---- | :---- | :------ | :-- | :-- | :----- | :------------ | :--- |
+| Gain | gain | 2 | 24 | 0 | 48 | √ | 0x12 | valid if manual-exposure |
+| Brightness | brightness/exposure_time | 2 | 120 | 0 | 240 | √ | 0x14 | valid if manual-exposure |
+| Contrast | contrast/black_level_calibration | 2 | 127 | 0 | 255 | √ | 0x10 | valid if manual-exposure |
+
+## Custom Protocol
+
+| Name | Field | Bytes | Default | Min | Max | Stored | Flash Address | Channel | Note |
+| :--- | :---- | :---- | :------ | :-- | :-- | :----- | :------------ | :------ | :----- |
+| Frame rate | frame_rate | 2 | 25 | 10 | √ | 0x21 | XU_CAM_CTRL | values: {10,15,20,25,30,35,40,45,50,55} |
+| IMU frequency | imu_frequency | 2 | 200 | 100 | 500 | √ | 0x23 | XU_CAM_CTRL | values: {100,200,250,333,500} |
+| Exposure mode | exposure_mode | 1 | 0 | 0 | 1 | √ | 0x0F | XU_CAM_CTRL | 0: enable auto-exposure; 1: manual-exposure |
+| Max gain | max_gain | 2 | 48 | 0 | 48 | √ | 0x1D | XU_CAM_CTRL | valid if auto-exposure |
+| Max exposure time | max_exposure_time | 2 | 240 | 0 | 240 | √ | 0x1B | XU_CAM_CTRL | valid if auto-exposure |
+| Desired brightness | desired_brightness | 2 | 192 | 0 | 255 | √ | 0x19 | XU_CAM_CTRL | valid if auto-exposure |
+| IR control | ir_control | 1 | 0 | 0 | 160 | × | - | XU_CAM_CTRL | |
+| HDR mode | hdr_mode | 1 | 0 | 0 | 1 | √ | 0x1F | XU_CAM_CTRL | 0: 10-bit; 1: 12-bit |
+| Zero drift calibration | zero_drift_calibration | | - | - | - | × | - | XU_HALF_DUPLEX | |
+| Erase chip | erase_chip | | - | - | - | × | - | XU_HALF_DUPLEX | |

doc/en/spec_control_channel.md

@@ -0,0 +1,53 @@
+# Control Channels {#spec_control_channel}
+
+| Name | Field | Address | Bandwidth | Node |
+| :----- | :----- | :----- | :----- | :----- |
+| Camera control channel | XU_CAM_CTRL_CHANNEL | 1 | 3 | |
+| Half-Duplex channel | XU_HALF_DUPLEX_CHANNEL | 2 | 20 | |
+| IMU write channel | XU_IMUDATA_WRITE_CHANNEL | 3 | 5 | |
+| IMU read channel | XU_IMUDATA_READ_CHANNEL | 4 | 2000 | |
+| File channel | XU_FILE_CHANNEL | 5 | 2000 | |
+
+## Camera Control Channel
+
+The channel provides get, set and query (min, max, default).
+
+## Half-Duplex Channel
+
+The channel only provides set, such as zero drift correction.
+
+## IMU Channel
+
+The channel is used to request and response IMU data, see @ref spec_imu_data.
+
+## File Channel
+
+The channel is used to read and write device information, image params, and IMU params.
+
+| Name | Header | Size | File | Checksum |
+| :--- | :- | :--- | :--- | :-------- |
+| Bytes | 1 | 2 | - | 1 |
+| Type | uint8_t | uint16_t | - | uint8_t |
+| Description | Flags | Content size | Content data | Checksum, XOR of all content bytes |
+
+| Header Bit Subscript | Description |
+| :------------------- | :---------- |
+| 0 | Device information |
+| 1 | Image params |
+| 2 | IMU params |
+| 3~6 | Undefined |
+| 7 | 0: Get; 1: Set |
+
+### File Content Packet
+
+| Name | ID | Size | Content |
+| :--- | :- | :--- | :------ |
+| Bytes | 1 | 2 | - |
+| Type | uint8_t | uint16_t | - |
+| Description | Content ID | Content size | Content data |
+
+| File | ID | Max Size |
+| :--- | :- | :------- |
+| Device information | 1 | 250 |
+| Image params | 2 | 250 |
+| IMU params | 4 | 500 |

doc/en/spec_hardware_info.md

@@ -0,0 +1,14 @@
+# Device Information {#spec_hardware_info}
+
+| Name | Field | Fixed Value | USB Descriptor | UVC Extension Unit | Bytes | Note |
+| :----- | :----- | :-------- | :-------------- | :----------------- | :-------- | :----- |
+| VID | vid | 0x04B4 | √ | × | 2 | |
+| PID | pid | 0x00F9 | √ | × | 2 | |
+| Device name | name | MYNT-EYE-? | √ | √ Get | 16 | MYNT-EYE-S1000 |
+| Serial number | serial_number | - | √ | √ Get | 16 | |
+| Firmware version | firmware_version | - | √ | √ Get | 2 | major,minor |
+| Hardware version | hardware_version | - | × | √ Get | 3 | major,minor,flag |
+| Spec version | spec_version | - | × | √ Get | 2 | major,minor |
+| Lens type | lens_type | - | × | √ Get/Set | 4 | vendor(2),product(2); default: 0 |
+| IMU type | imu_type | - | × | √ Get/Set | 4 | vendor(2),product(2); default: 0 |
+| Nominal baseline | nominal_baseline | - | × | √ Get/Set | 2 | unit: mm; default: 0 |

doc/en/spec_image_data.md

@@ -0,0 +1,20 @@
+# Image Data {#spec_image_data}
+
+| Name | Field | Unit | Bytes | Note |
+| :----- | :----- | :----- | :-------- | :----- |
+| Frame ID | frame_id | - | 2 | uint16_t; [0,65535] |
+| Timestamp | timestamp | 10 us | 4 | uint32_t |
+| Exposure Time | exposure_time | 10 us | 2 | uint16_t |
+
+## Image Packet
+
+| Name | Header | Size | Frame ID | Timestamp | Exposure Time | Checksum |
+| :--- | :----- | :--- | :------- | :-------- | :------------ | :------- |
+| Bytes | 1 | 1 | 2 | 4 | 2 | 1 |
+| Type | uint8_t | uint8_t | uint16_t | uint32_t | uint16_t | uint8_t |
+| Description | 0x3B | 0x08, content size | Frame ID | Timestamp | Exposure time | Checksum, XOR of all content bytes |
+
+* The image packet will be dropped, if checksum is incorrect.
+* The accuracy of the time unit: 0.01 ms / 10 us.
+  * The timestamp could indicate 11.9 hours, it will accumulate again after overflow.
+* The timestamp accumulation starts from the time of power-on, instead of opening.

doc/en/spec_image_params.md

@@ -0,0 +1,23 @@
+# Image Params {#spec_image_params}
+
+## Image Intrinsics
+
+| Name | Field | Unit | Bytes | Note |
+| :----- | :----- | :----- | :-------- | :----- |
+| Image width | width | px | 2 | uint16_t; [0,65535] |
+| Image height | height | px | 2 | uint16_t; [0,65535] |
+| Focal length | fx | - | 8 | double |
+| ^ | fy | - | 8 | double |
+| Principal point | cx | - | 8 | double |
+| ^ | cy | - | 8 | double |
+| Distortion model | model | - | 1 | uint8_t; pinhole,... |
+| Distortion coefficients | coeffs[5] | - | 40 | double; k1,k2,p1,p2,k3 |
+
+## Image Extrinsics
+
+Transformation matrix from left image to right image.
+
+| Name | Field | Unit | Bytes | Note |
+| :----- | :----- | :----- | :-------- | :----- |
+| Rotation matrix | rotation[3][3] | - | 72 | double |
+| Translation vector | translation[3] | - | 24 | double |

doc/en/spec_imu_data.md

@@ -0,0 +1,42 @@
+# IMU Data {#spec_imu_data}
+
+## IMU Request Packet
+
+| Name | Header | Serial Number |
+| :--- | :----- | :------------ |
+| Bytes | 1 | 4 |
+| Type | uint8_t | uint32_t |
+| Description | 0x5A | First request should be 0, otherwise the last one |
+
+## IMU Response Packet
+
+The IMU response packet contains multiple IMU packets, and each IMU packet contains multiple IMU segments.
+
+| Name | Header | State | Size | IMU Packets | Checksum |
+| :--- | :----- | :---- | :--- | :---------- | :------- |
+| Bytes | 1 | 1 | 2 | ... | 1 |
+| Type | uint8_t | uint8_t | uint16_t | - | uint8_t |
+| Description | 0x5B | 0 is success, others are failed | Content size | IMU packets | Checksum, XOR of all content bytes |
+
+### IMU Packet
+
+The IMU packet is an array of IMU datas.
+
+| Name | Serial Number | Timestamp | Count | IMU Datas |
+| :--- | :------------ | :-------- | :---- | :-------- |
+| Bytes | 4 | 4 | 1 | ... |
+| Type | uint32_t | uint32_t | uint8_t | - |
+| Description | Serial number | IMU basic timestamp | The number of IMU datas | IMU datas |
+
+### IMU Segment
+
+| Name | Offset | Frame ID | Accelerometer | Temperature | Gyroscope |
+| :--- | :----- | :------- | :------------ | :---------- | :-------- |
+| Bytes | 2 | 2 | 6 | 2 | 6 |
+| Type | int16_t | uint16_t | int16_t * 3 | int16_t | int16_t * 3 |
+| Description | The timestamp offset | Image frame ID | Accel x,y,z values | IMU temperature | Gyro x,y,z values |
+
+* Formula for converting the accel & gyro values to real ones: **real = data * range / 0x10000** .
+  * ``accel`` default ``range`` is **8 g**, ``gyro`` default ``range`` is **1000 deg/s**.
+* Formula for converting the temperature to real value: **real = data / ratio + offset** .
+  * default ``ratio`` is **326.8**, default ``offset`` is **25℃**.

doc/en/spec_imu_params.md

@@ -0,0 +1,23 @@
+# IMU Params {#spec_imu_params}
+
+## IMU Intrinsics
+
+| Name | Field | Unit | Bytes | Note |
+| :----- | :----- | :----- | :-------- | :----- |
+| Scale matrix | acc_scale[3][3] | - | 72 | double |
+| ^ | gyro_scale[3][3] | - | 72 | double |
+| Zero-drift | acc_drift[3] | - | 24 | double |
+| ^ | gyro_drift[3] | - | 24 | double |
+| Noise density | acc_noise[3] | - | 24 | double |
+| ^ | gyro_noise[3] | - | 24 | double |
+| Random walk | acc_bias[3] | - | 24 | double |
+| ^ | gyro_bias[3] | - | 24 | double |
+
+## IMU Extrinsics
+
+Transformation matrix from left image to IMU.
+
+| Name | Field | Unit | Bytes | Note |
+| :----- | :----- | :----- | :-------- | :----- |
+| Rotation matrix | rotation[3][3] | - | 72 | double |
+| Translation vector | translation[3] | - | 24 | double |

doc/en/specs_ctrl.md

@@ -0,0 +1,4 @@
+# Device Control Specification {#specs_ctrl}
+
+* @subpage spec_control_api
+* @subpage spec_control_channel

doc/en/specs_data.md

@@ -0,0 +1,7 @@
+# Device Data Specification {#specs_data}
+
+* @subpage spec_hardware_info
+* @subpage spec_image_params
+* @subpage spec_imu_params
+* @subpage spec_image_data
+* @subpage spec_imu_data

doc/zh-Hans/api.doxyfile

@@ -1,4 +1,4 @@
-# Doxyfile 1.8.13
+# Doxyfile 1.8.14
 
 # This file describes the settings to be used by the documentation system
 # doxygen (www.doxygen.org) for a project.
@@ -20,8 +20,8 @@
 # This tag specifies the encoding used for all characters in the config file
 # that follow. The default is UTF-8 which is also the encoding used for all text
 # before the first occurrence of this tag. Doxygen uses libiconv (or the iconv
-# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv
-# for the list of possible encodings.
+# built into libc) for the transcoding. See
+# https://www.gnu.org/software/libiconv/ for the list of possible encodings.
 # The default value is: UTF-8.
 
 DOXYFILE_ENCODING      = UTF-8
@@ -38,7 +38,7 @@ PROJECT_NAME           = "MYNT EYE SDK"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 2.0.0-rc0
+PROJECT_NUMBER         = 2.0.1-rc1
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -236,7 +236,8 @@ TAB_SIZE               = 2
 # will allow you to put the command \sideeffect (or @sideeffect) in the
 # documentation, which will result in a user-defined paragraph with heading
 # "Side Effects:". You can put \n's in the value part of an alias to insert
-# newlines.
+# newlines (in the resulting output). You can put ^^ in the value part of an
+# alias to insert a newline as if a physical newline was in the original file.
 
 ALIASES                =
 
@@ -337,7 +338,7 @@ BUILTIN_STL_SUPPORT    = NO
 CPP_CLI_SUPPORT        = NO
 
 # Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
-# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen
+# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen
 # will parse them like normal C++ but will assume all classes use public instead
 # of private inheritance when no explicit protection keyword is present.
 # The default value is: NO.
@@ -708,7 +709,7 @@ LAYOUT_FILE            =
 # The CITE_BIB_FILES tag can be used to specify one or more bib files containing
 # the reference definitions. This must be a list of .bib files. The .bib
 # extension is automatically appended if omitted. This requires the bibtex tool
-# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info.
+# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info.
 # For LaTeX the style of the bibliography can be controlled using
 # LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
 # search path. See also \cite for info how to create references.
@@ -791,14 +792,6 @@ WARN_LOGFILE           =
 # Note: If this tag is empty the current directory is searched.
 
 INPUT                  = mainpage.md \
-                         guides.md \
-                         guide_build_linux.md \
-                         guide_build_win.md \
-                         guide_samples.md \
-                         guide_tools.md \
-                         guide_log.md \
-                         guide_opencv.md \
-                         guide_ros.md \
                          specs_data.md \
                          spec_hardware_info.md \
                          spec_image_params.md \
@@ -813,7 +806,7 @@ INPUT                  = mainpage.md \
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
 # libiconv (or the iconv built into libc) for the transcoding. See the libiconv
-# documentation (see: http://www.gnu.org/software/libiconv) for the list of
+# documentation (see: https://www.gnu.org/software/libiconv/) for the list of
 # possible encodings.
 # The default value is: UTF-8.
 
@@ -1061,7 +1054,7 @@ SOURCE_TOOLTIPS        = YES
 # If the USE_HTAGS tag is set to YES then the references to source code will
 # point to the HTML generated by the htags(1) tool instead of doxygen built-in
 # source browser. The htags tool is part of GNU's global source tagging system
-# (see http://www.gnu.org/software/global/global.html). You will need version
+# (see https://www.gnu.org/software/global/global.html). You will need version
 # 4.8.6 or higher.
 #
 # To use it do the following:
@@ -1206,7 +1199,7 @@ HTML_EXTRA_FILES       =
 # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
 # will adjust the colors in the style sheet and background images according to
 # this color. Hue is specified as an angle on a colorwheel, see
-# http://en.wikipedia.org/wiki/Hue for more information. For instance the value
+# https://en.wikipedia.org/wiki/Hue for more information. For instance the value
 # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
 # purple, and 360 is red again.
 # Minimum value: 0, maximum value: 359, default value: 220.
@@ -1242,6 +1235,17 @@ HTML_COLORSTYLE_GAMMA  = 80
 
 HTML_TIMESTAMP         = NO
 
+# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML
+# documentation will contain a main index with vertical navigation menus that
+# are dynamically created via Javascript. If disabled, the navigation index will
+# consists of multiple levels of tabs that are statically embedded in every HTML
+# page. Disable this option to support browsers that do not have Javascript,
+# like the Qt help browser.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_MENUS     = YES
+
 # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
 # documentation will contain sections that can be hidden and shown after the
 # page has loaded.
@@ -1265,12 +1269,12 @@ HTML_INDEX_NUM_ENTRIES = 100
 
 # If the GENERATE_DOCSET tag is set to YES, additional index files will be
 # generated that can be used as input for Apple's Xcode 3 integrated development
-# environment (see: http://developer.apple.com/tools/xcode/), introduced with
+# environment (see: https://developer.apple.com/tools/xcode/), introduced with
 # OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a
 # Makefile in the HTML output directory. Running make will produce the docset in
 # that directory and running make install will install the docset in
 # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
-# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html
+# startup. See https://developer.apple.com/tools/creatingdocsetswithdoxygen.html
 # for more information.
 # The default value is: NO.
 # This tag requires that the tag GENERATE_HTML is set to YES.
@@ -1386,7 +1390,7 @@ QCH_FILE               =
 
 # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
 # Project output. For more information please see Qt Help Project / Namespace
-# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace).
+# (see: http://doc.qt.io/qt-4.8/qthelpproject.html#namespace).
 # The default value is: org.doxygen.Project.
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
@@ -1394,8 +1398,7 @@ QHP_NAMESPACE          = com.slightech.mynteye
 
 # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
 # Help Project output. For more information please see Qt Help Project / Virtual
-# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual-
-# folders).
+# Folders (see: http://doc.qt.io/qt-4.8/qthelpproject.html#virtual-folders).
 # The default value is: doc.
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
@@ -1403,23 +1406,21 @@ QHP_VIRTUAL_FOLDER     = doc
 
 # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
 # filter to add. For more information please see Qt Help Project / Custom
-# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
-# filters).
+# Filters (see: http://doc.qt.io/qt-4.8/qthelpproject.html#custom-filters).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
 QHP_CUST_FILTER_NAME   =
 
 # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
 # custom filter to add. For more information please see Qt Help Project / Custom
-# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
-# filters).
+# Filters (see: http://doc.qt.io/qt-4.8/qthelpproject.html#custom-filters).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
 QHP_CUST_FILTER_ATTRS  =
 
 # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
 # project's filter section matches. Qt Help Project / Filter Attributes (see:
-# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).
+# http://doc.qt.io/qt-4.8/qthelpproject.html#filter-attributes).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
 QHP_SECT_FILTER_ATTRS  =
@@ -1512,7 +1513,7 @@ EXT_LINKS_IN_WINDOW    = NO
 
 FORMULA_FONTSIZE       = 10
 
-# Use the FORMULA_TRANPARENT tag to determine whether or not the images
+# Use the FORMULA_TRANSPARENT tag to determine whether or not the images
 # generated for formulas are transparent PNGs. Transparent PNGs are not
 # supported properly for IE 6.0, but are supported on all modern browsers.
 #
@@ -1524,7 +1525,7 @@ FORMULA_FONTSIZE       = 10
 FORMULA_TRANSPARENT    = YES
 
 # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
-# http://www.mathjax.org) which uses client side Javascript for the rendering
+# https://www.mathjax.org) which uses client side Javascript for the rendering
 # instead of using pre-rendered bitmaps. Use this if you do not have LaTeX
 # installed or if you want to formulas look prettier in the HTML output. When
 # enabled you may also need to install MathJax separately and configure the path
@@ -1551,11 +1552,11 @@ MATHJAX_FORMAT         = HTML-CSS
 # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
 # Content Delivery Network so you can quickly see the result without installing
 # MathJax. However, it is strongly recommended to install a local copy of
-# MathJax from http://www.mathjax.org before deployment.
-# The default value is: http://cdn.mathjax.org/mathjax/latest.
+# MathJax from https://www.mathjax.org before deployment.
+# The default value is: https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.2/.
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
-MATHJAX_RELPATH        = http://cdn.mathjax.org/mathjax/latest
+MATHJAX_RELPATH        = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.2/
 
 # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax
 # extension names that should be enabled during MathJax rendering. For example
@@ -1613,7 +1614,7 @@ SERVER_BASED_SEARCH    = NO
 #
 # Doxygen ships with an example indexer (doxyindexer) and search engine
 # (doxysearch.cgi) which are based on the open source search engine library
-# Xapian (see: http://xapian.org/).
+# Xapian (see: https://xapian.org/).
 #
 # See the section "External Indexing and Searching" for details.
 # The default value is: NO.
@@ -1626,7 +1627,7 @@ EXTERNAL_SEARCH        = NO
 #
 # Doxygen ships with an example indexer (doxyindexer) and search engine
 # (doxysearch.cgi) which are based on the open source search engine library
-# Xapian (see: http://xapian.org/). See the section "External Indexing and
+# Xapian (see: https://xapian.org/). See the section "External Indexing and
 # Searching" for details.
 # This tag requires that the tag SEARCHENGINE is set to YES.
 
@@ -1815,7 +1816,7 @@ LATEX_SOURCE_CODE      = NO
 
 # The LATEX_BIB_STYLE tag can be used to specify the style to use for the
 # bibliography, e.g. plainnat, or ieeetr. See
-# http://en.wikipedia.org/wiki/BibTeX and \cite for more info.
+# https://en.wikipedia.org/wiki/BibTeX and \cite for more info.
 # The default value is: plain.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
@@ -1998,9 +1999,9 @@ DOCBOOK_PROGRAMLISTING = NO
 #---------------------------------------------------------------------------
 
 # If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an
-# AutoGen Definitions (see http://autogen.sf.net) file that captures the
-# structure of the code including all documentation. Note that this feature is
-# still experimental and incomplete at the moment.
+# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures
+# the structure of the code including all documentation. Note that this feature
+# is still experimental and incomplete at the moment.
 # The default value is: NO.
 
 GENERATE_AUTOGEN_DEF   = NO

doc/zh-Hans/mainpage.md

@@ -1,9 +1,11 @@
 # MYNT EYE SDK {#mainpage}
 
-* API 模块
+* <a class="el" href="annotated.html">API 类</a>
+* <a class="el" href="modules.html">API 模块</a>
   * \link enumerations 枚举类型\endlink
   * \link datatypes 数据类型\endlink
+  * \link utils 工具函数\endlink
   * \link calibration 内参与外参\endlink
-* 设备说明
+* <span style="font-weight:bold">设备说明</span>
   * @subpage specs_data
   * @subpage specs_ctrl

doc/zh-Hans/spec_control_api.md

@@ -1,4 +1,4 @@
-# 控制 API 说明 {#spec_contorl_api}
+# 控制 API 说明 {#spec_control_api}
 
 控制有两种实现方式，一是通过 UVC 标准协议，二是通过 UVC 拓展通道自定义协议。
 
@@ -8,21 +8,21 @@
 | :----- | :----- | :-------- | :-------- | :-------- | :-------- | :----------- | :----------- | :----- |
 | 增益 | gain | 2 | 24 | 0 | 48 | √ | 0x12 | 关闭自动曝光，手动设定的参数 |
 | 亮度 | brightness/exposure_time | 2 | 120 | 0 | 240 | √ | 0x14 | 关闭自动曝光，手动设定的参数 |
-| 对比度 | contrast/black_level_calibration | 2 | 127 | 0 | 255 | √ | 0x10 | |
+| 对比度 | contrast/black_level_calibration | 2 | 127 | 0 | 255 | √ | 0x10 | 关闭自动曝光，手动设定的参数 |
 
 > UVC 标准协议实现的控制，有现成的 API 进行 Get & Set ，包括 Min, Max, Default 。
 
 ## 自定义协议
 
-| 名称 | 字段 | 字节数 |  默认值 | 最小值 | 最大值 | 是否储存 | Flash 地址 | 所属通道 | 通道地址 | 说明 |
-| :----- | :----- | :-------- | :-------- | :-------- | :-------- | :----------- | :----------- | :----------- | :----------- | :----- |
-| 图像帧率 | frame_rate | 2 | 25 | 10 | 60 | √ | 0x21 | XU_CAM_CTRL | 0x0100 | 步进为5，即有效值为{10,15,20,25,30,35,40,45,50,55,60} |
-| IMU 频率 | imu_frequency | 2 | 200 | 100 | 500 | √ | 0x23 | XU_CAM_CTRL | 0x0100 | 有效值为{100,200,250,333,500} |
-| 曝光模式 | exposure_mode | 1 | 0 | 0 | 1 | √ | 0x0F | XU_CAM_CTRL | 0x0100 | 0：开启自动曝光； 1：关闭 |
-| 最大增益 | max_gain | 2 | 48 | 0 | 48 | √ | 0x1D | XU_CAM_CTRL | 0x0100 | 开始自动曝光，可设定的阈值 |
-| 最大曝光时间 | max_exposure_time | 2 | 240 | 0 | 240 | √ | 0x1B | XU_CAM_CTRL | 0x0100 | 开始自动曝光，可设定的阈值 |
-| 期望亮度 | desired_brightness | 2 | 192 | 0 | 255 | √ | 0x19 | XU_CAM_CTRL | 0x0100 | |
-| IR 控制 | ir_control | 1 | 0 | 0 | 160 | × | - | XU_CAM_CTRL | 0x0100 | |
-| HDR 模式 | hdr_mode | 1 | 0 | 0 | 1 | √ | 0x1F | XU_CAM_CTRL | 0x0100 | 0：10-bit；1：12-bit |
-| 零漂标定 | zero_drift_calibration | | - | - | - | × | - | XU_HALF_DUPLEX | 0x0200 | |
-| 擦除芯片 | erase_chip | | - | - | - | × | - | XU_HALF_DUPLEX | 0x0200 | |
+| 名称 | 字段 | 字节数 | 默认值 | 最小值 | 最大值 | 是否储存 | Flash 地址 | 所属通道 | 说明 |
+| :----- | :----- | :-------- | :-------- | :-------- | :-------- | :----------- | :----------- | :----------- | :----- |
+| 图像帧率 | frame_rate | 2 | 25 | 10 | √ | 0x21 | XU_CAM_CTRL | 步进为5，即有效值为{10,15,20,25,30,35,40,45,50,55} |
+| IMU 频率 | imu_frequency | 2 | 200 | 100 | 500 | √ | 0x23 | XU_CAM_CTRL | 有效值为{100,200,250,333,500} |
+| 曝光模式 | exposure_mode | 1 | 0 | 0 | 1 | √ | 0x0F | XU_CAM_CTRL | 0：开启自动曝光； 1：关闭 |
+| 最大增益 | max_gain | 2 | 48 | 0 | 48 | √ | 0x1D | XU_CAM_CTRL | 开始自动曝光，可设定的阈值 |
+| 最大曝光时间 | max_exposure_time | 2 | 240 | 0 | 240 | √ | 0x1B | XU_CAM_CTRL | 开始自动曝光，可设定的阈值 |
+| 期望亮度 | desired_brightness | 2 | 192 | 0 | 255 | √ | 0x19 | XU_CAM_CTRL | 开始自动曝光，可设定的阈值 |
+| IR 控制 | ir_control | 1 | 0 | 0 | 160 | × | - | XU_CAM_CTRL | |
+| HDR 模式 | hdr_mode | 1 | 0 | 0 | 1 | √ | 0x1F | XU_CAM_CTRL | 0：10-bit；1：12-bit |
+| 零漂标定 | zero_drift_calibration | | - | - | - | × | - | XU_HALF_DUPLEX | |
+| 擦除芯片 | erase_chip | | - | - | - | × | - | XU_HALF_DUPLEX | |

doc/zh-Hans/spec_control_channel.md

@@ -1,12 +1,12 @@
-# 拓展通道说明 {#spec_contorl_channel}
+# 拓展通道说明 {#spec_control_channel}
 
 | 名称 | 字段 | 地址 | 带宽 | 说明 |
 | :----- | :----- | :----- | :----- | :----- |
-| 相机控制通道 | XU_CAM_CTRL_CHANNEL | 0x0100 | 3 | |
-| 半双工通道 | XU_HALF_DUPLEX_CHANNEL | 0x0200 | 20 | |
-| IMU 请求通道 | XU_IMUDATA_WRITE_CHANNEL | 0x0300 | 5 | |
-| IMU 响应通道 | XU_IMUDATA_READ_CHANNEL | 0x0400 | 2000 | |
-| 文件通道 | XU_FILE_CHANNEL | 0x0500 | 2000 | |
+| 相机控制通道 | XU_CAM_CTRL_CHANNEL | 1 | 3 | |
+| 半双工通道 | XU_HALF_DUPLEX_CHANNEL | 2 | 20 | |
+| IMU 请求通道 | XU_IMUDATA_WRITE_CHANNEL | 3 | 5 | |
+| IMU 响应通道 | XU_IMUDATA_READ_CHANNEL | 4 | 2000 | |
+| 文件通道 | XU_FILE_CHANNEL | 5 | 2000 | |
 
 ## 相机控制通道
 
@@ -14,7 +14,7 @@
 
 ## 半双工通道
 
-半双工通道是那些仅需 Get 或 Set 的控制通道，如请求零漂矫正。
+半双工通道是那些仅需 Set 的控制通道，如请求零漂矫正。
 
 ## IMU 通道
 
@@ -25,18 +25,18 @@
 用来读写硬件信息、图像参数、 IMU 参数的通道。
 
 | Name | Header | Size | File | Checksum |
-| :--- | :- | :--- | :--- | :-------- |
+| :--- | :----- | :--- | :--- | :-------- |
 | 字节数 | 1 | 2 | - | 1 |
 | 类型 | uint8_t | uint16_t | - | uint8_t |
 | 描述 | 标识 | 文件内容大小 | 文件内容 | 校验码（文件内容所有字节异或） |
 
-| Header bit | Description |
-| :--------- | :---------- |
-| 0 | 0: Get; 1: Set |
-| 1~4 | 未定义 |
-| 5 | IMU 参数 |
-| 6 | 图像参数 |
-| 7 | 硬件信息 |
+| Header Bit Subscript | Description |
+| :------------------- | :---------- |
+| 0 | 硬件信息 |
+| 1 | 图像参数 |
+| 2 | IMU 参数 |
+| 3~6 | 未定义 |
+| 7 | 0: Get; 1: Set |
 
 ### 文件内容包
 

doc/zh-Hans/spec_hardware_info.md

@@ -5,7 +5,7 @@
 | VID | vid | 0x04B4 | √ | × | 2 | |
 | PID | pid | 0x00F9 | √ | × | 2 | |
 | 设备名称 | name | MYNT-EYE-? | √ | √ Get | 16 | MYNT-EYE-S1000 |
-| 序列号  | serial_number | - | √ | √ Get | 16 | |
+| 序列号 | serial_number | - | √ | √ Get | 16 | |
 | 固件版本 | firmware_version | - | √ | √ Get | 2 | major,minor |
 | 硬件版本 | hardware_version | - | × | √ Get | 3 | major,minor,flag |
 | 协议版本 | spec_version | - | × | √ Get | 2 | major,minor |

doc/zh-Hans/spec_image_data.md

@@ -10,13 +10,13 @@
 
 ## 图像数据包
 
-| Name | Header | Size | FrameID | Timestamp | ExposureTime | Checksum |
-| :--- | :----- | :--- | :------ | :-------- | :----------- | :------- |
+| Name | Header | Size | Frame ID | Timestamp | Exposure Time | Checksum |
+| :--- | :----- | :--- | :------- | :-------- | :------------ | :------- |
 | 字节数 | 1 | 1 | 2 | 4 | 2 | 1 |
 | 类型 | uint8_t | uint8_t | uint16_t | uint32_t | uint16_t | uint8_t |
 | 描述 | 0x3B | 0x08 （数据内容大小） | 帧 ID | 时间戳 | 曝光时间 | 校验码（数据内容所有字节异或） |
 
 * 数据包校验不过，会丢弃该帧。
-* 时间的单位精度为： 0.01 ms / 10 us 。
+* 时间单位的精度为： 0.01 ms / 10 us 。
   * 4 字节能表示的最大时间约是 11.9 小时，溢出后将重累计。
 * 时间累计是从上电时从开始，而不是从打开时开始。

doc/zh-Hans/spec_image_params.md

@@ -13,7 +13,7 @@
 | 畸变模型 | model | - | 1 | uint8_t; pinhole,... |
 | 畸变参数 | coeffs[5] | - | 40 | double; k1,k2,p1,p2,k3 |
 
-> 图像内参不同分辨率会不同。如果多分辨率的话，就会有多个。
+> 图像分辨率不同，内参不同。多分辨率的话，需有多个内参。
 
 ## 图像外参
 

doc/zh-Hans/spec_imu_data.md

@@ -30,11 +30,11 @@ IMU 包/小包，是一组 IMU 数据。
 
 ### IMU 段
 
-| Name | Offset | FrameID | Accelerometer | Temperature | Gyroscope |
-| :--- | :----- | :------ | :------------ | :---------- | :-------- |
+| Name | Offset | Frame ID | Accelerometer | Temperature | Gyroscope |
+| :--- | :----- | :------- | :------------ | :---------- | :-------- |
 | 字节数 | 2 | 2 | 6 | 2 | 6 |
 | 类型 | int16_t | uint16_t | int16_t * 3 | int16_t | int16_t * 3 |
-| Description | 相对基准时间戳的偏移量 | 图像帧 ID | 加速度计 x y z 三轴的值 | IMU 的温度 | 陀螺仪 x y z 三轴的值 |
+| 描述 | 相对基准时间戳的偏移量 | 图像帧 ID | 加速度计 x y z 三轴的值 | IMU 的温度 | 陀螺仪 x y z 三轴的值 |
 
 * 加速度计和陀螺仪的计量值换算成物理值公式： **real = data * range / 0x10000** 。
   * 加速度计量程默认值为 **8 g** ，陀螺仪量程默认值为 **1000 deg/s** 。

doc/zh-Hans/specs_ctrl.md

@@ -1,4 +1,4 @@
 # 设备控制说明 {#specs_ctrl}
 
-* @subpage spec_contorl_api
-* @subpage spec_contorl_channel
+* @subpage spec_control_api
+* @subpage spec_control_channel

include/mynteye/glog_init.h

@@ -25,14 +25,30 @@ struct glog_init {
   glog_init(int argc, char *argv[]) {
     (void)argc;
 
-    // FLAGS_logtostderr = true;
-    FLAGS_alsologtostderr = true;
+    // Set whether log messages go to stderr instead of logfiles
+    FLAGS_logtostderr = true;
+
+    // Set whether log messages go to stderr in addition to logfiles.
+    // FLAGS_alsologtostderr = true;
+
+    // Set color messages logged to stderr (if supported by terminal).
     FLAGS_colorlogtostderr = true;
 
+    // Log suppression level: messages logged at a lower level than this
+    // are suppressed.
+    FLAGS_minloglevel = google::GLOG_INFO;
+
+    // If specified, logfiles are written into this directory instead of the
+    // default logging directory.
     FLAGS_log_dir = ".";
+
+    // Sets the maximum log file size (in MB).
     FLAGS_max_log_size = 1024;
+
+    // Sets whether to avoid logging to the disk if the disk is full.
     FLAGS_stop_logging_if_full_disk = true;
 
+    // Show all VLOG(m) messages for m <= this.
     // FLAGS_v = 2;
 
     google::InitGoogleLogging(argv[0]);

include/mynteye/mynteye.h.in

@@ -58,4 +58,7 @@ MYNTEYE_API_VERSION_CHECK( \
 # define MYNTEYE_USE_NAMESPACE
 #endif
 
+const char MYNTEYE_SDK_ROOT_DIR[] = "@MYNTEYE_SDK_ROOT_DIR@";
+const char MYNTEYE_SDK_INSTALL_DIR[] = "@MYNTEYE_SDK_INSTALL_DIR@";
+
 #endif  // MYNTEYE_MYNTEYE_H_

include/mynteye/types.h

@@ -144,7 +144,7 @@ enum class Option : std::uint8_t {
   /**
    * Image frame rate, must set IMU_FREQUENCY together
    *
-   *   values: {10,15,20,25,30,35,40,45,50,55,60}, default: 25
+   *   values: {10,15,20,25,30,35,40,45,50,55}, default: 25
    */
   FRAME_RATE,
   /**

samples/CMakeLists.txt

@@ -79,4 +79,6 @@ add_subdirectory(uvc)
 
 # tutorials
 
-add_subdirectory(tutorials)
+if(WITH_API)
+  add_subdirectory(tutorials)
+endif()

samples/api/CMakeLists.txt

@@ -27,3 +27,15 @@ make_executable(camera_a
   LINK_LIBS mynteye ${OpenCV_LIBS}
   DLL_SEARCH_PATHS ${PRO_DIR}/_install/bin ${OpenCV_LIB_SEARCH_PATH}
 )
+
+## get_depth_with_region
+
+if(OpenCV_VERSION VERSION_LESS 4.0)
+
+make_executable(get_depth_with_region
+  SRCS get_depth_with_region.cc
+  LINK_LIBS mynteye ${OpenCV_LIBS}
+  DLL_SEARCH_PATHS ${PRO_DIR}/_install/bin ${OpenCV_LIB_SEARCH_PATH}
+)
+
+endif()

samples/api/get_depth_with_region.cc

@@ -0,0 +1,224 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include <glog/logging.h>
+
+#include "mynteye/api.h"
+
+namespace {
+
+class DepthRegion {
+ public:
+  explicit DepthRegion(std::uint32_t n)
+      : n_(std::move(n)), show_(false), selected_(false), point_(0, 0) {}
+
+  ~DepthRegion() = default;
+
+  /**
+   * 鼠标事件：默认不选中区域，随鼠标移动而显示。单击后，则会选中区域来显示。你可以再单击已选中区域或双击未选中区域，取消选中。
+   */
+  void OnMouse(const int &event, const int &x, const int &y, const int &flags) {
+    UNUSED(flags)
+    if (event != CV_EVENT_MOUSEMOVE && event != CV_EVENT_LBUTTONDOWN) {
+      return;
+    }
+    show_ = true;
+
+    if (event == CV_EVENT_MOUSEMOVE) {
+      if (!selected_) {
+        point_.x = x;
+        point_.y = y;
+      }
+    } else if (event == CV_EVENT_LBUTTONDOWN) {
+      if (selected_) {
+        if (x >= static_cast<int>(point_.x - n_) &&
+            x <= static_cast<int>(point_.x + n_) &&
+            y >= static_cast<int>(point_.y - n_) &&
+            y <= static_cast<int>(point_.y + n_)) {
+          selected_ = false;
+        }
+      } else {
+        selected_ = true;
+      }
+      point_.x = x;
+      point_.y = y;
+    }
+  }
+
+  template <typename T>
+  void ShowElems(
+      const cv::Mat &depth,
+      std::function<std::string(const T &elem)> elem2string,
+      int elem_space = 40,
+      std::function<std::string(
+          const cv::Mat &depth, const cv::Point &point, const std::uint32_t &n)>
+          getinfo = nullptr) {
+    if (!show_)
+      return;
+
+    int space = std::move(elem_space);
+    int n = 2 * n_ + 1;
+    cv::Mat im(space * n, space * n, CV_8UC3, cv::Scalar(255, 255, 255));
+
+    int x, y;
+    std::string str;
+    int baseline = 0;
+    for (int i = -n_; i <= n; ++i) {
+      x = point_.x + i;
+      if (x < 0 || x >= depth.cols)
+        continue;
+      for (int j = -n_; j <= n; ++j) {
+        y = point_.y + j;
+        if (y < 0 || y >= depth.rows)
+          continue;
+
+        str = elem2string(depth.at<T>(y, x));
+
+        cv::Scalar color(0, 0, 0);
+        if (i == 0 && j == 0)
+          color = cv::Scalar(0, 0, 255);
+
+        cv::Size sz =
+            cv::getTextSize(str, cv::FONT_HERSHEY_PLAIN, 1, 1, &baseline);
+
+        cv::putText(
+            im, str, cv::Point(
+                         (i + n_) * space + (space - sz.width) / 2,
+                         (j + n_) * space + (space + sz.height) / 2),
+            cv::FONT_HERSHEY_PLAIN, 1, color, 1);
+      }
+    }
+
+    if (getinfo) {
+      std::string info = getinfo(depth, point_, n_);
+      if (!info.empty()) {
+        cv::Size sz =
+            cv::getTextSize(info, cv::FONT_HERSHEY_PLAIN, 1, 1, &baseline);
+
+        cv::putText(
+            im, info, cv::Point(5, 5 + sz.height), cv::FONT_HERSHEY_PLAIN, 1,
+            cv::Scalar(255, 0, 255), 1);
+      }
+    }
+
+    cv::imshow("region", im);
+  }
+
+  void DrawRect(cv::Mat &image) {  // NOLINT
+    if (!show_)
+      return;
+    std::uint32_t n = (n_ > 1) ? n_ : 1;
+    n += 1;  // outside the region
+    cv::rectangle(
+        image, cv::Point(point_.x - n, point_.y - n),
+        cv::Point(point_.x + n, point_.y + n),
+        selected_ ? cv::Scalar(0, 255, 0) : cv::Scalar(0, 0, 255), 1);
+  }
+
+ private:
+  std::uint32_t n_;
+  bool show_;
+  bool selected_;
+  cv::Point point_;
+};
+
+void OnDepthMouseCallback(int event, int x, int y, int flags, void *userdata) {
+  DepthRegion *region = reinterpret_cast<DepthRegion *>(userdata);
+  region->OnMouse(event, x, y, flags);
+}
+
+}  // namespace
+
+MYNTEYE_USE_NAMESPACE
+
+int main(int argc, char *argv[]) {
+  auto &&api = API::Create(argc, argv);
+  if (!api)
+    return 1;
+
+  api->SetOptionValue(Option::IR_CONTROL, 80);
+
+  api->EnableStreamData(Stream::DISPARITY_NORMALIZED);
+  api->EnableStreamData(Stream::DEPTH);
+
+  api->Start(Source::VIDEO_STREAMING);
+
+  cv::namedWindow("frame");
+  cv::namedWindow("depth");
+  cv::namedWindow("region");
+
+  DepthRegion depth_region(3);
+  auto depth_info = [](
+      const cv::Mat &depth, const cv::Point &point, const std::uint32_t &n) {
+    UNUSED(depth)
+    std::ostringstream os;
+    os << "depth pos: [" << point.y << ", " << point.x << "]"
+       << "±" << n << ", unit: mm";
+    return os.str();
+  };
+
+  while (true) {
+    api->WaitForStreams();
+
+    auto &&left_data = api->GetStreamData(Stream::LEFT);
+    auto &&right_data = api->GetStreamData(Stream::RIGHT);
+
+    cv::Mat img;
+    cv::hconcat(left_data.frame, right_data.frame, img);
+    cv::imshow("frame", img);
+
+    auto &&disp_data = api->GetStreamData(Stream::DISPARITY_NORMALIZED);
+    auto &&depth_data = api->GetStreamData(Stream::DEPTH);
+    if (!disp_data.frame.empty() && !depth_data.frame.empty()) {
+      // Show disparity instead of depth, but show depth values in region.
+      auto &&depth_frame = disp_data.frame;
+
+#ifdef USE_OPENCV3
+      // ColormapTypes
+      //   http://docs.opencv.org/master/d3/d50/group__imgproc__colormap.html#ga9a805d8262bcbe273f16be9ea2055a65
+      cv::applyColorMap(depth_frame, depth_frame, cv::COLORMAP_JET);
+#endif
+
+      cv::setMouseCallback("depth", OnDepthMouseCallback, &depth_region);
+      // Note: DrawRect will change some depth values to show the rect.
+      depth_region.DrawRect(depth_frame);
+
+      cv::imshow("depth", depth_frame);
+
+      depth_region.ShowElems<ushort>(
+          depth_data.frame,
+          [](const ushort &elem) {
+            if (elem >= 10000) {
+              // Filter errors, or limit to valid range.
+              //
+              // reprojectImageTo3D(), missing values will set to 10000
+              //   https://docs.opencv.org/master/d9/d0c/group__calib3d.html#ga1bc1152bd57d63bc524204f21fde6e02
+              return std::string("invalid");
+            }
+            return std::to_string(elem);
+          },
+          80, depth_info);
+    }
+
+    char key = static_cast<char>(cv::waitKey(1));
+    if (key == 27 || key == 'q' || key == 'Q') {  // ESC/Q
+      break;
+    }
+  }
+
+  api->Stop(Source::VIDEO_STREAMING);
+  return 0;
+}

samples/tutorials/CMakeLists.txt

@@ -63,7 +63,7 @@ macro(make_executable2 NAME)
   endif()
 endmacro()
 
-if(WITH_API)
+# packages
 
 # If you install PCL to different directory, please set CMAKE_PREFIX_PATH to find it.
 #LIST(APPEND CMAKE_PREFIX_PATH /usr/local/share)
@@ -86,11 +86,13 @@ else()
   message(WARNING "PCL not found :(")
 endif()
 
-# data
+# beginner level
 
-make_executable2(get_device_info SRCS data/get_device_info.cc)
-make_executable2(get_img_params SRCS data/get_img_params.cc)
-make_executable2(get_imu_params SRCS data/get_imu_params.cc)
+## data
+
+make_executable2(get_device_info SRCS data/get_device_info.cc WITH_OPENCV)
+make_executable2(get_img_params SRCS data/get_img_params.cc WITH_OPENCV)
+make_executable2(get_imu_params SRCS data/get_imu_params.cc WITH_OPENCV)
 
 make_executable2(get_stereo SRCS data/get_stereo.cc WITH_OPENCV)
 make_executable2(get_stereo_rectified SRCS data/get_stereo_rectified.cc WITH_OPENCV)
@@ -109,7 +111,7 @@ make_executable2(get_from_callbacks
 )
 make_executable2(get_with_plugin SRCS data/get_with_plugin.cc WITH_OPENCV)
 
-# control
+## control
 
 make_executable2(ctrl_framerate SRCS control/framerate.cc WITH_OPENCV)
 make_executable2(ctrl_auto_exposure
@@ -122,4 +124,21 @@ make_executable2(ctrl_manual_exposure
 )
 make_executable2(ctrl_infrared SRCS control/infrared.cc WITH_OPENCV)
 
+# intermediate level
+
+make_executable2(get_all_device_info SRCS intermediate/get_all_device_info.cc WITH_OPENCV)
+
+if(PCL_FOUND)
+
+if(OpenCV_VERSION VERSION_LESS 4.0)
+
+make_executable2(get_depth_and_points
+  SRCS intermediate/get_depth_and_points.cc util/cv_painter.cc util/pc_viewer.cc
+  WITH_OPENCV WITH_PCL
+)
+
 endif()
+
+endif()
+
+# advanced level

samples/tutorials/control/auto_exposure.cc

@@ -29,7 +29,7 @@ int main(int argc, char *argv[]) {
   // auto-exposure: 0
   api->SetOptionValue(Option::EXPOSURE_MODE, 0);
 
-  // max_gain: range [0.48], default 48
+  // max_gain: range [0,48], default 48
   api->SetOptionValue(Option::MAX_GAIN, 48);
   // max_exposure_time: range [0,240], default 240
   api->SetOptionValue(Option::MAX_EXPOSURE_TIME, 240);

samples/tutorials/control/framerate.cc

@@ -30,7 +30,7 @@ int main(int argc, char *argv[]) {
   // Attention: must set FRAME_RATE and IMU_FREQUENCY together, otherwise won't
   // succeed.
 
-  // FRAME_RATE values: 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60
+  // FRAME_RATE values: 10, 15, 20, 25, 30, 35, 40, 45, 50, 55
   api->SetOptionValue(Option::FRAME_RATE, 25);
   // IMU_FREQUENCY values: 100, 200, 250, 333, 500
   api->SetOptionValue(Option::IMU_FREQUENCY, 500);

samples/tutorials/control/manual_exposure.cc

@@ -29,7 +29,7 @@ int main(int argc, char *argv[]) {
   // manual-exposure: 1
   api->SetOptionValue(Option::EXPOSURE_MODE, 1);
 
-  // gain: range [0.48], default 24
+  // gain: range [0,48], default 24
   api->SetOptionValue(Option::GAIN, 24);
   // brightness/exposure_time: range [0,240], default 120
   api->SetOptionValue(Option::BRIGHTNESS, 120);

samples/tutorials/data/get_img_params.cc

@@ -25,8 +25,8 @@ int main(int argc, char *argv[]) {
   LOG(INFO) << "Intrinsics left: {" << api->GetIntrinsics(Stream::LEFT) << "}";
   LOG(INFO) << "Intrinsics right: {" << api->GetIntrinsics(Stream::RIGHT)
             << "}";
-  LOG(INFO) << "Extrinsics left to right: {"
-            << api->GetExtrinsics(Stream::LEFT, Stream::RIGHT) << "}";
+  LOG(INFO) << "Extrinsics right to left: {"
+            << api->GetExtrinsics(Stream::RIGHT, Stream::LEFT) << "}";
 
   return 0;
 }

samples/tutorials/intermediate/get_all_device_info.cc

@@ -0,0 +1,50 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "mynteye/context.h"
+#include "mynteye/device.h"
+#include "mynteye/glog_init.h"
+
+MYNTEYE_USE_NAMESPACE
+
+int main(int argc, char *argv[]) {
+  glog_init _(argc, argv);
+
+  LOG(INFO) << "Detecting MYNT EYE devices";
+  Context context;
+  auto &&devices = context.devices();
+
+  size_t n = devices.size();
+  if (n <= 0) {
+    LOG(ERROR) << "No MYNT EYE devices :(";
+    return 1;
+  }
+
+  for (size_t i = 0; i < n; i++) {
+    auto &&device = devices[i];
+    LOG(INFO) << "MYNT EYE device index: " << i;
+    LOG(INFO) << "  Device name: " << device->GetInfo(Info::DEVICE_NAME);
+    LOG(INFO) << "  Serial number: " << device->GetInfo(Info::SERIAL_NUMBER);
+    LOG(INFO) << "  Firmware version: "
+              << device->GetInfo(Info::FIRMWARE_VERSION);
+    LOG(INFO) << "  Hardware version: "
+              << device->GetInfo(Info::HARDWARE_VERSION);
+    LOG(INFO) << "  Spec version: " << device->GetInfo(Info::SPEC_VERSION);
+    LOG(INFO) << "  Lens type: " << device->GetInfo(Info::LENS_TYPE);
+    LOG(INFO) << "  IMU type: " << device->GetInfo(Info::IMU_TYPE);
+    LOG(INFO) << "  Nominal baseline: "
+              << device->GetInfo(Info::NOMINAL_BASELINE);
+  }
+
+  return 0;
+}

samples/tutorials/intermediate/get_depth_and_points.cc

@@ -0,0 +1,241 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include <glog/logging.h>
+
+#include "mynteye/api.h"
+
+#include "util/cv_painter.h"
+#include "util/pc_viewer.h"
+
+namespace {
+
+class DepthRegion {
+ public:
+  explicit DepthRegion(std::uint32_t n)
+      : n_(std::move(n)), show_(false), selected_(false), point_(0, 0) {}
+
+  ~DepthRegion() = default;
+
+  /**
+   * 鼠标事件：默认不选中区域，随鼠标移动而显示。单击后，则会选中区域来显示。你可以再单击已选中区域或双击未选中区域，取消选中。
+   */
+  void OnMouse(const int &event, const int &x, const int &y, const int &flags) {
+    UNUSED(flags)
+    if (event != CV_EVENT_MOUSEMOVE && event != CV_EVENT_LBUTTONDOWN) {
+      return;
+    }
+    show_ = true;
+
+    if (event == CV_EVENT_MOUSEMOVE) {
+      if (!selected_) {
+        point_.x = x;
+        point_.y = y;
+      }
+    } else if (event == CV_EVENT_LBUTTONDOWN) {
+      if (selected_) {
+        if (x >= static_cast<int>(point_.x - n_) &&
+            x <= static_cast<int>(point_.x + n_) &&
+            y >= static_cast<int>(point_.y - n_) &&
+            y <= static_cast<int>(point_.y + n_)) {
+          selected_ = false;
+        }
+      } else {
+        selected_ = true;
+      }
+      point_.x = x;
+      point_.y = y;
+    }
+  }
+
+  template <typename T>
+  void ShowElems(
+      const cv::Mat &depth,
+      std::function<std::string(const T &elem)> elem2string,
+      int elem_space = 40,
+      std::function<std::string(
+          const cv::Mat &depth, const cv::Point &point, const std::uint32_t &n)>
+          getinfo = nullptr) {
+    if (!show_)
+      return;
+
+    int space = std::move(elem_space);
+    int n = 2 * n_ + 1;
+    cv::Mat im(space * n, space * n, CV_8UC3, cv::Scalar(255, 255, 255));
+
+    int x, y;
+    std::string str;
+    int baseline = 0;
+    for (int i = -n_; i <= n; ++i) {
+      x = point_.x + i;
+      if (x < 0 || x >= depth.cols)
+        continue;
+      for (int j = -n_; j <= n; ++j) {
+        y = point_.y + j;
+        if (y < 0 || y >= depth.rows)
+          continue;
+
+        str = elem2string(depth.at<T>(y, x));
+
+        cv::Scalar color(0, 0, 0);
+        if (i == 0 && j == 0)
+          color = cv::Scalar(0, 0, 255);
+
+        cv::Size sz =
+            cv::getTextSize(str, cv::FONT_HERSHEY_PLAIN, 1, 1, &baseline);
+
+        cv::putText(
+            im, str, cv::Point(
+                         (i + n_) * space + (space - sz.width) / 2,
+                         (j + n_) * space + (space + sz.height) / 2),
+            cv::FONT_HERSHEY_PLAIN, 1, color, 1);
+      }
+    }
+
+    if (getinfo) {
+      std::string info = getinfo(depth, point_, n_);
+      if (!info.empty()) {
+        cv::Size sz =
+            cv::getTextSize(info, cv::FONT_HERSHEY_PLAIN, 1, 1, &baseline);
+
+        cv::putText(
+            im, info, cv::Point(5, 5 + sz.height), cv::FONT_HERSHEY_PLAIN, 1,
+            cv::Scalar(255, 0, 255), 1);
+      }
+    }
+
+    cv::imshow("region", im);
+  }
+
+  void DrawRect(cv::Mat &image) {  // NOLINT
+    if (!show_)
+      return;
+    std::uint32_t n = (n_ > 1) ? n_ : 1;
+    n += 1;  // outside the region
+    cv::rectangle(
+        image, cv::Point(point_.x - n, point_.y - n),
+        cv::Point(point_.x + n, point_.y + n),
+        selected_ ? cv::Scalar(0, 255, 0) : cv::Scalar(0, 0, 255), 1);
+  }
+
+ private:
+  std::uint32_t n_;
+  bool show_;
+  bool selected_;
+  cv::Point point_;
+};
+
+void OnDepthMouseCallback(int event, int x, int y, int flags, void *userdata) {
+  DepthRegion *region = reinterpret_cast<DepthRegion *>(userdata);
+  region->OnMouse(event, x, y, flags);
+}
+
+}  // namespace
+
+MYNTEYE_USE_NAMESPACE
+
+int main(int argc, char *argv[]) {
+  auto &&api = API::Create(argc, argv);
+  if (!api)
+    return 1;
+
+  api->SetOptionValue(Option::IR_CONTROL, 80);
+
+  api->EnableStreamData(Stream::DISPARITY_NORMALIZED);
+  api->EnableStreamData(Stream::DEPTH);
+
+  api->Start(Source::VIDEO_STREAMING);
+
+  cv::namedWindow("frame");
+  cv::namedWindow("depth");
+  cv::namedWindow("region");
+
+  DepthRegion depth_region(3);
+  auto depth_info = [](
+      const cv::Mat &depth, const cv::Point &point, const std::uint32_t &n) {
+    UNUSED(depth)
+    std::ostringstream os;
+    os << "depth pos: [" << point.y << ", " << point.x << "]"
+       << "±" << n << ", unit: mm";
+    return os.str();
+  };
+
+  CVPainter painter;
+  PCViewer pcviewer;
+
+  while (true) {
+    api->WaitForStreams();
+
+    auto &&left_data = api->GetStreamData(Stream::LEFT);
+    auto &&right_data = api->GetStreamData(Stream::RIGHT);
+
+    cv::Mat img;
+    cv::hconcat(left_data.frame, right_data.frame, img);
+
+    painter.DrawImgData(img, *left_data.img);
+
+    cv::imshow("frame", img);
+
+    auto &&disp_data = api->GetStreamData(Stream::DISPARITY_NORMALIZED);
+    auto &&depth_data = api->GetStreamData(Stream::DEPTH);
+    if (!disp_data.frame.empty() && !depth_data.frame.empty()) {
+      // Show disparity instead of depth, but show depth values in region.
+      auto &&depth_frame = disp_data.frame;
+
+#ifdef USE_OPENCV3
+      // ColormapTypes
+      //   http://docs.opencv.org/master/d3/d50/group__imgproc__colormap.html#ga9a805d8262bcbe273f16be9ea2055a65
+      cv::applyColorMap(depth_frame, depth_frame, cv::COLORMAP_JET);
+#endif
+
+      cv::setMouseCallback("depth", OnDepthMouseCallback, &depth_region);
+      // Note: DrawRect will change some depth values to show the rect.
+      depth_region.DrawRect(depth_frame);
+
+      cv::imshow("depth", depth_frame);
+
+      depth_region.ShowElems<ushort>(
+          depth_data.frame,
+          [](const ushort &elem) {
+            if (elem >= 10000) {
+              // Filter errors, or limit to valid range.
+              //
+              // reprojectImageTo3D(), missing values will set to 10000
+              //   https://docs.opencv.org/master/d9/d0c/group__calib3d.html#ga1bc1152bd57d63bc524204f21fde6e02
+              return std::string("invalid");
+            }
+            return std::to_string(elem);
+          },
+          80, depth_info);
+    }
+
+    auto &&points_data = api->GetStreamData(Stream::POINTS);
+    if (!points_data.frame.empty()) {
+      pcviewer.Update(points_data.frame);
+    }
+
+    char key = static_cast<char>(cv::waitKey(1));
+    if (key == 27 || key == 'q' || key == 'Q') {  // ESC/Q
+      break;
+    }
+    if (pcviewer.WasStopped()) {
+      break;
+    }
+  }
+
+  api->Stop(Source::VIDEO_STREAMING);
+  return 0;
+}

samples/tutorials/util/pc_viewer.cc

@@ -35,7 +35,7 @@ std::shared_ptr<pcl::visualization::PCLVisualizer> CustomColorVis(
   // viewer->addCoordinateSystem(1.0);
   viewer->addCoordinateSystem(1000.0);
   viewer->initCameraParameters();
-  viewer->setCameraPosition(0, 0, -150, 0, 1, 0);
+  viewer->setCameraPosition(0, 0, -150, 0, -1, 0);
   return (viewer);
 }
 

samples/uvc/camera.cc

@@ -27,8 +27,17 @@
 
 struct frame {
   const void *data = nullptr;
+  std::function<void()> continuation = nullptr;
+  frame() {
+    // VLOG(2) << __func__;
+  }
   ~frame() {
+    // VLOG(2) << __func__;
     data = nullptr;
+    if (continuation) {
+      continuation();
+      continuation = nullptr;
+    }
   }
 };
 
@@ -41,7 +50,11 @@ int main(int argc, char *argv[]) {
 
   auto context = uvc::create_context();
   auto devices = uvc::query_devices(context);
-  LOG_IF(FATAL, devices.size() <= 0) << "No devices :(";
+  if (devices.size() <= 0) {
+    LOG(ERROR) << "No devices :(";
+    return 1;
+  }
+
   for (auto &&device : devices) {
     auto vid = uvc::get_vendor_id(*device);
     // auto pid = uvc::get_product_id(*device);
@@ -54,7 +67,10 @@ int main(int argc, char *argv[]) {
   // std::string dashes(80, '-');
 
   size_t n = mynteye_devices.size();
-  LOG_IF(FATAL, n <= 0) << "No MYNT EYE devices :(";
+  if (n <= 0) {
+    LOG(ERROR) << "No MYNT EYE devices :(";
+    return 1;
+  }
 
   LOG(INFO) << "MYNT EYE devices: ";
   for (size_t i = 0; i < n; i++) {
@@ -87,18 +103,25 @@ int main(int argc, char *argv[]) {
   std::mutex mtx;
   std::condition_variable cv;
 
-  std::vector<frame> frames;
-  const auto frame_ready = [&frames]() { return !frames.empty(); };
-  const auto frame_empty = [&frames]() { return frames.empty(); };
+  std::shared_ptr<frame> frame = nullptr;
+  const auto frame_ready = [&frame]() { return frame != nullptr; };
+  const auto frame_empty = [&frame]() { return frame == nullptr; };
 
   uvc::set_device_mode(
       *device, 752, 480, static_cast<int>(Format::YUYV), 25,
-      [&mtx, &cv, &frames, &frame_ready](const void *data) {
+      [&mtx, &cv, &frame, &frame_ready](
+          const void *data, std::function<void()> continuation) {
         // reinterpret_cast<const std::uint8_t *>(data);
         std::unique_lock<std::mutex> lock(mtx);
-        frame frame;
-        frame.data = data;  // not copy
-        frames.push_back(frame);
+        if (frame == nullptr) {
+          frame = std::make_shared<struct frame>();
+        } else {
+          if (frame->continuation) {
+            frame->continuation();
+          }
+        }
+        frame->data = data;  // not copy here
+        frame->continuation = continuation;
         if (frame_ready())
           cv.notify_one();
       });
@@ -119,13 +142,12 @@ int main(int argc, char *argv[]) {
         throw std::runtime_error("Timeout waiting for frame.");
     }
 
-    auto frame = frames.back();  // only last one is valid
-
-    cv::Mat img(480, 752, CV_8UC2, const_cast<void *>(frame.data));
+    // only lastest frame is valid
+    cv::Mat img(480, 752, CV_8UC2, const_cast<void *>(frame->data));
     cv::cvtColor(img, img, cv::COLOR_YUV2BGR_YUY2);
     cv::imshow("frame", img);
 
-    frames.clear();
+    frame = nullptr;
 
     char key = static_cast<char>(cv::waitKey(1));
     if (key == 27 || key == 'q' || key == 'Q') {  // ESC/Q

scripts/init.sh

@@ -24,11 +24,17 @@ source "$BASE_DIR/common/host.sh"
 
 PYTHON="python"
 if [ "$HOST_OS" = "Win" ]; then
-  # default python on MSYS
-  PYTHON="python2"
+  if ! _detect_cmd $PYTHON; then
+    PYTHON="python2"  # try python2 on MSYS
+  fi
 fi
 
-_detect $PYTHON
+_detect $PYTHON 1
+
+PYTHON_FOUND="${PYTHON}_FOUND"
+if [ -z "${!PYTHON_FOUND}" ]; then
+  _echo_en "$PYTHON not found"
+fi
 
 if [ "$HOST_OS" = "Linux" ]; then
   _detect_install() {
@@ -137,13 +143,21 @@ fi
 
 # detect pip
 if ! _detect_cmd pip; then
-  _echo_sn "Install pip"
-  [ -f "get-pip.py" ] || curl -O https://bootstrap.pypa.io/get-pip.py
-  $SUDO $PYTHON get-pip.py
+  if [ -n "${!PYTHON_FOUND}" ]; then
+    _echo_sn "Install pip"
+    [ -f "get-pip.py" ] || curl -O https://bootstrap.pypa.io/get-pip.py
+    $SUDO $PYTHON get-pip.py
+  else
+    _echo_en "Skipped install pip, as $PYTHON not found"
+  fi
 fi
 # pip install
-_echo_d "pip install --upgrade autopep8 cpplint pylint requests"
-$SUDO pip install --upgrade autopep8 cpplint pylint requests
+if _detect_cmd pip; then
+  _echo_d "pip install --upgrade autopep8 cpplint pylint requests"
+  $SUDO pip install --upgrade autopep8 cpplint pylint requests
+else
+  _echo_en "Skipped pip install packages, as pip not found"
+fi
 
 ## realpath
 
@@ -168,8 +182,12 @@ ROOT_DIR=$(realpath "$BASE_DIR/..")
 
 ## init
 
-_echo_s "Init git hooks"
-$PYTHON "$ROOT_DIR/tools/linter/init-git-hooks.py"
+if [ -n "${!PYTHON_FOUND}" ]; then
+  _echo_s "Init git hooks"
+  $PYTHON "$ROOT_DIR/tools/linter/init-git-hooks.py"
+else
+  _echo_en "Skipped init git hooks, as $PYTHON not found"
+fi
 
 ## cmake version
 

src/api/api.cc

@@ -13,8 +13,14 @@
 // limitations under the License.
 #include "api/api.h"
 
+#ifdef WITH_BOOST_FILESYSTEM
+#include <boost/filesystem.hpp>
+#include <boost/range/iterator_range.hpp>
+#endif
+
 #include <glog/logging.h>
 
+#include <algorithm>
 #include <thread>
 
 #include "mynteye/glog_init.h"
@@ -23,13 +29,210 @@
 #include "api/plugin.h"
 #include "api/synthetic.h"
 #include "device/device.h"
+#include "device/device_s.h"
 #include "internal/dl.h"
 
+#if defined(WITH_FILESYSTEM) && defined(WITH_NATIVE_FILESYSTEM)
+#if defined(OS_WIN)
+#include <windows.h>
+#endif
+#endif
+
 MYNTEYE_BEGIN_NAMESPACE
 
-API::API(std::shared_ptr<Device> device)
-    : device_(device), synthetic_(new Synthetic(this)) {
+namespace {
+
+#if defined(WITH_FILESYSTEM)
+
+#if defined(WITH_BOOST_FILESYSTEM)
+
+namespace fs = boost::filesystem;
+
+bool file_exists(const fs::path &p) {
+  try {
+    fs::file_status s = fs::status(p);
+    return fs::exists(s) && fs::is_regular_file(s);
+  } catch (fs::filesystem_error &e) {
+    LOG(ERROR) << e.what();
+    return false;
+  }
+}
+
+bool dir_exists(const fs::path &p) {
+  try {
+    fs::file_status s = fs::status(p);
+    return fs::exists(s) && fs::is_directory(s);
+  } catch (fs::filesystem_error &e) {
+    LOG(ERROR) << e.what();
+    return false;
+  }
+}
+
+#elif defined(WITH_NATIVE_FILESYSTEM)
+
+#if defined(OS_WIN)
+
+bool file_exists(const std::string &p) {
+  DWORD attrs = GetFileAttributes(p.c_str());
+  return (attrs != INVALID_FILE_ATTRIBUTES) &&
+         !(attrs & FILE_ATTRIBUTE_DIRECTORY);
+}
+
+bool dir_exists(const std::string &p) {
+  DWORD attrs = GetFileAttributes(p.c_str());
+  return (attrs != INVALID_FILE_ATTRIBUTES) &&
+         (attrs & FILE_ATTRIBUTE_DIRECTORY);
+}
+
+#else
+#error "Unsupported native filesystem"
+#endif
+
+#endif
+
+std::vector<std::string> get_plugin_paths() {
+  std::string info_path(MYNTEYE_SDK_INSTALL_DIR);
+  info_path.append(OS_SEP "share" OS_SEP "mynteye" OS_SEP "build.info");
+
+  cv::FileStorage fs(info_path, cv::FileStorage::READ);
+  if (!fs.isOpened()) {
+    LOG(WARNING) << "build.info not found: " << info_path;
+    return {};
+  }
+
+  auto to_lower = [](std::string &s) {  // NOLINT
+    std::transform(s.begin(), s.end(), s.begin(), ::tolower);
+  };
+
+  std::string host_os = fs["HOST_OS"];
+  to_lower(host_os);
+  std::string host_name = fs["HOST_NAME"];
+  to_lower(host_name);
+  std::string host_arch = fs["HOST_ARCH"];
+  to_lower(host_arch);
+  std::string host_compiler = fs["HOST_COMPILER"];
+  to_lower(host_compiler);
+
+  // std::string compiler_version = fs["COMPILER_VERSION"];
+  int compiler_version_major = fs["COMPILER_VERSION_MAJOR"];
+  // int compiler_version_minor = fs["COMPILER_VERSION_MINOR"];
+  // int compiler_version_patch = fs["COMPILER_VERSION_PATCH"];
+  // int compiler_version_tweak = fs["COMPILER_VERSION_TWEAK"];
+
+  std::string cuda_version = fs["CUDA_VERSION"];
+  // int cuda_version_major = fs["CUDA_VERSION_MAJOR"];
+  // int cuda_version_minor = fs["CUDA_VERSION_MINOR"];
+  // std::string cuda_version_string = fs["CUDA_VERSION_STRING"];
+
+  std::string opencv_version = fs["OpenCV_VERSION"];
+  // int opencv_version_major = fs["OpenCV_VERSION_MAJOR"];
+  // int opencv_version_minor = fs["OpenCV_VERSION_MINOR"];
+  // int opencv_version_patch = fs["OpenCV_VERSION_PATCH"];
+  // int opencv_version_tweak = fs["OpenCV_VERSION_TWEAK"];
+  // std::string opencv_version_status = fs["OpenCV_VERSION_STATUS"];
+  std::string opencv_with_world = fs["OpenCV_WITH_WORLD"];
+  to_lower(opencv_with_world);
+
+  std::string mynteye_version = fs["MYNTEYE_VERSION"];
+  // int mynteye_version_major = fs["MYNTEYE_VERSION_MAJOR"];
+  // int mynteye_version_minor = fs["MYNTEYE_VERSION_MINOR"];
+  // int mynteye_version_patch = fs["MYNTEYE_VERSION_PATCH"];
+  // int mynteye_version_tweak = fs["MYNTEYE_VERSION_TWEAK"];
+
+  fs.release();
+
+  std::string lib_prefix;
+  std::string lib_suffix;
+  if (host_os == "linux") {
+    if (host_compiler != "gnu" || compiler_version_major < 5)
+      return {};
+    lib_prefix = "lib";
+    lib_suffix = ".so";
+  } else if (host_os == "win") {
+    lib_prefix = "";
+    lib_suffix = ".dll";
+  } else if (host_os == "mac") {
+    lib_prefix = "lib";
+    lib_suffix = ".dylib";
+  } else {
+    return {};
+  }
+
+  std::vector<std::string> names;
+  {
+    std::vector<std::string> prefixes{
+        // lib_prefix + "plugin_b_ocl" + ocl_version,
+        lib_prefix + "plugin_g_cuda" + cuda_version,
+    };
+    std::string opencv_name("_opencv" + opencv_version);
+    if (opencv_with_world == "true") {
+      opencv_name.append("-world");
+    }
+    for (auto &&prefix : prefixes) {
+      names.push_back(prefix + opencv_name + "_mynteye" + mynteye_version);
+      names.push_back(prefix + opencv_name);
+      names.push_back(prefix);
+    }
+    for (auto &&name : names) {
+      name.append(lib_suffix);
+    }
+  }
+
+  std::vector<std::string> paths;
+
+  std::vector<std::string> plats;
+  if (host_name != host_os) {
+    plats.push_back(host_name + "-" + host_arch);
+  }
+  plats.push_back(host_os + "-" + host_arch);
+
+  std::vector<std::string> dirs{MYNTEYE_SDK_ROOT_DIR, MYNTEYE_SDK_INSTALL_DIR};
+  for (auto &&plat : plats) {
+    for (auto &&dir : dirs) {
+      auto &&plat_dir = dir + OS_SEP "plugins" + OS_SEP + plat;
+      // VLOG(2) << "plat_dir: " << plat_dir;
+      if (!dir_exists(plat_dir))
+        continue;
+      for (auto &&name : names) {
+        // VLOG(2) << "  name: " << name;
+        auto &&path = plat_dir + OS_SEP + name;
+        if (!file_exists(path))
+          continue;
+        paths.push_back(path);
+      }
+    }
+  }
+
+  return paths;
+}
+
+#endif
+
+}  // namespace
+
+API::API(std::shared_ptr<Device> device) : device_(device) {
   VLOG(2) << __func__;
+  if (std::dynamic_pointer_cast<StandardDevice>(device_) != nullptr) {
+    bool in_l_ok, in_r_ok, ex_r2l_ok;
+    device_->GetIntrinsics(Stream::LEFT, &in_l_ok);
+    device_->GetIntrinsics(Stream::RIGHT, &in_r_ok);
+    device_->GetExtrinsics(Stream::RIGHT, Stream::LEFT, &ex_r2l_ok);
+    if (!in_l_ok || !in_r_ok || !ex_r2l_ok) {
+#if defined(WITH_DEVICE_INFO_REQUIRED)
+      LOG(FATAL)
+#else
+      LOG(WARNING)
+#endif
+          << "Image params not found, but we need it to process the "
+             "images. Please `make tools` and use `img_params_writer` "
+             "to write the image params. If you update the SDK from "
+             "1.x, the `SN*.conf` is the file contains them. Besides, "
+             "you could also calibrate them by yourself. Read the guide "
+             "doc (https://github.com/slightech/MYNT-EYE-SDK-2-Guide) "
+             "to learn more.";
+    }
+  }
+  synthetic_.reset(new Synthetic(this));
 }
 
 API::~API() {
@@ -140,7 +343,7 @@ void API::SetMotionCallback(motion_callback_t callback) {
   static auto callback_ = callback;
   if (callback_) {
     device_->SetMotionCallback(
-        [](const device::MotionData &data) { callback_({data.imu}); });
+        [](const device::MotionData &data) { callback_({data.imu}); }, true);
   } else {
     device_->SetMotionCallback(nullptr);
   }
@@ -156,6 +359,18 @@ bool API::HasMotionCallback() const {
 
 void API::Start(const Source &source) {
   if (source == Source::VIDEO_STREAMING) {
+#ifdef WITH_FILESYSTEM
+    if (!synthetic_->HasPlugin()) {
+      try {
+        auto &&plugin_paths = get_plugin_paths();
+        if (plugin_paths.size() > 0) {
+          EnablePlugin(plugin_paths[0]);
+        }
+      } catch (...) {
+        LOG(WARNING) << "Incorrect yaml format: build.info";
+      }
+    }
+#endif
     synthetic_->StartVideoStreaming();
   } else if (source == Source::MOTION_TRACKING) {
     device_->StartMotionTracking();
@@ -163,7 +378,7 @@ void API::Start(const Source &source) {
     Start(Source::VIDEO_STREAMING);
     Start(Source::MOTION_TRACKING);
   } else {
-    LOG(FATAL) << "Unsupported source :(";
+    LOG(ERROR) << "Unsupported source :(";
   }
 }
 
@@ -178,7 +393,7 @@ void API::Stop(const Source &source) {
     std::this_thread::sleep_for(std::chrono::milliseconds(10));
     Stop(Source::VIDEO_STREAMING);
   } else {
-    LOG(FATAL) << "Unsupported source :(";
+    LOG(ERROR) << "Unsupported source :(";
   }
 }
 
@@ -220,7 +435,9 @@ void API::EnablePlugin(const std::string &path) {
 
   plugin_version_code_t *plugin_version_code =
       dl.Sym<plugin_version_code_t>("plugin_version_code");
-  LOG(INFO) << "Enable plugin, version code: " << plugin_version_code();
+  LOG(INFO) << "Enable plugin success";
+  LOG(INFO) << "  version code: " << plugin_version_code();
+  LOG(INFO) << "  path: " << path;
 
   plugin_create_t *plugin_create = dl.Sym<plugin_create_t>("plugin_create");
   plugin_destroy_t *plugin_destroy = dl.Sym<plugin_destroy_t>("plugin_destroy");

src/api/api.h

@@ -30,6 +30,12 @@ MYNTEYE_BEGIN_NAMESPACE
 class Device;
 class Synthetic;
 
+namespace device {
+
+class Frame;
+
+}  // namespace device
+
 namespace api {
 
 /**
@@ -41,6 +47,16 @@ struct MYNTEYE_API StreamData {
   std::shared_ptr<ImgData> img;
   /** Frame. */
   cv::Mat frame;
+  /** Raw frame. */
+  std::shared_ptr<device::Frame> frame_raw;
+
+  bool operator==(const StreamData &other) const {
+    if (img && other.img) {
+      return img->frame_id == other.img->frame_id &&
+             img->timestamp == other.img->timestamp;
+    }
+    return false;
+  }
 };
 
 /**
@@ -50,6 +66,14 @@ struct MYNTEYE_API StreamData {
 struct MYNTEYE_API MotionData {
   /** ImuData. */
   std::shared_ptr<ImuData> imu;
+
+  bool operator==(const MotionData &other) const {
+    if (imu && other.imu) {
+      return imu->frame_id == other.imu->frame_id &&
+             imu->timestamp == other.imu->timestamp;
+    }
+    return false;
+  }
 };
 
 }  // namespace api
@@ -65,7 +89,7 @@ class MYNTEYE_API API {
   using motion_callback_t = std::function<void(const api::MotionData &data)>;
 
   explicit API(std::shared_ptr<Device> device);
-  /*virtual*/ ~API();
+  virtual ~API();
 
   /**
    * Create the API instance.
@@ -222,12 +246,12 @@ class MYNTEYE_API API {
   void DisableStreamData(const Stream &stream);
 
   /**
-   * Get the datas of stream.
-   * @note default cache 4 datas at most.
+   * Get the latest data of stream.
    */
   api::StreamData GetStreamData(const Stream &stream);
   /**
-   * Get the latest data of stream.
+   * Get the datas of stream.
+   * @note default cache 4 datas at most.
    */
   std::vector<api::StreamData> GetStreamDatas(const Stream &stream);
 

src/api/processor/depth_processor.cc

@@ -15,10 +15,15 @@
 
 #include <glog/logging.h>
 
+#include <utility>
+
 MYNTEYE_BEGIN_NAMESPACE
 
-DepthProcessor::DepthProcessor() : Processor() {
-  VLOG(2) << __func__;
+const char DepthProcessor::NAME[] = "DepthProcessor";
+
+DepthProcessor::DepthProcessor(std::int32_t proc_period)
+    : Processor(std::move(proc_period)) {
+  VLOG(2) << __func__ << ": proc_period=" << proc_period;
 }
 
 DepthProcessor::~DepthProcessor() {

src/api/processor/depth_processor.h

@@ -23,9 +23,9 @@ MYNTEYE_BEGIN_NAMESPACE
 
 class DepthProcessor : public Processor {
  public:
-  static constexpr auto &&NAME = "DepthProcessor";
+  static const char NAME[];
 
-  DepthProcessor();
+  explicit DepthProcessor(std::int32_t proc_period = 0);
   virtual ~DepthProcessor();
 
   std::string Name() override;

src/api/processor/disparity_normalized_processor.cc

@@ -17,10 +17,17 @@
 
 #include <glog/logging.h>
 
+#include <utility>
+
 MYNTEYE_BEGIN_NAMESPACE
 
-DisparityNormalizedProcessor::DisparityNormalizedProcessor() : Processor() {
-  VLOG(2) << __func__;
+const char DisparityNormalizedProcessor::NAME[] =
+    "DisparityNormalizedProcessor";
+
+DisparityNormalizedProcessor::DisparityNormalizedProcessor(
+    std::int32_t proc_period)
+    : Processor(std::move(proc_period)) {
+  VLOG(2) << __func__ << ": proc_period=" << proc_period;
 }
 
 DisparityNormalizedProcessor::~DisparityNormalizedProcessor() {

src/api/processor/disparity_normalized_processor.h

@@ -23,9 +23,9 @@ MYNTEYE_BEGIN_NAMESPACE
 
 class DisparityNormalizedProcessor : public Processor {
  public:
-  static constexpr auto &&NAME = "DisparityNormalizedProcessor";
+  static const char NAME[];
 
-  DisparityNormalizedProcessor();
+  explicit DisparityNormalizedProcessor(std::int32_t proc_period = 0);
   virtual ~DisparityNormalizedProcessor();
 
   std::string Name() override;

src/api/processor/disparity_processor.cc

@@ -17,10 +17,15 @@
 
 #include <glog/logging.h>
 
+#include <utility>
+
 MYNTEYE_BEGIN_NAMESPACE
 
-DisparityProcessor::DisparityProcessor() : Processor() {
-  VLOG(2) << __func__;
+const char DisparityProcessor::NAME[] = "DisparityProcessor";
+
+DisparityProcessor::DisparityProcessor(std::int32_t proc_period)
+    : Processor(std::move(proc_period)) {
+  VLOG(2) << __func__ << ": proc_period=" << proc_period;
   int sgbmWinSize = 3;
   int numberOfDisparities = 64;
 

src/api/processor/disparity_processor.h

@@ -29,9 +29,9 @@ MYNTEYE_BEGIN_NAMESPACE
 
 class DisparityProcessor : public Processor {
  public:
-  static constexpr auto &&NAME = "DisparityProcessor";
+  static const char NAME[];
 
-  DisparityProcessor();
+  explicit DisparityProcessor(std::int32_t proc_period = 0);
   virtual ~DisparityProcessor();
 
   std::string Name() override;

src/api/processor/object.h

@@ -17,6 +17,8 @@
 
 #include <opencv2/core/core.hpp>
 
+#include <memory>
+
 #include "mynteye/mynteye.h"
 
 MYNTEYE_BEGIN_NAMESPACE
@@ -29,6 +31,7 @@ struct MYNTEYE_API Object {
   virtual ~Object() = default;
 
   virtual Object *Clone() const = 0;
+  virtual bool DecValidity() const = 0;
 
   /** Cast the obj to T pointer */
   template <typename T>
@@ -41,6 +44,11 @@ struct MYNTEYE_API Object {
   static const T *Cast(const Object *obj) {
     return dynamic_cast<const T *>(obj);
   }
+
+  template <typename T>
+  static std::shared_ptr<T> Cast(const std::shared_ptr<Object> &obj) {
+    return std::dynamic_pointer_cast<T>(obj);
+  }
 };
 
 /**
@@ -58,6 +66,10 @@ struct MYNTEYE_API ObjMat : public Object {
     mat->value = value.clone();
     return mat;
   }
+
+  bool DecValidity() const {
+    return !value.empty();
+  }
 };
 
 /**
@@ -80,6 +92,10 @@ struct MYNTEYE_API ObjMat2 : public Object {
     mat2->second = second.clone();
     return mat2;
   }
+
+  bool DecValidity() const {
+    return !first.empty() && !second.empty();
+  }
 };
 
 MYNTEYE_END_NAMESPACE

src/api/processor/points_processor.cc

@@ -21,8 +21,11 @@
 
 MYNTEYE_BEGIN_NAMESPACE
 
-PointsProcessor::PointsProcessor(cv::Mat Q) : Processor(), Q_(std::move(Q)) {
-  VLOG(2) << __func__;
+const char PointsProcessor::NAME[] = "PointsProcessor";
+
+PointsProcessor::PointsProcessor(cv::Mat Q, std::int32_t proc_period)
+    : Processor(std::move(proc_period)), Q_(std::move(Q)) {
+  VLOG(2) << __func__ << ": proc_period=" << proc_period;
 }
 
 PointsProcessor::~PointsProcessor() {

src/api/processor/points_processor.h

@@ -25,9 +25,9 @@ MYNTEYE_BEGIN_NAMESPACE
 
 class PointsProcessor : public Processor {
  public:
-  static constexpr auto &&NAME = "PointsProcessor";
+  static const char NAME[];
 
-  explicit PointsProcessor(cv::Mat Q);
+  explicit PointsProcessor(cv::Mat Q, std::int32_t proc_period = 0);
   virtual ~PointsProcessor();
 
   std::string Name() override;

src/api/processor/processor.cc

@@ -15,12 +15,17 @@
 
 #include <glog/logging.h>
 
+#include <exception>
 #include <utility>
 
+#include "internal/strings.h"
+#include "internal/times.h"
+
 MYNTEYE_BEGIN_NAMESPACE
 
-Processor::Processor()
-    : activated_(false),
+Processor::Processor(std::int32_t proc_period)
+    : proc_period_(std::move(proc_period)),
+      activated_(false),
       input_ready_(false),
       idle_(true),
       dropped_count_(0),
@@ -125,6 +130,10 @@ bool Processor::Process(const Object &in) {
       return false;
     }
   }
+  if (!in.DecValidity()) {
+    LOG(WARNING) << Name() << " process with invalid input";
+    return false;
+  }
   {
     std::lock_guard<std::mutex> lk(mtx_input_ready_);
     input_.reset(in.Clone());
@@ -134,9 +143,9 @@ bool Processor::Process(const Object &in) {
   return true;
 }
 
-Object *Processor::GetOutput() {
+std::shared_ptr<Object> Processor::GetOutput() {
   std::lock_guard<std::mutex> lk(mtx_result_);
-  return output_result_.get();
+  return std::shared_ptr<Object>(std::move(output_result_));
 }
 
 std::uint64_t Processor::GetDroppedCount() {
@@ -146,10 +155,34 @@ std::uint64_t Processor::GetDroppedCount() {
 
 void Processor::Run() {
   VLOG(2) << Name() << " thread start";
+
+  auto sleep = [this](const times::system_clock::time_point &time_beg) {
+    if (proc_period_ > 0) {
+      static times::system_clock::time_point time_prev = time_beg;
+      auto &&time_elapsed_ms =
+          times::count<times::milliseconds>(times::now() - time_prev);
+      time_prev = time_beg;
+
+      if (time_elapsed_ms < proc_period_) {
+        VLOG(2) << Name() << " process cost "
+                << times::count<times::milliseconds>(times::now() - time_beg)
+                << " ms, sleep " << (proc_period_ - time_elapsed_ms) << " ms";
+        std::this_thread::sleep_for(
+            std::chrono::milliseconds(proc_period_ - time_elapsed_ms));
+        return;
+      }
+    }
+    VLOG(2) << Name() << " process cost "
+            << times::count<times::milliseconds>(times::now() - time_beg)
+            << " ms";
+  };
+
   while (true) {
     std::unique_lock<std::mutex> lk(mtx_input_ready_);
     cond_input_ready_.wait(lk, [this] { return input_ready_; });
 
+    auto &&time_beg = times::now();
+
     if (!activated_) {
       SetIdle(true);
       input_ready_ = false;
@@ -165,14 +198,21 @@ void Processor::Run() {
       pre_callback_(input_.get());
     }
     bool ok = false;
-    if (callback_) {
-      if (callback_(input_.get(), output_.get(), parent_)) {
-        ok = true;
+    try {
+      if (callback_) {
+        if (callback_(input_.get(), output_.get(), parent_)) {
+          ok = true;
+        } else {
+          ok = OnProcess(input_.get(), output_.get(), parent_);
+        }
       } else {
         ok = OnProcess(input_.get(), output_.get(), parent_);
       }
-    } else {
-      ok = OnProcess(input_.get(), output_.get(), parent_);
+      // CV_Assert(false);
+    } catch (const std::exception &e) {
+      std::string msg(e.what());
+      strings::rtrim(msg);
+      LOG(ERROR) << Name() << " process error \"" << msg << "\"";
     }
     if (!ok) {
       VLOG(2) << Name() << " process failed";
@@ -194,6 +234,8 @@ void Processor::Run() {
 
     SetIdle(true);
     input_ready_ = false;
+
+    sleep(time_beg);
   }
   VLOG(2) << Name() << " thread end";
 }

src/api/processor/processor.h

@@ -37,7 +37,7 @@ class Processor /*: public std::enable_shared_from_this<Processor>*/ {
   using ProcessCallback = std::function<bool(
       Object *const in, Object *const out, Processor *const parent)>;
 
-  Processor();
+  explicit Processor(std::int32_t proc_period = 0);
   virtual ~Processor();
 
   virtual std::string Name();
@@ -65,7 +65,7 @@ class Processor /*: public std::enable_shared_from_this<Processor>*/ {
    * Returns the last output.
    * @note Returns null if not output now.
    */
-  Object *GetOutput();
+  std::shared_ptr<Object> GetOutput();
 
   std::uint64_t GetDroppedCount();
 
@@ -80,6 +80,8 @@ class Processor /*: public std::enable_shared_from_this<Processor>*/ {
 
   void SetIdle(bool idle);
 
+  std::int32_t proc_period_;
+
   bool activated_;
 
   bool input_ready_;

src/api/processor/rectify_processor.cc

@@ -18,16 +18,21 @@
 
 #include <glog/logging.h>
 
+#include <utility>
+
 #include "device/device.h"
 
 MYNTEYE_BEGIN_NAMESPACE
 
-RectifyProcessor::RectifyProcessor(std::shared_ptr<Device> device)
-    : Processor() {
-  VLOG(2) << __func__;
+const char RectifyProcessor::NAME[] = "RectifyProcessor";
+
+RectifyProcessor::RectifyProcessor(
+    std::shared_ptr<Device> device, std::int32_t proc_period)
+    : Processor(std::move(proc_period)) {
+  VLOG(2) << __func__ << ": proc_period=" << proc_period;
   InitParams(
       device->GetIntrinsics(Stream::LEFT), device->GetIntrinsics(Stream::RIGHT),
-      device->GetExtrinsics(Stream::LEFT, Stream::RIGHT));
+      device->GetExtrinsics(Stream::RIGHT, Stream::LEFT));
 }
 
 RectifyProcessor::~RectifyProcessor() {
@@ -53,7 +58,7 @@ bool RectifyProcessor::OnProcess(
 }
 
 void RectifyProcessor::InitParams(
-    Intrinsics in_left, Intrinsics in_right, Extrinsics ex_left_to_right) {
+    Intrinsics in_left, Intrinsics in_right, Extrinsics ex_right_to_left) {
   cv::Size size{in_left.width, in_left.height};
 
   cv::Mat M1 =
@@ -65,12 +70,12 @@ void RectifyProcessor::InitParams(
   cv::Mat D1(1, 5, CV_64F, in_left.coeffs);
   cv::Mat D2(1, 5, CV_64F, in_right.coeffs);
   cv::Mat R =
-      (cv::Mat_<double>(3, 3) << ex_left_to_right.rotation[0][0],
-       ex_left_to_right.rotation[0][1], ex_left_to_right.rotation[0][2],
-       ex_left_to_right.rotation[1][0], ex_left_to_right.rotation[1][1],
-       ex_left_to_right.rotation[1][2], ex_left_to_right.rotation[2][0],
-       ex_left_to_right.rotation[2][1], ex_left_to_right.rotation[2][2]);
-  cv::Mat T(3, 1, CV_64F, ex_left_to_right.translation);
+      (cv::Mat_<double>(3, 3) << ex_right_to_left.rotation[0][0],
+       ex_right_to_left.rotation[0][1], ex_right_to_left.rotation[0][2],
+       ex_right_to_left.rotation[1][0], ex_right_to_left.rotation[1][1],
+       ex_right_to_left.rotation[1][2], ex_right_to_left.rotation[2][0],
+       ex_right_to_left.rotation[2][1], ex_right_to_left.rotation[2][2]);
+  cv::Mat T(3, 1, CV_64F, ex_right_to_left.translation);
 
   VLOG(2) << "InitParams size: " << size;
   VLOG(2) << "M1: " << M1;

src/api/processor/rectify_processor.h

@@ -29,9 +29,10 @@ class Device;
 
 class RectifyProcessor : public Processor {
  public:
-  static constexpr auto &&NAME = "RectifyProcessor";
+  static const char NAME[];
 
-  explicit RectifyProcessor(std::shared_ptr<Device> device);
+  RectifyProcessor(
+      std::shared_ptr<Device> device, std::int32_t proc_period = 0);
   virtual ~RectifyProcessor();
 
   std::string Name() override;
@@ -46,7 +47,7 @@ class RectifyProcessor : public Processor {
 
  private:
   void InitParams(
-      Intrinsics in_left, Intrinsics in_right, Extrinsics ex_left_to_right);
+      Intrinsics in_left, Intrinsics in_right, Extrinsics ex_right_to_left);
 };
 
 MYNTEYE_END_NAMESPACE

src/api/synthetic.cc

@@ -29,6 +29,12 @@
 #include "api/processor/rectify_processor.h"
 #include "device/device.h"
 
+#define RECTIFY_PROC_PERIOD 0
+#define DISPARITY_PROC_PERIOD 0
+#define DISPARITY_NORM_PROC_PERIOD 0
+#define POINTS_PROC_PERIOD 0
+#define DEPTH_PROC_PERIOD 0
+
 MYNTEYE_BEGIN_NAMESPACE
 
 namespace {
@@ -40,7 +46,16 @@ cv::Mat frame2mat(const std::shared_ptr<device::Frame> &frame) {
 }
 
 api::StreamData data2api(const device::StreamData &data) {
-  return {data.img, frame2mat(data.frame)};
+  return {data.img, frame2mat(data.frame), data.frame};
+}
+
+void process_childs(
+    const std::shared_ptr<Processor> &proc, const std::string &name,
+    const Object &obj) {
+  auto &&processor = find_processor<Processor>(proc, name);
+  for (auto child : processor->GetChilds()) {
+    child->Process(obj);
+  }
 }
 
 }  // namespace
@@ -104,14 +119,16 @@ void Synthetic::StartVideoStreaming() {
     if (it->second == MODE_NATIVE) {
       auto &&stream = it->first;
       device->SetStreamCallback(
-          stream, [this, stream](const device::StreamData &data) {
+          stream,
+          [this, stream](const device::StreamData &data) {
             auto &&stream_data = data2api(data);
             ProcessNativeStream(stream, stream_data);
             // Need mutex if set callback after start
             if (HasStreamCallback(stream)) {
               stream_callbacks_.at(stream)(stream_data);
             }
-          });
+          },
+          true);
     }
   }
   device->Start(Source::VIDEO_STREAMING);
@@ -138,59 +155,59 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) {
     auto &&device = api_->device();
     return data2api(device->GetLatestStreamData(stream));
   } else if (mode == MODE_SYNTHETIC) {
+    if (stream == Stream::LEFT_RECTIFIED || stream == Stream::RIGHT_RECTIFIED) {
+      static std::shared_ptr<ObjMat2> output = nullptr;
+      auto &&processor = find_processor<RectifyProcessor>(processor_);
+      auto &&out = processor->GetOutput();
+      if (out != nullptr) {
+        // Obtain the output, out will be nullptr if get again immediately.
+        output = Object::Cast<ObjMat2>(out);
+      }
+      if (output != nullptr) {
+        if (stream == Stream::LEFT_RECTIFIED) {
+          return {nullptr, output->first, nullptr};
+        } else {
+          return {nullptr, output->second, nullptr};
+        }
+      }
+      VLOG(2) << "Rectify not ready now";
+      return {};
+    }
     switch (stream) {
-      case Stream::LEFT_RECTIFIED: {
-        auto &&processor = find_processor<RectifyProcessor>(processor_);
-        Object *out = processor->GetOutput();
-        if (out != nullptr) {
-          ObjMat2 *output = Object::Cast<ObjMat2>(out);
-          return {nullptr, output->first};
-        }
-        VLOG(2) << "Rectify not ready now";
-      } break;
-      case Stream::RIGHT_RECTIFIED: {
-        auto &&processor = find_processor<RectifyProcessor>(processor_);
-        Object *out = processor->GetOutput();
-        if (out != nullptr) {
-          ObjMat2 *output = Object::Cast<ObjMat2>(out);
-          return {nullptr, output->second};
-        }
-        VLOG(2) << "Rectify not ready now";
-      } break;
       case Stream::DISPARITY: {
         auto &&processor = find_processor<DisparityProcessor>(processor_);
-        Object *out = processor->GetOutput();
+        auto &&out = processor->GetOutput();
         if (out != nullptr) {
-          ObjMat *output = Object::Cast<ObjMat>(out);
-          return {nullptr, output->value};
+          auto &&output = Object::Cast<ObjMat>(out);
+          return {nullptr, output->value, nullptr};
         }
         VLOG(2) << "Disparity not ready now";
       } break;
       case Stream::DISPARITY_NORMALIZED: {
         auto &&processor =
             find_processor<DisparityNormalizedProcessor>(processor_);
-        Object *out = processor->GetOutput();
+        auto &&out = processor->GetOutput();
         if (out != nullptr) {
-          ObjMat *output = Object::Cast<ObjMat>(out);
-          return {nullptr, output->value};
+          auto &&output = Object::Cast<ObjMat>(out);
+          return {nullptr, output->value, nullptr};
         }
         VLOG(2) << "Disparity normalized not ready now";
       } break;
       case Stream::POINTS: {
         auto &&processor = find_processor<PointsProcessor>(processor_);
-        Object *out = processor->GetOutput();
+        auto &&out = processor->GetOutput();
         if (out != nullptr) {
-          ObjMat *output = Object::Cast<ObjMat>(out);
-          return {nullptr, output->value};
+          auto &&output = Object::Cast<ObjMat>(out);
+          return {nullptr, output->value, nullptr};
         }
         VLOG(2) << "Points not ready now";
       } break;
       case Stream::DEPTH: {
         auto &&processor = find_processor<DepthProcessor>(processor_);
-        Object *out = processor->GetOutput();
+        auto &&out = processor->GetOutput();
         if (out != nullptr) {
-          ObjMat *output = Object::Cast<ObjMat>(out);
-          return {nullptr, output->value};
+          auto &&output = Object::Cast<ObjMat>(out);
+          return {nullptr, output->value, nullptr};
         }
         VLOG(2) << "Depth not ready now";
       } break;
@@ -227,6 +244,10 @@ void Synthetic::SetPlugin(std::shared_ptr<Plugin> plugin) {
   plugin_ = plugin;
 }
 
+bool Synthetic::HasPlugin() const {
+  return plugin_ != nullptr;
+}
+
 void Synthetic::InitStreamSupports() {
   auto &&device = api_->device();
   if (device->Supports(Stream::LEFT) && device->Supports(Stream::RIGHT)) {
@@ -381,13 +402,16 @@ void Synthetic::DisableStreamData(const Stream &stream, std::uint32_t depth) {
 }
 
 void Synthetic::InitProcessors() {
-  auto &&rectify_processor = std::make_shared<RectifyProcessor>(api_->device());
-  auto &&disparity_processor = std::make_shared<DisparityProcessor>();
+  auto &&rectify_processor =
+      std::make_shared<RectifyProcessor>(api_->device(), RECTIFY_PROC_PERIOD);
+  auto &&disparity_processor =
+      std::make_shared<DisparityProcessor>(DISPARITY_PROC_PERIOD);
   auto &&disparitynormalized_processor =
-      std::make_shared<DisparityNormalizedProcessor>();
-  auto &&points_processor =
-      std::make_shared<PointsProcessor>(rectify_processor->Q);
-  auto &&depth_processor = std::make_shared<DepthProcessor>();
+      std::make_shared<DisparityNormalizedProcessor>(
+          DISPARITY_NORM_PROC_PERIOD);
+  auto &&points_processor = std::make_shared<PointsProcessor>(
+      rectify_processor->Q, POINTS_PROC_PERIOD);
+  auto &&depth_processor = std::make_shared<DepthProcessor>(DEPTH_PROC_PERIOD);
 
   using namespace std::placeholders;  // NOLINT
   rectify_processor->SetProcessCallback(
@@ -422,60 +446,50 @@ void Synthetic::InitProcessors() {
 
 void Synthetic::ProcessNativeStream(
     const Stream &stream, const api::StreamData &data) {
-  bool done = false;
-  static api::StreamData left_data, right_data;
-  if (stream == Stream::LEFT) {
-    left_data = data;
-    done = true;
-  } else if (stream == Stream::RIGHT) {
-    right_data = data;
-    done = true;
-  }
-  if (done && left_data.img && right_data.img &&
-      left_data.img->frame_id == right_data.img->frame_id) {
-    auto &&processor = find_processor<RectifyProcessor>(processor_);
-    processor->Process(ObjMat2{left_data.frame, right_data.frame});
-  }
-  if (done)
-    return;
-
-  auto &&process_childs = [this, &stream](
-      const std::string &name, const Object &obj) {
-    auto &&processor = find_processor<Processor>(processor_, name);
-    for (auto child : processor->GetChilds()) {
-      child->Process(obj);
+  if (stream == Stream::LEFT || stream == Stream::RIGHT) {
+    static api::StreamData left_data, right_data;
+    if (stream == Stream::LEFT) {
+      left_data = data;
+    } else if (stream == Stream::RIGHT) {
+      right_data = data;
+    }
+    if (left_data.img && right_data.img &&
+        left_data.img->frame_id == right_data.img->frame_id) {
+      auto &&processor = find_processor<RectifyProcessor>(processor_);
+      processor->Process(ObjMat2{left_data.frame, right_data.frame});
     }
-  };
-
-  static api::StreamData left_rect_data, right_rect_data;
-  if (stream == Stream::LEFT_RECTIFIED) {
-    left_rect_data = data;
-    done = true;
-  } else if (stream == Stream::RIGHT_RECTIFIED) {
-    right_rect_data = data;
-    done = true;
-  }
-  if (done && left_rect_data.img && right_rect_data.img &&
-      left_rect_data.img->frame_id == right_rect_data.img->frame_id) {
-    process_childs(
-        RectifyProcessor::NAME,
-        ObjMat2{left_rect_data.frame, right_rect_data.frame});
-  }
-  if (done)
     return;
+  }
+
+  if (stream == Stream::LEFT_RECTIFIED || stream == Stream::RIGHT_RECTIFIED) {
+    static api::StreamData left_rect_data, right_rect_data;
+    if (stream == Stream::LEFT_RECTIFIED) {
+      left_rect_data = data;
+    } else if (stream == Stream::RIGHT_RECTIFIED) {
+      right_rect_data = data;
+    }
+    if (left_rect_data.img && right_rect_data.img &&
+        left_rect_data.img->frame_id == right_rect_data.img->frame_id) {
+      process_childs(
+          processor_, RectifyProcessor::NAME,
+          ObjMat2{left_rect_data.frame, right_rect_data.frame});
+    }
+    return;
+  }
 
   switch (stream) {
     case Stream::DISPARITY: {
-      process_childs(DisparityProcessor::NAME, ObjMat{data.frame});
+      process_childs(processor_, DisparityProcessor::NAME, ObjMat{data.frame});
     } break;
     case Stream::DISPARITY_NORMALIZED: {
-      process_childs(DisparityNormalizedProcessor::NAME, ObjMat{data.frame});
+      process_childs(
+          processor_, DisparityNormalizedProcessor::NAME, ObjMat{data.frame});
     } break;
     case Stream::POINTS: {
-      process_childs(PointsProcessor::NAME, ObjMat{data.frame});
+      process_childs(processor_, PointsProcessor::NAME, ObjMat{data.frame});
     } break;
     case Stream::DEPTH: {
-      process_childs(DepthProcessor::NAME, ObjMat{data.frame});
+      process_childs(processor_, DepthProcessor::NAME, ObjMat{data.frame});
     } break;
     default:
       break;
@@ -531,17 +545,19 @@ bool Synthetic::OnDepthProcess(
 void Synthetic::OnRectifyPostProcess(Object *const out) {
   const ObjMat2 *output = Object::Cast<ObjMat2>(out);
   if (HasStreamCallback(Stream::LEFT_RECTIFIED)) {
-    stream_callbacks_.at(Stream::LEFT_RECTIFIED)({nullptr, output->first});
+    stream_callbacks_.at(Stream::LEFT_RECTIFIED)(
+        {nullptr, output->first, nullptr});
   }
   if (HasStreamCallback(Stream::RIGHT_RECTIFIED)) {
-    stream_callbacks_.at(Stream::RIGHT_RECTIFIED)({nullptr, output->second});
+    stream_callbacks_.at(Stream::RIGHT_RECTIFIED)(
+        {nullptr, output->second, nullptr});
   }
 }
 
 void Synthetic::OnDisparityPostProcess(Object *const out) {
   const ObjMat *output = Object::Cast<ObjMat>(out);
   if (HasStreamCallback(Stream::DISPARITY)) {
-    stream_callbacks_.at(Stream::DISPARITY)({nullptr, output->value});
+    stream_callbacks_.at(Stream::DISPARITY)({nullptr, output->value, nullptr});
   }
 }
 
@@ -549,21 +565,21 @@ void Synthetic::OnDisparityNormalizedPostProcess(Object *const out) {
   const ObjMat *output = Object::Cast<ObjMat>(out);
   if (HasStreamCallback(Stream::DISPARITY_NORMALIZED)) {
     stream_callbacks_.at(Stream::DISPARITY_NORMALIZED)(
-        {nullptr, output->value});
+        {nullptr, output->value, nullptr});
   }
 }
 
 void Synthetic::OnPointsPostProcess(Object *const out) {
   const ObjMat *output = Object::Cast<ObjMat>(out);
   if (HasStreamCallback(Stream::POINTS)) {
-    stream_callbacks_.at(Stream::POINTS)({nullptr, output->value});
+    stream_callbacks_.at(Stream::POINTS)({nullptr, output->value, nullptr});
   }
 }
 
 void Synthetic::OnDepthPostProcess(Object *const out) {
   const ObjMat *output = Object::Cast<ObjMat>(out);
   if (HasStreamCallback(Stream::DEPTH)) {
-    stream_callbacks_.at(Stream::DEPTH)({nullptr, output->value});
+    stream_callbacks_.at(Stream::DEPTH)({nullptr, output->value, nullptr});
   }
 }
 

src/api/synthetic.h

@@ -25,10 +25,11 @@
 MYNTEYE_BEGIN_NAMESPACE
 
 class API;
-class Object;
 class Plugin;
 class Processor;
 
+struct Object;
+
 class Synthetic {
  public:
   using stream_callback_t = API::stream_callback_t;
@@ -61,6 +62,7 @@ class Synthetic {
   std::vector<api::StreamData> GetStreamDatas(const Stream &stream);
 
   void SetPlugin(std::shared_ptr<Plugin> plugin);
+  bool HasPlugin() const;
 
  private:
   void InitStreamSupports();

src/device/device.cc

@@ -16,15 +16,18 @@
 #include <glog/logging.h>
 
 #include <algorithm>
+#include <iterator>
 #include <stdexcept>
 #include <utility>
 
 #include "device/device_s.h"
+#include "internal/async_callback.h"
 #include "internal/channels.h"
 #include "internal/config.h"
 #include "internal/motions.h"
 #include "internal/streams.h"
 #include "internal/strings.h"
+#include "internal/times.h"
 #include "internal/types.h"
 #include "uvc/uvc.h"
 
@@ -52,6 +55,27 @@ struct DeviceModel {
   }
 };
 
+bool CheckSupports(
+    const Device *const device, const Stream &stream, bool fatal = true) {
+  if (device->Supports(stream)) {
+    return true;
+  } else {
+    auto &&supports = stream_supports_map.at(device->GetModel());
+    std::ostringstream ss;
+    std::copy(
+        supports.begin(), supports.end(),
+        std::ostream_iterator<Stream>(ss, ", "));
+    if (fatal) {
+      LOG(FATAL) << "Unsupported stream: " << stream
+                 << ". Please use these: " << ss.str();
+    } else {
+      LOG(WARNING) << "Unsupported stream: " << stream
+                   << ". Please use these: " << ss.str();
+    }
+    return false;
+  }
+}
+
 }  // namespace
 
 Device::Device(const Model &model, std::shared_ptr<uvc::device> device)
@@ -112,7 +136,9 @@ bool Device::Supports(const AddOns &addon) const {
       return hw_flag[0];
     case AddOns::INFRARED2:
       return hw_flag[1];
-    default: { LOG(FATAL) << "Unknown add-on"; }
+    default:
+      LOG(WARNING) << "Unknown add-on";
+      return false;
   }
 }
 
@@ -135,8 +161,9 @@ void Device::ConfigStreamRequest(
   auto &&requests = GetStreamRequests(capability);
   if (std::find(requests.cbegin(), requests.cend(), request) ==
       requests.cend()) {
-    LOG(FATAL) << "Config stream request of " << capability
-               << " is not accpected";
+    LOG(WARNING) << "Config stream request of " << capability
+                 << " is not accpected";
+    return;
   }
   stream_config_requests_[capability] = request;
 }
@@ -148,50 +175,70 @@ std::shared_ptr<DeviceInfo> Device::GetInfo() const {
 std::string Device::GetInfo(const Info &info) const {
   CHECK_NOTNULL(device_info_);
   switch (info) {
-    case Info::DEVICE_NAME: {
+    case Info::DEVICE_NAME:
       return device_info_->name;
-    } break;
-    case Info::SERIAL_NUMBER: {
+    case Info::SERIAL_NUMBER:
       return device_info_->serial_number;
-    } break;
-    case Info::FIRMWARE_VERSION: {
+    case Info::FIRMWARE_VERSION:
       return device_info_->firmware_version.to_string();
-    } break;
-    case Info::HARDWARE_VERSION: {
+    case Info::HARDWARE_VERSION:
       return device_info_->hardware_version.to_string();
-    } break;
-    case Info::SPEC_VERSION: {
+    case Info::SPEC_VERSION:
       return device_info_->spec_version.to_string();
-    } break;
-    case Info::LENS_TYPE: {
+    case Info::LENS_TYPE:
       return device_info_->lens_type.to_string();
-    } break;
-    case Info::IMU_TYPE: {
+    case Info::IMU_TYPE:
       return device_info_->imu_type.to_string();
-    } break;
-    case Info::NOMINAL_BASELINE: {
+    case Info::NOMINAL_BASELINE:
       return std::to_string(device_info_->nominal_baseline);
-    } break;
-    default: { LOG(FATAL) << "Unknown device info"; }
+    default:
+      LOG(WARNING) << "Unknown device info";
+      return "";
   }
 }
 
 Intrinsics Device::GetIntrinsics(const Stream &stream) const {
+  bool ok;
+  return GetIntrinsics(stream, &ok);
+}
+
+Extrinsics Device::GetExtrinsics(const Stream &from, const Stream &to) const {
+  bool ok;
+  return GetExtrinsics(from, to, &ok);
+}
+
+MotionIntrinsics Device::GetMotionIntrinsics() const {
+  bool ok;
+  return GetMotionIntrinsics(&ok);
+}
+
+Extrinsics Device::GetMotionExtrinsics(const Stream &from) const {
+  bool ok;
+  return GetMotionExtrinsics(from, &ok);
+}
+
+Intrinsics Device::GetIntrinsics(const Stream &stream, bool *ok) const {
   try {
+    *ok = true;
     return stream_intrinsics_.at(stream);
   } catch (const std::out_of_range &e) {
+    *ok = false;
     LOG(WARNING) << "Intrinsics of " << stream << " not found";
     return {};
   }
 }
 
-Extrinsics Device::GetExtrinsics(const Stream &from, const Stream &to) const {
+Extrinsics Device::GetExtrinsics(
+    const Stream &from, const Stream &to, bool *ok) const {
   try {
+    *ok = true;
     return stream_from_extrinsics_.at(from).at(to);
   } catch (const std::out_of_range &e) {
     try {
+      *ok = true;
       return stream_from_extrinsics_.at(to).at(from).Inverse();
     } catch (const std::out_of_range &e) {
+      *ok = false;
       LOG(WARNING) << "Extrinsics from " << from << " to " << to
                    << " not found";
       return {};
@@ -199,19 +246,23 @@ Extrinsics Device::GetExtrinsics(const Stream &from, const Stream &to) const {
   }
 }
 
-MotionIntrinsics Device::GetMotionIntrinsics() const {
+MotionIntrinsics Device::GetMotionIntrinsics(bool *ok) const {
   if (motion_intrinsics_) {
+    *ok = true;
     return *motion_intrinsics_;
   } else {
+    *ok = false;
     LOG(WARNING) << "Motion intrinsics not found";
     return {};
   }
 }
 
-Extrinsics Device::GetMotionExtrinsics(const Stream &from) const {
+Extrinsics Device::GetMotionExtrinsics(const Stream &from, bool *ok) const {
   try {
+    *ok = true;
     return motion_from_extrinsics_.at(from);
   } catch (const std::out_of_range &e) {
+    *ok = false;
     LOG(WARNING) << "Motion extrinsics from " << from << " not found";
     return {};
   }
@@ -275,20 +326,32 @@ bool Device::RunOptionAction(const Option &option) const {
 }
 
 void Device::SetStreamCallback(
-    const Stream &stream, stream_callback_t callback) {
-  if (!Supports(stream)) {
-    LOG(WARNING) << "Unsupported stream: " << stream;
+    const Stream &stream, stream_callback_t callback, bool async) {
+  if (!CheckSupports(this, stream, false)) {
     return;
   }
   if (callback) {
     stream_callbacks_[stream] = callback;
+    if (async)
+      stream_async_callbacks_[stream] =
+          std::make_shared<stream_async_callback_t>(
+              to_string(stream), callback);  // max_data_size = 1
   } else {
     stream_callbacks_.erase(stream);
+    stream_async_callbacks_.erase(stream);
   }
 }
 
-void Device::SetMotionCallback(motion_callback_t callback) {
+void Device::SetMotionCallback(motion_callback_t callback, bool async) {
   motion_callback_ = callback;
+  if (callback) {
+    if (async)
+      motion_async_callback_ =
+          std::make_shared<motion_async_callback_t>("motion", callback, 1000);
+    // will drop old motion datas after callback cost > 2 s (1000 / 500 Hz)
+  } else {
+    motion_async_callback_ = nullptr;
+  }
 }
 
 bool Device::HasStreamCallback(const Stream &stream) const {
@@ -312,7 +375,7 @@ void Device::Start(const Source &source) {
     Start(Source::VIDEO_STREAMING);
     Start(Source::MOTION_TRACKING);
   } else {
-    LOG(FATAL) << "Unsupported source :(";
+    LOG(ERROR) << "Unsupported source :(";
   }
 }
 
@@ -327,7 +390,7 @@ void Device::Stop(const Source &source) {
     std::this_thread::sleep_for(std::chrono::milliseconds(10));
     Stop(Source::VIDEO_STREAMING);
   } else {
-    LOG(FATAL) << "Unsupported source :(";
+    LOG(ERROR) << "Unsupported source :(";
   }
 }
 
@@ -340,6 +403,7 @@ void Device::WaitForStreams() {
 std::vector<device::StreamData> Device::GetStreamDatas(const Stream &stream) {
   CHECK(video_streaming_);
   CHECK_NOTNULL(streams_);
+  CheckSupports(this, stream);
   std::lock_guard<std::mutex> _(mtx_streams_);
   return streams_->GetStreamDatas(stream);
 }
@@ -347,6 +411,7 @@ std::vector<device::StreamData> Device::GetStreamDatas(const Stream &stream) {
 device::StreamData Device::GetLatestStreamData(const Stream &stream) {
   CHECK(video_streaming_);
   CHECK_NOTNULL(streams_);
+  CheckSupports(this, stream);
   std::lock_guard<std::mutex> _(mtx_streams_);
   return streams_->GetLatestStreamData(stream);
 }
@@ -367,8 +432,8 @@ const StreamRequest &Device::GetStreamRequest(const Capabilities &capability) {
     return stream_config_requests_.at(capability);
   } catch (const std::out_of_range &e) {
     auto &&requests = GetStreamRequests(capability);
-    if (requests.size() == 1) {
-      VLOG(2) << "Get the only one stream request of " << capability;
+    if (requests.size() >= 1) {
+      VLOG(2) << "Select the first one stream request of " << capability;
       return requests[0];
     } else {
       LOG(FATAL) << "Please config the stream request of " << capability;
@@ -406,27 +471,27 @@ void Device::StartVideoStreaming() {
     uvc::set_device_mode(
         *device_, stream_request.width, stream_request.height,
         static_cast<int>(stream_request.format), stream_request.fps,
-        [this](const void *data) {
+        [this](const void *data, std::function<void()> continuation) {
           // drop the first stereo stream data
           static std::uint8_t drop_count = 1;
           if (drop_count > 0) {
             --drop_count;
+            continuation();
             return;
           }
-          std::lock_guard<std::mutex> _(mtx_streams_);
-          streams_->PushStream(Capabilities::STEREO, data);
-          if (HasStreamCallback(Stream::LEFT)) {
-            auto &&stream_datas = streams_->stream_datas(Stream::LEFT);
-            if (stream_datas.size() > 0) {
-              stream_callbacks_.at(Stream::LEFT)(stream_datas.back());
-            }
-          }
-          if (HasStreamCallback(Stream::RIGHT)) {
-            auto &&stream_datas = streams_->stream_datas(Stream::RIGHT);
-            if (stream_datas.size() > 0) {
-              stream_callbacks_.at(Stream::RIGHT)(stream_datas.back());
+          // auto &&time_beg = times::now();
+          {
+            std::lock_guard<std::mutex> _(mtx_streams_);
+            if (streams_->PushStream(Capabilities::STEREO, data)) {
+              CallbackPushedStreamData(Stream::LEFT);
+              CallbackPushedStreamData(Stream::RIGHT);
             }
           }
+          continuation();
+          OnStereoStreamUpdate();
+          // VLOG(2) << "Stereo video callback cost "
+          //     << times::count<times::milliseconds>(times::now() - time_beg)
+          //     << " ms";
         });
   } else {
     LOG(FATAL) << "Not any stream capabilities are supported by this device";
@@ -453,12 +518,9 @@ void Device::StartMotionTracking() {
     LOG(WARNING) << "Cannot start motion tracking without first stopping it";
     return;
   }
-  motions_->SetMotionCallback([this](const device::MotionData &data) {
-    if (motion_callback_) {
-      motion_callback_(data);
-    }
-  });
-  motions_->StartMotionTracking();
+  motions_->SetMotionCallback(
+      std::bind(&Device::CallbackMotionData, this, std::placeholders::_1));
+  // motions_->StartMotionTracking();
   motion_tracking_ = true;
 }
 
@@ -467,10 +529,12 @@ void Device::StopMotionTracking() {
     LOG(WARNING) << "Cannot stop motion tracking without first starting it";
     return;
   }
-  motions_->StopMotionTracking();
+  // motions_->StopMotionTracking();
   motion_tracking_ = false;
 }
 
+void Device::OnStereoStreamUpdate() {}
+
 void Device::ReadAllInfos() {
   device_info_ = std::make_shared<DeviceInfo>();
 
@@ -478,7 +542,13 @@ void Device::ReadAllInfos() {
   Channels::img_params_t img_params;
   Channels::imu_params_t imu_params;
   if (!channels_->GetFiles(device_info_.get(), &img_params, &imu_params)) {
-    LOG(FATAL) << "Read device infos failed :(";
+#if defined(WITH_DEVICE_INFO_REQUIRED)
+    LOG(FATAL)
+#else
+    LOG(WARNING)
+#endif
+        << "Read device infos failed. Please upgrade your firmware to the "
+           "latest version.";
   }
   VLOG(2) << "Device info: {name: " << device_info_->name
           << ", serial_number: " << device_info_->serial_number
@@ -495,11 +565,11 @@ void Device::ReadAllInfos() {
   if (img_params.ok) {
     SetIntrinsics(Stream::LEFT, img_params.in_left);
     SetIntrinsics(Stream::RIGHT, img_params.in_right);
-    SetExtrinsics(Stream::LEFT, Stream::RIGHT, img_params.ex_left_to_right);
+    SetExtrinsics(Stream::RIGHT, Stream::LEFT, img_params.ex_right_to_left);
     VLOG(2) << "Intrinsics left: {" << GetIntrinsics(Stream::LEFT) << "}";
     VLOG(2) << "Intrinsics right: {" << GetIntrinsics(Stream::RIGHT) << "}";
-    VLOG(2) << "Extrinsics left to right: {"
-            << GetExtrinsics(Stream::LEFT, Stream::RIGHT) << "}";
+    VLOG(2) << "Extrinsics right to left: {"
+            << GetExtrinsics(Stream::RIGHT, Stream::LEFT) << "}";
   } else {
     LOG(WARNING) << "Intrinsics & extrinsics not exist";
   }
@@ -514,4 +584,27 @@ void Device::ReadAllInfos() {
   }
 }
 
+void Device::CallbackPushedStreamData(const Stream &stream) {
+  if (HasStreamCallback(stream)) {
+    auto &&datas = streams_->stream_datas(stream);
+    // if (datas.size() > 0) {}
+    auto &&data = datas.back();
+    if (stream_async_callbacks_.find(stream) != stream_async_callbacks_.end()) {
+      stream_async_callbacks_.at(stream)->PushData(data);
+    } else {
+      stream_callbacks_.at(stream)(data);
+    }
+  }
+}
+
+void Device::CallbackMotionData(const device::MotionData &data) {
+  if (HasMotionCallback()) {
+    if (motion_async_callback_) {
+      motion_async_callback_->PushData(data);
+    } else {
+      motion_callback_(data);
+    }
+  }
+}
+
 MYNTEYE_END_NAMESPACE

src/device/device.h

@@ -47,6 +47,9 @@ class Channels;
 class Motions;
 class Streams;
 
+template <class Data>
+class AsyncCallback;
+
 /**
  * The Device class to communicate with MYNT® EYE device.
  */
@@ -59,6 +62,11 @@ class MYNTEYE_API Device {
 
   using stream_callbacks_t = std::map<Stream, stream_callback_t>;
 
+  using stream_async_callback_t = AsyncCallback<device::StreamData>;
+  using motion_async_callback_t = AsyncCallback<device::MotionData>;
+  using stream_async_callback_ptr_t = std::shared_ptr<stream_async_callback_t>;
+  using motion_async_callback_ptr_t = std::shared_ptr<motion_async_callback_t>;
+
   Device(const Model &model, std::shared_ptr<uvc::device> device);
   virtual ~Device();
 
@@ -132,6 +140,24 @@ class MYNTEYE_API Device {
    */
   Extrinsics GetMotionExtrinsics(const Stream &from) const;
 
+  /**
+   * Get the intrinsics of stream.
+   */
+  Intrinsics GetIntrinsics(const Stream &stream, bool *ok) const;
+  /**
+   * Get the extrinsics from one stream to another.
+   */
+  Extrinsics GetExtrinsics(
+      const Stream &from, const Stream &to, bool *ok) const;
+  /**
+   * Get the intrinsics of motion.
+   */
+  MotionIntrinsics GetMotionIntrinsics(bool *ok) const;
+  /**
+   * Get the extrinsics from one stream to motion.
+   */
+  Extrinsics GetMotionExtrinsics(const Stream &from, bool *ok) const;
+
   /**
    * Set the intrinsics of stream.
    */
@@ -176,11 +202,12 @@ class MYNTEYE_API Device {
   /**
    * Set the callback of stream.
    */
-  void SetStreamCallback(const Stream &stream, stream_callback_t callback);
+  void SetStreamCallback(
+      const Stream &stream, stream_callback_t callback, bool async = false);
   /**
    * Set the callback of motion.
    */
-  void SetMotionCallback(motion_callback_t callback);
+  void SetMotionCallback(motion_callback_t callback, bool async = false);
 
   /**
    * Has the callback of stream.
@@ -230,6 +257,18 @@ class MYNTEYE_API Device {
     return device_;
   }
 
+  std::shared_ptr<Streams> streams() const {
+    return streams_;
+  }
+
+  std::shared_ptr<Channels> channels() const {
+    return channels_;
+  }
+
+  std::shared_ptr<Motions> motions() const {
+    return motions_;
+  }
+
   const StreamRequest &GetStreamRequest(const Capabilities &capability);
 
   virtual void StartVideoStreaming();
@@ -238,6 +277,8 @@ class MYNTEYE_API Device {
   virtual void StartMotionTracking();
   virtual void StopMotionTracking();
 
+  virtual void OnStereoStreamUpdate();
+
   virtual std::vector<Stream> GetKeyStreams() const = 0;
 
   bool video_streaming_;
@@ -257,6 +298,9 @@ class MYNTEYE_API Device {
   stream_callbacks_t stream_callbacks_;
   motion_callback_t motion_callback_;
 
+  std::map<Stream, stream_async_callback_ptr_t> stream_async_callbacks_;
+  motion_async_callback_ptr_t motion_async_callback_;
+
   std::shared_ptr<Streams> streams_;
 
   std::map<Capabilities, StreamRequest> stream_config_requests_;
@@ -269,9 +313,8 @@ class MYNTEYE_API Device {
 
   void ReadAllInfos();
 
-  std::shared_ptr<Channels> channels() {
-    return channels_;
-  }
+  void CallbackPushedStreamData(const Stream &stream);
+  void CallbackMotionData(const device::MotionData &data);
 
   friend API;
   friend tools::DeviceWriter;

src/device/device_s.cc

@@ -15,6 +15,8 @@
 
 #include <glog/logging.h>
 
+#include "internal/motions.h"
+
 MYNTEYE_BEGIN_NAMESPACE
 
 StandardDevice::StandardDevice(std::shared_ptr<uvc::device> device)
@@ -30,4 +32,11 @@ std::vector<Stream> StandardDevice::GetKeyStreams() const {
   return {Stream::LEFT, Stream::RIGHT};
 }
 
+void StandardDevice::OnStereoStreamUpdate() {
+  if (motion_tracking_) {
+    auto &&motions = this->motions();
+    motions->DoMotionTrack();
+  }
+}
+
 MYNTEYE_END_NAMESPACE

src/device/device_s.h

@@ -28,6 +28,8 @@ class StandardDevice : public Device {
   virtual ~StandardDevice();
 
   std::vector<Stream> GetKeyStreams() const override;
+
+  void OnStereoStreamUpdate() override;
 };
 
 MYNTEYE_END_NAMESPACE

src/internal/async_callback.h

@@ -0,0 +1,62 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef MYNTEYE_INTERNAL_ASYNC_CALLBACK_H_  // NOLINT
+#define MYNTEYE_INTERNAL_ASYNC_CALLBACK_H_
+#pragma once
+
+#include <condition_variable>
+#include <functional>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "mynteye/mynteye.h"
+
+MYNTEYE_BEGIN_NAMESPACE
+
+template <class Data>
+class AsyncCallback {
+ public:
+  using callback_t = std::function<void(Data data)>;
+
+  AsyncCallback(
+      std::string name, callback_t callback, std::size_t max_data_size = 0);
+  ~AsyncCallback();
+
+  void PushData(Data data);
+
+ private:
+  void Run();
+
+  std::string name_;
+
+  callback_t callback_;
+
+  std::mutex mtx_;
+  std::condition_variable cv_;
+
+  bool running_;
+  std::thread thread_;
+
+  std::uint32_t count_;
+  std::vector<Data> datas_;
+  std::size_t max_data_size_;
+};
+
+MYNTEYE_END_NAMESPACE
+
+#include "internal/async_callback_impl.h"
+
+#endif  // MYNTEYE_INTERNAL_ASYNC_CALLBACK_H_ NOLINT

src/internal/async_callback_impl.h

@@ -0,0 +1,92 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef MYNTEYE_INTERNAL_ASYNC_CALLBACK_IMPL_H_  // NOLINT
+#define MYNTEYE_INTERNAL_ASYNC_CALLBACK_IMPL_H_
+#pragma once
+
+#include <glog/logging.h>
+
+#include <string>
+#include <utility>
+
+MYNTEYE_BEGIN_NAMESPACE
+
+template <class Data>
+AsyncCallback<Data>::AsyncCallback(
+    std::string name, callback_t callback, std::size_t max_data_size)
+    : name_(std::move(name)),
+      callback_(std::move(callback)),
+      count_(0),
+      max_data_size_(max_data_size) {
+  VLOG(2) << __func__;
+  running_ = true;
+  thread_ = std::thread(&AsyncCallback<Data>::Run, this);
+}
+
+template <class Data>
+AsyncCallback<Data>::~AsyncCallback() {
+  VLOG(2) << __func__;
+  {
+    std::lock_guard<std::mutex> _(mtx_);
+    running_ = false;
+    ++count_;
+  }
+  cv_.notify_one();
+  if (thread_.joinable()) {
+    thread_.join();
+  }
+}
+
+template <class Data>
+void AsyncCallback<Data>::PushData(Data data) {
+  std::lock_guard<std::mutex> _(mtx_);
+  if (max_data_size_ <= 0) {
+    datas_.clear();
+  } else if (max_data_size_ == datas_.size()) {  // >= 1
+    datas_.erase(datas_.begin());
+  }
+  datas_.push_back(data);
+  ++count_;
+  cv_.notify_one();
+}
+
+template <class Data>
+void AsyncCallback<Data>::Run() {
+  VLOG(2) << "AsyncCallback(" << name_ << ") thread start";
+  while (true) {
+    std::unique_lock<std::mutex> lock(mtx_);
+    cv_.wait(lock, [this] { return count_ > 0; });
+
+    if (!running_)
+      break;
+
+    if (callback_) {
+      for (auto &&data : datas_) {
+        callback_(data);
+      }
+    }
+
+    if (VLOG_IS_ON(2) && count_ > datas_.size()) {
+      VLOG(2) << "AsyncCallback(" << name_ << ") dropped "
+              << (count_ - datas_.size());
+    }
+    count_ = 0;
+    datas_.clear();
+  }
+  VLOG(2) << "AsyncCallback(" << name_ << ") thread end";
+}
+
+MYNTEYE_END_NAMESPACE
+
+#endif  // MYNTEYE_INTERNAL_ASYNC_CALLBACK_IMPL_H_ NOLINT

src/internal/channels.cc

@@ -31,6 +31,13 @@ MYNTEYE_BEGIN_NAMESPACE
 
 namespace {
 
+const uvc::xu mynteye_xu = {3,
+                            2,
+                            {0x947a6d9f,
+                             0x8a2f,
+                             0x418d,
+                             {0x85, 0x9e, 0x6c, 0x9a, 0xa0, 0x38, 0x10, 0x14}}};
+
 int XuCamCtrlId(Option option) {
   switch (option) {
     case Option::EXPOSURE_MODE:
@@ -177,7 +184,7 @@ std::int32_t Channels::GetControlValue(const Option &option) const {
       LOG(WARNING) << option << " get value useless";
       return -1;
     default:
-      LOG(FATAL) << "Unsupported option " << option;
+      LOG(ERROR) << "Unsupported option " << option;
   }
   return -1;
 }
@@ -217,7 +224,7 @@ void Channels::SetControlValue(const Option &option, std::int32_t value) {
     } break;
     case Option::FRAME_RATE: {
       if (!in_range() ||
-          !in_values({10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60}))
+          !in_values({10, 15, 20, 25, 30, 35, 40, 45, 50, 55}))
         break;
       XuCamCtrlSet(option, value);
     } break;
@@ -241,7 +248,7 @@ void Channels::SetControlValue(const Option &option, std::int32_t value) {
       LOG(WARNING) << option << " set value useless";
       break;
     default:
-      LOG(FATAL) << "Unsupported option " << option;
+      LOG(ERROR) << "Unsupported option " << option;
   }
 }
 
@@ -265,7 +272,8 @@ bool Channels::RunControlAction(const Option &option) const {
       LOG(WARNING) << option << " run action useless";
       return false;
     default:
-      LOG(FATAL) << "Unsupported option " << option;
+      LOG(ERROR) << "Unsupported option " << option;
+      return false;
   }
 }
 
@@ -273,6 +281,47 @@ void Channels::SetImuCallback(imu_callback_t callback) {
   imu_callback_ = callback;
 }
 
+void Channels::DoImuTrack() {
+  static ImuReqPacket req_packet{0};
+  static ImuResPacket res_packet;
+
+  req_packet.serial_number = imu_sn_;
+  if (!XuImuWrite(req_packet)) {
+    return;
+  }
+
+  if (!XuImuRead(&res_packet)) {
+    return;
+  }
+
+  if (res_packet.packets.size() == 0) {
+    return;
+  }
+
+  VLOG(2) << "Imu req sn: " << imu_sn_ << ", res count: " << []() {
+    std::size_t n = 0;
+    for (auto &&packet : res_packet.packets) {
+      n += packet.count;
+    }
+    return n;
+  }();
+
+  auto &&sn = res_packet.packets.back().serial_number;
+  if (imu_sn_ == sn) {
+    VLOG(2) << "New imu not ready, dropped";
+    return;
+  }
+  imu_sn_ = sn;
+
+  if (imu_callback_) {
+    for (auto &&packet : res_packet.packets) {
+      imu_callback_(packet);
+    }
+  }
+
+  res_packet.packets.clear();
+}
+
 void Channels::StartImuTracking(imu_callback_t callback) {
   if (is_imu_tracking_) {
     LOG(WARNING) << "Start imu tracking failed, is tracking already";
@@ -284,58 +333,20 @@ void Channels::StartImuTracking(imu_callback_t callback) {
   is_imu_tracking_ = true;
   imu_track_thread_ = std::thread([this]() {
     imu_sn_ = 0;
-    ImuReqPacket req_packet{imu_sn_};
-    ImuResPacket res_packet;
-    auto sleep_ms = [](std::intmax_t n) {
-      std::this_thread::sleep_for(std::chrono::milliseconds(n));
-    };
-    while (!imu_track_stop_) {
-      auto &&time_beg = times::now();
-
-      req_packet.serial_number = imu_sn_;
-      if (!XuImuWrite(req_packet)) {
-        continue;
-      }
-
-      if (!XuImuRead(&res_packet)) {
-        continue;
-      }
-
-      if (res_packet.packets.size() == 0) {
-        continue;
-      }
-
-      VLOG(2) << "Imu req sn: " << imu_sn_
-              << ", res count: " << [&res_packet]() {
-                   std::size_t n = 0;
-                   for (auto &&packet : res_packet.packets) {
-                     n += packet.count;
-                   }
-                   return n;
-                 }();
-
-      auto &&sn = res_packet.packets.back().serial_number;
-      if (imu_sn_ == sn) {
-        VLOG(2) << "New imu not ready, dropped";
-        continue;
-      }
-      imu_sn_ = sn;
-
-      if (imu_callback_) {
-        for (auto &&packet : res_packet.packets) {
-          imu_callback_(packet);
-        }
-      }
-
-      res_packet.packets.clear();
-
+    auto sleep = [](const times::system_clock::time_point &time_beg) {
       auto &&time_elapsed_ms =
           times::count<times::milliseconds>(times::now() - time_beg);
       if (time_elapsed_ms < IMU_TRACK_PERIOD) {
-        sleep_ms(IMU_TRACK_PERIOD - time_elapsed_ms);
+        std::this_thread::sleep_for(
+            std::chrono::milliseconds(IMU_TRACK_PERIOD - time_elapsed_ms));
         VLOG(2) << "Imu track cost " << time_elapsed_ms << " ms"
                 << ", sleep " << (IMU_TRACK_PERIOD - time_elapsed_ms) << " ms";
       }
+    };
+    while (!imu_track_stop_) {
+      auto &&time_beg = times::now();
+      DoImuTrack();
+      sleep(time_beg);
     }
   });
 }
@@ -504,7 +515,7 @@ std::size_t from_data(
   std::size_t i = 0;
   i += from_data(&img_params->in_left, data + i, spec_version);
   i += from_data(&img_params->in_right, data + i, spec_version);
-  i += from_data(&img_params->ex_left_to_right, data + i, spec_version);
+  i += from_data(&img_params->ex_right_to_left, data + i, spec_version);
   return i;
 }
 
@@ -776,7 +787,7 @@ std::size_t to_data(
   std::size_t i = 3;  // skip id, size
   i += to_data(&img_params->in_left, data + i, spec_version);
   i += to_data(&img_params->in_right, data + i, spec_version);
-  i += to_data(&img_params->ex_left_to_right, data + i, spec_version);
+  i += to_data(&img_params->ex_right_to_left, data + i, spec_version);
   // others
   std::size_t size = i - 3;
   data[0] = Channels::FID_IMG_PARAMS;
@@ -861,10 +872,23 @@ bool Channels::PuControlQuery(
   return uvc::pu_control_query(*device_, option, query, value);
 }
 
+bool Channels::XuControlRange(
+    channel_t channel, uint8_t id, int32_t *min, int32_t *max,
+    int32_t *def) const {
+  return XuControlRange(mynteye_xu, channel, id, min, max, def);
+}
+
+bool Channels::XuControlRange(
+    const uvc::xu &xu, uint8_t selector, uint8_t id, int32_t *min, int32_t *max,
+    int32_t *def) const {
+  CHECK_NOTNULL(device_);
+  return uvc::xu_control_range(*device_, xu, selector, id, min, max, def);
+}
+
 bool Channels::XuControlQuery(
     channel_t channel, uvc::xu_query query, uint16_t size,
     uint8_t *data) const {
-  return XuControlQuery({3}, channel >> 8, query, size, data);
+  return XuControlQuery(mynteye_xu, channel, query, size, data);
 }
 
 bool Channels::XuControlQuery(
@@ -995,32 +1019,12 @@ Channels::control_info_t Channels::PuControlInfo(Option option) const {
 Channels::control_info_t Channels::XuControlInfo(Option option) const {
   int id = XuCamCtrlId(option);
 
-  std::uint8_t data[3] = {static_cast<std::uint8_t>((id | 0x80) & 0xFF), 0, 0};
-  if (!XuCamCtrlQuery(uvc::XU_QUERY_SET, 3, data)) {
+  int32_t min = 0, max = 0, def = 0;
+  if (!XuControlRange(
+          CHANNEL_CAM_CTRL, static_cast<std::uint8_t>(id), &min, &max, &def)) {
     LOG(WARNING) << "Get XuControlInfo of " << option << " failed";
-    return {0, 0, 0};
   }
-
-  control_info_t info{0, 0, 0};
-
-  data[0] = id & 0xFF;
-  if (XuCamCtrlQuery(uvc::XU_QUERY_MIN, 3, data)) {
-    info.min = (data[1] << 8) | (data[2]);
-  } else {
-    LOG(WARNING) << "Get XuControlInfo.min of " << option << " failed";
-  }
-  if (XuCamCtrlQuery(uvc::XU_QUERY_MAX, 3, data)) {
-    info.max = (data[1] << 8) | (data[2]);
-  } else {
-    LOG(WARNING) << "Get XuControlInfo.max of " << option << " failed";
-  }
-  if (XuCamCtrlQuery(uvc::XU_QUERY_DEF, 3, data)) {
-    info.def = (data[1] << 8) | (data[2]);
-  } else {
-    LOG(WARNING) << "Get XuControlInfo.def of " << option << " failed";
-  }
-
-  return info;
+  return {min, max, def};
 }
 
 MYNTEYE_END_NAMESPACE

src/internal/channels.h

@@ -37,11 +37,11 @@ struct xu;
 class MYNTEYE_API Channels {
  public:
   typedef enum Channel {
-    CHANNEL_CAM_CTRL = 0x0100,
-    CHANNEL_HALF_DUPLEX = 0x0200,
-    CHANNEL_IMU_WRITE = 0x0300,
-    CHANNEL_IMU_READ = 0x0400,
-    CHANNEL_FILE = 0x0500,
+    CHANNEL_CAM_CTRL = 1,
+    CHANNEL_HALF_DUPLEX = 2,
+    CHANNEL_IMU_WRITE = 3,
+    CHANNEL_IMU_READ = 4,
+    CHANNEL_FILE = 5,
     CHANNEL_LAST
   } channel_t;
 
@@ -72,7 +72,7 @@ class MYNTEYE_API Channels {
     bool ok;
     Intrinsics in_left;
     Intrinsics in_right;
-    Extrinsics ex_left_to_right;
+    Extrinsics ex_right_to_left;
   } img_params_t;
 
   typedef struct ImuParams {
@@ -95,6 +95,8 @@ class MYNTEYE_API Channels {
   bool RunControlAction(const Option &option) const;
 
   void SetImuCallback(imu_callback_t callback);
+  void DoImuTrack();
+
   void StartImuTracking(imu_callback_t callback = nullptr);
   void StopImuTracking();
 
@@ -110,6 +112,13 @@ class MYNTEYE_API Channels {
       Option option, int32_t *min, int32_t *max, int32_t *def) const;
   bool PuControlQuery(Option option, uvc::pu_query query, int32_t *value) const;
 
+  bool XuControlRange(
+      channel_t channel, uint8_t id, int32_t *min, int32_t *max,
+      int32_t *def) const;
+  bool XuControlRange(
+      const uvc::xu &xu, uint8_t selector, uint8_t id, int32_t *min,
+      int32_t *max, int32_t *def) const;
+
   bool XuControlQuery(
       channel_t channel, uvc::xu_query query, uint16_t size,
       uint8_t *data) const;

src/internal/motions.cc

@@ -34,11 +34,8 @@ Motions::~Motions() {
 
 void Motions::SetMotionCallback(motion_callback_t callback) {
   motion_callback_ = callback;
-}
-
-void Motions::StartMotionTracking() {
-  if (!is_imu_tracking) {
-    channels_->StartImuTracking([this](const ImuPacket &packet) {
+  if (motion_callback_) {
+    channels_->SetImuCallback([this](const ImuPacket &packet) {
       if (!motion_callback_ && !motion_datas_enabled_) {
         LOG(WARNING) << "";
         return;
@@ -62,11 +59,22 @@ void Motions::StartMotionTracking() {
         std::lock_guard<std::mutex> _(mtx_datas_);
         motion_data_t data = {imu};
         motion_datas_.push_back(data);
-        if (motion_callback_) {
-          motion_callback_(data);
-        }
+
+        motion_callback_(data);
       }
     });
+  } else {
+    channels_->SetImuCallback(nullptr);
+  }
+}
+
+void Motions::DoMotionTrack() {
+  channels_->DoImuTrack();
+}
+
+void Motions::StartMotionTracking() {
+  if (!is_imu_tracking) {
+    channels_->StartImuTracking();
     is_imu_tracking = true;
   } else {
     LOG(WARNING) << "Imu is tracking already";

src/internal/motions.h

@@ -37,6 +37,7 @@ class Motions {
   ~Motions();
 
   void SetMotionCallback(motion_callback_t callback);
+  void DoMotionTrack();
 
   void StartMotionTracking();
   void StopMotionTracking();

src/internal/streams.cc

@@ -125,18 +125,20 @@ Streams::~Streams() {
 void Streams::ConfigStream(
     const Capabilities &capability, const StreamRequest &request) {
   if (!IsStreamCapability(capability)) {
-    LOG(FATAL) << "Cannot config stream without stream capability";
+    LOG(ERROR) << "Cannot config stream without stream capability";
+    return;
   }
   VLOG(2) << "Config stream request of " << capability << ", " << request;
   stream_config_requests_[capability] = request;
 }
 
-void Streams::PushStream(const Capabilities &capability, const void *data) {
+bool Streams::PushStream(const Capabilities &capability, const void *data) {
   if (!HasStreamConfigRequest(capability)) {
     LOG(FATAL) << "Cannot push stream without stream config request";
   }
   std::unique_lock<std::mutex> lock(mtx_);
   auto &&request = GetStreamConfigRequest(capability);
+  bool pushed = false;
   switch (capability) {
     case Capabilities::STEREO: {
       // alloc left
@@ -154,10 +156,12 @@ void Streams::PushStream(const Capabilities &capability, const void *data) {
             data, request, left_data.frame.get());
         unpack_img_pixels_map_[Stream::RIGHT](
             data, request, right_data.frame.get());
+        pushed = true;
       } else {
         // discard left
         DiscardStreamData(Stream::LEFT);
         LOG(WARNING) << "Image packet is unaccepted, frame dropped";
+        pushed = false;
       }
     } break;
     default:
@@ -165,13 +169,15 @@ void Streams::PushStream(const Capabilities &capability, const void *data) {
   }
   if (HasKeyStreamDatas())
     cv_.notify_one();
+  return pushed;
 }
 
 void Streams::WaitForStreams() {
   std::unique_lock<std::mutex> lock(mtx_);
   auto ready = std::bind(&Streams::HasKeyStreamDatas, this);
   if (!ready() && !cv_.wait_for(lock, std::chrono::seconds(2), ready)) {
-    LOG(FATAL) << "Timeout waiting for key frames";
+    LOG(FATAL) << "Timeout waiting for key frames. Please use USB 3.0, and not "
+                  "in virtual machine.";
   }
 }
 
@@ -191,21 +197,30 @@ std::size_t Streams::GetStreamDataMaxSize(const Stream &stream) const {
 
 Streams::stream_datas_t Streams::GetStreamDatas(const Stream &stream) {
   std::unique_lock<std::mutex> lock(mtx_);
-  if (!HasStreamDatas(stream) || stream_datas_map_.at(stream).empty()) {
-    LOG(WARNING) << "There are stream datas of " << stream
-                 << ", do you first call WaitForStreams?";
+  if (!HasStreamDatas(stream)) {
+    LOG(WARNING) << "There are no stream datas of " << stream
+                 << ". Did you call WaitForStreams() before this?";
     return {};
   }
-  stream_datas_t datas = stream_datas_map_.at(stream);
+  auto datas = stream_datas_map_.at(stream);
   stream_datas_map_[stream].clear();
   return datas;
 }
 
 Streams::stream_data_t Streams::GetLatestStreamData(const Stream &stream) {
-  return GetStreamDatas(stream).back();
+  std::unique_lock<std::mutex> lock(mtx_);
+  if (!HasStreamDatas(stream)) {
+    LOG(WARNING) << "There are no stream datas of " << stream
+                 << ". Did you call WaitForStreams() before this?";
+    return {};
+  }
+  auto data = stream_datas_map_.at(stream).back();
+  stream_datas_map_[stream].clear();
+  return data;
 }
 
 const Streams::stream_datas_t &Streams::stream_datas(const Stream &stream) {
+  std::unique_lock<std::mutex> lock(mtx_);
   try {
     return stream_datas_map_.at(stream);
   } catch (const std::out_of_range &e) {
@@ -232,7 +247,8 @@ const StreamRequest &Streams::GetStreamConfigRequest(
 }
 
 bool Streams::HasStreamDatas(const Stream &stream) const {
-  return stream_datas_map_.find(stream) != stream_datas_map_.end();
+  return stream_datas_map_.find(stream) != stream_datas_map_.end() &&
+         !stream_datas_map_.at(stream).empty();
 }
 
 void Streams::AllocStreamData(
@@ -243,27 +259,38 @@ void Streams::AllocStreamData(
 void Streams::AllocStreamData(
     const Stream &stream, const StreamRequest &request, const Format &format) {
   stream_data_t data;
+
+  if (HasStreamDatas(stream)) {
+    // If cached equal to limits_max, drop the oldest one.
+    if (stream_datas_map_.at(stream).size() == GetStreamDataMaxSize(stream)) {
+      auto &&datas = stream_datas_map_[stream];
+      // reuse the dropped data
+      data.img = datas.front().img;
+      data.frame = datas.front().frame;
+      datas.erase(datas.begin());
+      VLOG(2) << "Stream data of " << stream << " is dropped as out of limits";
+    }
+  }
+
   if (stream == Stream::LEFT || stream == Stream::RIGHT) {
-    data.img = std::make_shared<ImgData>();
+    if(!data.img) {
+      data.img = std::make_shared<ImgData>();
+    }
   } else {
     data.img = nullptr;
   }
-  data.frame =
-      std::make_shared<frame_t>(request.width, request.height, format, nullptr);
-  stream_datas_map_[stream].push_back(data);
-  // If cached more then limits_max, drop the oldest one.
-  if (stream_datas_map_.at(stream).size() > GetStreamDataMaxSize(stream)) {
-    auto &&datas = stream_datas_map_[stream];
-    datas.erase(datas.begin());
-    VLOG(2) << "Stream data of " << stream << " is dropped as out of limits";
+  if (!data.frame) {
+    data.frame = std::make_shared<frame_t>(
+        request.width, request.height, format, nullptr);
   }
+  stream_datas_map_[stream].push_back(data);
 }
 
 void Streams::DiscardStreamData(const Stream &stream) {
   // Must discard after alloc, otherwise at will out of range when no this key.
   if (stream_datas_map_.at(stream).size() > 0) {
     auto &&datas = stream_datas_map_[stream];
-    datas.erase(datas.end() - 1);
+    datas.pop_back();
   } else {
     VLOG(2) << "Stream data of " << stream << " is empty, could not discard";
   }
@@ -273,8 +300,6 @@ bool Streams::HasKeyStreamDatas() const {
   for (auto &&s : key_streams_) {
     if (!HasStreamDatas(s))
       return false;
-    if (stream_datas_map_.at(s).empty())
-      return false;
   }
   return true;
 }

src/internal/streams.h

@@ -44,7 +44,7 @@ class Streams {
   void ConfigStream(
       const Capabilities &capability, const StreamRequest &request);
 
-  void PushStream(const Capabilities &capability, const void *data);
+  bool PushStream(const Capabilities &capability, const void *data);
 
   void WaitForStreams();
 

src/internal/strings.cc

@@ -65,6 +65,13 @@ bool starts_with(const std::string &text, const std::string &prefix) {
   return text.compare(0, prefix.length(), prefix) == 0;
 }
 
+bool ends_with(const std::string &text, const std::string &suffix) {
+  if (suffix.length() > text.length())
+    return false;
+  return text.compare(
+             text.length() - suffix.length(), suffix.length(), suffix) == 0;
+}
+
 std::vector<std::string> split(
     const std::string &text, const std::string &delimiters) {
   std::vector<std::string> tokens;

src/internal/strings.h

@@ -41,6 +41,9 @@ int hex2int(const std::string &text);
 MYNTEYE_API
 bool starts_with(const std::string &text, const std::string &prefix);
 
+MYNTEYE_API
+bool ends_with(const std::string &text, const std::string &suffix);
+
 MYNTEYE_API
 std::vector<std::string> split(
     const std::string &text, const std::string &delimiters);

src/public/utils.cc

@@ -36,8 +36,9 @@ std::shared_ptr<Device> select() {
   LOG(INFO) << "MYNT EYE devices:";
   for (size_t i = 0; i < n; i++) {
     auto &&device = devices[i];
-    auto &&name = device->GetInfo(Info::DEVICE_NAME);
-    LOG(INFO) << "  index: " << i << ", name: " << name;
+    LOG(INFO) << "  index: " << i
+              << ", name: " << device->GetInfo(Info::DEVICE_NAME)
+              << ", sn: " << device->GetInfo(Info::SERIAL_NUMBER);
   }
 
   std::shared_ptr<Device> device = nullptr;
@@ -99,9 +100,6 @@ float get_real_exposure_time(
     case 55:
       real_max = 16.325;
       break;
-    case 60:
-      real_max = 15;
-      break;
     default:
       LOG(ERROR) << "Invalid frame rate: " << frame_rate;
       return exposure_time;

src/uvc/README.md

@@ -0,0 +1,10 @@
+
+## `uvc-v4l2.cc`
+
+* [Linux Media Subsystem Documentation](https://www.kernel.org/doc/html/latest/media/index.html)
+  * [The Linux USB Video Class (UVC) driver](https://www.kernel.org/doc/html/latest/media/v4l-drivers/uvcvideo.html)
+
+## `uvc-wmf.cc`
+
+* [Media Foundation](https://msdn.microsoft.com/en-us/library/ms694197(VS.85).aspx)
+* [USB Video Class Driver](https://docs.microsoft.com/en-us/windows-hardware/drivers/stream/usb-video-class-driver)

src/uvc/uvc-libuvc.cc

@@ -144,6 +144,20 @@ bool pu_control_query(
   return false;
 }
 
+bool xu_control_range(
+    const device &device, const xu &xu, uint8_t selector, uint8_t id, int32_t *min,
+    int32_t *max, int32_t *def) {
+  // TODO(JohnZhao)
+  UNUSED(device)
+  UNUSED(xu)
+  UNUSED(selector)
+  UNUSED(id)
+  UNUSED(min)
+  UNUSED(max)
+  UNUSED(def)
+  return false;
+}
+
 bool xu_control_query(
     const device &device, const xu &xu, uint8_t selector, xu_query query,
     uint16_t size, uint8_t *data) {

src/uvc/uvc-v4l2.cc

@@ -276,7 +276,7 @@ struct device {
 
     // Init memory mapped IO
     v4l2_requestbuffers req;
-    req.count = 4;
+    req.count = 24;
     req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
     req.memory = V4L2_MEMORY_MMAP;
     if (xioctl(fd, VIDIOC_REQBUFS, &req) < 0) {
@@ -381,11 +381,11 @@ struct device {
       }
 
       if (callback) {
-        callback(buffers[buf.index].start);
+        callback(buffers[buf.index].start, [buf, this]() mutable {
+          if (xioctl(fd, VIDIOC_QBUF, &buf) < 0)
+            throw_error("VIDIOC_QBUF");
+        });
       }
-
-      if (xioctl(fd, VIDIOC_QBUF, &buf) < 0)
-        LOG_ERROR(FATAL, "VIDIOC_QBUF");
     }
   }
 
@@ -504,6 +504,39 @@ bool pu_control_query(
   return device.pu_control_query(get_cid(option), code, value);
 }
 
+bool xu_control_range(
+    const device &device, const xu &xu, uint8_t selector, uint8_t id,
+    int32_t *min, int32_t *max, int32_t *def) {
+  bool ret = true;
+
+  std::uint8_t data[3]{static_cast<uint8_t>(id | 0x80), 0, 0};
+
+  if (!xu_control_query(device, xu, selector, XU_QUERY_SET, 3, data)) {
+    LOG(WARNING) << "xu_control_range query failed";
+    ret = false;
+  }
+
+  if (xu_control_query(device, xu, selector, XU_QUERY_MIN, 3, data)) {
+    *min = (data[1] << 8) | (data[2]);
+  } else {
+    LOG(WARNING) << "xu_control_range query min failed";
+    ret = false;
+  }
+  if (xu_control_query(device, xu, selector, XU_QUERY_MAX, 3, data)) {
+    *max = (data[1] << 8) | (data[2]);
+  } else {
+    LOG(WARNING) << "xu_control_range query max failed";
+    ret = false;
+  }
+  if (xu_control_query(device, xu, selector, XU_QUERY_DEF, 3, data)) {
+    *def = (data[1] << 8) | (data[2]);
+  } else {
+    LOG(WARNING) << "xu_control_range query def failed";
+    ret = false;
+  }
+  return ret;
+}
+
 bool xu_control_query(
     const device &device, const xu &xu, uint8_t selector, xu_query query,
     uint16_t size, uint8_t *data) {

src/uvc/uvc-wmf.cc

@@ -13,44 +13,476 @@
 // limitations under the License.
 #include "uvc/uvc.h"  // NOLINT
 
+#include <windows.h>
+#include <usbioctl.h>
+
+#include <Shlwapi.h>        // For QISearch, etc.
+#include <mfapi.h>          // For MFStartup, etc.
+#include <mfidl.h>          // For MF_DEVSOURCE_*, etc.
+#include <mfreadwrite.h>    // MFCreateSourceReaderFromMediaSource
+#include <mferror.h>
+
+#pragma comment(lib, "Shlwapi.lib")
+#pragma comment(lib, "mf.lib")
+#pragma comment(lib, "mfplat.lib")
+#pragma comment(lib, "mfreadwrite.lib")
+#pragma comment(lib, "mfuuid.lib")
+
+#pragma comment(lib, "setupapi.lib")
+#pragma comment(lib, "winusb.lib")
+
+#include <uuids.h>
+#include <vidcap.h>
+#include <ksmedia.h>
+#include <ksproxy.h>
+
+#include <Cfgmgr32.h>
+
+#pragma comment(lib, "cfgmgr32.lib")
+
+#include <SetupAPI.h>
+#include <WinUsb.h>
+
+#include <algorithm>
+#include <chrono>
+#include <functional>
+#include <map>
+#include <regex>
+#include <sstream>
+#include <thread>
+
+#include <strsafe.h>
+
 #include <glog/logging.h>
 
+#define VLOG_INFO VLOG(2)
+// #define VLOG_INFO LOG(INFO)
+
 MYNTEYE_BEGIN_NAMESPACE
 
 namespace uvc {
 
-struct context {
-  context() {
-    VLOG(2) << __func__;
+const std::map<uint32_t, uint32_t> fourcc_map = {
+  { 0x56595559, 0x32595559 },  // 'VYUY' => '2YUY'
+  { 0x59555956, 0x59555932 }   // 'YUYV' => 'YUY2'
+};
+
+struct throw_error {
+  throw_error() = default;
+
+  explicit throw_error(const std::string &s) {
+    ss << s;
   }
 
-  ~context() {
-    VLOG(2) << __func__;
+  ~throw_error() noexcept(false) {
+    throw std::runtime_error(ss.str());
   }
+
+  template<class T>
+  throw_error &operator<<(const T &val) {
+    ss << val;
+    return *this;
+  }
+
+  std::ostringstream ss;
+};
+
+static void check(const char *call, HRESULT hr) {
+  if (FAILED(hr)) {
+    throw_error() << call << "(...) returned 0x" << std::hex
+      << static_cast<uint32_t>(hr);
+  } else {
+    // VLOG_INFO << call << " SUCCESSED";
+  }
+}
+
+template<class T> class com_ptr {
+  T *p;
+  void ref(T *new_p) {
+    if (p == new_p) return;
+    unref();
+    p = new_p;
+    if (p) p->AddRef();
+  }
+
+  void unref() {
+    if (p) {
+      p->Release();
+      p = nullptr;
+    }
+  }
+public:
+  com_ptr() : p() {}
+  com_ptr(T *p) : com_ptr() {
+    ref(p);
+  }
+  com_ptr(const com_ptr &r) : com_ptr(r.p) {}
+  ~com_ptr() {
+    unref();
+  }
+
+  operator T *() const {
+    return p;
+  }
+  T &operator*() const {
+    return *p;
+  }
+  T *operator->() const {
+    return p;
+  }
+
+  T **operator&() {
+    unref();
+    return &p;
+  }
+  com_ptr &operator=(const com_ptr &r) {
+    ref(r.p);
+    return *this;
+  }
+};
+
+static std::string win_to_utf(const WCHAR *s) {
+  int len = WideCharToMultiByte(CP_UTF8, 0, s, -1, nullptr, 0, NULL, NULL);
+  if (len == 0) throw_error() << "WideCharToMultiByte(...) returned 0 and GetLastError() is " << GetLastError();
+  std::string buffer(len - 1, ' ');
+  len = WideCharToMultiByte(CP_UTF8, 0, s, -1, &buffer[0], (int)buffer.size()+1, NULL, NULL);
+  if (len == 0) throw_error() << "WideCharToMultiByte(...) returned 0 and GetLastError() is " << GetLastError();
+  return buffer;
+}
+
+std::vector<std::string> tokenize(std::string string, char separator) {
+  std::vector<std::string> tokens;
+  std::string::size_type i1 = 0;
+  while (true) {
+    auto i2 = string.find(separator, i1);
+    if (i2 == std::string::npos) {
+      tokens.push_back(string.substr(i1));
+      return tokens;
+    }
+    tokens.push_back(string.substr(i1, i2-i1));
+    i1 = i2 + 1;
+  }
+}
+
+/*
+static void print_guid(const char *call, int i, GUID guid) {
+  std::ostringstream ss;
+  ss << call << "(" << i << ") = ";
+  ss << "Data1: " << std::hex << guid.Data1 << ", ";
+  ss << "Data2: " << std::hex << guid.Data2 << ", ";
+  ss << "Data3: " << std::hex << guid.Data3 << ", ";
+  ss << "Data4: [ ";
+  for (int j = 0; j < 8; j++) {
+    ss << std::hex << (int)guid.Data4[j] << " ";
+  }
+  ss << "]";
+  LOG(INFO) << ss.str();
+}
+*/
+
+bool parse_usb_path(int &vid, int &pid, int &mi, std::string &unique_id, const std::string &path) {
+  auto name = path;
+  std::transform(begin(name), end(name), begin(name), ::tolower);
+  auto tokens = tokenize(name, '#');
+  if (tokens.size() < 1 || tokens[0] != R"(\\?\usb)") return false;  // Not a USB device
+  if (tokens.size() < 3) {
+    LOG(ERROR) << "malformed usb device path: " << name;
+    return false;
+  }
+
+  auto ids = tokenize(tokens[1], '&');
+  if (ids[0].size() != 8 || ids[0].substr(0,4) != "vid_" || !(std::istringstream(ids[0].substr(4,4)) >> std::hex >> vid)) {
+    LOG(ERROR) << "malformed vid string: " << tokens[1];
+    return false;
+  }
+
+  if (ids[1].size() != 8 || ids[1].substr(0,4) != "pid_" || !(std::istringstream(ids[1].substr(4,4)) >> std::hex >> pid)) {
+    LOG(ERROR) << "malformed pid string: " << tokens[1];
+    return false;
+  }
+
+  if (ids[2].size() != 5 || ids[2].substr(0,3) != "mi_" || !(std::istringstream(ids[2].substr(3,2)) >> mi)) {
+    LOG(ERROR) << "malformed mi string: " << tokens[1];
+    return false;
+  }
+
+  ids = tokenize(tokens[2], '&');
+  if (ids.size() < 2) {
+    LOG(ERROR) << "malformed id string: " << tokens[2];
+    return false;
+  }
+  unique_id = ids[1];
+  return true;
+}
+
+bool parse_usb_path_from_device_id(int &vid, int &pid, int &mi, std::string &unique_id, const std::string &device_id) {
+  auto name = device_id;
+  std::transform(begin(name), end(name), begin(name), ::tolower);
+  auto tokens = tokenize(name, '\\');
+  if (tokens.size() < 1 || tokens[0] != R"(usb)") return false;  // Not a USB device
+
+  auto ids = tokenize(tokens[1], '&');
+  if (ids[0].size() != 8 || ids[0].substr(0, 4) != "vid_" || !(std::istringstream(ids[0].substr(4, 4)) >> std::hex >> vid)) {
+    LOG(ERROR) << "malformed vid string: " << tokens[1];
+    return false;
+  }
+
+  if (ids[1].size() != 8 || ids[1].substr(0, 4) != "pid_" || !(std::istringstream(ids[1].substr(4, 4)) >> std::hex >> pid)) {
+    LOG(ERROR) << "malformed pid string: " << tokens[1];
+    return false;
+  }
+
+  if (ids[2].size() != 5 || ids[2].substr(0, 3) != "mi_" || !(std::istringstream(ids[2].substr(3, 2)) >> mi)) {
+    LOG(ERROR) << "malformed mi string: " << tokens[1];
+    return false;
+  }
+
+  ids = tokenize(tokens[2], '&');
+  if (ids.size() < 2) {
+    LOG(ERROR) << "malformed id string: " + tokens[2];
+    return false;
+  }
+  unique_id = ids[1];
+  return true;
+}
+
+struct context {
+  context() {
+    CoInitializeEx(NULL, COINIT_APARTMENTTHREADED);
+    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    MFStartup(MF_VERSION, MFSTARTUP_NOSOCKET);
+  }
+  ~context() {
+    MFShutdown();
+    CoUninitialize();
+  }
+};
+
+class reader_callback : public IMFSourceReaderCallback {
+  std::weak_ptr<device> owner;  // The device holds a reference to us, so use
+                                // weak_ptr to prevent a cycle
+  ULONG ref_count;
+  volatile bool streaming = false;
+public:
+  reader_callback(std::weak_ptr<device> owner) : owner(owner), ref_count() {}
+
+  bool is_streaming() const {
+    return streaming;
+  }
+  void on_start() {
+    streaming = true;
+  }
+
+#pragma warning( push )
+#pragma warning( disable: 4838 )
+  // Implement IUnknown
+  HRESULT STDMETHODCALLTYPE QueryInterface(REFIID riid, void **ppvObject) override {
+    static const QITAB table[] = {QITABENT(reader_callback, IUnknown), QITABENT(reader_callback, IMFSourceReaderCallback), {0}};
+    return QISearch(this, table, riid, ppvObject);
+  }
+#pragma warning( pop )
+
+  ULONG STDMETHODCALLTYPE AddRef() override { return InterlockedIncrement(&ref_count); }
+  ULONG STDMETHODCALLTYPE Release() override {
+    ULONG count = InterlockedDecrement(&ref_count);
+    if (count == 0) delete this;
+    return count;
+  }
+
+  // Implement IMFSourceReaderCallback
+  HRESULT STDMETHODCALLTYPE OnReadSample(HRESULT hrStatus, DWORD dwStreamIndex, DWORD dwStreamFlags, LONGLONG llTimestamp, IMFSample *sample) override;
+  HRESULT STDMETHODCALLTYPE OnFlush(DWORD dwStreamIndex) override { streaming = false; return S_OK; }
+  HRESULT STDMETHODCALLTYPE OnEvent(DWORD dwStreamIndex, IMFMediaEvent *pEvent) override { return S_OK; }
 };
 
 struct device {
   const std::shared_ptr<context> parent;
-
   int vid, pid;
+  std::string unique_id;
+  std::string name;
 
-  explicit device(std::shared_ptr<context> parent) : parent(parent) {
-    VLOG(2) << __func__;
+  com_ptr<reader_callback> reader_callback;
+  com_ptr<IMFActivate> mf_activate;
+  com_ptr<IMFMediaSource> mf_media_source;
+  com_ptr<IAMCameraControl> am_camera_control;
+  com_ptr<IAMVideoProcAmp> am_video_proc_amp;
+  std::map<int, com_ptr<IKsControl>> ks_controls;
+  com_ptr<IMFSourceReader> mf_source_reader;
+  video_channel_callback callback = nullptr;
+
+  device(std::shared_ptr<context> parent, int vid, int pid, std::string unique_id, std::string name)
+      : parent(move(parent)), vid(vid), pid(pid), unique_id(move(unique_id)), name(name) {
   }
 
   ~device() {
-    VLOG(2) << __func__;
+    stop_streaming();
   }
+
+
+  IKsControl *get_ks_control(const uvc::xu &xu) {
+    auto it = ks_controls.find(xu.node);
+    if (it != end(ks_controls)) return it->second;
+
+    get_media_source();
+
+    // Attempt to retrieve IKsControl
+    com_ptr<IKsTopologyInfo> ks_topology_info = NULL;
+    check("QueryInterface", mf_media_source->QueryInterface(__uuidof(IKsTopologyInfo), (void **)&ks_topology_info));
+
+    GUID node_type;
+    /*
+    DWORD numberOfNodes;
+    check("get_NumNodes", ks_topology_info->get_NumNodes(&numberOfNodes));
+    for (int i = 0; i < numberOfNodes; i++) {
+      check("get_NodeType", ks_topology_info->get_NodeType(i, &node_type));
+      print_guid("node_type", i, node_type);
+    }
+    */
+    check("get_NodeType", ks_topology_info->get_NodeType(xu.node, &node_type));
+    const GUID KSNODETYPE_DEV_SPECIFIC_LOCAL{0x941C7AC0L, 0xC559, 0x11D0, {0x8A, 0x2B, 0x00, 0xA0, 0xC9, 0x25, 0x5A, 0xC1}};
+    if (node_type != KSNODETYPE_DEV_SPECIFIC_LOCAL) throw_error() << "Invalid extension unit node ID: " << xu.node;
+
+    com_ptr<IUnknown> unknown;
+    check("CreateNodeInstance", ks_topology_info->CreateNodeInstance(xu.node, IID_IUnknown, (LPVOID *)&unknown));
+
+    com_ptr<IKsControl> ks_control;
+    check("QueryInterface", unknown->QueryInterface(__uuidof(IKsControl), (void **)&ks_control));
+    VLOG_INFO << "Obtained KS control node : " << xu.node;
+    return ks_controls[xu.node] = ks_control;
+  }
+
+  void start_streaming() {
+    if (mf_source_reader) {
+      reader_callback->on_start();
+      check("IMFSourceReader::ReadSample", mf_source_reader->ReadSample(MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, NULL, NULL, NULL, NULL));
+    }
+  }
+
+  void stop_streaming() {
+    if (mf_source_reader) mf_source_reader->Flush(MF_SOURCE_READER_FIRST_VIDEO_STREAM);
+
+    while (true) {
+      bool is_streaming = reader_callback->is_streaming();
+      if (is_streaming) std::this_thread::sleep_for(std::chrono::milliseconds(10));
+      else break;
+    }
+
+    mf_source_reader = nullptr;
+    am_camera_control = nullptr;
+    am_video_proc_amp = nullptr;
+    ks_controls.clear();
+    if (mf_media_source) {
+      mf_media_source = nullptr;
+      check("IMFActivate::ShutdownObject", mf_activate->ShutdownObject());
+    }
+    callback = {};
+  }
+
+  com_ptr<IMFMediaSource> get_media_source() {
+    if (!mf_media_source) {
+      check("IMFActivate::ActivateObject", mf_activate->ActivateObject(__uuidof(IMFMediaSource), (void **)&mf_media_source));
+      if (mf_media_source) {
+        check("IMFMediaSource::QueryInterface", mf_media_source->QueryInterface(__uuidof(IAMCameraControl), (void **)&am_camera_control));
+        if (SUCCEEDED(mf_media_source->QueryInterface(__uuidof(IAMVideoProcAmp), (void **)&am_video_proc_amp)));
+      } else throw_error() << "Invalid media source";
+    }
+    return mf_media_source;
+  }
+
 };
 
+HRESULT reader_callback::OnReadSample(HRESULT hrStatus, DWORD dwStreamIndex, DWORD dwStreamFlags, LONGLONG llTimestamp, IMFSample *sample) {
+  if (auto owner_ptr = owner.lock()) {
+    if (sample) {
+      com_ptr<IMFMediaBuffer> buffer = NULL;
+      if (SUCCEEDED(sample->GetBufferByIndex(0, &buffer))) {
+        BYTE *byte_buffer;
+        DWORD max_length, current_length;
+        if (SUCCEEDED(buffer->Lock(&byte_buffer, &max_length, &current_length))) {
+          auto continuation = [buffer, this]() {
+            buffer->Unlock();
+          };
+          owner_ptr->callback(byte_buffer, continuation);
+        }
+      }
+    }
+
+    if (auto owner_ptr_new = owner.lock()) {
+      auto hr = owner_ptr_new->mf_source_reader->ReadSample(MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, NULL, NULL, NULL, NULL);
+      switch (hr) {
+        case S_OK: break;
+        case MF_E_INVALIDREQUEST: LOG(ERROR) << "ReadSample returned MF_E_INVALIDREQUEST"; break;
+        case MF_E_INVALIDSTREAMNUMBER: LOG(ERROR) << "ReadSample returned MF_E_INVALIDSTREAMNUMBER"; break;
+        case MF_E_NOTACCEPTING: LOG(ERROR) << "ReadSample returned MF_E_NOTACCEPTING"; break;
+        case E_INVALIDARG: LOG(ERROR) << "ReadSample returned E_INVALIDARG"; break;
+        case MF_E_VIDEO_RECORDING_DEVICE_INVALIDATED: LOG(ERROR) << "ReadSample returned MF_E_VIDEO_RECORDING_DEVICE_INVALIDATED"; break;
+        default: LOG(ERROR) << "ReadSample returned HRESULT " << std::hex << (uint32_t)hr; break;
+      }
+      if (hr != S_OK) streaming = false;
+    }
+  }
+  return S_OK;
+}
+
 std::shared_ptr<context> create_context() {
   return std::make_shared<context>();
 }
 
-std::vector<std::shared_ptr<device>> query_devices(
-    std::shared_ptr<context> context) {
+std::vector<std::shared_ptr<device>> query_devices(std::shared_ptr<context> context) {
+  IMFAttributes *pAttributes = NULL;
+  check("MFCreateAttributes", MFCreateAttributes(&pAttributes, 1));
+  check("IMFAttributes::SetGUID", pAttributes->SetGUID(
+    MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID));
+
+  IMFActivate **ppDevices;
+  UINT32 numDevices;
+  check("MFEnumDeviceSources", MFEnumDeviceSources(pAttributes, &ppDevices, &numDevices));
+
   std::vector<std::shared_ptr<device>> devices;
-  UNUSED(context)
+  for (UINT32 i = 0; i < numDevices; ++i) {
+    com_ptr<IMFActivate> pDevice;
+    *&pDevice = ppDevices[i];
+
+    WCHAR *wchar_dev_name = NULL;
+    WCHAR *wchar_name = NULL;
+    UINT32 length;
+
+    pDevice->GetAllocatedString(
+      MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK, &wchar_dev_name,
+      &length);
+    auto dev_name = win_to_utf(wchar_dev_name);
+    CoTaskMemFree(wchar_dev_name);
+
+    pDevice->GetAllocatedString(MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME, &wchar_name, &length);
+    auto name = win_to_utf(wchar_name);  // Device description name
+    CoTaskMemFree(wchar_name);
+
+    int vid, pid, mi;
+    std::string unique_id;
+
+    if (!parse_usb_path(vid, pid, mi, unique_id, dev_name)) continue;
+
+    std::shared_ptr<device> dev;
+    for (auto & d : devices) {
+      if (d->vid == vid && d->pid == pid && d->unique_id == unique_id)
+        dev = d;
+    }
+    if (!dev) {
+      try {
+        dev = std::make_shared<device>(context, vid, pid, unique_id, name);
+        devices.push_back(dev);
+      } catch (const std::exception &e) {
+        VLOG_INFO << "Not a USB video device: " << e.what();
+      }
+    }
+
+    dev->reader_callback = new reader_callback(dev);
+    dev->mf_activate = pDevice;
+    dev->vid = vid;
+    dev->pid = pid;
+  }
+
+  CoTaskMemFree(ppDevices);
   return devices;
 }
 
@@ -63,65 +495,286 @@ int get_product_id(const device &device) {
 }
 
 std::string get_name(const device &device) {
-  UNUSED(device)
-  return "";
+  return device.name;
 }
 
 std::string get_video_name(const device &device) {
-  UNUSED(device)
-  return "";
+  return device.name;
+}
+
+static long get_cid(Option option) {
+  switch (option) {
+    case Option::GAIN:
+      return VideoProcAmp_Gain;
+    case Option::BRIGHTNESS:
+      return VideoProcAmp_Brightness;
+    case Option::CONTRAST:
+      return VideoProcAmp_Contrast;
+    default:
+      LOG(FATAL) << "No VideoProcAmp cid for " << option;
+  }
 }
 
 bool pu_control_range(
     const device &device, Option option, int32_t *min, int32_t *max,
     int32_t *def) {
-  UNUSED(device)
-  UNUSED(option)
-  UNUSED(min)
-  UNUSED(max)
-  UNUSED(def)
-  return false;
+  VLOG_INFO << __func__ << " " << option;
+  const_cast<uvc::device &>(device).get_media_source();
+  long minVal = 0, maxVal = 0, steppingDelta = 0, defVal = 0, capsFlag = 0;
+  check("IAMVideoProcAmp::GetRange",
+      const_cast<uvc::device &>(device).am_video_proc_amp->GetRange(
+        get_cid(option), &minVal, &maxVal, &steppingDelta, &defVal, &capsFlag));
+  if (min) *min = static_cast<int>(minVal);
+  if (max) *max = static_cast<int>(maxVal);
+  if (def) *def = static_cast<int>(defVal);
+  VLOG_INFO << __func__ << " " << option <<
+      ": min=" << *min << ", max=" << *max << ", def=" << *def;
+  return true;
+}
+
+static void pu_control_get(const device &device, long property, int32_t *value) {
+  long data, flags = 0;
+  check("IAMVideoProcAmp::Get",
+      const_cast<uvc::device &>(device).am_video_proc_amp->Get(
+        property, &data, &flags));
+  *value = data;
+}
+
+static void pu_control_set(const device &device, long property, int32_t *value) {
+  long data = *value;
+  check("IAMVideoProcAmp::Set",
+      const_cast<uvc::device &>(device).am_video_proc_amp->Set(
+        property, data, VideoProcAmp_Flags_Auto));
 }
 
 bool pu_control_query(
     const device &device, Option option, pu_query query, int32_t *value) {
-  UNUSED(device)
-  UNUSED(option)
-  UNUSED(query)
-  UNUSED(value)
-  return false;
+  CHECK_NOTNULL(value);
+  const_cast<uvc::device &>(device).get_media_source();
+  switch (query) {
+    case PU_QUERY_SET:
+      VLOG_INFO << "pu_control_set " << option << ": " << *value;
+      pu_control_set(device, get_cid(option), value);
+      VLOG_INFO << "pu_control_set " << option << " done";
+      return true;
+    case PU_QUERY_GET:
+      VLOG_INFO << "pu_control_get " << option;
+      pu_control_get(device, get_cid(option), value);
+      VLOG_INFO << "pu_control_get " << option << ": " << *value;
+      return true;
+    default:
+      LOG(ERROR) << "pu_control_query request code is unaccepted";
+      return false;
+  }
+}
+
+static std::string to_string(uint16_t size, uint8_t *data) {
+  std::ostringstream ss;
+  for (uint8_t *beg = data, *end = data + size; beg != end; beg++) {
+    ss << "0x" << std::hex << static_cast<int>(*beg) << ",";
+  }
+  return ss.str();
+}
+
+/*
+static std::vector<BYTE> xu_control_desc(const device &device, const xu &xu, ULONG id, ULONG flags) {
+  auto ks_control = const_cast<uvc::device &>(device).get_ks_control(xu);
+
+  KSP_NODE node;
+  memset(&node, 0, sizeof(KSP_NODE));
+  node.Property.Set = reinterpret_cast<const GUID &>(xu.id);
+  node.Property.Id = id;
+  node.Property.Flags = flags;
+  node.NodeId = xu.node;
+
+  KSPROPERTY_DESCRIPTION description;
+  ULONG bytes_received = 0;
+  check("IKsControl::KsProperty", ks_control->KsProperty(
+      (PKSPROPERTY)&node,
+      sizeof(node),
+      &description,
+      sizeof(KSPROPERTY_DESCRIPTION),
+      &bytes_received));
+
+  ULONG size = description.DescriptionSize;
+  std::vector<BYTE> buffer(size);
+
+  check("IKsControl::KsProperty", ks_control->KsProperty(
+      (PKSPROPERTY)&node,
+      sizeof(node),
+      buffer.data(),
+      size,
+      &bytes_received));
+
+  if (bytes_received != size) { throw_error() << "wrong data"; }
+
+  // VLOG_INFO << "buffer size=" << size << ", data=["
+  //     << to_string(size, buffer.data()) << "]";
+  return buffer;
+}
+
+bool xu_control_range(
+    const device &device, const xu &xu, uint8_t selector, int32_t *min,
+    int32_t *max, int32_t *def) {
+  VLOG_INFO << __func__ << " " << static_cast<int>(selector);
+  size_t prop_header_size = sizeof(KSPROPERTY_MEMBERSHEADER) + sizeof(KSPROPERTY_DESCRIPTION);
+  // get step, min and max values
+  {
+    auto &&buffer = xu_control_desc(device, xu, selector,
+        KSPROPERTY_TYPE_BASICSUPPORT | KSPROPERTY_TYPE_TOPOLOGY);
+
+    BYTE *values = buffer.data() + prop_header_size;
+
+    // size_t size = buffer.size() - prop_header_size;
+    // VLOG_INFO << "values size: " << size << ", data=["
+    //     << to_string(size, values) << "]";
+
+    *min = (values[1] << 8) | (values[2]);
+    values += 3;
+    *max = (values[1] << 8) | (values[2]);
+    // values += 3;
+    // *step = (values[1] << 8) | (values[2]);
+  }
+  // get def value
+  {
+    auto &&buffer = xu_control_desc(device, xu, selector,
+        KSPROPERTY_TYPE_DEFAULTVALUES | KSPROPERTY_TYPE_TOPOLOGY);
+
+    BYTE *values = buffer.data() + prop_header_size;
+
+    // size_t size = buffer.size() - prop_header_size;
+    // VLOG_INFO << "values size: " << size << ", data=["
+    //     << to_string(size, values) << "]";
+
+    *def = (values[1] << 8) | (values[2]);
+  }
+  VLOG_INFO << __func__ << " " << static_cast<int>(selector)
+      << ": min=" << *min << ", max=" << *max << ", def=" << *def;
+  return true;
+}
+*/
+
+static void xu_control_get(const device &device, const xu &xu, uint8_t selector,
+      uint16_t size, uint8_t *data) {
+  VLOG_INFO << __func__ << " " << static_cast<int>(selector);
+  auto &&ks_control = const_cast<uvc::device &>(device).get_ks_control(xu);
+
+  KSP_NODE node;
+  memset(&node, 0, sizeof(KSP_NODE));
+  node.Property.Set = reinterpret_cast<const GUID &>(xu.id);
+  node.Property.Id = selector;
+  node.Property.Flags = KSPROPERTY_TYPE_GET | KSPROPERTY_TYPE_TOPOLOGY;
+  node.NodeId = xu.node;
+
+  ULONG bytes_received = 0;
+  check("IKsControl::KsProperty", ks_control->KsProperty(
+      (PKSPROPERTY)&node, sizeof(node), data, size, &bytes_received));
+
+  if (bytes_received != size)
+    throw_error() << "xu_control_get did not return enough data";
+  VLOG_INFO << __func__ << " " << static_cast<int>(selector)
+      << ": size=" << size << ", data=[" << to_string(size, data) << "]";
+}
+
+static void xu_control_set(const device &device, const xu &xu, uint8_t selector,
+      uint16_t size, uint8_t *data) {
+  VLOG_INFO << __func__ << " " << static_cast<int>(selector)
+      << ": size=" << size << ", data=[" << to_string(size, data) << "]";
+  auto &&ks_control = const_cast<uvc::device &>(device).get_ks_control(xu);
+
+  KSP_NODE node;
+  memset(&node, 0, sizeof(KSP_NODE));
+  node.Property.Set = reinterpret_cast<const GUID &>(xu.id);
+  node.Property.Id = selector;
+  node.Property.Flags = KSPROPERTY_TYPE_SET | KSPROPERTY_TYPE_TOPOLOGY;
+  node.NodeId = xu.node;
+
+  ULONG bytes_received = 0;
+  check("IKsControl::KsProperty", ks_control->KsProperty(
+      (PKSPROPERTY)&node, sizeof(node), data, size, &bytes_received));
+  VLOG_INFO << __func__ << " " << static_cast<int>(selector) << " done";
+}
+
+static int32_t xu_control_range_basic(const device &device, const xu &xu, uint8_t selector, uint8_t id) {
+  std::uint8_t data[3]{id, 0, 0};
+  xu_control_set(device, xu, selector, 3, data);
+  xu_control_get(device, xu, selector, 3, data);
+  return (data[1] << 8) | (data[2]);
+}
+
+bool xu_control_range(
+    const device &device, const xu &xu, uint8_t selector, uint8_t id,
+    int32_t *min, int32_t *max, int32_t *def) {
+  VLOG_INFO << __func__ << " " << static_cast<int>(selector);
+  *min = xu_control_range_basic(
+      device, xu, selector, static_cast<uint8_t>(id | 0x90));
+  *max = xu_control_range_basic(
+      device, xu, selector, static_cast<uint8_t>(id | 0xa0));
+  *def = xu_control_range_basic(
+      device, xu, selector, static_cast<uint8_t>(id | 0xc0));
+  return true;
 }
 
 bool xu_control_query(
     const device &device, const xu &xu, uint8_t selector, xu_query query,
     uint16_t size, uint8_t *data) {
-  UNUSED(device)
-  UNUSED(xu)
-  UNUSED(selector)
-  UNUSED(query)
-  UNUSED(size)
-  UNUSED(data)
-  return false;
+  CHECK_NOTNULL(data);
+  switch (query) {
+    case XU_QUERY_SET:
+      xu_control_set(device, xu, selector, size, data);
+      return true;
+    case XU_QUERY_GET:
+      xu_control_get(device, xu, selector, size, data);
+      return true;
+    default:
+      LOG(ERROR) << "xu_control_query request code is unaccepted";
+      return false;
+  }
 }
 
-void set_device_mode(
-    device &device, int width, int height, int fourcc, int fps,  // NOLINT
-    video_channel_callback callback) {
-  UNUSED(device)
-  UNUSED(width)
-  UNUSED(height)
-  UNUSED(fourcc)
-  UNUSED(fps)
-  UNUSED(callback)
+void set_device_mode(device &device, int width, int height, int fourcc, int fps, video_channel_callback callback) {
+  if (!device.mf_source_reader) {
+    com_ptr<IMFAttributes> pAttributes;
+    check("MFCreateAttributes", MFCreateAttributes(&pAttributes, 1));
+    check("IMFAttributes::SetUnknown", pAttributes->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK, static_cast<IUnknown *>(device.reader_callback)));
+    check("MFCreateSourceReaderFromMediaSource", MFCreateSourceReaderFromMediaSource(device.get_media_source(), pAttributes, &device.mf_source_reader));
+  }
+
+  if (fourcc_map.count(fourcc)) fourcc = fourcc_map.at(fourcc);
+
+  for (DWORD j = 0; ; j++) {
+    com_ptr<IMFMediaType> media_type;
+    HRESULT hr = device.mf_source_reader->GetNativeMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, j, &media_type);
+    if (hr == MF_E_NO_MORE_TYPES) break;
+    check("IMFSourceReader::GetNativeMediaType", hr);
+
+    UINT32 uvc_width, uvc_height, uvc_fps_num, uvc_fps_denom;
+    GUID subtype;
+    check("MFGetAttributeSize", MFGetAttributeSize(media_type, MF_MT_FRAME_SIZE, &uvc_width, &uvc_height));
+    if (uvc_width != width || uvc_height != height) continue;
+
+    check("IMFMediaType::GetGUID", media_type->GetGUID(MF_MT_SUBTYPE, &subtype));
+    if (subtype.Data1 != fourcc) continue;
+
+    check("MFGetAttributeRatio", MFGetAttributeRatio(media_type, MF_MT_FRAME_RATE, &uvc_fps_num, &uvc_fps_denom));
+    if (uvc_fps_denom == 0) continue;
+    //int uvc_fps = uvc_fps_num / uvc_fps_denom;
+    //LOG(INFO) << "uvc_fps: " << uvc_fps;
+
+    check("IMFSourceReader::SetCurrentMediaType", device.mf_source_reader->SetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, NULL, media_type));
+
+    device.callback = callback;
+    return;
+  }
+  throw_error() << "no matching media type for pixel format " << std::hex << fourcc;
 }
 
-void start_streaming(device &device, int num_transfer_bufs) {  // NOLINT
-  UNUSED(device)
-  UNUSED(num_transfer_bufs)
+void start_streaming(device &device, int num_transfer_bufs) {
+  device.start_streaming();
 }
 
-void stop_streaming(device &device) {  // NOLINT
-  UNUSED(device)
+void stop_streaming(device &device) {
+  device.stop_streaming();
 }
 
 }  // namespace uvc

src/uvc/uvc.h

@@ -36,9 +36,17 @@ typedef enum pu_query {
   PU_QUERY_LAST
 } pu_query;
 
+struct MYNTEYE_API guid {
+  uint32_t data1;
+  uint16_t data2, data3;
+  uint8_t data4[8];
+};
+
 // Extension Unit
 struct MYNTEYE_API xu {
   uint8_t unit;
+  int node;
+  guid id;
 };
 
 typedef enum xu_query {
@@ -77,12 +85,16 @@ MYNTEYE_API bool pu_control_query(
     const device &device, Option option, pu_query query, int32_t *value);
 
 // Access XU (Extension Unit) controls
-MYNTEYE_API bool xu_control_query(
+MYNTEYE_API bool xu_control_range(
+    const device &device, const xu &xu, uint8_t selector, uint8_t id,
+    int32_t *min, int32_t *max, int32_t *def);
+MYNTEYE_API bool xu_control_query(  // XU_QUERY_SET, XU_QUERY_GET
     const device &device, const xu &xu, uint8_t selector, xu_query query,
     uint16_t size, uint8_t *data);
 
 // Control streaming
-typedef std::function<void(const void *frame)> video_channel_callback;
+typedef std::function<void(const void *frame,
+    std::function<void()> continuation)> video_channel_callback;
 
 MYNTEYE_API void set_device_mode(
     device &device, int width, int height, int fourcc, int fps,  // NOLINT

tools/README.md

@@ -69,7 +69,7 @@ roslaunch mynt_eye_ros_wrapper mynteye.launch
 ```
 
 ```bash
-rosbag record -O mynteye.bag /mynteye/left /mynteye/imu /mynteye/temp
+rosbag record -O mynteye.bag /mynteye/left/image_raw /mynteye/imu/data_raw /mynteye/temp/data_raw
 ```
 
 ## Analytics data (rosbag)

tools/analytics/imu_analytics.py

@@ -650,8 +650,8 @@ def _main():
     else:
       dataset = ROSBag(
           path,
-          topic_imu='/mynteye/imu',
-          topic_temp='/mynteye/temp')
+          topic_imu='/mynteye/imu/data_raw',
+          topic_temp='/mynteye/temp/data_raw')
     return dataset
 
   if args.binary:

tools/analytics/stamp_analytics.py

@@ -334,8 +334,8 @@ def _main():
         sys.exit('Error: dataset model not supported {}'.format(model))
     else:
       dataset = ROSBag(path,
-                       topic_img_left='/mynteye/left',
-                       topic_imu='/mynteye/imu')
+                       topic_img_left='/mynteye/left/image_raw',
+                       topic_imu='/mynteye/imu/data_raw')
     return dataset
 
   dataset = BinDataset(dataset_path, dataset_creator)

tools/checksum/md5sum.sh

@@ -21,16 +21,24 @@ source "$SCRIPTS_DIR/common/echo.sh"
 source "$SCRIPTS_DIR/common/host.sh"
 
 if [ "$HOST_OS" = "Linux" ]; then
-  MD5SUM="md5sum"
+  _md5sum() { md5sum "$1"; }
 elif [ "$HOST_OS" = "Mac" ]; then
-  MD5SUM="md5 -q"
+  _md5sum() { md5 -q "$1"; }
+elif [ "$HOST_OS" = "Win" ]; then
+  _md5sum() { certutil -hashfile "$1" MD5; }
 else  # unexpected
   _echo_e "Unknown host os :("
   exit 1
 fi
 
+PYTHON="python"
+if [ "$HOST_OS" = "Win" ]; then
+  # default python on MSYS
+  PYTHON="python2"
+fi
+
 _get_size() {
-PYTHON_ARG="$1" python - <<EOF
+PYTHON_ARG="$1" $PYTHON - <<EOF
 import math
 from os.path import getsize
 
@@ -52,7 +60,7 @@ _print_info() {
   file="$1"
   _echo "File: $file"
   _echo "Size: `_get_size "$file"`"
-  _echo "MD5: `$MD5SUM "$file"`"
+  _echo "MD5: `_md5sum "$file"`"
   _echo
 }
 

tools/config/mynteye/mynteye_config.yaml

@@ -3,7 +3,7 @@ dataset: "mynteye"
 
 # rosbag config
 rosbag:
-  topic_img_left: "/mynteye/left"
-  topic_img_right: "/mynteye/right"
-  topic_imu: "/mynteye/imu"
-  topic_temp: "/mynteye/temp"
+  topic_img_left: "/mynteye/left/image_raw"
+  topic_img_right: "/mynteye/right/image_raw"
+  topic_imu: "/mynteye/imu/data_raw"
+  topic_temp: "/mynteye/temp/data_raw"

tools/requirements.txt

@@ -1,3 +1,4 @@
 matplotlib>=1.5.1
 numpy>=1.11.0
 pandas>=0.22.0
+PyYAML>=3.11

tools/writer/config/img.params

@@ -22,7 +22,7 @@ in_right:
       model: 0
       coeffs: [ -5.1012886039889305e-01, 3.8764476500996770e-01, 0., 0.,
           0. ]
-ex_left_to_right:
+ex_right_to_left:
    rotation: [ 9.9701893306553813e-01, -9.5378124886236681e-04,
        -7.7151392794850615e-02, 1.4493996762830500e-03,
        9.9997867219985104e-01, 6.3682325649414354e-03,

tools/writer/device_writer.cc

@@ -72,7 +72,7 @@ bool DeviceWriter::WriteImgParams(const img_params_t &params) {
     LOG(INFO) << "Write img params success";
     LOG(INFO) << "Intrinsics left: {" << params.in_left << "}";
     LOG(INFO) << "Intrinsics right: {" << params.in_right << "}";
-    LOG(INFO) << "Extrinsics left to right: {" << params.ex_left_to_right
+    LOG(INFO) << "Extrinsics right to left: {" << params.ex_right_to_left
               << "}";
     return true;
   } else {
@@ -185,8 +185,8 @@ bool DeviceWriter::SaveImgParams(
     return false;
   }
   fs << "in_left" << std::vector<Intrinsics>{params.in_left} << "in_right"
-     << std::vector<Intrinsics>{params.in_right} << "ex_left_to_right"
-     << params.ex_left_to_right;
+     << std::vector<Intrinsics>{params.in_right} << "ex_right_to_left"
+     << params.ex_right_to_left;
   fs.release();
   return true;
 }
@@ -213,7 +213,7 @@ void DeviceWriter::SaveAllInfos(const std::string &dir) {
   SaveImgParams(
       {false, device_->GetIntrinsics(Stream::LEFT),
        device_->GetIntrinsics(Stream::RIGHT),
-       device_->GetExtrinsics(Stream::LEFT, Stream::RIGHT)},
+       device_->GetExtrinsics(Stream::RIGHT, Stream::LEFT)},
       dir + OS_SEP "img.params");
   auto &&m_in = device_->GetMotionIntrinsics();
   SaveImuParams(
@@ -311,8 +311,8 @@ DeviceWriter::dev_info_t DeviceWriter::LoadDeviceInfo(
     LOG(FATAL) << "Failed to load file: " << filepath;
   }
   DeviceInfo info;
-  info.lens_type = Type(fs["lens_type"].string());
-  info.imu_type = Type(fs["imu_type"].string());
+  info.lens_type = Type(std::string(fs["lens_type"]));
+  info.imu_type = Type(std::string(fs["imu_type"]));
   fs["nominal_baseline"] >> info.nominal_baseline;
   fs.release();
   return info;
@@ -348,11 +348,11 @@ DeviceWriter::img_params_t DeviceWriter::LoadImgParams(
 
     to_intrinsics(w, h, m, M1, D1, &params.in_left);
     to_intrinsics(w, h, m, M2, D2, &params.in_right);
-    to_extrinsics(R, T, &params.ex_left_to_right);
+    to_extrinsics(R, T, &params.ex_right_to_left);
   } else {
     fs["in_left"][0] >> params.in_left;
     fs["in_right"][0] >> params.in_right;
-    fs["ex_left_to_right"] >> params.ex_left_to_right;
+    fs["ex_right_to_left"] >> params.ex_right_to_left;
   }
 
   fs.release();

wrappers/README.md

@@ -1,3 +1,4 @@
 # Wrappers for MYNT® EYE SDK
 
 * [MYNT® EYE ROS Wrapper](https://github.com/slightech/MYNT-EYE-SDK-2/tree/master/wrappers/ros)
+* [MYNT® EYE Python Wrapper](https://github.com/slightech/MYNT-EYE-SDK-2/tree/master/wrappers/python)