Update README.md

.gitignore

@@ -1,5 +1,7 @@
 .DS_Store
-/.vscode/
+
+.vs/
+.vscode/
 
 _build/
 _install/

CMakeLists.txt

@@ -52,6 +52,19 @@ if(WITH_API)
   include(cmake/DetectOpenCV.cmake)
 endif()
 
+find_package(Boost COMPONENTS filesystem)
+if(Boost_FOUND)
+  set(WITH_BOOST_FILESYSTEM true)
+  add_definitions(-DWITH_FILESYSTEM)
+  add_definitions(-DWITH_BOOST_FILESYSTEM)
+  message(STATUS "With boost filesystem: ${Boost_VERSION}")
+  #message(STATUS "  Boost_LIBRARIES: ${Boost_LIBRARIES}")
+elseif(OS_WIN)
+  add_definitions(-DWITH_FILESYSTEM)
+  add_definitions(-DWITH_NATIVE_FILESYSTEM)
+  message(STATUS "With native filesystem")
+endif()
+
 LIST(APPEND CMAKE_MODULE_PATH cmake)
 
 include(CMakePackageConfigHelpers)
@@ -63,6 +76,14 @@ set(MYNTEYE_NAME ${PROJECT_NAME})
 set(MYNTEYE_NAMESPACE "mynteye")
 message(STATUS "Namespace: ${MYNTEYE_NAMESPACE}")
 
+if(MSVC)
+  string(REPLACE "/" "\\\\" MYNTEYE_SDK_ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+  string(REPLACE "/" "\\\\" MYNTEYE_SDK_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}")
+else()
+  file(TO_NATIVE_PATH "${CMAKE_CURRENT_SOURCE_DIR}" MYNTEYE_SDK_ROOT_DIR)
+  file(TO_NATIVE_PATH "${CMAKE_INSTALL_PREFIX}" MYNTEYE_SDK_INSTALL_DIR)
+endif()
+
 configure_file(
   include/mynteye/mynteye.h.in
   include/mynteye/mynteye.h @ONLY
@@ -167,6 +188,9 @@ set(MYNTEYE_LINKLIBS
 if(WITH_API)
   list(APPEND MYNTEYE_LINKLIBS ${OpenCV_LIBS})
 endif()
+if(WITH_BOOST_FILESYSTEM)
+  list(APPEND MYNTEYE_LINKLIBS ${Boost_LIBRARIES})
+endif()
 #message(STATUS "MYNTEYE_LINKLIBS: ${MYNTEYE_LINKLIBS}")
 
 add_library(${MYNTEYE_NAME} SHARED ${MYNTEYE_SRCS})
@@ -227,3 +251,56 @@ install(FILES
 install(EXPORT ${MYNTEYE_NAME}-targets
   DESTINATION ${MYNTEYE_CMAKE_INSTALLDIR}
 )
+
+## build.info
+
+macro(set_default_value VARIABLE DEFAULT)
+  if(NOT ${VARIABLE})
+    set(${VARIABLE} ${DEFAULT})
+  endif()
+endmacro()
+
+macro(set_version_values VARIABLE)
+  string(REPLACE "." ";" __version_list "${${VARIABLE}}")
+  list(LENGTH __version_list __len)
+  if(${__len} GREATER 0)
+    list(GET __version_list 0 ${VARIABLE}_MAJOR)
+  endif()
+  if(${__len} GREATER 1)
+    list(GET __version_list 1 ${VARIABLE}_MINOR)
+  endif()
+  if(${__len} GREATER 2)
+    list(GET __version_list 2 ${VARIABLE}_PATCH)
+  endif()
+  if(${__len} GREATER 3)
+    list(GET __version_list 3 ${VARIABLE}_TWEAK)
+  endif()
+endmacro()
+
+find_package(CUDA QUIET)
+
+set_version_values(CMAKE_CXX_COMPILER_VERSION)
+set_default_value(CMAKE_CXX_COMPILER_VERSION_MAJOR 0)
+set_default_value(CMAKE_CXX_COMPILER_VERSION_MINOR 0)
+set_default_value(CMAKE_CXX_COMPILER_VERSION_PATCH 0)
+set_default_value(CMAKE_CXX_COMPILER_VERSION_TWEAK 0)
+set_default_value(CUDA_VERSION_MAJOR 0)
+set_default_value(CUDA_VERSION_MINOR 0)
+set_default_value(OpenCV_VERSION_MAJOR 0)
+set_default_value(OpenCV_VERSION_MINOR 0)
+set_default_value(OpenCV_VERSION_PATCH 0)
+set_default_value(OpenCV_VERSION_TWEAK 0)
+set_default_value(${PROJECT_NAME}_VERSION_MAJOR 0)
+set_default_value(${PROJECT_NAME}_VERSION_MINOR 0)
+set_default_value(${PROJECT_NAME}_VERSION_PATCH 0)
+set_default_value(${PROJECT_NAME}_VERSION_TWEAK 0)
+
+configure_file(
+  cmake/templates/build.info.in
+  build.info @ONLY
+)
+
+install(FILES
+  ${CMAKE_CURRENT_BINARY_DIR}/build.info
+  DESTINATION ${CMAKE_INSTALL_PREFIX}/share/mynteye
+)

CommonDefs.mk

@@ -171,23 +171,32 @@ ifeq ($(HOST_OS),Win)
 ifeq ($(HOST_NAME),MinGW)
   CMAKE += -G "MinGW Makefiles"
 else ifeq ($(HOST_ARCH),x64)
-  VS_VERSION := $(shell echo "$(shell which cl)" | sed "s/.*Visual\sStudio\s\([0-9]\+\).*/\1/g")
-  ifeq (15,$(VS_VERSION))
+  ifeq ($(VS_CODE),)
+    WHICH_CL := $(shell which cl)
+    ifeq ($(WHICH_CL),)
+      $(error "Visual Studio version is unknown. Could set VS_CODE to specify it, e.g. make [TARGET] VS_CODE=2017")
+    endif
+    # C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\...
+    # C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\...
+    VS_CODE := $(shell echo "$(WHICH_CL)" | grep -Po "(?<=Visual Studio[ /])[0-9]+")
+  endif
+  # $(call mkinfo,"VS_CODE: $(VS_CODE)")
+  ifeq ($(filter $(VS_CODE),15 2017),$(VS_CODE))
     CMAKE += -G "Visual Studio 15 2017 Win64"
-  else ifeq (14,$(VS_VERSION))
+  else ifeq ($(filter $(VS_CODE),14 2015),$(VS_CODE))
     CMAKE += -G "Visual Studio 14 2015 Win64"
-  else ifeq (12,$(VS_VERSION))
+  else ifeq ($(filter $(VS_CODE),12 2013),$(VS_CODE))
     CMAKE += -G "Visual Studio 12 2013 Win64"
-  else ifeq (11,$(VS_VERSION))
+  else ifeq ($(filter $(VS_CODE),11 2012),$(VS_CODE))
     CMAKE += -G "Visual Studio 11 2012 Win64"
-  else ifeq (10,$(VS_VERSION))
+  else ifeq ($(filter $(VS_CODE),10 2010),$(VS_CODE))
     CMAKE += -G "Visual Studio 10 2010 Win64"
-  else ifeq (9,$(VS_VERSION))
+  else ifeq ($(filter $(VS_CODE),9 2008),$(VS_CODE))
     CMAKE += -G "Visual Studio 9 2008 Win64"
-  else ifeq (8,$(VS_VERSION))
+  else ifeq ($(filter $(VS_CODE),8 2005),$(VS_CODE))
     CMAKE += -G "Visual Studio 8 2005 Win64"
   else
-    $(call mkinfo,"Connot specify Visual Studio Win64")
+    $(error "Visual Studio version is not proper, VS_CODE: $(VS_CODE)")
   endif
 endif
 

README.md

@@ -4,8 +4,32 @@
 
 MYNT® EYE SDK 2.0 is a cross-platform library for MYNT® EYE cameras.
 
+The following platforms have been tested:
+
+* Windows 10
+* Ubuntu 16.04 / 14.04
+* Jetson TX2
+
+Please follow the guide doc to install the SDK on different platforms.
+
+## Documentations
+
+* [API Doc](https://github.com/slightech/MYNT-EYE-SDK-2/releases): API reference, some guides and data spec.
+* [Guide Doc](https://github.com/slightech/MYNT-EYE-SDK-2-Guide/releases): How to install and start using the SDK.
+
+> Supported languages: `zh-Hans`.
+
+## Firmwares
+
+[Google Drive]: https://drive.google.com/drive/folders/1tdFCcTBMNcImEGZ39tdOZmlX2SHKCr2f
+[百度网盘]: https://pan.baidu.com/s/1yPQDp2r0x4jvNwn2UjlMUQ
+
+Get firmwares from our online disks: [Google Drive][], [百度网盘][]. The latest version is `2.0.0-rc`.
+
 ## Usage
 
+In short,
+
 ```bash
 $ make
 Usage:
@@ -30,18 +54,6 @@ make samples
 ./samples/_output/bin/device/camera_d
 ```
 
-## Firmwares
-
-[Google Drive]: https://drive.google.com/drive/folders/1tdFCcTBMNcImEGZ39tdOZmlX2SHKCr2f
-[百度网盘]: https://pan.baidu.com/s/1yPQDp2r0x4jvNwn2UjlMUQ
-
-Get firmwares from our online disks: [Google Drive][], [百度网盘][].
-
-## Documentations
-
-* [API Doc](https://github.com/slightech/MYNT-EYE-SDK-2/releases)
-* [Guide Doc](https://github.com/slightech/MYNT-EYE-SDK-2-Guide/releases)
-
 ## Mirrors
 
 国内镜像：[码云](https://gitee.com/mynt/MYNT-EYE-SDK-2)。

cmake/Common.cmake

@@ -21,14 +21,40 @@ set(CUR_DIR ${CMAKE_CURRENT_LIST_DIR})
 
 if(MSVC OR MSYS OR MINGW)
   set(OS_WIN TRUE)
+  set(HOST_OS Win)
 elseif(APPLE)
   set(OS_MAC TRUE)
+  set(HOST_OS Mac)
 elseif(UNIX)
   set(OS_LINUX TRUE)
+  set(HOST_OS Linux)
 else()
   message(FATAL_ERROR "Unsupported OS.")
 endif()
 
+set(HOST_NAME "${HOST_OS}")
+
+if(OS_LINUX)
+  execute_process(COMMAND uname -a COMMAND tr -d '\n' OUTPUT_VARIABLE UNAME_A)
+  string(TOLOWER "${UNAME_A}" UNAME_A)
+  if(${UNAME_A} MATCHES ".*(tegra|jetsonbot).*")
+    set(OS_TEGRA TRUE)
+    set(HOST_NAME Tegra)
+  elseif(${UNAME_A} MATCHES ".*ubuntu.*")
+    set(OS_UBUNTU TRUE)
+    set(HOST_NAME Ubuntu)
+  endif()
+endif()
+
+include(${CMAKE_CURRENT_LIST_DIR}/TargetArch.cmake)
+target_architecture(HOST_ARCH)
+message(STATUS "HOST_ARCH: ${HOST_ARCH}")
+
+# CMAKE_CXX_COMPILER_ID
+#   https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_COMPILER_ID.html
+# CMAKE_CXX_COMPILER_VERSION
+#   https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_COMPILER_VERSION.html
+
 # set_outdir(ARCHIVE_OUTPUT_DIRECTORY
 #            LIBRARY_OUTPUT_DIRECTORY
 #            RUNTIME_OUTPUT_DIRECTORY)
@@ -44,17 +70,27 @@ macro(set_outdir ARCHIVE_OUTPUT_DIRECTORY LIBRARY_OUTPUT_DIRECTORY RUNTIME_OUTPU
   endforeach()
 endmacro()
 
+set(__exe2bat_relative_path false)
+
 macro(exe2bat exe_name exe_dir dll_search_paths)
   message(STATUS "Generating ${exe_name}.bat")
   set(__exe_name ${exe_name})
-  set(__dll_relative_search_paths "")
-  foreach(path ${dll_search_paths})
-    file(RELATIVE_PATH __relative_path "${exe_dir}" "${path}")
-    file(TO_NATIVE_PATH ${__relative_path} __relative_path)
-    list(APPEND __dll_relative_search_paths ${__relative_path})
-  endforeach()
-  #message(STATUS __dll_relative_search_paths: "${__dll_relative_search_paths}")
-  set(__dll_search_paths "${__dll_relative_search_paths}")
+  if(__exe2bat_relative_path)
+    set(__dll_relative_search_paths "")
+    foreach(path ${dll_search_paths})
+      file(RELATIVE_PATH __relative_path "${exe_dir}" "${path}")
+      file(TO_NATIVE_PATH ${__relative_path} __relative_path)
+      list(APPEND __dll_relative_search_paths ${__relative_path})
+    endforeach()
+    set(__dll_search_paths "${__dll_relative_search_paths}")
+  else()
+    set(__dll_native_search_paths "")
+    foreach(path ${dll_search_paths})
+      file(TO_NATIVE_PATH ${path} __native_path)
+      list(APPEND __dll_native_search_paths ${__native_path})
+    endforeach()
+    set(__dll_search_paths "${__dll_native_search_paths}")
+  endif()
   configure_file(
     "${CUR_DIR}/templates/exe.bat.in"
     "${exe_dir}/${__exe_name}.bat"

cmake/DetectOpenCV.cmake

@@ -19,8 +19,15 @@ find_package(OpenCV REQUIRED)
 message(STATUS "Found OpenCV: ${OpenCV_VERSION}")
 if(OpenCV_VERSION VERSION_LESS 3.0)
   add_definitions(-DUSE_OPENCV2)
-else()
+elseif(OpenCV_VERSION VERSION_LESS 4.0)
   add_definitions(-DUSE_OPENCV3)
+else()
+  add_definitions(-DUSE_OPENCV4)
+endif()
+
+list(FIND OpenCV_LIBS "opencv_world" __index)
+if(${__index} GREATER -1)
+  set(WITH_OPENCV_WORLD TRUE)
 endif()
 
 if(MSVC OR MSYS OR MINGW)

cmake/FindCUDA/make2cmake.cmake

@@ -0,0 +1,92 @@
+#  James Bigler, NVIDIA Corp (nvidia.com - jbigler)
+#  Abe Stephens, SCI Institute -- http://www.sci.utah.edu/~abe/FindCuda.html
+#
+#  Copyright (c) 2008 - 2009 NVIDIA Corporation.  All rights reserved.
+#
+#  Copyright (c) 2007-2009
+#  Scientific Computing and Imaging Institute, University of Utah
+#
+#  This code is licensed under the MIT License.  See the FindCUDA.cmake script
+#  for the text of the license.
+
+# The MIT License
+#
+# License for the specific language governing rights and limitations under
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+#
+
+#######################################################################
+# This converts a file written in makefile syntax into one that can be included
+# by CMake.
+
+file(READ ${input_file} depend_text)
+
+if (NOT "${depend_text}" STREQUAL "")
+
+  # message("FOUND DEPENDS")
+
+  string(REPLACE "\\ " " " depend_text ${depend_text})
+
+  # This works for the nvcc -M generated dependency files.
+  string(REGEX REPLACE "^.* : " "" depend_text ${depend_text})
+  string(REGEX REPLACE "[ \\\\]*\n" ";" depend_text ${depend_text})
+
+  set(dependency_list "")
+
+  foreach(file ${depend_text})
+
+    string(REGEX REPLACE "^ +" "" file ${file})
+
+    # OK, now if we had a UNC path, nvcc has a tendency to only output the first '/'
+    # instead of '//'.  Here we will test to see if the file exists, if it doesn't then
+    # try to prepend another '/' to the path and test again.  If it still fails remove the
+    # path.
+
+    if(NOT EXISTS "${file}")
+      if (EXISTS "/${file}")
+        set(file "/${file}")
+      else()
+        message(WARNING " Removing non-existent dependency file: ${file}")
+        set(file "")
+      endif()
+    endif()
+
+    if(NOT IS_DIRECTORY "${file}")
+      # If softlinks start to matter, we should change this to REALPATH.  For now we need
+      # to flatten paths, because nvcc can generate stuff like /bin/../include instead of
+      # just /include.
+      get_filename_component(file_absolute "${file}" ABSOLUTE)
+      list(APPEND dependency_list "${file_absolute}")
+    endif()
+
+  endforeach()
+
+else()
+  # message("FOUND NO DEPENDS")
+endif()
+
+# Remove the duplicate entries and sort them.
+list(REMOVE_DUPLICATES dependency_list)
+list(SORT dependency_list)
+
+foreach(file ${dependency_list})
+  set(cuda_nvcc_depend "${cuda_nvcc_depend} \"${file}\"\n")
+endforeach()
+
+file(WRITE ${output_file} "# Generated by: make2cmake.cmake\nSET(CUDA_NVCC_DEPEND\n ${cuda_nvcc_depend})\n\n")

cmake/FindCUDA/parse_cubin.cmake

@@ -0,0 +1,109 @@
+#  James Bigler, NVIDIA Corp (nvidia.com - jbigler)
+#  Abe Stephens, SCI Institute -- http://www.sci.utah.edu/~abe/FindCuda.html
+#
+#  Copyright (c) 2008 - 2009 NVIDIA Corporation.  All rights reserved.
+#
+#  Copyright (c) 2007-2009
+#  Scientific Computing and Imaging Institute, University of Utah
+#
+#  This code is licensed under the MIT License.  See the FindCUDA.cmake script
+#  for the text of the license.
+
+# The MIT License
+#
+# License for the specific language governing rights and limitations under
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+#
+
+#######################################################################
+# Parses a .cubin file produced by nvcc and reports statistics about the file.
+
+
+file(READ ${input_file} file_text)
+
+if (NOT "${file_text}" STREQUAL "")
+
+  string(REPLACE ";" "\\;" file_text ${file_text})
+  string(REPLACE "\ncode" ";code" file_text ${file_text})
+
+  list(LENGTH file_text len)
+
+  foreach(line ${file_text})
+
+    # Only look at "code { }" blocks.
+    if(line MATCHES "^code")
+
+      # Break into individual lines.
+      string(REGEX REPLACE "\n" ";" line ${line})
+
+      foreach(entry ${line})
+
+        # Extract kernel names.
+        if (${entry} MATCHES "[^g]name = ([^ ]+)")
+          set(entry "${CMAKE_MATCH_1}")
+
+          # Check to see if the kernel name starts with "_"
+          set(skip FALSE)
+          # if (${entry} MATCHES "^_")
+            # Skip the rest of this block.
+            # message("Skipping ${entry}")
+            # set(skip TRUE)
+          # else ()
+            message("Kernel:    ${entry}")
+          # endif ()
+
+        endif()
+
+        # Skip the rest of the block if necessary
+        if(NOT skip)
+
+          # Registers
+          if (${entry} MATCHES "reg([ ]+)=([ ]+)([^ ]+)")
+            set(entry "${CMAKE_MATCH_3}")
+            message("Registers: ${entry}")
+          endif()
+
+          # Local memory
+          if (${entry} MATCHES "lmem([ ]+)=([ ]+)([^ ]+)")
+            set(entry "${CMAKE_MATCH_3}")
+            message("Local:     ${entry}")
+          endif()
+
+          # Shared memory
+          if (${entry} MATCHES "smem([ ]+)=([ ]+)([^ ]+)")
+            set(entry "${CMAKE_MATCH_3}")
+            message("Shared:    ${entry}")
+          endif()
+
+          if (${entry} MATCHES "^}")
+            message("")
+          endif()
+
+        endif()
+
+
+      endforeach()
+
+    endif()
+
+  endforeach()
+
+else()
+  # message("FOUND NO DEPENDS")
+endif()

cmake/FindCUDA/run_nvcc.cmake

@@ -0,0 +1,288 @@
+#  James Bigler, NVIDIA Corp (nvidia.com - jbigler)
+#
+#  Copyright (c) 2008 - 2009 NVIDIA Corporation.  All rights reserved.
+#
+#  This code is licensed under the MIT License.  See the FindCUDA.cmake script
+#  for the text of the license.
+
+# The MIT License
+#
+# License for the specific language governing rights and limitations under
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+
+##########################################################################
+# This file runs the nvcc commands to produce the desired output file along with
+# the dependency file needed by CMake to compute dependencies.  In addition the
+# file checks the output of each command and if the command fails it deletes the
+# output files.
+
+# Input variables
+#
+# verbose:BOOL=<>          OFF: Be as quiet as possible (default)
+#                          ON : Describe each step
+#
+# build_configuration:STRING=<> Typically one of Debug, MinSizeRel, Release, or
+#                               RelWithDebInfo, but it should match one of the
+#                               entries in CUDA_HOST_FLAGS. This is the build
+#                               configuration used when compiling the code.  If
+#                               blank or unspecified Debug is assumed as this is
+#                               what CMake does.
+#
+# generated_file:STRING=<> File to generate.  This argument must be passed in.
+#
+# generated_cubin_file:STRING=<> File to generate.  This argument must be passed
+#                                                   in if build_cubin is true.
+
+if(NOT generated_file)
+  message(FATAL_ERROR "You must specify generated_file on the command line")
+endif()
+
+# Set these up as variables to make reading the generated file easier
+set(CMAKE_COMMAND "@CMAKE_COMMAND@") # path
+set(source_file "@source_file@") # path
+set(NVCC_generated_dependency_file "@NVCC_generated_dependency_file@") # path
+set(cmake_dependency_file "@cmake_dependency_file@") # path
+set(CUDA_make2cmake "@CUDA_make2cmake@") # path
+set(CUDA_parse_cubin "@CUDA_parse_cubin@") # path
+set(build_cubin @build_cubin@) # bool
+set(CUDA_HOST_COMPILER "@CUDA_HOST_COMPILER@") # path
+# We won't actually use these variables for now, but we need to set this, in
+# order to force this file to be run again if it changes.
+set(generated_file_path "@generated_file_path@") # path
+set(generated_file_internal "@generated_file@") # path
+set(generated_cubin_file_internal "@generated_cubin_file@") # path
+
+set(CUDA_NVCC_EXECUTABLE "@CUDA_NVCC_EXECUTABLE@") # path
+set(CUDA_NVCC_FLAGS @CUDA_NVCC_FLAGS@ ;; @CUDA_WRAP_OPTION_NVCC_FLAGS@) # list
+@CUDA_NVCC_FLAGS_CONFIG@
+set(nvcc_flags "@nvcc_flags@") # list
+set(CUDA_NVCC_INCLUDE_ARGS "@CUDA_NVCC_INCLUDE_ARGS@") # list (needs to be in quotes to handle spaces properly).
+set(format_flag "@format_flag@") # string
+
+if(build_cubin AND NOT generated_cubin_file)
+  message(FATAL_ERROR "You must specify generated_cubin_file on the command line")
+endif()
+
+# This is the list of host compilation flags.  It C or CXX should already have
+# been chosen by FindCUDA.cmake.
+@CUDA_HOST_FLAGS@
+
+# Take the compiler flags and package them up to be sent to the compiler via -Xcompiler
+set(nvcc_host_compiler_flags "")
+# If we weren't given a build_configuration, use Debug.
+if(NOT build_configuration)
+  set(build_configuration Debug)
+endif()
+string(TOUPPER "${build_configuration}" build_configuration)
+#message("CUDA_NVCC_HOST_COMPILER_FLAGS = ${CUDA_NVCC_HOST_COMPILER_FLAGS}")
+foreach(flag ${CMAKE_HOST_FLAGS} ${CMAKE_HOST_FLAGS_${build_configuration}})
+  # Extra quotes are added around each flag to help nvcc parse out flags with spaces.
+  set(nvcc_host_compiler_flags "${nvcc_host_compiler_flags},\"${flag}\"")
+endforeach()
+if (nvcc_host_compiler_flags)
+  set(nvcc_host_compiler_flags "-Xcompiler" ${nvcc_host_compiler_flags})
+endif()
+#message("nvcc_host_compiler_flags = \"${nvcc_host_compiler_flags}\"")
+# Add the build specific configuration flags
+list(APPEND CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS_${build_configuration}})
+
+# Any -ccbin existing in CUDA_NVCC_FLAGS gets highest priority
+list( FIND CUDA_NVCC_FLAGS "-ccbin" ccbin_found0 )
+list( FIND CUDA_NVCC_FLAGS "--compiler-bindir" ccbin_found1 )
+if( ccbin_found0 LESS 0 AND ccbin_found1 LESS 0 AND CUDA_HOST_COMPILER )
+  if (CUDA_HOST_COMPILER STREQUAL "@_CUDA_MSVC_HOST_COMPILER@" AND DEFINED CCBIN)
+    set(CCBIN -ccbin "${CCBIN}")
+  else()
+    set(CCBIN -ccbin "${CUDA_HOST_COMPILER}")
+  endif()
+endif()
+
+# cuda_execute_process - Executes a command with optional command echo and status message.
+#
+#   status  - Status message to print if verbose is true
+#   command - COMMAND argument from the usual execute_process argument structure
+#   ARGN    - Remaining arguments are the command with arguments
+#
+#   CUDA_result - return value from running the command
+#
+# Make this a macro instead of a function, so that things like RESULT_VARIABLE
+# and other return variables are present after executing the process.
+macro(cuda_execute_process status command)
+  set(_command ${command})
+  if(NOT "x${_command}" STREQUAL "xCOMMAND")
+    message(FATAL_ERROR "Malformed call to cuda_execute_process.  Missing COMMAND as second argument. (command = ${command})")
+  endif()
+  if(verbose)
+    execute_process(COMMAND "${CMAKE_COMMAND}" -E echo -- ${status})
+    # Now we need to build up our command string.  We are accounting for quotes
+    # and spaces, anything else is left up to the user to fix if they want to
+    # copy and paste a runnable command line.
+    set(cuda_execute_process_string)
+    foreach(arg ${ARGN})
+      # If there are quotes, excape them, so they come through.
+      string(REPLACE "\"" "\\\"" arg ${arg})
+      # Args with spaces need quotes around them to get them to be parsed as a single argument.
+      if(arg MATCHES " ")
+        list(APPEND cuda_execute_process_string "\"${arg}\"")
+      else()
+        list(APPEND cuda_execute_process_string ${arg})
+      endif()
+    endforeach()
+    # Echo the command
+    execute_process(COMMAND ${CMAKE_COMMAND} -E echo ${cuda_execute_process_string})
+  endif()
+  # Run the command
+  execute_process(COMMAND ${ARGN} RESULT_VARIABLE CUDA_result )
+endmacro()
+
+# Delete the target file
+cuda_execute_process(
+  "Removing ${generated_file}"
+  COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
+  )
+
+# For CUDA 2.3 and below, -G -M doesn't work, so remove the -G flag
+# for dependency generation and hope for the best.
+set(depends_CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS}")
+set(CUDA_VERSION @CUDA_VERSION@)
+if(CUDA_VERSION VERSION_LESS "3.0")
+  cmake_policy(PUSH)
+  # CMake policy 0007 NEW states that empty list elements are not
+  # ignored.  I'm just setting it to avoid the warning that's printed.
+  cmake_policy(SET CMP0007 NEW)
+  # Note that this will remove all occurrences of -G.
+  list(REMOVE_ITEM depends_CUDA_NVCC_FLAGS "-G")
+  cmake_policy(POP)
+endif()
+
+# nvcc doesn't define __CUDACC__ for some reason when generating dependency files.  This
+# can cause incorrect dependencies when #including files based on this macro which is
+# defined in the generating passes of nvcc invocation.  We will go ahead and manually
+# define this for now until a future version fixes this bug.
+set(CUDACC_DEFINE -D__CUDACC__)
+
+# Generate the dependency file
+cuda_execute_process(
+  "Generating dependency file: ${NVCC_generated_dependency_file}"
+  COMMAND "${CUDA_NVCC_EXECUTABLE}"
+  -M
+  ${CUDACC_DEFINE}
+  "${source_file}"
+  -o "${NVCC_generated_dependency_file}"
+  ${CCBIN}
+  ${nvcc_flags}
+  ${nvcc_host_compiler_flags}
+  ${depends_CUDA_NVCC_FLAGS}
+  -DNVCC
+  ${CUDA_NVCC_INCLUDE_ARGS}
+  )
+
+if(CUDA_result)
+  message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Generate the cmake readable dependency file to a temp file.  Don't put the
+# quotes just around the filenames for the input_file and output_file variables.
+# CMake will pass the quotes through and not be able to find the file.
+cuda_execute_process(
+  "Generating temporary cmake readable file: ${cmake_dependency_file}.tmp"
+  COMMAND "${CMAKE_COMMAND}"
+  -D "input_file:FILEPATH=${NVCC_generated_dependency_file}"
+  -D "output_file:FILEPATH=${cmake_dependency_file}.tmp"
+  -P "${CUDA_make2cmake}"
+  )
+
+if(CUDA_result)
+  message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Copy the file if it is different
+cuda_execute_process(
+  "Copy if different ${cmake_dependency_file}.tmp to ${cmake_dependency_file}"
+  COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${cmake_dependency_file}.tmp" "${cmake_dependency_file}"
+  )
+
+if(CUDA_result)
+  message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Delete the temporary file
+cuda_execute_process(
+  "Removing ${cmake_dependency_file}.tmp and ${NVCC_generated_dependency_file}"
+  COMMAND "${CMAKE_COMMAND}" -E remove "${cmake_dependency_file}.tmp" "${NVCC_generated_dependency_file}"
+  )
+
+if(CUDA_result)
+  message(FATAL_ERROR "Error generating ${generated_file}")
+endif()
+
+# Generate the code
+cuda_execute_process(
+  "Generating ${generated_file}"
+  COMMAND "${CUDA_NVCC_EXECUTABLE}"
+  "${source_file}"
+  ${format_flag} -o "${generated_file}"
+  ${CCBIN}
+  ${nvcc_flags}
+  ${nvcc_host_compiler_flags}
+  ${CUDA_NVCC_FLAGS}
+  -DNVCC
+  ${CUDA_NVCC_INCLUDE_ARGS}
+  )
+
+if(CUDA_result)
+  # Since nvcc can sometimes leave half done files make sure that we delete the output file.
+  cuda_execute_process(
+    "Removing ${generated_file}"
+    COMMAND "${CMAKE_COMMAND}" -E remove "${generated_file}"
+    )
+  message(FATAL_ERROR "Error generating file ${generated_file}")
+else()
+  if(verbose)
+    message("Generated ${generated_file} successfully.")
+  endif()
+endif()
+
+# Cubin resource report commands.
+if( build_cubin )
+  # Run with -cubin to produce resource usage report.
+  cuda_execute_process(
+    "Generating ${generated_cubin_file}"
+    COMMAND "${CUDA_NVCC_EXECUTABLE}"
+    "${source_file}"
+    ${CUDA_NVCC_FLAGS}
+    ${nvcc_flags}
+    ${CCBIN}
+    ${nvcc_host_compiler_flags}
+    -DNVCC
+    -cubin
+    -o "${generated_cubin_file}"
+    ${CUDA_NVCC_INCLUDE_ARGS}
+    )
+
+  # Execute the parser script.
+  cuda_execute_process(
+    "Executing the parser script"
+    COMMAND  "${CMAKE_COMMAND}"
+    -D "input_file:STRING=${generated_cubin_file}"
+    -P "${CUDA_parse_cubin}"
+    )
+
+endif()

cmake/TargetArch.cmake

@@ -0,0 +1,167 @@
+# Based on the Qt 5 processor detection code, so should be very accurate
+# http://code.qt.io/cgit/qt/qtbase.git/tree/src/corelib/global/qprocessordetection.h
+# Currently handles arm (v5, v6, v7, v8), x86 (32/64), ia64, and ppc (32/64)
+
+# Regarding POWER/PowerPC, just as is noted in the Qt source,
+# "There are many more known variants/revisions that we do not handle/detect."
+
+set(archdetect_c_code "
+#if defined(__arm__) || defined(__TARGET_ARCH_ARM) || defined(_M_ARM) \\
+    || defined(__aarch64__) || defined(__ARM64__)
+  #if defined(__aarch64__) || defined(__ARM64__)
+    #define cmake_ARM_64
+  #else
+    #define cmake_ARM_32
+  #endif
+  #if defined(__ARM_ARCH) && __ARM_ARCH > 1
+    #define cmake_ARM __ARM_ARCH
+  #elif defined(__TARGET_ARCH_ARM) && __TARGET_ARCH_ARM > 1
+    #define cmake_ARM __TARGET_ARCH_ARM
+  #elif defined(_M_ARM) && _M_ARM > 1
+    #define cmake_ARM _M_ARM
+  #elif defined(__ARM64_ARCH_8__) \\
+      || defined(__aarch64__) \\
+      || defined(__ARMv8__) \\
+      || defined(__ARMv8_A__)
+    #define cmake_ARM 8
+  #elif defined(__ARM_ARCH_7__) \\
+      || defined(__ARM_ARCH_7A__) \\
+      || defined(__ARM_ARCH_7R__) \\
+      || defined(__ARM_ARCH_7M__) \\
+      || defined(__ARM_ARCH_7S__) \\
+      || defined(_ARM_ARCH_7) \\
+      || defined(__CORE_CORTEXA__)
+    #define cmake_ARM 7
+  #elif defined(__ARM_ARCH_6__) \\
+      || defined(__ARM_ARCH_6J__) \\
+      || defined(__ARM_ARCH_6T2__) \\
+      || defined(__ARM_ARCH_6Z__) \\
+      || defined(__ARM_ARCH_6K__) \\
+      || defined(__ARM_ARCH_6ZK__) \\
+      || defined(__ARM_ARCH_6M__)
+    #define cmake_ARM 6
+  #elif defined(__ARM_ARCH_5TEJ__) \\
+      || defined(__ARM_ARCH_5TE__)
+    #define cmake_ARM 5
+  #else
+    #define cmake_ARM 0
+  #endif
+  #if cmake_ARM >= 8
+    #error cmake_ARCH armv8
+  #elif cmake_ARM >= 7
+    #error cmake_ARCH armv7
+  #elif cmake_ARM >= 6
+    #error cmake_ARCH armv6
+  #elif cmake_ARM >= 5
+    #error cmake_ARCH armv5
+  #else
+    #if defined(cmake_ARM_64)
+      #error cmake_ARCH arm64
+    #else
+      #error cmake_ARCH arm
+    #endif
+  #endif
+#elif defined(__i386) || defined(__i386__) || defined(_M_IX86)
+  #error cmake_ARCH i386
+#elif defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(_M_X64)
+  #error cmake_ARCH x86_64
+#elif defined(__ia64) || defined(__ia64__) || defined(_M_IA64)
+  #error cmake_ARCH ia64
+#elif defined(__ppc__) || defined(__ppc) || defined(__powerpc__) \\
+    || defined(_ARCH_COM) || defined(_ARCH_PWR) || defined(_ARCH_PPC) \\
+    || defined(_M_MPPC) || defined(_M_PPC)
+  #if defined(__ppc64__) || defined(__powerpc64__) || defined(__64BIT__)
+    #error cmake_ARCH ppc64
+  #else
+    #error cmake_ARCH ppc
+  #endif
+#endif
+
+#error cmake_ARCH unknown
+")
+
+# Set ppc_support to TRUE before including this file or ppc and ppc64
+# will be treated as invalid architectures since they are no longer supported by Apple
+
+function(target_architecture output_var)
+  if(APPLE AND CMAKE_OSX_ARCHITECTURES)
+    # On OS X we use CMAKE_OSX_ARCHITECTURES *if* it was set
+    # First let's normalize the order of the values
+
+    # Note that it's not possible to compile PowerPC applications if you are using
+    # the OS X SDK version 10.6 or later - you'll need 10.4/10.5 for that, so we
+    # disable it by default
+    # See this page for more information:
+    # http://stackoverflow.com/questions/5333490/how-can-we-restore-ppc-ppc64-as-well-as-full-10-4-10-5-sdk-support-to-xcode-4
+
+    # Architecture defaults to i386 or ppc on OS X 10.5 and earlier, depending on the CPU type detected at runtime.
+    # On OS X 10.6+ the default is x86_64 if the CPU supports it, i386 otherwise.
+
+    foreach(osx_arch ${CMAKE_OSX_ARCHITECTURES})
+      if("${osx_arch}" STREQUAL "ppc" AND ppc_support)
+        set(osx_arch_ppc TRUE)
+      elseif("${osx_arch}" STREQUAL "i386")
+        set(osx_arch_i386 TRUE)
+      elseif("${osx_arch}" STREQUAL "x86_64")
+        set(osx_arch_x86_64 TRUE)
+      elseif("${osx_arch}" STREQUAL "ppc64" AND ppc_support)
+        set(osx_arch_ppc64 TRUE)
+      else()
+        message(FATAL_ERROR "Invalid OS X arch name: ${osx_arch}")
+      endif()
+    endforeach()
+
+    # Now add all the architectures in our normalized order
+    if(osx_arch_ppc)
+      list(APPEND ARCH ppc)
+    endif()
+
+    if(osx_arch_i386)
+      list(APPEND ARCH i386)
+    endif()
+
+    if(osx_arch_x86_64)
+      list(APPEND ARCH x86_64)
+    endif()
+
+    if(osx_arch_ppc64)
+      list(APPEND ARCH ppc64)
+    endif()
+  else()
+    file(WRITE "${CMAKE_BINARY_DIR}/arch.c" "${archdetect_c_code}")
+
+    enable_language(C)
+
+    # Detect the architecture in a rather creative way...
+    # This compiles a small C program which is a series of ifdefs that selects a
+    # particular #error preprocessor directive whose message string contains the
+    # target architecture. The program will always fail to compile (both because
+    # file is not a valid C program, and obviously because of the presence of the
+    # #error preprocessor directives... but by exploiting the preprocessor in this
+    # way, we can detect the correct target architecture even when cross-compiling,
+    # since the program itself never needs to be run (only the compiler/preprocessor)
+    try_run(
+      run_result_unused
+      compile_result_unused
+      "${CMAKE_BINARY_DIR}"
+      "${CMAKE_BINARY_DIR}/arch.c"
+      COMPILE_OUTPUT_VARIABLE ARCH
+      CMAKE_FLAGS CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}
+    )
+
+    # Parse the architecture name from the compiler output
+    string(REGEX MATCH "cmake_ARCH ([a-zA-Z0-9_]+)" ARCH "${ARCH}")
+
+    # Get rid of the value marker leaving just the architecture name
+    string(REPLACE "cmake_ARCH " "" ARCH "${ARCH}")
+
+    # If we are compiling with an unknown architecture this variable should
+    # already be set to "unknown" but in the case that it's empty (i.e. due
+    # to a typo in the code), then set it to unknown
+    if (NOT ARCH)
+      set(ARCH unknown)
+    endif()
+  endif()
+
+  set(${output_var} "${ARCH}" PARENT_SCOPE)
+endfunction()

cmake/templates/build.info.in

@@ -0,0 +1,27 @@
+%YAML:1.0
+---
+HOST_OS: "@HOST_OS@"
+HOST_NAME: "@HOST_NAME@"
+HOST_ARCH: "@HOST_ARCH@"
+HOST_COMPILER: "@CMAKE_CXX_COMPILER_ID@"
+COMPILER_VERSION: "@CMAKE_CXX_COMPILER_VERSION@"
+COMPILER_VERSION_MAJOR: @CMAKE_CXX_COMPILER_VERSION_MAJOR@
+COMPILER_VERSION_MINOR: @CMAKE_CXX_COMPILER_VERSION_MINOR@
+COMPILER_VERSION_PATCH: @CMAKE_CXX_COMPILER_VERSION_PATCH@
+COMPILER_VERSION_TWEAK: @CMAKE_CXX_COMPILER_VERSION_TWEAK@
+CUDA_VERSION: "@CUDA_VERSION@"
+CUDA_VERSION_MAJOR: @CUDA_VERSION_MAJOR@
+CUDA_VERSION_MINOR: @CUDA_VERSION_MINOR@
+CUDA_VERSION_STRING: "@CUDA_VERSION_STRING@"
+OpenCV_VERSION: "@OpenCV_VERSION@"
+OpenCV_VERSION_MAJOR: @OpenCV_VERSION_MAJOR@
+OpenCV_VERSION_MINOR: @OpenCV_VERSION_MINOR@
+OpenCV_VERSION_PATCH: @OpenCV_VERSION_PATCH@
+OpenCV_VERSION_TWEAK: @OpenCV_VERSION_TWEAK@
+OpenCV_VERSION_STATUS: "@OpenCV_VERSION_STATUS@"
+OpenCV_WITH_WORLD: "@WITH_OPENCV_WORLD@"
+MYNTEYE_VERSION: "@mynteye_VERSION@"
+MYNTEYE_VERSION_MAJOR: @mynteye_VERSION_MAJOR@
+MYNTEYE_VERSION_MINOR: @mynteye_VERSION_MINOR@
+MYNTEYE_VERSION_PATCH: @mynteye_VERSION_PATCH@
+MYNTEYE_VERSION_TWEAK: @mynteye_VERSION_TWEAK@

cmake/templates/exe.bat.in

@@ -7,7 +7,7 @@ if "%MY_PATH_SET_@__exe_name@%"=="" (
   set "PATH=@__dll_search_paths@;%PATH%"
 )
 
-cd "%MY_DIR%"
+::cd "%MY_DIR%"
 "%MY_DIR%@__exe_name@.exe" %*
-cd "%MY_PWD%"
+::cd "%MY_PWD%"
 ::pause >nul

doc/zh-Hans/api.doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = "MYNT EYE SDK"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = 2.0.0-rc1
+PROJECT_NUMBER         = 2.0.0-rc
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

doc/zh-Hans/spec_control_channel.md

@@ -2,11 +2,11 @@
 
 | 名称 | 字段 | 地址 | 带宽 | 说明 |
 | :----- | :----- | :----- | :----- | :----- |
-| 相机控制通道 | XU_CAM_CTRL_CHANNEL | 0x0100 | 3 | |
-| 半双工通道 | XU_HALF_DUPLEX_CHANNEL | 0x0200 | 20 | |
-| IMU 请求通道 | XU_IMUDATA_WRITE_CHANNEL | 0x0300 | 5 | |
-| IMU 响应通道 | XU_IMUDATA_READ_CHANNEL | 0x0400 | 2000 | |
-| 文件通道 | XU_FILE_CHANNEL | 0x0500 | 2000 | |
+| 相机控制通道 | XU_CAM_CTRL_CHANNEL | 1 | 3 | |
+| 半双工通道 | XU_HALF_DUPLEX_CHANNEL | 2 | 20 | |
+| IMU 请求通道 | XU_IMUDATA_WRITE_CHANNEL | 3 | 5 | |
+| IMU 响应通道 | XU_IMUDATA_READ_CHANNEL | 4 | 2000 | |
+| 文件通道 | XU_FILE_CHANNEL | 5 | 2000 | |
 
 ## 相机控制通道
 

include/mynteye/glog_init.h

@@ -29,7 +29,7 @@ struct glog_init {
     FLAGS_alsologtostderr = true;
     FLAGS_colorlogtostderr = true;
 
-    FLAGS_log_dir = ".";
+    // FLAGS_log_dir = ".";
     FLAGS_max_log_size = 1024;
     FLAGS_stop_logging_if_full_disk = true;
 

include/mynteye/mynteye.h.in

@@ -58,4 +58,7 @@ MYNTEYE_API_VERSION_CHECK( \
 # define MYNTEYE_USE_NAMESPACE
 #endif
 
+constexpr char MYNTEYE_SDK_ROOT_DIR[] = "@MYNTEYE_SDK_ROOT_DIR@";
+constexpr char MYNTEYE_SDK_INSTALL_DIR[] = "@MYNTEYE_SDK_INSTALL_DIR@";
+
 #endif  // MYNTEYE_MYNTEYE_H_

samples/api/CMakeLists.txt

@@ -27,3 +27,15 @@ make_executable(camera_a
   LINK_LIBS mynteye ${OpenCV_LIBS}
   DLL_SEARCH_PATHS ${PRO_DIR}/_install/bin ${OpenCV_LIB_SEARCH_PATH}
 )
+
+## get_depth_with_region
+
+if(OpenCV_VERSION VERSION_LESS 4.0)
+
+make_executable(get_depth_with_region
+  SRCS get_depth_with_region.cc
+  LINK_LIBS mynteye ${OpenCV_LIBS}
+  DLL_SEARCH_PATHS ${PRO_DIR}/_install/bin ${OpenCV_LIB_SEARCH_PATH}
+)
+
+endif()

samples/api/get_depth_with_region.cc

@@ -0,0 +1,227 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include <glog/logging.h>
+
+#include "mynteye/api.h"
+
+#define WIN_FLAGS \
+  cv::WINDOW_AUTOSIZE | cv::WINDOW_KEEPRATIO | cv::WINDOW_GUI_NORMAL
+
+namespace {
+
+class DepthRegion {
+ public:
+  explicit DepthRegion(std::uint32_t n)
+      : n_(std::move(n)), show_(false), selected_(false), point_(0, 0) {}
+
+  ~DepthRegion() = default;
+
+  /**
+   * 鼠标事件：默认不选中区域，随鼠标移动而显示。单击后，则会选中区域来显示。你可以再单击已选中区域或双击未选中区域，取消选中。
+   */
+  void OnMouse(const int &event, const int &x, const int &y, const int &flags) {
+    UNUSED(flags)
+    if (event != CV_EVENT_MOUSEMOVE && event != CV_EVENT_LBUTTONDOWN) {
+      return;
+    }
+    show_ = true;
+
+    if (event == CV_EVENT_MOUSEMOVE) {
+      if (!selected_) {
+        point_.x = x;
+        point_.y = y;
+      }
+    } else if (event == CV_EVENT_LBUTTONDOWN) {
+      if (selected_) {
+        if (x >= static_cast<int>(point_.x - n_) &&
+            x <= static_cast<int>(point_.x + n_) &&
+            y >= static_cast<int>(point_.y - n_) &&
+            y <= static_cast<int>(point_.y + n_)) {
+          selected_ = false;
+        }
+      } else {
+        selected_ = true;
+      }
+      point_.x = x;
+      point_.y = y;
+    }
+  }
+
+  template <typename T>
+  void ShowElems(
+      const cv::Mat &depth,
+      std::function<std::string(const T &elem)> elem2string,
+      int elem_space = 40,
+      std::function<std::string(
+          const cv::Mat &depth, const cv::Point &point, const std::uint32_t &n)>
+          getinfo = nullptr) {
+    if (!show_)
+      return;
+
+    int space = std::move(elem_space);
+    int n = 2 * n_ + 1;
+    cv::Mat im(space * n, space * n, CV_8UC3, cv::Scalar(255, 255, 255));
+
+    int x, y;
+    std::string str;
+    int baseline = 0;
+    for (int i = -n_; i <= n; ++i) {
+      x = point_.x + i;
+      if (x < 0 || x >= depth.cols)
+        continue;
+      for (int j = -n_; j <= n; ++j) {
+        y = point_.y + j;
+        if (y < 0 || y >= depth.rows)
+          continue;
+
+        str = elem2string(depth.at<T>(y, x));
+
+        cv::Scalar color(0, 0, 0);
+        if (i == 0 && j == 0)
+          color = cv::Scalar(0, 0, 255);
+
+        cv::Size sz =
+            cv::getTextSize(str, cv::FONT_HERSHEY_PLAIN, 1, 1, &baseline);
+
+        cv::putText(
+            im, str, cv::Point(
+                         (i + n_) * space + (space - sz.width) / 2,
+                         (j + n_) * space + (space + sz.height) / 2),
+            cv::FONT_HERSHEY_PLAIN, 1, color, 1);
+      }
+    }
+
+    if (getinfo) {
+      std::string info = getinfo(depth, point_, n_);
+      if (!info.empty()) {
+        cv::Size sz =
+            cv::getTextSize(info, cv::FONT_HERSHEY_PLAIN, 1, 1, &baseline);
+
+        cv::putText(
+            im, info, cv::Point(5, 5 + sz.height), cv::FONT_HERSHEY_PLAIN, 1,
+            cv::Scalar(255, 0, 255), 1);
+      }
+    }
+
+    cv::imshow("region", im);
+  }
+
+  void DrawRect(cv::Mat &image) {  // NOLINT
+    if (!show_)
+      return;
+    std::uint32_t n = (n_ > 1) ? n_ : 1;
+    n += 1;  // outside the region
+    cv::rectangle(
+        image, cv::Point(point_.x - n, point_.y - n),
+        cv::Point(point_.x + n, point_.y + n),
+        selected_ ? cv::Scalar(0, 255, 0) : cv::Scalar(0, 0, 255), 1);
+  }
+
+ private:
+  std::uint32_t n_;
+  bool show_;
+  bool selected_;
+  cv::Point point_;
+};
+
+void OnDepthMouseCallback(int event, int x, int y, int flags, void *userdata) {
+  DepthRegion *region = reinterpret_cast<DepthRegion *>(userdata);
+  region->OnMouse(event, x, y, flags);
+}
+
+}  // namespace
+
+MYNTEYE_USE_NAMESPACE
+
+int main(int argc, char *argv[]) {
+  auto &&api = API::Create(argc, argv);
+  if (!api)
+    return 1;
+
+  api->SetOptionValue(Option::IR_CONTROL, 80);
+
+  api->EnableStreamData(Stream::DISPARITY_NORMALIZED);
+  api->EnableStreamData(Stream::DEPTH);
+
+  api->Start(Source::VIDEO_STREAMING);
+
+  cv::namedWindow("frame", WIN_FLAGS);
+  cv::namedWindow("depth", WIN_FLAGS);
+  cv::namedWindow("region", WIN_FLAGS);
+
+  DepthRegion depth_region(3);
+  auto depth_info = [](
+      const cv::Mat &depth, const cv::Point &point, const std::uint32_t &n) {
+    UNUSED(depth)
+    std::ostringstream os;
+    os << "depth pos: [" << point.y << ", " << point.x << "]"
+       << "±" << n << ", unit: mm";
+    return os.str();
+  };
+
+  while (true) {
+    api->WaitForStreams();
+
+    auto &&left_data = api->GetStreamData(Stream::LEFT);
+    auto &&right_data = api->GetStreamData(Stream::RIGHT);
+
+    cv::Mat img;
+    cv::hconcat(left_data.frame, right_data.frame, img);
+    cv::imshow("frame", img);
+
+    auto &&disp_data = api->GetStreamData(Stream::DISPARITY_NORMALIZED);
+    auto &&depth_data = api->GetStreamData(Stream::DEPTH);
+    if (!disp_data.frame.empty() && !depth_data.frame.empty()) {
+      // Show disparity instead of depth, but show depth values in region.
+      auto &&depth_frame = disp_data.frame;
+
+#ifdef USE_OPENCV3
+      // ColormapTypes
+      //   http://docs.opencv.org/master/d3/d50/group__imgproc__colormap.html#ga9a805d8262bcbe273f16be9ea2055a65
+      cv::applyColorMap(depth_frame, depth_frame, cv::COLORMAP_JET);
+#endif
+
+      cv::setMouseCallback("depth", OnDepthMouseCallback, &depth_region);
+      // Note: DrawRect will change some depth values to show the rect.
+      depth_region.DrawRect(depth_frame);
+
+      cv::imshow("depth", depth_frame);
+
+      depth_region.ShowElems<ushort>(
+          depth_data.frame,
+          [](const ushort &elem) {
+            if (elem >= 10000) {
+              // Filter errors, or limit to valid range.
+              //
+              // reprojectImageTo3D(), missing values will set to 10000
+              //   https://docs.opencv.org/master/d9/d0c/group__calib3d.html#ga1bc1152bd57d63bc524204f21fde6e02
+              return std::string("invalid");
+            }
+            return std::to_string(elem);
+          },
+          80, depth_info);
+    }
+
+    char key = static_cast<char>(cv::waitKey(1));
+    if (key == 27 || key == 'q' || key == 'Q') {  // ESC/Q
+      break;
+    }
+  }
+
+  api->Stop(Source::VIDEO_STREAMING);
+  return 0;
+}

samples/tutorials/CMakeLists.txt

@@ -86,11 +86,13 @@ else()
   message(WARNING "PCL not found :(")
 endif()
 
-# data
+# beginner level
 
-make_executable2(get_device_info SRCS data/get_device_info.cc)
-make_executable2(get_img_params SRCS data/get_img_params.cc)
-make_executable2(get_imu_params SRCS data/get_imu_params.cc)
+## data
+
+make_executable2(get_device_info SRCS data/get_device_info.cc WITH_OPENCV)
+make_executable2(get_img_params SRCS data/get_img_params.cc WITH_OPENCV)
+make_executable2(get_imu_params SRCS data/get_imu_params.cc WITH_OPENCV)
 
 make_executable2(get_stereo SRCS data/get_stereo.cc WITH_OPENCV)
 make_executable2(get_stereo_rectified SRCS data/get_stereo_rectified.cc WITH_OPENCV)
@@ -109,7 +111,7 @@ make_executable2(get_from_callbacks
 )
 make_executable2(get_with_plugin SRCS data/get_with_plugin.cc WITH_OPENCV)
 
-# control
+## control
 
 make_executable2(ctrl_framerate SRCS control/framerate.cc WITH_OPENCV)
 make_executable2(ctrl_auto_exposure
@@ -123,3 +125,20 @@ make_executable2(ctrl_manual_exposure
 make_executable2(ctrl_infrared SRCS control/infrared.cc WITH_OPENCV)
 
 endif()
+
+# intermediate level
+
+if(PCL_FOUND)
+
+if(OpenCV_VERSION VERSION_LESS 4.0)
+
+make_executable2(get_depth_and_points
+  SRCS intermediate/get_depth_and_points.cc util/cv_painter.cc util/pc_viewer.cc
+  WITH_OPENCV WITH_PCL
+)
+
+endif()
+
+endif()
+
+# advanced level

samples/tutorials/intermediate/get_depth_and_points.cc

@@ -0,0 +1,244 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include <glog/logging.h>
+
+#include "mynteye/api.h"
+
+#include "util/cv_painter.h"
+#include "util/pc_viewer.h"
+
+#define WIN_FLAGS \
+  cv::WINDOW_AUTOSIZE | cv::WINDOW_KEEPRATIO | cv::WINDOW_GUI_NORMAL
+
+namespace {
+
+class DepthRegion {
+ public:
+  explicit DepthRegion(std::uint32_t n)
+      : n_(std::move(n)), show_(false), selected_(false), point_(0, 0) {}
+
+  ~DepthRegion() = default;
+
+  /**
+   * 鼠标事件：默认不选中区域，随鼠标移动而显示。单击后，则会选中区域来显示。你可以再单击已选中区域或双击未选中区域，取消选中。
+   */
+  void OnMouse(const int &event, const int &x, const int &y, const int &flags) {
+    UNUSED(flags)
+    if (event != CV_EVENT_MOUSEMOVE && event != CV_EVENT_LBUTTONDOWN) {
+      return;
+    }
+    show_ = true;
+
+    if (event == CV_EVENT_MOUSEMOVE) {
+      if (!selected_) {
+        point_.x = x;
+        point_.y = y;
+      }
+    } else if (event == CV_EVENT_LBUTTONDOWN) {
+      if (selected_) {
+        if (x >= static_cast<int>(point_.x - n_) &&
+            x <= static_cast<int>(point_.x + n_) &&
+            y >= static_cast<int>(point_.y - n_) &&
+            y <= static_cast<int>(point_.y + n_)) {
+          selected_ = false;
+        }
+      } else {
+        selected_ = true;
+      }
+      point_.x = x;
+      point_.y = y;
+    }
+  }
+
+  template <typename T>
+  void ShowElems(
+      const cv::Mat &depth,
+      std::function<std::string(const T &elem)> elem2string,
+      int elem_space = 40,
+      std::function<std::string(
+          const cv::Mat &depth, const cv::Point &point, const std::uint32_t &n)>
+          getinfo = nullptr) {
+    if (!show_)
+      return;
+
+    int space = std::move(elem_space);
+    int n = 2 * n_ + 1;
+    cv::Mat im(space * n, space * n, CV_8UC3, cv::Scalar(255, 255, 255));
+
+    int x, y;
+    std::string str;
+    int baseline = 0;
+    for (int i = -n_; i <= n; ++i) {
+      x = point_.x + i;
+      if (x < 0 || x >= depth.cols)
+        continue;
+      for (int j = -n_; j <= n; ++j) {
+        y = point_.y + j;
+        if (y < 0 || y >= depth.rows)
+          continue;
+
+        str = elem2string(depth.at<T>(y, x));
+
+        cv::Scalar color(0, 0, 0);
+        if (i == 0 && j == 0)
+          color = cv::Scalar(0, 0, 255);
+
+        cv::Size sz =
+            cv::getTextSize(str, cv::FONT_HERSHEY_PLAIN, 1, 1, &baseline);
+
+        cv::putText(
+            im, str, cv::Point(
+                         (i + n_) * space + (space - sz.width) / 2,
+                         (j + n_) * space + (space + sz.height) / 2),
+            cv::FONT_HERSHEY_PLAIN, 1, color, 1);
+      }
+    }
+
+    if (getinfo) {
+      std::string info = getinfo(depth, point_, n_);
+      if (!info.empty()) {
+        cv::Size sz =
+            cv::getTextSize(info, cv::FONT_HERSHEY_PLAIN, 1, 1, &baseline);
+
+        cv::putText(
+            im, info, cv::Point(5, 5 + sz.height), cv::FONT_HERSHEY_PLAIN, 1,
+            cv::Scalar(255, 0, 255), 1);
+      }
+    }
+
+    cv::imshow("region", im);
+  }
+
+  void DrawRect(cv::Mat &image) {  // NOLINT
+    if (!show_)
+      return;
+    std::uint32_t n = (n_ > 1) ? n_ : 1;
+    n += 1;  // outside the region
+    cv::rectangle(
+        image, cv::Point(point_.x - n, point_.y - n),
+        cv::Point(point_.x + n, point_.y + n),
+        selected_ ? cv::Scalar(0, 255, 0) : cv::Scalar(0, 0, 255), 1);
+  }
+
+ private:
+  std::uint32_t n_;
+  bool show_;
+  bool selected_;
+  cv::Point point_;
+};
+
+void OnDepthMouseCallback(int event, int x, int y, int flags, void *userdata) {
+  DepthRegion *region = reinterpret_cast<DepthRegion *>(userdata);
+  region->OnMouse(event, x, y, flags);
+}
+
+}  // namespace
+
+MYNTEYE_USE_NAMESPACE
+
+int main(int argc, char *argv[]) {
+  auto &&api = API::Create(argc, argv);
+  if (!api)
+    return 1;
+
+  api->SetOptionValue(Option::IR_CONTROL, 80);
+
+  api->EnableStreamData(Stream::DISPARITY_NORMALIZED);
+  api->EnableStreamData(Stream::DEPTH);
+
+  api->Start(Source::VIDEO_STREAMING);
+
+  cv::namedWindow("frame", WIN_FLAGS);
+  cv::namedWindow("depth", WIN_FLAGS);
+  cv::namedWindow("region", WIN_FLAGS);
+
+  DepthRegion depth_region(3);
+  auto depth_info = [](
+      const cv::Mat &depth, const cv::Point &point, const std::uint32_t &n) {
+    UNUSED(depth)
+    std::ostringstream os;
+    os << "depth pos: [" << point.y << ", " << point.x << "]"
+       << "±" << n << ", unit: mm";
+    return os.str();
+  };
+
+  CVPainter painter;
+  PCViewer pcviewer;
+
+  while (true) {
+    api->WaitForStreams();
+
+    auto &&left_data = api->GetStreamData(Stream::LEFT);
+    auto &&right_data = api->GetStreamData(Stream::RIGHT);
+
+    cv::Mat img;
+    cv::hconcat(left_data.frame, right_data.frame, img);
+
+    painter.DrawImgData(img, *left_data.img);
+
+    cv::imshow("frame", img);
+
+    auto &&disp_data = api->GetStreamData(Stream::DISPARITY_NORMALIZED);
+    auto &&depth_data = api->GetStreamData(Stream::DEPTH);
+    if (!disp_data.frame.empty() && !depth_data.frame.empty()) {
+      // Show disparity instead of depth, but show depth values in region.
+      auto &&depth_frame = disp_data.frame;
+
+#ifdef USE_OPENCV3
+      // ColormapTypes
+      //   http://docs.opencv.org/master/d3/d50/group__imgproc__colormap.html#ga9a805d8262bcbe273f16be9ea2055a65
+      cv::applyColorMap(depth_frame, depth_frame, cv::COLORMAP_JET);
+#endif
+
+      cv::setMouseCallback("depth", OnDepthMouseCallback, &depth_region);
+      // Note: DrawRect will change some depth values to show the rect.
+      depth_region.DrawRect(depth_frame);
+
+      cv::imshow("depth", depth_frame);
+
+      depth_region.ShowElems<ushort>(
+          depth_data.frame,
+          [](const ushort &elem) {
+            if (elem >= 10000) {
+              // Filter errors, or limit to valid range.
+              //
+              // reprojectImageTo3D(), missing values will set to 10000
+              //   https://docs.opencv.org/master/d9/d0c/group__calib3d.html#ga1bc1152bd57d63bc524204f21fde6e02
+              return std::string("invalid");
+            }
+            return std::to_string(elem);
+          },
+          80, depth_info);
+    }
+
+    auto &&points_data = api->GetStreamData(Stream::POINTS);
+    if (!points_data.frame.empty()) {
+      pcviewer.Update(points_data.frame);
+    }
+
+    char key = static_cast<char>(cv::waitKey(1));
+    if (key == 27 || key == 'q' || key == 'Q') {  // ESC/Q
+      break;
+    }
+    if (pcviewer.WasStopped()) {
+      break;
+    }
+  }
+
+  api->Stop(Source::VIDEO_STREAMING);
+  return 0;
+}

samples/tutorials/util/pc_viewer.cc

@@ -35,7 +35,7 @@ std::shared_ptr<pcl::visualization::PCLVisualizer> CustomColorVis(
   // viewer->addCoordinateSystem(1.0);
   viewer->addCoordinateSystem(1000.0);
   viewer->initCameraParameters();
-  viewer->setCameraPosition(0, 0, -150, 0, 1, 0);
+  viewer->setCameraPosition(0, 0, -150, 0, -1, 0);
   return (viewer);
 }
 

samples/uvc/camera.cc

@@ -27,8 +27,17 @@
 
 struct frame {
   const void *data = nullptr;
+  std::function<void()> continuation = nullptr;
+  frame() {
+    // VLOG(2) << __func__;
+  }
   ~frame() {
+    // VLOG(2) << __func__;
     data = nullptr;
+    if (continuation) {
+      continuation();
+      continuation = nullptr;
+    }
   }
 };
 
@@ -41,7 +50,11 @@ int main(int argc, char *argv[]) {
 
   auto context = uvc::create_context();
   auto devices = uvc::query_devices(context);
-  LOG_IF(FATAL, devices.size() <= 0) << "No devices :(";
+  if (devices.size() <= 0) {
+    LOG(ERROR) << "No devices :(";
+    return 1;
+  }
+
   for (auto &&device : devices) {
     auto vid = uvc::get_vendor_id(*device);
     // auto pid = uvc::get_product_id(*device);
@@ -54,7 +67,10 @@ int main(int argc, char *argv[]) {
   // std::string dashes(80, '-');
 
   size_t n = mynteye_devices.size();
-  LOG_IF(FATAL, n <= 0) << "No MYNT EYE devices :(";
+  if (n <= 0) {
+    LOG(ERROR) << "No MYNT EYE devices :(";
+    return 1;
+  }
 
   LOG(INFO) << "MYNT EYE devices: ";
   for (size_t i = 0; i < n; i++) {
@@ -87,18 +103,25 @@ int main(int argc, char *argv[]) {
   std::mutex mtx;
   std::condition_variable cv;
 
-  std::vector<frame> frames;
-  const auto frame_ready = [&frames]() { return !frames.empty(); };
-  const auto frame_empty = [&frames]() { return frames.empty(); };
+  std::shared_ptr<frame> frame = nullptr;
+  const auto frame_ready = [&frame]() { return frame != nullptr; };
+  const auto frame_empty = [&frame]() { return frame == nullptr; };
 
   uvc::set_device_mode(
       *device, 752, 480, static_cast<int>(Format::YUYV), 25,
-      [&mtx, &cv, &frames, &frame_ready](const void *data) {
+      [&mtx, &cv, &frame, &frame_ready](
+          const void *data, std::function<void()> continuation) {
         // reinterpret_cast<const std::uint8_t *>(data);
         std::unique_lock<std::mutex> lock(mtx);
-        frame frame;
-        frame.data = data;  // not copy
-        frames.push_back(frame);
+        if (frame == nullptr) {
+          frame = std::make_shared<struct frame>();
+        } else {
+          if (frame->continuation) {
+            frame->continuation();
+          }
+        }
+        frame->data = data;  // not copy here
+        frame->continuation = continuation;
         if (frame_ready())
           cv.notify_one();
       });
@@ -119,13 +142,12 @@ int main(int argc, char *argv[]) {
         throw std::runtime_error("Timeout waiting for frame.");
     }
 
-    auto frame = frames.back();  // only last one is valid
-
-    cv::Mat img(480, 752, CV_8UC2, const_cast<void *>(frame.data));
+    // only lastest frame is valid
+    cv::Mat img(480, 752, CV_8UC2, const_cast<void *>(frame->data));
     cv::cvtColor(img, img, cv::COLOR_YUV2BGR_YUY2);
     cv::imshow("frame", img);
 
-    frames.clear();
+    frame = nullptr;
 
     char key = static_cast<char>(cv::waitKey(1));
     if (key == 27 || key == 'q' || key == 'Q') {  // ESC/Q

src/api/api.cc

@@ -13,8 +13,14 @@
 // limitations under the License.
 #include "api/api.h"
 
+#ifdef WITH_BOOST_FILESYSTEM
+#include <boost/filesystem.hpp>
+#include <boost/range/iterator_range.hpp>
+#endif
+
 #include <glog/logging.h>
 
+#include <algorithm>
 #include <thread>
 
 #include "mynteye/glog_init.h"
@@ -25,8 +31,182 @@
 #include "device/device.h"
 #include "internal/dl.h"
 
+#if defined(WITH_FILESYSTEM) && defined(WITH_NATIVE_FILESYSTEM)
+#if defined(OS_WIN)
+#include <windows.h>
+#endif
+#endif
+
 MYNTEYE_BEGIN_NAMESPACE
 
+namespace {
+
+#if defined(WITH_FILESYSTEM)
+
+#if defined(WITH_BOOST_FILESYSTEM)
+
+namespace fs = boost::filesystem;
+
+bool file_exists(const fs::path &p) {
+  try {
+    fs::file_status s = fs::status(p);
+    return fs::exists(s) && fs::is_regular_file(s);
+  } catch (fs::filesystem_error &e) {
+    LOG(ERROR) << e.what();
+    return false;
+  }
+}
+
+bool dir_exists(const fs::path &p) {
+  try {
+    fs::file_status s = fs::status(p);
+    return fs::exists(s) && fs::is_directory(s);
+  } catch (fs::filesystem_error &e) {
+    LOG(ERROR) << e.what();
+    return false;
+  }
+}
+
+#elif defined(WITH_NATIVE_FILESYSTEM)
+
+#if defined(OS_WIN)
+
+bool file_exists(const std::string &p) {
+  DWORD attrs = GetFileAttributes(p.c_str());
+  return (attrs != INVALID_FILE_ATTRIBUTES && !(attrs & FILE_ATTRIBUTE_DIRECTORY));
+}
+
+bool dir_exists(const std::string &p) {
+  DWORD attrs = GetFileAttributes(p.c_str());
+  return (attrs != INVALID_FILE_ATTRIBUTES && (attrs & FILE_ATTRIBUTE_DIRECTORY));
+}
+
+#else
+#error "Unsupported native filesystem"
+#endif
+
+#endif
+
+std::vector<std::string> get_plugin_paths() {
+  std::string info_path(MYNTEYE_SDK_INSTALL_DIR);
+  info_path.append(OS_SEP "share" OS_SEP "mynteye" OS_SEP "build.info");
+
+  cv::FileStorage fs(info_path, cv::FileStorage::READ);
+  if (!fs.isOpened()) {
+    LOG(WARNING) << "build.info not found: " << info_path;
+    return {};
+  }
+
+  auto to_lower = [](std::string &s) {  // NOLINT
+    std::transform(s.begin(), s.end(), s.begin(), ::tolower);
+  };
+
+  std::string host_os = fs["HOST_OS"];
+  to_lower(host_os);
+  std::string host_name = fs["HOST_NAME"];
+  to_lower(host_name);
+  std::string host_arch = fs["HOST_ARCH"];
+  to_lower(host_arch);
+  std::string host_compiler = fs["HOST_COMPILER"];
+  to_lower(host_compiler);
+
+  // std::string compiler_version = fs["COMPILER_VERSION"];
+  int compiler_version_major = fs["COMPILER_VERSION_MAJOR"];
+  // int compiler_version_minor = fs["COMPILER_VERSION_MINOR"];
+  // int compiler_version_patch = fs["COMPILER_VERSION_PATCH"];
+  // int compiler_version_tweak = fs["COMPILER_VERSION_TWEAK"];
+
+  std::string cuda_version = fs["CUDA_VERSION"];
+  // int cuda_version_major = fs["CUDA_VERSION_MAJOR"];
+  // int cuda_version_minor = fs["CUDA_VERSION_MINOR"];
+  // std::string cuda_version_string = fs["CUDA_VERSION_STRING"];
+
+  std::string opencv_version = fs["OpenCV_VERSION"];
+  // int opencv_version_major = fs["OpenCV_VERSION_MAJOR"];
+  // int opencv_version_minor = fs["OpenCV_VERSION_MINOR"];
+  // int opencv_version_patch = fs["OpenCV_VERSION_PATCH"];
+  // int opencv_version_tweak = fs["OpenCV_VERSION_TWEAK"];
+  // std::string opencv_version_status = fs["OpenCV_VERSION_STATUS"];
+  std::string opencv_with_world = fs["OpenCV_WITH_WORLD"];
+  to_lower(opencv_with_world);
+
+  std::string mynteye_version = fs["MYNTEYE_VERSION"];
+  // int mynteye_version_major = fs["MYNTEYE_VERSION_MAJOR"];
+  // int mynteye_version_minor = fs["MYNTEYE_VERSION_MINOR"];
+  // int mynteye_version_patch = fs["MYNTEYE_VERSION_PATCH"];
+  // int mynteye_version_tweak = fs["MYNTEYE_VERSION_TWEAK"];
+
+  fs.release();
+
+  std::string lib_prefix;
+  std::string lib_suffix;
+  if (host_os == "linux") {
+    if (host_compiler != "gnu" || compiler_version_major < 5)
+      return {};
+    lib_prefix = "lib";
+    lib_suffix = ".so";
+  } else if (host_os == "win") {
+    lib_prefix = "";
+    lib_suffix = ".dll";
+  } else if (host_os == "mac") {
+    lib_prefix = "lib";
+    lib_suffix = ".dylib";
+  } else {
+    return {};
+  }
+
+  std::vector<std::string> names;
+  {
+    std::vector<std::string> prefixes{
+        // lib_prefix + "plugin_b_ocl" + ocl_version,
+        lib_prefix + "plugin_g_cuda" + cuda_version,
+    };
+    std::string opencv_name("_opencv" + opencv_version);
+    if (opencv_with_world == "true") {
+      opencv_name.append("-world");
+    }
+    for (auto &&prefix : prefixes) {
+      names.push_back(prefix + opencv_name + "_mynteye" + mynteye_version);
+      names.push_back(prefix + opencv_name);
+      names.push_back(prefix);
+    }
+    for (auto &&name : names) {
+      name.append(lib_suffix);
+    }
+  }
+
+  std::vector<std::string> paths;
+
+  std::vector<std::string> plats;
+  if (host_name != host_os) {
+    plats.push_back(host_name + "-" + host_arch);
+  }
+  plats.push_back(host_os + "-" + host_arch);
+
+  std::vector<std::string> dirs{MYNTEYE_SDK_ROOT_DIR, MYNTEYE_SDK_INSTALL_DIR};
+  for (auto &&plat : plats) {
+    for (auto &&dir : dirs) {
+      auto &&plat_dir = dir + OS_SEP "plugins" + OS_SEP + plat;
+      // VLOG(2) << "plat_dir: " << plat_dir;
+      if (!dir_exists(plat_dir))
+        continue;
+      for (auto &&name : names) {
+        // VLOG(2) << "  name: " << name;
+        auto &&path = plat_dir + OS_SEP + name;
+        if (!file_exists(path))
+          continue;
+        paths.push_back(path);
+      }
+    }
+  }
+
+  return paths;
+}
+
+#endif
+
+}  // namespace
+
 API::API(std::shared_ptr<Device> device)
     : device_(device), synthetic_(new Synthetic(this)) {
   VLOG(2) << __func__;
@@ -140,7 +320,7 @@ void API::SetMotionCallback(motion_callback_t callback) {
   static auto callback_ = callback;
   if (callback_) {
     device_->SetMotionCallback(
-        [](const device::MotionData &data) { callback_({data.imu}); });
+        [](const device::MotionData &data) { callback_({data.imu}); }, true);
   } else {
     device_->SetMotionCallback(nullptr);
   }
@@ -156,6 +336,18 @@ bool API::HasMotionCallback() const {
 
 void API::Start(const Source &source) {
   if (source == Source::VIDEO_STREAMING) {
+#ifdef WITH_FILESYSTEM
+    if (!synthetic_->HasPlugin()) {
+      try {
+        auto &&plugin_paths = get_plugin_paths();
+        if (plugin_paths.size() > 0) {
+          EnablePlugin(plugin_paths[0]);
+        }
+      } catch (...) {
+        LOG(WARNING) << "Incorrect yaml format: build.info";
+      }
+    }
+#endif
     synthetic_->StartVideoStreaming();
   } else if (source == Source::MOTION_TRACKING) {
     device_->StartMotionTracking();
@@ -163,7 +355,7 @@ void API::Start(const Source &source) {
     Start(Source::VIDEO_STREAMING);
     Start(Source::MOTION_TRACKING);
   } else {
-    LOG(FATAL) << "Unsupported source :(";
+    LOG(ERROR) << "Unsupported source :(";
   }
 }
 
@@ -178,7 +370,7 @@ void API::Stop(const Source &source) {
     std::this_thread::sleep_for(std::chrono::milliseconds(10));
     Stop(Source::VIDEO_STREAMING);
   } else {
-    LOG(FATAL) << "Unsupported source :(";
+    LOG(ERROR) << "Unsupported source :(";
   }
 }
 
@@ -220,7 +412,9 @@ void API::EnablePlugin(const std::string &path) {
 
   plugin_version_code_t *plugin_version_code =
       dl.Sym<plugin_version_code_t>("plugin_version_code");
-  LOG(INFO) << "Enable plugin, version code: " << plugin_version_code();
+  LOG(INFO) << "Enable plugin success";
+  LOG(INFO) << "  version code: " << plugin_version_code();
+  LOG(INFO) << "  path: " << path;
 
   plugin_create_t *plugin_create = dl.Sym<plugin_create_t>("plugin_create");
   plugin_destroy_t *plugin_destroy = dl.Sym<plugin_destroy_t>("plugin_destroy");

src/api/api.h

@@ -30,6 +30,12 @@ MYNTEYE_BEGIN_NAMESPACE
 class Device;
 class Synthetic;
 
+namespace device {
+
+class Frame;
+
+}  // namespace device
+
 namespace api {
 
 /**
@@ -41,6 +47,8 @@ struct MYNTEYE_API StreamData {
   std::shared_ptr<ImgData> img;
   /** Frame. */
   cv::Mat frame;
+  /** Raw frame. */
+  std::shared_ptr<device::Frame> frame_raw;
 
   bool operator==(const StreamData &other) const {
     if (img && other.img) {

src/api/processor/depth_processor.cc

@@ -15,10 +15,13 @@
 
 #include <glog/logging.h>
 
+#include <utility>
+
 MYNTEYE_BEGIN_NAMESPACE
 
-DepthProcessor::DepthProcessor() : Processor() {
-  VLOG(2) << __func__;
+DepthProcessor::DepthProcessor(std::int32_t proc_period)
+    : Processor(std::move(proc_period)) {
+  VLOG(2) << __func__ << ": proc_period=" << proc_period;
 }
 
 DepthProcessor::~DepthProcessor() {

src/api/processor/depth_processor.h

@@ -25,7 +25,7 @@ class DepthProcessor : public Processor {
  public:
   static constexpr auto &&NAME = "DepthProcessor";
 
-  DepthProcessor();
+  explicit DepthProcessor(std::int32_t proc_period = 0);
   virtual ~DepthProcessor();
 
   std::string Name() override;

src/api/processor/disparity_normalized_processor.cc

@@ -17,10 +17,14 @@
 
 #include <glog/logging.h>
 
+#include <utility>
+
 MYNTEYE_BEGIN_NAMESPACE
 
-DisparityNormalizedProcessor::DisparityNormalizedProcessor() : Processor() {
-  VLOG(2) << __func__;
+DisparityNormalizedProcessor::DisparityNormalizedProcessor(
+    std::int32_t proc_period)
+    : Processor(std::move(proc_period)) {
+  VLOG(2) << __func__ << ": proc_period=" << proc_period;
 }
 
 DisparityNormalizedProcessor::~DisparityNormalizedProcessor() {

src/api/processor/disparity_normalized_processor.h

@@ -25,7 +25,7 @@ class DisparityNormalizedProcessor : public Processor {
  public:
   static constexpr auto &&NAME = "DisparityNormalizedProcessor";
 
-  DisparityNormalizedProcessor();
+  explicit DisparityNormalizedProcessor(std::int32_t proc_period = 0);
   virtual ~DisparityNormalizedProcessor();
 
   std::string Name() override;

src/api/processor/disparity_processor.cc

@@ -17,10 +17,13 @@
 
 #include <glog/logging.h>
 
+#include <utility>
+
 MYNTEYE_BEGIN_NAMESPACE
 
-DisparityProcessor::DisparityProcessor() : Processor() {
-  VLOG(2) << __func__;
+DisparityProcessor::DisparityProcessor(std::int32_t proc_period)
+    : Processor(std::move(proc_period)) {
+  VLOG(2) << __func__ << ": proc_period=" << proc_period;
   int sgbmWinSize = 3;
   int numberOfDisparities = 64;
 

src/api/processor/disparity_processor.h

@@ -31,7 +31,7 @@ class DisparityProcessor : public Processor {
  public:
   static constexpr auto &&NAME = "DisparityProcessor";
 
-  DisparityProcessor();
+  explicit DisparityProcessor(std::int32_t proc_period = 0);
   virtual ~DisparityProcessor();
 
   std::string Name() override;

src/api/processor/points_processor.cc

@@ -21,8 +21,9 @@
 
 MYNTEYE_BEGIN_NAMESPACE
 
-PointsProcessor::PointsProcessor(cv::Mat Q) : Processor(), Q_(std::move(Q)) {
-  VLOG(2) << __func__;
+PointsProcessor::PointsProcessor(cv::Mat Q, std::int32_t proc_period)
+    : Processor(std::move(proc_period)), Q_(std::move(Q)) {
+  VLOG(2) << __func__ << ": proc_period=" << proc_period;
 }
 
 PointsProcessor::~PointsProcessor() {

src/api/processor/points_processor.h

@@ -27,7 +27,7 @@ class PointsProcessor : public Processor {
  public:
   static constexpr auto &&NAME = "PointsProcessor";
 
-  explicit PointsProcessor(cv::Mat Q);
+  explicit PointsProcessor(cv::Mat Q, std::int32_t proc_period = 0);
   virtual ~PointsProcessor();
 
   std::string Name() override;

src/api/processor/processor.cc

@@ -17,10 +17,13 @@
 
 #include <utility>
 
+#include "internal/times.h"
+
 MYNTEYE_BEGIN_NAMESPACE
 
-Processor::Processor()
-    : activated_(false),
+Processor::Processor(std::int32_t proc_period)
+    : proc_period_(std::move(proc_period)),
+      activated_(false),
       input_ready_(false),
       idle_(true),
       dropped_count_(0),
@@ -146,10 +149,34 @@ std::uint64_t Processor::GetDroppedCount() {
 
 void Processor::Run() {
   VLOG(2) << Name() << " thread start";
+
+  auto sleep = [this](const times::system_clock::time_point &time_beg) {
+    if (proc_period_ > 0) {
+      static times::system_clock::time_point time_prev = time_beg;
+      auto &&time_elapsed_ms =
+          times::count<times::milliseconds>(times::now() - time_prev);
+      time_prev = time_beg;
+
+      if (time_elapsed_ms < proc_period_) {
+        VLOG(2) << Name() << " process cost "
+                << times::count<times::milliseconds>(times::now() - time_beg)
+                << " ms, sleep " << (proc_period_ - time_elapsed_ms) << " ms";
+        std::this_thread::sleep_for(
+            std::chrono::milliseconds(proc_period_ - time_elapsed_ms));
+        return;
+      }
+    }
+    VLOG(2) << Name() << " process cost "
+            << times::count<times::milliseconds>(times::now() - time_beg)
+            << " ms";
+  };
+
   while (true) {
     std::unique_lock<std::mutex> lk(mtx_input_ready_);
     cond_input_ready_.wait(lk, [this] { return input_ready_; });
 
+    auto &&time_beg = times::now();
+
     if (!activated_) {
       SetIdle(true);
       input_ready_ = false;
@@ -194,6 +221,8 @@ void Processor::Run() {
 
     SetIdle(true);
     input_ready_ = false;
+
+    sleep(time_beg);
   }
   VLOG(2) << Name() << " thread end";
 }

src/api/processor/processor.h

@@ -37,7 +37,7 @@ class Processor /*: public std::enable_shared_from_this<Processor>*/ {
   using ProcessCallback = std::function<bool(
       Object *const in, Object *const out, Processor *const parent)>;
 
-  Processor();
+  explicit Processor(std::int32_t proc_period = 0);
   virtual ~Processor();
 
   virtual std::string Name();
@@ -80,6 +80,8 @@ class Processor /*: public std::enable_shared_from_this<Processor>*/ {
 
   void SetIdle(bool idle);
 
+  std::int32_t proc_period_;
+
   bool activated_;
 
   bool input_ready_;

src/api/processor/rectify_processor.cc

@@ -18,13 +18,16 @@
 
 #include <glog/logging.h>
 
+#include <utility>
+
 #include "device/device.h"
 
 MYNTEYE_BEGIN_NAMESPACE
 
-RectifyProcessor::RectifyProcessor(std::shared_ptr<Device> device)
-    : Processor() {
-  VLOG(2) << __func__;
+RectifyProcessor::RectifyProcessor(
+    std::shared_ptr<Device> device, std::int32_t proc_period)
+    : Processor(std::move(proc_period)) {
+  VLOG(2) << __func__ << ": proc_period=" << proc_period;
   InitParams(
       device->GetIntrinsics(Stream::LEFT), device->GetIntrinsics(Stream::RIGHT),
       device->GetExtrinsics(Stream::LEFT, Stream::RIGHT));

src/api/processor/rectify_processor.h

@@ -31,7 +31,8 @@ class RectifyProcessor : public Processor {
  public:
   static constexpr auto &&NAME = "RectifyProcessor";
 
-  explicit RectifyProcessor(std::shared_ptr<Device> device);
+  RectifyProcessor(
+      std::shared_ptr<Device> device, std::int32_t proc_period = 0);
   virtual ~RectifyProcessor();
 
   std::string Name() override;

src/api/synthetic.cc

@@ -29,6 +29,12 @@
 #include "api/processor/rectify_processor.h"
 #include "device/device.h"
 
+#define RECTIFY_PROC_PERIOD 0
+#define DISPARITY_PROC_PERIOD 0
+#define DISPARITY_NORM_PROC_PERIOD 0
+#define POINTS_PROC_PERIOD 0
+#define DEPTH_PROC_PERIOD 0
+
 MYNTEYE_BEGIN_NAMESPACE
 
 namespace {
@@ -40,7 +46,7 @@ cv::Mat frame2mat(const std::shared_ptr<device::Frame> &frame) {
 }
 
 api::StreamData data2api(const device::StreamData &data) {
-  return {data.img, frame2mat(data.frame)};
+  return {data.img, frame2mat(data.frame), data.frame};
 }
 
 }  // namespace
@@ -104,14 +110,16 @@ void Synthetic::StartVideoStreaming() {
     if (it->second == MODE_NATIVE) {
       auto &&stream = it->first;
       device->SetStreamCallback(
-          stream, [this, stream](const device::StreamData &data) {
+          stream,
+          [this, stream](const device::StreamData &data) {
             auto &&stream_data = data2api(data);
             ProcessNativeStream(stream, stream_data);
             // Need mutex if set callback after start
             if (HasStreamCallback(stream)) {
               stream_callbacks_.at(stream)(stream_data);
             }
-          });
+          },
+          true);
     }
   }
   device->Start(Source::VIDEO_STREAMING);
@@ -144,7 +152,7 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) {
         Object *out = processor->GetOutput();
         if (out != nullptr) {
           ObjMat2 *output = Object::Cast<ObjMat2>(out);
-          return {nullptr, output->first};
+          return {nullptr, output->first, nullptr};
         }
         VLOG(2) << "Rectify not ready now";
       } break;
@@ -153,7 +161,7 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) {
         Object *out = processor->GetOutput();
         if (out != nullptr) {
           ObjMat2 *output = Object::Cast<ObjMat2>(out);
-          return {nullptr, output->second};
+          return {nullptr, output->second, nullptr};
         }
         VLOG(2) << "Rectify not ready now";
       } break;
@@ -162,7 +170,7 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) {
         Object *out = processor->GetOutput();
         if (out != nullptr) {
           ObjMat *output = Object::Cast<ObjMat>(out);
-          return {nullptr, output->value};
+          return {nullptr, output->value, nullptr};
         }
         VLOG(2) << "Disparity not ready now";
       } break;
@@ -172,7 +180,7 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) {
         Object *out = processor->GetOutput();
         if (out != nullptr) {
           ObjMat *output = Object::Cast<ObjMat>(out);
-          return {nullptr, output->value};
+          return {nullptr, output->value, nullptr};
         }
         VLOG(2) << "Disparity normalized not ready now";
       } break;
@@ -181,7 +189,7 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) {
         Object *out = processor->GetOutput();
         if (out != nullptr) {
           ObjMat *output = Object::Cast<ObjMat>(out);
-          return {nullptr, output->value};
+          return {nullptr, output->value, nullptr};
         }
         VLOG(2) << "Points not ready now";
       } break;
@@ -190,7 +198,7 @@ api::StreamData Synthetic::GetStreamData(const Stream &stream) {
         Object *out = processor->GetOutput();
         if (out != nullptr) {
           ObjMat *output = Object::Cast<ObjMat>(out);
-          return {nullptr, output->value};
+          return {nullptr, output->value, nullptr};
         }
         VLOG(2) << "Depth not ready now";
       } break;
@@ -227,6 +235,10 @@ void Synthetic::SetPlugin(std::shared_ptr<Plugin> plugin) {
   plugin_ = plugin;
 }
 
+bool Synthetic::HasPlugin() const {
+  return plugin_ != nullptr;
+}
+
 void Synthetic::InitStreamSupports() {
   auto &&device = api_->device();
   if (device->Supports(Stream::LEFT) && device->Supports(Stream::RIGHT)) {
@@ -381,13 +393,16 @@ void Synthetic::DisableStreamData(const Stream &stream, std::uint32_t depth) {
 }
 
 void Synthetic::InitProcessors() {
-  auto &&rectify_processor = std::make_shared<RectifyProcessor>(api_->device());
-  auto &&disparity_processor = std::make_shared<DisparityProcessor>();
+  auto &&rectify_processor =
+      std::make_shared<RectifyProcessor>(api_->device(), RECTIFY_PROC_PERIOD);
+  auto &&disparity_processor =
+      std::make_shared<DisparityProcessor>(DISPARITY_PROC_PERIOD);
   auto &&disparitynormalized_processor =
-      std::make_shared<DisparityNormalizedProcessor>();
-  auto &&points_processor =
-      std::make_shared<PointsProcessor>(rectify_processor->Q);
-  auto &&depth_processor = std::make_shared<DepthProcessor>();
+      std::make_shared<DisparityNormalizedProcessor>(
+          DISPARITY_NORM_PROC_PERIOD);
+  auto &&points_processor = std::make_shared<PointsProcessor>(
+      rectify_processor->Q, POINTS_PROC_PERIOD);
+  auto &&depth_processor = std::make_shared<DepthProcessor>(DEPTH_PROC_PERIOD);
 
   using namespace std::placeholders;  // NOLINT
   rectify_processor->SetProcessCallback(
@@ -531,17 +546,19 @@ bool Synthetic::OnDepthProcess(
 void Synthetic::OnRectifyPostProcess(Object *const out) {
   const ObjMat2 *output = Object::Cast<ObjMat2>(out);
   if (HasStreamCallback(Stream::LEFT_RECTIFIED)) {
-    stream_callbacks_.at(Stream::LEFT_RECTIFIED)({nullptr, output->first});
+    stream_callbacks_.at(Stream::LEFT_RECTIFIED)(
+        {nullptr, output->first, nullptr});
   }
   if (HasStreamCallback(Stream::RIGHT_RECTIFIED)) {
-    stream_callbacks_.at(Stream::RIGHT_RECTIFIED)({nullptr, output->second});
+    stream_callbacks_.at(Stream::RIGHT_RECTIFIED)(
+        {nullptr, output->second, nullptr});
   }
 }
 
 void Synthetic::OnDisparityPostProcess(Object *const out) {
   const ObjMat *output = Object::Cast<ObjMat>(out);
   if (HasStreamCallback(Stream::DISPARITY)) {
-    stream_callbacks_.at(Stream::DISPARITY)({nullptr, output->value});
+    stream_callbacks_.at(Stream::DISPARITY)({nullptr, output->value, nullptr});
   }
 }
 
@@ -549,21 +566,21 @@ void Synthetic::OnDisparityNormalizedPostProcess(Object *const out) {
   const ObjMat *output = Object::Cast<ObjMat>(out);
   if (HasStreamCallback(Stream::DISPARITY_NORMALIZED)) {
     stream_callbacks_.at(Stream::DISPARITY_NORMALIZED)(
-        {nullptr, output->value});
+        {nullptr, output->value, nullptr});
   }
 }
 
 void Synthetic::OnPointsPostProcess(Object *const out) {
   const ObjMat *output = Object::Cast<ObjMat>(out);
   if (HasStreamCallback(Stream::POINTS)) {
-    stream_callbacks_.at(Stream::POINTS)({nullptr, output->value});
+    stream_callbacks_.at(Stream::POINTS)({nullptr, output->value, nullptr});
   }
 }
 
 void Synthetic::OnDepthPostProcess(Object *const out) {
   const ObjMat *output = Object::Cast<ObjMat>(out);
   if (HasStreamCallback(Stream::DEPTH)) {
-    stream_callbacks_.at(Stream::DEPTH)({nullptr, output->value});
+    stream_callbacks_.at(Stream::DEPTH)({nullptr, output->value, nullptr});
   }
 }
 

src/api/synthetic.h

@@ -25,10 +25,11 @@
 MYNTEYE_BEGIN_NAMESPACE
 
 class API;
-class Object;
 class Plugin;
 class Processor;
 
+struct Object;
+
 class Synthetic {
  public:
   using stream_callback_t = API::stream_callback_t;
@@ -61,6 +62,7 @@ class Synthetic {
   std::vector<api::StreamData> GetStreamDatas(const Stream &stream);
 
   void SetPlugin(std::shared_ptr<Plugin> plugin);
+  bool HasPlugin() const;
 
  private:
   void InitStreamSupports();

src/device/device.cc

@@ -20,11 +20,13 @@
 #include <utility>
 
 #include "device/device_s.h"
+#include "internal/async_callback.h"
 #include "internal/channels.h"
 #include "internal/config.h"
 #include "internal/motions.h"
 #include "internal/streams.h"
 #include "internal/strings.h"
+#include "internal/times.h"
 #include "internal/types.h"
 #include "uvc/uvc.h"
 
@@ -112,7 +114,9 @@ bool Device::Supports(const AddOns &addon) const {
       return hw_flag[0];
     case AddOns::INFRARED2:
       return hw_flag[1];
-    default: { LOG(FATAL) << "Unknown add-on"; }
+    default:
+      LOG(WARNING) << "Unknown add-on";
+      return false;
   }
 }
 
@@ -135,8 +139,9 @@ void Device::ConfigStreamRequest(
   auto &&requests = GetStreamRequests(capability);
   if (std::find(requests.cbegin(), requests.cend(), request) ==
       requests.cend()) {
-    LOG(FATAL) << "Config stream request of " << capability
-               << " is not accpected";
+    LOG(WARNING) << "Config stream request of " << capability
+                 << " is not accpected";
+    return;
   }
   stream_config_requests_[capability] = request;
 }
@@ -148,31 +153,25 @@ std::shared_ptr<DeviceInfo> Device::GetInfo() const {
 std::string Device::GetInfo(const Info &info) const {
   CHECK_NOTNULL(device_info_);
   switch (info) {
-    case Info::DEVICE_NAME: {
+    case Info::DEVICE_NAME:
       return device_info_->name;
-    } break;
-    case Info::SERIAL_NUMBER: {
+    case Info::SERIAL_NUMBER:
       return device_info_->serial_number;
-    } break;
-    case Info::FIRMWARE_VERSION: {
+    case Info::FIRMWARE_VERSION:
       return device_info_->firmware_version.to_string();
-    } break;
-    case Info::HARDWARE_VERSION: {
+    case Info::HARDWARE_VERSION:
       return device_info_->hardware_version.to_string();
-    } break;
-    case Info::SPEC_VERSION: {
+    case Info::SPEC_VERSION:
       return device_info_->spec_version.to_string();
-    } break;
-    case Info::LENS_TYPE: {
+    case Info::LENS_TYPE:
       return device_info_->lens_type.to_string();
-    } break;
-    case Info::IMU_TYPE: {
+    case Info::IMU_TYPE:
       return device_info_->imu_type.to_string();
-    } break;
-    case Info::NOMINAL_BASELINE: {
+    case Info::NOMINAL_BASELINE:
       return std::to_string(device_info_->nominal_baseline);
-    } break;
-    default: { LOG(FATAL) << "Unknown device info"; }
+    default:
+      LOG(WARNING) << "Unknown device info";
+      return "";
   }
 }
 
@@ -275,20 +274,32 @@ bool Device::RunOptionAction(const Option &option) const {
 }
 
 void Device::SetStreamCallback(
-    const Stream &stream, stream_callback_t callback) {
+    const Stream &stream, stream_callback_t callback, bool async) {
   if (!Supports(stream)) {
     LOG(WARNING) << "Unsupported stream: " << stream;
     return;
   }
   if (callback) {
     stream_callbacks_[stream] = callback;
+    if (async)
+      stream_async_callbacks_[stream] =
+          std::make_shared<stream_async_callback_t>(
+              to_string(stream), callback);
   } else {
     stream_callbacks_.erase(stream);
+    stream_async_callbacks_.erase(stream);
   }
 }
 
-void Device::SetMotionCallback(motion_callback_t callback) {
+void Device::SetMotionCallback(motion_callback_t callback, bool async) {
   motion_callback_ = callback;
+  if (callback) {
+    if (async)
+      motion_async_callback_ =
+          std::make_shared<motion_async_callback_t>("motion", callback, true);
+  } else {
+    motion_async_callback_ = nullptr;
+  }
 }
 
 bool Device::HasStreamCallback(const Stream &stream) const {
@@ -312,7 +323,7 @@ void Device::Start(const Source &source) {
     Start(Source::VIDEO_STREAMING);
     Start(Source::MOTION_TRACKING);
   } else {
-    LOG(FATAL) << "Unsupported source :(";
+    LOG(ERROR) << "Unsupported source :(";
   }
 }
 
@@ -327,7 +338,7 @@ void Device::Stop(const Source &source) {
     std::this_thread::sleep_for(std::chrono::milliseconds(10));
     Stop(Source::VIDEO_STREAMING);
   } else {
-    LOG(FATAL) << "Unsupported source :(";
+    LOG(ERROR) << "Unsupported source :(";
   }
 }
 
@@ -367,8 +378,8 @@ const StreamRequest &Device::GetStreamRequest(const Capabilities &capability) {
     return stream_config_requests_.at(capability);
   } catch (const std::out_of_range &e) {
     auto &&requests = GetStreamRequests(capability);
-    if (requests.size() == 1) {
-      VLOG(2) << "Get the only one stream request of " << capability;
+    if (requests.size() >= 1) {
+      VLOG(2) << "Select the first one stream request of " << capability;
       return requests[0];
     } else {
       LOG(FATAL) << "Please config the stream request of " << capability;
@@ -406,27 +417,27 @@ void Device::StartVideoStreaming() {
     uvc::set_device_mode(
         *device_, stream_request.width, stream_request.height,
         static_cast<int>(stream_request.format), stream_request.fps,
-        [this](const void *data) {
+        [this](const void *data, std::function<void()> continuation) {
           // drop the first stereo stream data
           static std::uint8_t drop_count = 1;
           if (drop_count > 0) {
             --drop_count;
+            continuation();
             return;
           }
-          std::lock_guard<std::mutex> _(mtx_streams_);
-          streams_->PushStream(Capabilities::STEREO, data);
-          if (HasStreamCallback(Stream::LEFT)) {
-            auto &&stream_datas = streams_->stream_datas(Stream::LEFT);
-            if (stream_datas.size() > 0) {
-              stream_callbacks_.at(Stream::LEFT)(stream_datas.back());
-            }
-          }
-          if (HasStreamCallback(Stream::RIGHT)) {
-            auto &&stream_datas = streams_->stream_datas(Stream::RIGHT);
-            if (stream_datas.size() > 0) {
-              stream_callbacks_.at(Stream::RIGHT)(stream_datas.back());
+          // auto &&time_beg = times::now();
+          {
+            std::lock_guard<std::mutex> _(mtx_streams_);
+            if (streams_->PushStream(Capabilities::STEREO, data)) {
+              CallbackPushedStreamData(Stream::LEFT);
+              CallbackPushedStreamData(Stream::RIGHT);
             }
           }
+          continuation();
+          OnStereoStreamUpdate();
+          // VLOG(2) << "Stereo video callback cost "
+          //     << times::count<times::milliseconds>(times::now() - time_beg)
+          //     << " ms";
         });
   } else {
     LOG(FATAL) << "Not any stream capabilities are supported by this device";
@@ -453,12 +464,9 @@ void Device::StartMotionTracking() {
     LOG(WARNING) << "Cannot start motion tracking without first stopping it";
     return;
   }
-  motions_->SetMotionCallback([this](const device::MotionData &data) {
-    if (motion_callback_) {
-      motion_callback_(data);
-    }
-  });
-  motions_->StartMotionTracking();
+  motions_->SetMotionCallback(
+      std::bind(&Device::CallbackMotionData, this, std::placeholders::_1));
+  // motions_->StartMotionTracking();
   motion_tracking_ = true;
 }
 
@@ -467,10 +475,12 @@ void Device::StopMotionTracking() {
     LOG(WARNING) << "Cannot stop motion tracking without first starting it";
     return;
   }
-  motions_->StopMotionTracking();
+  // motions_->StopMotionTracking();
   motion_tracking_ = false;
 }
 
+void Device::OnStereoStreamUpdate() {}
+
 void Device::ReadAllInfos() {
   device_info_ = std::make_shared<DeviceInfo>();
 
@@ -478,7 +488,8 @@ void Device::ReadAllInfos() {
   Channels::img_params_t img_params;
   Channels::imu_params_t imu_params;
   if (!channels_->GetFiles(device_info_.get(), &img_params, &imu_params)) {
-    LOG(FATAL) << "Read device infos failed :(";
+    LOG(FATAL) << "Read device infos failed. Please upgrade your firmware to "
+                  "the latest version.";
   }
   VLOG(2) << "Device info: {name: " << device_info_->name
           << ", serial_number: " << device_info_->serial_number
@@ -514,4 +525,27 @@ void Device::ReadAllInfos() {
   }
 }
 
+void Device::CallbackPushedStreamData(const Stream &stream) {
+  if (HasStreamCallback(stream)) {
+    auto &&datas = streams_->stream_datas(stream);
+    // if (datas.size() > 0) {}
+    auto &&data = datas.back();
+    if (stream_async_callbacks_.find(stream) != stream_async_callbacks_.end()) {
+      stream_async_callbacks_.at(stream)->PushData(data);
+    } else {
+      stream_callbacks_.at(stream)(data);
+    }
+  }
+}
+
+void Device::CallbackMotionData(const device::MotionData &data) {
+  if (HasMotionCallback()) {
+    if (motion_async_callback_) {
+      motion_async_callback_->PushData(data);
+    } else {
+      motion_callback_(data);
+    }
+  }
+}
+
 MYNTEYE_END_NAMESPACE

src/device/device.h

@@ -47,6 +47,9 @@ class Channels;
 class Motions;
 class Streams;
 
+template <class Data>
+class AsyncCallback;
+
 /**
  * The Device class to communicate with MYNT® EYE device.
  */
@@ -59,6 +62,11 @@ class MYNTEYE_API Device {
 
   using stream_callbacks_t = std::map<Stream, stream_callback_t>;
 
+  using stream_async_callback_t = AsyncCallback<device::StreamData>;
+  using motion_async_callback_t = AsyncCallback<device::MotionData>;
+  using stream_async_callback_ptr_t = std::shared_ptr<stream_async_callback_t>;
+  using motion_async_callback_ptr_t = std::shared_ptr<motion_async_callback_t>;
+
   Device(const Model &model, std::shared_ptr<uvc::device> device);
   virtual ~Device();
 
@@ -176,11 +184,12 @@ class MYNTEYE_API Device {
   /**
    * Set the callback of stream.
    */
-  void SetStreamCallback(const Stream &stream, stream_callback_t callback);
+  void SetStreamCallback(
+      const Stream &stream, stream_callback_t callback, bool async = false);
   /**
    * Set the callback of motion.
    */
-  void SetMotionCallback(motion_callback_t callback);
+  void SetMotionCallback(motion_callback_t callback, bool async = false);
 
   /**
    * Has the callback of stream.
@@ -230,6 +239,18 @@ class MYNTEYE_API Device {
     return device_;
   }
 
+  std::shared_ptr<Streams> streams() const {
+    return streams_;
+  }
+
+  std::shared_ptr<Channels> channels() const {
+    return channels_;
+  }
+
+  std::shared_ptr<Motions> motions() const {
+    return motions_;
+  }
+
   const StreamRequest &GetStreamRequest(const Capabilities &capability);
 
   virtual void StartVideoStreaming();
@@ -238,6 +259,8 @@ class MYNTEYE_API Device {
   virtual void StartMotionTracking();
   virtual void StopMotionTracking();
 
+  virtual void OnStereoStreamUpdate();
+
   virtual std::vector<Stream> GetKeyStreams() const = 0;
 
   bool video_streaming_;
@@ -257,6 +280,9 @@ class MYNTEYE_API Device {
   stream_callbacks_t stream_callbacks_;
   motion_callback_t motion_callback_;
 
+  std::map<Stream, stream_async_callback_ptr_t> stream_async_callbacks_;
+  motion_async_callback_ptr_t motion_async_callback_;
+
   std::shared_ptr<Streams> streams_;
 
   std::map<Capabilities, StreamRequest> stream_config_requests_;
@@ -269,9 +295,8 @@ class MYNTEYE_API Device {
 
   void ReadAllInfos();
 
-  std::shared_ptr<Channels> channels() {
-    return channels_;
-  }
+  void CallbackPushedStreamData(const Stream &stream);
+  void CallbackMotionData(const device::MotionData &data);
 
   friend API;
   friend tools::DeviceWriter;

src/device/device_s.cc

@@ -15,6 +15,8 @@
 
 #include <glog/logging.h>
 
+#include "internal/motions.h"
+
 MYNTEYE_BEGIN_NAMESPACE
 
 StandardDevice::StandardDevice(std::shared_ptr<uvc::device> device)
@@ -30,4 +32,11 @@ std::vector<Stream> StandardDevice::GetKeyStreams() const {
   return {Stream::LEFT, Stream::RIGHT};
 }
 
+void StandardDevice::OnStereoStreamUpdate() {
+  if (motion_tracking_) {
+    auto &&motions = this->motions();
+    motions->DoMotionTrack();
+  }
+}
+
 MYNTEYE_END_NAMESPACE

src/device/device_s.h

@@ -28,6 +28,8 @@ class StandardDevice : public Device {
   virtual ~StandardDevice();
 
   std::vector<Stream> GetKeyStreams() const override;
+
+  void OnStereoStreamUpdate() override;
 };
 
 MYNTEYE_END_NAMESPACE

src/internal/async_callback.h

@@ -0,0 +1,61 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef MYNTEYE_INTERNAL_ASYNC_CALLBACK_H_  // NOLINT
+#define MYNTEYE_INTERNAL_ASYNC_CALLBACK_H_
+#pragma once
+
+#include <condition_variable>
+#include <functional>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "mynteye/mynteye.h"
+
+MYNTEYE_BEGIN_NAMESPACE
+
+template <class Data>
+class AsyncCallback {
+ public:
+  using callback_t = std::function<void(Data data)>;
+
+  AsyncCallback(std::string name, callback_t callback, bool concat = false);
+  ~AsyncCallback();
+
+  void PushData(Data data);
+
+ private:
+  void Run();
+
+  std::string name_;
+
+  callback_t callback_;
+
+  std::mutex mtx_;
+  std::condition_variable cv_;
+
+  bool running_;
+  std::thread thread_;
+
+  std::uint32_t count_;
+  bool concat_;
+  std::vector<Data> datas_;
+};
+
+MYNTEYE_END_NAMESPACE
+
+#include "internal/async_callback_impl.h"
+
+#endif  // MYNTEYE_INTERNAL_ASYNC_CALLBACK_H_ NOLINT

src/internal/async_callback_impl.h

@@ -0,0 +1,90 @@
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef MYNTEYE_INTERNAL_ASYNC_CALLBACK_IMPL_H_  // NOLINT
+#define MYNTEYE_INTERNAL_ASYNC_CALLBACK_IMPL_H_
+#pragma once
+
+#include <glog/logging.h>
+
+#include <string>
+#include <utility>
+
+MYNTEYE_BEGIN_NAMESPACE
+
+template <class Data>
+AsyncCallback<Data>::AsyncCallback(
+    std::string name, callback_t callback, bool concat)
+    : name_(std::move(name)),
+      callback_(std::move(callback)),
+      count_(0),
+      concat_(concat) {
+  VLOG(2) << __func__;
+  running_ = true;
+  thread_ = std::thread(&AsyncCallback<Data>::Run, this);
+}
+
+template <class Data>
+AsyncCallback<Data>::~AsyncCallback() {
+  VLOG(2) << __func__;
+  {
+    std::lock_guard<std::mutex> _(mtx_);
+    running_ = false;
+    ++count_;
+  }
+  cv_.notify_one();
+  if (thread_.joinable()) {
+    thread_.join();
+  }
+}
+
+template <class Data>
+void AsyncCallback<Data>::PushData(Data data) {
+  std::lock_guard<std::mutex> _(mtx_);
+  if (!concat_) {
+    datas_.clear();
+  }
+  datas_.push_back(data);
+  ++count_;
+  cv_.notify_one();
+}
+
+template <class Data>
+void AsyncCallback<Data>::Run() {
+  VLOG(2) << "AsyncCallback(" << name_ << ") thread start";
+  while (true) {
+    std::unique_lock<std::mutex> lock(mtx_);
+    cv_.wait(lock, [this] { return count_ > 0; });
+
+    if (!running_)
+      break;
+
+    if (callback_) {
+      for (auto &&data : datas_) {
+        callback_(data);
+      }
+    }
+
+    if (VLOG_IS_ON(2) && count_ > datas_.size()) {
+      VLOG(2) << "AsyncCallback(" << name_ << ") dropped "
+              << (count_ - datas_.size());
+    }
+    count_ = 0;
+    datas_.clear();
+  }
+  VLOG(2) << "AsyncCallback(" << name_ << ") thread end";
+}
+
+MYNTEYE_END_NAMESPACE
+
+#endif  // MYNTEYE_INTERNAL_ASYNC_CALLBACK_IMPL_H_ NOLINT

src/internal/channels.cc

@@ -31,6 +31,13 @@ MYNTEYE_BEGIN_NAMESPACE
 
 namespace {
 
+const uvc::xu mynteye_xu = {3,
+                            2,
+                            {0x947a6d9f,
+                             0x8a2f,
+                             0x418d,
+                             {0x85, 0x9e, 0x6c, 0x9a, 0xa0, 0x38, 0x10, 0x14}}};
+
 int XuCamCtrlId(Option option) {
   switch (option) {
     case Option::EXPOSURE_MODE:
@@ -177,7 +184,7 @@ std::int32_t Channels::GetControlValue(const Option &option) const {
       LOG(WARNING) << option << " get value useless";
       return -1;
     default:
-      LOG(FATAL) << "Unsupported option " << option;
+      LOG(ERROR) << "Unsupported option " << option;
   }
   return -1;
 }
@@ -241,7 +248,7 @@ void Channels::SetControlValue(const Option &option, std::int32_t value) {
       LOG(WARNING) << option << " set value useless";
       break;
     default:
-      LOG(FATAL) << "Unsupported option " << option;
+      LOG(ERROR) << "Unsupported option " << option;
   }
 }
 
@@ -265,7 +272,8 @@ bool Channels::RunControlAction(const Option &option) const {
       LOG(WARNING) << option << " run action useless";
       return false;
     default:
-      LOG(FATAL) << "Unsupported option " << option;
+      LOG(ERROR) << "Unsupported option " << option;
+      return false;
   }
 }
 
@@ -273,6 +281,47 @@ void Channels::SetImuCallback(imu_callback_t callback) {
   imu_callback_ = callback;
 }
 
+void Channels::DoImuTrack() {
+  static ImuReqPacket req_packet{0};
+  static ImuResPacket res_packet;
+
+  req_packet.serial_number = imu_sn_;
+  if (!XuImuWrite(req_packet)) {
+    return;
+  }
+
+  if (!XuImuRead(&res_packet)) {
+    return;
+  }
+
+  if (res_packet.packets.size() == 0) {
+    return;
+  }
+
+  VLOG(2) << "Imu req sn: " << imu_sn_ << ", res count: " << []() {
+    std::size_t n = 0;
+    for (auto &&packet : res_packet.packets) {
+      n += packet.count;
+    }
+    return n;
+  }();
+
+  auto &&sn = res_packet.packets.back().serial_number;
+  if (imu_sn_ == sn) {
+    VLOG(2) << "New imu not ready, dropped";
+    return;
+  }
+  imu_sn_ = sn;
+
+  if (imu_callback_) {
+    for (auto &&packet : res_packet.packets) {
+      imu_callback_(packet);
+    }
+  }
+
+  res_packet.packets.clear();
+}
+
 void Channels::StartImuTracking(imu_callback_t callback) {
   if (is_imu_tracking_) {
     LOG(WARNING) << "Start imu tracking failed, is tracking already";
@@ -284,8 +333,6 @@ void Channels::StartImuTracking(imu_callback_t callback) {
   is_imu_tracking_ = true;
   imu_track_thread_ = std::thread([this]() {
     imu_sn_ = 0;
-    ImuReqPacket req_packet{imu_sn_};
-    ImuResPacket res_packet;
     auto sleep = [](const times::system_clock::time_point &time_beg) {
       auto &&time_elapsed_ms =
           times::count<times::milliseconds>(times::now() - time_beg);
@@ -298,48 +345,7 @@ void Channels::StartImuTracking(imu_callback_t callback) {
     };
     while (!imu_track_stop_) {
       auto &&time_beg = times::now();
-
-      req_packet.serial_number = imu_sn_;
-      if (!XuImuWrite(req_packet)) {
-        sleep(time_beg);
-        continue;
-      }
-
-      if (!XuImuRead(&res_packet)) {
-        sleep(time_beg);
-        continue;
-      }
-
-      if (res_packet.packets.size() == 0) {
-        sleep(time_beg);
-        continue;
-      }
-
-      VLOG(2) << "Imu req sn: " << imu_sn_
-              << ", res count: " << [&res_packet]() {
-                   std::size_t n = 0;
-                   for (auto &&packet : res_packet.packets) {
-                     n += packet.count;
-                   }
-                   return n;
-                 }();
-
-      auto &&sn = res_packet.packets.back().serial_number;
-      if (imu_sn_ == sn) {
-        VLOG(2) << "New imu not ready, dropped";
-        sleep(time_beg);
-        continue;
-      }
-      imu_sn_ = sn;
-
-      if (imu_callback_) {
-        for (auto &&packet : res_packet.packets) {
-          imu_callback_(packet);
-        }
-      }
-
-      res_packet.packets.clear();
-
+      DoImuTrack();
       sleep(time_beg);
     }
   });
@@ -866,10 +872,23 @@ bool Channels::PuControlQuery(
   return uvc::pu_control_query(*device_, option, query, value);
 }
 
+bool Channels::XuControlRange(
+    channel_t channel, uint8_t id, int32_t *min, int32_t *max,
+    int32_t *def) const {
+  return XuControlRange(mynteye_xu, channel, id, min, max, def);
+}
+
+bool Channels::XuControlRange(
+    const uvc::xu &xu, uint8_t selector, uint8_t id, int32_t *min, int32_t *max,
+    int32_t *def) const {
+  CHECK_NOTNULL(device_);
+  return uvc::xu_control_range(*device_, xu, selector, id, min, max, def);
+}
+
 bool Channels::XuControlQuery(
     channel_t channel, uvc::xu_query query, uint16_t size,
     uint8_t *data) const {
-  return XuControlQuery({3}, channel >> 8, query, size, data);
+  return XuControlQuery(mynteye_xu, channel, query, size, data);
 }
 
 bool Channels::XuControlQuery(
@@ -1000,32 +1019,12 @@ Channels::control_info_t Channels::PuControlInfo(Option option) const {
 Channels::control_info_t Channels::XuControlInfo(Option option) const {
   int id = XuCamCtrlId(option);
 
-  std::uint8_t data[3] = {static_cast<std::uint8_t>((id | 0x80) & 0xFF), 0, 0};
-  if (!XuCamCtrlQuery(uvc::XU_QUERY_SET, 3, data)) {
+  int32_t min = 0, max = 0, def = 0;
+  if (!XuControlRange(
+          CHANNEL_CAM_CTRL, static_cast<std::uint8_t>(id), &min, &max, &def)) {
     LOG(WARNING) << "Get XuControlInfo of " << option << " failed";
-    return {0, 0, 0};
   }
-
-  control_info_t info{0, 0, 0};
-
-  data[0] = id & 0xFF;
-  if (XuCamCtrlQuery(uvc::XU_QUERY_MIN, 3, data)) {
-    info.min = (data[1] << 8) | (data[2]);
-  } else {
-    LOG(WARNING) << "Get XuControlInfo.min of " << option << " failed";
-  }
-  if (XuCamCtrlQuery(uvc::XU_QUERY_MAX, 3, data)) {
-    info.max = (data[1] << 8) | (data[2]);
-  } else {
-    LOG(WARNING) << "Get XuControlInfo.max of " << option << " failed";
-  }
-  if (XuCamCtrlQuery(uvc::XU_QUERY_DEF, 3, data)) {
-    info.def = (data[1] << 8) | (data[2]);
-  } else {
-    LOG(WARNING) << "Get XuControlInfo.def of " << option << " failed";
-  }
-
-  return info;
+  return {min, max, def};
 }
 
 MYNTEYE_END_NAMESPACE

src/internal/channels.h

@@ -37,11 +37,11 @@ struct xu;
 class MYNTEYE_API Channels {
  public:
   typedef enum Channel {
-    CHANNEL_CAM_CTRL = 0x0100,
-    CHANNEL_HALF_DUPLEX = 0x0200,
-    CHANNEL_IMU_WRITE = 0x0300,
-    CHANNEL_IMU_READ = 0x0400,
-    CHANNEL_FILE = 0x0500,
+    CHANNEL_CAM_CTRL = 1,
+    CHANNEL_HALF_DUPLEX = 2,
+    CHANNEL_IMU_WRITE = 3,
+    CHANNEL_IMU_READ = 4,
+    CHANNEL_FILE = 5,
     CHANNEL_LAST
   } channel_t;
 
@@ -95,6 +95,8 @@ class MYNTEYE_API Channels {
   bool RunControlAction(const Option &option) const;
 
   void SetImuCallback(imu_callback_t callback);
+  void DoImuTrack();
+
   void StartImuTracking(imu_callback_t callback = nullptr);
   void StopImuTracking();
 
@@ -110,6 +112,13 @@ class MYNTEYE_API Channels {
       Option option, int32_t *min, int32_t *max, int32_t *def) const;
   bool PuControlQuery(Option option, uvc::pu_query query, int32_t *value) const;
 
+  bool XuControlRange(
+      channel_t channel, uint8_t id, int32_t *min, int32_t *max,
+      int32_t *def) const;
+  bool XuControlRange(
+      const uvc::xu &xu, uint8_t selector, uint8_t id, int32_t *min,
+      int32_t *max, int32_t *def) const;
+
   bool XuControlQuery(
       channel_t channel, uvc::xu_query query, uint16_t size,
       uint8_t *data) const;

src/internal/motions.cc

@@ -34,11 +34,8 @@ Motions::~Motions() {
 
 void Motions::SetMotionCallback(motion_callback_t callback) {
   motion_callback_ = callback;
-}
-
-void Motions::StartMotionTracking() {
-  if (!is_imu_tracking) {
-    channels_->StartImuTracking([this](const ImuPacket &packet) {
+  if (motion_callback_) {
+    channels_->SetImuCallback([this](const ImuPacket &packet) {
       if (!motion_callback_ && !motion_datas_enabled_) {
         LOG(WARNING) << "";
         return;
@@ -62,11 +59,22 @@ void Motions::StartMotionTracking() {
         std::lock_guard<std::mutex> _(mtx_datas_);
         motion_data_t data = {imu};
         motion_datas_.push_back(data);
-        if (motion_callback_) {
-          motion_callback_(data);
-        }
+
+        motion_callback_(data);
       }
     });
+  } else {
+    channels_->SetImuCallback(nullptr);
+  }
+}
+
+void Motions::DoMotionTrack() {
+  channels_->DoImuTrack();
+}
+
+void Motions::StartMotionTracking() {
+  if (!is_imu_tracking) {
+    channels_->StartImuTracking();
     is_imu_tracking = true;
   } else {
     LOG(WARNING) << "Imu is tracking already";

src/internal/motions.h

@@ -37,6 +37,7 @@ class Motions {
   ~Motions();
 
   void SetMotionCallback(motion_callback_t callback);
+  void DoMotionTrack();
 
   void StartMotionTracking();
   void StopMotionTracking();

src/internal/streams.cc

@@ -125,18 +125,20 @@ Streams::~Streams() {
 void Streams::ConfigStream(
     const Capabilities &capability, const StreamRequest &request) {
   if (!IsStreamCapability(capability)) {
-    LOG(FATAL) << "Cannot config stream without stream capability";
+    LOG(ERROR) << "Cannot config stream without stream capability";
+    return;
   }
   VLOG(2) << "Config stream request of " << capability << ", " << request;
   stream_config_requests_[capability] = request;
 }
 
-void Streams::PushStream(const Capabilities &capability, const void *data) {
+bool Streams::PushStream(const Capabilities &capability, const void *data) {
   if (!HasStreamConfigRequest(capability)) {
     LOG(FATAL) << "Cannot push stream without stream config request";
   }
   std::unique_lock<std::mutex> lock(mtx_);
   auto &&request = GetStreamConfigRequest(capability);
+  bool pushed = false;
   switch (capability) {
     case Capabilities::STEREO: {
       // alloc left
@@ -154,10 +156,12 @@ void Streams::PushStream(const Capabilities &capability, const void *data) {
             data, request, left_data.frame.get());
         unpack_img_pixels_map_[Stream::RIGHT](
             data, request, right_data.frame.get());
+        pushed = true;
       } else {
         // discard left
         DiscardStreamData(Stream::LEFT);
         LOG(WARNING) << "Image packet is unaccepted, frame dropped";
+        pushed = false;
       }
     } break;
     default:
@@ -165,13 +169,15 @@ void Streams::PushStream(const Capabilities &capability, const void *data) {
   }
   if (HasKeyStreamDatas())
     cv_.notify_one();
+  return pushed;
 }
 
 void Streams::WaitForStreams() {
   std::unique_lock<std::mutex> lock(mtx_);
   auto ready = std::bind(&Streams::HasKeyStreamDatas, this);
   if (!ready() && !cv_.wait_for(lock, std::chrono::seconds(2), ready)) {
-    LOG(FATAL) << "Timeout waiting for key frames";
+    LOG(FATAL) << "Timeout waiting for key frames. Please use USB 3.0, and not "
+                  "in virtual machine.";
   }
 }
 
@@ -191,21 +197,30 @@ std::size_t Streams::GetStreamDataMaxSize(const Stream &stream) const {
 
 Streams::stream_datas_t Streams::GetStreamDatas(const Stream &stream) {
   std::unique_lock<std::mutex> lock(mtx_);
-  if (!HasStreamDatas(stream) || stream_datas_map_.at(stream).empty()) {
-    LOG(WARNING) << "There are stream datas of " << stream
-                 << ", do you first call WaitForStreams?";
+  if (!HasStreamDatas(stream)) {
+    LOG(WARNING) << "There are no stream datas of " << stream
+                 << ". Did you call WaitForStreams() before this?";
     return {};
   }
-  stream_datas_t datas = stream_datas_map_.at(stream);
+  auto datas = stream_datas_map_.at(stream);
   stream_datas_map_[stream].clear();
   return datas;
 }
 
 Streams::stream_data_t Streams::GetLatestStreamData(const Stream &stream) {
-  return GetStreamDatas(stream).back();
+  std::unique_lock<std::mutex> lock(mtx_);
+  if (!HasStreamDatas(stream)) {
+    LOG(WARNING) << "There are no stream datas of " << stream
+                 << ". Did you call WaitForStreams() before this?";
+    return {};
+  }
+  auto data = stream_datas_map_.at(stream).back();
+  stream_datas_map_[stream].clear();
+  return data;
 }
 
 const Streams::stream_datas_t &Streams::stream_datas(const Stream &stream) {
+  std::unique_lock<std::mutex> lock(mtx_);
   try {
     return stream_datas_map_.at(stream);
   } catch (const std::out_of_range &e) {
@@ -232,7 +247,8 @@ const StreamRequest &Streams::GetStreamConfigRequest(
 }
 
 bool Streams::HasStreamDatas(const Stream &stream) const {
-  return stream_datas_map_.find(stream) != stream_datas_map_.end();
+  return stream_datas_map_.find(stream) != stream_datas_map_.end() &&
+         !stream_datas_map_.at(stream).empty();
 }
 
 void Streams::AllocStreamData(
@@ -243,27 +259,38 @@ void Streams::AllocStreamData(
 void Streams::AllocStreamData(
     const Stream &stream, const StreamRequest &request, const Format &format) {
   stream_data_t data;
+
+  if (HasStreamDatas(stream)) {
+    // If cached equal to limits_max, drop the oldest one.
+    if (stream_datas_map_.at(stream).size() == GetStreamDataMaxSize(stream)) {
+      auto &&datas = stream_datas_map_[stream];
+      // reuse the dropped data
+      data.img = datas.front().img;
+      data.frame = datas.front().frame;
+      datas.erase(datas.begin());
+      VLOG(2) << "Stream data of " << stream << " is dropped as out of limits";
+    }
+  }
+
   if (stream == Stream::LEFT || stream == Stream::RIGHT) {
-    data.img = std::make_shared<ImgData>();
+    if(!data.img) {
+      data.img = std::make_shared<ImgData>();
+    }
   } else {
     data.img = nullptr;
   }
-  data.frame =
-      std::make_shared<frame_t>(request.width, request.height, format, nullptr);
-  stream_datas_map_[stream].push_back(data);
-  // If cached more then limits_max, drop the oldest one.
-  if (stream_datas_map_.at(stream).size() > GetStreamDataMaxSize(stream)) {
-    auto &&datas = stream_datas_map_[stream];
-    datas.erase(datas.begin());
-    VLOG(2) << "Stream data of " << stream << " is dropped as out of limits";
+  if (!data.frame) {
+    data.frame = std::make_shared<frame_t>(
+        request.width, request.height, format, nullptr);
   }
+  stream_datas_map_[stream].push_back(data);
 }
 
 void Streams::DiscardStreamData(const Stream &stream) {
   // Must discard after alloc, otherwise at will out of range when no this key.
   if (stream_datas_map_.at(stream).size() > 0) {
     auto &&datas = stream_datas_map_[stream];
-    datas.erase(datas.end() - 1);
+    datas.pop_back();
   } else {
     VLOG(2) << "Stream data of " << stream << " is empty, could not discard";
   }
@@ -273,8 +300,6 @@ bool Streams::HasKeyStreamDatas() const {
   for (auto &&s : key_streams_) {
     if (!HasStreamDatas(s))
       return false;
-    if (stream_datas_map_.at(s).empty())
-      return false;
   }
   return true;
 }

src/internal/streams.h

@@ -44,7 +44,7 @@ class Streams {
   void ConfigStream(
       const Capabilities &capability, const StreamRequest &request);
 
-  void PushStream(const Capabilities &capability, const void *data);
+  bool PushStream(const Capabilities &capability, const void *data);
 
   void WaitForStreams();
 

src/internal/strings.cc

@@ -65,6 +65,13 @@ bool starts_with(const std::string &text, const std::string &prefix) {
   return text.compare(0, prefix.length(), prefix) == 0;
 }
 
+bool ends_with(const std::string &text, const std::string &suffix) {
+  if (suffix.length() > text.length())
+    return false;
+  return text.compare(
+             text.length() - suffix.length(), suffix.length(), suffix) == 0;
+}
+
 std::vector<std::string> split(
     const std::string &text, const std::string &delimiters) {
   std::vector<std::string> tokens;

src/internal/strings.h

@@ -41,6 +41,9 @@ int hex2int(const std::string &text);
 MYNTEYE_API
 bool starts_with(const std::string &text, const std::string &prefix);
 
+MYNTEYE_API
+bool ends_with(const std::string &text, const std::string &suffix);
+
 MYNTEYE_API
 std::vector<std::string> split(
     const std::string &text, const std::string &delimiters);

src/uvc/README.md

@@ -0,0 +1,10 @@
+
+## `uvc-v4l2.cc`
+
+* [Linux Media Subsystem Documentation](https://www.kernel.org/doc/html/latest/media/index.html)
+  * [The Linux USB Video Class (UVC) driver](https://www.kernel.org/doc/html/latest/media/v4l-drivers/uvcvideo.html)
+
+## `uvc-wmf.cc`
+
+* [Media Foundation](https://msdn.microsoft.com/en-us/library/ms694197(VS.85).aspx)
+* [USB Video Class Driver](https://docs.microsoft.com/en-us/windows-hardware/drivers/stream/usb-video-class-driver)

src/uvc/uvc-libuvc.cc

@@ -144,6 +144,20 @@ bool pu_control_query(
   return false;
 }
 
+bool xu_control_range(
+    const device &device, const xu &xu, uint8_t selector, uint8_t id, int32_t *min,
+    int32_t *max, int32_t *def) {
+  // TODO(JohnZhao)
+  UNUSED(device)
+  UNUSED(xu)
+  UNUSED(selector)
+  UNUSED(id)
+  UNUSED(min)
+  UNUSED(max)
+  UNUSED(def)
+  return false;
+}
+
 bool xu_control_query(
     const device &device, const xu &xu, uint8_t selector, xu_query query,
     uint16_t size, uint8_t *data) {

src/uvc/uvc-v4l2.cc

@@ -276,7 +276,7 @@ struct device {
 
     // Init memory mapped IO
     v4l2_requestbuffers req;
-    req.count = 4;
+    req.count = 24;
     req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
     req.memory = V4L2_MEMORY_MMAP;
     if (xioctl(fd, VIDIOC_REQBUFS, &req) < 0) {
@@ -381,11 +381,11 @@ struct device {
       }
 
       if (callback) {
-        callback(buffers[buf.index].start);
+        callback(buffers[buf.index].start, [buf, this]() mutable {
+          if (xioctl(fd, VIDIOC_QBUF, &buf) < 0)
+            throw_error("VIDIOC_QBUF");
+        });
       }
-
-      if (xioctl(fd, VIDIOC_QBUF, &buf) < 0)
-        LOG_ERROR(FATAL, "VIDIOC_QBUF");
     }
   }
 
@@ -504,6 +504,39 @@ bool pu_control_query(
   return device.pu_control_query(get_cid(option), code, value);
 }
 
+bool xu_control_range(
+    const device &device, const xu &xu, uint8_t selector, uint8_t id,
+    int32_t *min, int32_t *max, int32_t *def) {
+  bool ret = true;
+
+  std::uint8_t data[3]{static_cast<uint8_t>(id | 0x80), 0, 0};
+
+  if (!xu_control_query(device, xu, selector, XU_QUERY_SET, 3, data)) {
+    LOG(WARNING) << "xu_control_range query failed";
+    ret = false;
+  }
+
+  if (xu_control_query(device, xu, selector, XU_QUERY_MIN, 3, data)) {
+    *min = (data[1] << 8) | (data[2]);
+  } else {
+    LOG(WARNING) << "xu_control_range query min failed";
+    ret = false;
+  }
+  if (xu_control_query(device, xu, selector, XU_QUERY_MAX, 3, data)) {
+    *max = (data[1] << 8) | (data[2]);
+  } else {
+    LOG(WARNING) << "xu_control_range query max failed";
+    ret = false;
+  }
+  if (xu_control_query(device, xu, selector, XU_QUERY_DEF, 3, data)) {
+    *def = (data[1] << 8) | (data[2]);
+  } else {
+    LOG(WARNING) << "xu_control_range query def failed";
+    ret = false;
+  }
+  return ret;
+}
+
 bool xu_control_query(
     const device &device, const xu &xu, uint8_t selector, xu_query query,
     uint16_t size, uint8_t *data) {

src/uvc/uvc-wmf.cc

@@ -1,129 +1,782 @@
-// Copyright 2018 Slightech Co., Ltd. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-#include "uvc/uvc.h"  // NOLINT
-
-#include <glog/logging.h>
-
-MYNTEYE_BEGIN_NAMESPACE
-
-namespace uvc {
-
-struct context {
-  context() {
-    VLOG(2) << __func__;
-  }
-
-  ~context() {
-    VLOG(2) << __func__;
-  }
-};
-
-struct device {
-  const std::shared_ptr<context> parent;
-
-  int vid, pid;
-
-  explicit device(std::shared_ptr<context> parent) : parent(parent) {
-    VLOG(2) << __func__;
-  }
-
-  ~device() {
-    VLOG(2) << __func__;
-  }
-};
-
-std::shared_ptr<context> create_context() {
-  return std::make_shared<context>();
-}
-
-std::vector<std::shared_ptr<device>> query_devices(
-    std::shared_ptr<context> context) {
-  std::vector<std::shared_ptr<device>> devices;
-  UNUSED(context)
-  return devices;
-}
-
-int get_vendor_id(const device &device) {
-  return device.vid;
-}
-
-int get_product_id(const device &device) {
-  return device.pid;
-}
-
-std::string get_name(const device &device) {
-  UNUSED(device)
-  return "";
-}
-
-std::string get_video_name(const device &device) {
-  UNUSED(device)
-  return "";
-}
-
-bool pu_control_range(
-    const device &device, Option option, int32_t *min, int32_t *max,
-    int32_t *def) {
-  UNUSED(device)
-  UNUSED(option)
-  UNUSED(min)
-  UNUSED(max)
-  UNUSED(def)
-  return false;
-}
-
-bool pu_control_query(
-    const device &device, Option option, pu_query query, int32_t *value) {
-  UNUSED(device)
-  UNUSED(option)
-  UNUSED(query)
-  UNUSED(value)
-  return false;
-}
-
-bool xu_control_query(
-    const device &device, const xu &xu, uint8_t selector, xu_query query,
-    uint16_t size, uint8_t *data) {
-  UNUSED(device)
-  UNUSED(xu)
-  UNUSED(selector)
-  UNUSED(query)
-  UNUSED(size)
-  UNUSED(data)
-  return false;
-}
-
-void set_device_mode(
-    device &device, int width, int height, int fourcc, int fps,  // NOLINT
-    video_channel_callback callback) {
-  UNUSED(device)
-  UNUSED(width)
-  UNUSED(height)
-  UNUSED(fourcc)
-  UNUSED(fps)
-  UNUSED(callback)
-}
-
-void start_streaming(device &device, int num_transfer_bufs) {  // NOLINT
-  UNUSED(device)
-  UNUSED(num_transfer_bufs)
-}
-
-void stop_streaming(device &device) {  // NOLINT
-  UNUSED(device)
-}
-
-}  // namespace uvc
-
-MYNTEYE_END_NAMESPACE
+// Copyright 2018 Slightech Co., Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "uvc/uvc.h"  // NOLINT
+
+#include <windows.h>
+#include <usbioctl.h>
+
+#include <Shlwapi.h>        // For QISearch, etc.
+#include <mfapi.h>          // For MFStartup, etc.
+#include <mfidl.h>          // For MF_DEVSOURCE_*, etc.
+#include <mfreadwrite.h>    // MFCreateSourceReaderFromMediaSource
+#include <mferror.h>
+
+#pragma comment(lib, "Shlwapi.lib")
+#pragma comment(lib, "mf.lib")
+#pragma comment(lib, "mfplat.lib")
+#pragma comment(lib, "mfreadwrite.lib")
+#pragma comment(lib, "mfuuid.lib")
+
+#pragma comment(lib, "setupapi.lib")
+#pragma comment(lib, "winusb.lib")
+
+#include <uuids.h>
+#include <vidcap.h>
+#include <ksmedia.h>
+#include <ksproxy.h>
+
+#include <Cfgmgr32.h>
+
+#pragma comment(lib, "cfgmgr32.lib")
+
+#include <SetupAPI.h>
+#include <WinUsb.h>
+
+#include <algorithm>
+#include <chrono>
+#include <functional>
+#include <map>
+#include <regex>
+#include <sstream>
+#include <thread>
+
+#include <strsafe.h>
+
+#include <glog/logging.h>
+
+#define VLOG_INFO VLOG(2)
+// #define VLOG_INFO LOG(INFO)
+
+MYNTEYE_BEGIN_NAMESPACE
+
+namespace uvc {
+
+const std::map<uint32_t, uint32_t> fourcc_map = {
+  { 0x56595559, 0x32595559 },  // 'VYUY' => '2YUY'
+  { 0x59555956, 0x59555932 }   // 'YUYV' => 'YUY2'
+};
+
+struct throw_error {
+  throw_error() = default;
+
+  explicit throw_error(const std::string &s) {
+    ss << s;
+  }
+
+  ~throw_error() noexcept(false) {
+    throw std::runtime_error(ss.str());
+  }
+
+  template<class T>
+  throw_error &operator<<(const T &val) {
+    ss << val;
+    return *this;
+  }
+
+  std::ostringstream ss;
+};
+
+static void check(const char *call, HRESULT hr) {
+  if (FAILED(hr)) {
+    throw_error() << call << "(...) returned 0x" << std::hex
+      << static_cast<uint32_t>(hr);
+  } else {
+    // VLOG_INFO << call << " SUCCESSED";
+  }
+}
+
+template<class T> class com_ptr {
+  T *p;
+  void ref(T *new_p) {
+    if (p == new_p) return;
+    unref();
+    p = new_p;
+    if (p) p->AddRef();
+  }
+
+  void unref() {
+    if (p) {
+      p->Release();
+      p = nullptr;
+    }
+  }
+public:
+  com_ptr() : p() {}
+  com_ptr(T *p) : com_ptr() {
+    ref(p);
+  }
+  com_ptr(const com_ptr &r) : com_ptr(r.p) {}
+  ~com_ptr() {
+    unref();
+  }
+
+  operator T *() const {
+    return p;
+  }
+  T &operator*() const {
+    return *p;
+  }
+  T *operator->() const {
+    return p;
+  }
+
+  T **operator&() {
+    unref();
+    return &p;
+  }
+  com_ptr &operator=(const com_ptr &r) {
+    ref(r.p);
+    return *this;
+  }
+};
+
+static std::string win_to_utf(const WCHAR *s) {
+  int len = WideCharToMultiByte(CP_UTF8, 0, s, -1, nullptr, 0, NULL, NULL);
+  if (len == 0) throw_error() << "WideCharToMultiByte(...) returned 0 and GetLastError() is " << GetLastError();
+  std::string buffer(len - 1, ' ');
+  len = WideCharToMultiByte(CP_UTF8, 0, s, -1, &buffer[0], (int)buffer.size()+1, NULL, NULL);
+  if (len == 0) throw_error() << "WideCharToMultiByte(...) returned 0 and GetLastError() is " << GetLastError();
+  return buffer;
+}
+
+std::vector<std::string> tokenize(std::string string, char separator) {
+  std::vector<std::string> tokens;
+  std::string::size_type i1 = 0;
+  while (true) {
+    auto i2 = string.find(separator, i1);
+    if (i2 == std::string::npos) {
+      tokens.push_back(string.substr(i1));
+      return tokens;
+    }
+    tokens.push_back(string.substr(i1, i2-i1));
+    i1 = i2 + 1;
+  }
+}
+
+/*
+static void print_guid(const char *call, int i, GUID guid) {
+  std::ostringstream ss;
+  ss << call << "(" << i << ") = ";
+  ss << "Data1: " << std::hex << guid.Data1 << ", ";
+  ss << "Data2: " << std::hex << guid.Data2 << ", ";
+  ss << "Data3: " << std::hex << guid.Data3 << ", ";
+  ss << "Data4: [ ";
+  for (int j = 0; j < 8; j++) {
+    ss << std::hex << (int)guid.Data4[j] << " ";
+  }
+  ss << "]";
+  LOG(INFO) << ss.str();
+}
+*/
+
+bool parse_usb_path(int &vid, int &pid, int &mi, std::string &unique_id, const std::string &path) {
+  auto name = path;
+  std::transform(begin(name), end(name), begin(name), ::tolower);
+  auto tokens = tokenize(name, '#');
+  if (tokens.size() < 1 || tokens[0] != R"(\\?\usb)") return false;  // Not a USB device
+  if (tokens.size() < 3) {
+    LOG(ERROR) << "malformed usb device path: " << name;
+    return false;
+  }
+
+  auto ids = tokenize(tokens[1], '&');
+  if (ids[0].size() != 8 || ids[0].substr(0,4) != "vid_" || !(std::istringstream(ids[0].substr(4,4)) >> std::hex >> vid)) {
+    LOG(ERROR) << "malformed vid string: " << tokens[1];
+    return false;
+  }
+
+  if (ids[1].size() != 8 || ids[1].substr(0,4) != "pid_" || !(std::istringstream(ids[1].substr(4,4)) >> std::hex >> pid)) {
+    LOG(ERROR) << "malformed pid string: " << tokens[1];
+    return false;
+  }
+
+  if (ids[2].size() != 5 || ids[2].substr(0,3) != "mi_" || !(std::istringstream(ids[2].substr(3,2)) >> mi)) {
+    LOG(ERROR) << "malformed mi string: " << tokens[1];
+    return false;
+  }
+
+  ids = tokenize(tokens[2], '&');
+  if (ids.size() < 2) {
+    LOG(ERROR) << "malformed id string: " << tokens[2];
+    return false;
+  }
+  unique_id = ids[1];
+  return true;
+}
+
+bool parse_usb_path_from_device_id(int &vid, int &pid, int &mi, std::string &unique_id, const std::string &device_id) {
+  auto name = device_id;
+  std::transform(begin(name), end(name), begin(name), ::tolower);
+  auto tokens = tokenize(name, '\\');
+  if (tokens.size() < 1 || tokens[0] != R"(usb)") return false;  // Not a USB device
+
+  auto ids = tokenize(tokens[1], '&');
+  if (ids[0].size() != 8 || ids[0].substr(0, 4) != "vid_" || !(std::istringstream(ids[0].substr(4, 4)) >> std::hex >> vid)) {
+    LOG(ERROR) << "malformed vid string: " << tokens[1];
+    return false;
+  }
+
+  if (ids[1].size() != 8 || ids[1].substr(0, 4) != "pid_" || !(std::istringstream(ids[1].substr(4, 4)) >> std::hex >> pid)) {
+    LOG(ERROR) << "malformed pid string: " << tokens[1];
+    return false;
+  }
+
+  if (ids[2].size() != 5 || ids[2].substr(0, 3) != "mi_" || !(std::istringstream(ids[2].substr(3, 2)) >> mi)) {
+    LOG(ERROR) << "malformed mi string: " << tokens[1];
+    return false;
+  }
+
+  ids = tokenize(tokens[2], '&');
+  if (ids.size() < 2) {
+    LOG(ERROR) << "malformed id string: " + tokens[2];
+    return false;
+  }
+  unique_id = ids[1];
+  return true;
+}
+
+struct context {
+  context() {
+    CoInitializeEx(NULL, COINIT_APARTMENTTHREADED);
+    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    MFStartup(MF_VERSION, MFSTARTUP_NOSOCKET);
+  }
+  ~context() {
+    MFShutdown();
+    CoUninitialize();
+  }
+};
+
+class reader_callback : public IMFSourceReaderCallback {
+  std::weak_ptr<device> owner;  // The device holds a reference to us, so use
+                                // weak_ptr to prevent a cycle
+  ULONG ref_count;
+  volatile bool streaming = false;
+public:
+  reader_callback(std::weak_ptr<device> owner) : owner(owner), ref_count() {}
+
+  bool is_streaming() const {
+    return streaming;
+  }
+  void on_start() {
+    streaming = true;
+  }
+
+#pragma warning( push )
+#pragma warning( disable: 4838 )
+  // Implement IUnknown
+  HRESULT STDMETHODCALLTYPE QueryInterface(REFIID riid, void **ppvObject) override {
+    static const QITAB table[] = {QITABENT(reader_callback, IUnknown), QITABENT(reader_callback, IMFSourceReaderCallback), {0}};
+    return QISearch(this, table, riid, ppvObject);
+  }
+#pragma warning( pop )
+
+  ULONG STDMETHODCALLTYPE AddRef() override { return InterlockedIncrement(&ref_count); }
+  ULONG STDMETHODCALLTYPE Release() override {
+    ULONG count = InterlockedDecrement(&ref_count);
+    if (count == 0) delete this;
+    return count;
+  }
+
+  // Implement IMFSourceReaderCallback
+  HRESULT STDMETHODCALLTYPE OnReadSample(HRESULT hrStatus, DWORD dwStreamIndex, DWORD dwStreamFlags, LONGLONG llTimestamp, IMFSample *sample) override;
+  HRESULT STDMETHODCALLTYPE OnFlush(DWORD dwStreamIndex) override { streaming = false; return S_OK; }
+  HRESULT STDMETHODCALLTYPE OnEvent(DWORD dwStreamIndex, IMFMediaEvent *pEvent) override { return S_OK; }
+};
+
+struct device {
+  const std::shared_ptr<context> parent;
+  int vid, pid;
+  std::string unique_id;
+  std::string name;
+
+  com_ptr<reader_callback> reader_callback;
+  com_ptr<IMFActivate> mf_activate;
+  com_ptr<IMFMediaSource> mf_media_source;
+  com_ptr<IAMCameraControl> am_camera_control;
+  com_ptr<IAMVideoProcAmp> am_video_proc_amp;
+  std::map<int, com_ptr<IKsControl>> ks_controls;
+  com_ptr<IMFSourceReader> mf_source_reader;
+  video_channel_callback callback = nullptr;
+
+  device(std::shared_ptr<context> parent, int vid, int pid, std::string unique_id, std::string name)
+      : parent(move(parent)), vid(vid), pid(pid), unique_id(move(unique_id)), name(name) {
+  }
+
+  ~device() {
+    stop_streaming();
+  }
+
+
+  IKsControl *get_ks_control(const uvc::xu &xu) {
+    auto it = ks_controls.find(xu.node);
+    if (it != end(ks_controls)) return it->second;
+
+    get_media_source();
+
+    // Attempt to retrieve IKsControl
+    com_ptr<IKsTopologyInfo> ks_topology_info = NULL;
+    check("QueryInterface", mf_media_source->QueryInterface(__uuidof(IKsTopologyInfo), (void **)&ks_topology_info));
+
+    GUID node_type;
+    /*
+    DWORD numberOfNodes;
+    check("get_NumNodes", ks_topology_info->get_NumNodes(&numberOfNodes));
+    for (int i = 0; i < numberOfNodes; i++) {
+      check("get_NodeType", ks_topology_info->get_NodeType(i, &node_type));
+      print_guid("node_type", i, node_type);
+    }
+    */
+    check("get_NodeType", ks_topology_info->get_NodeType(xu.node, &node_type));
+    const GUID KSNODETYPE_DEV_SPECIFIC_LOCAL{0x941C7AC0L, 0xC559, 0x11D0, {0x8A, 0x2B, 0x00, 0xA0, 0xC9, 0x25, 0x5A, 0xC1}};
+    if (node_type != KSNODETYPE_DEV_SPECIFIC_LOCAL) throw_error() << "Invalid extension unit node ID: " << xu.node;
+
+    com_ptr<IUnknown> unknown;
+    check("CreateNodeInstance", ks_topology_info->CreateNodeInstance(xu.node, IID_IUnknown, (LPVOID *)&unknown));
+
+    com_ptr<IKsControl> ks_control;
+    check("QueryInterface", unknown->QueryInterface(__uuidof(IKsControl), (void **)&ks_control));
+    VLOG_INFO << "Obtained KS control node : " << xu.node;
+    return ks_controls[xu.node] = ks_control;
+  }
+
+  void start_streaming() {
+    if (mf_source_reader) {
+      reader_callback->on_start();
+      check("IMFSourceReader::ReadSample", mf_source_reader->ReadSample(MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, NULL, NULL, NULL, NULL));
+    }
+  }
+
+  void stop_streaming() {
+    if (mf_source_reader) mf_source_reader->Flush(MF_SOURCE_READER_FIRST_VIDEO_STREAM);
+
+    while (true) {
+      bool is_streaming = reader_callback->is_streaming();
+      if (is_streaming) std::this_thread::sleep_for(std::chrono::milliseconds(10));
+      else break;
+    }
+
+    mf_source_reader = nullptr;
+    am_camera_control = nullptr;
+    am_video_proc_amp = nullptr;
+    ks_controls.clear();
+    if (mf_media_source) {
+      mf_media_source = nullptr;
+      check("IMFActivate::ShutdownObject", mf_activate->ShutdownObject());
+    }
+    callback = {};
+  }
+
+  com_ptr<IMFMediaSource> get_media_source() {
+    if (!mf_media_source) {
+      check("IMFActivate::ActivateObject", mf_activate->ActivateObject(__uuidof(IMFMediaSource), (void **)&mf_media_source));
+      if (mf_media_source) {
+        check("IMFMediaSource::QueryInterface", mf_media_source->QueryInterface(__uuidof(IAMCameraControl), (void **)&am_camera_control));
+        if (SUCCEEDED(mf_media_source->QueryInterface(__uuidof(IAMVideoProcAmp), (void **)&am_video_proc_amp)));
+      } else throw_error() << "Invalid media source";
+    }
+    return mf_media_source;
+  }
+
+};
+
+HRESULT reader_callback::OnReadSample(HRESULT hrStatus, DWORD dwStreamIndex, DWORD dwStreamFlags, LONGLONG llTimestamp, IMFSample *sample) {
+  if (auto owner_ptr = owner.lock()) {
+    if (sample) {
+      com_ptr<IMFMediaBuffer> buffer = NULL;
+      if (SUCCEEDED(sample->GetBufferByIndex(0, &buffer))) {
+        BYTE *byte_buffer;
+        DWORD max_length, current_length;
+        if (SUCCEEDED(buffer->Lock(&byte_buffer, &max_length, &current_length))) {
+          auto continuation = [buffer, this]() {
+            buffer->Unlock();
+          };
+          owner_ptr->callback(byte_buffer, continuation);
+        }
+      }
+    }
+
+    if (auto owner_ptr_new = owner.lock()) {
+      auto hr = owner_ptr_new->mf_source_reader->ReadSample(MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, NULL, NULL, NULL, NULL);
+      switch (hr) {
+        case S_OK: break;
+        case MF_E_INVALIDREQUEST: LOG(ERROR) << "ReadSample returned MF_E_INVALIDREQUEST"; break;
+        case MF_E_INVALIDSTREAMNUMBER: LOG(ERROR) << "ReadSample returned MF_E_INVALIDSTREAMNUMBER"; break;
+        case MF_E_NOTACCEPTING: LOG(ERROR) << "ReadSample returned MF_E_NOTACCEPTING"; break;
+        case E_INVALIDARG: LOG(ERROR) << "ReadSample returned E_INVALIDARG"; break;
+        case MF_E_VIDEO_RECORDING_DEVICE_INVALIDATED: LOG(ERROR) << "ReadSample returned MF_E_VIDEO_RECORDING_DEVICE_INVALIDATED"; break;
+        default: LOG(ERROR) << "ReadSample returned HRESULT " << std::hex << (uint32_t)hr; break;
+      }
+      if (hr != S_OK) streaming = false;
+    }
+  }
+  return S_OK;
+}
+
+std::shared_ptr<context> create_context() {
+  return std::make_shared<context>();
+}
+
+std::vector<std::shared_ptr<device>> query_devices(std::shared_ptr<context> context) {
+  IMFAttributes *pAttributes = NULL;
+  check("MFCreateAttributes", MFCreateAttributes(&pAttributes, 1));
+  check("IMFAttributes::SetGUID", pAttributes->SetGUID(
+    MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID));
+
+  IMFActivate **ppDevices;
+  UINT32 numDevices;
+  check("MFEnumDeviceSources", MFEnumDeviceSources(pAttributes, &ppDevices, &numDevices));
+
+  std::vector<std::shared_ptr<device>> devices;
+  for (UINT32 i = 0; i < numDevices; ++i) {
+    com_ptr<IMFActivate> pDevice;
+    *&pDevice = ppDevices[i];
+
+    WCHAR *wchar_dev_name = NULL;
+    WCHAR *wchar_name = NULL;
+    UINT32 length;
+
+    pDevice->GetAllocatedString(
+      MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK, &wchar_dev_name,
+      &length);
+    auto dev_name = win_to_utf(wchar_dev_name);
+    CoTaskMemFree(wchar_dev_name);
+
+    pDevice->GetAllocatedString(MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME, &wchar_name, &length);
+    auto name = win_to_utf(wchar_name);  // Device description name
+    CoTaskMemFree(wchar_name);
+
+    int vid, pid, mi;
+    std::string unique_id;
+
+    if (!parse_usb_path(vid, pid, mi, unique_id, dev_name)) continue;
+
+    std::shared_ptr<device> dev;
+    for (auto & d : devices) {
+      if (d->vid == vid && d->pid == pid && d->unique_id == unique_id)
+        dev = d;
+    }
+    if (!dev) {
+      try {
+        dev = std::make_shared<device>(context, vid, pid, unique_id, name);
+        devices.push_back(dev);
+      } catch (const std::exception &e) {
+        VLOG_INFO << "Not a USB video device: " << e.what();
+      }
+    }
+
+    dev->reader_callback = new reader_callback(dev);
+    dev->mf_activate = pDevice;
+    dev->vid = vid;
+    dev->pid = pid;
+  }
+
+  CoTaskMemFree(ppDevices);
+  return devices;
+}
+
+int get_vendor_id(const device &device) {
+  return device.vid;
+}
+
+int get_product_id(const device &device) {
+  return device.pid;
+}
+
+std::string get_name(const device &device) {
+  return device.name;
+}
+
+std::string get_video_name(const device &device) {
+  return device.name;
+}
+
+static long get_cid(Option option) {
+  switch (option) {
+    case Option::GAIN:
+      return VideoProcAmp_Gain;
+    case Option::BRIGHTNESS:
+      return VideoProcAmp_Brightness;
+    case Option::CONTRAST:
+      return VideoProcAmp_Contrast;
+    default:
+      LOG(FATAL) << "No VideoProcAmp cid for " << option;
+  }
+}
+
+bool pu_control_range(
+    const device &device, Option option, int32_t *min, int32_t *max,
+    int32_t *def) {
+  VLOG_INFO << __func__ << " " << option;
+  const_cast<uvc::device &>(device).get_media_source();
+  long minVal = 0, maxVal = 0, steppingDelta = 0, defVal = 0, capsFlag = 0;
+  check("IAMVideoProcAmp::GetRange",
+      const_cast<uvc::device &>(device).am_video_proc_amp->GetRange(
+        get_cid(option), &minVal, &maxVal, &steppingDelta, &defVal, &capsFlag));
+  if (min) *min = static_cast<int>(minVal);
+  if (max) *max = static_cast<int>(maxVal);
+  if (def) *def = static_cast<int>(defVal);
+  VLOG_INFO << __func__ << " " << option <<
+      ": min=" << *min << ", max=" << *max << ", def=" << *def;
+  return true;
+}
+
+static void pu_control_get(const device &device, long property, int32_t *value) {
+  long data, flags = 0;
+  check("IAMVideoProcAmp::Get",
+      const_cast<uvc::device &>(device).am_video_proc_amp->Get(
+        property, &data, &flags));
+  *value = data;
+}
+
+static void pu_control_set(const device &device, long property, int32_t *value) {
+  long data = *value;
+  check("IAMVideoProcAmp::Set",
+      const_cast<uvc::device &>(device).am_video_proc_amp->Set(
+        property, data, VideoProcAmp_Flags_Auto));
+}
+
+bool pu_control_query(
+    const device &device, Option option, pu_query query, int32_t *value) {
+  CHECK_NOTNULL(value);
+  const_cast<uvc::device &>(device).get_media_source();
+  switch (query) {
+    case PU_QUERY_SET:
+      VLOG_INFO << "pu_control_set " << option << ": " << *value;
+      pu_control_set(device, get_cid(option), value);
+      VLOG_INFO << "pu_control_set " << option << " done";
+      return true;
+    case PU_QUERY_GET:
+      VLOG_INFO << "pu_control_get " << option;
+      pu_control_get(device, get_cid(option), value);
+      VLOG_INFO << "pu_control_get " << option << ": " << *value;
+      return true;
+    default:
+      LOG(ERROR) << "pu_control_query request code is unaccepted";
+      return false;
+  }
+}
+
+static std::string to_string(uint16_t size, uint8_t *data) {
+  std::ostringstream ss;
+  for (uint8_t *beg = data, *end = data + size; beg != end; beg++) {
+    ss << "0x" << std::hex << static_cast<int>(*beg) << ",";
+  }
+  return ss.str();
+}
+
+/*
+static std::vector<BYTE> xu_control_desc(const device &device, const xu &xu, ULONG id, ULONG flags) {
+  auto ks_control = const_cast<uvc::device &>(device).get_ks_control(xu);
+
+  KSP_NODE node;
+  memset(&node, 0, sizeof(KSP_NODE));
+  node.Property.Set = reinterpret_cast<const GUID &>(xu.id);
+  node.Property.Id = id;
+  node.Property.Flags = flags;
+  node.NodeId = xu.node;
+
+  KSPROPERTY_DESCRIPTION description;
+  ULONG bytes_received = 0;
+  check("IKsControl::KsProperty", ks_control->KsProperty(
+      (PKSPROPERTY)&node,
+      sizeof(node),
+      &description,
+      sizeof(KSPROPERTY_DESCRIPTION),
+      &bytes_received));
+
+  ULONG size = description.DescriptionSize;
+  std::vector<BYTE> buffer(size);
+
+  check("IKsControl::KsProperty", ks_control->KsProperty(
+      (PKSPROPERTY)&node,
+      sizeof(node),
+      buffer.data(),
+      size,
+      &bytes_received));
+
+  if (bytes_received != size) { throw_error() << "wrong data"; }
+
+  // VLOG_INFO << "buffer size=" << size << ", data=["
+  //     << to_string(size, buffer.data()) << "]";
+  return buffer;
+}
+
+bool xu_control_range(
+    const device &device, const xu &xu, uint8_t selector, int32_t *min,
+    int32_t *max, int32_t *def) {
+  VLOG_INFO << __func__ << " " << static_cast<int>(selector);
+  size_t prop_header_size = sizeof(KSPROPERTY_MEMBERSHEADER) + sizeof(KSPROPERTY_DESCRIPTION);
+  // get step, min and max values
+  {
+    auto &&buffer = xu_control_desc(device, xu, selector,
+        KSPROPERTY_TYPE_BASICSUPPORT | KSPROPERTY_TYPE_TOPOLOGY);
+
+    BYTE *values = buffer.data() + prop_header_size;
+
+    // size_t size = buffer.size() - prop_header_size;
+    // VLOG_INFO << "values size: " << size << ", data=["
+    //     << to_string(size, values) << "]";
+
+    *min = (values[1] << 8) | (values[2]);
+    values += 3;
+    *max = (values[1] << 8) | (values[2]);
+    // values += 3;
+    // *step = (values[1] << 8) | (values[2]);
+  }
+  // get def value
+  {
+    auto &&buffer = xu_control_desc(device, xu, selector,
+        KSPROPERTY_TYPE_DEFAULTVALUES | KSPROPERTY_TYPE_TOPOLOGY);
+
+    BYTE *values = buffer.data() + prop_header_size;
+
+    // size_t size = buffer.size() - prop_header_size;
+    // VLOG_INFO << "values size: " << size << ", data=["
+    //     << to_string(size, values) << "]";
+
+    *def = (values[1] << 8) | (values[2]);
+  }
+  VLOG_INFO << __func__ << " " << static_cast<int>(selector)
+      << ": min=" << *min << ", max=" << *max << ", def=" << *def;
+  return true;
+}
+*/
+
+static void xu_control_get(const device &device, const xu &xu, uint8_t selector,
+      uint16_t size, uint8_t *data) {
+  VLOG_INFO << __func__ << " " << static_cast<int>(selector);
+  auto &&ks_control = const_cast<uvc::device &>(device).get_ks_control(xu);
+
+  KSP_NODE node;
+  memset(&node, 0, sizeof(KSP_NODE));
+  node.Property.Set = reinterpret_cast<const GUID &>(xu.id);
+  node.Property.Id = selector;
+  node.Property.Flags = KSPROPERTY_TYPE_GET | KSPROPERTY_TYPE_TOPOLOGY;
+  node.NodeId = xu.node;
+
+  ULONG bytes_received = 0;
+  check("IKsControl::KsProperty", ks_control->KsProperty(
+      (PKSPROPERTY)&node, sizeof(node), data, size, &bytes_received));
+
+  if (bytes_received != size)
+    throw_error() << "xu_control_get did not return enough data";
+  VLOG_INFO << __func__ << " " << static_cast<int>(selector)
+      << ": size=" << size << ", data=[" << to_string(size, data) << "]";
+}
+
+static void xu_control_set(const device &device, const xu &xu, uint8_t selector,
+      uint16_t size, uint8_t *data) {
+  VLOG_INFO << __func__ << " " << static_cast<int>(selector)
+      << ": size=" << size << ", data=[" << to_string(size, data) << "]";
+  auto &&ks_control = const_cast<uvc::device &>(device).get_ks_control(xu);
+
+  KSP_NODE node;
+  memset(&node, 0, sizeof(KSP_NODE));
+  node.Property.Set = reinterpret_cast<const GUID &>(xu.id);
+  node.Property.Id = selector;
+  node.Property.Flags = KSPROPERTY_TYPE_SET | KSPROPERTY_TYPE_TOPOLOGY;
+  node.NodeId = xu.node;
+
+  ULONG bytes_received = 0;
+  check("IKsControl::KsProperty", ks_control->KsProperty(
+      (PKSPROPERTY)&node, sizeof(node), data, size, &bytes_received));
+  VLOG_INFO << __func__ << " " << static_cast<int>(selector) << " done";
+}
+
+static int32_t xu_control_range_basic(const device &device, const xu &xu, uint8_t selector, uint8_t id) {
+  std::uint8_t data[3]{id, 0, 0};
+  xu_control_set(device, xu, selector, 3, data);
+  xu_control_get(device, xu, selector, 3, data);
+  return (data[1] << 8) | (data[2]);
+}
+
+bool xu_control_range(
+    const device &device, const xu &xu, uint8_t selector, uint8_t id,
+    int32_t *min, int32_t *max, int32_t *def) {
+  VLOG_INFO << __func__ << " " << static_cast<int>(selector);
+  *min = xu_control_range_basic(
+      device, xu, selector, static_cast<uint8_t>(id | 0x90));
+  *max = xu_control_range_basic(
+      device, xu, selector, static_cast<uint8_t>(id | 0xa0));
+  *def = xu_control_range_basic(
+      device, xu, selector, static_cast<uint8_t>(id | 0xc0));
+  return true;
+}
+
+bool xu_control_query(
+    const device &device, const xu &xu, uint8_t selector, xu_query query,
+    uint16_t size, uint8_t *data) {
+  CHECK_NOTNULL(data);
+  switch (query) {
+    case XU_QUERY_SET:
+      xu_control_set(device, xu, selector, size, data);
+      return true;
+    case XU_QUERY_GET:
+      xu_control_get(device, xu, selector, size, data);
+      return true;
+    default:
+      LOG(ERROR) << "xu_control_query request code is unaccepted";
+      return false;
+  }
+}
+
+void set_device_mode(device &device, int width, int height, int fourcc, int fps, video_channel_callback callback) {
+  if (!device.mf_source_reader) {
+    com_ptr<IMFAttributes> pAttributes;
+    check("MFCreateAttributes", MFCreateAttributes(&pAttributes, 1));
+    check("IMFAttributes::SetUnknown", pAttributes->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK, static_cast<IUnknown *>(device.reader_callback)));
+    check("MFCreateSourceReaderFromMediaSource", MFCreateSourceReaderFromMediaSource(device.get_media_source(), pAttributes, &device.mf_source_reader));
+  }
+
+  if (fourcc_map.count(fourcc)) fourcc = fourcc_map.at(fourcc);
+
+  for (DWORD j = 0; ; j++) {
+    com_ptr<IMFMediaType> media_type;
+    HRESULT hr = device.mf_source_reader->GetNativeMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, j, &media_type);
+    if (hr == MF_E_NO_MORE_TYPES) break;
+    check("IMFSourceReader::GetNativeMediaType", hr);
+
+    UINT32 uvc_width, uvc_height, uvc_fps_num, uvc_fps_denom;
+    GUID subtype;
+    check("MFGetAttributeSize", MFGetAttributeSize(media_type, MF_MT_FRAME_SIZE, &uvc_width, &uvc_height));
+    if (uvc_width != width || uvc_height != height) continue;
+
+    check("IMFMediaType::GetGUID", media_type->GetGUID(MF_MT_SUBTYPE, &subtype));
+    if (subtype.Data1 != fourcc) continue;
+
+    check("MFGetAttributeRatio", MFGetAttributeRatio(media_type, MF_MT_FRAME_RATE, &uvc_fps_num, &uvc_fps_denom));
+    if (uvc_fps_denom == 0) continue;
+    //int uvc_fps = uvc_fps_num / uvc_fps_denom;
+    //LOG(INFO) << "uvc_fps: " << uvc_fps;
+
+    check("IMFSourceReader::SetCurrentMediaType", device.mf_source_reader->SetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, NULL, media_type));
+
+    device.callback = callback;
+    return;
+  }
+  throw_error() << "no matching media type for pixel format " << std::hex << fourcc;
+}
+
+void start_streaming(device &device, int num_transfer_bufs) {
+  device.start_streaming();
+}
+
+void stop_streaming(device &device) {
+  device.stop_streaming();
+}
+
+}  // namespace uvc
+
+MYNTEYE_END_NAMESPACE

src/uvc/uvc.h

@@ -36,9 +36,17 @@ typedef enum pu_query {
   PU_QUERY_LAST
 } pu_query;
 
+struct MYNTEYE_API guid {
+  uint32_t data1;
+  uint16_t data2, data3;
+  uint8_t data4[8];
+};
+
 // Extension Unit
 struct MYNTEYE_API xu {
   uint8_t unit;
+  int node;
+  guid id;
 };
 
 typedef enum xu_query {
@@ -77,12 +85,16 @@ MYNTEYE_API bool pu_control_query(
     const device &device, Option option, pu_query query, int32_t *value);
 
 // Access XU (Extension Unit) controls
-MYNTEYE_API bool xu_control_query(
+MYNTEYE_API bool xu_control_range(
+    const device &device, const xu &xu, uint8_t selector, uint8_t id,
+    int32_t *min, int32_t *max, int32_t *def);
+MYNTEYE_API bool xu_control_query(  // XU_QUERY_SET, XU_QUERY_GET
     const device &device, const xu &xu, uint8_t selector, xu_query query,
     uint16_t size, uint8_t *data);
 
 // Control streaming
-typedef std::function<void(const void *frame)> video_channel_callback;
+typedef std::function<void(const void *frame,
+    std::function<void()> continuation)> video_channel_callback;
 
 MYNTEYE_API void set_device_mode(
     device &device, int width, int height, int fourcc, int fps,  // NOLINT

tools/README.md

@@ -69,7 +69,7 @@ roslaunch mynt_eye_ros_wrapper mynteye.launch
 ```
 
 ```bash
-rosbag record -O mynteye.bag /mynteye/left /mynteye/imu /mynteye/temp
+rosbag record -O mynteye.bag /mynteye/left/image_raw /mynteye/imu/data_raw /mynteye/temp/data_raw
 ```
 
 ## Analytics data (rosbag)

tools/analytics/imu_analytics.py

@@ -650,8 +650,8 @@ def _main():
     else:
       dataset = ROSBag(
           path,
-          topic_imu='/mynteye/imu',
-          topic_temp='/mynteye/temp')
+          topic_imu='/mynteye/imu/data_raw',
+          topic_temp='/mynteye/temp/data_raw')
     return dataset
 
   if args.binary:

tools/analytics/stamp_analytics.py

@@ -334,8 +334,8 @@ def _main():
         sys.exit('Error: dataset model not supported {}'.format(model))
     else:
       dataset = ROSBag(path,
-                       topic_img_left='/mynteye/left',
-                       topic_imu='/mynteye/imu')
+                       topic_img_left='/mynteye/left/image_raw',
+                       topic_imu='/mynteye/imu/data_raw')
     return dataset
 
   dataset = BinDataset(dataset_path, dataset_creator)

tools/config/mynteye/mynteye_config.yaml

@@ -3,7 +3,7 @@ dataset: "mynteye"
 
 # rosbag config
 rosbag:
-  topic_img_left: "/mynteye/left"
-  topic_img_right: "/mynteye/right"
-  topic_imu: "/mynteye/imu"
-  topic_temp: "/mynteye/temp"
+  topic_img_left: "/mynteye/left/image_raw"
+  topic_img_right: "/mynteye/right/image_raw"
+  topic_imu: "/mynteye/imu/data_raw"
+  topic_temp: "/mynteye/temp/data_raw"

tools/requirements.txt

@@ -1,3 +1,4 @@
 matplotlib>=1.5.1
 numpy>=1.11.0
 pandas>=0.22.0
+PyYAML>=3.11

wrappers/ros/README.md

@@ -32,6 +32,15 @@ source wrappers/ros/devel/setup.bash
 roslaunch mynt_eye_ros_wrapper display.launch
 ```
 
+## Test
+
+Test `get_info` service,
+
+```bash
+source wrappers/ros/devel/setup.bash
+rosrun mynt_eye_ros_wrapper get_device_info.py
+```
+
 ## ROS Indigo
 
 How to install ROS Indigo (Ubuntu 14.04),

wrappers/ros/src/mynt_eye_ros_wrapper/CMakeLists.txt

@@ -66,6 +66,11 @@ add_message_files(
   Temp.msg
 )
 
+add_service_files(
+  FILES
+  GetInfo.srv
+)
+
 generate_messages(
   DEPENDENCIES
   std_msgs
@@ -113,6 +118,11 @@ endif()
 
 # install
 
+#install(PROGRAMS
+#  scripts/get_device_info.py
+#  DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
+#)
+
 install(TARGETS mynteye_wrapper mynteye_wrapper_node
   ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
   LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}

wrappers/ros/src/mynt_eye_ros_wrapper/launch/mynteye.launch

@@ -4,17 +4,17 @@
 
   <!-- node params -->
 
-  <arg name="left_topic" default="left" />
-  <arg name="right_topic" default="right" />
-  <arg name="left_rect_topic" default="left_rect" />
-  <arg name="right_rect_topic" default="right_rect" />
-  <arg name="disparity_topic" default="disparity" />
-  <arg name="disparity_norm_topic" default="disparity_norm" />
-  <arg name="points_topic" default="points" />
-  <arg name="depth_topic" default="depth" />
+  <arg name="left_topic" default="left/image_raw" />
+  <arg name="right_topic" default="right/image_raw" />
+  <arg name="left_rect_topic" default="left/image_rect" />
+  <arg name="right_rect_topic" default="right/image_rect" />
+  <arg name="disparity_topic" default="disparity/image_raw" />
+  <arg name="disparity_norm_topic" default="disparity/image_norm" />
+  <arg name="depth_topic" default="depth/image_raw" />
+  <arg name="points_topic" default="points/data_raw" />
 
-  <arg name="imu_topic" default="imu" />
-  <arg name="temp_topic" default="temp" />
+  <arg name="imu_topic" default="imu/data_raw" />
+  <arg name="temp_topic" default="temp/data_raw" />
 
   <arg name="base_frame_id" default="$(arg mynteye)_link" />
   <arg name="left_frame_id" default="$(arg mynteye)_left_frame" />

wrappers/ros/src/mynt_eye_ros_wrapper/rviz/mynteye.rviz

@@ -55,7 +55,7 @@ Visualization Manager:
       Displays:
         - Class: rviz/Image
           Enabled: true
-          Image Topic: /mynteye/left
+          Image Topic: /mynteye/left/image_raw
           Max Value: 1
           Median window: 5
           Min Value: 0
@@ -67,7 +67,7 @@ Visualization Manager:
           Value: true
         - Class: rviz/Image
           Enabled: true
-          Image Topic: /mynteye/right
+          Image Topic: /mynteye/right/image_raw
           Max Value: 1
           Median window: 5
           Min Value: 0
@@ -81,7 +81,7 @@ Visualization Manager:
           Displays:
             - Class: rviz/Image
               Enabled: true
-              Image Topic: /mynteye/left_rect
+              Image Topic: /mynteye/left/image_rect
               Max Value: 1
               Median window: 5
               Min Value: 0
@@ -93,7 +93,7 @@ Visualization Manager:
               Value: true
             - Class: rviz/Image
               Enabled: true
-              Image Topic: /mynteye/right_rect
+              Image Topic: /mynteye/right/image_rect
               Max Value: 1
               Median window: 5
               Min Value: 0
@@ -111,7 +111,7 @@ Visualization Manager:
       Displays:
         - Class: rviz/Image
           Enabled: true
-          Image Topic: /mynteye/disparity
+          Image Topic: /mynteye/disparity/image_raw
           Max Value: 1
           Median window: 5
           Min Value: 0
@@ -123,7 +123,7 @@ Visualization Manager:
           Value: true
         - Class: rviz/Image
           Enabled: true
-          Image Topic: /mynteye/disparity_norm
+          Image Topic: /mynteye/disparity/image_norm
           Max Value: 1
           Median window: 5
           Min Value: 0
@@ -137,7 +137,7 @@ Visualization Manager:
       Name: Disparity
     - Class: rviz/Image
       Enabled: true
-      Image Topic: /mynteye/depth
+      Image Topic: /mynteye/depth/image_raw
       Max Value: 1
       Median window: 5
       Min Value: 0
@@ -157,7 +157,7 @@ Visualization Manager:
       Channel Name: intensity
       Class: rviz/PointCloud2
       Color: 255; 255; 255
-      Color Transformer: ""
+      Color Transformer: RGB8
       Decay Time: 0
       Enabled: true
       Invert Rainbow: false
@@ -166,13 +166,13 @@ Visualization Manager:
       Min Color: 0; 0; 0
       Min Intensity: 0
       Name: Points
-      Position Transformer: ""
+      Position Transformer: XYZ
       Queue Size: 10
       Selectable: true
       Size (Pixels): 3
       Size (m): 0.00999999978
       Style: Flat Squares
-      Topic: /mynteye/points
+      Topic: /mynteye/points/data_raw
       Unreliable: false
       Use Fixed Frame: true
       Use rainbow: true
@@ -183,7 +183,7 @@ Visualization Manager:
       Enabled: false
       History Length: 1
       Name: Imu
-      Topic: /mynteye/imu
+      Topic: /mynteye/imu/data_raw
       Unreliable: false
       Value: false
   Enabled: true
@@ -247,7 +247,7 @@ Window Geometry:
     collapsed: false
   LeftRect:
     collapsed: false
-  QMainWindow State: 000000ff00000000fd00000004000000000000016a00000396fc020000000ffb0000001200530065006c0065006300740069006f006e00000001e10000009b0000006100fffffffb0000001e0054006f006f006c002000500072006f007000650072007400690065007302000001ed000001df00000185000000a3fb000000120056006900650077007300200054006f006f02000001df000002110000018500000122fb000000200054006f006f006c002000500072006f0070006500720074006900650073003203000002880000011d000002210000017afb000000100044006900730070006c006100790073010000002800000396000000d700fffffffb0000002000730065006c0065006300740069006f006e00200062007500660066006500720200000138000000aa0000023a00000294fb00000014005700690064006500530074006500720065006f02000000e6000000d2000003ee0000030bfb0000000c004b0069006e0065006300740200000186000001060000030c00000261fb00000008004c00650066007403000001bd000000860000024e00000190fb0000000a00520069006700680074030000040c000000860000025000000190fb00000010004c006500660074005200650063007402000001bd000002170000024e00000190fb00000012005200690067006800740052006500630074020000040c000002170000024f00000190fb0000000a0044006500700074006803000001bc000002170000024f00000190fb0000001200440069007300700061007200690074007902000001bd000002170000024e0000018ffb0000001a004400690073007000610072006900740079004e006f0072006d020000040c000002170000025100000190000000010000010f00000396fc0200000003fb0000001e0054006f006f006c002000500072006f00700065007200740069006500730100000041000000780000000000000000fb0000000a00560069006500770073010000002800000396000000ad00fffffffb0000001200530065006c0065006300740069006f006e010000025a000000b200000000000000000000000200000490000000a9fc0100000001fb0000000a00560069006500770073030000004e00000080000002e100000197000000030000073f0000003efc0100000002fb0000000800540069006d006501000000000000073f0000030000fffffffb0000000800540069006d00650100000000000004500000000000000000000004ba0000039600000004000000040000000800000008fc0000000100000002000000010000000a0054006f006f006c00730100000000ffffffff0000000000000000
+  QMainWindow State: 000000ff00000000fd00000004000000000000017e00000396fc020000000ffb0000001200530065006c0065006300740069006f006e00000001e10000009b0000006100fffffffb0000001e0054006f006f006c002000500072006f007000650072007400690065007302000001ed000001df00000185000000a3fb000000120056006900650077007300200054006f006f02000001df000002110000018500000122fb000000200054006f006f006c002000500072006f0070006500720074006900650073003203000002880000011d000002210000017afb000000100044006900730070006c006100790073010000002800000396000000d700fffffffb0000002000730065006c0065006300740069006f006e00200062007500660066006500720200000138000000aa0000023a00000294fb00000014005700690064006500530074006500720065006f02000000e6000000d2000003ee0000030bfb0000000c004b0069006e0065006300740200000186000001060000030c00000261fb00000008004c00650066007403000001bd000000860000024e00000190fb0000000a00520069006700680074030000040c000000860000025000000190fb00000010004c006500660074005200650063007402000001bd000002170000024e00000190fb00000012005200690067006800740052006500630074020000040c000002170000024f00000190fb0000000a0044006500700074006803000001bc000002170000024f00000190fb0000001200440069007300700061007200690074007902000001bd000002170000024e0000018ffb0000001a004400690073007000610072006900740079004e006f0072006d020000040c000002170000025100000190000000010000010f00000396fc0200000003fb0000001e0054006f006f006c002000500072006f00700065007200740069006500730100000041000000780000000000000000fb0000000a00560069006500770073010000002800000396000000ad00fffffffb0000001200530065006c0065006300740069006f006e010000025a000000b200000000000000000000000200000490000000a9fc0100000001fb0000000a00560069006500770073030000004e00000080000002e100000197000000030000073f0000003efc0100000002fb0000000800540069006d006501000000000000073f0000030000fffffffb0000000800540069006d00650100000000000004500000000000000000000004a60000039600000004000000040000000800000008fc0000000100000002000000010000000a0054006f006f006c00730100000000ffffffff0000000000000000
   Right:
     collapsed: false
   RightRect:

wrappers/ros/src/mynt_eye_ros_wrapper/scripts/get_device_info.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2018 Slightech Co., Ltd. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=missing-docstring
+
+from __future__ import print_function
+
+import sys
+
+import rospy
+
+# pylint: disable=wildcard-import, unused-wildcard-import
+from mynt_eye_ros_wrapper.srv import *
+
+
+def get_device_info(**keys):
+  if not keys:
+    sys.exit('get_device_info without keys :(')
+
+  rospy.wait_for_service('mynteye/get_info', 1)
+
+  try:
+    get_info = rospy.ServiceProxy('mynteye/get_info', GetInfo)
+
+    result = {}
+    for key_name, key_value in keys.items():
+      result[key_name] = get_info(key_value).value
+
+    return result
+  except rospy.ServiceException as error:
+    sys.exit('Failed to call service: {}'.format(error))
+
+
+def main():
+  keys = {
+      'DEVICE_NAME': GetInfoRequest.DEVICE_NAME,
+      'SERIAL_NUMBER': GetInfoRequest.SERIAL_NUMBER,
+      'FIRMWARE_VERSION': GetInfoRequest.FIRMWARE_VERSION,
+      'HARDWARE_VERSION': GetInfoRequest.HARDWARE_VERSION,
+      'SPEC_VERSION': GetInfoRequest.SPEC_VERSION,
+      'LENS_TYPE': GetInfoRequest.LENS_TYPE,
+      'IMU_TYPE': GetInfoRequest.IMU_TYPE,
+      'NOMINAL_BASELINE': GetInfoRequest.NOMINAL_BASELINE,
+  }
+  for k, v in get_device_info(**keys).items():
+    print('{}: {}'.format(k, v))
+
+
+if __name__ == '__main__':
+  main()

wrappers/ros/src/mynt_eye_ros_wrapper/src/wrapper_nodelet.cc

@@ -23,6 +23,7 @@
 #include <tf/tf.h>
 #include <tf2_ros/static_transform_broadcaster.h>
 
+#include <mynt_eye_ros_wrapper/GetInfo.h>
 #include <mynt_eye_ros_wrapper/Temp.h>
 
 #include <glog/logging.h>
@@ -56,7 +57,7 @@ class ROSWrapperNodelet : public nodelet::Nodelet {
       double time_end = ros::Time::now().toSec();
       double time_elapsed = time_end - time_beg_;
 
-      LOG(INFO) << "Time elapsed: " << time_elapsed << "s";
+      LOG(INFO) << "Time elapsed: " << time_elapsed << " s";
       LOG(INFO) << "Left count: " << left_count_
                 << ", fps: " << (left_count_ / time_elapsed);
       LOG(INFO) << "Right count: " << right_count_
@@ -191,10 +192,53 @@ class ROSWrapperNodelet : public nodelet::Nodelet {
       }
     }
 
+    // services
+
+    const std::string DEVICE_INFO_SERVICE = "get_info";
+    get_info_service_ = nh_.advertiseService(
+        DEVICE_INFO_SERVICE, &ROSWrapperNodelet::getInfo, this);
+    NODELET_INFO_STREAM("Advertized service " << DEVICE_INFO_SERVICE);
+
     publishStaticTransforms();
     publishTopics();
   }
 
+  bool getInfo(
+      mynt_eye_ros_wrapper::GetInfo::Request &req,     // NOLINT
+      mynt_eye_ros_wrapper::GetInfo::Response &res) {  // NOLINT
+    using Request = mynt_eye_ros_wrapper::GetInfo::Request;
+    switch (req.key) {
+      case Request::DEVICE_NAME:
+        res.value = api_->GetInfo(Info::DEVICE_NAME);
+        break;
+      case Request::SERIAL_NUMBER:
+        res.value = api_->GetInfo(Info::SERIAL_NUMBER);
+        break;
+      case Request::FIRMWARE_VERSION:
+        res.value = api_->GetInfo(Info::FIRMWARE_VERSION);
+        break;
+      case Request::HARDWARE_VERSION:
+        res.value = api_->GetInfo(Info::HARDWARE_VERSION);
+        break;
+      case Request::SPEC_VERSION:
+        res.value = api_->GetInfo(Info::SPEC_VERSION);
+        break;
+      case Request::LENS_TYPE:
+        res.value = api_->GetInfo(Info::LENS_TYPE);
+        break;
+      case Request::IMU_TYPE:
+        res.value = api_->GetInfo(Info::IMU_TYPE);
+        break;
+      case Request::NOMINAL_BASELINE:
+        res.value = api_->GetInfo(Info::NOMINAL_BASELINE);
+        break;
+      default:
+        NODELET_WARN_STREAM("Info of key " << req.key << " not exist");
+        return false;
+    }
+    return true;
+  }
+
   void publishTopics() {
     api_->SetStreamCallback(
         Stream::LEFT, [this](const api::StreamData &data) {
@@ -728,6 +772,8 @@ class ROSWrapperNodelet : public nodelet::Nodelet {
 
   tf2_ros::StaticTransformBroadcaster static_tf_broadcaster_;
 
+  ros::ServiceServer get_info_service_;
+
   // node params
 
   std::string base_frame_id_;

wrappers/ros/src/mynt_eye_ros_wrapper/srv/GetInfo.srv

@@ -0,0 +1,12 @@
+uint32 DEVICE_NAME=0
+uint32 SERIAL_NUMBER=1
+uint32 FIRMWARE_VERSION=2
+uint32 HARDWARE_VERSION=3
+uint32 SPEC_VERSION=4
+uint32 LENS_TYPE=5
+uint32 IMU_TYPE=6
+uint32 NOMINAL_BASELINE=7
+
+uint32 key
+---
+string value