CameraInputAndroid-napi.cpp

Go to the documentation of this file.

/*
-----------------------------------------------------------------------------
This source file is part of OpenSpace3D
For the latest info, see http://www.openspace3d.com
 
Copyright (c) 2012 I-maginer
 
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU Lesser General Public License as published by the Free Software
Foundation; either version 2 of the License, or (at your option) any later
version.
 
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
 
You should have received a copy of the GNU Lesser General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place - Suite 330, Boston, MA 02111-1307, USA, or go to
http://www.gnu.org/copyleft/lesser.txt
 
-----------------------------------------------------------------------------
*/
 
#include "ArManager.h"
#include "ICameraInput.h"
#include "CameraInputAndroid.h"
 
#include <stdexcept>
#include <opencv2/opencv.hpp>
 
 
// Constants
const int CameraBackendAndroid::CAMERA_FACING_BACK = 0;
const int CameraBackendAndroid::CAMERA_FACING_FRONT = 1;
const std::string CameraBackendAndroid::android_package = "org/imaginer/scol/CameraBackend";
 
// Construction
CameraBackendAndroid::CameraBackendAndroid()
{
  mUpdated = false;
  mIsOpen = false;
  mIndex = 0;
  initJava();
}
 
void CameraBackendAndroid::initJava()
{
  mApp = (struct android_app*)SCgetExtra("this_inst");
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
}
 
CameraBackendAndroid::jmethodbox CameraBackendAndroid::grabMethod(JNIEnv* env, const char* name, const char* signature)
{
  CameraBackendAndroid::jmethodbox result;
  result.clazz = 0;
  result.instance = 0;
  result.method = 0;
 
  // Get class
  jobject oActivity = mApp->activity->clazz;
  jclass cActivity = env->GetObjectClass(oActivity);
  if (cActivity)
  {
    jfieldID camera_backend_fieldID = env->GetFieldID(cActivity, "mCameraBackend", "Lorg/imaginer/scol/CameraBackend;");
    if (camera_backend_fieldID)
      result.instance = env->GetObjectField(oActivity, camera_backend_fieldID);
    if (result.instance)
      result.clazz = env->GetObjectClass(result.instance);
    if (result.clazz)
      result.method = env->GetMethodID(result.clazz, name, signature);
 
    env->DeleteLocalRef(cActivity);
  }
  return result;
}
 
CameraBackendAndroid::~CameraBackendAndroid()
{
  mApp->activity->vm->DetachCurrentThread();
  mFrameBuffer.release();
  mFrameRGB.release();
}
 
// Java methods
void CameraBackendAndroid::startPreview(int width, int height, int index)
{
  mIndex = index;
 
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  //wait for default authorization
  jobject oActivity = mApp->activity->clazz;
  jclass cActivity = env->GetObjectClass(oActivity);
 
  if (cActivity)
  {
    bool authPassed = false;
    while (!authPassed)
    {
      jfieldID reqBoolean = env->GetFieldID(cActivity, "mMinAuthPassed", "Z");
      authPassed = (bool) env->GetBooleanField(oActivity, reqBoolean);
      if (!authPassed)
        boost::this_thread::sleep_for(boost::chrono::milliseconds(100));
    }
 
    //check camera permission
    jmethodID camera_permission = env->GetMethodID(cActivity, "RequestCameraPermission", "()V");
    if (camera_permission)
      env->CallVoidMethod(oActivity, camera_permission);
 
    if (env->ExceptionCheck())
    {
      env->ExceptionClear();
      MMechostr(MSKRUNTIME, "Android camera exception : Check android.permission.CAMERA\n");
    }
 
    authPassed = false;
    while (!authPassed)
    {
      jfieldID reqBoolean = env->GetFieldID(cActivity, "mCameraAuthPassed", "Z");
      authPassed = (bool) env->GetBooleanField(oActivity, reqBoolean);
      if (!authPassed)
        boost::this_thread::sleep_for(boost::chrono::milliseconds(100));
    }
    env->DeleteLocalRef(cActivity);
  }
 
  LOGI(">>>>>>> JgrabMethod look for 'openCamera'");
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "openCamera", "(III)V");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'openCamera'");
    return;
  }
 
  LOGI(">>>>>>> JgrabMethod call method");
  env->CallVoidMethod(method_box.instance, method_box.method,  (jint)width, (jint)height, (jint)index);
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
 
  if (env->ExceptionCheck())
  {
    env->ExceptionDescribe();
    env->ExceptionClear();
    LOGI(">>>>>>> 'openCamera' error");
    return;
  }
 
  if (isCameraOpened())
  {
    mSize = getPreviewSize();
    mFrameBuffer = cv::Mat(mSize.height + mSize.height / 2, mSize.width, CV_8UC1);
    mFrameRGB = cv::Mat(mSize.height,  mSize.width, CV_8UC3);
 
    mIsOpen = true;
  }
  else
  {
    mSize = cv::Size(0, 0);
  }
}
 
void CameraBackendAndroid::stopPreview()
{
  if (mIsOpen)
  {
    mIsOpen = false;
 
    JNIEnv* env = 0;
    if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
      mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
    CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "closeCamera", "()V");
    if (!method_box.instance)
    {
      LOGI(">>>>>>> JgrabMethod failed for 'closeCamera'");
      return;
    }
 
    env->CallVoidMethod(method_box.instance, method_box.method);
    env->DeleteLocalRef(method_box.instance);
    env->DeleteLocalRef(method_box.clazz);
 
    if (env->ExceptionCheck())
    {
      env->ExceptionClear();
      LOGI(">>>>>>> JgrabMethod failed for 'closeCamera'");
    }
  }
}
 
void CameraBackendAndroid::torchOn()
{
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "torchOn", "()V");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'torchOn'");
    return;
  }
 
  env->CallVoidMethod(method_box.instance, method_box.method);
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
 
  if (env->ExceptionCheck())
  {
    env->ExceptionClear();
    LOGI(">>>>>>> JgrabMethod failed for 'torchOn'");
  }
}
 
void CameraBackendAndroid::torchOff()
{
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "torchOff", "()V");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'torchOn'");
    return;
  }
 
  env->CallVoidMethod(method_box.instance, method_box.method);
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
 
  if (env->ExceptionCheck())
  {
    env->ExceptionClear();
    LOGI(">>>>>>> JgrabMethod failed for 'torchOn'");
  }
}
 
int CameraBackendAndroid::getDeviceOrientation()
{
  int ret = 0;
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "getDeviceOrientation", "()I");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'getDeviceOrientation'");
    return false;
  }
 
  ret = (int)env->CallIntMethod(method_box.instance, method_box.method);
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
 
  if (env->ExceptionCheck())
    env->ExceptionClear();
 
  //LOGI(">>>>>>> getDeviceOrientation: %d", ret);
  return ret;
}
 
 
bool CameraBackendAndroid::isPortrait()
{
  int degree = getDeviceOrientation();
  if (degree == 90 || degree == 270)
    return true;
  
  return false;
}
 
bool CameraBackendAndroid::isCameraOpened()
{
  bool ret = false;
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "isCameraOpened", "()Z");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'isCameraOpened'");
    return false;
  }
 
  ret = (bool)env->CallBooleanMethod(method_box.instance, method_box.method);
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
 
  if (env->ExceptionCheck())
    env->ExceptionClear();
 
  return ret;
}
 
bool CameraBackendAndroid::isOpen()
{
  return mIsOpen;
}
 
cv::Size CameraBackendAndroid::getPreviewSize()
{
  cv::Size nSize(0, 0);
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "getPreviewSize", "()Landroid/graphics/Point;");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'getPreviewSize'");
    return nSize;
  }
 
  jobject previewSize_object = env->CallObjectMethod(method_box.instance, method_box.method);
 
  if (env->ExceptionCheck())
  {
    env->ExceptionClear();
    return nSize;
  }
 
  if (previewSize_object == 0)
    return nSize;
 
  jclass cameraSize_class = env->GetObjectClass(previewSize_object);
  jfieldID size_width_field = env->GetFieldID(cameraSize_class, "x", "I");
  jfieldID size_height_field = env->GetFieldID(cameraSize_class, "y", "I");
 
  int width = env->GetIntField(previewSize_object, size_width_field);
  int height = env->GetIntField(previewSize_object, size_height_field);
  nSize = cv::Size(width, height);
 
  env->DeleteLocalRef(cameraSize_class);
  env->DeleteLocalRef(previewSize_object);
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
  return nSize;
}
 
cv::Mat CameraBackendAndroid::grabFrame()
{
  mUpdated = false;
 
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  // get camera facing
  CameraBackendAndroid::jmethodbox facing_method_box = grabMethod(env, "isCameraFacing", "()Z");
  if (!facing_method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'isCameraFacing'");
    return cv::Mat();
  }
 
  bool camIsFacing = (bool)env->CallBooleanMethod(facing_method_box.instance, facing_method_box.method);
 
  if (env->ExceptionCheck())
    env->ExceptionClear();
 
  env->DeleteLocalRef(facing_method_box.instance);
  env->DeleteLocalRef(facing_method_box.clazz);
 
  // get image buffer
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "grabFrame", "()[B");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'grabFrame'");
    return cv::Mat();
  }
 
  jbyteArray frame_buffer_object = (jbyteArray)env->CallObjectMethod(method_box.instance, method_box.method);
  if (!frame_buffer_object || env->ExceptionCheck())
  {
    // read exception
    env->ExceptionClear();
    env->DeleteLocalRef(method_box.instance);
    env->DeleteLocalRef(method_box.clazz);
    return cv::Mat();
  }
 
  int length = env->GetArrayLength((jarray)frame_buffer_object);
  env->GetByteArrayRegion(frame_buffer_object, 0, length, (jbyte*)mFrameBuffer.data);
  env->DeleteLocalRef(frame_buffer_object);
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
 
  if (mFrameBuffer.data == NULL || env->ExceptionCheck())
  {
    // read exception
    env->ExceptionClear();
    MMechostr(MSKRUNTIME, "Could not get camera frame");
    return cv::Mat();
  }
 
  if (mFrameBuffer.data)
  {
    cv::Mat rgbmat;
    cv::cvtColor(mFrameBuffer, rgbmat, cv::COLOR_YUV2BGR_NV21, 3);
 
    if (!rgbmat.empty())
    {
      //rotate picture
 
      int rotation = getDeviceOrientation();
      //image mirror for front view is managed by user
      if (rotation == 180)
      {
        cv::flip(rgbmat, mFrameRGB, camIsFacing ? 1 : -1);
      }
      else if (rotation == 90)
      {
        cv::transpose(rgbmat, mFrameRGB);
        cv::flip(mFrameRGB, mFrameRGB, camIsFacing ? 0 : 1);
      }
      else if (rotation == 270)
      {
        cv::transpose(rgbmat, mFrameRGB);
        if (mIndex != 1)
          cv::flip(mFrameRGB, mFrameRGB, 0);
      }
        /*else if(camIsFacing)
        {
          cv::flip(rgbmat, mFrameRGB, -1);
        }*/
      else
      {
        rgbmat.copyTo(mFrameRGB);
      }
 
      /*cv::Point center(rgbmat.cols / 2, rgbmat.rows / 2);
      cv::Mat rotationMatrix = cv::getRotationMatrix2D(center, rotation + 90, 1.0);
      cv::warpAffine(rgbmat, mFrameRGB, rotationMatrix, (rotation == 0 || rotation == 180) ? cv::Size(rgbmat.cols, rgbmat.rows) : cv::Size(rgbmat.rows, rgbmat.cols));
      */
 
      mUpdated = true;
    }
 
    if (!mUpdated || mFrameRGB.empty())
      return cv::Mat();
  }
  return mFrameRGB;
}
 
bool CameraBackendAndroid::setFocusPoint(int x, int y)
{
  int w = (int)ANativeWindow_getWidth(mApp->window);
  int h = (int)ANativeWindow_getHeight(mApp->window);
 
  JNIEnv* env = 0;
  if (mApp->activity->vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK)
    mApp->activity->vm->AttachCurrentThread(&env, NULL);
 
  CameraBackendAndroid::jmethodbox method_box = grabMethod(env, "doTouchFocus", "(IIII)V");
  if (!method_box.instance)
  {
    LOGI(">>>>>>> JgrabMethod failed for 'setFocusPoint'");
    return false;
  }
 
  env->CallVoidMethod(method_box.instance, method_box.method, (jint)x, (jint)y, (jint)w, (jint)h);
 
  if (env->ExceptionCheck())
    env->ExceptionClear();
 
  // Release references
  env->DeleteLocalRef(method_box.instance);
  env->DeleteLocalRef(method_box.clazz);
  return true;
}
 
cv::Size CameraBackendAndroid::getSize()
{
  return mSize;
}
 
 
CameraInputAndroid::CameraInputAndroid(int index)
  : ICameraInput(index)
{
  mVI = new CameraBackendAndroid();
  mBufferSize = cv::Size(640, 480);
  mInitializing = false;
  MMechostr(MSKDEBUG, "CameraInputAndroid : Created CameraInputAndroid with size %d:%d", mBufferSize.width, mBufferSize.height);
}
 
bool CameraInputAndroid::Initialize()
{
  if (!mInitializing)
  {
    mInitializing = true;
    MMechostr(MSKDEBUG, "CameraInputAndroid::Initialize : Initializing camera backend with size %d:%d", mBufferSize.width, mBufferSize.height);
    mVI->startPreview(mBufferSize.width, mBufferSize.height, mIndex);
    cv::Size nSize = mVI->getSize();
 
    mInitializing = false;
    if (nSize.width == 0 || nSize.height == 0)
      return false;
 
    mBufferSize = nSize;
  }
  return true;
}
 
void CameraInputAndroid::Close()
{
  if (!mInitializing && mVI->isOpen())
    mVI->stopPreview();
}
 
CameraInputAndroid::~CameraInputAndroid()
{
  Close();
  mFrameCached.release();
  SAFE_DELETE(mVI);
}
 
bool CameraInputAndroid::IsOpened()
{
  return mVI->isOpen();
}
 
cv::Mat CameraInputAndroid::UpdateImage()
{
  if (!IsOpened())
    throw std::logic_error("CameraInputAndroid::UpdateImage : Device not initialised");
 
  mVI->grabFrame().copyTo(mFrameCached);
 
  if (mFrameCached.empty())
    throw std::logic_error("CameraInputAndroid::UpdateImage : Frame is empty");
 
  return mFrameCached;
}
 
int CameraInputAndroid::GetWidth()
{
  return mVI->isPortrait() ? mBufferSize.height : mBufferSize.width;
}
 
int CameraInputAndroid::GetHeight()
{
  return mVI->isPortrait() ? mBufferSize.width : mBufferSize.height;
}
 
void CameraInputAndroid::SetSize(int width, int height)
{
  if (IsOpened())
  {
    //MMechostr(MSKDEBUG, "CameraInputAndroid::SetSize : IsOpened : %d x %d\n", width, height);
    if (width < height)
    {
      int pw = height;
      height = width;
      width = pw;
    }
 
    mVI->stopPreview();
    mVI->startPreview(width / 2, height / 2, mIndex);
    cv::Size nSize = mVI->getSize();
    mBufferSize = nSize;
  }
  else
  {
    mBufferSize = cv::Size(width, height);
  }
 
  MMechostr(MSKDEBUG, "CameraInputAndroid::SetSize : Camera set size : %ix%i orientation : %i\n", mBufferSize.width, mBufferSize.height, mVI->getDeviceOrientation());
}
 
bool CameraInputAndroid::TakeSnapshot(std::string path)
{
  std::cout << "CameraInputAndroid::TakeSnapshot : taking a snapshot of camera" << mIndex << "... ";
  if (!IsOpened())
  {
    std::cout << "CameraInputAndroid::TakeSnapshot : failed. camera not opened yet." << std::endl;
    return false;
  }
 
  cv::Mat frame = UpdateImage();
  if (frame.empty())
  {
    std::cout << "CameraInputAndroid::TakeSnapshot : failed. Frame is empty." << std::endl;
    return false;
  }
 
  if (!cv::imwrite(path, frame))
  {
    std::cout << "CameraInputAndroid::TakeSnapshot : failed. Couldn't write to " << path << "." << std::endl;
    return false;
  }
 
  std::cout << "CameraInputAndroid::TakeSnapshot : success. Frame written to " << path << "." << std::endl;
  return true;
}
 
void CameraInputAndroid::RenderToScreen()
{
  cv::Mat frame;
  try
  {
    MMechostr(MSKDEBUG, "CameraInputAndroid::RenderToScreen : updating frame");
    frame = UpdateImage();
  }
  catch (std::exception e)
  {
    MMechostr(MSKDEBUG, "CameraInputAndroid::RenderToScreen : %s", e.what());
    return;
  }
 
  if (frame.empty())
  {
    MMechostr(MSKDEBUG, "CameraInputAndroid::RenderToScreen : frame empty!");
    return;
  }
 
  ANativeWindow* window = (ANativeWindow*)SCgetExtra("hscol");
  if (!window)
  {
    MMechostr(MSKDEBUG, "CameraInputAndroid::RenderToScreen : error: window empty !");
    return;
  }
  ANativeWindow_Buffer buffer;
 
  // Try lock buffer
  if (ANativeWindow_lock(window, &buffer, 0) < 0)
    return;
 
  //MMechostr(MSKDEBUG, "buffer info :: width = %d height = %d stride = %d format = %d", buffer.width, buffer.height, buffer.stride, buffer.format);
 
  cv::resize(frame, frame, cv::Size(buffer.width, buffer.height), 0, 0, cv::INTER_CUBIC);
 
  for (unsigned int y = 0; y < frame.rows; y++)
  {
    for (unsigned int x = 0; x < frame.cols; x++)
    {
      unsigned long srcByte = (x * 3) + (frame.cols * 3 * y);
      unsigned long destByte = (x * 4) + (buffer.width * 4 * y);
 
      ((unsigned char*)buffer.bits)[destByte + 2] = frame.data[srcByte];
      ((unsigned char*)buffer.bits)[destByte + 1] = frame.data[srcByte + 1];
      ((unsigned char*)buffer.bits)[destByte + 0] = frame.data[srcByte + 2];
      ((unsigned char*)buffer.bits)[destByte + 3] = 255;
    }
  }
 
  // Unlock buffer
  ANativeWindow_unlockAndPost(window);
}
 
bool CameraInputAndroid::SetFocusPoint(int x, int y)
{
  return mVI->setFocusPoint(x, y);
}
 
void CameraInputAndroid::SetTorchState(bool state)
{
  if (state)
    mVI->torchOn();
  else
    mVI->torchOff();
}