I am getting the camera frame from the android camera Preview Callback in Byte array and pass it to jni code. As we can't use byte in c++ so i am converting it to the integer array as follows:
JNIEXPORT void JNICALL Java_com_omobio_armadillo_Armadillo_onAndroidCameraFrameNative(
JNIEnv* env, jobject, jbyteArray data, jint dataLen, jint width,
jint height, jint bitsPerComponent) {
Armadillo *armadillo = Armadillo::singleton();
jbyte *jArr = env->GetByteArrayElements(data, NULL);
int dataChar[dataLen];
for (int i = 0; i < dataLen; i++) {
dataChar[i] = (int) jArr[i];
}
Then I am paasing it to the the CCImage to create a texture as folllows:
void AppClass::drawAndroidCameraFrame() {
CCLOG("drawAndroidCameraFrame");
int nextBufferIndex = !_bufferIndex;
if (mIsNewFrameReceived) {
mIsNewFrameReceived = false;
return;
}
CCLOG("drawAndroidCameraFrame - creating CCImage");
_image[nextBufferIndex] = new CCImage();
_image[nextBufferIndex]->initWithImageData(mFramePData, mFrameDataLen,
mFrameFormat, mFrameWidth, mFrameHeight, mBitsPerComponent);
if (mIsNewFrameReceived) {
CCLOG("drawAndroidCameraFrame = relasing frame image");
_image[nextBufferIndex]->release();
mIsNewFrameReceived = false;
CCLOG("camera frame process cancelled 2");
return;
}
CCLOG("drawAndroidCameraFrame - creating texture2d");
_texture[nextBufferIndex] = new CCTexture2D();
_texture[nextBufferIndex]->initWithImage(_image[nextBufferIndex]);
if (!_videoSprite) {
CCLOG("Creating new sprite");
if (mIsNewFrameReceived) {
CCLOG("drawAndroidCameraFrame - releasing image an texture");
_image[nextBufferIndex]->release();
_texture[nextBufferIndex]->release();
mIsNewFrameReceived = false;
CCLOG("camera frame process cancelled 3");
return;
}
CCLOG("drawAndroidCameraFrame - creating video sprite");
_videoSprite = new CollisionBitmapSprite();
_videoSprite->initWithTexture(_texture[nextBufferIndex]);
//get director
CCDirector *director = CCDirector::sharedDirector();
// ask director the window size
CCSize size = director->getWinSize();
// position the sprite on the center of the screen
_videoSprite->setPosition(ccp(size.width/2, size.height/2));
//get scale factor
CCSize* imageSize = new CCSize(_image[nextBufferIndex]->getWidth(),
_image[nextBufferIndex]->getHeight());
CCSize scale = getCameraFrameScaleFactor(*imageSize);
// CCLOG ("Scale factor is x=%f and y=%f", scale.width, scale.height);
_videoSprite->setScaleX(scale.width);
_videoSprite->setScaleY(scale.height);
if (mIsNewFrameReceived) {
_image[nextBufferIndex]->release();
_texture[nextBufferIndex]->release();
mIsNewFrameReceived = false;
CCLOG("camera frame process cancelled 4");
return;
}
_videoSprite->setTexture(_texture[nextBufferIndex]);
Shaders::addProgram(_videoSprite, (char *) Shaders::textureVertShader,
mFrameWidth, mFrameHeight);
GLuint i =Shaders::addProgram(_videoSprite, (char *) Shaders::vertShader,
(char *) Shaders::yuvtorgb);
Shaders::setYuvtorgbParameters(_videoSprite,i);
addChild(_videoSprite, -1);
} else {
_videoSprite->setTexture(_texture[nextBufferIndex]);
}
// CCLOG ("Armadillo::drawCameraFrame completed successfully");
//release memory
if (_image[_bufferIndex]) {
_image[_bufferIndex]->release();
}
if (_texture[_bufferIndex]) {
_texture[_bufferIndex]->release();
}
_bufferIndex = nextBufferIndex;
}
As the image is in YUV(N21) format so I am applying the shader to the frame which can convert the image frame in to rgb. The shader program is as follows:
Fragment Shader:
const char *Shaders::yuvtorgb = MULTI_LINE_STRING(
precision highp float;
varying vec2 v_yTexCoord;
varying vec4 v_effectTexCoord;
uniform sampler2D y_texture;
uniform sampler2D u_texture;
uniform sampler2D v_texture;
void main()
{
float y = texture2D(y_texture, v_yTexCoord).r;
float u = texture2D( u_texture, v_yTexCoord ).r;
float v = texture2D( v_texture, v_yTexCoord ).r;
y = 1.1643 * ( y - 0.0625 );
u = u - 0.5;
v = v - 0.5;
float r = y + 1.5958 * v;
float g = y - 0.39173 * u - 0.81290 * v;
float b = y + 2.017 * u;
gl_FragColor = vec4(r,g,b, 1.0);
}
);
Vertex Shader:
const char *Shaders::vertShader = MULTI_LINE_STRING(
attribute vec4 a_position;
attribute vec2 a_yTexCoord;
attribute vec4 a_effectTexCoord;
varying vec2 v_yTexCoord;
varying vec4 v_effectTexCoord;
uniform mat4 u_MVPMatrix;
void main()
{
v_yTexCoord = a_yTexCoord;
v_effectTexCoord = a_effectTexCoord;
gl_Position = u_MVPMatrix * a_position;
}
);
Add Program method:
GLuint Shaders::addProgram(CCSprite *sprite, char *vertShader,
char*fragShader) {
CCGLProgram *glProgram = new CCGLProgram();
if (!glProgram->initWithVertexShaderByteArray(vertShader, fragShader)) {
CCLOG("Shader problem: %s\n %s \n%s", glProgram->vertexShaderLog(), glProgram->fragmentShaderLog(), glProgram->programLog());
}
glProgram->addAttribute(kCCAttributeNamePosition, kCCVertexAttrib_Position);
glProgram->addAttribute(kCCAttributeNameTexCoord,
kCCVertexAttrib_TexCoords);
if (!glProgram->link()) {
CCLOG(
"Shader problem: %s\n %s \n%s", glProgram->vertexShaderLog(), glProgram->fragmentShaderLog(), glProgram->programLog());
}
glProgram->updateUniforms();
sprite->setShaderProgram(glProgram);
return glProgram->getProgram();
}
Then I am applying the shader to the frame sprite:
GLuint i =Shaders::addProgram(_videoSprite, (char *) Shaders::vertShader,
(char *) Shaders::yuvtorgb);
I am getting a image frame in green and pink color. The dark portion changes to green color and bright portion displays in pink color.
The resulting image url is of Green and pink color.
I am stuck over her and not finding any proper solution. Can any one help for solving the issue?