From 465a5d4e0c06cb5fd42130a4365effcd8e45c228 Mon Sep 17 00:00:00 2001 From: Guillaume Beraudo Date: Wed, 17 Nov 2010 15:52:42 +0100 Subject: [PATCH 1/2] First working Android video version. --- res/layout/videocall.xml | 5 +- src/org/linphone/VideoCallActivity.java | 12 +- .../linphone/core/AndroidCameraRecord.java | 230 +++++++++++++----- .../core/AndroidCameraRecordBufferedImpl.java | 69 ++++++ .../core/AndroidCameraRecordImpl.java | 55 ++++- .../core/tutorials/JavaCameraRecordImpl.java | 15 +- .../core/tutorials/TestVideoActivity.java | 19 +- 7 files changed, 316 insertions(+), 89 deletions(-) create mode 100644 src/org/linphone/core/AndroidCameraRecordBufferedImpl.java diff --git a/res/layout/videocall.xml b/res/layout/videocall.xml index ba58b1963..5259b4136 100644 --- a/res/layout/videocall.xml +++ b/res/layout/videocall.xml @@ -1,6 +1,7 @@ - + android:layout_height="fill_parent" android:layout_width="fill_parent" android:orientation="vertical"> + + diff --git a/src/org/linphone/VideoCallActivity.java b/src/org/linphone/VideoCallActivity.java index ac5a2b2bd..244174006 100644 --- a/src/org/linphone/VideoCallActivity.java +++ b/src/org/linphone/VideoCallActivity.java @@ -20,17 +20,27 @@ package org.linphone; +import org.linphone.core.AndroidCameraRecord; + import android.app.Activity; import android.os.Bundle; +import android.os.Handler; import android.view.SurfaceView; public class VideoCallActivity extends Activity { SurfaceView mVideoView; + SurfaceView mVideoCaptureView; + private Handler mHandler = new Handler() ; + public void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.videocall); mVideoView = (SurfaceView) findViewById(R.id.video_surface); - LinphoneService.instance().getLinphoneCore().setVideoWindow((Object) mVideoView) ; + LinphoneService.instance().getLinphoneCore().setVideoWindow((Object) mVideoView); + +// mVideoCaptureView = new SurfaceView(getApplicationContext()); + mVideoCaptureView = (SurfaceView) findViewById(R.id.video_capture_surface); + AndroidCameraRecord.setSurfaceView(mVideoCaptureView, mHandler); } } diff --git a/src/org/linphone/core/AndroidCameraRecord.java b/src/org/linphone/core/AndroidCameraRecord.java index ae07c6d56..6ebce9856 100644 --- a/src/org/linphone/core/AndroidCameraRecord.java +++ b/src/org/linphone/core/AndroidCameraRecord.java @@ -19,102 +19,176 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. package org.linphone.core; import android.hardware.Camera; +import android.hardware.Camera.ErrorCallback; import android.hardware.Camera.PreviewCallback; +import android.os.Build; +import android.os.Handler; import android.util.Log; import android.view.SurfaceHolder; import android.view.SurfaceView; +import android.view.SurfaceHolder.Callback; -public abstract class AndroidCameraRecord implements SurfaceHolder.Callback { +public abstract class AndroidCameraRecord { + + public static final int ANDROID_VERSION = Integer.parseInt(Build.VERSION.SDK); + protected static Camera camera; + private static SurfaceView surfaceView; + + protected int fps; + protected int height; + protected int width; + private int longTermVisibility; - protected Camera camera; - private static SurfaceView surfaceView; // should be initialized first... - protected int rate; - private int visibility = SurfaceView.GONE; // Automatically hidden private boolean visibilityChangeable = false; + private PreviewCallback storedPreviewCallback; - protected final SurfaceView getSurfaceView() { - return surfaceView; + private static AndroidCameraRecord instance; + private static Handler handler; + private static boolean previewStarted; + + public AndroidCameraRecord() { + // TODO check if another instance is loaded and kill it. + instance = this; } - /** - * AndroidCameraRecord.setSurfaceView() should be called first. - * @param rate + public void setParameters(int height, int width, float fps, boolean hide) { + this.fps = Math.round(fps); + this.height = height; + this.width = width; + this.longTermVisibility = hide ? SurfaceView.GONE : SurfaceView.VISIBLE; + + if (surfaceView != null) { + Log.d("Linphone", "Surfaceview defined and ready; starting video capture"); + instance.startPreview(); + } else { + Log.w("Linphone", "Surfaceview not defined; postponning video capture"); + } + } + + /* + * AndroidCameraRecord.setSurfaceView() should be called first, from the Activity code. + * It will start automatically */ - public AndroidCameraRecord(int rate) { + private void startPreview() { + assert surfaceView != null; + + if (previewStarted) { + Log.w("Linphone", "Already started"); + return; + } + + if (surfaceView.getVisibility() != SurfaceView.VISIBLE) { + // Illegal state + Log.e("Linphone", "Illegal state: video capture surface view is not visible"); + return; + } + + camera=Camera.open(); + camera.setErrorCallback(new ErrorCallback() { + public void onError(int error, Camera camera) { + Log.e("Linphone", "Camera error : " + error); + } + }); + + + Camera.Parameters parameters=camera.getParameters(); + + parameters.setPreviewSize(width, height); + parameters.setPreviewFrameRate(fps); + camera.setParameters(parameters); + + SurfaceHolder holder = surfaceView.getHolder(); - holder.addCallback(this); - - this.rate = rate; - } - - - /** - * AndroidCameraRecord.setSurfaceView() should be called first. - * @param rate - * @param visilibity - */ - public AndroidCameraRecord(int rate, int visilibity) { - this(rate); - this.visibility = visilibity; - } - - - public void surfaceCreated(SurfaceHolder holder) { try { camera.setPreviewDisplay(holder); } catch (Throwable t) { - Log.e("PictureDemo-surfaceCallback", "Exception in setPreviewDisplay()", t); + Log.e("Linphone", "Exception in Video capture setPreviewDisplay()", t); } - } + try { + camera.startPreview(); + previewStarted = true; + } catch (Throwable e) { + Log.e("Linphone", "Can't start camera preview"); + } - public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { - Camera.Parameters parameters=camera.getParameters(); + previewStarted = true; - parameters.setPreviewSize(width, height); - parameters.setPreviewFrameRate(rate); - camera.setParameters(parameters); - - camera.startPreview(); + + + // Register callback to get capture buffer + if (storedPreviewCallback != null) { + reallySetPreviewCallback(camera, storedPreviewCallback); + } + visibilityChangeable = true; - if (surfaceView.getVisibility() != visibility) { + if (surfaceView.getVisibility() != longTermVisibility) { updateVisibility(); } - + + onCameraStarted(camera); } + - public void surfaceDestroyed(SurfaceHolder holder) { - camera.stopPreview(); - camera.release(); - camera=null; - } - public void setPreviewCallBack(PreviewCallback cb) { - camera.setPreviewCallback(cb); - } - private void updateVisibility() { - if (!visibilityChangeable) { - throw new IllegalStateException("Visilibity not changeable now"); + /** + * Hook. + * @param camera + */ + public void onCameraStarted(Camera camera) {} + + public void setOrStorePreviewCallBack(PreviewCallback cb) { + if (camera == null) { + Log.w("Linphone", "Capture camera not ready, storing callback"); + this.storedPreviewCallback = cb; + return; } - - surfaceView.setVisibility(visibility); + + reallySetPreviewCallback(camera, cb); } - public void setVisibility(int visibility) { - if (visibility == this.visibility) return; - this.visibility = visibility; - updateVisibility(); - } + + public static final void setSurfaceView(final SurfaceView sv, Handler mHandler) { + AndroidCameraRecord.handler = mHandler; + SurfaceHolder holder = sv.getHolder(); + holder.setType(SurfaceHolder.SURFACE_TYPE_PUSH_BUFFERS); - public static final void setSurfaceView(SurfaceView sv) { - AndroidCameraRecord.surfaceView = sv; + holder.addCallback(new Callback() { + public void surfaceDestroyed(SurfaceHolder holder) { + AndroidCameraRecord.surfaceView = null; + + if (camera == null) { + Log.e("AndroidCameraRecord.surfaceDestroyed", "illegal state"); + return; + } + camera.setPreviewCallback(null); // TODO check if used whatever the SDK version + camera.stopPreview(); + camera.release(); + camera=null; + previewStarted = false; + Log.w("Linphone", "The video capture Surface view has been destroyed"); + } + + public void surfaceCreated(SurfaceHolder holder) { + AndroidCameraRecord.surfaceView = sv; + + if (instance != null) { + instance.startPreview(); + } + } + + public void surfaceChanged(SurfaceHolder holder, int format, int width, + int height) { + // Do nothing + } + }); } @@ -124,4 +198,40 @@ public abstract class AndroidCameraRecord implements SurfaceHolder.Callback { * @param buffer buffer to reuse */ public void addBackCaptureBuffer(byte[] buffer) {} + + + + private void updateVisibility() { + if (!visibilityChangeable) { + throw new IllegalStateException("Visilibity not changeable now"); + } + + handler.post(new Runnable() { + public void run() { + Log.d("Linphone", "Changing video capture surface view visibility :" + longTermVisibility); + surfaceView.setVisibility(longTermVisibility); + } + }); + } + + public void setVisibility(int visibility) { + if (visibility == this.longTermVisibility) return; + + this.longTermVisibility = visibility; + updateVisibility(); + } + + + public void stopCaptureCallback() { + if (camera != null) { + reallySetPreviewCallback(camera, null); + } + } + + protected void reallySetPreviewCallback(Camera camera, PreviewCallback cb) { + camera.setPreviewCallback(cb); + } + } + + diff --git a/src/org/linphone/core/AndroidCameraRecordBufferedImpl.java b/src/org/linphone/core/AndroidCameraRecordBufferedImpl.java new file mode 100644 index 000000000..c65f479ad --- /dev/null +++ b/src/org/linphone/core/AndroidCameraRecordBufferedImpl.java @@ -0,0 +1,69 @@ +/* +AndroidCameraRecord8Impl.java +Copyright (C) 2010 Belledonne Communications, Grenoble, France + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +*/ +package org.linphone.core; + +import android.hardware.Camera; +import android.hardware.Camera.PreviewCallback; +import android.hardware.Camera.Size; +import android.util.Log; + +/** + * + * Android >= 8 (2.2) version. + * @author Guillaume Beraudo + * + */ +public class AndroidCameraRecordBufferedImpl extends AndroidCameraRecordImpl { + + public AndroidCameraRecordBufferedImpl(long filterCtxPtr) { + super(filterCtxPtr); + } + + @Override + protected void reallySetPreviewCallback(Camera camera, PreviewCallback cb) { + Log.d("Linphone", "Setting optimized callback with buffer (Android >= 8). Remember to manage the pool of buffers!!!"); + camera.setPreviewCallbackWithBuffer(cb); + } + + @Override + public void addBackCaptureBuffer(byte[] buffer) { + camera.addCallbackBuffer(buffer); + } + + @Override + public void onCameraStarted(Camera camera) { + super.onCameraStarted(camera); + + Size s = camera.getParameters().getPreviewSize(); + int wishedBufferSize = s.height * s.width * 3 / 2; + + camera.addCallbackBuffer(new byte[wishedBufferSize]); + camera.addCallbackBuffer(new byte[wishedBufferSize]); +/* + for (int i=1; i < 30; i++) { + camera.addCallbackBuffer(new byte[wishedBufferSize]); + }*/ + } + + @Override + protected void badBufferLengthReceived(byte[] buffer, int expectedBufferLength) { + super.badBufferLengthReceived(buffer, expectedBufferLength); + addBackCaptureBuffer(buffer); + } +} diff --git a/src/org/linphone/core/AndroidCameraRecordImpl.java b/src/org/linphone/core/AndroidCameraRecordImpl.java index c9173d1da..aab29a183 100644 --- a/src/org/linphone/core/AndroidCameraRecordImpl.java +++ b/src/org/linphone/core/AndroidCameraRecordImpl.java @@ -20,6 +20,7 @@ package org.linphone.core; import android.hardware.Camera; import android.hardware.Camera.PreviewCallback; +import android.hardware.Camera.Size; import android.util.Log; /** @@ -31,19 +32,63 @@ import android.util.Log; public class AndroidCameraRecordImpl extends AndroidCameraRecord implements PreviewCallback { private long filterCtxPtr; + private double timeElapsedBetweenFrames = 0; + private long lastFrameTime = 0; - public AndroidCameraRecordImpl(long filterCtxPtr, int rate) { - super(rate); - this.filterCtxPtr = filterCtxPtr; - setPreviewCallBack(this); + public AndroidCameraRecordImpl(long filterCtxPtr) { + super(); + + try { + this.filterCtxPtr = filterCtxPtr; + setOrStorePreviewCallBack(this); + } catch (Throwable e) { + Log.e("Linphone", "Error"); + } + } private native void putImage(long filterCtxPtr, byte[] buffer); + public void onPreviewFrame(byte[] data, Camera camera) { - Log.d("onPreviewFrame: ", Integer.toString(data.length)); + if (data == null) { + Log.e("Linphone", "onPreviewFrame Called with null buffer"); + return; + } + + Size s = camera.getParameters().getPreviewSize(); + int expectedBuffLength = s.width * s.height * 3 /2; + if (expectedBuffLength != data.length) { + badBufferLengthReceived(data, expectedBuffLength); + return; + } + + long curTime = System.currentTimeMillis(); + if (lastFrameTime == 0) { + lastFrameTime = curTime; + putImage(filterCtxPtr, data); + return; + } + + double currentTimeElapsed = 0.8 * (curTime - lastFrameTime) / 1000 + 0.2 * timeElapsedBetweenFrames; + if (1 / currentTimeElapsed > fps) { +// Log.d("Linphone", "Clipping frame " + Math.round(1 / currentTimeElapsed) + " > " + fps); + addBackCaptureBuffer(data); + return; + } + lastFrameTime = curTime; + timeElapsedBetweenFrames = currentTimeElapsed; + + // Log.d("onPreviewFrame: ", Integer.toString(data.length)); putImage(filterCtxPtr, data); } + + // Hook + protected void badBufferLengthReceived(byte[] data, int expectedBuffLength) { + Log.e("Linphone", "onPreviewFrame called with bad buffer length " + data.length + + " whereas expected is " + expectedBuffLength + " don't calling putImage"); + } + } diff --git a/src/org/linphone/core/tutorials/JavaCameraRecordImpl.java b/src/org/linphone/core/tutorials/JavaCameraRecordImpl.java index 45499db78..30c1838f6 100644 --- a/src/org/linphone/core/tutorials/JavaCameraRecordImpl.java +++ b/src/org/linphone/core/tutorials/JavaCameraRecordImpl.java @@ -30,21 +30,16 @@ public class JavaCameraRecordImpl extends AndroidCameraRecord implements Preview private TextView debug; private long count = 0; private float averageCalledRate; - private int averageWindowSize = 2 * rate; private long startTime; private long endTime; - public JavaCameraRecordImpl(int rate) { - super(rate); - setPreviewCallBack(this); + public JavaCameraRecordImpl() { + super(); + setOrStorePreviewCallBack(this); } - public JavaCameraRecordImpl(int rate, int visilibity) { - super(rate, visilibity); - setPreviewCallBack(this); - } public void setDebug(TextView debug) { this.debug = debug; @@ -52,9 +47,9 @@ public class JavaCameraRecordImpl extends AndroidCameraRecord implements Preview public void onPreviewFrame(byte[] data, Camera camera) { - if ((count % averageWindowSize) == 0) { + if ((count % 2 * fps) == 0) { endTime = System.currentTimeMillis(); - averageCalledRate = (100000 * averageWindowSize) / (endTime - startTime); + averageCalledRate = (100000 * 2 * fps) / (endTime - startTime); averageCalledRate /= 100f; startTime = endTime; } diff --git a/src/org/linphone/core/tutorials/TestVideoActivity.java b/src/org/linphone/core/tutorials/TestVideoActivity.java index 6526e1cce..e016cad25 100644 --- a/src/org/linphone/core/tutorials/TestVideoActivity.java +++ b/src/org/linphone/core/tutorials/TestVideoActivity.java @@ -23,7 +23,7 @@ import org.linphone.core.AndroidCameraRecord; import android.app.Activity; import android.os.Bundle; -import android.view.SurfaceHolder; +import android.os.Handler; import android.view.SurfaceView; import android.widget.TextView; @@ -36,14 +36,11 @@ import android.widget.TextView; public class TestVideoActivity extends Activity { private SurfaceView surfaceView; - private static final int rate = 15; + private static final int rate = 7; - - - @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -51,14 +48,14 @@ public class TestVideoActivity extends Activity { surfaceView=(SurfaceView)findViewById(R.id.videotest_surfaceView); - SurfaceHolder holder=surfaceView.getHolder(); - holder.setFixedSize(320, 240); - holder.setType(SurfaceHolder.SURFACE_TYPE_PUSH_BUFFERS); +// SurfaceHolder holder=surfaceView.getHolder(); +// holder.setType(SurfaceHolder.SURFACE_TYPE_PUSH_BUFFERS); - AndroidCameraRecord.setSurfaceView(surfaceView); + AndroidCameraRecord.setSurfaceView(surfaceView, new Handler()); - JavaCameraRecordImpl manager = new JavaCameraRecordImpl(rate, SurfaceView.VISIBLE); - manager.setDebug((TextView) findViewById(R.id.videotest_debug)); + JavaCameraRecordImpl recorder = new JavaCameraRecordImpl(); + recorder.setDebug((TextView) findViewById(R.id.videotest_debug)); + recorder.setParameters(288, 352, rate, false); } From d2d1e257b6af2094a6aa7dfc053d44dd22233690 Mon Sep 17 00:00:00 2001 From: guillaume Date: Thu, 18 Nov 2010 15:01:15 +0100 Subject: [PATCH 2/2] addbackbuffer managed directly in java code --- src/org/linphone/core/AndroidCameraRecord.java | 8 -------- .../core/AndroidCameraRecordBufferedImpl.java | 11 +++-------- src/org/linphone/core/AndroidCameraRecordImpl.java | 9 ++------- 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/src/org/linphone/core/AndroidCameraRecord.java b/src/org/linphone/core/AndroidCameraRecord.java index 6ebce9856..17d9f8f9a 100644 --- a/src/org/linphone/core/AndroidCameraRecord.java +++ b/src/org/linphone/core/AndroidCameraRecord.java @@ -192,14 +192,6 @@ public abstract class AndroidCameraRecord { } - /** - * Hook to add back a buffer for reuse in capture. - * Override in a version supporting addPreviewCallBackWithBuffer() - * @param buffer buffer to reuse - */ - public void addBackCaptureBuffer(byte[] buffer) {} - - private void updateVisibility() { if (!visibilityChangeable) { diff --git a/src/org/linphone/core/AndroidCameraRecordBufferedImpl.java b/src/org/linphone/core/AndroidCameraRecordBufferedImpl.java index c65f479ad..715f17765 100644 --- a/src/org/linphone/core/AndroidCameraRecordBufferedImpl.java +++ b/src/org/linphone/core/AndroidCameraRecordBufferedImpl.java @@ -41,11 +41,6 @@ public class AndroidCameraRecordBufferedImpl extends AndroidCameraRecordImpl { camera.setPreviewCallbackWithBuffer(cb); } - @Override - public void addBackCaptureBuffer(byte[] buffer) { - camera.addCallbackBuffer(buffer); - } - @Override public void onCameraStarted(Camera camera) { super.onCameraStarted(camera); @@ -62,8 +57,8 @@ public class AndroidCameraRecordBufferedImpl extends AndroidCameraRecordImpl { } @Override - protected void badBufferLengthReceived(byte[] buffer, int expectedBufferLength) { - super.badBufferLengthReceived(buffer, expectedBufferLength); - addBackCaptureBuffer(buffer); + public void onPreviewFrame(byte[] data, Camera camera) { + super.onPreviewFrame(data, camera); + camera.addCallbackBuffer(data); } } diff --git a/src/org/linphone/core/AndroidCameraRecordImpl.java b/src/org/linphone/core/AndroidCameraRecordImpl.java index aab29a183..6186e458a 100644 --- a/src/org/linphone/core/AndroidCameraRecordImpl.java +++ b/src/org/linphone/core/AndroidCameraRecordImpl.java @@ -60,7 +60,8 @@ public class AndroidCameraRecordImpl extends AndroidCameraRecord implements Prev Size s = camera.getParameters().getPreviewSize(); int expectedBuffLength = s.width * s.height * 3 /2; if (expectedBuffLength != data.length) { - badBufferLengthReceived(data, expectedBuffLength); + Log.e("Linphone", "onPreviewFrame called with bad buffer length " + data.length + + " whereas expected is " + expectedBuffLength + " don't calling putImage"); return; } @@ -74,7 +75,6 @@ public class AndroidCameraRecordImpl extends AndroidCameraRecord implements Prev double currentTimeElapsed = 0.8 * (curTime - lastFrameTime) / 1000 + 0.2 * timeElapsedBetweenFrames; if (1 / currentTimeElapsed > fps) { // Log.d("Linphone", "Clipping frame " + Math.round(1 / currentTimeElapsed) + " > " + fps); - addBackCaptureBuffer(data); return; } lastFrameTime = curTime; @@ -85,10 +85,5 @@ public class AndroidCameraRecordImpl extends AndroidCameraRecord implements Prev } - // Hook - protected void badBufferLengthReceived(byte[] data, int expectedBuffLength) { - Log.e("Linphone", "onPreviewFrame called with bad buffer length " + data.length - + " whereas expected is " + expectedBuffLength + " don't calling putImage"); - } }