The document discusses developing natural user interface applications using Intel RealSense depth cameras. It provides an overview of the RealSense SDK capabilities like emotion detection, face tracking, and hand tracking. The SDK supports C++, C#, Java, JavaScript and can integrate with game engines like Unity. The document also includes code examples for initializing the camera, processing frames for tasks like emotion detection and segmentation, and integrating features like touchless control, speech synthesis and face tracking.

1. Developing Natural User
Interface Applications With
RealSenseTM Devices
Pete O’Hanlon @peteohanlon

2. Evolution of Perceptual Computing
 Allows computer to perceive depth in a similar way we do
 Uses 2D and 3D depth camera
 Allows us to interact with computers in a completely new way

3. Different types of camera
 Front facing camera
 Track hands and fingers
 Capture face movements
 Detect and differentiate between foreground and background
 Rear facing camera
 Scan, measure rooms and objects
 Snapshot camera

4. The SDK
 Runs on Windows 8.1 as well as in the browser and Android
 Available for different languages
 C++
 C#
 Java
 JavaScript
 Processing (OS framework for developing visual oriented apps – Java like)
 Runs on popular game engines
 Unity supported

5. What you need
Hardware
 4th generation Intel® Core™ Haswell
 Minimum 8GB free hard disk
 USB 3 port
 RealSenseTM 3D Camera
Software
 Windows 8.1 64 bit
 Visual Studio 2010+
 .NET 4+
 Unity PRO 4.1.0+
 Web Browser
 Processing 2.1.2+
 JDK 1.7.0_11+

6. IR Camera
Full HD 1080p Image Sensor
IR Laser Projector

7. SDK Based Application
Language specific interfaces
SDK interfaces
SDK Core I/O Algorithm Algorithm

8. Natural User Interfaces

9. Framework features
 Emotion detection
 Face tracking
 Facial recognition
 Hand tracking
 Gesture recognition
 Object tracking
 Speech recognition
 Speech synthesis
 Touchless controller
 Segmentation

10. The potential
 Touch free kiosks
 Games
 Interactive books
 Collaboration
 Enhanced tutoring
 Many more…

11. Developing
Setting up

13. Initialisation
private PXCMSenseManager senseManager;
private PXCMHandModule handModule;
private PXCMHandConfiguration handConfiguration;
private PXCMSpeechSynthesis speechSynthesis;
private PXCMAudio.AudioInfo audioInfo;
private PXCMTouchlessController touchlessController;
private PXCMFaceModule faceModule;
private PXCMFaceConfiguration faceConfiguration;
senseManager = PXCMSenseManager.CreateInstance();

14. if (status >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMCapture.Device device =
senseManager.captureManager.device;
device.SetDepthConfidenceThreshold(10);
device.SetIVCAMLaserPower(16);
device.SetIVCAMAccuracy(
PXCMCapture.Device.
IVCAMAccuracy.IVCAM_ACCURACY_COARSE);
device.SetIVCAMMotionRangeTradeOff(21);
device.SetIVCAMFilterOption(6);
if (senseManager.StreamFrames(false)
>= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
// We are streaming data,,,
}
}

15. Emotion detection

16. senseManager.EnableEmotion();
private pxcmStatus ProcessFrame(…
{
switch (moduleId)
{
case PXCMEmotion.CUID:
ProcessEmotionData(module);
break;
}
return pxcmStatus.PXCM_STATUS_NO_ERROR;
}

17. private void ProcessEmotionData(
PXCMBase module)
{
PXCMEmotion emotion =
module.QueryInstance<PXCMEmotion>();
int numberOfFaces = emotion.QueryNumFaces();
for (int face = 0; face < numberOfFaces;
face++)
{
PXCMEmotion.EmotionData[] emotionData =
new PXCMEmotion.EmotionData[10];
emotion.QueryAllEmotionData(face,
out emotionData);
FindStrongestEmotion(emotionData);
}
}

18. private void FindStrongestEmotion(
PXCMEmotion.EmotionData[] emotionData)
{
PXCMEmotion.EmotionData strongestEmotion =
emotionData[0];
int index = 0;
for (int emotionIndex = 1; emotionIndex < 7;
emotionIndex++)
{
if (emotionData[emotionIndex].evidence >
strongestEmotion.evidence)
{
index = emotionIndex;
strongestEmotion =
emotionData[emotionIndex];
}
}
}

19. Segmentation

20. senseManager.Enable3DSeg();
private pxcmStatus ProcessFrame(
int moduleId,
PXCMBase module,
PXCMCapture.Sample captureSample)
{
switch (moduleId)
{
case PXCM3DSeg.CUID:
ProcessSegmentData(module);
break;
}
return pxcmStatus.PXCM_STATUS_NO_ERROR;
}

21. private void ProcessSegmentData(
PXCMBase module)
{
PXCM3DSeg segment =
module.QueryInstance<PXCM3DSeg>();
if (segment != null)
{
using (PXCMImage image =
segment.AcquireSegmentedImage())
{
if (image != null)
{
ExtractForeground (image);
}
}
}
}

22. private void ExtractForeground(PXCMImage image)
{
PXCMImage.ImageData segmentImage;
image.AcquireAccess(
PXCMImage.Access.ACCESS_READ_WRITE,
PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32,
out segmentImage);
try
{
ProcessImage(image, segmentImage);
}
finally
{
image.ReleaseAccess(segmentImage);
}
WriteableBitmap bmp =
segmentImage.ToWritableBitmap(
0,
image.info.width,
image.info.height, 96, 96);
}

23. PXCMImage.ImageInfo info = image.QueryInfo();
for (int y = 0; y < height; y++)
{
unsafe
{
byte* pixel = (byte*)segmentImage.planes[0]
+ y * segmentImage.pitches[0];
for (int x = 0; x < width; x++){
if (pixel[3] == 0){
for (int ch = 0; ch < 3; ch++)
{
pixel[ch] = (byte)0;
}
}
pixel += 4;
}
}
}

24. Hands
Touchless Control

25. senseManager.EnableTouchlessController(null);
touchlessController =
senseManager.QueryTouchlessController();
touchlessController.SubscribeEvent(
new PXCMTouchlessController
.OnFiredUXEventDelegate(
OnTouchlessControllerUXEvent));
PXCMTouchlessController.ProfileInfo
profileInfo;
touchlessController.QueryProfile(out
profileInfo);
profileInfo.config = PXCMTouchlessController
.ProfileInfo.Configuration
.Configuration_Scroll_Horizontally;
touchlessController.SetProfile(profileInfo);

26. private void OnTouchlessControllerUXEvent(
PXCMTouchlessController.UXEventData data)
{
switch (data.type)
{
case PXCMTouchlessController
.UXEventData.UXEventType
.UXEvent_StartScroll:
break;
case PXCMTouchlessController
.UXEventData.UXEventType
.UXEvent_Scroll:
break;
case PXCMTouchlessController
.UXEventData.UXEventType
.UXEvent_Select:
break;
}
}

27. Speech Synthesis

28. private void SetupSpeechSynthesis()
{
if (senseManager.session.CreateImpl<PXCMSpeechSynthesis>
(out speechSynthesis)
>= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMSpeechSynthesis
.ProfileInfo profileInfo;
if (speechSynthesis.QueryProfile(0,
out profileInfo)
>= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
profileInfo.voice =
PXCMSpeechSynthesis.VoiceType
.VOICE_ANY;
audioInfo = profileInfo.outputs;
speechSynthesis
.SetProfile(profileInfo);
}
}
}

29. public void Say(string sentence)
{
if (speechSynthesis != null)
{
speechSynthesis.BuildSentence(1, sentence);
int buffer =
speechSynthesis.QueryBufferNum(1);
VoiceOut voiceOut =
new VoiceOut(audioInfo);
for (int i = 0; i < buffer; i++)
{
PXCMAudio audio =
speechSynthesis.QueryBuffer(1, i);
voiceOut.RenderAudio(audio);
}
voiceOut.Close();
speechSynthesis.ReleaseSentence(1);
}
}

30. Face tracking

31. senseManager.EnableFace();
PXCMFaceModule faceModule = senseManager.QueryFace();
PXCMFaceConfiguration config =
faceModule.CreateActiveConfiguration();
config.detection.isEnabled = true;
config.detection.maxTrackedFaces = 2;
config.landmarks.isEnabled = true;
config.ApplyChanges();

32. private pxcmStatus ProcessFrame(
int moduleId,
PXCMBase module,
PXCMCapture.Sample captureSample)
{
switch (moduleId)
{
case PXCMFaceModule.CUID:
ProcessFace(module);
break;
}
return pxcmStatus.PXCM_STATUS_NO_ERROR;
}

33. private void ProcessFace(PXCMBase module)
{
using (PXCMFaceModule faceModule =
module.QueryInstance<PXCMFaceModule>())
{
using (PXCMFaceData faceData = faceModule.CreateOutput())
{
int faceCount = faceData.QueryNumberOfDetectedFaces();
for (int faceIndex = 0; faceIndex < faceCount;
faceIndex++)
{
PXCMFaceData.Face face =
faceData.QueryFaceByIndex(faceIndex);
GetFaceBounds(face);
GetLandmarkData(face);
}
}
}
}

34. private static void GetLandmarkData(
PXCMFaceData.Face face)
{
PXCMFaceData.LandmarkPoint[] landmarks;
PXCMFaceData.LandmarksData landmarkData =
face.QueryLandmarks();
landmarkData.QueryPoints(out landmarks);
}
private static void GetFaceBounds(
PXCMFaceData.Face face)
{
PXCMFaceData.DetectionData detectionData =
face.QueryDetection();
PXCMRectI32 boundingRect;
detectionData.QueryBoundingRect(
out boundingRect);
}

35. Starting the loop

36. PXCMSenseManager.Handler handler = new
PXCMSenseManager.Handler
{
onModuleProcessedFrame = ProcessFrame
};
private pxcmStatus ProcessFrame(
int moduleId,
PXCMBase module,
PXCMCapture.Sample captureSample)
{
return pxcmStatus.PXCM_STATUS_NO_ERROR;
}
pxcmStatus status = senseManager.Init(handler);

37. What next?
 Order your camera
 http://click.intel.com/intel-realsense-developer-kit.html
 Download the SDK
 https://software.intel.com/en-us/realsense/intel-realsense-sdk-for-windows
 Code