Tanoda
LeapImageRetriever.cs
Go to the documentation of this file.
1/******************************************************************************
2 * Copyright (C) Ultraleap, Inc. 2011-2020. *
3 * *
4 * Use subject to the terms of the Apache License 2.0 available at *
5 * http://www.apache.org/licenses/LICENSE-2.0, or another agreement *
6 * between Ultraleap and you, your company or other organization. *
7 ******************************************************************************/
8
9using UnityEngine;
10using UnityEngine.Serialization;
11using System;
12using System.Collections;
13using Leap.Unity.Query;
14
15namespace Leap.Unity {
16
24 [RequireComponent(typeof(Camera))]
25 [RequireComponent(typeof(LeapServiceProvider))]
26 public class LeapImageRetriever : MonoBehaviour {
27 public const string GLOBAL_COLOR_SPACE_GAMMA_NAME = "_LeapGlobalColorSpaceGamma";
28 public const string GLOBAL_GAMMA_CORRECTION_EXPONENT_NAME = "_LeapGlobalGammaCorrectionExponent";
29 public const string GLOBAL_CAMERA_PROJECTION_NAME = "_LeapGlobalProjection";
30 public const int IMAGE_WARNING_WAIT = 10;
31 public const int LEFT_IMAGE_INDEX = 0;
32 public const int RIGHT_IMAGE_INDEX = 1;
33 public const float IMAGE_SETTING_POLL_RATE = 2.0f;
34
35 [SerializeField]
36 [FormerlySerializedAs("gammaCorrection")]
37 private float _gammaCorrection = 1.0f;
38
39 private LeapServiceProvider _provider;
40 private EyeTextureData _eyeTextureData = new EyeTextureData();
41
42 //Image that we have requested from the service. Are requested in Update and retrieved in OnPreRender
44 protected Image _currentImage = null;
45
46 private long _prevSequenceId;
47 private bool _needQueueReset;
48
50 get {
51 return _eyeTextureData;
52 }
53 }
54
55 public class LeapTextureData {
56 private Texture2D _combinedTexture = null;
57 private byte[] _intermediateArray = null;
58
59 public Texture2D CombinedTexture {
60 get {
61 return _combinedTexture;
62 }
63 }
64
65 public bool CheckStale(Image image) {
66 if (_combinedTexture == null || _intermediateArray == null) {
67 return true;
68 }
69
70 if (image.Width != _combinedTexture.width || image.Height * 2 != _combinedTexture.height) {
71 return true;
72 }
73
74 if (_combinedTexture.format != getTextureFormat(image)) {
75 return true;
76 }
77
78 return false;
79 }
80
81 public void Reconstruct(Image image, string globalShaderName, string pixelSizeName) {
82 int combinedWidth = image.Width;
83 int combinedHeight = image.Height * 2;
84
85 TextureFormat format = getTextureFormat(image);
86
87 if (_combinedTexture != null) {
88 DestroyImmediate(_combinedTexture);
89 }
90
91 _combinedTexture = new Texture2D(combinedWidth, combinedHeight, format, false, true);
92 _combinedTexture.wrapMode = TextureWrapMode.Clamp;
93 _combinedTexture.filterMode = FilterMode.Bilinear;
94 _combinedTexture.name = globalShaderName;
95 _combinedTexture.hideFlags = HideFlags.DontSave;
96
97 _intermediateArray = new byte[combinedWidth * combinedHeight * bytesPerPixel(format)];
98
99 Shader.SetGlobalTexture(globalShaderName, _combinedTexture);
100 Shader.SetGlobalVector(pixelSizeName, new Vector2(1.0f / image.Width, 1.0f / image.Height));
101 }
102
103 public void UpdateTexture(Image image) {
104 _combinedTexture.LoadRawTextureData(image.Data(Image.CameraType.LEFT));
105 _combinedTexture.Apply();
106 }
107
108 private TextureFormat getTextureFormat(Image image) {
109 switch (image.Format) {
110 case Image.FormatType.INFRARED:
111 return TextureFormat.Alpha8;
112 default:
113 throw new Exception("Unexpected image format " + image.Format + "!");
114 }
115 }
116
117 private int bytesPerPixel(TextureFormat format) {
118 switch (format) {
119 case TextureFormat.Alpha8:
120 return 1;
121 default:
122 throw new Exception("Unexpected texture format " + format);
123 }
124 }
125 }
126
127 public class LeapDistortionData {
128 private Texture2D _combinedTexture = null;
129
130 public Texture2D CombinedTexture {
131 get {
132 return _combinedTexture;
133 }
134 }
135
136 public bool CheckStale() {
137 return _combinedTexture == null;
138 }
139
140 public void Reconstruct(Image image, string shaderName) {
141 int combinedWidth = image.DistortionWidth / 2;
142 int combinedHeight = image.DistortionHeight * 2;
143
144 if (_combinedTexture != null) {
145 DestroyImmediate(_combinedTexture);
146 }
147
148 Color32[] colorArray = new Color32[combinedWidth * combinedHeight];
149 _combinedTexture = new Texture2D(combinedWidth, combinedHeight, TextureFormat.RGBA32, false, true);
150 _combinedTexture.filterMode = FilterMode.Bilinear;
151 _combinedTexture.wrapMode = TextureWrapMode.Clamp;
152 _combinedTexture.hideFlags = HideFlags.DontSave;
153
154 addDistortionData(image, colorArray, 0);
155
156 _combinedTexture.SetPixels32(colorArray);
157 _combinedTexture.Apply();
158
159 Shader.SetGlobalTexture(shaderName, _combinedTexture);
160 }
161
162 private void addDistortionData(Image image, Color32[] colors, int startIndex) {
163 float[] distortionData = image.Distortion(Image.CameraType.LEFT).
164 Query().
165 Concat(image.Distortion(Image.CameraType.RIGHT)).
166 ToArray();
167
168 for (int i = 0; i < distortionData.Length; i += 2) {
169 byte b0, b1, b2, b3;
170 encodeFloat(distortionData[i], out b0, out b1);
171 encodeFloat(distortionData[i + 1], out b2, out b3);
172 colors[i / 2 + startIndex] = new Color32(b0, b1, b2, b3);
173 }
174 }
175
176 private void encodeFloat(float value, out byte byte0, out byte byte1) {
177 // The distortion range is -0.6 to +1.7. Normalize to range [0..1).
178 value = (value + 0.6f) / 2.3f;
179 float enc_0 = value;
180 float enc_1 = value * 255.0f;
181
182 enc_0 = enc_0 - (int)enc_0;
183 enc_1 = enc_1 - (int)enc_1;
184
185 enc_0 -= 1.0f / 255.0f * enc_1;
186
187 byte0 = (byte)(enc_0 * 256.0f);
188 byte1 = (byte)(enc_1 * 256.0f);
189 }
190 }
191
192 public class EyeTextureData {
193 private const string GLOBAL_RAW_TEXTURE_NAME = "_LeapGlobalRawTexture";
194 private const string GLOBAL_DISTORTION_TEXTURE_NAME = "_LeapGlobalDistortion";
195 private const string GLOBAL_RAW_PIXEL_SIZE_NAME = "_LeapGlobalRawPixelSize";
196
199 private bool _isStale = false;
200
201 public static void ResetGlobalShaderValues() {
202 Texture2D empty = new Texture2D(1, 1, TextureFormat.ARGB32, false, false);
203 empty.name = "EmptyTexture";
204 empty.hideFlags = HideFlags.DontSave;
205 empty.SetPixel(0, 0, new Color(0, 0, 0, 0));
206
207 Shader.SetGlobalTexture(GLOBAL_RAW_TEXTURE_NAME, empty);
208 Shader.SetGlobalTexture(GLOBAL_DISTORTION_TEXTURE_NAME, empty);
209 }
210
211 public EyeTextureData() {
214 }
215
216 public bool CheckStale(Image image) {
217 return TextureData.CheckStale(image) ||
219 _isStale;
220 }
221
222 public void MarkStale() {
223 _isStale = true;
224 }
225
226 public void Reconstruct(Image image) {
227 TextureData.Reconstruct(image, GLOBAL_RAW_TEXTURE_NAME, GLOBAL_RAW_PIXEL_SIZE_NAME);
228 Distortion.Reconstruct(image, GLOBAL_DISTORTION_TEXTURE_NAME);
229 _isStale = false;
230 }
231
232 public void UpdateTextures(Image image) {
234 }
235 }
236
237#if UNITY_EDITOR
238 void OnValidate() {
239 if (Application.isPlaying) {
241 } else {
242 EyeTextureData.ResetGlobalShaderValues();
243 }
244 }
245#endif
246
247 private void Awake() {
248 _provider = GetComponent<LeapServiceProvider>();
249 if (_provider == null) {
250 _provider = GetComponentInChildren<LeapServiceProvider>();
251 }
252
253 //Enable pooling to reduce overhead of images
254 LeapInternal.MemoryManager.EnablePooling = true;
255
257 }
258
259 private void OnEnable() {
260 subscribeToService();
261 }
262
263 private void OnDisable() {
264 unsubscribeFromService();
265 }
266
267 private void OnDestroy() {
268 StopAllCoroutines();
269 Controller controller = _provider.GetLeapController();
270 if (controller != null) {
271 _provider.GetLeapController().DistortionChange -= onDistortionChange;
272 }
273 }
274
275 private void LateUpdate() {
276 Frame imageFrame = _provider.CurrentFrame;
277
278 _currentImage = null;
279
280 if (_needQueueReset) {
281 while (_imageQueue.TryDequeue()) { }
282 _needQueueReset = false;
283 }
284
285 /* Use the most recent image that is not newer than the current frame
286 * This means that the shown image might be slightly older than the current
287 * frame if for some reason a frame arrived before an image did.
288 *
289 * Usually however, this is just important when robust mode is enabled.
290 * At that time, image ids never line up with tracking ids.
291 */
292 Image potentialImage;
293 while (_imageQueue.TryPeek(out potentialImage)) {
294 if (potentialImage.SequenceId > imageFrame.Id) {
295 break;
296 }
297
298 _currentImage = potentialImage;
299 _imageQueue.TryDequeue();
300 }
301 }
302
303 private void OnPreRender() {
304 if (_currentImage != null) {
305 if (_eyeTextureData.CheckStale(_currentImage)) {
306 _eyeTextureData.Reconstruct(_currentImage);
307 }
308
309 _eyeTextureData.UpdateTextures(_currentImage);
310 }
311 }
312
313 private void subscribeToService() {
314 if (_serviceCoroutine != null) {
315 return;
316 }
317
318 _serviceCoroutine = StartCoroutine(serviceCoroutine());
319 }
320
321 private void unsubscribeFromService() {
322 if (_serviceCoroutine != null) {
323 StopCoroutine(_serviceCoroutine);
324 _serviceCoroutine = null;
325 }
326
327 var controller = _provider.GetLeapController();
328 if (controller != null) {
329 controller.ClearPolicy(Controller.PolicyFlag.POLICY_IMAGES);
330 controller.ImageReady -= onImageReady;
331 controller.DistortionChange -= onDistortionChange;
332 }
333 }
334
335 private Coroutine _serviceCoroutine = null;
336 private IEnumerator serviceCoroutine() {
337 Controller controller = null;
338 do {
339 controller = _provider.GetLeapController();
340 yield return null;
341 } while (controller == null);
342
343 controller.SetPolicy(Controller.PolicyFlag.POLICY_IMAGES);
344 controller.ImageReady += onImageReady;
345 controller.DistortionChange += onDistortionChange;
346 }
347
348 private void onImageReady(object sender, ImageEventArgs args) {
349 Image image = args.image;
350
351 if (!_imageQueue.TryEnqueue(image)) {
352 Debug.LogWarning("Image buffer filled up. This is unexpected and means images are being provided faster than " +
353 "LeapImageRetriever can consume them. This might happen if the application has stalled " +
354 "or we recieved a very high volume of images suddenly.");
355 _needQueueReset = true;
356 }
357
358 if (image.SequenceId < _prevSequenceId) {
359 //We moved back in time, so we should reset the queue so it doesn't get stuck
360 //on the previous image, which will be very old.
361 //this typically happens when the service is restarted while the application is running.
362 _needQueueReset = true;
363 }
364 _prevSequenceId = image.SequenceId;
365 }
366
368 float gamma = 1f;
369 if (QualitySettings.activeColorSpace != ColorSpace.Linear) {
370 gamma = -Mathf.Log10(Mathf.GammaToLinearSpace(0.1f));
371 }
372 Shader.SetGlobalFloat(GLOBAL_COLOR_SPACE_GAMMA_NAME, gamma);
373 Shader.SetGlobalFloat(GLOBAL_GAMMA_CORRECTION_EXPONENT_NAME, 1.0f / _gammaCorrection);
374 }
375
376 void onDistortionChange(object sender, LeapEventArgs args) {
377 _eyeTextureData.MarkStale();
378 }
379 }
380}
UnityEngine.Debug Debug
Definition: TanodaServer.cs:19
System.Drawing.Image Image
Definition: TestScript.cs:37
UnityEngine.Color Color
Definition: TestScript.cs:32
EventHandler< DistortionEventArgs > DistortionChange
Dispatched when the image distortion map changes. The distortion map can change when the Leap device ...
void ClearPolicy(PolicyFlag policy)
Requests clearing a policy.
The Image class represents a stereo image pair from the Leap Motion device.
Definition: Image.cs:20
int Height
The image height.
Definition: Image.cs:223
CameraType
Definition: Image.cs:386
float[] Distortion(CameraType camera)
The distortion calibration map for this image.
Definition: Image.cs:117
int DistortionWidth
The stride of the distortion map.
Definition: Image.cs:282
int Width
The image width.
Definition: Image.cs:213
int DistortionHeight
The distortion map height. Currently fixed at 64.
Definition: Image.cs:294
byte[] Data(CameraType camera)
The buffer containing the image data.
Definition: Image.cs:59
FormatType Format
The image format.
Definition: Image.cs:247
Int64 SequenceId
The image sequence ID.
Definition: Image.cs:203
FormatType
Enumerates the possible image formats.
Definition: Image.cs:376
A generic object with no arguments beyond the event type.
Definition: Events.cs:42
void Reconstruct(Image image, string shaderName)
void Reconstruct(Image image, string globalShaderName, string pixelSizeName)
Acquires images from a LeapServiceProvider and uploads image data as shader global data for use by an...
const string GLOBAL_GAMMA_CORRECTION_EXPONENT_NAME
ProduceConsumeBuffer< Image > _imageQueue
The LeapServiceProvider provides tracked Leap Hand data and images from the device via the Leap servi...
Controller GetLeapController()
Returns the Leap Controller instance.
A Query object is a type of immutable ordered collection of elements that can be used to perform usef...
Definition: Query.cs:90