#32 Add hand tracking by using frame.fillPoses for devices that support it

Rufus31415 · Rufus31415 · commit 7b37e75ede96 · 2021-05-01T14:23:36.000+02:00
diff --git a/SimpleWebXR-Demo/Assets/SimpleWebXR/Scripts/MRTK/Profiles/SimpleWebXRHandTrackingProfile.asset b/SimpleWebXR-Demo/Assets/SimpleWebXR/Scripts/MRTK/Profiles/SimpleWebXRHandTrackingProfile.asset
@@ -0,0 +1,25 @@
+%YAML 1.1
+%TAG !u! tag:unity3d.com,2011:
+--- !u!114 &11400000
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 0}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 8275efdbe76bdff49a97a8e82fba118d, type: 3}
+  m_Name: SimpleWebXRHandTrackingProfile
+  m_EditorClassIdentifier: 
+  isCustomProfile: 1
+  jointPrefab: {fileID: 1955475817299902, guid: 6a3f88d2571cd234a86d95ee5856b9ec,
+    type: 3}
+  palmPrefab: {fileID: 6797406804172968804, guid: 750bdc3344567a447960aae3eda2b462,
+    type: 3}
+  fingertipPrefab: {fileID: 7094064642998883381, guid: b37dde41a983d664c8a09a91313733e7,
+    type: 3}
+  handMeshPrefab: {fileID: 1887883006053652, guid: a86f479797fea8f4189f924b3b6ad979,
+    type: 3}
+  handMeshVisualizationModes: -1
+  handJointVisualizationModes: 0
diff --git a/SimpleWebXR-Demo/Assets/SimpleWebXR/Scripts/MRTK/Profiles/SimpleWebXRHandTrackingProfile.asset.meta b/SimpleWebXR-Demo/Assets/SimpleWebXR/Scripts/MRTK/Profiles/SimpleWebXRHandTrackingProfile.asset.meta
diff --git a/SimpleWebXR-Demo/Assets/SimpleWebXR/Scripts/MRTK/Profiles/SimpleWebXRInputSystemProfile.asset b/SimpleWebXR-Demo/Assets/SimpleWebXR/Scripts/MRTK/Profiles/SimpleWebXRInputSystemProfile.asset
@@ -74,5 +74,5 @@ MonoBehaviour:
     type: 2}
   controllerVisualizationProfile: {fileID: 11400000, guid: 345c06fdf3732db46b96299bd3cba653,
     type: 2}
-  handTrackingProfile: {fileID: 11400000, guid: 7f1e3cd673742f94ca860ac7ae733024,
+  handTrackingProfile: {fileID: 11400000, guid: 1ee5870617471ac4e82a5d1fe41dac1b,
     type: 2}
diff --git a/SimpleWebXR-Demo/Assets/SimpleWebXR/Scripts/MRTK/SimpleWebXRHand.cs b/SimpleWebXR-Demo/Assets/SimpleWebXR/Scripts/MRTK/SimpleWebXRHand.cs
@@ -28,6 +28,10 @@ public override bool TryGetJoint(TrackedHandJoint joint, out MixedRealityPose po
 
         #endregion IMixedRealityHand Implementation
 
+        private ArticulatedHandDefinition handDefinition;
+        private ArticulatedHandDefinition HandDefinition => handDefinition ?? (handDefinition = Definition as ArticulatedHandDefinition);
+
+
 
         public override MixedRealityInteractionMapping[] DefaultInteractions => new[]
         {
@@ -50,24 +54,35 @@ public override void SetupDefaultInteractions()
 
         public override bool IsInPointingPose => true;
 
+        private MixedRealityPose GetJointMixedRealityPose(WebXRJoint joint)
+        {
+
+            var position = MixedRealityPlayspace.TransformPoint(joint.Position);
+            var rotation = MixedRealityPlayspace.Rotation * joint.Rotation;
+
+            return new MixedRealityPose(position, rotation);
+        }
+
         public void UpdateController(WebXRInputSource controller)
         {
             if (!Enabled) return;
 
             IsPositionAvailable = IsRotationAvailable = controller.Hand.Available;
 
 
-            for (int i = 0; i < WebXRHand.JOINT_COUNT; i++)
+            jointPoses[TrackedHandJoint.Wrist] = GetJointMixedRealityPose(controller.Hand.Joints[WebXRHand.WRIST]);
+
+
+            for (int i = WebXRHand.THUMB_METACARPAL; i < WebXRHand.JOINT_COUNT; i++)
             {
                 var joint = controller.Hand.Joints[i];
 
-                var position = MixedRealityPlayspace.TransformPoint(joint.Position);
-                var rotation = MixedRealityPlayspace.Rotation * joint.Rotation;
-
-                jointPoses[(TrackedHandJoint)(i + 1)] = new MixedRealityPose(position, rotation);
+                jointPoses[(TrackedHandJoint)(i + 2)] = GetJointMixedRealityPose(joint);
             }
 
-            var indexPose = jointPoses[(TrackedHandJoint)WebXRHand.INDEX_PHALANX_TIP + 1];
+            jointPoses[TrackedHandJoint.Palm] = new MixedRealityPose((jointPoses[TrackedHandJoint.MiddleMetacarpal].Position + jointPoses[TrackedHandJoint.MiddleMetacarpal].Position)/2, jointPoses[TrackedHandJoint.MiddleMetacarpal].Rotation) ;
+
+            var indexPose = jointPoses[TrackedHandJoint.IndexTip];
 
             bool isSelecting;
             MixedRealityPose spatialPointerPose;
@@ -152,6 +167,9 @@ public void UpdateController(WebXRInputSource controller)
                             CoreServices.InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, Interactions[i].MixedRealityInputAction, Interactions[i].PoseData);
                         }
                         break;
+                    case DeviceInputType.ThumbStick:
+                        HandDefinition?.UpdateCurrentTeleportPose(Interactions[i]);
+                        break;
                 }
             }
         }
diff --git a/SimpleWebXR-Demo/ProjectSettings/EditorSettings.asset b/SimpleWebXR-Demo/ProjectSettings/EditorSettings.asset
@@ -3,19 +3,33 @@
 --- !u!159 &1
 EditorSettings:
   m_ObjectHideFlags: 0
-  serializedVersion: 7
+  serializedVersion: 9
   m_ExternalVersionControlSupport: Visible Meta Files
   m_SerializationMode: 2
   m_LineEndingsForNewScripts: 2
   m_DefaultBehaviorMode: 0
+  m_PrefabRegularEnvironment: {fileID: 0}
+  m_PrefabUIEnvironment: {fileID: 0}
   m_SpritePackerMode: 0
   m_SpritePackerPaddingPower: 1
   m_EtcTextureCompressorBehavior: 1
   m_EtcTextureFastCompressor: 1
   m_EtcTextureNormalCompressor: 2
   m_EtcTextureBestCompressor: 4
-  m_ProjectGenerationIncludedExtensions: txt;xml;fnt;cd
+  m_ProjectGenerationIncludedExtensions: txt;xml;fnt;cd;asmref
   m_ProjectGenerationRootNamespace: 
-  m_UserGeneratedProjectSuffix: 
   m_CollabEditorSettings:
     inProgressEnabled: 1
+  m_EnableTextureStreamingInEditMode: 1
+  m_EnableTextureStreamingInPlayMode: 1
+  m_AsyncShaderCompilation: 1
+  m_EnterPlayModeOptionsEnabled: 0
+  m_EnterPlayModeOptions: 3
+  m_ShowLightmapResolutionOverlay: 1
+  m_UseLegacyProbeSampleCount: 1
+  m_AssetPipelineMode: 0
+  m_CacheServerMode: 0
+  m_CacheServerEndpoint: 
+  m_CacheServerNamespacePrefix: default
+  m_CacheServerEnableDownload: 1
+  m_CacheServerEnableUpload: 1
diff --git a/com.rufus31415.simplewebxr/Runtime/Plugins/WebGL/SimpleWebXR.jslib b/com.rufus31415.simplewebxr/Runtime/Plugins/WebGL/SimpleWebXR.jslib
@@ -85,20 +85,20 @@ mergeInto(LibraryManager.library, {
     _isArSupported = false;
 
     if (!navigator.xr) {
-      document.dispatchEvent(new CustomEvent("SimpleWebXRInitialized", {success: false}));
+      document.dispatchEvent(new CustomEvent("SimpleWebXRInitialized", { success: false }));
       return
     };
 
     // Check if WebXR immersive VR is supported (check immersive-vr before immersive-ar to make it work on Oculus Quest Browser)
     navigator.xr.isSessionSupported('immersive-vr').then(function (supported) {
       _isVrSupported = supported;
-      document.dispatchEvent(new CustomEvent("SimpleWebXRSessionSupported", {xrSessionMode:'immersive-vr', supported: supported}));
+      document.dispatchEvent(new CustomEvent("SimpleWebXRSessionSupported", { xrSessionMode: 'immersive-vr', supported: supported }));
     });
 
     // Check if WebXR immersive AR is supported
     navigator.xr.isSessionSupported('immersive-ar').then(function (supported) {
       _isArSupported = supported;
-      document.dispatchEvent(new CustomEvent("SimpleWebXRSessionSupported", {xrSessionMode:'immersive-ar', supported: supported}));
+      document.dispatchEvent(new CustomEvent("SimpleWebXRSessionSupported", { xrSessionMode: 'immersive-ar', supported: supported }));
     });
 
     // Initialize pointers to shared arrays that contains data (projection matrix, position, orientation, input sources)
@@ -126,7 +126,7 @@ mergeInto(LibraryManager.library, {
       // Share position
       var position = view.transform.position;
 
-      if(_firstFrame) _yOffset = position.y;
+      if (_firstFrame) _yOffset = position.y;
 
       _dataArray[floatStartId + 16] = position.x;
       _dataArray[floatStartId + 17] = position.y - _yOffset;
@@ -220,10 +220,10 @@ mergeInto(LibraryManager.library, {
           _byteArray[byteStartId + 3] = 0
         }
 
-        if(_dataArray[103+id]>0 && inputSource.gamepad.hapticActuators && inputSource.gamepad.hapticActuators.length>0){
+        if (_dataArray[103 + id] > 0 && inputSource.gamepad.hapticActuators && inputSource.gamepad.hapticActuators.length > 0) {
           // Trigger of haptic vibration pulse(intensity [0..1], duration in ms)
-          inputSource.gamepad.hapticActuators[0].pulse(_dataArray[101+id], _dataArray[103+id]);
-          _dataArray[103+id] = 0; // reset flag once it's done
+          inputSource.gamepad.hapticActuators[0].pulse(_dataArray[101 + id], _dataArray[103 + id]);
+          _dataArray[103 + id] = 0; // reset flag once it's done
         }
       }
       else {
@@ -253,31 +253,80 @@ mergeInto(LibraryManager.library, {
 
       // Hand detection
       // https://immersive-web.github.io/webxr-hand-input/#skeleton-joints-section
-      if (inputSource.hand && inputSource.hand.length == 25) {
-        _byteArray[46 + id] = 1; // hand supported
-        var delta = (_useLocalSpaceForInput ? 0 : _dataArray[100])
-        for (var j = 0; j < 25; j++) {
-          if (inputSource.hand[j] !== null) {
-            var joint = frame.getJointPose(inputSource.hand[j], _useLocalSpaceForInput ? _arSession.localSpace : _arSession.localFloorSpace);
-            if (joint !== null) {
-              var i = id * 200 + j * 8;
-              _handArray[i] = joint.transform.position.x;
-              _handArray[i + 1] = joint.transform.position.y - delta - _yOffset;
-              _handArray[i + 2] = joint.transform.position.z;
-              _handArray[i + 3] = joint.transform.orientation.x;
-              _handArray[i + 4] = joint.transform.orientation.y;
-              _handArray[i + 5] = joint.transform.orientation.z;
-              _handArray[i + 6] = joint.transform.orientation.w;
-              if (joint.radius !== null) {
-                _handArray[i + 7] = joint.radius;
+      if (inputSource.hand) {
+
+        // For browsers that support fillPoses
+        if (typeof frame.fillPoses === "function") {
+          _byteArray[46 + id] = 1; // hand supported
+
+          var delta = (_useLocalSpaceForInput ? 0 : _dataArray[100])
+
+          var refSpace = _useLocalSpaceForInput ? _arSession.localSpace : _arSession.localFloorSpace;
+
+          var radii = new Float32Array(25);
+          var poses = new Float32Array(16 * 25);
+
+          if (inputSource.hand.values) {
+            frame.fillPoses(inputSource.hand.values(), refSpace, poses);
+            frame.fillJointRadii(inputSource.hand.values(), radii);
+          } else {
+            frame.fillPoses(inputSource.hand, refSpace, poses);
+            frame.fillJointRadii(inputSource.hand, radii);
+          }
+
+          for (var j = 0; j < 25; j++) {
+            var jointIndex = j * 16;
+
+            var i = id * 200 + j * 8;
+            _handArray[i] = poses[jointIndex + 12];
+            _handArray[i + 1] = poses[jointIndex + 13] - delta - _yOffset;
+            _handArray[i + 2] = poses[jointIndex + 14];
+
+
+            _handArray[i + 7] = radii[j];
+
+            var quaternion = new Float32Array(4);
+
+            quaternion[3] = Math.sqrt(Math.max(0, 1 + poses[jointIndex + 0] + poses[jointIndex + 5] + poses[jointIndex + 10])) / 2;
+            quaternion[0] = Math.sqrt(Math.max(0, 1 + poses[jointIndex + 0] - poses[jointIndex + 5] - poses[jointIndex + 10])) / 2;
+            quaternion[1] = Math.sqrt(Math.max(0, 1 - poses[jointIndex + 0] + poses[jointIndex + 5] - poses[jointIndex + 10])) / 2;
+            quaternion[2] = Math.sqrt(Math.max(0, 1 - poses[jointIndex + 0] - poses[jointIndex + 5] + poses[jointIndex + 10])) / 2;
+            quaternion[0] *= Math.sign(quaternion[0] * (poses[jointIndex + 6] - poses[jointIndex + 9]));
+            quaternion[1] *= Math.sign(quaternion[1] * (poses[jointIndex + 8] - poses[jointIndex + 2]));
+            quaternion[2] *= Math.sign(quaternion[2] * (poses[jointIndex + 1] - poses[jointIndex + 4]));
+
+            _handArray[i + 3] = quaternion[0];
+            _handArray[i + 4] = quaternion[1];
+            _handArray[i + 5] = quaternion[2];
+            _handArray[i + 6] = quaternion[3];
+          }
+        }
+        else {
+          _byteArray[46 + id] = 1; // hand supported
+
+          for (var j = 0; j < 25; j++) {
+            if (inputSource.hand[j] !== null) {
+              var joint = frame.getJointPose(inputSource.hand[j], refSpace);
+              if (joint !== null) {
+                var i = id * 200 + j * 8;
+                _handArray[i] = joint.transform.position.x;
+                _handArray[i + 1] = joint.transform.position.y - delta - _yOffset;
+                _handArray[i + 2] = joint.transform.position.z;
+                _handArray[i + 3] = joint.transform.orientation.x;
+                _handArray[i + 4] = joint.transform.orientation.y;
+                _handArray[i + 5] = joint.transform.orientation.z;
+                _handArray[i + 6] = joint.transform.orientation.w;
+                if (joint.radius !== null) {
+                  _handArray[i + 7] = joint.radius;
+                }
+                else {
+                  _handArray[i + 7] = NaN;
+                }
               }
               else {
-                _handArray[i + 7] = NaN;
+                _byteArray[46 + id] = 0; // hand not fully supported
               }
             }
-            else {
-              _byteArray[46 + id] = 0; // hand not fully supported
-            }
           }
         }
       }
@@ -434,7 +483,7 @@ mergeInto(LibraryManager.library, {
       }
     }(GLctx.bindFramebuffer);
 
-    document.dispatchEvent(new CustomEvent("SimpleWebXRInitialized", {success: true}));
+    document.dispatchEvent(new CustomEvent("SimpleWebXRInitialized", { success: true }));
   },
 
   /****************************************************************************/
@@ -468,7 +517,7 @@ mergeInto(LibraryManager.library, {
       GLctx.ARSessionStarted = _isArSupported;
       session.isInSession = true; // add field in session to indicate that a session in running
 
-      document.dispatchEvent(new CustomEvent("SimpleWebXRSessionStarted", {session: session, GLctx: GLctx}));
+      document.dispatchEvent(new CustomEvent("SimpleWebXRSessionStarted", { session: session, GLctx: GLctx }));
 
       var glLayer = new XRWebGLLayer(session, GLctx);
 
@@ -531,7 +580,7 @@ mergeInto(LibraryManager.library, {
     _orientationInfo[0] = 0;
 
     _onDeviceOrientation = function (event) {
-      if(_orientationInfo[0] == 0)  document.dispatchEvent(new CustomEvent("SimpleWebXRDeviceOrientationStarted"));
+      if (_orientationInfo[0] == 0) document.dispatchEvent(new CustomEvent("SimpleWebXRDeviceOrientationStarted"));
 
       _orientationInfo[0] = 1;
       _orientationArray[0] = event.alpha;
