From eb24ee08b58362296cbcca2f647005bef50df319 Mon Sep 17 00:00:00 2001 From: BlueRose <378100977@qq.com> Date: Wed, 9 Aug 2023 17:02:24 +0800 Subject: [PATCH] vault backup: 2023-08-09 17:02:24 --- 01-Diary/周小结/未命名.md | 6 +- 02-Note/DAWA/AI/AIVirtualIdel动画方案.md | 193 ++++++++++++++++++++++- 2 files changed, 197 insertions(+), 2 deletions(-) diff --git a/01-Diary/周小结/未命名.md b/01-Diary/周小结/未命名.md index 72199de..ac3f515 100644 --- a/01-Diary/周小结/未命名.md +++ b/01-Diary/周小结/未命名.md @@ -1 +1,5 @@ -## 查看UEOSC & VMC4UE实现 \ No newline at end of file +# 存在问题 +1. 推流方案对延迟要求较高，如果通过AI服务器 => UE服务器 =>客户端的方式，会大大提高延迟。 + +# 工作内容 +1. 查看UEOSC & VMC4UE实现 \ No newline at end of file diff --git a/02-Note/DAWA/AI/AIVirtualIdel动画方案.md b/02-Note/DAWA/AI/AIVirtualIdel动画方案.md index 1cf8b1a..f023c56 100644 --- a/02-Note/DAWA/AI/AIVirtualIdel动画方案.md +++ b/02-Note/DAWA/AI/AIVirtualIdel动画方案.md @@ -209,4 +209,195 @@ https://docs.unrealengine.com/5.1/en-US/remote-control-for-unreal-engine/ ![[AnimNode]] # VMC APP代码参考 -- https://github.com/digital-standard/ThreeDPoseTracker \ No newline at end of file +- [VirtualMotionCaptureProtocol](https://github.com/sh-akira/VirtualMotionCaptureProtocol)提供了最基础的实现。 +- ~~[EasyVirtualMotionCaptureForUnity](https://github.com/gpsnmeajp/EasyVirtualMotionCaptureForUnity)~~ +- ThirdParts + - https://github.com/digital-standard/ThreeDPoseTracker + +## VirtualMotionCaptureProtocol +Message方式： +```c# +void Update() +{ + //モデルが更新されたときのみ読み込み + if (Model != null && OldModel != Model) + { + animator = Model.GetComponent(); + blendShapeProxy = Model.GetComponent(); + OldModel = Model; + } + + if (Model != null && animator != null && uClient != null) + { + //Root + var RootTransform = Model.transform; + if (RootTransform != null) + { + uClient.Send("/VMC/Ext/Root/Pos", + "root", + RootTransform.position.x, RootTransform.position.y, RootTransform.position.z, + RootTransform.rotation.x, RootTransform.rotation.y, RootTransform.rotation.z, RootTransform.rotation.w); + } + + //Bones + foreach (HumanBodyBones bone in Enum.GetValues(typeof(HumanBodyBones))) + { + if (bone != HumanBodyBones.LastBone) + { + var Transform = animator.GetBoneTransform(bone); + if (Transform != null) + { + uClient.Send("/VMC/Ext/Bone/Pos", + bone.ToString(), + Transform.localPosition.x, Transform.localPosition.y, Transform.localPosition.z, + Transform.localRotation.x, Transform.localRotation.y, Transform.localRotation.z, Transform.localRotation.w); + } + } + } + + //ボーン位置を仮想トラッカーとして送信 + SendBoneTransformForTracker(HumanBodyBones.Head, "Head"); + SendBoneTransformForTracker(HumanBodyBones.Spine, "Spine"); + SendBoneTransformForTracker(HumanBodyBones.LeftHand, "LeftHand"); + SendBoneTransformForTracker(HumanBodyBones.RightHand, "RightHand"); + SendBoneTransformForTracker(HumanBodyBones.LeftFoot, "LeftFoot"); + SendBoneTransformForTracker(HumanBodyBones.RightFoot, "RightFoot"); + + //BlendShape + if (blendShapeProxy != null) + { + foreach (var b in blendShapeProxy.GetValues()) + { + uClient.Send("/VMC/Ext/Blend/Val", + b.Key.ToString(), + (float)b.Value + ); + } + uClient.Send("/VMC/Ext/Blend/Apply"); + } + + //Available + uClient.Send("/VMC/Ext/OK", 1); + } + else + { + uClient.Send("/VMC/Ext/OK", 0); + } + uClient.Send("/VMC/Ext/T", Time.time); + + //Load request + uClient.Send("/VMC/Ext/VRM", filepath, ""); +} + +void SendBoneTransformForTracker(HumanBodyBones bone, string DeviceSerial) +{ + var DeviceTransform = animator.GetBoneTransform(bone); + if (DeviceTransform != null) { + uClient.Send("/VMC/Ext/Tra/Pos", + (string)DeviceSerial, + (float)DeviceTransform.position.x, + (float)DeviceTransform.position.y, + (float)DeviceTransform.position.z, + (float)DeviceTransform.rotation.x, + (float)DeviceTransform.rotation.y, + (float)DeviceTransform.rotation.z, + (float)DeviceTransform.rotation.w); + } +} +``` + +Bundle将数据打包成一个Bundle，创建Bundle时会填入一个时间戳。之后 +```c# +foreach (HumanBodyBones bone in Enum.GetValues(typeof(HumanBodyBones))) +{ +... + boneBundle.Add(new Message("/VMC/Ext/Bone/Pos", +                            bone.ToString(), +                            Transform.localPosition.x, Transform.localPosition.y, Transform.localPosition.z, +                            Transform.localRotation.x, Transform.localRotation.y, Transform.localRotation.z, Transform.localRotation.w)); +... +} +``` +Bundle方式： +```c# + void Update() + { + //Only model updated + if (Model != null && OldModel != Model) + { + animator = Model.GetComponent(); + blendShapeProxy = Model.GetComponent(); + OldModel = Model; + } + + if (Model != null && animator != null && uClient != null) + { + //Root + var RootTransform = Model.transform; + if (RootTransform != null) + { + uClient.Send("/VMC/Ext/Root/Pos", + "root", + RootTransform.position.x, RootTransform.position.y, RootTransform.position.z, + RootTransform.rotation.x, RootTransform.rotation.y, RootTransform.rotation.z, RootTransform.rotation.w); + } + + //Bones + var boneBundle = new Bundle(Timestamp.Now); + foreach (HumanBodyBones bone in Enum.GetValues(typeof(HumanBodyBones))) + { + if (bone != HumanBodyBones.LastBone) + { + var Transform = animator.GetBoneTransform(bone); + if (Transform != null) + { + boneBundle.Add(new Message("/VMC/Ext/Bone/Pos", + bone.ToString(), + Transform.localPosition.x, Transform.localPosition.y, Transform.localPosition.z, + Transform.localRotation.x, Transform.localRotation.y, Transform.localRotation.z, Transform.localRotation.w)); + } + } + } + uClient.Send(boneBundle); + + //Virtual Tracker send from bone position + var trackerBundle = new Bundle(Timestamp.Now); + SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.Head, "Head"); + SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.Spine, "Spine"); + SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.LeftHand, "LeftHand"); + SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.RightHand, "RightHand"); + SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.LeftFoot, "LeftFoot"); + SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.RightFoot, "RightFoot"); + uClient.Send(trackerBundle); + + //BlendShape + if (blendShapeProxy != null) + { + var blendShapeBundle = new Bundle(Timestamp.Now); + + foreach (var b in blendShapeProxy.GetValues()) + { + blendShapeBundle.Add(new Message("/VMC/Ext/Blend/Val", + b.Key.ToString(), + (float)b.Value + )); + } + blendShapeBundle.Add(new Message("/VMC/Ext/Blend/Apply")); + uClient.Send(blendShapeBundle); + } + + //Available + uClient.Send("/VMC/Ext/OK", 1); + } + else + { + uClient.Send("/VMC/Ext/OK", 0); + } + uClient.Send("/VMC/Ext/T", Time.time); + + //Load request + uClient.Send("/VMC/Ext/VRM", vrmfilepath, ""); + + } + +``` \ No newline at end of file