vault backup: 2023-08-09 17:02:24
This commit is contained in:
parent
60bedd8df5
commit
eb24ee08b5
@ -1 +1,5 @@
|
|||||||
## 查看UEOSC & VMC4UE实现
|
# 存在问题
|
||||||
|
1. 推流方案对延迟要求较高，如果通过AI服务器 => UE服务器 =>客户端的方式，会大大提高延迟。
|
||||||
|
|
||||||
|
# 工作内容
|
||||||
|
1. 查看UEOSC & VMC4UE实现
|
||||||
@ -209,4 +209,195 @@ https://docs.unrealengine.com/5.1/en-US/remote-control-for-unreal-engine/
|
|||||||
![[AnimNode]]
|
![[AnimNode]]
|
||||||
|
|
||||||
# VMC APP代码参考
|
# VMC APP代码参考
|
||||||
- https://github.com/digital-standard/ThreeDPoseTracker
|
- [VirtualMotionCaptureProtocol](https://github.com/sh-akira/VirtualMotionCaptureProtocol)提供了最基础的实现。
|
||||||
|
- ~~[EasyVirtualMotionCaptureForUnity](https://github.com/gpsnmeajp/EasyVirtualMotionCaptureForUnity)~~
|
||||||
|
- ThirdParts
|
||||||
|
- https://github.com/digital-standard/ThreeDPoseTracker
|
||||||
|
|
||||||
|
## VirtualMotionCaptureProtocol
|
||||||
|
Message方式：
|
||||||
|
```c#
|
||||||
|
void Update()
|
||||||
|
{
|
||||||
|
//モデルが更新されたときのみ読み込み
|
||||||
|
if (Model != null && OldModel != Model)
|
||||||
|
{
|
||||||
|
animator = Model.GetComponent<Animator>();
|
||||||
|
blendShapeProxy = Model.GetComponent<VRMBlendShapeProxy>();
|
||||||
|
OldModel = Model;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Model != null && animator != null && uClient != null)
|
||||||
|
{
|
||||||
|
//Root
|
||||||
|
var RootTransform = Model.transform;
|
||||||
|
if (RootTransform != null)
|
||||||
|
{
|
||||||
|
uClient.Send("/VMC/Ext/Root/Pos",
|
||||||
|
"root",
|
||||||
|
RootTransform.position.x, RootTransform.position.y, RootTransform.position.z,
|
||||||
|
RootTransform.rotation.x, RootTransform.rotation.y, RootTransform.rotation.z, RootTransform.rotation.w);
|
||||||
|
}
|
||||||
|
|
||||||
|
//Bones
|
||||||
|
foreach (HumanBodyBones bone in Enum.GetValues(typeof(HumanBodyBones)))
|
||||||
|
{
|
||||||
|
if (bone != HumanBodyBones.LastBone)
|
||||||
|
{
|
||||||
|
var Transform = animator.GetBoneTransform(bone);
|
||||||
|
if (Transform != null)
|
||||||
|
{
|
||||||
|
uClient.Send("/VMC/Ext/Bone/Pos",
|
||||||
|
bone.ToString(),
|
||||||
|
Transform.localPosition.x, Transform.localPosition.y, Transform.localPosition.z,
|
||||||
|
Transform.localRotation.x, Transform.localRotation.y, Transform.localRotation.z, Transform.localRotation.w);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//ボーン位置を仮想トラッカーとして送信
|
||||||
|
SendBoneTransformForTracker(HumanBodyBones.Head, "Head");
|
||||||
|
SendBoneTransformForTracker(HumanBodyBones.Spine, "Spine");
|
||||||
|
SendBoneTransformForTracker(HumanBodyBones.LeftHand, "LeftHand");
|
||||||
|
SendBoneTransformForTracker(HumanBodyBones.RightHand, "RightHand");
|
||||||
|
SendBoneTransformForTracker(HumanBodyBones.LeftFoot, "LeftFoot");
|
||||||
|
SendBoneTransformForTracker(HumanBodyBones.RightFoot, "RightFoot");
|
||||||
|
|
||||||
|
//BlendShape
|
||||||
|
if (blendShapeProxy != null)
|
||||||
|
{
|
||||||
|
foreach (var b in blendShapeProxy.GetValues())
|
||||||
|
{
|
||||||
|
uClient.Send("/VMC/Ext/Blend/Val",
|
||||||
|
b.Key.ToString(),
|
||||||
|
(float)b.Value
|
||||||
|
);
|
||||||
|
}
|
||||||
|
uClient.Send("/VMC/Ext/Blend/Apply");
|
||||||
|
}
|
||||||
|
|
||||||
|
//Available
|
||||||
|
uClient.Send("/VMC/Ext/OK", 1);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
uClient.Send("/VMC/Ext/OK", 0);
|
||||||
|
}
|
||||||
|
uClient.Send("/VMC/Ext/T", Time.time);
|
||||||
|
|
||||||
|
//Load request
|
||||||
|
uClient.Send("/VMC/Ext/VRM", filepath, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
void SendBoneTransformForTracker(HumanBodyBones bone, string DeviceSerial)
|
||||||
|
{
|
||||||
|
var DeviceTransform = animator.GetBoneTransform(bone);
|
||||||
|
if (DeviceTransform != null) {
|
||||||
|
uClient.Send("/VMC/Ext/Tra/Pos",
|
||||||
|
(string)DeviceSerial,
|
||||||
|
(float)DeviceTransform.position.x,
|
||||||
|
(float)DeviceTransform.position.y,
|
||||||
|
(float)DeviceTransform.position.z,
|
||||||
|
(float)DeviceTransform.rotation.x,
|
||||||
|
(float)DeviceTransform.rotation.y,
|
||||||
|
(float)DeviceTransform.rotation.z,
|
||||||
|
(float)DeviceTransform.rotation.w);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Bundle将数据打包成一个Bundle，创建Bundle时会填入一个时间戳。之后
|
||||||
|
```c#
|
||||||
|
foreach (HumanBodyBones bone in Enum.GetValues(typeof(HumanBodyBones)))
|
||||||
|
{
|
||||||
|
...
|
||||||
|
boneBundle.Add(new Message("/VMC/Ext/Bone/Pos",
|
||||||
|
bone.ToString(),
|
||||||
|
Transform.localPosition.x, Transform.localPosition.y, Transform.localPosition.z,
|
||||||
|
Transform.localRotation.x, Transform.localRotation.y, Transform.localRotation.z, Transform.localRotation.w));
|
||||||
|
...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
Bundle方式：
|
||||||
|
```c#
|
||||||
|
void Update()
|
||||||
|
{
|
||||||
|
//Only model updated
|
||||||
|
if (Model != null && OldModel != Model)
|
||||||
|
{
|
||||||
|
animator = Model.GetComponent<Animator>();
|
||||||
|
blendShapeProxy = Model.GetComponent<VRMBlendShapeProxy>();
|
||||||
|
OldModel = Model;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Model != null && animator != null && uClient != null)
|
||||||
|
{
|
||||||
|
//Root
|
||||||
|
var RootTransform = Model.transform;
|
||||||
|
if (RootTransform != null)
|
||||||
|
{
|
||||||
|
uClient.Send("/VMC/Ext/Root/Pos",
|
||||||
|
"root",
|
||||||
|
RootTransform.position.x, RootTransform.position.y, RootTransform.position.z,
|
||||||
|
RootTransform.rotation.x, RootTransform.rotation.y, RootTransform.rotation.z, RootTransform.rotation.w);
|
||||||
|
}
|
||||||
|
|
||||||
|
//Bones
|
||||||
|
var boneBundle = new Bundle(Timestamp.Now);
|
||||||
|
foreach (HumanBodyBones bone in Enum.GetValues(typeof(HumanBodyBones)))
|
||||||
|
{
|
||||||
|
if (bone != HumanBodyBones.LastBone)
|
||||||
|
{
|
||||||
|
var Transform = animator.GetBoneTransform(bone);
|
||||||
|
if (Transform != null)
|
||||||
|
{
|
||||||
|
boneBundle.Add(new Message("/VMC/Ext/Bone/Pos",
|
||||||
|
bone.ToString(),
|
||||||
|
Transform.localPosition.x, Transform.localPosition.y, Transform.localPosition.z,
|
||||||
|
Transform.localRotation.x, Transform.localRotation.y, Transform.localRotation.z, Transform.localRotation.w));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
uClient.Send(boneBundle);
|
||||||
|
|
||||||
|
//Virtual Tracker send from bone position
|
||||||
|
var trackerBundle = new Bundle(Timestamp.Now);
|
||||||
|
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.Head, "Head");
|
||||||
|
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.Spine, "Spine");
|
||||||
|
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.LeftHand, "LeftHand");
|
||||||
|
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.RightHand, "RightHand");
|
||||||
|
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.LeftFoot, "LeftFoot");
|
||||||
|
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.RightFoot, "RightFoot");
|
||||||
|
uClient.Send(trackerBundle);
|
||||||
|
|
||||||
|
//BlendShape
|
||||||
|
if (blendShapeProxy != null)
|
||||||
|
{
|
||||||
|
var blendShapeBundle = new Bundle(Timestamp.Now);
|
||||||
|
|
||||||
|
foreach (var b in blendShapeProxy.GetValues())
|
||||||
|
{
|
||||||
|
blendShapeBundle.Add(new Message("/VMC/Ext/Blend/Val",
|
||||||
|
b.Key.ToString(),
|
||||||
|
(float)b.Value
|
||||||
|
));
|
||||||
|
}
|
||||||
|
blendShapeBundle.Add(new Message("/VMC/Ext/Blend/Apply"));
|
||||||
|
uClient.Send(blendShapeBundle);
|
||||||
|
}
|
||||||
|
|
||||||
|
//Available
|
||||||
|
uClient.Send("/VMC/Ext/OK", 1);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
uClient.Send("/VMC/Ext/OK", 0);
|
||||||
|
}
|
||||||
|
uClient.Send("/VMC/Ext/T", Time.time);
|
||||||
|
|
||||||
|
//Load request
|
||||||
|
uClient.Send("/VMC/Ext/VRM", vrmfilepath, "");
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
Loading…
x
Reference in New Issue
Block a user