vault backup: 2023-08-09 17:02:24
This commit is contained in:
parent
60bedd8df5
commit
eb24ee08b5
@ -1 +1,5 @@
|
||||
## 查看UEOSC & VMC4UE实现
|
||||
# 存在问题
|
||||
1. 推流方案对延迟要求较高，如果通过AI服务器 => UE服务器 =>客户端的方式，会大大提高延迟。
|
||||
|
||||
# 工作内容
|
||||
1. 查看UEOSC & VMC4UE实现
|
||||
@ -209,4 +209,195 @@ https://docs.unrealengine.com/5.1/en-US/remote-control-for-unreal-engine/
|
||||
![[AnimNode]]
|
||||
|
||||
# VMC APP代码参考
|
||||
- [VirtualMotionCaptureProtocol](https://github.com/sh-akira/VirtualMotionCaptureProtocol)提供了最基础的实现。
|
||||
- ~~[EasyVirtualMotionCaptureForUnity](https://github.com/gpsnmeajp/EasyVirtualMotionCaptureForUnity)~~
|
||||
- ThirdParts
|
||||
- https://github.com/digital-standard/ThreeDPoseTracker
|
||||
|
||||
## VirtualMotionCaptureProtocol
|
||||
Message方式：
|
||||
```c#
|
||||
void Update()
|
||||
{
|
||||
//モデルが更新されたときのみ読み込み
|
||||
if (Model != null && OldModel != Model)
|
||||
{
|
||||
animator = Model.GetComponent<Animator>();
|
||||
blendShapeProxy = Model.GetComponent<VRMBlendShapeProxy>();
|
||||
OldModel = Model;
|
||||
}
|
||||
|
||||
if (Model != null && animator != null && uClient != null)
|
||||
{
|
||||
//Root
|
||||
var RootTransform = Model.transform;
|
||||
if (RootTransform != null)
|
||||
{
|
||||
uClient.Send("/VMC/Ext/Root/Pos",
|
||||
"root",
|
||||
RootTransform.position.x, RootTransform.position.y, RootTransform.position.z,
|
||||
RootTransform.rotation.x, RootTransform.rotation.y, RootTransform.rotation.z, RootTransform.rotation.w);
|
||||
}
|
||||
|
||||
//Bones
|
||||
foreach (HumanBodyBones bone in Enum.GetValues(typeof(HumanBodyBones)))
|
||||
{
|
||||
if (bone != HumanBodyBones.LastBone)
|
||||
{
|
||||
var Transform = animator.GetBoneTransform(bone);
|
||||
if (Transform != null)
|
||||
{
|
||||
uClient.Send("/VMC/Ext/Bone/Pos",
|
||||
bone.ToString(),
|
||||
Transform.localPosition.x, Transform.localPosition.y, Transform.localPosition.z,
|
||||
Transform.localRotation.x, Transform.localRotation.y, Transform.localRotation.z, Transform.localRotation.w);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//ボーン位置を仮想トラッカーとして送信
|
||||
SendBoneTransformForTracker(HumanBodyBones.Head, "Head");
|
||||
SendBoneTransformForTracker(HumanBodyBones.Spine, "Spine");
|
||||
SendBoneTransformForTracker(HumanBodyBones.LeftHand, "LeftHand");
|
||||
SendBoneTransformForTracker(HumanBodyBones.RightHand, "RightHand");
|
||||
SendBoneTransformForTracker(HumanBodyBones.LeftFoot, "LeftFoot");
|
||||
SendBoneTransformForTracker(HumanBodyBones.RightFoot, "RightFoot");
|
||||
|
||||
//BlendShape
|
||||
if (blendShapeProxy != null)
|
||||
{
|
||||
foreach (var b in blendShapeProxy.GetValues())
|
||||
{
|
||||
uClient.Send("/VMC/Ext/Blend/Val",
|
||||
b.Key.ToString(),
|
||||
(float)b.Value
|
||||
);
|
||||
}
|
||||
uClient.Send("/VMC/Ext/Blend/Apply");
|
||||
}
|
||||
|
||||
//Available
|
||||
uClient.Send("/VMC/Ext/OK", 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
uClient.Send("/VMC/Ext/OK", 0);
|
||||
}
|
||||
uClient.Send("/VMC/Ext/T", Time.time);
|
||||
|
||||
//Load request
|
||||
uClient.Send("/VMC/Ext/VRM", filepath, "");
|
||||
}
|
||||
|
||||
void SendBoneTransformForTracker(HumanBodyBones bone, string DeviceSerial)
|
||||
{
|
||||
var DeviceTransform = animator.GetBoneTransform(bone);
|
||||
if (DeviceTransform != null) {
|
||||
uClient.Send("/VMC/Ext/Tra/Pos",
|
||||
(string)DeviceSerial,
|
||||
(float)DeviceTransform.position.x,
|
||||
(float)DeviceTransform.position.y,
|
||||
(float)DeviceTransform.position.z,
|
||||
(float)DeviceTransform.rotation.x,
|
||||
(float)DeviceTransform.rotation.y,
|
||||
(float)DeviceTransform.rotation.z,
|
||||
(float)DeviceTransform.rotation.w);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Bundle将数据打包成一个Bundle，创建Bundle时会填入一个时间戳。之后
|
||||
```c#
|
||||
foreach (HumanBodyBones bone in Enum.GetValues(typeof(HumanBodyBones)))
|
||||
{
|
||||
...
|
||||
boneBundle.Add(new Message("/VMC/Ext/Bone/Pos",
|
||||
bone.ToString(),
|
||||
Transform.localPosition.x, Transform.localPosition.y, Transform.localPosition.z,
|
||||
Transform.localRotation.x, Transform.localRotation.y, Transform.localRotation.z, Transform.localRotation.w));
|
||||
...
|
||||
}
|
||||
```
|
||||
Bundle方式：
|
||||
```c#
|
||||
void Update()
|
||||
{
|
||||
//Only model updated
|
||||
if (Model != null && OldModel != Model)
|
||||
{
|
||||
animator = Model.GetComponent<Animator>();
|
||||
blendShapeProxy = Model.GetComponent<VRMBlendShapeProxy>();
|
||||
OldModel = Model;
|
||||
}
|
||||
|
||||
if (Model != null && animator != null && uClient != null)
|
||||
{
|
||||
//Root
|
||||
var RootTransform = Model.transform;
|
||||
if (RootTransform != null)
|
||||
{
|
||||
uClient.Send("/VMC/Ext/Root/Pos",
|
||||
"root",
|
||||
RootTransform.position.x, RootTransform.position.y, RootTransform.position.z,
|
||||
RootTransform.rotation.x, RootTransform.rotation.y, RootTransform.rotation.z, RootTransform.rotation.w);
|
||||
}
|
||||
|
||||
//Bones
|
||||
var boneBundle = new Bundle(Timestamp.Now);
|
||||
foreach (HumanBodyBones bone in Enum.GetValues(typeof(HumanBodyBones)))
|
||||
{
|
||||
if (bone != HumanBodyBones.LastBone)
|
||||
{
|
||||
var Transform = animator.GetBoneTransform(bone);
|
||||
if (Transform != null)
|
||||
{
|
||||
boneBundle.Add(new Message("/VMC/Ext/Bone/Pos",
|
||||
bone.ToString(),
|
||||
Transform.localPosition.x, Transform.localPosition.y, Transform.localPosition.z,
|
||||
Transform.localRotation.x, Transform.localRotation.y, Transform.localRotation.z, Transform.localRotation.w));
|
||||
}
|
||||
}
|
||||
}
|
||||
uClient.Send(boneBundle);
|
||||
|
||||
//Virtual Tracker send from bone position
|
||||
var trackerBundle = new Bundle(Timestamp.Now);
|
||||
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.Head, "Head");
|
||||
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.Spine, "Spine");
|
||||
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.LeftHand, "LeftHand");
|
||||
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.RightHand, "RightHand");
|
||||
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.LeftFoot, "LeftFoot");
|
||||
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.RightFoot, "RightFoot");
|
||||
uClient.Send(trackerBundle);
|
||||
|
||||
//BlendShape
|
||||
if (blendShapeProxy != null)
|
||||
{
|
||||
var blendShapeBundle = new Bundle(Timestamp.Now);
|
||||
|
||||
foreach (var b in blendShapeProxy.GetValues())
|
||||
{
|
||||
blendShapeBundle.Add(new Message("/VMC/Ext/Blend/Val",
|
||||
b.Key.ToString(),
|
||||
(float)b.Value
|
||||
));
|
||||
}
|
||||
blendShapeBundle.Add(new Message("/VMC/Ext/Blend/Apply"));
|
||||
uClient.Send(blendShapeBundle);
|
||||
}
|
||||
|
||||
//Available
|
||||
uClient.Send("/VMC/Ext/OK", 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
uClient.Send("/VMC/Ext/OK", 0);
|
||||
}
|
||||
uClient.Send("/VMC/Ext/T", Time.time);
|
||||
|
||||
//Load request
|
||||
uClient.Send("/VMC/Ext/VRM", vrmfilepath, "");
|
||||
|
||||
}
|
||||
|
||||
```
|
||||
Loading…
x
Reference in New Issue
Block a user