BlueRose/BlueRoseNote
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

vault backup: 2023-08-09 17:02:24

Browse Source
This commit is contained in:
BlueRose 2023-08-09 17:02:24 +08:00
parent 60bedd8df5
commit eb24ee08b5
2 changed files with 197 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
01-Diary/周小结/未命名.md
View File

@ -1 +1,5 @@
## 查看UEOSC & VMC4UE实现
# 存在问题
1. 推流方案对延迟要求较高，如果通过AI服务器 => UE服务器 =>客户端的方式，会大大提高延迟。
# 工作内容
1. 查看UEOSC & VMC4UE实现

193
02-Note/DAWA/AI/AIVirtualIdel动画方案.md
View File

@ -209,4 +209,195 @@ https://docs.unrealengine.com/5.1/en-US/remote-control-for-unreal-engine/
![[AnimNode]]
# VMC APP代码参考
- https://github.com/digital-standard/ThreeDPoseTracker
- [VirtualMotionCaptureProtocol](https://github.com/sh-akira/VirtualMotionCaptureProtocol)提供了最基础的实现。
- ~~[EasyVirtualMotionCaptureForUnity](https://github.com/gpsnmeajp/EasyVirtualMotionCaptureForUnity)~~
- ThirdParts
- https://github.com/digital-standard/ThreeDPoseTracker
## VirtualMotionCaptureProtocol
Message方式：
```c#
void Update()
{
//モデルが更新されたときのみ読み込み
if (Model != null && OldModel != Model)
{
animator = Model.GetComponent<Animator>();
blendShapeProxy = Model.GetComponent<VRMBlendShapeProxy>();
OldModel = Model;
}
if (Model != null && animator != null && uClient != null)
{
//Root
var RootTransform = Model.transform;
if (RootTransform != null)
{
uClient.Send("/VMC/Ext/Root/Pos",
"root",
RootTransform.position.x, RootTransform.position.y, RootTransform.position.z,
RootTransform.rotation.x, RootTransform.rotation.y, RootTransform.rotation.z, RootTransform.rotation.w);
}
//Bones
foreach (HumanBodyBones bone in Enum.GetValues(typeof(HumanBodyBones)))
{
if (bone != HumanBodyBones.LastBone)
{
var Transform = animator.GetBoneTransform(bone);
if (Transform != null)
{
uClient.Send("/VMC/Ext/Bone/Pos",
bone.ToString(),
Transform.localPosition.x, Transform.localPosition.y, Transform.localPosition.z,
Transform.localRotation.x, Transform.localRotation.y, Transform.localRotation.z, Transform.localRotation.w);
}
}
}
//ボーン位置を仮想トラッカーとして送信
SendBoneTransformForTracker(HumanBodyBones.Head, "Head");
SendBoneTransformForTracker(HumanBodyBones.Spine, "Spine");
SendBoneTransformForTracker(HumanBodyBones.LeftHand, "LeftHand");
SendBoneTransformForTracker(HumanBodyBones.RightHand, "RightHand");
SendBoneTransformForTracker(HumanBodyBones.LeftFoot, "LeftFoot");
SendBoneTransformForTracker(HumanBodyBones.RightFoot, "RightFoot");
//BlendShape
if (blendShapeProxy != null)
{
foreach (var b in blendShapeProxy.GetValues())
{
uClient.Send("/VMC/Ext/Blend/Val",
b.Key.ToString(),
(float)b.Value
);
}
uClient.Send("/VMC/Ext/Blend/Apply");
}
//Available
uClient.Send("/VMC/Ext/OK", 1);
}
else
{
uClient.Send("/VMC/Ext/OK", 0);
}
uClient.Send("/VMC/Ext/T", Time.time);
//Load request
uClient.Send("/VMC/Ext/VRM", filepath, "");
}
void SendBoneTransformForTracker(HumanBodyBones bone, string DeviceSerial)
{
var DeviceTransform = animator.GetBoneTransform(bone);
if (DeviceTransform != null) {
uClient.Send("/VMC/Ext/Tra/Pos",
(string)DeviceSerial,
(float)DeviceTransform.position.x,
(float)DeviceTransform.position.y,
(float)DeviceTransform.position.z,
(float)DeviceTransform.rotation.x,
(float)DeviceTransform.rotation.y,
(float)DeviceTransform.rotation.z,
(float)DeviceTransform.rotation.w);
}
}
```
Bundle将数据打包成一个Bundle，创建Bundle时会填入一个时间戳。之后
```c#
foreach (HumanBodyBones bone in Enum.GetValues(typeof(HumanBodyBones)))
{
...
 boneBundle.Add(new Message("/VMC/Ext/Bone/Pos",
                            bone.ToString(),
                            Transform.localPosition.x, Transform.localPosition.y, Transform.localPosition.z,
                            Transform.localRotation.x, Transform.localRotation.y, Transform.localRotation.z, Transform.localRotation.w));
...
}
```
Bundle方式：
```c#
void Update()
{
//Only model updated
if (Model != null && OldModel != Model)
{
animator = Model.GetComponent<Animator>();
blendShapeProxy = Model.GetComponent<VRMBlendShapeProxy>();
OldModel = Model;
}
if (Model != null && animator != null && uClient != null)
{
//Root
var RootTransform = Model.transform;
if (RootTransform != null)
{
uClient.Send("/VMC/Ext/Root/Pos",
"root",
RootTransform.position.x, RootTransform.position.y, RootTransform.position.z,
RootTransform.rotation.x, RootTransform.rotation.y, RootTransform.rotation.z, RootTransform.rotation.w);
}
//Bones
var boneBundle = new Bundle(Timestamp.Now);
foreach (HumanBodyBones bone in Enum.GetValues(typeof(HumanBodyBones)))
{
if (bone != HumanBodyBones.LastBone)
{
var Transform = animator.GetBoneTransform(bone);
if (Transform != null)
{
boneBundle.Add(new Message("/VMC/Ext/Bone/Pos",
bone.ToString(),
Transform.localPosition.x, Transform.localPosition.y, Transform.localPosition.z,
Transform.localRotation.x, Transform.localRotation.y, Transform.localRotation.z, Transform.localRotation.w));
}
}
}
uClient.Send(boneBundle);
//Virtual Tracker send from bone position
var trackerBundle = new Bundle(Timestamp.Now);
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.Head, "Head");
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.Spine, "Spine");
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.LeftHand, "LeftHand");
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.RightHand, "RightHand");
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.LeftFoot, "LeftFoot");
SendBoneTransformForTracker(ref trackerBundle, HumanBodyBones.RightFoot, "RightFoot");
uClient.Send(trackerBundle);
//BlendShape
if (blendShapeProxy != null)
{
var blendShapeBundle = new Bundle(Timestamp.Now);
foreach (var b in blendShapeProxy.GetValues())
{
blendShapeBundle.Add(new Message("/VMC/Ext/Blend/Val",
b.Key.ToString(),
(float)b.Value
));
}
blendShapeBundle.Add(new Message("/VMC/Ext/Blend/Apply"));
uClient.Send(blendShapeBundle);
}
//Available
uClient.Send("/VMC/Ext/OK", 1);
}
else
{
uClient.Send("/VMC/Ext/OK", 0);
}
uClient.Send("/VMC/Ext/T", Time.time);
//Load request
uClient.Send("/VMC/Ext/VRM", vrmfilepath, "");
}
```