From b24c6d77ae381a25e2af7634301dd6aa855ed41f Mon Sep 17 00:00:00 2001 From: bd_ Date: Sun, 6 Nov 2022 18:44:05 -0800 Subject: [PATCH] FPV: Retarget meshes to avoid polygons spliced between Head and retargeted Head --- .../Editor/FirstPersonVisibleMeshProcessor.cs | 280 ++++++++++++++++++ .../FirstPersonVisibleMeshProcessor.cs.meta | 3 + .../Editor/FirstPersonVisibleProcessor.cs | 43 ++- 3 files changed, 319 insertions(+), 7 deletions(-) create mode 100644 Packages/net.fushizen.modular-avatar/Editor/FirstPersonVisibleMeshProcessor.cs create mode 100644 Packages/net.fushizen.modular-avatar/Editor/FirstPersonVisibleMeshProcessor.cs.meta diff --git a/Packages/net.fushizen.modular-avatar/Editor/FirstPersonVisibleMeshProcessor.cs b/Packages/net.fushizen.modular-avatar/Editor/FirstPersonVisibleMeshProcessor.cs new file mode 100644 index 00000000..07daaf90 --- /dev/null +++ b/Packages/net.fushizen.modular-avatar/Editor/FirstPersonVisibleMeshProcessor.cs @@ -0,0 +1,280 @@ +using System.Collections.Generic; +using System.Linq; +using Unity.Collections; +using UnityEditor; +using UnityEngine; + +namespace net.fushizen.modular_avatar.core.editor +{ + /// + /// Many avatars have hair meshes which are partially painted to the Head bone, and partially to a bone we might + /// want to make visible in the first-person view. To handle this case, we must retarget those meshes to use our + /// proxy head bone instead. + /// + internal class FirstPersonVisibleMeshProcessor + { + private SkinnedMeshRenderer _renderer; + private HashSet _visibleBones; + private Transform _proxyHead; + + public FirstPersonVisibleMeshProcessor( + SkinnedMeshRenderer renderer, + HashSet visibleBones, + Transform proxyHead + ) + { + _renderer = renderer; + _visibleBones = visibleBones; + _proxyHead = proxyHead; + } + + public bool NeedsRetargeting() + { + return _renderer.bones.Any(_visibleBones.Contains) && _renderer.bones.Any(b => !_visibleBones.Contains(b)); + } + + public void Retarget() + { + if (!NeedsRetargeting()) return; + + bool anyVisible = false; + var originalMesh = _renderer.sharedMesh; + var newMesh = Object.Instantiate(originalMesh); + AssetDatabase.CreateAsset(newMesh, Util.GenerateAssetPath()); + + // Identify the manifolds which need to be retargeted. Generally, we define a manifold as the maximal set of + // points which are connected by primitives. If this manifold contains both visible and retargeted bones, + // we need to retarget any retargeted bones in there. + // + // To do this, we use my favorite algorithm: The Union-Find set algorithm! This is one of my favorite + // algorithms/data-structures and I'm very happy to finally have a reason to apply it. Yay! + var vertexCount = originalMesh.vertexCount; + ManifoldNode[] nodes = new ManifoldNode[vertexCount]; + var boneWeights = originalMesh.GetAllBoneWeights(); + var bonesPerVertex = originalMesh.GetBonesPerVertex(); + + AnalyzeManifolds(); + // Bail out early if the bones are unused + if (!anyVisible) return; +/* + var vcol = new List(); + for (int v = 0; v < vertexCount; v++) + { + var n = nodes[v].Find(); + //vcol.Add(new Color((byte) (nodes[v].HasRetargetedBone ? 1 : 0), (byte) (nodes[v].HasVisibleBone ? 1 : 0), nodes[v].b, 1)); + vcol.Add(new Color(n.HasRetargetedBone ? 1 : 0, n.HasVisibleBone ? 1 : 0, 0, 1)); + } +*/ + // Now construct a new bone weight array + var bindposes = new List(); + originalMesh.GetBindposes(bindposes); + + var bones = new List(_renderer.bones); + var proxyIndices = new Dictionary(); + + var newWeights = new List(); + + int src_w_base = 0; + for (int v = 0; v < vertexCount; v++) + { + var weightPerVertex = bonesPerVertex[v]; + + bool remapManifold = nodes[v].HasRetargetedBone && nodes[v].HasVisibleBone; + + for (int w = 0; w < weightPerVertex; w++) + { + var weight = boneWeights[src_w_base + w]; + var bone = _renderer.bones[weight.boneIndex]; + + // Check for broken bone bindings, and if so just copy over. + // Also just copy over if the manifold doesn't need adjustment. + if (bone == null || !remapManifold || _visibleBones.Contains(bone)) + { + newWeights.Add(weight); + } + else + { + // This bone needs to be remapped, so do the thing. + int newIndex = RemapBone(weight.boneIndex); + weight.boneIndex = newIndex; + newWeights.Add(weight); + //vcol[v] = vcol[v] + Color.blue; + } + } + + src_w_base += weightPerVertex; + } + + using (var nativeWeights = new NativeArray(newWeights.ToArray(), Allocator.Temp)) + { + newMesh.SetBoneWeights( + bonesPerVertex, + nativeWeights + ); + } + + newMesh.bindposes = bindposes.ToArray(); + + _renderer.bones = bones.ToArray(); + _renderer.sharedMesh = newMesh; + + //newMesh.colors = vcol.ToArray(); + + int RemapBone(int originalIndex) + { + if (proxyIndices.TryGetValue(originalIndex, out var index)) return index; + + index = bones.Count; + bones.Add(_proxyHead); + + // The original bindpose is the inverse of the transform matrix of the bone as it was in the 3D editor, + // which does not necessarily match where the bone is right now. That is, we can imagine that some + // additional unknown transform T has been applied on top of the bone pose B, and the bindpose is + // therefore K = (T * B)^-1 = B^-1 * T^-1. Poses are computed as P' = B * K * P = B * B^-1 * T^-1 * P + // + // What we want to find is the bindpose that maps onto the proxy head transform. We can imagine that + // unity will eventually multiply some point P by pose Q, then the head pose like so: P' = H * Q * P + // We want to get the same result: B * K * P = H * Q * P; thus B * K = H * Q. + // Since H, B, K are known, we can solve like so: Q = H^-1 * B * K + + var k = bindposes[originalIndex]; + var b = _renderer.bones[originalIndex].localToWorldMatrix; + var hInv = _proxyHead.worldToLocalMatrix; + + bindposes.Add(hInv * b * k); + proxyIndices[originalIndex] = index; + + return index; + } + + void AnalyzeManifolds() + { + int boneIndex = 0; + for (int i = 0; i < vertexCount; i++) + { + nodes[i] = new ManifoldNode(); + + var weightsForVertex = bonesPerVertex[i]; + for (int w = 0; w < weightsForVertex; w++) + { + var weight = boneWeights[boneIndex + w]; + var bone = _renderer.bones[weight.boneIndex]; + if (bone == null) continue; + if (_visibleBones.Contains(bone)) + { + anyVisible = true; + nodes[i].HasVisibleBone = true; + } + else + { + nodes[i].HasRetargetedBone = true; + } + } + + boneIndex += weightsForVertex; + } + + if (!anyVisible) return; + + for (int s = 0; s < newMesh.subMeshCount; s++) + { + var topology = newMesh.GetTopology(s); + if (topology != MeshTopology.Triangles) continue; + var indices = newMesh.GetIndices(s); + + for (int t = 0; t < indices.Length; t += 3) + { + nodes[indices[t]].Union(nodes[indices[t + 1]]); + nodes[indices[t]].Union(nodes[indices[t + 2]]); + } + } + + HashSet roots = new HashSet(); + + for (int i = 0; i < vertexCount; i++) + { + var root = nodes[i].Find(); + roots.Add(root); + } + + Debug.Log("=== " + originalMesh.name + " ==="); + int index = 0; + foreach (var r in roots.OrderByDescending(r => r.Rank)) + { + Debug.Log("Root: " + r.Rank + " rank" + (r.HasVisibleBone ? " visible" : "") + + (r.HasRetargetedBone ? " retargeted" : "")); + r.b = index++ / (float) roots.Count; + } + } + } + + + private class ManifoldNode + { + private bool _hasVisibleBone, _hasRetargetedBone; + private ManifoldNode parent; + private int rank; + public float b; + + public int Rank => rank; + + public bool HasVisibleBone + { + get => Find()._hasVisibleBone; + set => Find()._hasVisibleBone |= value; + } + + public bool HasRetargetedBone + { + get => Find()._hasRetargetedBone; + set => Find()._hasRetargetedBone |= value; + } + + public ManifoldNode() + { + _hasVisibleBone = false; + _hasRetargetedBone = false; + parent = this; + rank = 1; + } + + public ManifoldNode Find() + { + // Path halving algorithm + ManifoldNode node = this; + while (node.parent != node) + { + node.parent = node.parent.parent; + node = node.parent; + } + + return node; + } + + internal void Union(ManifoldNode other) + { + var x = Find(); + var y = other.Find(); + + if (x == y) return; + if (x.rank < y.rank) + { + var tmp = x; + x = y; + y = tmp; + } + + y.parent = x; + x.rank += y.rank; + + if (x._hasRetargetedBone != y._hasRetargetedBone) + { + //Debug.Log("!"); + } + + x._hasRetargetedBone |= y._hasRetargetedBone; + x._hasVisibleBone |= y._hasVisibleBone; + } + } + } +} \ No newline at end of file diff --git a/Packages/net.fushizen.modular-avatar/Editor/FirstPersonVisibleMeshProcessor.cs.meta b/Packages/net.fushizen.modular-avatar/Editor/FirstPersonVisibleMeshProcessor.cs.meta new file mode 100644 index 00000000..d38263ea --- /dev/null +++ b/Packages/net.fushizen.modular-avatar/Editor/FirstPersonVisibleMeshProcessor.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 416b45ba9bfd4c799fa0a65611626027 +timeCreated: 1667783021 \ No newline at end of file diff --git a/Packages/net.fushizen.modular-avatar/Editor/FirstPersonVisibleProcessor.cs b/Packages/net.fushizen.modular-avatar/Editor/FirstPersonVisibleProcessor.cs index af6635ac..6291083a 100644 --- a/Packages/net.fushizen.modular-avatar/Editor/FirstPersonVisibleProcessor.cs +++ b/Packages/net.fushizen.modular-avatar/Editor/FirstPersonVisibleProcessor.cs @@ -22,7 +22,8 @@ namespace net.fushizen.modular_avatar.core.editor private HashSet _activeBones = new HashSet(); private Transform _headBone; - private Dictionary _proxyBones = new Dictionary(); + private HashSet _visibleBones = new HashSet(); + private Transform _proxyHead; public FirstPersonVisibleProcessor(VRCAvatarDescriptor avatar) { @@ -56,17 +57,31 @@ namespace net.fushizen.modular_avatar.core.editor public void Process() { + bool didWork = false; + foreach (var target in _avatar.GetComponentsInChildren(true)) { - Process(target); + var w = Process(target); + didWork = didWork || w; + } + + if (didWork) + { + // Process meshes + foreach (var smr in _avatar.GetComponentsInChildren(true)) + { + new FirstPersonVisibleMeshProcessor(smr, _visibleBones, _proxyHead).Retarget(); + } } } - void Process(ModularAvatarFirstPersonVisible target) + bool Process(ModularAvatarFirstPersonVisible target) { + bool didWork = false; + if (Validate(target) == ReadyStatus.Ready) { - var proxy = CreateProxy(_headBone); + var proxy = CreateProxy(); var xform = target.transform; @@ -75,14 +90,28 @@ namespace net.fushizen.modular_avatar.core.editor xform.localScale = new Vector3(oscale.x / pscale.x, oscale.y / pscale.y, oscale.z / pscale.z); target.transform.SetParent(proxy, true); + + didWork = true; + } + + if (didWork) + { + foreach (var xform in target.GetComponentsInChildren(true)) + { + _visibleBones.Add(xform); + } } Object.DestroyImmediate(target); + + return didWork; } - private Transform CreateProxy(Transform src) + private Transform CreateProxy() { - if (_proxyBones.TryGetValue(src, out var proxy)) return proxy; + if (_proxyHead != null) return _proxyHead; + + var src = _headBone; GameObject obj = new GameObject(src.name + " (FirstPersonVisible)"); Transform parent = _headBone.parent; @@ -103,7 +132,7 @@ namespace net.fushizen.modular_avatar.core.editor constraint.rotationOffsets = new[] {Vector3.zero}; constraint.translationOffsets = new[] {Vector3.zero}; - _proxyBones.Add(src, obj.transform); + _proxyHead = obj.transform; return obj.transform; }