从一组混合形状中高效地更新 ARSCNFaceGeometry

Question

我正在使用 ARSCNFaceGeometry 并且需要在我的游戏循环中更新面部模型的混合形状。我目前的解决方案是用新的 ARFaceGeometry:

调用 ARSCNFaceGeometry.update

class Face {
    let geometry: ARSCNFaceGeometry
    
    init(device: MTLDevice) {
        guard let geometry = ARSCNFaceGeometry(device: device, fillMesh: true) else {
            fatalError("Could not create ARSCNFaceGeometry")
        }
        self.geometry = geometry
    }
    
    func update(blendShapes: [ARFaceAnchor.BlendShapeLocation: NSNumber]) {
        let faceGeometry = ARFaceGeometry(blendShapes: blendShapes)!
        geometry.update(from: faceGeometry)
    }
}

然而，这不适合实时使用，因为单独的 ARFaceGeometry 行大约需要 0.01 秒（仅供参考，在 60fps 下我们的总帧预算为 0.0166 秒）。

经过大约几百次更新后，我们似乎运行出现了某种错误，使整个游戏循环达到 10-20fps。这是来自分析器的 6 秒样本，它看到 ARFaceGeometry 花费 2.3 秒！:

是否有更有效的方法从一组混合形状更新现有 ARSCNFaceGeometry？

例如，对于自定义 3D 模型，我可以只更新 SCNNode.morpher 上的混合变形值。 ARSCNFaceGeometry 是否有等价物？

Answer 1

Apple 开发者文档说：

To update a SceneKit model of a face actively tracked in an AR session, call this method in your ARSCNViewDelegate object’s renderer(_:didUpdate:for:) callback, passing the geometry property from the ARFaceAnchor object that callback provides.

第一个代码版本

extension ViewController: ARSCNViewDelegate {

    func renderer(_ renderer: SCNSceneRenderer,
              nodeFor anchor: ARAnchor) -> SCNNode? {

        let faceGeo = ARSCNFaceGeometry(device: sceneView.device!)
        let node = SCNNode(geometry: faceGeo)
        node.geometry?.firstMaterial?.fillMode = .lines
        return node
    }

    func renderer(_ renderer: SCNSceneRenderer,
              didUpdate node: SCNNode,
                  for anchor: ARAnchor) {

        if let faceAnchor = anchor as? ARFaceAnchor,
           let faceGeo = node.geometry as? ARSCNFaceGeometry {

            if faceAnchor.lookAtPoint.x >=  0 {
                faceGeo.firstMaterial?.diffuse.contents = UIColor.red
            } else {
                faceGeo.firstMaterial?.diffuse.contents = UIColor.cyan
            }

            faceGeo.update(from: faceAnchor.geometry)
            self.facialExrpession(anchor: faceAnchor)

            DispatchQueue.main.async {
                self.label.text = self.textBoard
            }
        }
    }
}

...

extension ViewController {

    func facialExrpession(anchor: ARFaceAnchor) {

        self.textBoard = "PLEASE SMILE!"
        let smileLeft = anchor.blendShapes[.mouthSmileLeft]
        let smileRight = anchor.blendShapes[.mouthSmileRight]

        if ((smileLeft?.decimalValue ?? 0.0) + 
            (smileRight?.decimalValue ?? 0.0)) > 0.5 {
            self.textBoard += "You are smiling"
        }
    }
}

第二个代码版本

import ARKit
import Metal

class BlendShapes: NSObject, ARSCNViewDelegate {

    typealias BlendShapeDict = [ARFaceAnchor.BlendShapeLocation: NSNumber]
    var collectBlendShapes: ((BlendShapeDict) -> Void)?

    let geometry = ARSCNFaceGeometry(device: MTLCreateSystemDefaultDevice()!, 
                                   fillMesh: true)
    
    func renderer(_ renderer: SCNSceneRenderer,
              didUpdate node: SCNNode,
                  for anchor: ARAnchor) {

        guard let facialAnchor = anchor as? ARFaceAnchor else { return }
        let blendShapes: (Any)? = collectBlendShapes?(facialAnchor.blendShapes)
        
        geometry!.update(from: blendShapes as! ARFaceGeometry)
    }
}

Alternatively, you can create, configure, and visualize face models independent of an AR session by creating face geometry objects using the ARFaceGeometry init(blendShapes:) initializer and passing them to this method.

Answer 2

我无法找到 ARFaceGeometry 性能问题。为了解决这个问题，我决定从 ARFaceGeometry.

构建我自己的模型

首先，我从 ARFaceGeometry 生成了一个模型文件。该文件包括基础几何体以及应用每个单独的混合形状时的几何体：

let LIST_OF_BLEND_SHAPES: [ARFaceAnchor.BlendShapeLocation] = [
    .eyeBlinkLeft,
    .eyeLookDownLeft,
    // ... fill in rest
]

func printFaceModelJson() {
    // Get the geometry without any blend shapes applied 
    let base = ARFaceGeometry(blendShapes: [:])!
    
    // First print out a single copy of the indices.
    // These are shared between the models
    let indexList = base.triangleIndices.map({ "\([=10=])" }).joined(separator: ",")
    print("indexList: [\(indexList)]")
    
    // Then print the starting geometry (i.e. no blend shapes applied)
    printFaceNodeJson(blendShape: nil)
    
    // And print the model with each blend shape applied
    for blend in LIST_OF_BLEND_SHAPES {
        printFaceNodeJson(blendShape: blend)
    }
}

func printFaceNodeJson(
    blendShape: ARFaceAnchor.BlendShapeLocation?
) {
    let geometry = ARFaceGeometry(blendShapes: blendShape != nil ? [blendShape!: 1.0] : [:])!
    
    let verticies = geometry.vertices.flatMap({ v in [v[0], v[1], v[2]] })
    let vertexList = verticies.map({ "\([=10=])" }).joined(separator: ",")
    print("{ \"blendShape\": \(blendShape != nil ?  "\"" + blendShape!.rawValue + "\"" : "null"), \"verticies\": [\(vertexList)] }")
}

我运行此代码脱机生成模型文件（手动快速将输出转换为正确的json）。您还可以使用适当的 3D 模型文件格式，这可能会导致模型文件更小。

然后对于我的应用程序，我从 json 模型文件重建模型：

class ARMaskFaceModel {
    
    let node: SCNNode
    
    init() {
        let data = loadMaskJsonDataFromFile() // implement this!
        
        let elements = [SCNGeometryElement(indices: data.indicies, primitiveType: .triangles)]
        
        // Create the base geometry
        let baseGeometryData = data.blends[0]
        let geometry = SCNGeometry(sources: [
            SCNGeometrySource(vertices: baseGeometryData.verticies)
        ], elements: elements)
        
        node = SCNNode(geometry: geometry)
        
        // Then load each of the blend shape geometries into a morpher
        let morpher = SCNMorpher()
        morpher.targets = data.blends.dropFirst().map({ x in
            SCNGeometry(sources: [
                SCNGeometrySource(vertices: x.verticies)
            ], elements: elements)
        })
        node.morpher = morpher
    }

    /// Apply blend shapes to the model
    func update(blendShapes: [ARFaceAnchor.BlendShapeLocation : NSNumber]) {
        var i = 0
        for blendShape in LIST_OF_BLEND_SHAPES {
            if i > node.morpher?.targets.count ?? 0 {
                return
            }
            node.morpher?.setWeight(CGFloat(truncating: blendShapes[blendShape] ?? 0.0), forTargetAt: i)
            i += 1
        }
    }
}

不理想，但它工作正常并且性能更好，即使没有任何优化。我们现在一直保持在 60fps。此外，它也适用于较旧的 phone！（虽然 printFaceModelJson 在支持实时面部跟踪的 phone 上必须是运行）

从一组混合形状中高效地更新 ARSCNFaceGeometry

Efficiently updating ARSCNFaceGeometry from a set of blend shapes

augmented-reality

ios

scenekit

swift

arkit

第一个代码版本

第二个代码版本