将 ARKit 面部跟踪 3D 网格投影到 2D 图像坐标
Projecting the ARKit face tracking 3D mesh to 2D image coordinates
我正在使用 ARKit 收集面部网格 3D 顶点。我已阅读：Mapping image onto 3D face mesh and Tracking and Visualizing Faces.
我有以下结构：
struct CaptureData {
var vertices: [SIMD3<Float>]
var verticesformatted: String {
let verticesDescribed = vertices.map({ "\([=10=].x):\([=10=].y):\([=10=].z)" }).joined(separator: "~")
return "<\(verticesDescribed)>"
}
}
我有一个捕捉顶点的 Strat 按钮：
@IBAction private func startPressed() {
captureData = [] // Clear data
currentCaptureFrame = 0 //inital capture frame
fpsTimer = Timer.scheduledTimer(withTimeInterval: 1/fps, repeats: true, block: {(timer) -> Void in self.recordData()})
}
private var fpsTimer = Timer()
private var captureData: [CaptureData] = [CaptureData]()
private var currentCaptureFrame = 0
以及停止捕获（保存数据）的停止按钮：
@IBAction private func stopPressed() {
do {
fpsTimer.invalidate() //turn off the timer
let capturedData = captureData.map{[=13=].verticesformatted}.joined(separator:"")
let dir: URL = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).last! as URL
let url = dir.appendingPathComponent("facedata.txt")
try capturedData.appendLineToURL(fileURL: url as URL)
}
catch {
print("Could not write to file")
}
}
重新编码数据的功能
private func recordData() {
guard let data = getFrameData() else { return }
captureData.append(data)
currentCaptureFrame += 1
}
获取帧数据的函数
private func getFrameData() -> CaptureData? {
let arFrame = sceneView?.session.currentFrame!
guard let anchor = arFrame?.anchors[0] as? ARFaceAnchor else {return nil}
let vertices = anchor.geometry.vertices
let data = CaptureData(vertices: vertices)
return data
}
ARSCN 扩展名：
extension ViewController: ARSCNViewDelegate {
func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor) {
guard let faceAnchor = anchor as? ARFaceAnchor else { return }
currentFaceAnchor = faceAnchor
if node.childNodes.isEmpty, let contentNode = selectedContentController.renderer(renderer, nodeFor: faceAnchor) {
node.addChildNode(contentNode)
}
selectedContentController.session = sceneView?.session
selectedContentController.sceneView = sceneView
}
/// - Tag: ARFaceGeometryUpdate
func renderer(_ renderer: SCNSceneRenderer, didUpdate node: SCNNode, for anchor: ARAnchor) {
guard anchor == currentFaceAnchor,
let contentNode = selectedContentController.contentNode,
contentNode.parent == node
else { return }
selectedContentController.session = sceneView?.session
selectedContentController.sceneView = sceneView
selectedContentController.renderer(renderer, didUpdate: contentNode, for: anchor)
}
}
我正在尝试使用 Tracking and Visualizing Faces:
中的示例代码
// Transform the vertex to the camera coordinate system.
float4 vertexCamera = scn_node.modelViewTransform * _geometry.position;
// Camera projection and perspective divide to get normalized viewport coordinates (clip space).
float4 vertexClipSpace = scn_frame.projectionTransform * vertexCamera;
vertexClipSpace /= vertexClipSpace.w;
// XY in clip space is [-1,1]x[-1,1], so adjust to UV texture coordinates: [0,1]x[0,1].
// Image coordinates are Y-flipped (upper-left origin).
float4 vertexImageSpace = float4(vertexClipSpace.xy * 0.5 + 0.5, 0.0, 1.0);
vertexImageSpace.y = 1.0 - vertexImageSpace.y;
// Apply ARKit's display transform (device orientation * front-facing camera flip).
float4 transformedVertex = displayTransform * vertexImageSpace;
// Output as texture coordinates for use in later rendering stages.
_geometry.texcoords[0] = transformedVertex.xy;
我也读过投影点（但不确定哪个更适用）：
func projectPoint(_ point: SCNVector3) -> SCNVector3
我的问题是如何使用上面的示例代码将收集到的3D人脸网格顶点转换为2D图像坐标？？
我想获取 3D 网格顶点及其对应的 2D 坐标。
目前，我可以像这样捕捉面部网格点：
我想将我的网格点转换为图像坐标并将它们一起显示，如下所示：
预期结果：
有什么建议吗？提前致谢！
也许你可以使用 SCNSceneRenderer 的 projectPoint 功能。
extension ARFaceAnchor{
// struct to store the 3d vertex and the 2d projection point
struct VerticesAndProjection {
var vertex: SIMD3<Float>
var projected: CGPoint
}
// return a struct with vertices and projection
func verticeAndProjection(to view: ARSCNView) -> [VerticesAndProjection]{
let points = geometry.vertices.compactMap({ (vertex) -> VerticesAndProjection? in
let col = SIMD4<Float>(SCNVector4())
let pos = SIMD4<Float>(SCNVector4(vertex.x, vertex.y, vertex.z, 1))
let pworld = transform * simd_float4x4(col, col, col, pos)
let vect = view.projectPoint(SCNVector3(pworld.position.x, pworld.position.y, pworld.position.z))
let p = CGPoint(x: CGFloat(vect.x), y: CGFloat(vect.y))
return VerticesAndProjection(vertex:vertex, projected: p)
})
return points
}
}
这是获取位置的便捷方法：
extension matrix_float4x4 {
/// Get the position of the transform matrix.
public var position: SCNVector3 {
get{
return SCNVector3(self[3][0], self[3][1], self[3][2])
}
}
}
如果您想检查投影是否正常，请将调试子视图添加到 ARSCNView 实例，然后使用其他几个扩展在视图上绘制二维点，例如：
extension UIView{
private struct drawCircleProperty{
static let circleFillColor = UIColor.green
static let circleStrokeColor = UIColor.black
static let circleRadius: CGFloat = 3.0
}
func drawCircle(point: CGPoint) {
let circlePath = UIBezierPath(arcCenter: point, radius: drawCircleProperty.circleRadius, startAngle: CGFloat(0), endAngle: CGFloat(Double.pi * 2.0), clockwise: true)
let shapeLayer = CAShapeLayer()
shapeLayer.path = circlePath.cgPath
shapeLayer.fillColor = drawCircleProperty.circleFillColor.cgColor
shapeLayer.strokeColor = drawCircleProperty.circleStrokeColor.cgColor
self.layer.addSublayer(shapeLayer)
}
func drawCircles(points: [CGPoint]){
self.clearLayers()
for point in points{
self.drawCircle(point: point)
}
}
func clearLayers(){
if let subLayers = self.layer.sublayers {
for subLayer in subLayers {
subLayer.removeFromSuperlayer()
}
}
}
您可以计算投影并绘制点：
let points:[ARFaceAnchor.VerticesAndProjection] = faceAnchor.verticeAndProjection(to: sceneView)
// keep only the projected points
let projected = points.map{ [=13=].projected}
// draw the points !
self.debugView?.drawCircles(points: projected)
我可以看到投影在2d屏幕上的所有3d顶点（https://thispersondoesnotexist.com生成的图片）。
我将此代码添加到 Apple 演示项目，可在此处获得 https://github.com/hugoliv/projectvertices.git
我正在使用 ARKit 收集面部网格 3D 顶点。我已阅读：Mapping image onto 3D face mesh and Tracking and Visualizing Faces.
我有以下结构：
struct CaptureData {
var vertices: [SIMD3<Float>]
var verticesformatted: String {
let verticesDescribed = vertices.map({ "\([=10=].x):\([=10=].y):\([=10=].z)" }).joined(separator: "~")
return "<\(verticesDescribed)>"
}
}
我有一个捕捉顶点的 Strat 按钮：
@IBAction private func startPressed() {
captureData = [] // Clear data
currentCaptureFrame = 0 //inital capture frame
fpsTimer = Timer.scheduledTimer(withTimeInterval: 1/fps, repeats: true, block: {(timer) -> Void in self.recordData()})
}
private var fpsTimer = Timer()
private var captureData: [CaptureData] = [CaptureData]()
private var currentCaptureFrame = 0
以及停止捕获（保存数据）的停止按钮：
@IBAction private func stopPressed() {
do {
fpsTimer.invalidate() //turn off the timer
let capturedData = captureData.map{[=13=].verticesformatted}.joined(separator:"")
let dir: URL = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).last! as URL
let url = dir.appendingPathComponent("facedata.txt")
try capturedData.appendLineToURL(fileURL: url as URL)
}
catch {
print("Could not write to file")
}
}
重新编码数据的功能
private func recordData() {
guard let data = getFrameData() else { return }
captureData.append(data)
currentCaptureFrame += 1
}
获取帧数据的函数
private func getFrameData() -> CaptureData? {
let arFrame = sceneView?.session.currentFrame!
guard let anchor = arFrame?.anchors[0] as? ARFaceAnchor else {return nil}
let vertices = anchor.geometry.vertices
let data = CaptureData(vertices: vertices)
return data
}
ARSCN 扩展名：
extension ViewController: ARSCNViewDelegate {
func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor) {
guard let faceAnchor = anchor as? ARFaceAnchor else { return }
currentFaceAnchor = faceAnchor
if node.childNodes.isEmpty, let contentNode = selectedContentController.renderer(renderer, nodeFor: faceAnchor) {
node.addChildNode(contentNode)
}
selectedContentController.session = sceneView?.session
selectedContentController.sceneView = sceneView
}
/// - Tag: ARFaceGeometryUpdate
func renderer(_ renderer: SCNSceneRenderer, didUpdate node: SCNNode, for anchor: ARAnchor) {
guard anchor == currentFaceAnchor,
let contentNode = selectedContentController.contentNode,
contentNode.parent == node
else { return }
selectedContentController.session = sceneView?.session
selectedContentController.sceneView = sceneView
selectedContentController.renderer(renderer, didUpdate: contentNode, for: anchor)
}
}
我正在尝试使用 Tracking and Visualizing Faces:
中的示例代码// Transform the vertex to the camera coordinate system.
float4 vertexCamera = scn_node.modelViewTransform * _geometry.position;
// Camera projection and perspective divide to get normalized viewport coordinates (clip space).
float4 vertexClipSpace = scn_frame.projectionTransform * vertexCamera;
vertexClipSpace /= vertexClipSpace.w;
// XY in clip space is [-1,1]x[-1,1], so adjust to UV texture coordinates: [0,1]x[0,1].
// Image coordinates are Y-flipped (upper-left origin).
float4 vertexImageSpace = float4(vertexClipSpace.xy * 0.5 + 0.5, 0.0, 1.0);
vertexImageSpace.y = 1.0 - vertexImageSpace.y;
// Apply ARKit's display transform (device orientation * front-facing camera flip).
float4 transformedVertex = displayTransform * vertexImageSpace;
// Output as texture coordinates for use in later rendering stages.
_geometry.texcoords[0] = transformedVertex.xy;
我也读过投影点（但不确定哪个更适用）：
func projectPoint(_ point: SCNVector3) -> SCNVector3
我的问题是如何使用上面的示例代码将收集到的3D人脸网格顶点转换为2D图像坐标？？
我想获取 3D 网格顶点及其对应的 2D 坐标。
目前，我可以像这样捕捉面部网格点：
我想将我的网格点转换为图像坐标并将它们一起显示，如下所示：
预期结果：
有什么建议吗？提前致谢！
也许你可以使用 SCNSceneRenderer 的 projectPoint 功能。
extension ARFaceAnchor{
// struct to store the 3d vertex and the 2d projection point
struct VerticesAndProjection {
var vertex: SIMD3<Float>
var projected: CGPoint
}
// return a struct with vertices and projection
func verticeAndProjection(to view: ARSCNView) -> [VerticesAndProjection]{
let points = geometry.vertices.compactMap({ (vertex) -> VerticesAndProjection? in
let col = SIMD4<Float>(SCNVector4())
let pos = SIMD4<Float>(SCNVector4(vertex.x, vertex.y, vertex.z, 1))
let pworld = transform * simd_float4x4(col, col, col, pos)
let vect = view.projectPoint(SCNVector3(pworld.position.x, pworld.position.y, pworld.position.z))
let p = CGPoint(x: CGFloat(vect.x), y: CGFloat(vect.y))
return VerticesAndProjection(vertex:vertex, projected: p)
})
return points
}
}
这是获取位置的便捷方法：
extension matrix_float4x4 {
/// Get the position of the transform matrix.
public var position: SCNVector3 {
get{
return SCNVector3(self[3][0], self[3][1], self[3][2])
}
}
}
如果您想检查投影是否正常，请将调试子视图添加到 ARSCNView 实例，然后使用其他几个扩展在视图上绘制二维点，例如：
extension UIView{
private struct drawCircleProperty{
static let circleFillColor = UIColor.green
static let circleStrokeColor = UIColor.black
static let circleRadius: CGFloat = 3.0
}
func drawCircle(point: CGPoint) {
let circlePath = UIBezierPath(arcCenter: point, radius: drawCircleProperty.circleRadius, startAngle: CGFloat(0), endAngle: CGFloat(Double.pi * 2.0), clockwise: true)
let shapeLayer = CAShapeLayer()
shapeLayer.path = circlePath.cgPath
shapeLayer.fillColor = drawCircleProperty.circleFillColor.cgColor
shapeLayer.strokeColor = drawCircleProperty.circleStrokeColor.cgColor
self.layer.addSublayer(shapeLayer)
}
func drawCircles(points: [CGPoint]){
self.clearLayers()
for point in points{
self.drawCircle(point: point)
}
}
func clearLayers(){
if let subLayers = self.layer.sublayers {
for subLayer in subLayers {
subLayer.removeFromSuperlayer()
}
}
}
您可以计算投影并绘制点：
let points:[ARFaceAnchor.VerticesAndProjection] = faceAnchor.verticeAndProjection(to: sceneView)
// keep only the projected points
let projected = points.map{ [=13=].projected}
// draw the points !
self.debugView?.drawCircles(points: projected)
我可以看到投影在2d屏幕上的所有3d顶点（https://thispersondoesnotexist.com生成的图片）。
我将此代码添加到 Apple 演示项目，可在此处获得 https://github.com/hugoliv/projectvertices.git