Swift

Question

我有一个应用程序可以播放从用户库中选择的视频。我希望该应用程序最终能够做的是在视频上渲染叠加层（在播放时），然后将结果输出到新的媒体文件。为此，我需要捕获解码帧，以便在视频播放结束后渲染此叠加层并输出到文件。

这是我第一个使用 AVFoundation 的应用程序，我花了一两天时间试图通过 google 和 Apple 文档找出如何实现这一点，我想我在 AVPlayerItemVideoOutput 对象。但是，永远不会执行委托回调。

我发现 AVPlayerItemVideoOutput 必须在 之后创建 AVPlayerItem 处于状态 readyToPlay。因此，在 PlayerUIView 的初始化程序中，我向 AVPlayerItem 添加了一个观察者以观察其状态。

init(frame: CGRect, url: Binding<URL?>) {
        _url = url
        // Setup the player
        player = AVPlayer(url: url.wrappedValue!)
        super.init(frame: frame)
        
        playerLayer.player = player
        playerLayer.videoGravity = .resizeAspect
        layer.addSublayer(playerLayer)
        
        //displayLink = CADisplayLink()
        
        // Setup looping
        player.actionAtItemEnd = .none
        NotificationCenter.default.addObserver(self,
                                               selector: #selector(playerItemDidReachEnd(notification:)),
                                               name: .AVPlayerItemDidPlayToEndTime,
                                               object: player.currentItem)
        
        player.currentItem?.addObserver(self, forKeyPath: #keyPath(AVPlayerItem.status), options: [.old, .new], context: nil)
        
        // Start the movie
        player.play()
    }

我在中间创建了一个 CADisplayLink - 被注释掉了 - 因为我看到它可以以某种方式用于此但不确定它应该如何或做什么。从名称上也担心它从显示的视频中获取帧，而不是从我想要的实际解码视频帧中获取帧。

第一次将状态设置为readyToPlay时，我创建并添加了AVPlayerItemVideoOutput。

override func observeValue(forKeyPath keyPath: String?, of object: Any?, change: [NSKeyValueChangeKey : Any]?, context: UnsafeMutableRawPointer?) {
        if let item = object as? AVPlayerItem {
            if item.status == AVPlayerItem.Status.readyToPlay && item.outputs.count == 0 {
                let settings = [ String(kCVPixelBufferPixelFormatTypeKey): kCVPixelFormatType_24RGB ]
                let output = AVPlayerItemVideoOutput(pixelBufferAttributes: settings)
                
                output.setDelegate(PlayerOutput(output: output), queue: DispatchQueue(label: ""))
                
                player.currentItem?.add(output)
            }
        }
    }

关于委托 PlayerOutput 我希望在有新框架可用时收到通知。此时我将访问 AVPlayerItemVideoOutput 对象以访问像素缓冲区。

class PlayerOutput : NSObject, AVPlayerItemOutputPullDelegate {
        
        func outputMediaDataWillChange(_ sender: AVPlayerItemOutput) {
            let videoOutput = sender as! AVPlayerItemVideoOutput
            let newPixelBuff = videoOutput.hasNewPixelBuffer(forItemTime: CMTime(seconds: 1, preferredTimescale: 10))
        }
    }

但是，此回调从未进行过。我在代码中设置了一个断点，它永远不会被击中。从 AVFoundation 中其他地方的命名和类似代码来看，我假设它会被每个新帧命中，所以我可以访问缓冲区中的帧，但我没有看到任何事情发生。有什么我遗漏或做错了什么吗？

我觉得我不太using/understanding这些类是正确的，它们的用途是什么，但它在命名法上与AVCaptureVideoDataOutput等相似类我已经成功地在应用程序的其他地方实现了，但它们的工作方式似乎不太一样。很难找到任何例子来做我想用 AVPlayer 做的事情。

编辑： 当前代码的工作示例：

import SwiftUI
import AVFoundation

struct CustomCameraPhotoView: View {

    @State private var image: Image?
    @State private var showingCustomCamera = false
    @State private var showImagePicker = false
    @State private var inputImage: UIImage?
    @State private var url: URL?

    var body: some View {
                ZStack {
                    if url != nil
                    {
                        PlayerView(url: $url)
                    }
                    else
                    {
                        Button(action: {
                            self.showImagePicker = true
                            }) {
                            Text("Select a Video").foregroundColor(.white).font(.headline)
                        }
                        
                    }
                }.edgesIgnoringSafeArea(.all)
            .sheet(isPresented: $showImagePicker,
                   onDismiss: loadImage) {
                    PhotoCaptureView(showImagePicker: self.$showImagePicker, image: self.$image, url: self.$url)
                }.edgesIgnoringSafeArea(.leading).edgesIgnoringSafeArea(.trailing)
    }
    func loadImage() {
        guard let inputImage = inputImage else { return }
        image = Image(uiImage: inputImage)
    }
}

struct PlayerView: UIViewControllerRepresentable {
    
    @Binding var url: URL?
    
    func updateUIViewController(_ uiView: UIViewController, context: UIViewControllerRepresentableContext<PlayerView>) {
    }
    
    func makeCoordinator() ->  PlayerCoordinator{
    //Make Coordinator which will commnicate with the    ImagePickerViewController
        PlayerCoordinator()
    }

    func makeUIViewController(context: Context) -> UIViewController {
        let view = PlayerUIView(frame: .zero, url: $url)
        let controller = PlayerController()
        controller.view = view
        
        return controller
    }
}

class PlayerCoordinator : NSObject, UINavigationControllerDelegate {
    
}

class PlayerController: UIViewController {
    override var shouldAutorotate: Bool {
        return false
    }

    override var supportedInterfaceOrientations: UIInterfaceOrientationMask {
        return .all
    }
}

class PlayerUIView: UIView {
    private let playerLayer = AVPlayerLayer()
    private var playerOutput = PlayerOutput()
    private let _myVideoOutputQueue = DispatchQueue(label: "VideoFrames", qos: .background, attributes: .concurrent, autoreleaseFrequency: .workItem, target: nil)
    
    var displayLink: CADisplayLink?
    var player: AVPlayer
    
    @Binding var url: URL?

    required init?(coder: NSCoder) {
        fatalError("init(coder:) has not been implemented")
    }
    
    init(frame: CGRect, url: Binding<URL?>) {
        _url = url
        // Setup the player
        player = AVPlayer(url: url.wrappedValue!)
        super.init(frame: frame)
        
        playerLayer.player = player
        playerLayer.videoGravity = .resizeAspect
        layer.addSublayer(playerLayer)
        
        let settings = [ String(kCVPixelBufferPixelFormatTypeKey): kCVPixelFormatType_32BGRA ]
        let output = AVPlayerItemVideoOutput(pixelBufferAttributes: settings)
        
        output.setDelegate(self.playerOutput, queue: self._myVideoOutputQueue)
        
        player.currentItem?.add(output)
        
        //displayLink = CADisplayLink()
        
        // Setup looping
        player.actionAtItemEnd = .none
        NotificationCenter.default.addObserver(self,
                                               selector: #selector(playerItemDidReachEnd(notification:)),
                                               name: .AVPlayerItemDidPlayToEndTime,
                                               object: player.currentItem)
        
        player.currentItem?.addObserver(self, forKeyPath: #keyPath(AVPlayerItem.status), options: [.old, .new], context: nil)

        // Start the movie
        player.play()
    }
    
    override func observeValue(forKeyPath keyPath: String?, of object: Any?, change: [NSKeyValueChangeKey : Any]?, context: UnsafeMutableRawPointer?) {
        if let item = object as? AVPlayerItem {
            if item.status == AVPlayerItem.Status.readyToPlay && item.outputs.count == 0 {
                
            }
        }
    }
    
    @objc
    func playerItemDidReachEnd(notification: Notification) {
        self.url = nil
    }

    override func layoutSubviews() {
        super.layoutSubviews()
        playerLayer.frame = bounds
    }
    
    class PlayerOutput : NSObject, AVPlayerItemOutputPullDelegate {
        
        func outputMediaDataWillChange(_ sender: AVPlayerItemOutput) {
            let videoOutput = sender as! AVPlayerItemVideoOutput
            let newPixelBuff = videoOutput.hasNewPixelBuffer(forItemTime: CMTime(seconds: 1, preferredTimescale: 10))
        }
    }
}

struct ImagePicker : UIViewControllerRepresentable {
    @Binding var isShown : Bool
    @Binding var image : Image?
    @Binding var url : URL?
    
    func updateUIViewController(_ uiViewController: UIImagePickerController, context: UIViewControllerRepresentableContext<ImagePicker>)
    {
       //Update UIViewcontrolleer Method
    }
    func makeCoordinator() ->  ImagePickerCoordinator{
    //Make Coordinator which will commnicate with the    ImagePickerViewController
        ImagePickerCoordinator(isShown: $isShown, image: $image, url: $url)
    }
    func makeUIViewController(context: UIViewControllerRepresentableContext<ImagePicker>) -> UIImagePickerController
    {
        let picker = UIImagePickerController()
        picker.sourceType = .photoLibrary
        picker.delegate = context.coordinator
        picker.mediaTypes = ["public.movie"]
        picker.videoQuality = .typeHigh
         return picker
    }
}

class ImagePickerCoordinator : NSObject, UINavigationControllerDelegate, UIImagePickerControllerDelegate{
@Binding var isShown : Bool
@Binding var image : Image?
    @Binding var url: URL?
    init(isShown : Binding<Bool>, image: Binding<Image?>, url: Binding<URL?>) {
      _isShown = isShown
      _image = image
      _url = url
   }
//Selected Image
   func imagePickerController(_ picker: UIImagePickerController, didFinishPickingMediaWithInfo info: [UIImagePickerController.InfoKey : Any]) {
    let uiImage = info[UIImagePickerController.InfoKey.mediaURL]   as! URL
    url = uiImage
//image = Image(uiImage: uiImage)
   isShown = false
}
//Image selection got cancel
   func imagePickerControllerDidCancel(_ picker: UIImagePickerController) {
   isShown = false
   }
}

struct PhotoCaptureView: View {
   @Binding var showImagePicker : Bool
   @Binding var image : Image?
    @Binding var url : URL?
 
   var body: some View {
    ImagePicker(isShown: $showImagePicker, image: $image, url: $url)
   }
}

struct ContentView_Previews: PreviewProvider {
    static var previews: some View {
        CustomCameraPhotoView()
    }
}

Answer 1

编辑： 你怀疑 outputMediaDataWillChange 是一个与 AVCaptureVideoDataOutput 完全不同的野兽是对的。它不是按帧调用，而是表示一些播放器状态更改（我不太清楚到底是什么情况）。有（至少）两个读取和修改视频输出的选项。

您可以将 videoComposition 添加到当前 playerItem 中，您可以在其中使用 CoreImage 工具调整视频（例如添加叠加层）。在这个例子中，我对输出应用了一个简单的模糊滤镜，但 CoreImage 允许你做更复杂的事情。

let blurFilter = CIFilter(name: "CIGaussianBlur")
if let playerItem = player.currentItem {
    let asset = playerItem.asset
    
    playerItem.videoComposition = AVMutableVideoComposition(asset: asset) { (filteringRequest) in
        let source = filteringRequest.sourceImage
        blurFilter?.setValue(source, forKey: kCIInputImageKey)
        
        filteringRequest.finish(with: blurFilter?.outputImage ?? source, context: nil)
    }
}

如果您需要实际的像素缓冲区，那么 CADisplayLink 您走对了。 Display link 允许您将操作与显示刷新率同步。您可以通过以下方式从视频输出中抓取帧：

lazy var displayLink: CADisplayLink = CADisplayLink(target: self, selector: #selector(displayLinkDidRefresh(link:)))

init(frame: CGRect, url: Binding<URL?>) {
    ...
    // activate the displayLink
    displayLink.add(to: .main, forMode: .common)
    ...
}

@objc func displayLinkDidRefresh(link: CADisplayLink) {
    guard let videoOutput = self.videoOutput else { return }
    
    let itemTime = player.currentTime()
    if videoOutput.hasNewPixelBuffer(forItemTime: itemTime) {
        var presentationItemTime: CMTime = .zero
        if let pixelBuffer = videoOutput.copyPixelBuffer(forItemTime: itemTime, itemTimeForDisplay: &presentationItemTime) {
            
            // process the pixelbuffer here
        }
    }
}

完整的最小示例：

import SwiftUI
import AVFoundation

struct CustomCameraPhotoView: View {

    @State private var image: Image?
    @State private var showingCustomCamera = false
    @State private var showImagePicker = false
    @State private var inputImage: UIImage?
    @State private var url: URL?

    var body: some View {
                ZStack {
                    if url != nil {
                        PlayerView(url: $url)
                    } else {
                        Button(action: {
                            self.showImagePicker = true
                            }) {
                            Text("Select a Video")
                                .font(.headline)
                        }
                    }
                }   .edgesIgnoringSafeArea(.all)
                    .sheet(isPresented: $showImagePicker,
                           onDismiss: loadImage) {
                            PhotoCaptureView(showImagePicker: self.$showImagePicker, image: self.$image, url: self.$url)
                    }
                    .edgesIgnoringSafeArea(.leading).edgesIgnoringSafeArea(.trailing)
    }
    
    func loadImage() {
        guard let inputImage = inputImage else { return }
        image = Image(uiImage: inputImage)
    }
}

struct PlayerView: UIViewControllerRepresentable {
    
    @Binding var url: URL?
    
    func updateUIViewController(_ uiView: UIViewController, context: UIViewControllerRepresentableContext<PlayerView>) {
    }

    func makeUIViewController(context: Context) -> UIViewController {
        let view = PlayerUIView(frame: .zero, url: $url)
        let controller = PlayerController()
        controller.view = view
        
        return controller
    }
}

class PlayerController: UIViewController {
    override var shouldAutorotate: Bool { false }
    override var supportedInterfaceOrientations: UIInterfaceOrientationMask { .all }
}

class PlayerUIView: UIView {
    private let playerLayer = AVPlayerLayer()
    private let _myVideoOutputQueue = DispatchQueue(label: "VideoFrames", qos: .background, attributes: .concurrent, autoreleaseFrequency: .workItem, target: nil)
    
    lazy var displayLink: CADisplayLink = CADisplayLink(target: self, selector: #selector(displayLinkDidRefresh(link:)))
    var player: AVPlayer
    var videoOutput: AVPlayerItemVideoOutput
    
    @Binding var url: URL?

    required init?(coder: NSCoder) {
        fatalError("init(coder:) has not been implemented")
    }
    
    init(frame: CGRect, url: Binding<URL?>) {
        _url = url
        // Setup the player
        player = AVPlayer(url: url.wrappedValue!)
        
        let settings = [ String(kCVPixelBufferPixelFormatTypeKey): kCVPixelFormatType_32BGRA ]
        let output = AVPlayerItemVideoOutput(pixelBufferAttributes: settings)
        self.videoOutput = output
        
        super.init(frame: frame)
        
        playerLayer.player = player
        playerLayer.videoGravity = .resizeAspect
        layer.addSublayer(playerLayer)
        
        attachVideoComposition()
        
        player.currentItem?.add(output)
        displayLink.add(to: .main, forMode: .common)

        // Start the movie
        player.play()
    }
    
    private func attachVideoComposition() {
        let blurFilter = CIFilter(name: "CIGaussianBlur")
        if let playerItem = player.currentItem {
            let asset = playerItem.asset
            
            playerItem.videoComposition = AVMutableVideoComposition(asset: asset) { (filteringRequest) in
                let source = filteringRequest.sourceImage
                blurFilter?.setValue(source, forKey: kCIInputImageKey)
                
                // Apply CoreImage provessing here
                
                filteringRequest.finish(with: blurFilter?.outputImage ?? source, context: nil)
            }
        }
    }
    
    override func observeValue(forKeyPath keyPath: String?, of object: Any?, change: [NSKeyValueChangeKey : Any]?, context: UnsafeMutableRawPointer?) {
        if let item = object as? AVPlayerItem {
            if item.status == AVPlayerItem.Status.readyToPlay && item.outputs.count == 0 {
                
            }
        }
    }

    override func layoutSubviews() {
        super.layoutSubviews()
        playerLayer.frame = bounds
    }
    
    @objc func displayLinkDidRefresh(link: CADisplayLink) {
        let itemTime = player.currentTime()
        if videoOutput.hasNewPixelBuffer(forItemTime: itemTime) {
            var presentationItemTime: CMTime = .zero
            if let pixelBuffer = videoOutput.copyPixelBuffer(forItemTime: itemTime, itemTimeForDisplay: &presentationItemTime) {
                
                // process the pixelbuffer here
                print(pixelBuffer)
            }
        }
    }
}

struct ImagePicker : UIViewControllerRepresentable {
    @Binding var isShown : Bool
    @Binding var image : Image?
    @Binding var url : URL?
    
    func updateUIViewController(_ uiViewController: UIImagePickerController, context: UIViewControllerRepresentableContext<ImagePicker>) {
       //Update UIViewcontrolleer Method
    }
    
    func makeCoordinator() -> ImagePickerCoordinator{
        ImagePickerCoordinator(isShown: $isShown, image: $image, url: $url)
    }
    
    func makeUIViewController(context: UIViewControllerRepresentableContext<ImagePicker>) -> UIImagePickerController {
        
        let picker = UIImagePickerController()
        picker.sourceType = .photoLibrary
        picker.delegate = context.coordinator
        picker.mediaTypes = ["public.movie"]
        picker.videoQuality = .typeHigh
        return picker
    }
}

class ImagePickerCoordinator : NSObject, UINavigationControllerDelegate, UIImagePickerControllerDelegate {
    
    @Binding var isShown : Bool
    @Binding var image : Image?
    @Binding var url: URL?
    
    init(isShown : Binding<Bool>, image: Binding<Image?>, url: Binding<URL?>) {
      _isShown = isShown
      _image = image
      _url = url
   }

   func imagePickerController(_ picker: UIImagePickerController, didFinishPickingMediaWithInfo info: [UIImagePickerController.InfoKey : Any]) {
    
        url = info[UIImagePickerController.InfoKey.mediaURL] as? URL

        isShown = false
    }

    func imagePickerControllerDidCancel(_ picker: UIImagePickerController) {
        isShown = false
    }
}

struct PhotoCaptureView: View {
    @Binding var showImagePicker : Bool
    @Binding var image           : Image?
    @Binding var url             : URL?
 
    var body: some View {
        ImagePicker(isShown: $showImagePicker, image: $image, url: $url)
    }
}

struct ContentView_Previews: PreviewProvider {
    static var previews: some View {
        CustomCameraPhotoView()
    }
}

原回答： 代表总是（？）被定义为弱成员。参见documentation。在调用委托之前，您的 PlayerOutput 对象将退出范围。使某个对象的 PlayerOutput 成员在播放期间处于活动状态，您的代码应该按原样工作。

Swift - 使用 AVPlayerItemVideoOutput 访问解码帧：未调用 outputMediaDataWillChange

Swift - Accessing decoded frames with AVPlayerItemVideoOutput: outputMediaDataWillChange is not called

avfoundation

ios