使用 AVMutableComposition 和 AVAssetExportSession 创建的 MP4 视频在 Quicktime 中工作，但在所有其他视频工具中显示损坏
MP4 videos created with AVMutableComposition and AVAssetExportSession work in Quicktime, but appear corrupted in all other video tools
我正在尝试在 macOS 上使用 Swift、AVFoundation 和 AVKit 将多部电影片段一起编辑成一个剪辑。以下 Swift 代码是我正在尝试做的一个很好的例子：
import AVFoundation
import AVKit
let source1 = AVAsset(url: URL(string: "https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4")!)
let source2 = AVAsset(url: URL(string: "http://techslides.com/demos/sample-videos/small.mp4")!)
let comp = AVMutableComposition()
comp.addMutableTrack(withMediaType: .video, preferredTrackID: kCMPersistentTrackID_Invalid)
comp.addMutableTrack(withMediaType: .audio, preferredTrackID: kCMPersistentTrackID_Invalid)
func cmtime(_ i: Double) -> CMTime {
return CMTime(seconds: i, preferredTimescale: 600)
}
func insertSecond(into: AVMutableComposition, from: AVAsset, start: CMTime, at: CMTime) throws {
let videoTrack = into.tracks(withMediaType: .video).first!
let audioTrack = into.tracks(withMediaType: .audio).first!
try videoTrack.insertTimeRange(
CMTimeRange(start: start , duration: cmtime(1.0)),
of: from.tracks(withMediaType: .video).first!,
at: at
)
try audioTrack.insertTimeRange(
CMTimeRange(start: start, duration: cmtime(1.0)),
of: from.tracks(withMediaType: .audio).first!,
at: at
)
}
try insertSecond(into: comp, from: source1, start: cmtime(3.0), at: cmtime(0.0))
try insertSecond(into: comp, from: source2, start: cmtime(2.0), at: cmtime(1.0))
try insertSecond(into: comp, from: source1, start: cmtime(100.0), at: cmtime(2.0))
try insertSecond(into: comp, from: source2, start: cmtime(3.0), at: cmtime(3.0))
try insertSecond(into: comp, from: source1, start: cmtime(350.0), at: cmtime(4.0))
if let sess = AVAssetExportSession(asset: comp, presetName: "AVAssetExportPresetHighestQuality") {
sess.outputURL = URL(fileURLWithPath: "/tmp/output.mp4")
sess.outputFileType = .mp4
sess.exportAsynchronously {
print("done")
print(sess.error ?? "success")
}
}
运行 此代码确实成功生成了 output.mp4 文件，并且该文件可以在 Quicktime 中播放，没有任何问题。您应该能够将上述代码粘贴到 Playground 中以重现视频（源视频都是托管在网络上的公开示例视频）。我也有 uploaded it to S3 here，所以你可以下载并分析它，而不必自己 运行 代码。
但是，尝试使用任何其他视频软件打开或处理它会导致错误。
VLC 将尝试播放该文件，但播放起来非常困难。视频经常冻结，与音频不同步，包括 Quicktime 根本不显示的帧，并完全跳过某些部分。
Firefox 也会尝试播放文件，但显然无法正确解码并且视频输出有问题。 Chrome 播放第一秒后卡住。
我尝试使用 ffprobe 和 ffmpeg 进一步诊断。
运行 ffprobe -show_frames output.mp4 1>/dev/null 显示：
[mov,mp4,m4a,3gp,3g2,mj2 @ 0x7fe841801800] DTS -24000 < 24000 out of order
[h264 @ 0x7fe843022800] reference count overflow
[h264 @ 0x7fe843022800] decode_slice_header error
[h264 @ 0x7fe843022800] no frame!
[h264 @ 0x7fe843022800] deblocking_filter_idc 6 out of range
[h264 @ 0x7fe843022800] decode_slice_header error
[h264 @ 0x7fe843022800] no frame!
[h264 @ 0x7fe843022800] deblocking_filter_idc 6 out of range
[h264 @ 0x7fe843022800] decode_slice_header error
[h264 @ 0x7fe843022800] no frame!
[h264 @ 0x7fe843022800] top block unavailable for requested intra mode -1
[h264 @ 0x7fe843022800] error while decoding MB 5 0, bytestream 947
[h264 @ 0x7fe843022800] concealing 3600 DC, 3600 AC, 3600 MV errors in P frame
[h264 @ 0x7fe843022800] mmco: unref short failure
[h264 @ 0x7fe843022800] cabac_init_idc 4 overflow
[h264 @ 0x7fe843022800] decode_slice_header error
[h264 @ 0x7fe843022800] no frame!
[h264 @ 0x7fe843022800] deblocking filter parameters -43 0 out of range
[h264 @ 0x7fe843022800] decode_slice_header error
[h264 @ 0x7fe843022800] no frame!
尝试使用 ffmpeg (ffmpeg -i output.mp4 output.avi) 转码为另一种格式有很多警告和错误：
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 73, current: 71; changing to 74. This may result in incorrect timestamps in the output file.
[mov,mp4,m4a,3gp,3g2,mj2 @ 0x7fe1f4802800] DTS -24000 < 24000 out of order
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 74, current: 72; changing to 75. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 75, current: 73; changing to 76. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 76, current: 74; changing to 77. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 77, current: 75; changing to 78. This may result in incorrect timestamps in the output file.
[h264 @ 0x7fe1f4849600] reference count overflow
[h264 @ 0x7fe1f4849600] decode_slice_header error
[h264 @ 0x7fe1f4849600] no frame!
Error while decoding stream #0:1: Invalid data found when processing input
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 145, current: 143; changing to 146. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 146, current: 144; changing to 147. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 147, current: 145; changing to 148. This may result in incorrect timestamps in the output file.
[h264 @ 0x7fe1f483d800] deblocking_filter_idc 6 out of range
[h264 @ 0x7fe1f483d800] decode_slice_header error
[h264 @ 0x7fe1f483d800] no frame!
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 148, current: 146; changing to 149. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 149, current: 147; changing to 150. This may result in incorrect timestamps in the output file.
[h264 @ 0x7fe1f4849600] deblocking_filter_idc 6 out of range
[h264 @ 0x7fe1f4849600] decode_slice_header error
[h264 @ 0x7fe1f4849600] no frame!
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 150, current: 148; changing to 151. This may result in incorrect timestamps in the output file.
Error while decoding stream #0:1: Invalid data found when processing input
Last message repeated 1 times
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 151, current: 149; changing to 152. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 152, current: 150; changing to 153. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 153, current: 151; changing to 154. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 154, current: 152; changing to 155. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 155, current: 153; changing to 156. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 156, current: 154; changing to 157. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 157, current: 155; changing to 158. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 158, current: 156; changing to 159. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 159, current: 157; changing to 160. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 160, current: 158; changing to 161. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 161, current: 159; changing to 162. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 162, current: 160; changing to 163. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 163, current: 161; changing to 164. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 164, current: 162; changing to 165. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 165, current: 163; changing to 166. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 166, current: 164; changing to 167. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 167, current: 165; changing to 168. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 168, current: 166; changing to 169. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 169, current: 167; changing to 170. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 170, current: 168; changing to 171. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 171, current: 169; changing to 172. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 172, current: 170; changing to 173. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 173, current: 171; changing to 174. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 174, current: 172; changing to 175. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 175, current: 173; changing to 176. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 176, current: 174; changing to 177. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 177, current: 175; changing to 178. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 178, current: 176; changing to 179. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 179, current: 177; changing to 180. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 180, current: 178; changing to 181. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 181, current: 179; changing to 182. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 182, current: 180; changing to 183. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 183, current: 181; changing to 184. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 184, current: 182; changing to 185. This may result in incorrect timestamps in the output file.
[h264 @ 0x7fe1f483d800] top block unavailable for requested intra mode -1
[h264 @ 0x7fe1f483d800] error while decoding MB 5 0, bytestream 947
[h264 @ 0x7fe1f483d800] concealing 3600 DC, 3600 AC, 3600 MV errors in P frame
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 185, current: 183; changing to 186. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 186, current: 184; changing to 187. This may result in incorrect timestamps in the output file.
[h264 @ 0x7fe1f4849600] mmco: unref short failure
[h264 @ 0x7fe1f4849600] cabac_init_idc 4 overflow
[h264 @ 0x7fe1f4849600] decode_slice_header error
[h264 @ 0x7fe1f4849600] no frame!
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 187, current: 185; changing to 188. This may result in incorrect timestamps in the output file.
[h264 @ 0x7fe1f485fa00] deblocking filter parameters -43 0 out of range
[h264 @ 0x7fe1f485fa00] decode_slice_header error
[h264 @ 0x7fe1f485fa00] no frame!
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 188, current: 186; changing to 189. This may result in incorrect timestamps in the output file.
Error while decoding stream #0:1: Invalid data found when processing input
Last message repeated 1 times
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 189, current: 187; changing to 190. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 190, current: 188; changing to 191. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 191, current: 189; changing to 192. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 192, current: 190; changing to 193. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 193, current: 191; changing to 194. This may result in incorrect timestamps in the output file.
上面的代码只是一个例子，我在这段代码的许多变体中看到了严重程度不同的类似问题。我尝试了很多东西，包括：
- 使用本地文件 URL 而不是 HTTPS url
- 使用
.mov 而不是 .mp4 个文件
- 使用
AVMovie 和 AVMutableMovie（以及各种设置调整，例如将 AVURLAssetPreferPreciseDurationAndTimingKey 设置为 true）而不是 AVMutableComposition
- 调整
AVAssetExportSession 上的设置（预设等）
- 以不同的方式构建
CMTime 个对象
但无济于事——我似乎无法让 AVFoundation 生成其他工具可以处理的视频文件。
感谢任何帮助，甚至是关于输出文件编码异常的任何想法，you can download here 如果你不能或不想 运行 以上 Swift代码自己复现。
这个问题几乎可以肯定是因为视频和音频的剪辑格式不匹配：
AVAssetExportSession 可以做很多事情，但它似乎选择将所有格式转储到那里，这可能解释了您的兼容性问题。我明白为什么玩家会感到困惑，这两种视频格式甚至没有相同的纵横比。也许这种行为是一个错误，或者在某些情况下它很有意义。我不知道。
所以你可以：
- 确保你所有的剪辑都有匹配的格式（我喜欢这个，也许你不希望你的剪辑混搭在信箱样式/纵横比变化之间跳跃）
- 使用
AVMutableVideoCompositionLayerInstruction + AVAssetExportSession 或 AVAssetReader + AVAssetWriter 使格式匹配
p.s。我怀疑这里的视频不匹配比音频更重要，所以也许测试会表明您可以忽略音频？
正如@RhythmicFistman 所说，您的视频流是多个具有不同属性的 H264 流的串联。 H264 流的参数通常可以存储在带内（称为附件 B）或存储在 MP4 等容器中时的全局元数据内（stsd）。 AVF 在这里所做的是添加多个 stsd 条目。
stsd: s= 326 (0x00000146), o= 1982552 (0x001e4058)
version: 0
flags: 0x000000
sample_descriptions (0x00000002):
size: 0000009b
data_format: avc1 (61 76 63 31)
...
size: 0000009b
data_format: avc1 (61 76 63 31)
...
大多数播放器会忽略额外的条目，但解码器需要此比特流配置以进行（重新）初始化。
前进的道路有两条。
使用相同的编码属性重新编码每个单独的片段，以便 post-concat，缺少解码器重新初始化实际上不是问题，
或
获取 AVF 或 mp4box 等其他工具，将流连接为 avc3 流，由此比特流参数也存储在带内。解码器应该遇到新的参数集并重新初始化。
我同意其他播放器的播放问题源于每个曲目有不止一种格式描述。但实际上在合成之前不需要昂贵的音轨转码，AVFoundation 可以为你做这件事……如果你愿意跳过几个环节。
关键是 AVMutableComposition 可以有多个特定媒体类型的曲目，并且 AVAssetExportSession 可以 "mix" 将这样的作品缩减为每种媒体只有一个曲目类型。 AVFoundation 通过提供 mutableTrackCompatibleWithTrack: 来承认每个轨道有多种格式描述的问题。因此，当您想要从给定的源曲目中插入片段时，您可以向 AVMutableComposition 询问合适的目标曲目，如果返回 none，请添加一个新曲目。
如前所述，有几点需要牢记：
您不能将 "void time" 中超出当前结尾的某处插入给定的目标轨道。要解决此问题，请记下目标轨道的当前结束时间，在该时间追加片段，然后之后在前一个轨道结束处插入一个正确持续时间的空片段。下面的示例显示了这一点，在简化假设下，您总是在追加。如果您想在现有音轨中的任何位置插入，您将需要更复杂的逻辑。
实际上 AVAssetExportSession 将所有内容混合到每种媒体类型只有一个轨道，您必须在导出会话中设置 AVAudioMix 和 AVVideoComposition。
下面的示例代码基于您的原始示例，生成的 output.mp4 可以在 VLC、Chrome 和 Firefox 中正确播放，并且在使用 ffmpeg 检查时不会抛出任何错误。
import AVFoundation
import Foundation
let source0 = AVAsset(url: URL(string: "https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4")!)
let source1 = AVAsset(url: URL(string: "http://techslides.com/demos/sample-videos/small.mp4")!)
let comp = AVMutableComposition()
func cmtime(_ i: Double) -> CMTime {
return CMTime(seconds: i, preferredTimescale: 600)
}
func insertTrackSecond( srcAsset: AVAsset, dstComp: AVMutableComposition, mediaType: AVMediaType, start: CMTime, at: CMTime) throws {
let srcTrack: AVAssetTrack = srcAsset.tracks(withMediaType: mediaType).first!
// get a compatible destination track or, if not available, create a new one
let dstTrack: AVMutableCompositionTrack = dstComp.mutableTrack(compatibleWith: srcTrack) ?? dstComp.addMutableTrack(withMediaType: mediaType, preferredTrackID: kCMPersistentTrackID_Invalid)!
// can't insert into "void" time beyond the current end of track. Instead, note current end time, append there, and *after* appending, insert empty range
var dstTrackEnd: CMTime = CMTimeRangeGetEnd( dstTrack.timeRange)
if CMTIME_IS_INVALID( dstTrackEnd) {
dstTrackEnd = kCMTimeZero
}
try dstTrack.insertTimeRange( CMTimeRangeMake( start, cmtime( 1.0)), of: srcTrack, at: dstTrackEnd)
// now add empty range, if necessary
if CMTimeCompare( dstTrackEnd, at) == -1 {
dstTrack.insertEmptyTimeRange( CMTimeRangeFromTimeToTime( dstTrackEnd, at))
}
}
func insertSecond( srcAsset: AVAsset, dstComp: AVMutableComposition, start: CMTime, at: CMTime) throws
{
try insertTrackSecond(srcAsset: srcAsset, dstComp: dstComp, mediaType: .video, start: start, at: at)
try insertTrackSecond(srcAsset: srcAsset, dstComp: dstComp, mediaType: .audio, start: start, at: at)
}
try insertSecond( srcAsset: source0, dstComp: comp, start: cmtime(3.0), at: cmtime(0.0))
try insertSecond( srcAsset: source1, dstComp: comp, start: cmtime(2.0), at: cmtime(1.0))
try insertSecond( srcAsset: source0, dstComp: comp, start: cmtime(100.0), at: cmtime(2.0))
try insertSecond( srcAsset: source1, dstComp: comp, start: cmtime(3.0), at: cmtime(3.0))
try insertSecond( srcAsset: source0, dstComp: comp, start: cmtime(350.0), at: cmtime(4.0))
if let sess = AVAssetExportSession(asset: comp, presetName: "AVAssetExportPresetHighestQuality") {
sess.outputURL = URL(fileURLWithPath: "/tmp/output.mp4")
sess.outputFileType = .mp4
// this leaves smaller videotracks at the origin, in their "natural" size. Manipulate the "preferredTransform" property of the mutable composition tracks for nicer results
sess.videoComposition = AVVideoComposition.init(propertiesOf: comp)
// not assigning an audio mix results in an output with multiple audio tracks
var inputParameters = [AVAudioMixInputParameters]()
for audioTrack: AVAssetTrack in comp.tracks(withMediaType:.audio) {
inputParameters.append( AVMutableAudioMixInputParameters.init(track:audioTrack))
}
let audioMix: AVMutableAudioMix = AVMutableAudioMix();
audioMix.inputParameters = inputParameters;
sess.audioMix = audioMix;
let semaphore: DispatchSemaphore = DispatchSemaphore(value:0);
sess.exportAsynchronously {
print("done")
print(sess.error ?? "success")
semaphore.signal()
}
semaphore.wait()
}
我正在尝试在 macOS 上使用 Swift、AVFoundation 和 AVKit 将多部电影片段一起编辑成一个剪辑。以下 Swift 代码是我正在尝试做的一个很好的例子：
import AVFoundation
import AVKit
let source1 = AVAsset(url: URL(string: "https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4")!)
let source2 = AVAsset(url: URL(string: "http://techslides.com/demos/sample-videos/small.mp4")!)
let comp = AVMutableComposition()
comp.addMutableTrack(withMediaType: .video, preferredTrackID: kCMPersistentTrackID_Invalid)
comp.addMutableTrack(withMediaType: .audio, preferredTrackID: kCMPersistentTrackID_Invalid)
func cmtime(_ i: Double) -> CMTime {
return CMTime(seconds: i, preferredTimescale: 600)
}
func insertSecond(into: AVMutableComposition, from: AVAsset, start: CMTime, at: CMTime) throws {
let videoTrack = into.tracks(withMediaType: .video).first!
let audioTrack = into.tracks(withMediaType: .audio).first!
try videoTrack.insertTimeRange(
CMTimeRange(start: start , duration: cmtime(1.0)),
of: from.tracks(withMediaType: .video).first!,
at: at
)
try audioTrack.insertTimeRange(
CMTimeRange(start: start, duration: cmtime(1.0)),
of: from.tracks(withMediaType: .audio).first!,
at: at
)
}
try insertSecond(into: comp, from: source1, start: cmtime(3.0), at: cmtime(0.0))
try insertSecond(into: comp, from: source2, start: cmtime(2.0), at: cmtime(1.0))
try insertSecond(into: comp, from: source1, start: cmtime(100.0), at: cmtime(2.0))
try insertSecond(into: comp, from: source2, start: cmtime(3.0), at: cmtime(3.0))
try insertSecond(into: comp, from: source1, start: cmtime(350.0), at: cmtime(4.0))
if let sess = AVAssetExportSession(asset: comp, presetName: "AVAssetExportPresetHighestQuality") {
sess.outputURL = URL(fileURLWithPath: "/tmp/output.mp4")
sess.outputFileType = .mp4
sess.exportAsynchronously {
print("done")
print(sess.error ?? "success")
}
}
运行 此代码确实成功生成了 output.mp4 文件，并且该文件可以在 Quicktime 中播放，没有任何问题。您应该能够将上述代码粘贴到 Playground 中以重现视频（源视频都是托管在网络上的公开示例视频）。我也有 uploaded it to S3 here，所以你可以下载并分析它，而不必自己 运行 代码。
但是，尝试使用任何其他视频软件打开或处理它会导致错误。
VLC 将尝试播放该文件，但播放起来非常困难。视频经常冻结，与音频不同步，包括 Quicktime 根本不显示的帧，并完全跳过某些部分。
Firefox 也会尝试播放文件，但显然无法正确解码并且视频输出有问题。 Chrome 播放第一秒后卡住。
我尝试使用 ffprobe 和 ffmpeg 进一步诊断。
运行 ffprobe -show_frames output.mp4 1>/dev/null 显示：
[mov,mp4,m4a,3gp,3g2,mj2 @ 0x7fe841801800] DTS -24000 < 24000 out of order
[h264 @ 0x7fe843022800] reference count overflow
[h264 @ 0x7fe843022800] decode_slice_header error
[h264 @ 0x7fe843022800] no frame!
[h264 @ 0x7fe843022800] deblocking_filter_idc 6 out of range
[h264 @ 0x7fe843022800] decode_slice_header error
[h264 @ 0x7fe843022800] no frame!
[h264 @ 0x7fe843022800] deblocking_filter_idc 6 out of range
[h264 @ 0x7fe843022800] decode_slice_header error
[h264 @ 0x7fe843022800] no frame!
[h264 @ 0x7fe843022800] top block unavailable for requested intra mode -1
[h264 @ 0x7fe843022800] error while decoding MB 5 0, bytestream 947
[h264 @ 0x7fe843022800] concealing 3600 DC, 3600 AC, 3600 MV errors in P frame
[h264 @ 0x7fe843022800] mmco: unref short failure
[h264 @ 0x7fe843022800] cabac_init_idc 4 overflow
[h264 @ 0x7fe843022800] decode_slice_header error
[h264 @ 0x7fe843022800] no frame!
[h264 @ 0x7fe843022800] deblocking filter parameters -43 0 out of range
[h264 @ 0x7fe843022800] decode_slice_header error
[h264 @ 0x7fe843022800] no frame!
尝试使用 ffmpeg (ffmpeg -i output.mp4 output.avi) 转码为另一种格式有很多警告和错误：
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 73, current: 71; changing to 74. This may result in incorrect timestamps in the output file.
[mov,mp4,m4a,3gp,3g2,mj2 @ 0x7fe1f4802800] DTS -24000 < 24000 out of order
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 74, current: 72; changing to 75. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 75, current: 73; changing to 76. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 76, current: 74; changing to 77. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 77, current: 75; changing to 78. This may result in incorrect timestamps in the output file.
[h264 @ 0x7fe1f4849600] reference count overflow
[h264 @ 0x7fe1f4849600] decode_slice_header error
[h264 @ 0x7fe1f4849600] no frame!
Error while decoding stream #0:1: Invalid data found when processing input
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 145, current: 143; changing to 146. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 146, current: 144; changing to 147. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 147, current: 145; changing to 148. This may result in incorrect timestamps in the output file.
[h264 @ 0x7fe1f483d800] deblocking_filter_idc 6 out of range
[h264 @ 0x7fe1f483d800] decode_slice_header error
[h264 @ 0x7fe1f483d800] no frame!
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 148, current: 146; changing to 149. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 149, current: 147; changing to 150. This may result in incorrect timestamps in the output file.
[h264 @ 0x7fe1f4849600] deblocking_filter_idc 6 out of range
[h264 @ 0x7fe1f4849600] decode_slice_header error
[h264 @ 0x7fe1f4849600] no frame!
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 150, current: 148; changing to 151. This may result in incorrect timestamps in the output file.
Error while decoding stream #0:1: Invalid data found when processing input
Last message repeated 1 times
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 151, current: 149; changing to 152. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 152, current: 150; changing to 153. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 153, current: 151; changing to 154. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 154, current: 152; changing to 155. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 155, current: 153; changing to 156. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 156, current: 154; changing to 157. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 157, current: 155; changing to 158. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 158, current: 156; changing to 159. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 159, current: 157; changing to 160. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 160, current: 158; changing to 161. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 161, current: 159; changing to 162. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 162, current: 160; changing to 163. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 163, current: 161; changing to 164. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 164, current: 162; changing to 165. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 165, current: 163; changing to 166. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 166, current: 164; changing to 167. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 167, current: 165; changing to 168. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 168, current: 166; changing to 169. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 169, current: 167; changing to 170. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 170, current: 168; changing to 171. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 171, current: 169; changing to 172. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 172, current: 170; changing to 173. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 173, current: 171; changing to 174. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 174, current: 172; changing to 175. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 175, current: 173; changing to 176. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 176, current: 174; changing to 177. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 177, current: 175; changing to 178. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 178, current: 176; changing to 179. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 179, current: 177; changing to 180. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 180, current: 178; changing to 181. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 181, current: 179; changing to 182. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 182, current: 180; changing to 183. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 183, current: 181; changing to 184. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 184, current: 182; changing to 185. This may result in incorrect timestamps in the output file.
[h264 @ 0x7fe1f483d800] top block unavailable for requested intra mode -1
[h264 @ 0x7fe1f483d800] error while decoding MB 5 0, bytestream 947
[h264 @ 0x7fe1f483d800] concealing 3600 DC, 3600 AC, 3600 MV errors in P frame
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 185, current: 183; changing to 186. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 186, current: 184; changing to 187. This may result in incorrect timestamps in the output file.
[h264 @ 0x7fe1f4849600] mmco: unref short failure
[h264 @ 0x7fe1f4849600] cabac_init_idc 4 overflow
[h264 @ 0x7fe1f4849600] decode_slice_header error
[h264 @ 0x7fe1f4849600] no frame!
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 187, current: 185; changing to 188. This may result in incorrect timestamps in the output file.
[h264 @ 0x7fe1f485fa00] deblocking filter parameters -43 0 out of range
[h264 @ 0x7fe1f485fa00] decode_slice_header error
[h264 @ 0x7fe1f485fa00] no frame!
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 188, current: 186; changing to 189. This may result in incorrect timestamps in the output file.
Error while decoding stream #0:1: Invalid data found when processing input
Last message repeated 1 times
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 189, current: 187; changing to 190. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 190, current: 188; changing to 191. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 191, current: 189; changing to 192. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 192, current: 190; changing to 193. This may result in incorrect timestamps in the output file.
[avi @ 0x7fe1f5804e00] Non-monotonous DTS in output stream 0:1; previous: 193, current: 191; changing to 194. This may result in incorrect timestamps in the output file.
上面的代码只是一个例子，我在这段代码的许多变体中看到了严重程度不同的类似问题。我尝试了很多东西，包括：
- 使用本地文件 URL 而不是 HTTPS url
- 使用
.mov而不是.mp4个文件 - 使用
AVMovie和AVMutableMovie（以及各种设置调整，例如将AVURLAssetPreferPreciseDurationAndTimingKey设置为 true）而不是AVMutableComposition - 调整
AVAssetExportSession 上的设置（预设等）
- 以不同的方式构建
CMTime个对象
但无济于事——我似乎无法让 AVFoundation 生成其他工具可以处理的视频文件。
感谢任何帮助，甚至是关于输出文件编码异常的任何想法，you can download here 如果你不能或不想 运行 以上 Swift代码自己复现。
这个问题几乎可以肯定是因为视频和音频的剪辑格式不匹配：
AVAssetExportSession 可以做很多事情，但它似乎选择将所有格式转储到那里，这可能解释了您的兼容性问题。我明白为什么玩家会感到困惑，这两种视频格式甚至没有相同的纵横比。也许这种行为是一个错误，或者在某些情况下它很有意义。我不知道。
所以你可以：
- 确保你所有的剪辑都有匹配的格式（我喜欢这个，也许你不希望你的剪辑混搭在信箱样式/纵横比变化之间跳跃）
- 使用
AVMutableVideoCompositionLayerInstruction+AVAssetExportSession或AVAssetReader+AVAssetWriter 使格式匹配
p.s。我怀疑这里的视频不匹配比音频更重要，所以也许测试会表明您可以忽略音频？
正如@RhythmicFistman 所说，您的视频流是多个具有不同属性的 H264 流的串联。 H264 流的参数通常可以存储在带内（称为附件 B）或存储在 MP4 等容器中时的全局元数据内（stsd）。 AVF 在这里所做的是添加多个 stsd 条目。
stsd: s= 326 (0x00000146), o= 1982552 (0x001e4058)
version: 0
flags: 0x000000
sample_descriptions (0x00000002):
size: 0000009b
data_format: avc1 (61 76 63 31)
...
size: 0000009b
data_format: avc1 (61 76 63 31)
...
大多数播放器会忽略额外的条目，但解码器需要此比特流配置以进行（重新）初始化。
前进的道路有两条。
使用相同的编码属性重新编码每个单独的片段，以便 post-concat，缺少解码器重新初始化实际上不是问题，
或
获取 AVF 或 mp4box 等其他工具，将流连接为 avc3 流，由此比特流参数也存储在带内。解码器应该遇到新的参数集并重新初始化。
我同意其他播放器的播放问题源于每个曲目有不止一种格式描述。但实际上在合成之前不需要昂贵的音轨转码，AVFoundation 可以为你做这件事……如果你愿意跳过几个环节。
关键是 AVMutableComposition 可以有多个特定媒体类型的曲目，并且 AVAssetExportSession 可以 "mix" 将这样的作品缩减为每种媒体只有一个曲目类型。 AVFoundation 通过提供 mutableTrackCompatibleWithTrack: 来承认每个轨道有多种格式描述的问题。因此，当您想要从给定的源曲目中插入片段时，您可以向 AVMutableComposition 询问合适的目标曲目，如果返回 none，请添加一个新曲目。
如前所述，有几点需要牢记：
您不能将 "void time" 中超出当前结尾的某处插入给定的目标轨道。要解决此问题，请记下目标轨道的当前结束时间，在该时间追加片段，然后之后在前一个轨道结束处插入一个正确持续时间的空片段。下面的示例显示了这一点，在简化假设下，您总是在追加。如果您想在现有音轨中的任何位置插入，您将需要更复杂的逻辑。
实际上
AVAssetExportSession将所有内容混合到每种媒体类型只有一个轨道，您必须在导出会话中设置 AVAudioMix 和 AVVideoComposition。
下面的示例代码基于您的原始示例，生成的 output.mp4 可以在 VLC、Chrome 和 Firefox 中正确播放，并且在使用 ffmpeg 检查时不会抛出任何错误。
import AVFoundation
import Foundation
let source0 = AVAsset(url: URL(string: "https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4")!)
let source1 = AVAsset(url: URL(string: "http://techslides.com/demos/sample-videos/small.mp4")!)
let comp = AVMutableComposition()
func cmtime(_ i: Double) -> CMTime {
return CMTime(seconds: i, preferredTimescale: 600)
}
func insertTrackSecond( srcAsset: AVAsset, dstComp: AVMutableComposition, mediaType: AVMediaType, start: CMTime, at: CMTime) throws {
let srcTrack: AVAssetTrack = srcAsset.tracks(withMediaType: mediaType).first!
// get a compatible destination track or, if not available, create a new one
let dstTrack: AVMutableCompositionTrack = dstComp.mutableTrack(compatibleWith: srcTrack) ?? dstComp.addMutableTrack(withMediaType: mediaType, preferredTrackID: kCMPersistentTrackID_Invalid)!
// can't insert into "void" time beyond the current end of track. Instead, note current end time, append there, and *after* appending, insert empty range
var dstTrackEnd: CMTime = CMTimeRangeGetEnd( dstTrack.timeRange)
if CMTIME_IS_INVALID( dstTrackEnd) {
dstTrackEnd = kCMTimeZero
}
try dstTrack.insertTimeRange( CMTimeRangeMake( start, cmtime( 1.0)), of: srcTrack, at: dstTrackEnd)
// now add empty range, if necessary
if CMTimeCompare( dstTrackEnd, at) == -1 {
dstTrack.insertEmptyTimeRange( CMTimeRangeFromTimeToTime( dstTrackEnd, at))
}
}
func insertSecond( srcAsset: AVAsset, dstComp: AVMutableComposition, start: CMTime, at: CMTime) throws
{
try insertTrackSecond(srcAsset: srcAsset, dstComp: dstComp, mediaType: .video, start: start, at: at)
try insertTrackSecond(srcAsset: srcAsset, dstComp: dstComp, mediaType: .audio, start: start, at: at)
}
try insertSecond( srcAsset: source0, dstComp: comp, start: cmtime(3.0), at: cmtime(0.0))
try insertSecond( srcAsset: source1, dstComp: comp, start: cmtime(2.0), at: cmtime(1.0))
try insertSecond( srcAsset: source0, dstComp: comp, start: cmtime(100.0), at: cmtime(2.0))
try insertSecond( srcAsset: source1, dstComp: comp, start: cmtime(3.0), at: cmtime(3.0))
try insertSecond( srcAsset: source0, dstComp: comp, start: cmtime(350.0), at: cmtime(4.0))
if let sess = AVAssetExportSession(asset: comp, presetName: "AVAssetExportPresetHighestQuality") {
sess.outputURL = URL(fileURLWithPath: "/tmp/output.mp4")
sess.outputFileType = .mp4
// this leaves smaller videotracks at the origin, in their "natural" size. Manipulate the "preferredTransform" property of the mutable composition tracks for nicer results
sess.videoComposition = AVVideoComposition.init(propertiesOf: comp)
// not assigning an audio mix results in an output with multiple audio tracks
var inputParameters = [AVAudioMixInputParameters]()
for audioTrack: AVAssetTrack in comp.tracks(withMediaType:.audio) {
inputParameters.append( AVMutableAudioMixInputParameters.init(track:audioTrack))
}
let audioMix: AVMutableAudioMix = AVMutableAudioMix();
audioMix.inputParameters = inputParameters;
sess.audioMix = audioMix;
let semaphore: DispatchSemaphore = DispatchSemaphore(value:0);
sess.exportAsynchronously {
print("done")
print(sess.error ?? "success")
semaphore.signal()
}
semaphore.wait()
}