使用 libswresample 将音频从 48000 重采样到 44100
Resampling audio using libswresample from 48000 to 44100
我正在尝试使用 libswresample API 将解码后的音频帧从 48KHz 重采样到 44.1KHz。我的代码如下：
// 'frame' is the original decoded audio frame
AVFrame *output_frame = av_frame_alloc();
// Without this, there is no sound at all at the output (PTS stuff I guess)
av_frame_copy_props(output_frame, frame);
output_frame->channel_layout = audioStream->codec->channel_layout;
output_frame->sample_rate = audioStream->codec->sample_rate;
output_frame->format = audioStream->codec->sample_fmt;
SwrContext *swr;
// Configure resampling context
swr = swr_alloc_set_opts(NULL, // we're allocating a new context
AV_CH_LAYOUT_STEREO, // out_ch_layout
AV_SAMPLE_FMT_FLTP, // out_sample_fmt
44100, // out_sample_rate
AV_CH_LAYOUT_STEREO, // in_ch_layout
AV_SAMPLE_FMT_FLTP, // in_sample_fmt
48000, // in_sample_rate
0, // log_offset
NULL); // log_ctx
// Initialize resampling context
swr_init(swr);
// Perform conversion
swr_convert_frame(swr, output_frame, frame);
// Close resampling context
swr_close(swr);
swr_free(&swr);
// Free the original frame and replace it with the new one
av_frame_unref(frame);
return output_frame;
使用此代码，我能够在输出端听到音频，但它也很嘈杂。根据我的阅读，这段没有 av_frame_copy_props() 的代码应该足够了，但由于某种原因它不起作用。有什么想法吗？
编辑：输入流使用AAC对音频进行编码，样本数为1024。但是，转换后，样本数为925。
编辑： 我试着反过来做。由于我的应用程序接收来自任何来源的流，因此一些音频流是 48KHz 而另一些是 44.1KHz。所以我尝试从 44.1 重采样到 48 以避免重采样损失。但是现在每帧都有超过1024个样本，编码失败。
编辑： 我尝试使用 libavfilter 代替以下过滤器链：
int init_filter_graph(AVStream *audio_st) {
// create new graph
filter_graph = avfilter_graph_alloc();
if (!filter_graph) {
av_log(NULL, AV_LOG_ERROR, "unable to create filter graph: out of memory\n");
return -1;
}
AVFilter *abuffer = avfilter_get_by_name("abuffer");
AVFilter *aformat = avfilter_get_by_name("aformat");
AVFilter *asetnsamples = avfilter_get_by_name("asetnsamples");
AVFilter *abuffersink = avfilter_get_by_name("abuffersink");
int err;
// create abuffer filter
AVCodecContext *avctx = audio_st->codec;
AVRational time_base = audio_st->time_base;
snprintf(strbuf, sizeof(strbuf),
"time_base=%d/%d:sample_rate=%d:sample_fmt=%s:channel_layout=0x%" PRIx64,
time_base.num, time_base.den, avctx->sample_rate,
av_get_sample_fmt_name(avctx->sample_fmt),
avctx->channel_layout);
fprintf(stderr, "abuffer: %s\n", strbuf);
err = avfilter_graph_create_filter(&abuffer_ctx, abuffer,
NULL, strbuf, NULL, filter_graph);
if (err < 0) {
av_log(NULL, AV_LOG_ERROR, "error initializing abuffer filter\n");
return err;
}
// create aformat filter
snprintf(strbuf, sizeof(strbuf),
"sample_fmts=%s:sample_rates=%d:channel_layouts=0x%" PRIx64,
av_get_sample_fmt_name(AV_SAMPLE_FMT_FLTP), 44100,
AV_CH_LAYOUT_STEREO);
fprintf(stderr, "aformat: %s\n", strbuf);
err = avfilter_graph_create_filter(&aformat_ctx, aformat,
NULL, strbuf, NULL, filter_graph);
if (err < 0) {
av_log(NULL, AV_LOG_ERROR, "unable to create aformat filter\n");
return err;
}
// create asetnsamples filter
snprintf(strbuf, sizeof(strbuf),
"n=1024:p=0");
fprintf(stderr, "asetnsamples: %s\n", strbuf);
err = avfilter_graph_create_filter(&asetnsamples_ctx, asetnsamples,
NULL, strbuf, NULL, filter_graph);
if (err < 0) {
av_log(NULL, AV_LOG_ERROR, "unable to create asetnsamples filter\n");
return err;
}
// create abuffersink filter
err = avfilter_graph_create_filter(&abuffersink_ctx, abuffersink,
NULL, NULL, NULL, filter_graph);
if (err < 0) {
av_log(NULL, AV_LOG_ERROR, "unable to create aformat filter\n");
return err;
}
// connect inputs and outputs
if (err >= 0) err = avfilter_link(abuffer_ctx, 0, aformat_ctx, 0);
if (err >= 0) err = avfilter_link(aformat_ctx, 0, asetnsamples_ctx, 0);
if (err >= 0) err = avfilter_link(asetnsamples_ctx, 0, abuffersink_ctx, 0);
if (err < 0) {
av_log(NULL, AV_LOG_ERROR, "error connecting filters\n");
return err;
}
err = avfilter_graph_config(filter_graph, NULL);
if (err < 0) {
av_log(NULL, AV_LOG_ERROR, "error configuring the filter graph\n");
return err;
}
return 0;
}
现在生成的帧有 1024 个样本，但音频仍然不稳定。
不要为每一帧重新创建 SwrContext。它需要将一些数据从一帧传输到下一帧以平滑重新采样帧之间的 "edges"。
开始播放音频时创建一个 SwrContext 并为每一帧调用 swr_convert_frame。
最后，我用 here 的解决方案解决了这个问题。
这是为我的设置创建滤波器的代码（重采样到 44.1KHz）
AVFilterGraph *filter_graph = NULL;
AVFilterContext *buffersrc_ctx = NULL;
AVFilterContext *buffersink_ctx = NULL;
QString filter_description = "aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,asetnsamples=n=1024:p=0";
/**
* Initialize conversion filter */
int initialize_audio_filter(AVStream *inputStream) {
char args[512];
int ret;
AVFilter *buffersrc = avfilter_get_by_name("abuffer");
AVFilter *buffersink = avfilter_get_by_name("abuffersink");
AVFilterInOut *outputs = avfilter_inout_alloc();
AVFilterInOut *inputs = avfilter_inout_alloc();
filter_graph = avfilter_graph_alloc();
const enum AVSampleFormat out_sample_fmts[] = {AV_SAMPLE_FMT_FLTP, AV_SAMPLE_FMT_NONE};
const int64_t out_channel_layouts[] = {AV_CH_LAYOUT_STEREO, -1};
const int out_sample_rates[] = {44100, -1};
snprintf(args, sizeof(args), "time_base=%d/%d:sample_rate=%d:sample_fmt=%s:channel_layout=0x%" PRIx64,
inputStream->codec->time_base.num, inputStream->codec->time_base.den,
inputStream->codec->sample_rate,
av_get_sample_fmt_name(inputStream->codec->sample_fmt),
inputStream->codec->channel_layout);
ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in", args, NULL, filter_graph);
if (ret < 0) {
svsCritical("", QString("Could not create filter graph, error: %1").arg(svsAvErrorToFormattedString(ret)))
return -1;
}
ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out", NULL, NULL, filter_graph);
if (ret < 0) {
svsCritical("", QString("Cannot create buffer sink, error: %1").arg(svsAvErrorToFormattedString(ret)))
return ret;
}
ret = av_opt_set_int_list(buffersink_ctx, "sample_fmts", out_sample_fmts, -1,
AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
svsCritical("", QString("Cannot set output sample format, error: %1").arg(svsAvErrorToFormattedString(ret)))
return ret;
}
ret = av_opt_set_int_list(buffersink_ctx, "channel_layouts", out_channel_layouts, -1,
AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
svsCritical("", QString("Cannot set output channel layout, error: %1").arg(svsAvErrorToFormattedString(ret)))
return ret;
}
ret = av_opt_set_int_list(buffersink_ctx, "sample_rates", out_sample_rates, -1,
AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
svsCritical("", QString("Cannot set output sample rate, error: %1").arg(svsAvErrorToFormattedString(ret)))
return ret;
}
/* Endpoints for the filter graph. */
outputs -> name = av_strdup("in");
outputs -> filter_ctx = buffersrc_ctx;
outputs -> pad_idx = 0;
outputs -> next = NULL;
/* Endpoints for the filter graph. */
inputs -> name = av_strdup("out");
inputs -> filter_ctx = buffersink_ctx;
inputs -> pad_idx = 0;
inputs -> next = NULL;
if ((ret = avfilter_graph_parse_ptr(filter_graph, filter_description.toStdString().c_str(), &inputs, &outputs, NULL)) < 0) {
svsCritical("", QString("Could not add the filter to graph, error: %1").arg(svsAvErrorToFormattedString(ret)))
}
if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0) {
svsCritical("", QString("Could not configure the graph, error: %1").arg(svsAvErrorToFormattedString(ret)))
}
/* Print summary of the sink buffer
* Note: args buffer is reused to store channel layout string */
AVFilterLink *outlink = buffersink_ctx->inputs[0];
av_get_channel_layout_string(args, sizeof(args), -1, outlink->channel_layout);
svsInfo("", QString::asprintf("Output: srate:%dHz fmt:%s chlayout:%s\n",
(int) outlink->sample_rate,
(char *) av_x_if_null(av_get_sample_fmt_name((AVSampleFormat) outlink->format), "?"),
args))
return 0;
}
以及过滤器用法：
AVFrame* resampleAudio(const QString& key, AVFrame *frame) {
/* Push the decoded frame into the filtergraph */
qint32 ret;
ret = av_buffersrc_add_frame_flags(buffersrc_ctx, frame, AV_BUFFERSRC_FLAG_KEEP_REF);
if(ret < 0) {
svsWarning(key, QString("Error adding frame to buffer: %1").arg(svsAvErrorToFormattedString(ret)))
// Delete input frame and return null
av_frame_unref(frame);
return nullptr;
}
AVFrame *resampled_frame = av_frame_alloc();
/* Pull filtered frames from the filtergraph */
ret = av_buffersink_get_frame(buffersink_ctx, resampled_frame);
/* Set the timestamp on the resampled frame */
resampled_frame->best_effort_timestamp = resampled_frame->pts;
if(ret < 0) {
// This is very common. For 48KHz -> 44.1KHz for some input frames the
// filter has not data enough to generate another one.
av_frame_unref(frame);
av_frame_unref(resampled_frame);
return nullptr;
}
av_frame_unref(frame);
return resampled_frame;
}
重要的是在重采样帧上设置 best_effort_timestamp 以使其工作。但是这一帧的PTS是滤镜设置的
我正在尝试使用 libswresample API 将解码后的音频帧从 48KHz 重采样到 44.1KHz。我的代码如下：
// 'frame' is the original decoded audio frame
AVFrame *output_frame = av_frame_alloc();
// Without this, there is no sound at all at the output (PTS stuff I guess)
av_frame_copy_props(output_frame, frame);
output_frame->channel_layout = audioStream->codec->channel_layout;
output_frame->sample_rate = audioStream->codec->sample_rate;
output_frame->format = audioStream->codec->sample_fmt;
SwrContext *swr;
// Configure resampling context
swr = swr_alloc_set_opts(NULL, // we're allocating a new context
AV_CH_LAYOUT_STEREO, // out_ch_layout
AV_SAMPLE_FMT_FLTP, // out_sample_fmt
44100, // out_sample_rate
AV_CH_LAYOUT_STEREO, // in_ch_layout
AV_SAMPLE_FMT_FLTP, // in_sample_fmt
48000, // in_sample_rate
0, // log_offset
NULL); // log_ctx
// Initialize resampling context
swr_init(swr);
// Perform conversion
swr_convert_frame(swr, output_frame, frame);
// Close resampling context
swr_close(swr);
swr_free(&swr);
// Free the original frame and replace it with the new one
av_frame_unref(frame);
return output_frame;
使用此代码，我能够在输出端听到音频，但它也很嘈杂。根据我的阅读，这段没有 av_frame_copy_props() 的代码应该足够了，但由于某种原因它不起作用。有什么想法吗？
编辑：输入流使用AAC对音频进行编码，样本数为1024。但是，转换后，样本数为925。
编辑： 我试着反过来做。由于我的应用程序接收来自任何来源的流，因此一些音频流是 48KHz 而另一些是 44.1KHz。所以我尝试从 44.1 重采样到 48 以避免重采样损失。但是现在每帧都有超过1024个样本，编码失败。
编辑： 我尝试使用 libavfilter 代替以下过滤器链：
int init_filter_graph(AVStream *audio_st) {
// create new graph
filter_graph = avfilter_graph_alloc();
if (!filter_graph) {
av_log(NULL, AV_LOG_ERROR, "unable to create filter graph: out of memory\n");
return -1;
}
AVFilter *abuffer = avfilter_get_by_name("abuffer");
AVFilter *aformat = avfilter_get_by_name("aformat");
AVFilter *asetnsamples = avfilter_get_by_name("asetnsamples");
AVFilter *abuffersink = avfilter_get_by_name("abuffersink");
int err;
// create abuffer filter
AVCodecContext *avctx = audio_st->codec;
AVRational time_base = audio_st->time_base;
snprintf(strbuf, sizeof(strbuf),
"time_base=%d/%d:sample_rate=%d:sample_fmt=%s:channel_layout=0x%" PRIx64,
time_base.num, time_base.den, avctx->sample_rate,
av_get_sample_fmt_name(avctx->sample_fmt),
avctx->channel_layout);
fprintf(stderr, "abuffer: %s\n", strbuf);
err = avfilter_graph_create_filter(&abuffer_ctx, abuffer,
NULL, strbuf, NULL, filter_graph);
if (err < 0) {
av_log(NULL, AV_LOG_ERROR, "error initializing abuffer filter\n");
return err;
}
// create aformat filter
snprintf(strbuf, sizeof(strbuf),
"sample_fmts=%s:sample_rates=%d:channel_layouts=0x%" PRIx64,
av_get_sample_fmt_name(AV_SAMPLE_FMT_FLTP), 44100,
AV_CH_LAYOUT_STEREO);
fprintf(stderr, "aformat: %s\n", strbuf);
err = avfilter_graph_create_filter(&aformat_ctx, aformat,
NULL, strbuf, NULL, filter_graph);
if (err < 0) {
av_log(NULL, AV_LOG_ERROR, "unable to create aformat filter\n");
return err;
}
// create asetnsamples filter
snprintf(strbuf, sizeof(strbuf),
"n=1024:p=0");
fprintf(stderr, "asetnsamples: %s\n", strbuf);
err = avfilter_graph_create_filter(&asetnsamples_ctx, asetnsamples,
NULL, strbuf, NULL, filter_graph);
if (err < 0) {
av_log(NULL, AV_LOG_ERROR, "unable to create asetnsamples filter\n");
return err;
}
// create abuffersink filter
err = avfilter_graph_create_filter(&abuffersink_ctx, abuffersink,
NULL, NULL, NULL, filter_graph);
if (err < 0) {
av_log(NULL, AV_LOG_ERROR, "unable to create aformat filter\n");
return err;
}
// connect inputs and outputs
if (err >= 0) err = avfilter_link(abuffer_ctx, 0, aformat_ctx, 0);
if (err >= 0) err = avfilter_link(aformat_ctx, 0, asetnsamples_ctx, 0);
if (err >= 0) err = avfilter_link(asetnsamples_ctx, 0, abuffersink_ctx, 0);
if (err < 0) {
av_log(NULL, AV_LOG_ERROR, "error connecting filters\n");
return err;
}
err = avfilter_graph_config(filter_graph, NULL);
if (err < 0) {
av_log(NULL, AV_LOG_ERROR, "error configuring the filter graph\n");
return err;
}
return 0;
}
现在生成的帧有 1024 个样本，但音频仍然不稳定。
不要为每一帧重新创建 SwrContext。它需要将一些数据从一帧传输到下一帧以平滑重新采样帧之间的 "edges"。
开始播放音频时创建一个 SwrContext 并为每一帧调用 swr_convert_frame。
最后，我用 here 的解决方案解决了这个问题。
这是为我的设置创建滤波器的代码（重采样到 44.1KHz）
AVFilterGraph *filter_graph = NULL;
AVFilterContext *buffersrc_ctx = NULL;
AVFilterContext *buffersink_ctx = NULL;
QString filter_description = "aresample=44100,aformat=sample_fmts=fltp:channel_layouts=stereo,asetnsamples=n=1024:p=0";
/**
* Initialize conversion filter */
int initialize_audio_filter(AVStream *inputStream) {
char args[512];
int ret;
AVFilter *buffersrc = avfilter_get_by_name("abuffer");
AVFilter *buffersink = avfilter_get_by_name("abuffersink");
AVFilterInOut *outputs = avfilter_inout_alloc();
AVFilterInOut *inputs = avfilter_inout_alloc();
filter_graph = avfilter_graph_alloc();
const enum AVSampleFormat out_sample_fmts[] = {AV_SAMPLE_FMT_FLTP, AV_SAMPLE_FMT_NONE};
const int64_t out_channel_layouts[] = {AV_CH_LAYOUT_STEREO, -1};
const int out_sample_rates[] = {44100, -1};
snprintf(args, sizeof(args), "time_base=%d/%d:sample_rate=%d:sample_fmt=%s:channel_layout=0x%" PRIx64,
inputStream->codec->time_base.num, inputStream->codec->time_base.den,
inputStream->codec->sample_rate,
av_get_sample_fmt_name(inputStream->codec->sample_fmt),
inputStream->codec->channel_layout);
ret = avfilter_graph_create_filter(&buffersrc_ctx, buffersrc, "in", args, NULL, filter_graph);
if (ret < 0) {
svsCritical("", QString("Could not create filter graph, error: %1").arg(svsAvErrorToFormattedString(ret)))
return -1;
}
ret = avfilter_graph_create_filter(&buffersink_ctx, buffersink, "out", NULL, NULL, filter_graph);
if (ret < 0) {
svsCritical("", QString("Cannot create buffer sink, error: %1").arg(svsAvErrorToFormattedString(ret)))
return ret;
}
ret = av_opt_set_int_list(buffersink_ctx, "sample_fmts", out_sample_fmts, -1,
AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
svsCritical("", QString("Cannot set output sample format, error: %1").arg(svsAvErrorToFormattedString(ret)))
return ret;
}
ret = av_opt_set_int_list(buffersink_ctx, "channel_layouts", out_channel_layouts, -1,
AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
svsCritical("", QString("Cannot set output channel layout, error: %1").arg(svsAvErrorToFormattedString(ret)))
return ret;
}
ret = av_opt_set_int_list(buffersink_ctx, "sample_rates", out_sample_rates, -1,
AV_OPT_SEARCH_CHILDREN);
if (ret < 0) {
svsCritical("", QString("Cannot set output sample rate, error: %1").arg(svsAvErrorToFormattedString(ret)))
return ret;
}
/* Endpoints for the filter graph. */
outputs -> name = av_strdup("in");
outputs -> filter_ctx = buffersrc_ctx;
outputs -> pad_idx = 0;
outputs -> next = NULL;
/* Endpoints for the filter graph. */
inputs -> name = av_strdup("out");
inputs -> filter_ctx = buffersink_ctx;
inputs -> pad_idx = 0;
inputs -> next = NULL;
if ((ret = avfilter_graph_parse_ptr(filter_graph, filter_description.toStdString().c_str(), &inputs, &outputs, NULL)) < 0) {
svsCritical("", QString("Could not add the filter to graph, error: %1").arg(svsAvErrorToFormattedString(ret)))
}
if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0) {
svsCritical("", QString("Could not configure the graph, error: %1").arg(svsAvErrorToFormattedString(ret)))
}
/* Print summary of the sink buffer
* Note: args buffer is reused to store channel layout string */
AVFilterLink *outlink = buffersink_ctx->inputs[0];
av_get_channel_layout_string(args, sizeof(args), -1, outlink->channel_layout);
svsInfo("", QString::asprintf("Output: srate:%dHz fmt:%s chlayout:%s\n",
(int) outlink->sample_rate,
(char *) av_x_if_null(av_get_sample_fmt_name((AVSampleFormat) outlink->format), "?"),
args))
return 0;
}
以及过滤器用法：
AVFrame* resampleAudio(const QString& key, AVFrame *frame) {
/* Push the decoded frame into the filtergraph */
qint32 ret;
ret = av_buffersrc_add_frame_flags(buffersrc_ctx, frame, AV_BUFFERSRC_FLAG_KEEP_REF);
if(ret < 0) {
svsWarning(key, QString("Error adding frame to buffer: %1").arg(svsAvErrorToFormattedString(ret)))
// Delete input frame and return null
av_frame_unref(frame);
return nullptr;
}
AVFrame *resampled_frame = av_frame_alloc();
/* Pull filtered frames from the filtergraph */
ret = av_buffersink_get_frame(buffersink_ctx, resampled_frame);
/* Set the timestamp on the resampled frame */
resampled_frame->best_effort_timestamp = resampled_frame->pts;
if(ret < 0) {
// This is very common. For 48KHz -> 44.1KHz for some input frames the
// filter has not data enough to generate another one.
av_frame_unref(frame);
av_frame_unref(resampled_frame);
return nullptr;
}
av_frame_unref(frame);
return resampled_frame;
}
重要的是在重采样帧上设置 best_effort_timestamp 以使其工作。但是这一帧的PTS是滤镜设置的