FFMPEG C++/Kotlin 将 ByteArray (ByteBuffer) 中的数据编码为字幕并将其与视频混合

Question

我需要将一些编码为 base64 字符串的字节数据混合到 mpeg-ts 容器中。找不到任何解决方案来将此数据编码为 AVPacket。使用“avcodec_encode_subtitle”假设 AVSubtitle 具有正确的数据。但是我如何用我的数据创建/分配 AVSubtitle 呢？

更新。我设法创建的唯一变体（在 Kotlin 中，因为我的应用程序是 KMP）是这样的，但它在编码时给出错误“处理输入时发现无效数据”

val data = AVSubtitleRect()
data.type(SUBTITLE_ASS)
data.ass(BytePointer(buffer))
val subtitle = AVSubtitle()
val pointerPointer = PointerPointer<AVSubtitleRect>(1)
pointerPointer.put(data)
subtitle.rects(pointerPointer)
subtitle.pts(packet.pts())

val bufferSize = 1024 * 1024
val encodedBuffer = BytePointer(av_malloc(bufferSize.toLong()))

val result = avcodec_encode_subtitle(subtitleContext, encodedBuffer, bufferSize, subtitle)

if (result >= 0) {
    val outSubtitlePacket = AVPacket()
    outSubtitlePacket.apply {
        data(encodedBuffer)
        outSubtitlePacket.size(result)
        outSubtitlePacket.stream_index(subtitleStreamIndex)
        duration(packet.duration())
        dts(packet.dts())
        pts(packet.pts())
        pos(packet.pos())
    }

    av_interleaved_write_frame(avOutputCtx, outSubtitlePacket)

    av_packet_unref(outSubtitlePacket)
    av_free(encodedBuffer)
}

C 等值

// Allocate the subtitle rect
AVSubtitleRect *data = av_mallocz(sizeof(AVSubtitleRect));
if (!data) {
    fprintf(stderr, "Could not allocate AVSubtitleRect.\n");
    return AVERROR(ENOMEM);
}
data->type = SUBTITLE_ASS;
data->ass = av_strdup(buffer);
if (!data->ass) {
    fprintf(stderr, "Could not allocate ASS buffer.\n");
    av_free(data);
    return AVERROR(ENOMEM);
}

// Allocate the subtitle
AVSubtitle subtitle;
memset(&subtitle, 0, sizeof(subtitle));
subtitle.rects = av_mallocz(sizeof(*subtitle.rects));
if (!subtitle.rects) {
    fprintf(stderr, "Could not allocate AVSubtitle rects.\n");
    av_free(data->ass);
    av_free(data);
    return AVERROR(ENOMEM);
}
subtitle.rects[0] = data;
subtitle.num_rects = 1;
subtitle.pts = packet->pts;

// Allocate buffer for encoded subtitle
int bufferSize = 1024 * 1024;
uint8_t *encodedBuffer = av_malloc(bufferSize);
if (!encodedBuffer) {
    fprintf(stderr, "Could not allocate buffer for encoded subtitle.\n");
    av_free(subtitle.rects);
    av_free(data->ass);
    av_free(data);
    return AVERROR(ENOMEM);
}

// Encode the subtitle
int result = avcodec_encode_subtitle(subtitleContext, encodedBuffer, bufferSize, &subtitle);
if (result >= 0) {
    // Create the output subtitle packet
    AVPacket outSubtitlePacket;
    av_init_packet(&outSubtitlePacket);
    outSubtitlePacket.data = encodedBuffer;
    outSubtitlePacket.size = result;
    outSubtitlePacket.stream_index = subtitleStreamIndex;
    outSubtitlePacket.duration = packet->duration;
    outSubtitlePacket.dts = packet->dts;
    outSubtitlePacket.pts = packet->pts;
    outSubtitlePacket.pos = packet->pos;

    // Write the subtitle packet
    result = av_interleaved_write_frame(avOutputCtx, &outSubtitlePacket);
    if (result < 0) {
        fprintf(stderr, "Error while writing subtitle packet: %s\n", av_err2str(result));
    }

    av_packet_unref(&outSubtitlePacket);
    av_free(encodedBuffer);
} else {
    fprintf(stderr, "Failed to encode subtitles. Reason: %s\n", av_err2str(result));
    av_free(encodedBuffer);
}

av_free(subtitle.rects);
av_free(data->ass);
av_free(data);

Answer 1

我已经成功做到了这一点。但我在调用“av_interleaved_write_frame”时遇到访问冲突异常

if (!subtitleCodec)
    subtitleCodec = avcodec_find_encoder(AV_CODEC_ID_ASS);

if (!subtitleStream)
    subtitleStream = avformat_new_stream(avOutputCtxRec, subtitleCodec);
if (!subtitleStream) {
    std::cout << "Could not allocate subtitle stream" << std::endl;
    return;
}

if (!subtitleCtx) {
    subtitleCtx = avcodec_alloc_context3(subtitleCodec);
    subtitleCtx->codec_id = AV_CODEC_ID_ASS;
    subtitleCtx->codec_type = AVMEDIA_TYPE_SUBTITLE;
    subtitleCtx->width = width;
    subtitleCtx->height = height;
    subtitleCtx->framerate = av_make_q(1000, 1);
    subtitleCtx->time_base = av_make_q(1, 1000);
}

subtitleStream->id = 1;
subtitleStream->codecpar->codec_id = AV_CODEC_ID_ASS;
subtitleStream->codecpar->codec_type = AVMEDIA_TYPE_SUBTITLE;
subtitleStream->codecpar->width = width;
subtitleStream->codecpar->height = height;
subtitleStream->codecpar->framerate = av_make_q(1000, 1);
subtitleStream->time_base = av_make_q(1, 1000);
subtitleStream->avg_frame_rate = av_make_q(1000, 1);

if (avOutputCtxRec->oformat->flags & AVFMT_GLOBALHEADER)
    subtitleCtx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;

if (avcodec_open2(subtitleCtx, subtitleCodec, NULL) < 0) {
    std::cout << "Could not open subtitle codec" << std::endl;
    return;
}

if (avcodec_parameters_from_context(subtitleStream->codecpar, subtitleCtx) < 0) {
    std::cout << "Could not copy codec parameters" << std::endl;
    return;
}

AVSubtitle subtitle;
memset(&subtitle, 0, sizeof(AVSubtitle));

subtitle.num_rects = 1;
subtitle.rects = (AVSubtitleRect**)av_mallocz(sizeof(AVSubtitleRect*));
subtitle.rects[0] = (AVSubtitleRect*)av_mallocz(sizeof(AVSubtitleRect));

AVSubtitleRect* rect = subtitle.rects[0];
rect->type = SUBTITLE_ASS;
rect->ass = av_strdup(text.c_str());

subtitle.start_display_time = 0;
subtitle.end_display_time = 0;

auto now = std::chrono::high_resolution_clock::now();
auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
if (subtitleTimeStamp == 0)
    subtitleTimeStamp = millis;
subtitle.pts = millis - subtitleTimeStamp;

int buf_size = 1024 * 1024;
uint8_t* buf = (uint8_t*)av_malloc(buf_size);
if (!buf) {
    av_freep(&subtitle.rects[0]->ass);
    av_freep(&subtitle.rects[0]);
    av_freep(&subtitle.rects);
    std::cout << "Could not allocate buffer" << std::endl;
    return;
}

int encoded_size = avcodec_encode_subtitle(subtitleCtx, buf, buf_size, &subtitle);
if (encoded_size < 0) {
    av_free(buf);
    av_freep(&subtitle.rects[0]->ass);
    av_freep(&subtitle.rects[0]);
    av_freep(&subtitle.rects);
    std::cout << "Failed to encode subtitle" << std::endl;
    return;
}

AVPacket* pkt = av_packet_alloc();
if (av_packet_from_data(pkt, buf, encoded_size) < 0) {
    av_free(buf);
    av_packet_free(&pkt);
    std::cout << "Error setting packet data" << std::endl;
    return;
}
pkt->stream_index = subtitleStream->index;
pkt->pts = av_rescale_q(subtitle.pts, subtitleCtx->time_base, subtitleStream->time_base);
pkt->dts = pkt->pts;
pkt->duration = 0;

if (av_interleaved_write_frame(avOutputCtxRec, pkt) < 0) {
    std::cout << "Error writing subtitle packet" << std::endl;
}

av_packet_unref(pkt);
av_packet_free(&pkt);
av_freep(&subtitle.rects[0]->ass);
av_freep(&subtitle.rects[0]);
av_freep(&subtitle.rects);
std::cout << "Subtitle written" << std::endl;