最近，我一直在尝试使用 C# 和一个名为 Sipsorcery 的库来实现 60 fps 低延迟屏幕共享。到目前为止，我已尽最大努力最大化帧数，但只能达到 20 fps 左右。
我尝试看看是编码还是截图的问题。我使用 Visual Studio CPU 使用情况分析器，发现它主要是编码器。尽管如此，我还是重新做了截图功能并使用了SharpDX；我注意到性能略有提高，但没有接近 60 fps 这让我相信编码器是问题所在。也许它可以使用 FFmpeg 或某些 GPU 编码。如果我输入的详细信息不正确，请提前抱歉，因为我对视频编码和解码还比较陌生。
这是我到目前为止所做的：
//VideoTestPatternSource.cs
using System;
using System.Collections.Generic;
using System.Drawing.Imaging;
using System.Drawing;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using Microsoft.Extensions.Logging;
using SIPSorceryMedia.Abstractions;
using SharpDX;
using SharpDX.Direct3D11;
using SharpDX.DXGI;
using Device = SharpDX.Direct3D11.Device;
using MapFlags = SharpDX.Direct3D11.MapFlags;
using System.Runtime.InteropServices;
namespace SIPSorcery.Media
{
public class VideoSource : IVideoSource, IDisposable
{
private const int VIDEO_SAMPLING_RATE = 90000;
private const int MAXIMUM_FRAMES_PER_SECOND = 20;
private const int DEFAULT_FRAMES_PER_SECOND = MAXIMUM_FRAMES_PER_SECOND; // Changed from 30 to 60
private const int MINIMUM_FRAMES_PER_SECOND = MAXIMUM_FRAMES_PER_SECOND - 5;
private const int TIMER_DISPOSE_WAIT_MILLISECONDS = 1000;
private const int VP8_SUGGESTED_FORMAT_ID = 96;
private const int H264_SUGGESTED_FORMAT_ID = 100;
public static readonly List<VideoFormat> SupportedFormats = new List<VideoFormat>
{
new VideoFormat(VideoCodecsEnum.VP8, VP8_SUGGESTED_FORMAT_ID, VIDEO_SAMPLING_RATE),
new VideoFormat(VideoCodecsEnum.H264, H264_SUGGESTED_FORMAT_ID, VIDEO_SAMPLING_RATE, "packetization-mode=1")
};
private int _frameSpacing;
private System.Threading.Timer _sendTestPatternTimer;
private bool _isStarted;
private bool _isPaused;
private bool _isClosed;
private bool _isMaxFrameRate;
private int _frameCount;
private SIPSorceryMedia.Abstractions.IVideoEncoder _videoEncoder;
private MediaFormatManager<VideoFormat> _formatManager;
public event RawVideoSampleDelegate OnVideoSourceRawSample;
#pragma warning disable CS0067
public event RawVideoSampleFasterDelegate OnVideoSourceRawSampleFaster;
#pragma warning restore CS0067
public event EncodedSampleDelegate OnVideoSourceEncodedSample;
public event SourceErrorDelegate OnVideoSourceError;
private readonly int _screenWidth;
private readonly int _screenHeight;
private Factory1 _factory;
private Adapter1 _adapter;
private Device _device;
private Output1 _output1;
private OutputDuplication _duplicatedOutput;
private Texture2D _screenTexture;
public VideoSource(int width, int height, SIPSorceryMedia.Abstractions.IVideoEncoder encoder = null)
{
_screenWidth = width;
_screenHeight = height;
if (encoder != null)
{
_videoEncoder = encoder;
_formatManager = new MediaFormatManager<VideoFormat>(SupportedFormats);
}
try
{
InitializeDirectX();
_sendTestPatternTimer = new System.Threading.Timer(GeneratePattern, null, Timeout.Infinite, Timeout.Infinite);
_frameSpacing = 1000 / DEFAULT_FRAMES_PER_SECOND;
}
catch (Exception ex)
{
MessageBox.Show($"Failed to initialize: {ex.Message}");
}
}
private void InitializeDirectX()
{
_factory = new Factory1();
_adapter = _factory.GetAdapter1(0);
_device = new Device(_adapter);
var output = _adapter.GetOutput(0);
_output1 = output.QueryInterface<Output1>();
var textureDesc = new Texture2DDescription
{
CpuAccessFlags = CpuAccessFlags.Read,
BindFlags = BindFlags.None,
Format = Format.B8G8R8A8_UNorm,
Width = _screenWidth,
Height = _screenHeight,
OptionFlags = ResourceOptionFlags.None,
MipLevels = 1,
ArraySize = 1,
SampleDescription = { Count = 1, Quality = 0 },
Usage = ResourceUsage.Staging
};
_screenTexture = new Texture2D(_device, textureDesc);
_duplicatedOutput = _output1.DuplicateOutput(_device);
}
public void RestrictFormats(Func<VideoFormat, bool> filter) => _formatManager.RestrictFormats(filter);
public List<VideoFormat> GetVideoSourceFormats() => _formatManager.GetSourceFormats();
public void SetVideoSourceFormat(VideoFormat videoFormat) => _formatManager.SetSelectedFormat(videoFormat);
public List<VideoFormat> GetVideoSinkFormats() => _formatManager.GetSourceFormats();
public void SetVideoSinkFormat(VideoFormat videoFormat) => _formatManager.SetSelectedFormat(videoFormat);
public void ForceKeyFrame() => _videoEncoder?.ForceKeyFrame();
public bool HasEncodedVideoSubscribers() => OnVideoSourceEncodedSample != null;
public void ExternalVideoSourceRawSample(uint durationMilliseconds, int width, int height, byte[] sample, VideoPixelFormatsEnum pixelFormat) =>
throw new NotImplementedException("The test pattern video source does not offer any encoding services for external sources.");
public void ExternalVideoSourceRawSampleFaster(uint durationMilliseconds, RawImage rawImage) =>
throw new NotImplementedException("The test pattern video source does not offer any encoding services for external sources.");
public Task<bool> InitialiseVideoSourceDevice() =>
throw new NotImplementedException("The test pattern video source does not use a device.");
public bool IsVideoSourcePaused() => _isPaused;
public void SetFrameRate(int framesPerSecond)
{
if (framesPerSecond < MINIMUM_FRAMES_PER_SECOND || framesPerSecond > MAXIMUM_FRAMES_PER_SECOND)
{
MessageBox.Show($"Frames per second not in the allowed range of {MINIMUM_FRAMES_PER_SECOND} to {MAXIMUM_FRAMES_PER_SECOND}, ignoring.");
}
else
{
_frameSpacing = 1000 / framesPerSecond;
if (_isStarted)
{
_sendTestPatternTimer?.Change(0, _frameSpacing);
}
}
}
public Task PauseVideo()
{
_isPaused = true;
_sendTestPatternTimer?.Change(Timeout.Infinite, Timeout.Infinite);
return Task.CompletedTask;
}
public Task ResumeVideo()
{
_isPaused = false;
_sendTestPatternTimer?.Change(0, _frameSpacing);
return Task.CompletedTask;
}
public Task StartVideo()
{
if (!_isStarted)
{
_isStarted = true;
if (_isMaxFrameRate)
{
GenerateMaxFrames();
}
else
{
try
{
_sendTestPatternTimer?.Change(0, _frameSpacing);
}
catch (Exception ex)
{
MessageBox.Show("The following error occured: " + ex);
}
}
}
return Task.CompletedTask;
}
public Task CloseVideo()
{
if (!_isClosed)
{
_isClosed = true;
ManualResetEventSlim mre = new ManualResetEventSlim();
_sendTestPatternTimer?.Dispose(mre.WaitHandle);
return Task.Run(() => mre.Wait(TIMER_DISPOSE_WAIT_MILLISECONDS));
}
return Task.CompletedTask;
}
private void GenerateMaxFrames()
{
DateTime lastGenerateTime = DateTime.Now;
while (!_isClosed && _isMaxFrameRate)
{
_frameSpacing = Convert.ToInt32(DateTime.Now.Subtract(lastGenerateTime).TotalMilliseconds);
GeneratePattern(null);
lastGenerateTime = DateTime.Now;
}
}
private void GeneratePattern(object state)
{
lock (_sendTestPatternTimer)
{
if (!_isClosed && (OnVideoSourceRawSample != null || OnVideoSourceEncodedSample != null))
{
_frameCount++;
var buffer = Snapshot(_screenWidth, _screenHeight);
if (OnVideoSourceRawSample != null)
{
OnVideoSourceRawSample?.Invoke((uint)_frameSpacing, _screenWidth, _screenHeight, buffer, VideoPixelFormatsEnum.Bgra);
}
if (_videoEncoder != null && OnVideoSourceEncodedSample != null && !_formatManager.SelectedFormat.IsEmpty())
{
var encodedBuffer = _videoEncoder.EncodeVideo(_screenWidth, _screenHeight, buffer, VideoPixelFormatsEnum.Bgra, _formatManager.SelectedFormat.Codec);
if (encodedBuffer != null)
{
uint fps = (_frameSpacing > 0) ? 1000 / (uint)_frameSpacing : MAXIMUM_FRAMES_PER_SECOND;
uint durationRtpTS = VIDEO_SAMPLING_RATE / fps;
OnVideoSourceEncodedSample.Invoke(durationRtpTS, encodedBuffer);
}
}
if (_frameCount == int.MaxValue)
{
_frameCount = 0;
}
}
}
}
private byte[] Snapshot(int width, int height)
{
SharpDX.DXGI.Resource screenResource;
OutputDuplicateFrameInformation duplicateFrameInformation;
try
{
_duplicatedOutput.AcquireNextFrame(10000, out duplicateFrameInformation, out screenResource);
using (var screenTexture2D = screenResource.QueryInterface<Texture2D>())
_device.ImmediateContext.CopyResource(screenTexture2D, _screenTexture);
var mapSource = _device.ImmediateContext.MapSubresource(_screenTexture, 0, MapMode.Read, MapFlags.None);
int stride = width * 4; // 4 bytes per pixel (RGBA)
byte[] buffer = new byte[stride * height];
IntPtr sourcePtr = mapSource.DataPointer;
for (int y = 0; y < height; y++)
{
// Copy each row from source to buffer
Marshal.Copy(sourcePtr, buffer, y * stride, stride);
sourcePtr = IntPtr.Add(sourcePtr, mapSource.RowPitch);
}
_device.ImmediateContext.UnmapSubresource(_screenTexture, 0);
screenResource.Dispose();
_duplicatedOutput.ReleaseFrame();
// Correct pixel inversion (swap R and B channels)
for (int i = 0; i < buffer.Length; i += 4)
{
byte temp = buffer[i]; // Save R
buffer[i] = buffer[i + 2]; // Swap R (buffer[i]) with B (buffer[i+2])
buffer[i + 2] = temp; // Swap B with R (saved R)
// buffer[i+1] (G) remains unchanged
// buffer[i+3] (A) remains unchanged
}
return buffer;
}
catch (SharpDXException e)
{
if (e.ResultCode.Code != SharpDX.DXGI.ResultCode.WaitTimeout.Result.Code)
{
throw;
}
return new byte[width * height * 4];
}
}
public void Dispose()
{
_isClosed = true;
_sendTestPatternTimer?.Dispose();
_videoEncoder?.Dispose();
_screenTexture?.Dispose();
_duplicatedOutput?.Dispose();
_output1?.Dispose();
_device?.Dispose();
_adapter?.Dispose();
_factory?.Dispose();
}
}
}
以下是加载内容以供参考：
//Program.cs
using System;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using SIPSorcery.Media;
using SIPSorcery.Net;
using SIPSorceryMedia.Encoders;
using SIPSorceryMedia.FFmpeg;
using WebSocketSharp.Server;
namespace TESTPATTERNSERVER
{
class Program
{
private const int WEBSOCKET_PORT = 8081;
static void Main()
{
Console.WriteLine("WebRTC Get Started");
// Start web socket.
Console.WriteLine("Starting web socket server...");
var webSocketServer = new WebSocketServer(IPAddress.Any, WEBSOCKET_PORT);
webSocketServer.AddWebSocketService<WebRTCWebSocketPeer>("/", (peer) => peer.CreatePeerConnection = () => CreatePeerConnection());
webSocketServer.Start();
Console.WriteLine($"Waiting for web socket connections on {webSocketServer.Address}:{webSocketServer.Port}...");
Console.WriteLine("Press any key exit.");
Console.ReadLine();
}
private static Task<RTCPeerConnection> CreatePeerConnection()
{
var pc = new RTCPeerConnection(null);
var testPatternSource = new VideoSource(1920, 1080, new VP8VideoEncoder());
var videoEndPoint = new SIPSorceryMedia.FFmpeg.FFmpegVideoEndPoint();
MediaStreamTrack videoTrack = new MediaStreamTrack(videoEndPoint.GetVideoSourceFormats(), MediaStreamStatusEnum.SendOnly);
pc.addTrack(videoTrack);
testPatternSource.OnVideoSourceEncodedSample += pc.SendVideo;
pc.OnVideoFormatsNegotiated += (formats) => testPatternSource.SetVideoSourceFormat(formats.First());
pc.onconnectionstatechange += async (state) =>
{
Console.WriteLine($"Peer connection state change to {state}.");
switch (state)
{
case RTCPeerConnectionState.connected:
await testPatternSource.StartVideo();
break;
case RTCPeerConnectionState.failed:
pc.Close("ice disconnection");
break;
case RTCPeerConnectionState.closed:
await testPatternSource.CloseVideo();
testPatternSource.Dispose();
break;
}
};
return Task.FromResult(pc);
}
}
}
提前致谢！
这主要是关于低延迟使用的视频编解码器的讨论，因为我不熟悉 WebRTC。
低延迟和高效率很难结合起来。 h264、VP8 等高效视频编解码器使用运动矢量和许多其他技巧来利用帧间相似性，称为预测帧。但这需要帧缓冲区才能工作。这还要求每一帧都完好无损且按顺序到达。因此，任何丢失的数据包都需要重新发送，从而增加延迟。时不时还会出现完全编码的帧，这可能会由于帧大小不均匀而导致卡顿。
因此，获得低延迟的最简单选择是跳过所有预测帧内容并将每个帧编码为完整帧。 IE。 MJPEG 或同等格式。理想情况下使用高性能库进行编码/解码。
当然有更好的选择，这篇文章深入探讨了有助于减少延迟的各种 h264 设置。但 h264 比 jpeg 复杂得多，因此除非您可以使用硬件编码器，否则可能性能不佳。
但是大多数视频编解码器以及 Jpeg 都是针对现实世界的图像进行调整的。如果您打算共享桌面应用程序的视频，可能有专门的编码可以更好地处理文本和大型统一区域等内容。但最好的编码器可能是专有的或正在申请专利，因为能够有效地传输视频需要相当多的资金。