使用 Unity Render Streaming 设置 WebRTC 的分辨率，以特定分辨率流式传输视频文件

Question

我的计算机上有一个 360 度视频的视频文件，我正在使用 Skybox 将其流式传输到 Unity。视频原始分辨率为1920x960，但当我在Unity中查看时，分辨率明显较低。事实上，我正在使用 WebApp，我修改了双向示例以仅发送视频，并且我正在使用 Unity 渲染流插件中的接收器示例来可视化流。我可以看出分辨率正在根据我当前的带宽进行调整。但这不是我想要的。我想做的是强制流保持 1920x960 的分辨率，但我做不到。

我尝试使用 applyConstraints，将编解码器更改为 h264，将默认分辨率设置为 1920x960，将其添加到分辨率数组中的可能分辨率列表中。但是，它似乎覆盖了我的尝试并强制基于当前带宽的当前分辨率。

ReceiverSample.cs

using System;
using UnityEngine;
using UnityEngine.UI;

namespace Unity.RenderStreaming
{
    public class ReceiverSample : MonoBehaviour
    {
#pragma warning disable 0649
        [SerializeField] private Button startButton;
        [SerializeField] private Button stopButton;
        [SerializeField] private ReceiveVideoViewer receiveVideoViewer;
        [SerializeField] private SingleConnection connection;
        [SerializeField] private Material skyboxMaterial;  
        [SerializeField] private Camera mainCamera;  
        [SerializeField] private Canvas canvas;  
#pragma warning restore 0649

        private string connectionId = "68052";  

        void Awake()
        {
            Texture2D fallbackTexture = Resources.Load<Texture2D>("exemplo"); 
            if (fallbackTexture == null)
            {
                Debug.LogError("Fallback TestImage not found in Resources folder!");
            }

            // Skybox material setup
            if (skyboxMaterial == null)
            {
                skyboxMaterial = Resources.Load<Material>("360Material");
                if (skyboxMaterial == null)
                {
                    Debug.LogError("360Material not found in Resources folder!");
                }
                else
                {
                    Debug.Log("360Material successfully assigned.");
                    skyboxMaterial.SetTexture("_MainTex", fallbackTexture);  
                    RenderSettings.skybox = skyboxMaterial;
                    DynamicGI.UpdateEnvironment();  
                }
            }

            if (mainCamera == null)
            {
                mainCamera = Camera.main;  
                if (mainCamera == null)
                {
                    Debug.LogError("Main camera not found!");
                }
                else
                {
                    Debug.Log("Main camera successfully assigned.");
                }
            }

            receiveVideoViewer.OnUpdateReceiveTexture += texture =>
            {
                if (texture == null)
                {
                    Debug.LogError("No texture received!");
                    return;
                }

                Debug.Log($"Received texture with dimensions: {texture.width}x{texture.height}");

                skyboxMaterial.SetTexture("_MainTex", texture);
                RenderSettings.skybox = skyboxMaterial;
                DynamicGI.UpdateEnvironment();  
                Debug.Log("Skybox material updated with video texture.");

                if (canvas != null)
                {
                    Debug.Log("Switching Canvas to World Space mode.");
                    canvas.renderMode = RenderMode.WorldSpace; 
                    canvas.transform.position = new Vector3(0, 1000, 0);  
                    canvas.transform.localScale = new Vector3(0.1f, 0.1f, 0.1f);  
                    Invoke(nameof(DeactivateCanvas), 2f);  
                }
            };
        }

        void Start()
        {
            mainCamera.transform.position = new Vector3(476f, 238f, -596f);  // Set camera position to X: 476, Y: 238, Z: -596

            OnStart();
        }

        private void OnStart()
        {
            connection.CreateConnection(connectionId, true);  

            startButton?.gameObject.SetActive(false);  
            stopButton?.gameObject.SetActive(true); 
        }

        private void OnStop()
        {
            connection.DeleteConnection(connectionId);
            connectionId = String.Empty;
            startButton?.gameObject.SetActive(true);   
            stopButton?.gameObject.SetActive(false); 
        }

        private void DeactivateCanvas()
        {
            if (canvas != null)
            {
                canvas.gameObject.SetActive(false);
            }
        }
    }
}

main.js

import { SendVideo } from "./sendvideo.js";
import { getServerConfig, getRTCConfiguration } from "../../js/config.js";
import { createDisplayStringArray } from "../../js/stats.js";
import { RenderStreaming } from "../../module/renderstreaming.js";
import { Signaling, WebSocketSignaling } from "../../module/signaling.js";

const defaultStreamWidth = 1920; // Set to custom resolution width
const defaultStreamHeight = 960; // Set to custom resolution height
const streamSizeList = [
  { width: 640, height: 360 },
  { width: 1280, height: 720 },
  { width: 1920, height: 1080 },
  { width: 2560, height: 1440 },
  { width: 3840, height: 2160 },
  { width: 360, height: 640 },
  { width: 720, height: 1280 },
  { width: 1080, height: 1920 },
  { width: 1440, height: 2560 },
  { width: 2160, height: 3840 },
  { width: 1920, height: 960 } // Add custom resolution to the list
];

const localVideo = document.getElementById('localVideo');
const remoteVideo = document.getElementById('remoteVideo');
const localVideoStatsDiv = document.getElementById('localVideoStats');
const remoteVideoStatsDiv = document.getElementById('remoteVideoStats');
const textForConnectionId = document.getElementById('textForConnectionId');
textForConnectionId.value = getRandom();
const videoSelect = document.querySelector('select#videoSource');
const audioSelect = document.querySelector('select#audioSource');
const videoResolutionSelect = document.querySelector('select#videoResolution');
const cameraWidthInput = document.querySelector('input#cameraWidth');
const cameraHeightInput = document.querySelector('input#cameraHeight');

const codecPreferences = document.getElementById('codecPreferences');
const supportsSetCodecPreferences = window.RTCRtpTransceiver &&
  'setCodecPreferences' in window.RTCRtpTransceiver.prototype;
const messageDiv = document.getElementById('message');
messageDiv.style.display = 'none';

let useCustomResolution = true; // Ensure custom resolution is used

setUpInputSelect();
showCodecSelect();

/** @type {SendVideo} */
let sendVideo = new SendVideo(localVideo, remoteVideo);
/** @type {RenderStreaming} */
let renderstreaming;
let useWebSocket;
let connectionId;

const startButton = document.getElementById('startVideoButton');
startButton.addEventListener('click', startVideo);
const setupButton = document.getElementById('setUpButton');
setupButton.addEventListener('click', setUp);
const hangUpButton = document.getElementById('hangUpButton');
hangUpButton.addEventListener('click', hangUp);

window.addEventListener('beforeunload', async () => {
  if (!renderstreaming)
    return;
  await renderstreaming.stop();
}, true);

setupConfig();

async function setupConfig() {
  const res = await getServerConfig();
  useWebSocket = res.useWebSocket;
  showWarningIfNeeded(res.startupMode);
}

function showWarningIfNeeded(startupMode) {
  const warningDiv = document.getElementById("warning");
  if (startupMode == "public") {
    warningDiv.innerHTML = "<h4>Warning</h4> This sample is not working on Public Mode.";
    warningDiv.hidden = false;
  }
}

async function startVideo() {
  startButton.disabled = true;

  await sendVideo.startLocalVideo();

  // Set custom resolution for video streaming
  cameraWidthInput.value = defaultStreamWidth;
  cameraHeightInput.value = defaultStreamHeight;
  cameraWidthInput.disabled = false;
  cameraHeightInput.disabled = false;
  useCustomResolution = true;

  // Enable setup button after initializing local video
  setupButton.disabled = false;
}

async function setUp() {
  setupButton.disabled = true;
  hangUpButton.disabled = false;
  connectionId = textForConnectionId.value;
  codecPreferences.disabled = true;

  const signaling = useWebSocket ? new WebSocketSignaling() : new Signaling();
  const config = getRTCConfiguration();
  renderstreaming = new RenderStreaming(signaling, config);
  renderstreaming.onConnect = () => {
    const tracks = sendVideo.getLocalTracks();
    for (const track of tracks) {
      renderstreaming.addTransceiver(track, { direction: 'sendonly' });
    }
    setCodecPreferences();
    showStatsMessage();
  };
  renderstreaming.onDisconnect = () => {
    hangUp();
  };

  await renderstreaming.start();
  await renderstreaming.createConnection(connectionId);
}

function setCodecPreferences() {
  /** @type {RTCRtpCodecCapability[] | null} */
  let selectedCodecs = null;
  if (supportsSetCodecPreferences) {
    const preferredCodec = codecPreferences.options[codecPreferences.selectedIndex];
    if (preferredCodec.value !== '') {
      const [mimeType, sdpFmtpLine] = preferredCodec.value.split(' ');
      const { codecs } = RTCRtpSender.getCapabilities('video');
      const selectedCodecIndex = codecs.findIndex(c => c.mimeType === mimeType && c.sdpFmtpLine === sdpFmtpLine);
      const selectCodec = codecs[selectedCodecIndex];
      selectedCodecs = [selectCodec];
    }
  }

  if (selectedCodecs == null) {
    return;
  }
  const transceivers = renderstreaming.getTransceivers().filter(t => t.receiver.track.kind == "video");
  if (transceivers && transceivers.length > 0) {
    transceivers.forEach(t => t.setCodecPreferences(selectedCodecs));
  }
}

async function hangUp() {
  clearStatsMessage();
  messageDiv.style.display = 'block';
  messageDiv.innerText = `Disconnect peer on ${connectionId}.`;

  hangUpButton.disabled = true;
  setupButton.disabled = false;
  await renderstreaming.deleteConnection();
  await renderstreaming.stop();
  renderstreaming = null;
  remoteVideo.srcObject = null;

  textForConnectionId.value = getRandom();
  connectionId = null;
  if (supportsSetCodecPreferences) {
    codecPreferences.disabled = false;
  }
}

function getRandom() {
  const max = 99999;
  const length = String(max).length;
  const number = Math.floor(Math.random() * max);
  return (Array(length).join('0') + number).slice(-length);
}

async function setUpInputSelect() {
  const deviceInfos = await navigator.mediaDevices.enumerateDevices();

  for (let i = 0; i !== deviceInfos.length; ++i) {
    const deviceInfo = deviceInfos[i];
    if (deviceInfo.kind === 'videoinput') {
      const option = document.createElement('option');
      option.value = deviceInfo.deviceId;
      option.text = deviceInfo.label || `camera ${videoSelect.length + 1}`;
      videoSelect.appendChild(option);
    } else if (deviceInfo.kind === 'audioinput') {
      const option = document.createElement('option');
      option.value = deviceInfo.deviceId;
      option.text = deviceInfo.label || `mic ${audioSelect.length + 1}`;
      audioSelect.appendChild(option);
    }
  }

  for (let i = 0; i < streamSizeList.length; i++) {
    const streamSize = streamSizeList[i];
    const option = document.createElement('option');
    option.value = i;
    option.text = `${streamSize.width} x ${streamSize.height}`;
    videoResolutionSelect.appendChild(option);
  }

  const option = document.createElement('option');
  option.value = streamSizeList.length;
  option.text = 'Custom';
  videoResolutionSelect.appendChild(option);
  videoResolutionSelect.value = streamSizeList.length - 1; // Default to custom resolution

  videoResolutionSelect.addEventListener('change', (event) => {
    const isCustom = event.target.value >= streamSizeList.length;
    cameraWidthInput.disabled = !isCustom;
    cameraHeightInput.disabled = !isCustom;
    useCustomResolution = isCustom;
  });
}

function showCodecSelect() {
  if (!supportsSetCodecPreferences) {
    messageDiv.style.display = 'block';
    messageDiv.innerHTML = `Current Browser does not support <a href="https://developer.mozilla.org/en-US/docs/Web/API/RTCRtpTransceiver/setCodecPreferences">RTCRtpTransceiver.setCodecPreferences</a>.`;
    return;
  }

  const codecs = RTCRtpSender.getCapabilities('video').codecs;
  codecs.forEach(codec => {
    if (['video/red', 'video/ulpfec', 'video/rtx'].includes(codec.mimeType)) {
      return;
    }
    const option = document.createElement('option');
    option.value = (codec.mimeType + ' ' + (codec.sdpFmtpLine || '')).trim();
    option.innerText = option.value;
    codecPreferences.appendChild(option);
  });
  codecPreferences.disabled = false;
}

let lastStats;
let intervalId;

function showStatsMessage() {
  intervalId = setInterval(async () => {
    if (localVideo.videoWidth) {
      localVideoStatsDiv.innerHTML = `<strong>Sending resolution:</strong> ${localVideo.videoWidth} x ${localVideo.videoHeight} px`;
    }
    if (remoteVideo.videoWidth) {
      remoteVideoStatsDiv.innerHTML = `<strong>Receiving resolution:</strong> ${remoteVideo.videoWidth} x ${remoteVideo.videoHeight} px`;
    }

    if (renderstreaming == null || connectionId == null) {
      return;
    }

    const stats = await renderstreaming.getStats();
    if (stats == null) {
      return;
    }

    const array = createDisplayStringArray(stats, lastStats);
    if (array.length) {
      messageDiv.style.display = 'block';
      messageDiv.innerHTML = array.join('<br>');
    }
    lastStats = stats;
  }, 1000);
}

function clearStatsMessage() {
  if (intervalId) {
    clearInterval(intervalId);
  }
  lastStats = null;
  intervalId = null;
  localVideoStatsDiv.innerHTML = '';
  remoteVideoStatsDiv.innerHTML = '';
  messageDiv.style.display = 'none';
  messageDiv.innerHTML = '';
}

发送视频.js

import * as Logger from "../../module/logger.js";

export class SendVideo {
  constructor(localVideoElement, remoteVideoElement) {
    this.localVideo = localVideoElement;
    this.remoteVideo = remoteVideoElement;
    this.peerConnection = null;
  }

  async startLocalVideo() {
    try {
      const videoElement = document.createElement('video');
      videoElement.src = '/videos/video.mp4'; // Path to your video file
      videoElement.muted = true;
      await videoElement.play();

      const videoConstraints = {
        video: {
          width: { exact: 1920 },  // Force width to 1920
          height: { exact: 960 },  // Force height to 960
          frameRate: { ideal: 30 }, // Optional: Set frame rate target to 30 FPS
        }
      };

      const stream = videoElement.captureStream();
      this.localVideo.srcObject = stream;

      // Apply the video constraints to the tracks
      const videoTrack = stream.getVideoTracks()[0];
      await videoTrack.applyConstraints(videoConstraints);

      await this.localVideo.play();

      // Initialize WebRTC connection and data channel
      this.initializePeerConnection();

    } catch (err) {
      Logger.error(`Error starting local video: ${err}`);
    }
  }

  // Set up WebRTC connection and DataChannel
  initializePeerConnection() {
    this.peerConnection = new RTCPeerConnection();

    // Create a DataChannel (optional)
    const dataChannel = this.peerConnection.createDataChannel("myDataChannel");
    dataChannel.onopen = () => {
      console.log("DataChannel open: Connection established with Unity client");
    };
    dataChannel.onclose = () => {
      console.log("DataChannel closed");
    };

    // Set up connection state change event
    this.peerConnection.onconnectionstatechange = () => {
      if (this.peerConnection.connectionState === 'connected') {
        console.log("WebRTC connection established with Unity");
      }
    };

    // Add the video track to the peer connection
    const videoTrack = this.localVideo.srcObject.getVideoTracks()[0];
    const sender = this.peerConnection.addTrack(videoTrack);

    // Force H.264 codec by modifying transceiver settings
    const transceivers = this.peerConnection.getTransceivers();
    transceivers.forEach(transceiver => {
      if (transceiver.sender.track.kind === 'video') {
        const capabilities = RTCRtpSender.getCapabilities('video');
        const h264Codec = capabilities.codecs.find(codec => codec.mimeType === 'video/H264');
        if (h264Codec) {
          transceiver.setCodecPreferences([h264Codec]);
          console.log("H.264 codec has been set for the video stream.");
        } else {
          console.log("H.264 codec is not supported.");
        }
      }
    });

    // Create an SDP offer, modify to force H.264 and set resolution constraints
    this.peerConnection.createOffer().then(offer => {
      let sdp = offer.sdp;

      // Remove VP8 and VP9 codecs to force H.264
      sdp = sdp.replace(/a=rtpmap:\d+ VP8\/90000\n/g, '');
      sdp = sdp.replace(/a=rtpmap:\d+ VP9\/90000\n/g, '');

      // Enforce the resolution constraint for 224x224 in SDP
      sdp = sdp.replace(/a=mid:video\n/g, 'a=mid:video\nb=AS:500\n'); // Adjust bitrate as needed for 224x224

      // Apply the modified SDP offer
      offer.sdp = sdp;
      return this.peerConnection.setLocalDescription(offer);
    }).then(() => {
      console.log("Offer created with H.264 codec and resolution constraints");
    }).catch(err => console.error("Error creating offer:", err));
  }

  getLocalTracks() {
    return this.localVideo.srcObject.getTracks();
  }

  addRemoteTrack(track) {
    if (this.remoteVideo.srcObject == null) {
      this.remoteVideo.srcObject = new MediaStream();
    }
    this.remoteVideo.srcObject.addTrack(track);
  }
}

我想知道如何将分辨率和/或其他流媒体参数强制为我想要的，或者我到底需要在哪里调整这些参数。

Answer 1

我使用默认编解码器来传输视频，对于 1080p 视频，我需要使用 VP9 或 h256