发送到 websocket 的消息未使用 TwiML 双向调用进行播放

Question

我目前正在编写一个使用 Google Cloud Speech (TTS) 创建音频内容的脚本，我使用的编码是 LINEAR16。然后，我使用 WaveFile 库将结果转换为 mulaw/8000 base64，然后最终将 base64 编码结果发送到我的 websocket 服务器。

但是，由于某种原因，当我发送媒体消息时，通话期间没有播放任何内容（甚至没有噪音）。

我采取了以下步骤：

1。我用 NodeJS 创建了 websocket 服务器：

const WebSocket = require("ws");
const wss = new WebSocket.Server({ port: 8080 });

const path = require("path");

wss.on("connection", function connection(ws) {
    console.log("New Connection Initiated");


    ws.on("message", function incoming(message) {
        let msg;
        try {
            msg = JSON.parse(message);
        } catch (e) {
            console.error('Error parsing message:', e);
            return;
        }
        switch (msg.event) {
            case "connected":
                console.log(`A new call has connected.`);
                break;
            case "start":
                console.log(`Starting Media Stream ${msg.streamSid}`);
                break;
            case "media":
                // Write Media Packets to the recognize stream
                if (msg.test == 1)
                    console.log(msg)
                break;
            case "stop":
                console.log(`Call Has Ended`);
                break;
        }
    });
});

console.log("Listening on Port 8080");

2。我使用 TwiML API 发起呼叫：

$sid = "SID_HERE";
$token = "TOKEN_HERE";

$client = new Twilio\Rest\Client($sid, $token);

$xml ='
<Response>
    <Say>This is your Voice Assistant speaking!</Say>
    <Connect>
        <Stream url="wss://MY_WEBSOCKET_SERVER_HERE">
            <Parameter name="aCutomParameter" value="aCustomValue that was set in TwiML" />
        </Stream>
    </Connect>
</Response>';


$call = $client->account->calls->create(
    '+TARGET_NUMBER_HERE',
    '+TWILIO_NUMBER_HERE',
    [
        'twiml' => $xml
    ]
);

3.我把音频和标记发在这里：

const textToSpeech = require('@google-cloud/text-to-speech');
const WebSocket = require('ws');
const ttsClient = new textToSpeech.TextToSpeechClient();
const wavefile = require('wavefile');
const fs = require('fs');

const path = require("path");

require("dotenv").config();

async function generateTtsAudio(text) {
    const request = {
        input: { text: text },
        voice: { languageCode: 'en-US', ssmlGender: 'NEUTRAL' },
        audioConfig: { audioEncoding: 'LINEAR16', sampleRateHertz: 8000 },
    };

    const [response] = await ttsClient.synthesizeSpeech(request);
    return response.audioContent;
}

async function sendTtsToWebSocket(text, wsUrl, streamSid) {
    const audioContent = await generateTtsAudio(text);


    const wav = new wavefile.WaveFile(audioContent);
        wav.toBitDepth('8');
        wav.toSampleRate(8000);
        av.toMuLaw();

    const base64Audio = Buffer.from(wav.data.samples).toString('base64');
    console.log(base64Audio);

    const ws = new WebSocket(wsUrl);

    ws.on('open', function open() {
        const message = {
            event: 'media',
            streamSid: streamSid,
            media: { payload: base64Audio }
        };

        const markMessage = {
            event: 'mark',
            streamSid: streamSid,
            "mark": {
                name: "testmark"
            }
        };

        ws.send(JSON.stringify(message));
        ws.send(JSON.stringify(markMessage));
    });

    ws.on('error', function error(error) {
        console.error('WebSocket Error:', error);
    });
}

const wsUrl = 'wss://MY_WEBSOCKET_SERVER_HERE';
const streamSid = 'STREAM_SID_HERE'; // I get it when starting the call
const text = 'Hello, this is a TTS test message.';

sendTtsToWebSocket(text, wsUrl, streamSid)
    .catch(console.error);

但是通话过程中仍然没有声音

Answer 1

终于找到问题了。

Twilio 甚至没有看到我发送的媒体消息，因为我必须将媒体消息发送到连接到 websocket 的所有客户端才能看到它们：

    ws.on("message", function incoming(message) {

    wss.clients.forEach(function each(client) {
        if (client !== ws && client.readyState === WebSocket.OPEN) {
            client.send(message.toString());
        }
    });

这成功了，现在它按预期工作并且正在播放媒体消息。