如何发送包含 HTML 创建图像的 JSON 文件

Question

我正在使用我的第一个基于 React 的应用程序，并尝试将图像发送到我的基于 Flask 的服务器，但它说图像没有被附加。其基础是拍摄实时图像，然后将其发送到人工智能模型以处理图像，我发现直接上传文件是成功的，但实时视频则不然。

这是我遇到问题的主要文件

"use client";
import "./myscript.css"
import { useEffect, useRef } from 'react';

const MyWebcam = () => {
    const videoRef = useRef(null);


  useEffect(() => {
    const getUserMedia = async () => {
      try {
        const stream = await navigator.mediaDevices.getUserMedia({ video: true });
        if (videoRef.current) {
          videoRef.current.srcObject = stream;
          videoRef.current.addEventListener('loadedmetadata', () => {})
        }
      } catch (error) {
        console.error(error);
      }
    };

    getUserMedia();

  }, []);

  const takePicture = async () => {
    if (!videoRef.current || !videoRef.current.videoWidth || !videoRef.current.videoHeight) {
        console.log(videoRef.current)
        console.error('Video not ready');
    }
    else{
      const canvas = document.createElement('canvas');
      canvas.width = videoRef.current.videoWidth;
      canvas.height = videoRef.current.videoHeight;
      const context = canvas.getContext('2d');
      context.drawImage(videoRef.current, 0, 0, canvas.width, canvas.height);
      const imgUrl = canvas.toDataURL('image/png');
      const img= document.getElementById("picture")
      img.src = imgUrl

      let data = context.getImageData(0, 0, canvas.width, canvas.height)
      const formData = new FormData()
      formData.append('file', data.data)
      console.log("pre JSON fetching")
      console.log(formData)
  try {
    const response = await fetch("http://localhost:8080/results", {
      method: 'POST',
      body: formData
    });
    console.log(response)
    const data = await response.json();
    console.log('Response:', data);
    let ex =  document.getElementById("translation")
    ex.textContent = data.message
      }
  catch(error){
      console.error(error)        }



    }
    
  }

  return (
<div className="fullscreen-container">
  <div className="input-div">
    <h2>Video Capture</h2>
    <div className="video-container">
      <video ref={videoRef} autoPlay width="900" height="900" />
      <div className="image-container">
        <img id="picture" width="900" height="900" alt="Captured" />
      </div>
    </div>
    <button type="button" onClick={takePicture}>Take Picture</button>
    <p id="translation">Translation</p>
  </div>
</div>



  );
};

export default MyWebcam;

这是我要发送到的服务器

from flask import Flask, jsonify, request
from flask_cors import CORS, cross_origin
from model import ASLModel
import torch
from pathlib import Path
from torchvision import transforms, datasets
from PIL import Image


'Access-Control-Allow-Origin: *'


#app instance
app = Flask('__name__')
CORS(app, resources={r"/*": {"origins": "*"}})
app.config['CORS_HEADERS'] = 'Content-Type'



@app.route("/results", methods=["POST", "GET"])
@cross_origin()
def results():
    if 'file' not in request.files:
        print(request.files)
        return jsonify({'error': 'no file found'})
    print(request.files)
    img = request.files['file']

    trainData = datasets.ImageFolder(root="archive/asl_alphabet_train")
    classNames=trainData.classes
    device = "cpu"
    MODEL_PATH = Path("models")
    MODEL_PATH.mkdir(parents=True, exist_ok=True)
    MODEL_NAME = "ASL_CNN_MODEL.pth"
    MODEL_SAVE_PATH = MODEL_PATH / MODEL_NAME


    LoadedModel = ASLModel(input_shape=3, hidden_units=30, output_shapes=29)
    LoadedModel.load_state_dict(torch.load(MODEL_SAVE_PATH), False)
    LoadedModel.to(device=device)
    transform=transforms.Compose([transforms.Resize(size=(500, 500)), transforms.ToTensor()])
    LoadedModel.eval()

    img = Image.open(img)
    img.show()
    img = transform(img)    
    
    sample = torch.unsqueeze(img, dim=0).to(device)

    predLogit = LoadedModel(sample)

    result = torch.softmax(predLogit.squeeze(), dim=0)
    result = result.argmax()
    
    print("result", classNames[int(result)])
    
    response = jsonify({'message': classNames[int(result)]})
    return response



if __name__ == "__main__":
    app.run(debug=True, port=8080)#port 5000 has issues with requests

这是允许您上传图像的页面的文件

'use client'
import './upload.css'

const MyInput = () => {
    var img
    const ImageUploaded = () => {
            let input = document.getElementById("file") //Get image
                img = input.files[0]
                let final= document.getElementById("picture") //access displayed tag for image
                final.src = URL.createObjectURL(img)
            
    }


    const SendToModel = async () => {
        // fetch("http://localhost:8080/results").then(
        //     response => response.json()).then(
        //         data => {
                    
        //             let ex =  document.getElementById("translation")
        //             ex.textContent = data.message
        //         }
        //     )


            const formData = new FormData()
            formData.append('file', img)
            console.log("pre JSON fetching")
            console.log(formData)
        try {
        const response = await fetch("http://localhost:8080/results", {
            method: 'POST',
            body: formData
        });
        console.log(response)
        const data = await response.json();
        console.log('Response:', data);
        let ex =  document.getElementById("translation")
        ex.textContent = data.message
            }
        catch(error){
            console.error(error)        }
    }


return (        


<div className="fullscreen-container">
<div className="input-div">
    <h2>Upload Images</h2>
    <p>Drag and drop images here or <span className="browse">browse</span></p>
    <input type="file" id="file" onChange={ImageUploaded} />
    <img id="picture" alt="Uploaded" />
    <button id="submit" onClick={SendToModel}>Submit</button>
    <p id="translation">Translation</p>
</div>
</div>
)}
export default MyInput

任何和所有帮助将不胜感激，因为“如果文件不在 request.files 中”语句仅在我尝试发送实时图像时才会被命中

Answer 1

我仔细查看了你的代码，发现了一个问题。您尝试使用 getImageData 从画布获取图像数据并将内容作为文件发送到 Flask 服务器。但在这种情况下，Flask 服务器不会将数据识别为文件，因为它不支持该格式。要解决此问题，您需要将画布图像转换为Blob或File，这将允许您将其作为实际文件发送。

我提供更新的代码：

在前端，更新 takePicture 函数：

const takePicture = async () => {
  if (!videoRef.current || !videoRef.current.videoWidth || !videoRef.current.videoHeight) {
    console.log(videoRef.current)
    console.error('Video not ready');
  } else {
    const canvas = document.createElement('canvas');
    canvas.width = videoRef.current.videoWidth;
    canvas.height = videoRef.current.videoHeight;
    const context = canvas.getContext('2d');
    context.drawImage(videoRef.current, 0, 0, canvas.width, canvas.height);

    // Convert the canvas to a Blob
    canvas.toBlob(async (blob) => {
      const formData = new FormData();
      formData.append('file', blob, 'image.png'); // Add the Blob to the FormData

      try {
        const response = await fetch("http://localhost:8080/results", {
          method: 'POST',
          body: formData
        });
        const data = await response.json();
        console.log('Response:', data);
        let ex = document.getElementById("translation");
        ex.textContent = data.message;
      } catch (error) {
        console.error(error);
      }
    }, 'image/png');
  }
};

希望对您有帮助。