无法使用其新的.NET库将不在wwwroot下的本地图像发送到openai

Question

我有一个 ASP.NET Core 6 Web API，其中包含 OpenAI 的新官方库 (https://github.com/openai/openai-dotnet)。

我想做的是使用本地图像文件来OpenAI。该文件不在

wwwroot

下，而是在

backend/assets/1.jpg

下。

我编写了一个基本服务来设置向 OpenAI 发送请求所需的所有信息。但问题是我无法发送图像。

我不断收到“网址太长”或“图片无效”等错误，

这是我的代码 -

OpenAiService

:

using OpenAI.Chat;

namespace backend.Services
{
    public class OpenAiService
    {
        private readonly ChatClient _chatClient;
        private readonly ChatCompletionOptions _options;

        public OpenAiService(IConfiguration configuration)
        {
            var apiKey = configuration.GetValue<string>("OpenAI:Key");
            _chatClient = new ChatClient("gpt-4o", apiKey);

            _options = new ChatCompletionOptions()
            {
                MaxTokens = 300,
            };
        }

        public async Task<string> ExtractListOfItems()
        {
            var imagePath = Path.Combine(Directory.GetCurrentDirectory(), "Assets", "1.jpg");
            var localUrl = $"https://localhost:7068/assets/{Path.GetFileName(imagePath)}";
            
            var messages = new List<ChatMessage>
            {
                new UserChatMessage(new List<ChatMessageContentPart>
                {
                    ChatMessageContentPart.CreateTextMessageContentPart("Extract the items from the following image and return a list of items including prices and amount."),
                    ChatMessageContentPart.CreateImageMessageContentPart(new Uri(localUrl))
                })
            };

            var completion = await _chatClient.CompleteChatAsync(messages, _options);
            return completion.Value.ToString();
        }
    }
}

用于测试的演示控制器：

using Microsoft.AspNetCore.Mvc;
using System.Threading.Tasks;
using backend.Services;
using OpenAI;
using OpenAI.Chat;

namespace backend.Controllers;

[ApiController]
[Route("[controller]")]
public class OpenAiDemoController : ControllerBase
{
    private readonly OpenAiService _openAiService;

    public OpenAiDemoController(OpenAiService openAiService)
    {
        _openAiService = openAiService;
    }
    
    [HttpPost]
    [Route("extract-items")]
    public async Task<IActionResult> CompleteSentence()
    {
        var completion = await _openAiService.ExtractListOfItems();
        return Ok(completion);
    }
}

program.cs

文件：

using backend.Configurations;
using backend.Services;
using Microsoft.Extensions.FileProviders;

var builder = WebApplication.CreateBuilder(args);

// Add services to the container.

builder.Services.Configure<OpenAiConfig>(builder.Configuration.GetSection("OpenAI"));

//add services
builder.Services.AddSingleton<OpenAiService>();

builder.Services.AddControllers();
builder.Services.AddEndpointsApiExplorer();
builder.Services.AddSwaggerGen();

// builder.Services.AddScoped<IOpenAiService, OpenAiService>();

builder.Services.AddCors(opt =>
{
    opt.AddPolicy("AllowAll", builder =>
    {
        builder.AllowAnyOrigin()
            .AllowAnyMethod()
            .AllowAnyHeader();
    });
});

var app = builder.Build();

app.UseStaticFiles(new StaticFileOptions
{
    FileProvider = new PhysicalFileProvider(
        Path.Combine(builder.Environment.ContentRootPath, "Assets")),
    RequestPath = "/assets"
});

app.UseStaticFiles(); // This serves files from wwwroot

app.UseStaticFiles(new StaticFileOptions
{
    FileProvider = new PhysicalFileProvider(
        Path.Combine(builder.Environment.ContentRootPath, "Assets")),
    RequestPath = "/assets"
});

// Configure the HTTP request pipeline.
if (app.Environment.IsDevelopment())
{
    app.UseSwagger();
    app.UseSwaggerUI();
}

app.UseHttpsRedirection();
app.UseCors("AllowAll");
app.UseAuthorization();

app.MapControllers();

app.Run();

知道我做错了什么吗？

Answer 1

我自己解决了这个问题。 openai官方repo中有一个例子帮助我解决了这个问题。

https://github.com/openai/openai-dotnet/blob/main/examples/Chat/Example05_ChatWithVisionAsync.cs

不过，我不确定我的实现是否大部分正确。我将保留此开放以供任何其他建议。

服务：

using OpenAI.Chat;


namespace backend.Services
{
    public class OpenAiService
    {
        private readonly ChatClient _chatClient;
        private readonly ChatCompletionOptions _options;

        public OpenAiService(IConfiguration configuration)
        {
            var apiKey = configuration.GetValue<string>("OpenAI:Key");
            _chatClient = new ChatClient("gpt-4o", apiKey);
            _options = new ChatCompletionOptions()
            {
                MaxTokens = 300,
            };
        }

        public async Task<string> ExtractListOfItems()
        {
            var imageFilePath = Path.Combine("Assets", "1.jpg");
            await using Stream imageStream = File.OpenRead(imageFilePath);
            var imageBytes = BinaryData.FromStream(imageStream);

            var messages = new List<ChatMessage>
            {
                new UserChatMessage(new List<ChatMessageContentPart>
                {
                    ChatMessageContentPart.CreateTextMessageContentPart("describe the image. "),
                    ChatMessageContentPart.CreateImageMessageContentPart(imageBytes, "image/png")
                })
            };

            var completion = await _chatClient.CompleteChatAsync(messages, _options);
            return completion.Value.ToString();
        }
        
    }
}

在控制器中的用法：

using Microsoft.AspNetCore.Mvc;
using System.Threading.Tasks;
using backend.Services;
using OpenAI;
using OpenAI.Chat;

namespace backend.Controllers;

[ApiController]
[Route("[controller]")]
public class OpenAiDemoController : ControllerBase
{
    private readonly OpenAiService _openAiService;

    public OpenAiDemoController(OpenAiService openAiService)
    {
        _openAiService = openAiService;
    }
    
    [HttpPost]
    [Route("extract-items")]
    public async Task<IActionResult> CompleteSentence()
    {
        var completion = await _openAiService.ExtractListOfItems();
        return Ok(completion);
    }
    
}

无需申请静态文件中间件。