在不导致429错误（请求过多）的情况下发出API请求的有效方法？

Question

我有一个 API POST 方法，可以编辑数据库中表的单行中的值。该 API 每分钟只能处理 200 个请求，否则会抛出 429。我有 35,000 行必须从另一个表写入该表（因此总共需要 35,000 个请求）。而不是简单地在发出每个请求之前给出延迟（我现在已经完成了），最有效的方法（最少的时间）是什么？

 private static async void callMetadataAPI()
    {
        Stopwatch watch = new Stopwatch();
        watch.Start();
        string strConnection = GetConnectionString("");
        string queryString = "select * from test.dbo.test_table";
        using (SqlConnection connection = new SqlConnection(strConnection))
        {
            SqlCommand command = new SqlCommand(queryString, connection);
            connection.Open();
            SqlDataReader reader = command.ExecuteReader();

            try
            {
                while (reader.Read())
                {
                    HttpResponseMessage success = await runMetadataAPI(reader);
                    success.EnsureSuccessStatusCode();
                }
                
            }
            catch (Exception e)
            {
                ExceptionHandler(e);
            }
            finally
            {
                reader.Close();
                connection.Close();
            }
        }
        await Task.WhenAll();
        watch.Stop();
       
    }
    private static async Task<HttpResponseMessage> runMetadataAPI(SqlDataReader reader)
    {
        System.Threading.Thread.Sleep(1000);
        HttpClient client = new HttpClient();
        client.BaseAddress = new Uri("https://test.com");
        client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("JWT", "token");
        string requestUri = "/api/metadata";
        var metaDataPostData = new MetaDataPostData();
        metaDataPostData.CId= reader["CId"].ToString();
        metaDataPostData.SId = "somevalue"; 
        Metadata[] metaData = new Metadata[1];
        metaData[0] = new Metadata();
        metaData[0].Key = "columnname";
        metaData[0].Value = "columnvalue";
        metaDataPostData.Metadata = metaData;
        var json = JsonConvert.SerializeObject(metaDataPostData);
        var data = new StringContent(json, Encoding.UTF8, "application/json");
        HttpResponseMessage response = await client.PostAsync(requestUri, data);
        return response;
    }
class MetaDataPostData
{
    public Metadata[] Metadata { get; set; }
    public string SId{ get; set; }

    public string CId { get; set; }

}
class Metadata
{
    public string Key { get; set; }
    public string Value { get; set; }

}

鉴于 API 无法修改，非常感谢任何建议。目前，此代码大约需要 19 分钟才能完成 1000 个请求！我不确定这是否是对整个数据执行此操作的正确方法。

我只是通过在

 System.Threading.Thread.Sleep(1000)

函数（使用 API 的函数）中给予延迟

runMetadataAPI

来防止此代码中的 429。

Answer 1

如果您知道每分钟的请求数为 200，那么我会分批运行您的请求，并在批次之间等待 1 分钟。当前，您的 HttpClient 已重新创建每个请求。您可能希望在类构造函数中创建它，然后重新使用。 像这样的东西可以工作：

public class MyService() {
    private readonly HttpClient _client;

    public MyService() {
        _client = new HttpClient();
        _client.BaseAddress = new Uri("https://test.com");
        _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("JWT", "token");
    }

    private static async Task callMetadataAPI() 
    {
        Stopwatch watch = new Stopwatch();
        watch.Start();
        var tasks = new List<Task<HttpResponseMessage>>();
        string strConnection = GetConnectionString("");
        string queryString = "select * from test.dbo.test_table";
        using (SqlConnection connection = new SqlConnection(strConnection))
        {
            SqlCommand command = new SqlCommand(queryString, connection);
            connection.Open();
            SqlDataReader reader = command.ExecuteReader();

            try
            {            
                var batchSize = 200;
                var currentIdx = 0;
                while (reader.Read())
                {
                    var batchFinished = currentIdx % batchSize == 0;
                    var metaDataTask = runMetadataAPI(reader, batchFinished);
                    tasks.Add(metaDataTask);
                    currentIdx++;
                }

            }
            catch (Exception e)
            {
                ExceptionHandler(e);
            }
            finally
            {
                reader.Close();
                connection.Close();
            }
        }
        await Task.WhenAll(tasks);
        watch.Stop();
    }

    private static async Task<HttpResponseMessage> runMetadataApi(SqlDataReader reader, bool batchFinished) {
        if(batchFinished) {
            await Task.Delay(60000);
        }
        string requestUri = "/api/metadata";
        var metaDataPostData = new MetaDataPostData();
        metaDataPostData.CId= reader["CId"].ToString();
        metaDataPostData.SId = "somevalue"; 
        Metadata[] metaData = new Metadata[1];
        metaData[0] = new Metadata();
        metaData[0].Key = "columnname";
        metaData[0].Value = "columnvalue";
        metaDataPostData.Metadata = metaData;
        var json = JsonConvert.SerializeObject(metaDataPostData);
        var data = new StringContent(json, Encoding.UTF8, "application/json");
        return client.PostAsync(requestUri, data);
    }
}