ONNX I/O 绑定

Question

我需要使用 ONNX 运行时模型的 I/O 绑定来绑定张量输入和输出。但我没有得到输出。输出张量返回 NULL 指针。我将附上下面的代码。

std::vector<Ort::Value> input_tensors;
std::vector<Ort::Value> output_tensors;
std::vector<const char*> input_node_names_c_str;
std::vector<const char*> output_node_names_c_str;
int64_t input_height = input_node_dims[0].at(2);
int64_t input_width = input_node_dims[0].at(3);

// // Pass gpu_graph_id to RunOptions through RunConfigs
Ort::RunOptions run_option;
// gpu_graph_id is optional if the session uses only one cuda graph
run_option.AddConfigEntry("gpu_graph_id", "1");

// Dimension expansion [CHW -> NCHW]
std::vector<int64_t> input_tensor_shape = {1, 3, input_height, input_width};
std::vector<int64_t> output_tensor_shape = {1, 300, 84};
size_t input_tensor_size = vector_product(input_tensor_shape);
size_t output_tensor_size = vector_product(output_tensor_shape);
std::vector<float> input_tensor_values(p_blob, p_blob + input_tensor_size);

Ort::IoBinding io_binding{session};
Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);

input_tensors.push_back(Ort::Value::CreateTensor<float>(
        memory_info, input_tensor_values.data(), input_tensor_size,
        input_tensor_shape.data(), input_tensor_shape.size()
));

// Check if input and output node names are empty
for (const auto& inputNodeName : input_node_names) {
    if (std::string(inputNodeName).empty()) {
        std::cerr << "Empty input node name found." << std::endl;
    }
}

// format conversion
for (const auto& inputName : input_node_names) {
    input_node_names_c_str.push_back(inputName.c_str());
}

for (const auto& outputName : output_node_names) {
    output_node_names_c_str.push_back(outputName.c_str());
}

io_binding.BindInput(input_node_names_c_str[0], input_tensors[0]);

Ort::MemoryInfo output_mem_info{"Cuda", OrtDeviceAllocator, 0,
                                OrtMemTypeDefault};

cudaMalloc(&output_data_ptr, output_tensor_size * sizeof(float));
output_tensors.push_back(Ort::Value::CreateTensor<float>(
    output_mem_info,  static_cast<float*>(output_data_ptr),output_tensor_size, 
    output_tensor_shape.data(),output_tensor_shape.size()));                            

io_binding.BindOutput(output_node_names_c_str[0],  output_tensors[0]);
session.Run(run_option, io_binding);

//Get output results
auto* rawOutput = output_tensors[0].GetTensorData<float>();
cout<<rawOutput<<endl; //suhail
cudaFree(output_data_ptr); //suhail
std::vector<int64_t> outputShape = output_tensors[0].GetTensorTypeAndShapeInfo().GetShape();
for(auto i:outputShape){cout<<i<<" ";} cout<<endl; //suhail
size_t count = output_tensors[0].GetTensorTypeAndShapeInfo().GetElementCount();
cout<<count<<endl; //suhail
std::vector<float> output(rawOutput, rawOutput + count);

我交叉检查了输入张量数据和形状以及输出张量。但我得到了 NULL 指针。我该如何解决这个问题。任何人都有 I/O 绑定的经验。请给我解决这个问题的提示。

Answer 1

试试这个

auto rawOutput = output_tensors.front().GetTensorMutableData<float>();

而不是

auto* rawOutput = output_tensors[0].GetTensorData<float>();