YOLO系列之yolov5+OpenCV-DNN和onnxruntime推理

秋意为谁白

848人浏览 · 2023-10-24 08:00:00

秋意为谁白 · 2023-10-24 08:00:00 发布

OpenCV-DNN、onnxruntime都可以用于深度学习模型推理，OpenCV从 3.4 版本开始支持 CUDA 和 cuDNN 加速，允许在 NVIDIA GPU 上进行高效推理，支持多种深度学习框架导出的模型格式（如 Caffe、TensorFlow、Darknet、ONNX 等），简化了从训练到部署的过程，但在规模部署或需要利用最新硬件加速技术时可能不如专门针对推理优化的工具（如 ONNX Runtime）高效，同时对于其他类型的深度学习应用（如自然语言处理）的支持较少。而onnxruntime不局限于特定的操作系统或硬件架构，能够跨多个平台部署，包括 Windows、Linux、macOS 和边缘设备，相对于DNN提供了更强的动态模型支持能力，尤其是在处理复杂的数据流和操作方面更为灵活,但对于非 ONNX 模型的支持相对有限。

1. OpenCV-DNN(yolov5-Det)

注意事项：

pt->onnx 一定要选择静态尺寸，dnn只支持静态尺寸，否则会报错。

parser.add_argument('--dynamic', default=True, help='ONNX/TF/TensorRT: dynamic axes')

opencv version: 3.4.15

#include <iostream>
#include<opencv2//opencv.hpp>
#include<math.h>
#include<time.h>

using namespace std;
using namespace cv;
using namespace dnn;


// reference https://github.com/UNeedCryDear/yolov5-opencv-dnn-cpp/blob/main/yolo.cpp

std::vector<std::string> extensions{ "jpg", "bmp", "png", "jpeg" };
static const string kWinName = "YOLOV5 object detection in OpenCV";

static vector<string> loadLabels(const string& filename)
{
    vector<string> labels;
    ifstream ifs(filename.c_str());
    string line;
    while (getline(ifs, line))
    {
        labels.push_back(line);
    }
    return labels;
}

void GetImgFilenames(std::string path, std::vector<std::string>& imgFilenames)
{
    // imgFilenames.clear();
    if (path.find(".") != std::string::npos)
    {
        imgFilenames.push_back(path);
    }
    else
    {
        std::string fpath = path.append("*.*");
        std::vector<cv::String> allfiles;  //cv::String
        cv::glob(fpath, allfiles);
        for (int i = 0; i < allfiles.size(); i++)
        {
            size_t iPos = allfiles[i].rfind('.');
            std::string fileEx = allfiles[i].substr(iPos + 1, allfiles[i].length());
            if (std::find(extensions.begin(), extensions.end(), fileEx) != extensions.end())
            {
                imgFilenames.push_back(allfiles[i]);
            }
        }
    }
}

std::vector<cv::Scalar> colormap(int numClass)
{
    std::vector<cv::Scalar> colors;

    for (int i = 0; i < numClass; i++) {
        int b = rand() % 256;
        int g = rand() % 256;
        int r = rand() % 256;
        colors.push_back(Scalar(b, g, r));
    }
    return colors;
}

#define isP6 false //是否使用P6模型

struct OutputDet {
    int cls;             // 类别id
    float confidence;   //置信度
    cv::Rect bndbox;       //矩形框
};

class YoloDet {
public:
    
    bool modelInit(cv::dnn::Net& net, std::string& netPath, std::string classFile, bool isCuda);
    bool Detect(cv::Mat srcImg, cv::dnn::Net& net, std::vector<OutputDet>& output);
    void Visvalize(cv::Mat img, std::vector<OutputDet> predOuts);
    void letterbox(cv::Mat srcImg, cv::Mat& pendImg);

private:
#if(defined isP6 && isP6==true)
    const float netAnchors[4][6] = { { 19,27, 44,40, 38,94 },{ 96,68, 86,152, 180,137 },{ 140,301, 303,264, 238,542 },{ 436,615, 739,380, 925,792 } };

    const int inpWidth = 1280;  
    const int inpHeight = 1280; 

    const int strideSize = 4;  //stride size
#else
    const float netAnchors[3][6] = { { 10,13, 16,30, 33,23 },{ 30,61, 62,45, 59,119 },{ 116,90, 156,198, 373,326 } };

    const int inpWidth = 640;   
    const int inpHeight = 640;  

    const int strideSize = 3;   //stride size
#endif // isP6

    const float netStride[4] = { 8, 16.0,32,64 };

    float objThreshold = 0.25;
    float confThreshold = 0.25;

    float nmsThreshold = 0.45;
    float nmsScoreThreshold = objThreshold * confThreshold;

    bool keep_ratio = true;
    cv::Vec4d param;
    vector<string> classNames;
    int numClasses;
    std::vector<cv::Scalar> colors;

    // std::vector<std::string> classNames = { "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light",
    //  "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow",
    //  "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
    //  "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard",
    //  "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
    //  "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch",
    //  "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone",
    //  "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear",
    //  "hair drier", "toothbrush" };
};

bool YoloDet::modelInit(Net& net, string& netPath, std::string classFile, bool isCuda = false) {
    try {
        net = readNet(netPath);
    }
    catch (const std::exception&) {
        return false;
    }
    
    net.setPreferableBackend(cv::dnn::DNN_BACKEND_OPENCV);
    net.setPreferableTarget(cv::dnn::DNN_TARGET_CPU);
    // 编译 OpenCV 时启用 GPU 支持: 如果从源代码编译OpenCV, 需要在CMake配置阶段启用CUDA支持(通过设置 WITH_CUDA 选项为 ON), 且正确指定CUDA和cuDNN的路径
    // net.setPreferableBackend(cv::dnn::DNN_BACKEND_CUDA);
    // net.setPreferableTarget(cv::dnn::DNN_TARGET_CUDA);
    
    ifstream ifs(classFile);
    string line;
    while (getline(ifs, line)) this->classNames.push_back(line);
    
    if (classNames.empty()) {
        cout << "Failed to load labels: " << classFile << endl;

    }
    this->numClasses = classNames.size();
    this->colors = colormap(numClasses);
    
    return true;
}

void YoloDet::letterbox(cv::Mat srcImg, cv::Mat& pendImg)
{
   

    int col = srcImg.cols;
    int row = srcImg.rows;

    pendImg = srcImg.clone();
    int maxSize = col>=row?col:row;
    // int maxSize = MAX(col, row);

    if (maxSize > 1.2 * col || maxSize > 1.2 * row) {
        pendImg = cv::Mat::zeros(maxSize, maxSize, CV_8UC3);
        srcImg.copyTo(pendImg(Rect(0, 0, col, row)));
    }
    float ratioh = (float)pendImg.rows / inpHeight;
    float ratiow = (float)pendImg.cols / inpWidth;

    cv::Vec4d p(ratioh, ratiow, 0, 0);
    this-> param = p;

}

bool YoloDet::Detect(cv::Mat srcImg, cv::dnn::Net& net, vector<OutputDet>& output) {
    

    cv::Mat netInputImg;
    this->letterbox(srcImg.clone(), netInputImg);
    cv::Mat blob;
    cv::dnn::blobFromImage(netInputImg, blob, 1 / 255.0, cv::Size(inpWidth, inpHeight), cv::Scalar(0, 0, 0), true, false);
    //blobFromImage(netInputImg, blob, 1 / 255.0, cv::Size(inpWidth, inpHeight), cv::Scalar(104, 117, 123), true, false);
    //blobFromImage(netInputImg, blob, 1 / 255.0, cv::Size(inpWidth, inpHeight), cv::Scalar(114, 114,114), true, false);
    net.setInput(blob);
    std::vector<cv::Mat> netOutputImg;
    net.forward(netOutputImg, net.getUnconnectedOutLayersNames());

#if CV_VERSION_MAJOR==4&&CV_VERSION_MINOR==6
    std::sort(netOutputImg.begin(), netOutputImg.end(), [](Mat& A, Mat& B) {return A.size[1] > B.size[1]; });//opencv 4.6
#endif

    
    int netWidth = numClasses + 5;  
    float* rawOutput = (float*)netOutputImg[0].data;

    std::vector<int> clsIDs;
    std::vector<float> confidences;
    std::vector<cv::Rect> boxes;
    //vector<BoxInfo> boxes;
    for (int n = 0; n < this-> strideSize; n++) 
    {    
        const float stride = pow(2, n + 3);
        int grid_x = (int)ceil((this->inpWidth / stride));
        int grid_y = (int)ceil((this->inpHeight / stride));
        for (int anchor = 0; anchor < 3; anchor++) 
        {   //anchors
            for (int i = 0; i < grid_y; ++i) 
            {
                for (int j = 0; j < grid_x; ++j) 
                {
                    float boxScore = rawOutput[4]; ;//bndbox-confidence
                    if (boxScore >= objThreshold) 
                    {
                        cv::Mat confs(1, classNames.size(), CV_32FC1, rawOutput + 5);
                        Point clsID;
                        double maxConf;
                        minMaxLoc(confs, 0, &maxConf, 0, &clsID);
                        maxConf = (float)maxConf;
                        if (maxConf >= confThreshold) 
                        {
                            float x = rawOutput[0];  //x
                            float y = rawOutput[1];  //y
                            float w = rawOutput[2];  //w
                            float h = rawOutput[3];  //h
                    
                            // float left = (x - param[3] - 0.5 * w) * param[1];
                            // float top = (y - param[2] - 0.5 * h) * param[0];
                            int left = (x - 0.5 * w) * param[1];
                            int top = (y - 0.5 * h) * param[0];
                        
                            w =  w * param[1];
                            h =  h * param[0];
                            
                            clsIDs.push_back(clsID.x);
                            confidences.push_back(maxConf * boxScore);
                            boxes.push_back(cv::Rect(int(left), int(top), int(w), int(h)));
                        }
                    }
                    rawOutput += netWidth;
                }
            }
        }
    }

    cv::Mat img = srcImg.clone();
    vector<int> resultNMS;
    NMSBoxes(boxes, confidences, nmsScoreThreshold, nmsThreshold, resultNMS);
    Rect refineRect(0, 0, img.cols, img.rows);
    // std::vector<OutputDet> tempOutput;
    for (int i = 0; i < resultNMS.size(); ++i) {

        int ix = resultNMS[i];
        OutputDet predOut;
        predOut.cls = clsIDs[ix];
        predOut.confidence = confidences[ix];
        predOut.bndbox = boxes[ix] & refineRect;
        output.push_back(predOut);
    }
    if (output.size())
        return true;
    else
        return false;
}

void YoloDet::Visvalize(cv::Mat img, vector<OutputDet> predOuts) {

    for (int i = 0; i < predOuts.size(); i++) {
        int left, top;
        left = predOuts[i].bndbox.x;
        top = predOuts[i].bndbox.y;
        rectangle(img, predOuts[i].bndbox, colors[predOuts[i].cls], 2, 8);
        
        string label = classNames[predOuts[i].cls] + ":" + to_string(predOuts[i].confidence);

        int baseLine;
        Size labelSize = getTextSize(label, FONT_HERSHEY_SIMPLEX, 0.5, 1, &baseLine);
        top = max(top, labelSize.height);
        putText(img, label, Point(left, top), FONT_HERSHEY_SIMPLEX, 1, colors[predOuts[i].cls], 2);
        
    }

    // cv::namedWindow(kWinName, WINDOW_NORMAL);
    cv::imshow(kWinName, img);
    cv::waitKey(0);
    cv::destroyAllWindows();

}

//  不支持动态尺寸
int runner(std::string modelPath, std::string imgPath, std::string classFile){
    
    YoloDet detector;
    cv::dnn::Net net;
    if (detector.modelInit(net, modelPath, classFile, true)) {
        cout << "model loaded successfully!" << endl;
    }
    else {
        cout << "model loaded failed!" << endl;
        return -1;
    }

    std::vector<std::string> imgFiles;
    GetImgFilenames(imgPath, imgFiles);
    cout << imgFiles.size() << endl;
    for (int i = 0; i < imgFiles.size(); i++)
    {
        vector<OutputDet> predOuts;
        Mat img = imread(imgFiles[i]);
        if (detector.Detect(img, net, predOuts)) {
            detector.Visvalize(img, predOuts);

        }
        else {
            cout << "Detect Failed!" << endl;
        }

    }
    return 0;
    
}

int main() {
    std::string imgPath = "images/";
    std::string modelPath = "yolov5s.onnx";
    std::string classFile = "class.names";
    runner(modelPath, imgPath, classFile); 
    return 0;
}

2. onnxruntime(yolov5-Det/Seg、yolov8-Det/Seg)

onnxruntime支持动态onnx模型推理，所以转化onnx时，不需要特意注意dynamic。

utils.h:

#pragma once
#ifndef UTILS_H
#define UTILS_H

// Cpp native
#include <fstream>
#include <vector>
#include <string>
#include <random>
#include <opencv2/opencv.hpp>


#include <onnxruntime_cxx_api.h>


using namespace std;
using namespace cv;
using namespace Ort;
using namespace cv::dnn;


struct ParamsSeg {
	
	/*int segChannels = 32;
	int segWidth = 160;
	int segHeight = 160;*/
	int netWidth = 640;
	int netHeight = 640;
	float mask_threshold = 0.5;
	cv::Size oriShape;
	cv::Vec4d params;
};

struct Detection
{
	int clsID{ 0 };
	std::string className{};
	float confidence{ 0.0 };
	cv::Scalar color{};
	cv::Rect box{};
	cv::Mat mask{};
};

class Inference
{

public:

	Inference(const std::string& onnxModelPath, const cv::Size& modelInputShape = { 640, 640 }, const std::string& classesTxtFile = "", const bool& runWithCuda = true, const int& batchSize=1);
	std::vector<Detection> runner(const cv::Mat& input);

private:

	std::vector<std::string> loadClassesFromFile(const string& Path);
	cv::Mat letterBox(const cv::Mat& src, cv::Size size);
	size_t vectorProduct(const std::vector<int64_t>& vector);
	void GetMask(const cv::Mat& maskProposals, const cv::Mat& maskProtos, vector<Detection>& output, const ParamsSeg& paramsSeg);

	cv::Size2f inferSize{640, 640};
	std::string modelPath{};
	std::string classesPath{};
	bool cudaEnabled{};
	bool isDynamic{};

	float conf_threshold{ 0.25 };
	float score_threshold{ 0.45 };
	float nms_threshold{ 0.50 };
	bool hasMask{ false };
	float mask_threshold{ 0.5 };

	std::vector<std::string> classes{ "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier", "toothbrush" };
	
	Env env = Env(ORT_LOGGING_LEVEL_ERROR, "yolo");
	Ort::Session* ortSession = nullptr;
	SessionOptions sessionOptions = SessionOptions();

	vector<char*> inputNodeNames;
	vector<char*> outputNodeNames;
	std::vector<int64_t> inputTensorShapes;
	std::vector<int64_t> outputTensorShapes;

	std::vector<std::vector<int64_t>> inputShapes;
	std::vector<std::vector<int64_t>> outputShapes;

};

#endif // UTILS_H

utils.cpp:

#include "utils.h"
#include <iostream>
using namespace std;
using namespace cv;

size_t Inference::vectorProduct(const std::vector<int64_t>& vector)
{
	if (vector.empty())
		return 0;

	size_t product = 1;
	for (const auto& element : vector)
		product *= element;

	return product;
}


std::vector<std::string> Inference::loadClassesFromFile(const string& classFilePath)
{
	std::vector<std::string> classnames = {};
	std::ifstream inputFile(classFilePath);
	if (inputFile.is_open())
	{
		std::string classLine;
		while (std::getline(inputFile, classLine))
			classnames.push_back(classLine);
		inputFile.close();
	}
	return classnames;
}

Inference::Inference(const std::string& onnxModelPath, const cv::Size& modelInputShape, const std::string& classesTxtFile, const bool& runWithCuda, const int& batchSize)
{

	std::wstring widestr = std::wstring(onnxModelPath.begin(), onnxModelPath.end());
	std::vector<std::string> availableProviders = Ort::GetAvailableProviders();
	auto cudaAvailable = std::find(availableProviders.begin(), availableProviders.end(), "CUDAExecutionProvider");
	OrtCUDAProviderOptions cudaOption;
	cudaEnabled = runWithCuda;


	if (cudaEnabled && (cudaAvailable == availableProviders.end()))
	{
		std::cout << "GPU is not supported by your ONNXRuntime build. Fallback to CPU." << std::endl;
		std::cout << "Inference device: CPU" << std::endl;
	}
	else if (cudaEnabled && (cudaAvailable != availableProviders.end()))
	{
		std::cout << "Inference device: GPU" << std::endl;
		sessionOptions.AppendExecutionProvider_CUDA(cudaOption);
	}
	else
	{
		std::cout << "Inference device: CPU" << std::endl;
	}

	sessionOptions.SetGraphOptimizationLevel(ORT_ENABLE_BASIC);
	ortSession = new Session(env, widestr.c_str(), sessionOptions);
	size_t numInputNodes = ortSession->GetInputCount();
	size_t numOutputNodes = ortSession->GetOutputCount();

	AllocatorWithDefaultOptions allocator;

	Ort::TypeInfo inputTypeInfo = ortSession->GetInputTypeInfo(0);
	this->inputTensorShapes = inputTypeInfo.GetTensorTypeAndShapeInfo().GetShape();
	this->inputShapes.push_back(inputTensorShapes);
	this->inputNodeNames.push_back(ortSession->GetInputName(0, allocator));

	Ort::TypeInfo outputTypeInfo = ortSession->GetOutputTypeInfo(0);
	this->outputTensorShapes = outputTypeInfo.GetTensorTypeAndShapeInfo().GetShape();
	this->outputShapes.push_back(outputTensorShapes);


	if (numOutputNodes > 1)
	{
		this->hasMask = true;
		std::cout << "Instance Segmentation" << std::endl;
		for (int i = 0; i < numOutputNodes; ++i)
		{
			this->outputNodeNames.push_back(ortSession->GetOutputName(i, allocator));
		}
	}
	else
		std::cout << "Object Detection" << std::endl;
	this->outputNodeNames.push_back(ortSession->GetOutputName(0, allocator));


	if (this->inputTensorShapes[0] == -1)
	{
		isDynamic = true;
		this->inputTensorShapes[0] = batchSize;

	}
	if (this->inputTensorShapes[2] == -1 || this->inputTensorShapes[3] == -1) {
		this->isDynamic = true;
		this->inputTensorShapes[2] = modelInputShape.width;
		this->inputTensorShapes[3] = modelInputShape.height;
	}


	if (this->classes.empty()) {
		this->classes = loadClassesFromFile(classesTxtFile.c_str());
		cout << "Failed to load labels: " << classesTxtFile << endl;
	}

	this->inferSize = modelInputShape;
}



cv::Mat Inference::letterBox(const cv::Mat& src, cv::Size size)
{

	int w = src.cols; // width
	int h = src.rows; // height

	float r = std::min(float(size.height) / w, float(size.width) / h);
	int nw = round(w * r);
	int nh = round(h * r);
	int dw = size.width - nw;
	int dh = size.height - nh;

	cv::Mat letBoxImg;

	cv::resize(src, letBoxImg, cv::Size(nw, nh));

	dw = dw / 2;
	dh = dh / 2;


	int top = int(round(dh - 0.1));
	int bottom = int(round(dh + 0.1));
	int left = int(round(dw - 0.1));
	int right = int(round(dw + 0.1));
	cv::copyMakeBorder(letBoxImg, letBoxImg, top, bottom, left, right, 0, cv::Scalar(114, 114, 114));

	return letBoxImg;
}

std::vector<Detection> Inference::runner(const cv::Mat& img) {

	cv::Mat net_input = letterBox(img, this->inferSize);
	auto Device = MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);

	Mat blob = cv::dnn::blobFromImage(net_input, 1 / 255.0, this->inferSize, cv::Scalar(), true, false);  // �Ƿ�����������

	size_t inputTensorSize = vectorProduct(inputTensorShapes);


	std::vector<Ort::Value> inputTensors;
	inputTensors.push_back(Ort::Value::CreateTensor<float>(Device,
		(float*)blob.data, inputTensorSize, inputTensorShapes.data(), inputTensorShapes.size()));

	std::vector<Ort::Value> outputTensors = ortSession->Run(Ort::RunOptions{ nullptr },
		inputNodeNames.data(), inputTensors.data(), inputNodeNames.size(), outputNodeNames.data(), outputNodeNames.size());

	float* rawOutputs = outputTensors[0].GetTensorMutableData<float>();

	std::vector<int64_t> outputShape = outputTensors[0].GetTensorTypeAndShapeInfo().GetShape();


	bool yolov8 = (int)outputShape[1] < int(outputShape[2]) ? true : false;


	// yolov5 has an output of shape (batchSize, 25200, 85) (Num classes + box[x,y,w,h] + confidence[c])
	// yolov8 has an output of shape (batchSize, 84,  8400) (Num classes + box[x,y,w,h])

	double gain = std::min(float(net_input.rows) / img.rows, float(net_input.cols) / img.cols);
	double dw = (net_input.cols - int(float(img.cols) * gain)) / 2;
	double dh = (net_input.rows - int(float(img.rows) * gain)) / 2;


	std::vector<int> clsIDs;
	std::vector<float> confidences;
	std::vector<cv::Rect> boxes;

	std::vector<std::vector<float>> pickledProposals;

	if (yolov8) {

		int netWidth = (int)outputShape[1];

		cv::Mat output = cv::Mat(Size((int)outputShape[2], (int)outputShape[1]), CV_32F, rawOutputs).t(); // [1, 84 ,8400] -> [1, 8400, 84]
		float* pData = (float*)output.data;

		for (int i = 0; i < output.rows; ++i)
		{
			cv::Mat scores(1, classes.size(), CV_32FC1, pData + 4);
			cv::Point clsID;
			double maxClsScore;
			minMaxLoc(scores, 0, &maxClsScore, 0, &clsID);

			if (maxClsScore > score_threshold) {

				if (this->hasMask)
				{
					std::vector<float> temp(pData + 4 + classes.size(), pData + netWidth);
					pickledProposals.push_back(temp);
				}

				confidences.push_back(maxClsScore);
				clsIDs.push_back(clsID.x);

				double x = pData[0];
				double y = pData[1];
				double w = pData[2];
				double h = pData[3];

				int width = w / gain;
				int height = h / gain;
				int left = (double(x - dw) - 0.5 * w) / gain;
				int top = (double(y - dh) - 0.5 * h) / gain;

				boxes.push_back(cv::Rect(left, top, width, height));
			}
			pData += netWidth;
		}
	}


	else
	{
		int netWidth = (int)outputShape[2];
		int rows = (int)outputShape[1];

		float* pData = rawOutputs;
		for (int i = 0; i < rows; ++i)
		{
			float confidence = pData[4];

			if (confidence >= conf_threshold)

			{
				float* classes_scores = pData + 5;

				cv::Mat scores(1, classes.size(), CV_32FC1, classes_scores);
				cv::Point clsID;
				double max_class_score;

				minMaxLoc(scores, 0, &max_class_score, 0, &clsID);

				if (max_class_score > score_threshold)

				{

					if (this->hasMask)
					{
						std::vector<float> temp(pData + 5 + classes.size(), pData + netWidth);
						pickledProposals.push_back(temp);
					}
					confidences.push_back(confidence);
					clsIDs.push_back(clsID.x);

					double x = pData[0];
					double y = pData[1];
					double w = pData[2];
					double h = pData[3];

					int width = w / gain;
					int height = h / gain;
					int left = (double(x - dw) - 0.5 * w) / gain;
					int top = (double(y - dh) - 0.5 * h) / gain;

					boxes.push_back(cv::Rect(left, top, width, height));
				}
			}
			pData += netWidth;
		}

	}

	std::vector<int> results;
	cv::dnn::NMSBoxes(boxes, confidences, score_threshold, nms_threshold, results);
	std::vector<Detection> detections{};
	cv::Rect holeImgRect(0, 0, img.cols, img.rows);
	std::vector<vector<float>> pre_maskProposals;

	for (unsigned long i = 0; i < results.size(); ++i)
	{
		int ind = results[i];

		Detection result;
		result.clsID = clsIDs[ind];
		result.confidence = confidences[ind];

		std::random_device rd;
		std::mt19937 gen(rd());
		std::uniform_int_distribution<int> dis(100, 255);
		result.color = cv::Scalar(dis(gen), dis(gen), dis(gen));
		result.className = classes[result.clsID];
		result.box = boxes[ind] & holeImgRect;
		if (hasMask) {
			pre_maskProposals.push_back(pickledProposals[ind]);
		}
		detections.push_back(result);


	}

	if (hasMask)
	{
		ParamsSeg paramsSeg;
		paramsSeg.params = cv::Vec4d{ float(net_input.rows) / img.rows, float(net_input.cols) / img.cols, dw, dh };

		paramsSeg.oriShape = img.size();
		paramsSeg.netHeight = net_input.rows;
		paramsSeg.netWidth = net_input.cols;
		paramsSeg.mask_threshold = mask_threshold;

		std::vector<int64_t> outputTensorShape_mask = outputTensors[1].GetTensorTypeAndShapeInfo().GetShape();

		vector<int> protosShape_mask = { 1,(int)outputTensorShape_mask[1],(int)outputTensorShape_mask[2],(int)outputTensorShape_mask[3] };
		// int maskProtosLength = (int)outputTensorShape_mask[1] * (int)outputTensorShape_mask[2] * (int)outputTensorShape_mask[3];
		cv::Mat maskProtos = Mat(protosShape_mask, CV_32F, outputTensors[1].GetTensorMutableData<float>());
		cv::Mat maskProposals;
		for (int i = 0; i < pre_maskProposals.size(); ++i) {
			maskProposals.push_back(cv::Mat(pre_maskProposals[i]).t());
		}
		
		GetMask(maskProposals, maskProtos, detections, paramsSeg);
	}
	return detections;

}




void Inference::GetMask(const cv::Mat& maskProposals, const cv::Mat& maskProtos, std::vector<Detection>& output, const ParamsSeg& paramsSeg) {
	//cout << maskProtos.size << endl;

	int net_width = paramsSeg.netWidth;
	int net_height = paramsSeg.netHeight;
	int channeels_seg = maskProtos.size[1];
	int height_seg = maskProtos.size[2];
	int width_seg = maskProtos.size[3];
	float mask_threshold = paramsSeg.mask_threshold;
	cv::Vec4f params = paramsSeg.params;
	cv::Size oriShape = paramsSeg.oriShape;

	cv::Mat protos = maskProtos.reshape(0, { channeels_seg,width_seg * height_seg });
	cv::Mat matmul_res = (maskProposals * protos).t();
	cv::Mat masks = matmul_res.reshape(output.size(), { width_seg, height_seg });
	std::vector<cv::Mat> maskSplit;
	split(masks, maskSplit);
	for (int i = 0; i < output.size(); ++i) {
		cv::Mat dest, mask;
		//sigmoid
		cv::exp(-maskSplit[i], dest);
		dest = 1.0 / (1.0 + dest);

		cv::Rect roi(int(params[2] / net_width * width_seg), int(params[3] / net_height * height_seg), int(width_seg - params[2] / 2), int(height_seg - params[3] / 2));
		dest = dest(roi);
		cv::resize(dest, mask, oriShape, INTER_NEAREST);

		//crop
		cv::Rect rectBox = output[i].box;
		mask = mask(rectBox) > mask_threshold;
		output[i].mask = mask;
	}
}

main.cpp:

#include <iostream>
#include <vector>
#include <opencv2/opencv.hpp>
#include <onnxruntime_cxx_api.h>
#include "utils.h"

using namespace Ort;
using namespace std;
using namespace cv;

const string kWinName = "Yolo object detection in ONNXRuntime";
const std::vector<std::string> extensions{ "jpg", "bmp", "png", "jpeg" };


std::vector<std::string> GetImgFilenames(std::string path)
{
    std::vector<std::string> imgFilenames;

    std::string fpath = path.append("*.*");
    std::vector<cv::String> allfiles;  //cv::String
    cv::glob(fpath, allfiles);
    for (int i = 0; i < allfiles.size(); i++)
    {
        size_t iPos = allfiles[i].rfind('.');
        std::string fileEx = allfiles[i].substr(iPos + 1, allfiles[i].length());

        if (std::find(extensions.begin(), extensions.end(), fileEx) != extensions.end())
        {
            imgFilenames.push_back(allfiles[i]);
        }
    }
    return imgFilenames;
}

int main() 
{   

    /* ------------
        1) 环境配置：
            onnxruntime-win-x64-gpu-1.11.0 
            opencv_4.9.0
        2) 功能：
            调用onnxruntime实现yolov5/yolov8/yolov5seg/yolov8seg的推理
    */
    string rootPath = "./source/bus.jpg";
	string modelPath = "models/yolov5s-seg.onnx";
	string classFile = "class.names";
	int batchSize = 1;
	bool runWithCuda = true;

	Inference inference(modelPath, cv::Size(640, 640), classFile, runWithCuda, batchSize);

	std::vector<std::string> imgFiles = GetImgFilenames(rootPath);

    /*
    for (int i = 0; i < imgFiles.size(); ++i)
    {
        cv::Mat img = cv::imread(imgFiles[i]);
    */

    cv::Mat img = cv::imread(rootPath); 
    std::vector<Detection> output = inference.runner(img);

    cv::Mat maskColored = img.clone();

    for (int i = 0; i < output.size(); ++i)
    {
        Detection detection = output[i];

        cv::Rect box = detection.box;
        cv::Scalar color = detection.color;

        // Detection box
        cv::rectangle(img, box, color, 2);

        // Segment mask
        if (detection.mask.rows && detection.mask.cols > 0)
        {
            maskColored(detection.box).setTo(color, detection.mask);
            cv::addWeighted(img, 0.7, maskColored, 0.3, 1, img);
        }

        // label text
        std::string classString = detection.className + ' ' + std::to_string(detection.confidence).substr(0, 4);
        cv::Size textSize = cv::getTextSize(classString, cv::FONT_HERSHEY_DUPLEX, 1, 1, 0);
        cv::Rect textBox(box.x, box.y - 40, textSize.width + 10, textSize.height + 20);

        cv::rectangle(img, textBox, color, cv::FILLED);
        cv::putText(img, classString, cv::Point(box.x + 5, box.y - 10), cv::FONT_HERSHEY_DUPLEX, 1, cv::Scalar(0, 0, 0), 1, 0);
    }
    
    cv::imshow(kWinName, img);
    cv::waitKey(0);
    cv::destroyAllWindows();

	return 0;
}