【GiantPandaCV】本文為大家介紹了一個手把手使用OpenVINO部署NanoDet的教程，並開源部署的全部代碼，在Intel i7-7700HQ CPU做到了6ms一幀的速度。
0x0. nanodet簡介NanoDet （https://github.com/RangiLyu/nanodet）是一個速度超快和輕量級的Anchor-free 目標檢測模型。想了解算法本身的可以去搜一搜之前機器之心的介紹。
0x1. 環境配置OpenVINO和OpenCV安裝包（編譯好了，也可以自己從官網下載自己編譯）可以從連結: https://pan.baidu.com/s/1zxtPKm-Q48Is5mzKbjGHeg 密碼: gw5c下載
tar -xvzf l_openvino_toolkit_p_2020.4.287.tgz
cd l_openvino_toolkit_p_2020.4.287
sudo ./install_GUI.sh 一路next安裝
cd /opt/intel/openvino/install_dependencies
sudo ./install_openvino_dependencies.sh
vi ~/.bashrcsource /opt/intel/openvino/bin/setupvars.sh
source /opt/intel/openvino/opencv/setupvars.shcd /opt/intel/openvino/deployment_tools/model_optimizer/install_prerequisites
sudo ./install_prerequisites_onnx.sh（模型是從onnx轉為IR文件，只需配置onnx依賴）tar -xvzf opencv-3.4.2.zip 解壓OpenCV到用戶根目錄即可，以便後續調用。（這是我編譯好的版本，有需要可以自己編譯）
0x2. NanoDet模型訓練和轉換ONNXgit clone https://github.com/Wulingtian/nanodet.git
定位到nanodet目錄，進入tools目錄，打開export.py文件，配置cfg_path model_path out_path三個參數
定位到nanodet目錄，運行 python tools/export.py 得到轉換後的onnx模型
python /opt/intel/openvino/deployment_tools/model_optimizer/mo_onnx.py --input_model onnx模型 --output_dir 期望模型輸出的路徑。得到IR文件
0x3. NanoDet模型部署sudo apt install cmake 安裝cmake
git clone https://github.com/Wulingtian/nanodet_openvino.git （求star！）
cd nanodet_openvino 打開CMakeLists.txt文件，修改OpenCV_INCLUDE_DIRS和OpenCV_LIBS_DIR，之前已經把OpenCV解壓到根目錄了，所以按照你自己的路徑指定
定位到nanodet_openvino，cd models 把之前生成的IR模型（包括bin和xml文件）文件放到該目錄下
定位到nanodet_openvino， cd test_imgs 把需要測試的圖片放到該目錄下
定位到nanodet_openvino，編輯main.cpp，xml_path參數修改為"../models/你的模型名稱.xml"
編輯 num_class 設置類別數，例如：我訓練的模型是安全帽檢測，只有1類，那麼設置為1
編輯 src 設置測試圖片路徑，src參數修改為"../test_imgs/你的測試圖片"
mkdir build; cd build; cmake .. ;make
./detect_test 輸出平均推理時間，以及保存預測圖片到當前目錄下，至此，部署完成！
0x4. 核心代碼一覽//主要對圖片進行預處理，包括resize和歸一化
std::vector<float> Detector::prepareImage(cv::Mat &src_img){
std::vector<float> result(INPUT_W * INPUT_H * 3);
float *data = result.data();
float ratio = float(INPUT_W) / float(src_img.cols) < float(INPUT_H) / float(src_img.rows) ? float(INPUT_W) / float(src_img.cols) : float(INPUT_H) / float(src_img.rows);
cv::Mat flt_img = cv::Mat::zeros(cv::Size(INPUT_W, INPUT_H), CV_8UC3);
cv::Mat rsz_img = cv::Mat::zeros(cv::Size(src_img.cols*ratio, src_img.rows*ratio), CV_8UC3);
cv::resize(src_img, rsz_img, cv::Size(), ratio, ratio);
rsz_img.copyTo(flt_img(cv::Rect(0, 0, rsz_img.cols, rsz_img.rows)));
flt_img.convertTo(flt_img, CV_32FC3);
int channelLength = INPUT_W * INPUT_H;
std::vector<cv::Mat> split_img = {
cv::Mat(INPUT_W, INPUT_H, CV_32FC1, data + channelLength * 2),
cv::Mat(INPUT_W, INPUT_H, CV_32FC1, data + channelLength),
cv::Mat(INPUT_W, INPUT_H, CV_32FC1, data)
};
cv::split(flt_img, split_img);
for (int i = 0; i < 3; i++) {
split_img[i] = (split_img[i] - img_mean[i]) / img_std[i];
}
return result;
}
//加載IR模型，初始化網絡
bool Detector::init(string xml_path,double cof_threshold,double nms_area_threshold,int input_w, int input_h, int num_class, int r_rows, int r_cols, std::vector<int> s, std::vector<float> i_mean,std::vector<float> i_std){
_xml_path = xml_path;
_cof_threshold = cof_threshold;
_nms_area_threshold = nms_area_threshold;
INPUT_W = input_w;
INPUT_H = input_h;
NUM_CLASS = num_class;
refer_rows = r_rows;
refer_cols = r_cols;
strides = s;
img_mean = i_mean;
img_std = i_std;
Core ie;
auto cnnNetwork = ie.ReadNetwork(_xml_path);
InputsDataMap inputInfo(cnnNetwork.getInputsInfo());
InputInfo::Ptr& input = inputInfo.begin()->second;
_input_name = inputInfo.begin()->first;
input->setPrecision(Precision::FP32);
input->getInputData()->setLayout(Layout::NCHW);
ICNNNetwork::InputShapes inputShapes = cnnNetwork.getInputShapes();
SizeVector& inSizeVector = inputShapes.begin()->second;
cnnNetwork.reshape(inputShapes);
_outputinfo = OutputsDataMap(cnnNetwork.getOutputsInfo());
for (auto &output : _outputinfo) {
output.second->setPrecision(Precision::FP32);
}
_network = ie.LoadNetwork(cnnNetwork, "CPU");
return true;
}
//模型推理及獲取輸出結果
vector<Detector::Bbox> Detector::process_frame(Mat& inframe){
cv::Mat showImage = inframe.clone();
std::vector<float> pr_img = prepareImage(inframe);
InferRequest::Ptr infer_request = _network.CreateInferRequestPtr();
Blob::Ptr frameBlob = infer_request->GetBlob(_input_name);
InferenceEngine::LockedMemory<void> blobMapped = InferenceEngine::as<InferenceEngine::MemoryBlob>(frameBlob)->wmap();
float* blob_data = blobMapped.as<float*>();
memcpy(blob_data, pr_img.data(), 3 * INPUT_H * INPUT_W * sizeof(float));
infer_request->Infer();
vector<Rect> origin_rect;
vector<float> origin_rect_cof;
int i=0;
vector<Bbox> bboxes;
for (auto &output : _outputinfo) {
auto output_name = output.first;
Blob::Ptr blob = infer_request->GetBlob(output_name);
LockedMemory<const void> blobMapped = as<MemoryBlob>(blob)->rmap();
float *output_blob = blobMapped.as<float *>();
bboxes = postProcess(showImage,output_blob);
++i;
}
return bboxes;
}
//對模型輸出結果進行解碼及nms
std::vector<Detector::Bbox> Detector::postProcess(const cv::Mat &src_img,
float *output) {
GenerateReferMatrix();
std::vector<Detector::Bbox> result;
float *out = output;
float ratio = std::max(float(src_img.cols) / float(INPUT_W), float(src_img.rows) / float(INPUT_H));
cv::Mat result_matrix = cv::Mat(refer_rows, NUM_CLASS + 4, CV_32FC1, out);
for (int row_num = 0; row_num < refer_rows; row_num++) {
Detector::Bbox box;
auto *row = result_matrix.ptr<float>(row_num);
auto max_pos = std::max_element(row + 4, row + NUM_CLASS + 4);
box.prob = row[max_pos - row];
if (box.prob < _cof_threshold)
continue;
box.classes = max_pos - row - 4;
auto *anchor = refer_matrix.ptr<float>(row_num);
box.x = (anchor[0] - row[0] * anchor[2] + anchor[0] + row[2] * anchor[2]) / 2 * ratio;
box.y = (anchor[1] - row[1] * anchor[2] + anchor[1] + row[3] * anchor[2]) / 2 * ratio;
box.w = (row[2] + row[0]) * anchor[2] * ratio;
box.h = (row[3] + row[1]) * anchor[2] * ratio;
result.push_back(box);
}
NmsDetect(result);
return result;
}//主要對圖片進行預處理，包括resize和歸一化
std::vector<float> Detector::prepareImage(cv::Mat &src_img){
std::vector<float> result(INPUT_W * INPUT_H * 3);
float *data = result.data();
float ratio = float(INPUT_W) / float(src_img.cols) < float(INPUT_H) / float(src_img.rows) ? float(INPUT_W) / float(src_img.cols) : float(INPUT_H) / float(src_img.rows);
cv::Mat flt_img = cv::Mat::zeros(cv::Size(INPUT_W, INPUT_H), CV_8UC3);
cv::Mat rsz_img = cv::Mat::zeros(cv::Size(src_img.cols*ratio, src_img.rows*ratio), CV_8UC3);
cv::resize(src_img, rsz_img, cv::Size(), ratio, ratio);
rsz_img.copyTo(flt_img(cv::Rect(0, 0, rsz_img.cols, rsz_img.rows)));
flt_img.convertTo(flt_img, CV_32FC3);
int channelLength = INPUT_W * INPUT_H;
std::vector<cv::Mat> split_img = {
cv::Mat(INPUT_W, INPUT_H, CV_32FC1, data + channelLength * 2),
cv::Mat(INPUT_W, INPUT_H, CV_32FC1, data + channelLength),
cv::Mat(INPUT_W, INPUT_H, CV_32FC1, data)
};
cv::split(flt_img, split_img);
for (int i = 0; i < 3; i++) {
split_img[i] = (split_img[i] - img_mean[i]) / img_std[i];
}
return result;
}
//加載IR模型，初始化網絡
bool Detector::init(string xml_path,double cof_threshold,double nms_area_threshold,int input_w, int input_h, int num_class, int r_rows, int r_cols, std::vector<int> s, std::vector<float> i_mean,std::vector<float> i_std){
_xml_path = xml_path;
_cof_threshold = cof_threshold;
_nms_area_threshold = nms_area_threshold;
INPUT_W = input_w;
INPUT_H = input_h;
NUM_CLASS = num_class;
refer_rows = r_rows;
refer_cols = r_cols;
strides = s;
img_mean = i_mean;
img_std = i_std;
Core ie;
auto cnnNetwork = ie.ReadNetwork(_xml_path);
InputsDataMap inputInfo(cnnNetwork.getInputsInfo());
InputInfo::Ptr& input = inputInfo.begin()->second;
_input_name = inputInfo.begin()->first;
input->setPrecision(Precision::FP32);
input->getInputData()->setLayout(Layout::NCHW);
ICNNNetwork::InputShapes inputShapes = cnnNetwork.getInputShapes();
SizeVector& inSizeVector = inputShapes.begin()->second;
cnnNetwork.reshape(inputShapes);
_outputinfo = OutputsDataMap(cnnNetwork.getOutputsInfo());
for (auto &output : _outputinfo) {
output.second->setPrecision(Precision::FP32);
}
_network = ie.LoadNetwork(cnnNetwork, "CPU");
return true;
}
//模型推理及獲取輸出結果
vector<Detector::Bbox> Detector::process_frame(Mat& inframe){
cv::Mat showImage = inframe.clone();
std::vector<float> pr_img = prepareImage(inframe);
InferRequest::Ptr infer_request = _network.CreateInferRequestPtr();
Blob::Ptr frameBlob = infer_request->GetBlob(_input_name);
InferenceEngine::LockedMemory<void> blobMapped = InferenceEngine::as<InferenceEngine::MemoryBlob>(frameBlob)->wmap();
float* blob_data = blobMapped.as<float*>();
memcpy(blob_data, pr_img.data(), 3 * INPUT_H * INPUT_W * sizeof(float));
infer_request->Infer();
vector<Rect> origin_rect;
vector<float> origin_rect_cof;
int i=0;
vector<Bbox> bboxes;
for (auto &output : _outputinfo) {
auto output_name = output.first;
Blob::Ptr blob = infer_request->GetBlob(output_name);
LockedMemory<const void> blobMapped = as<MemoryBlob>(blob)->rmap();
float *output_blob = blobMapped.as<float *>();
bboxes = postProcess(showImage,output_blob);
++i;
}
return bboxes;
}
//對模型輸出結果進行解碼及nms
std::vector<Detector::Bbox> Detector::postProcess(const cv::Mat &src_img,
float *output) {
GenerateReferMatrix();
std::vector<Detector::Bbox> result;
float *out = output;
float ratio = std::max(float(src_img.cols) / float(INPUT_W), float(src_img.rows) / float(INPUT_H));
cv::Mat result_matrix = cv::Mat(refer_rows, NUM_CLASS + 4, CV_32FC1, out);
for (int row_num = 0; row_num < refer_rows; row_num++) {
Detector::Bbox box;
auto *row = result_matrix.ptr<float>(row_num);
auto max_pos = std::max_element(row + 4, row + NUM_CLASS + 4);
box.prob = row[max_pos - row];
if (box.prob < _cof_threshold)
continue;
box.classes = max_pos - row - 4;
auto *anchor = refer_matrix.ptr<float>(row_num);
box.x = (anchor[0] - row[0] * anchor[2] + anchor[0] + row[2] * anchor[2]) / 2 * ratio;
box.y = (anchor[1] - row[1] * anchor[2] + anchor[1] + row[3] * anchor[2]) / 2 * ratio;
box.w = (row[2] + row[0]) * anchor[2] * ratio;
box.h = (row[3] + row[1]) * anchor[2] * ratio;
result.push_back(box);
}
NmsDetect(result);
return result;
}
0x5. 推理時間展示及預測結果展示平均推理時間6ms左右，CPU下實時推理安全帽檢測結果至此完成了NanoDet在X86 CPU上的部署，希望有幫助到大家。
歡迎關注GiantPandaCV/PandaCV, 在這裡你將看到獨家的深度學習分享，堅持原創，每天分享我們學習到的新鮮知識。( • ̀ω•́ )✧
有對文章相關的問題，或者想要加入交流群，歡迎添加BBuf微信：
二維碼為了方便讀者獲取資料以及我們公眾號的作者發布一些Github工程的更新，我們成立了一個QQ群，二維碼如下，感興趣可以加入。
公眾號QQ交流群