Skip to content

Commit

Permalink
add ax620e support yolo11, yolo11-seg, yolo11-pose
Browse files Browse the repository at this point in the history
  • Loading branch information
@BUG1989
BUG1989 committed Nov 4, 2024
1 parent d890c56 commit 853c692
Show file tree
Hide file tree
Showing 4 changed files with 513 additions and 0 deletions.
1 change: 1 addition & 0 deletions benchmark/Benchmark_AX630C.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ Benchmark 是了解硬件平台网络模型运行速度的最佳途径。**仅
| YOLOv6s | 640 | 13.708 |
| YOLOv7-Tiny | 640 | 12.508 |
| YOLOv8s | 640 | 16.912 |
| YOLO11s | 640 | 15.933 |
| YOLOX_s | 640 | 15.425 |
| PPYOLOE+_s | 640 | 14.885 |
| Deit_t | 224 | 3.350 |
Expand Down
2 changes: 2 additions & 0 deletions examples/ax620e/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ axera_example(ax_yolov9 ax_yolov9_steps.cc)
axera_example(ax_yolov10 ax_yolov10s_steps.cc)
axera_example(ax_yolov10_u ax_yolov10_ultralytics_steps.cc)
axera_example(ax_yolo11 ax_yolo11_steps.cc)
axera_example(ax_yolo11_seg ax_yolo11_seg_steps.cc)
axera_example(ax_yolo11_pose ax_yolo11_pose_steps.cc)
axera_example(ax_yolox ax_yolox_steps.cc)
axera_example(ax_yolo_world ax_yolo_world_steps.cc)
axera_example(ax_yolo_world_open_vocabulary ax_yolo_world_open_vocabulary_steps.cc)
Expand Down
252 changes: 252 additions & 0 deletions examples/ax620e/ax_yolo11_pose_steps.cc
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,252 @@
/*
* AXERA is pleased to support the open source community by making ax-samples available.
*
* Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved.
*
* Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* https://opensource.org/licenses/BSD-3-Clause
*
* Unless required by applicable law or agreed to in writing, software distributed
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/

/*
* Note: For the YOLO11 series exported by the ultralytics project.
* Author: ZHEQIUSHUI
* Author: QQC
*/

#include <cstdio>
#include <cstring>
#include <numeric>

#include <opencv2/opencv.hpp>
#include "base/common.hpp"
#include "base/detection.hpp"
#include "middleware/io.hpp"

#include "utilities/args.hpp"
#include "utilities/cmdline.hpp"
#include "utilities/file.hpp"
#include "utilities/timer.hpp"

#include <ax_sys_api.h>
#include <ax_engine_api.h>

const int DEFAULT_IMG_H = 640;
const int DEFAULT_IMG_W = 640;

const char* CLASS_NAMES[] = {
"person",
};
const std::vector<std::vector<uint8_t> > KPS_COLORS = {{0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, {51, 153, 255}};
const std::vector<std::vector<uint8_t> > LIMB_COLORS = {{51, 153, 255}, {51, 153, 255}, {51, 153, 255}, {51, 153, 255}, {255, 51, 255}, {255, 51, 255}, {255, 51, 255}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {255, 128, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}, {0, 255, 0}};
const std::vector<std::vector<uint8_t> > SKELETON = {{16, 14}, {14, 12}, {17, 15}, {15, 13}, {12, 13}, {6, 12}, {7, 13}, {6, 7}, {6, 8}, {7, 9}, {8, 10}, {9, 11}, {2, 3}, {1, 2}, {1, 3}, {2, 4}, {3, 5}, {4, 6}, {5, 7}};

int NUM_CLASS = 1;
int NUM_POINT = 17;

const int DEFAULT_LOOP_COUNT = 1;

const float PROB_THRESHOLD = 0.45f;
const float NMS_THRESHOLD = 0.45f;
namespace ax
{
void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const cv::Mat& mat, int input_w, int input_h, const std::vector<float>& time_costs)
{
std::vector<detection::Object> proposals;
std::vector<detection::Object> objects;
timer timer_postprocess;

float* output_ptr[3] = {(float*)io_data->pOutputs[0].pVirAddr, // 1*80*80*65
(float*)io_data->pOutputs[1].pVirAddr, // 1*40*40*65
(float*)io_data->pOutputs[2].pVirAddr}; // 1*20*20*65
float* output_kps_ptr[3] = {(float*)io_data->pOutputs[3].pVirAddr, // 1*80*80*51
(float*)io_data->pOutputs[4].pVirAddr, // 1*40*40*51
(float*)io_data->pOutputs[5].pVirAddr}; // 1*20*20*51

for (int i = 0; i < 3; ++i)
{
auto feat_ptr = output_ptr[i];
auto feat_kps_ptr = output_kps_ptr[i];
int32_t stride = (1 << i) * 8;
detection::generate_proposals_yolov8_pose_native(stride, feat_ptr, feat_kps_ptr, PROB_THRESHOLD, proposals, input_w, input_h, NUM_POINT, NUM_CLASS);
}

detection::get_out_bbox_kps(proposals, objects, NMS_THRESHOLD, input_h, input_w, mat.rows, mat.cols);
fprintf(stdout, "post process cost time:%.2f ms \n", timer_postprocess.cost());
fprintf(stdout, "--------------------------------------\n");
auto total_time = std::accumulate(time_costs.begin(), time_costs.end(), 0.f);
auto min_max_time = std::minmax_element(time_costs.begin(), time_costs.end());
fprintf(stdout,
"Repeat %d times, avg time %.2f ms, max_time %.2f ms, min_time %.2f ms\n",
(int)time_costs.size(),
total_time / (float)time_costs.size(),
*min_max_time.second,
*min_max_time.first);
fprintf(stdout, "--------------------------------------\n");
fprintf(stdout, "detection num: %zu\n", objects.size());

detection::draw_keypoints(mat, objects, KPS_COLORS, LIMB_COLORS, SKELETON, "yolo11_pose_out");
}

bool run_model(const std::string& model, const std::vector<uint8_t>& data, const int& repeat, cv::Mat& mat, int input_h, int input_w)
{
// 1. init engine
AX_ENGINE_NPU_ATTR_T npu_attr;
memset(&npu_attr, 0, sizeof(npu_attr));
npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_DISABLE;
auto ret = AX_ENGINE_Init(&npu_attr);
if (0 != ret)
{
return ret;
}

// 2. load model
std::vector<char> model_buffer;
if (!utilities::read_file(model, model_buffer))
{
fprintf(stderr, "Read Run-Joint model(%s) file failed.\n", model.c_str());
return false;
}

// 3. create handle
AX_ENGINE_HANDLE handle;
ret = AX_ENGINE_CreateHandle(&handle, model_buffer.data(), model_buffer.size());
SAMPLE_AX_ENGINE_DEAL_HANDLE
fprintf(stdout, "Engine creating handle is done.\n");

// 4. create context
ret = AX_ENGINE_CreateContext(handle);
SAMPLE_AX_ENGINE_DEAL_HANDLE
fprintf(stdout, "Engine creating context is done.\n");

// 5. set io
AX_ENGINE_IO_INFO_T* io_info;
ret = AX_ENGINE_GetIOInfo(handle, &io_info);
SAMPLE_AX_ENGINE_DEAL_HANDLE
fprintf(stdout, "Engine get io info is done. \n");

// 6. alloc io
AX_ENGINE_IO_T io_data;
ret = middleware::prepare_io(io_info, &io_data, std::make_pair(AX_ENGINE_ABST_DEFAULT, AX_ENGINE_ABST_CACHED));
SAMPLE_AX_ENGINE_DEAL_HANDLE
fprintf(stdout, "Engine alloc io is done. \n");

// 7. insert input
ret = middleware::push_input(data, &io_data, io_info);
SAMPLE_AX_ENGINE_DEAL_HANDLE_IO
fprintf(stdout, "Engine push input is done. \n");
fprintf(stdout, "--------------------------------------\n");

// 8. warn up
for (int i = 0; i < 5; ++i)
{
AX_ENGINE_RunSync(handle, &io_data);
}

// 9. run model
std::vector<float> time_costs(repeat, 0);
for (int i = 0; i < repeat; ++i)
{
timer tick;
ret = AX_ENGINE_RunSync(handle, &io_data);
time_costs[i] = tick.cost();
SAMPLE_AX_ENGINE_DEAL_HANDLE_IO
}

// 10. get result
post_process(io_info, &io_data, mat, input_w, input_h, time_costs);
fprintf(stdout, "--------------------------------------\n");

middleware::free_io(&io_data);
return AX_ENGINE_DestroyHandle(handle);
}
} // namespace ax

int main(int argc, char* argv[])
{
cmdline::parser cmd;
cmd.add<std::string>("model", 'm', "joint file(a.k.a. joint model)", true, "");
cmd.add<std::string>("image", 'i', "image file", true, "");
cmd.add<std::string>("size", 'g', "input_h, input_w", false, std::to_string(DEFAULT_IMG_H) + "," + std::to_string(DEFAULT_IMG_W));

cmd.add<int>("repeat", 'r', "repeat count", false, DEFAULT_LOOP_COUNT);
cmd.parse_check(argc, argv);

// 0. get app args, can be removed from user's app
auto model_file = cmd.get<std::string>("model");
auto image_file = cmd.get<std::string>("image");

auto model_file_flag = utilities::file_exist(model_file);
auto image_file_flag = utilities::file_exist(image_file);

if (!model_file_flag | !image_file_flag)
{
auto show_error = [](const std::string& kind, const std::string& value) {
fprintf(stderr, "Input file %s(%s) is not exist, please check it.\n", kind.c_str(), value.c_str());
};

if (!model_file_flag) { show_error("model", model_file); }
if (!image_file_flag) { show_error("image", image_file); }

return -1;
}

auto input_size_string = cmd.get<std::string>("size");

std::array<int, 2> input_size = {DEFAULT_IMG_H, DEFAULT_IMG_W};

auto input_size_flag = utilities::parse_string(input_size_string, input_size);

if (!input_size_flag)
{
auto show_error = [](const std::string& kind, const std::string& value) {
fprintf(stderr, "Input %s(%s) is not allowed, please check it.\n", kind.c_str(), value.c_str());
};

show_error("size", input_size_string);

return -1;
}

auto repeat = cmd.get<int>("repeat");

// 1. print args
fprintf(stdout, "--------------------------------------\n");
fprintf(stdout, "model file : %s\n", model_file.c_str());
fprintf(stdout, "image file : %s\n", image_file.c_str());
fprintf(stdout, "img_h, img_w : %d %d\n", input_size[0], input_size[1]);
fprintf(stdout, "--------------------------------------\n");

// 2. read image & resize & transpose
std::vector<uint8_t> image(input_size[0] * input_size[1] * 3, 0);
cv::Mat mat = cv::imread(image_file);
if (mat.empty())
{
fprintf(stderr, "Read image failed.\n");
return -1;
}
common::get_input_data_letterbox(mat, image, input_size[0], input_size[1]);

// 3. sys_init
AX_SYS_Init();

// 4. - engine model - can only use AX_ENGINE** inside
{
// AX_ENGINE_NPUReset(); // todo ??
ax::run_model(model_file, image, repeat, mat, input_size[0], input_size[1]);

// 4.3 engine de init
AX_ENGINE_Deinit();
// AX_ENGINE_NPUReset();
}
// 4. - engine model -

AX_SYS_Deinit();
return 0;
}
Loading

0 comments on commit 853c692

Please sign in to comment.