Skip to content

Commit

Permalink
add depth anything (#125)
Browse files Browse the repository at this point in the history 
* add depth anything

* add depth anything

* edit readme
  • Loading branch information
@ZHEQIUSHUI
ZHEQIUSHUI authored Feb 4, 2024
1 parent d26c3f3 commit 4646905
Show file tree
Hide file tree
Showing 4 changed files with 266 additions and 2 deletions.
Binary file added docs/ax650/depth_anything_out.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
1 change: 1 addition & 0 deletions examples/ax650/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ axera_example(ax_pfld ax_pfld_steps.cc)
axera_example(ax_dinov2 ax_dinov2_steps.cc)
axera_example(ax_simcc_pose ax_simcc_pose_steps.cc)
axera_example(ax_glpdepth ax_glpdepth_steps.cc)
axera_example(ax_depth_anything ax_depth_anything_steps.cc)
axera_example(ax_rtdetr ax_rtdetr_steps.cc)

axera_example(ax_imgproc ax_imgproc_steps.cc)
Expand Down
26 changes: 24 additions & 2 deletions examples/ax650/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -510,9 +510,9 @@ Repeat 1 times, avg time 4.79 ms, max_time 4.79 ms, min_time 4.79 ms

### GLPDepth
```
/opt/test # ./ax_glpdepth -m install/bin/models/glpdepth_896x1152.axmodel -i test.jpg
/opt/test # ./ax_glpdepth -m glpdepth_896x1152.axmodel -i test.jpg
--------------------------------------
model file : install/bin/models/glpdepth_896x1152.axmodel
model file : glpdepth_896x1152.axmodel
image file : test.jpg
img_h, img_w : 896 1152
--------------------------------------
Expand All @@ -529,6 +529,28 @@ Repeat 1 times, avg time 258.29 ms, max_time 258.29 ms, min_time 258.29 ms
```
<img src="../../docs/ax650/glpdepth_out.png">

### Depth-Anything
```
/opt/test # ./ax_depth_anything -m depth_anything.axmodel -i ssd_horse.jpg
--------------------------------------
model file : depth_anything.axmodel
image file : ssd_horse.jpg
img_h, img_w : 518 518
--------------------------------------
Engine creating handle is done.
Engine creating context is done.
Engine get io info is done.
Engine alloc io is done.
Engine push input is done.
--------------------------------------
post process cost time:9.36 ms
--------------------------------------
Repeat 1 times, avg time 56.67 ms, max_time 56.67 ms, min_time 56.67 ms
--------------------------------------
```
<img src="../../docs/ax650/depth_anything_out.png">

### YoloV8-Seg
```
/opt/test # ./ax_yolov8_seg -m yolov8n_seg.axmodel -i ssd_horse.jpg
Expand Down
241 changes: 241 additions & 0 deletions examples/ax650/ax_depth_anything_steps.cc
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,241 @@
/*
* AXERA is pleased to support the open source community by making ax-samples available.
*
* Copyright (c) 2022, AXERA Semiconductor (Shanghai) Co., Ltd. All rights reserved.
*
* Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
*
* https://opensource.org/licenses/BSD-3-Clause
*
* Unless required by applicable law or agreed to in writing, software distributed
* under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/

/*
* Author: ZHEQIUSHUI
*/

#include <cstdio>
#include <cstring>
#include <numeric>

#include <opencv2/opencv.hpp>
#include "base/common.hpp"
#include "base/detection.hpp"
#include "middleware/io.hpp"

#include "utilities/args.hpp"
#include "utilities/cmdline.hpp"
#include "utilities/file.hpp"
#include "utilities/timer.hpp"

#include <ax_sys_api.h>
#include <ax_engine_api.h>

const int DEFAULT_IMG_H = 518;
const int DEFAULT_IMG_W = 518;

const int DEFAULT_LOOP_COUNT = 1;

namespace ax
{
void post_process(AX_ENGINE_IO_INFO_T* io_info, AX_ENGINE_IO_T* io_data, const cv::Mat& mat, int input_w, int input_h, const std::vector<float>& time_costs)
{
timer timer_postprocess;
auto& output = io_data->pOutputs[0];
auto& info = io_info->pOutputs[0];

cv::Mat feature(info.pShape[2], info.pShape[3], CV_32FC1, output.pVirAddr);

double minVal, maxVal;
cv::minMaxLoc(feature, &minVal, &maxVal);

feature -= minVal;
feature /= (maxVal - minVal);
feature = 1.f - feature;
feature *= 255;

feature.convertTo(feature, CV_8UC1);

cv::Mat dst(info.pShape[2], info.pShape[3], CV_8UC3);
cv::applyColorMap(feature, dst, cv::ColormapTypes::COLORMAP_MAGMA);
cv::resize(dst, dst, cv::Size(mat.cols, mat.rows));

fprintf(stdout, "post process cost time:%.2f ms \n", timer_postprocess.cost());
fprintf(stdout, "--------------------------------------\n");
auto total_time = std::accumulate(time_costs.begin(), time_costs.end(), 0.f);
auto min_max_time = std::minmax_element(time_costs.begin(), time_costs.end());
fprintf(stdout,
"Repeat %d times, avg time %.2f ms, max_time %.2f ms, min_time %.2f ms\n",
(int)time_costs.size(),
total_time / (float)time_costs.size(),
*min_max_time.second,
*min_max_time.first);
fprintf(stdout, "--------------------------------------\n");
cv::hconcat(std::vector<cv::Mat>{mat, dst}, dst);
cv::imwrite("depth_anything_out.png", dst);
}

bool run_model(const std::string& model, const std::vector<uint8_t>& data, const int& repeat, cv::Mat& mat, int input_h, int input_w)
{
// 1. init engine
#ifdef AXERA_TARGET_CHIP_AX620E
auto ret = AX_ENGINE_Init();
#else
AX_ENGINE_NPU_ATTR_T npu_attr;
memset(&npu_attr, 0, sizeof(npu_attr));
npu_attr.eHardMode = AX_ENGINE_VIRTUAL_NPU_DISABLE;
auto ret = AX_ENGINE_Init(&npu_attr);
#endif
if (0 != ret)
{
return ret;
}

// 2. load model
std::vector<char> model_buffer;
if (!utilities::read_file(model, model_buffer))
{
fprintf(stderr, "Read Run-Joint model(%s) file failed.\n", model.c_str());
return false;
}

// 3. create handle
AX_ENGINE_HANDLE handle;
ret = AX_ENGINE_CreateHandle(&handle, model_buffer.data(), model_buffer.size());
SAMPLE_AX_ENGINE_DEAL_HANDLE
fprintf(stdout, "Engine creating handle is done.\n");

// 4. create context
ret = AX_ENGINE_CreateContext(handle);
SAMPLE_AX_ENGINE_DEAL_HANDLE
fprintf(stdout, "Engine creating context is done.\n");

// 5. set io
AX_ENGINE_IO_INFO_T* io_info;
ret = AX_ENGINE_GetIOInfo(handle, &io_info);
SAMPLE_AX_ENGINE_DEAL_HANDLE
fprintf(stdout, "Engine get io info is done. \n");

// 6. alloc io
AX_ENGINE_IO_T io_data;
ret = middleware::prepare_io(io_info, &io_data, std::make_pair(AX_ENGINE_ABST_DEFAULT, AX_ENGINE_ABST_CACHED));
SAMPLE_AX_ENGINE_DEAL_HANDLE
fprintf(stdout, "Engine alloc io is done. \n");

// 7. insert input
ret = middleware::push_input(data, &io_data, io_info);
SAMPLE_AX_ENGINE_DEAL_HANDLE_IO
fprintf(stdout, "Engine push input is done. \n");
fprintf(stdout, "--------------------------------------\n");

// 8. warn up
for (int i = 0; i < 5; ++i)
{
AX_ENGINE_RunSync(handle, &io_data);
}

// 9. run model
std::vector<float> time_costs(repeat, 0);
for (int i = 0; i < repeat; ++i)
{
timer tick;
ret = AX_ENGINE_RunSync(handle, &io_data);
time_costs[i] = tick.cost();
SAMPLE_AX_ENGINE_DEAL_HANDLE_IO
}

// 10. get result
post_process(io_info, &io_data, mat, input_w, input_h, time_costs);
fprintf(stdout, "--------------------------------------\n");

middleware::free_io(&io_data);
return AX_ENGINE_DestroyHandle(handle);
}
} // namespace ax

int main(int argc, char* argv[])
{
cmdline::parser cmd;
cmd.add<std::string>("model", 'm', "joint file(a.k.a. joint model)", true, "");
cmd.add<std::string>("image", 'i', "image file", true, "");
cmd.add<std::string>("size", 'g', "input_h, input_w", false, std::to_string(DEFAULT_IMG_H) + "," + std::to_string(DEFAULT_IMG_W));

cmd.add<int>("repeat", 'r', "repeat count", false, DEFAULT_LOOP_COUNT);
cmd.parse_check(argc, argv);

// 0. get app args, can be removed from user's app
auto model_file = cmd.get<std::string>("model");
auto image_file = cmd.get<std::string>("image");

auto model_file_flag = utilities::file_exist(model_file);
auto image_file_flag = utilities::file_exist(image_file);

if (!model_file_flag | !image_file_flag)
{
auto show_error = [](const std::string& kind, const std::string& value) {
fprintf(stderr, "Input file %s(%s) is not exist, please check it.\n", kind.c_str(), value.c_str());
};

if (!model_file_flag) { show_error("model", model_file); }
if (!image_file_flag) { show_error("image", image_file); }

return -1;
}

auto input_size_string = cmd.get<std::string>("size");

std::array<int, 2> input_size = {DEFAULT_IMG_H, DEFAULT_IMG_W};

auto input_size_flag = utilities::parse_string(input_size_string, input_size);

if (!input_size_flag)
{
auto show_error = [](const std::string& kind, const std::string& value) {
fprintf(stderr, "Input %s(%s) is not allowed, please check it.\n", kind.c_str(), value.c_str());
};

show_error("size", input_size_string);

return -1;
}

auto repeat = cmd.get<int>("repeat");

// 1. print args
fprintf(stdout, "--------------------------------------\n");
fprintf(stdout, "model file : %s\n", model_file.c_str());
fprintf(stdout, "image file : %s\n", image_file.c_str());
fprintf(stdout, "img_h, img_w : %d %d\n", input_size[0], input_size[1]);
fprintf(stdout, "--------------------------------------\n");

// 2. read image & resize & transpose
std::vector<uint8_t> image(input_size[0] * input_size[1] * 3, 0);
cv::Mat mat = cv::imread(image_file);
if (mat.empty())
{
fprintf(stderr, "Read image failed.\n");
return -1;
}
common::get_input_data_no_letterbox(mat, image, input_size[0], input_size[1], true);

// 3. sys_init
AX_SYS_Init();

// 4. - engine model - can only use AX_ENGINE** inside
{
// AX_ENGINE_NPUReset(); // todo ??
ax::run_model(model_file, image, repeat, mat, input_size[0], input_size[1]);

// 4.3 engine de init
AX_ENGINE_Deinit();
// AX_ENGINE_NPUReset();
}
// 4. - engine model -

AX_SYS_Deinit();
return 0;
}

0 comments on commit 4646905

Please sign in to comment.