YOLOv8实时目标检测
基于rknn_model_zoo/examples/yolov8/cpp进行改造，采用OpenCV采集usb摄像头数据，并通过yolov8模型进行实时目标检测。
环境准备
1、板端准备好要修改的rknn_model_zoo代码
git clone https://github.com/airockchip/rknn_model_zoo.git --depth 1
2、yoloV8模型参考RKNN上手指南，将yolov8模型转换为rknn模型，并将模型拷贝到板端。
代码修改
可以通过交叉编译rknn_model_zoo的yolov8示例代码，再拷贝到板端进行测试。如果板端CPU性能足够，可以直接在板端进行编译。本文采用板端编译的方式。
注意
做好原有代码的备份，避免修改错误导致编译失败。
1、修改rknn_model_zoo/examples/yolov8/cpp/CMakeLists.txt，添加库的依赖，解决编译错误问题。
CMakeLists.txt
cmake_minimum_required(VERSION 3.4.1)
project(rknn_yolov8_demo)
# Install model files
install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../model/
DESTINATION ${CMAKE_INSTALL_PREFIX}/model
FILES_MATCHING
PATTERN "*.rknn"
PATTERN "*.txt"
PATTERN "README.md" EXCLUDE)
if (ENABLE_ASAN)
message(STATUS "BUILD WITH ADDRESS SANITIZER")
set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -fno-omit-frame-pointer -fsanitize=address")
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer -fsanitize=address")
set (CMAKE_LINKER_FLAGS_DEBUG "${CMAKE_LINKER_FLAGS_DEBUG} -fno-omit-frame-pointer -fsanitize=address")
endif ()
# Find pthread package
find_package(Threads REQUIRED)
set(rknpu_yolov8_file rknpu2/yolov8.cc)
if (TARGET_SOC STREQUAL "rv1106" OR TARGET_SOC STREQUAL "rv1103")
add_definitions(-DRV1106_1103)
set(rknpu_yolov8_file rknpu2/yolov8_rv1106_1103.cc)
#dma
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../../3rdparty/allocator/dma)
endif()
if(TARGET_SOC STREQUAL "rk1808" OR TARGET_SOC STREQUAL "rv1109" OR TARGET_SOC STREQUAL "rv1126")
add_definitions(-DRKNPU1)
set(rknpu_yolov8_file rknpu1/yolov8.cc)
endif()
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../../../3rdparty/ 3rdparty.out)
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../../../utils/ utils.out)
set(CMAKE_INSTALL_RPATH "$ORIGIN/../lib")
file(GLOB SRCS ${CMAKE_CURRENT_SOURCE_DIR}/*.cc)
# Include directories
include_directories(
/usr/include/opencv4
/usr/include/aarch64-linux-gnu/opencv4
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_SOURCE_DIR}/../../../include
${CMAKE_SOURCE_DIR}/rknpu2
${CMAKE_SOURCE_DIR}/../../../3rdparty
${CMAKE_SOURCE_DIR}/../../../3rdparty/rknpu2/include
)
# Link directories
link_directories(
${CMAKE_SOURCE_DIR}/../../../3rdparty/rknpu2/lib
/usr/lib/aarch64-linux-gnu
)
# Remove duplicate target definitions
if(NOT TARGET imageutils)
add_library(imageutils STATIC ${CMAKE_SOURCE_DIR}/../../../utils/image_utils.cc)
endif()
if(NOT TARGET fileutils)
add_library(fileutils STATIC ${CMAKE_SOURCE_DIR}/../../../utils/file_utils.cc)
endif()
if(NOT TARGET imagedrawing)
add_library(imagedrawing STATIC ${CMAKE_SOURCE_DIR}/../../../utils/image_drawing.cc)
endif()
# Add executables (only if they don't exist)
if(NOT TARGET ${PROJECT_NAME})
add_executable(${PROJECT_NAME} main.cc postprocess.cc ${rknpu_yolov8_file})
target_link_libraries(${PROJECT_NAME}
imageutils
fileutils
imagedrawing
${LIBRKNNRT}
opencv_core
opencv_imgproc
opencv_imgcodecs
opencv_videoio
opencv_highgui
${CMAKE_THREAD_LIBS_INIT}
dl
rga
)
endif()
# Install
install(TARGETS ${PROJECT_NAME} DESTINATION ./)
2、修改rknn_model_zoo/examples/yolov8/cpp/main.cc，通过OpenCV采集usb摄像头数据，并通过yolov8模型进行实时目标检测。
main.cc
// Copyright (c) 2023 by Rockchip Electronics Co., Ltd. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*-------------------------------------------
Includes
-------------------------------------------*/
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <fstream>
#include <vector>
#include <string>
#include "yolov8.h"
#include "image_utils.h"
#include "file_utils.h"
#include "image_drawing.h"
#include <opencv2/opencv.hpp>
#include <chrono>
#if defined(RV1106_1103)
#include "dma_alloc.hpp"
#endif
/*-------------------------------------------
Main Function
-------------------------------------------*/
// Add FPS counter class
class FPSCounter {
private:
std::chrono::time_point<std::chrono::steady_clock> last_time;
float fps;
int frame_count;
public:
FPSCounter() : fps(0), frame_count(0) {
last_time = std::chrono::steady_clock::now();
}
void update() {
frame_count++;
auto current_time = std::chrono::steady_clock::now();
auto time_diff = std::chrono::duration_cast<std::chrono::milliseconds>(current_time - last_time).count();
if (time_diff >= 1000) { // Update every second
fps = frame_count * 1000.0f / time_diff;
frame_count = 0;
last_time = current_time;
}
}
float getFPS() { return fps; }
};
std::vector<std::string> load_labels(const char* filename) {
std::vector<std::string> labels;
std::ifstream file(filename);
if (!file.is_open()) {
printf("Failed to open label file: %s\n", filename);
return labels;
}
std::string line;
while (std::getline(file, line)) {
if (!line.empty()) {
labels.push_back(line);
}
}
printf("Loaded %zu labels from %s\n", labels.size(), filename);
return labels;
}
void print_usage(const char* name) {
printf("Usage: %s <rknn_model> <video_device> [show_image] [label_file]\n", name);
printf(" show_image: 0 - disable display, 1 - enable display (default: 1)\n");
printf(" label_file: path to label file (default: coco_80_labels_list.txt)\n");
}
int main(int argc, char** argv)
{
if (argc < 3) {
print_usage(argv[0]);
return -1;
}
const char* model_path = argv[1];
const char* video_device = argv[2];
bool enable_display = (argc > 3) ? atoi(argv[3]) : 1; // 默认启用显示
const char* label_file = (argc > 4) ? argv[4] : "/home/toybrick/dev/rknn_model_zoo/examples/yolov8/model/coco_80_labels_list.txt";
// Load labels
std::vector<std::string> labels = load_labels(label_file);
if (labels.empty()) {
printf("No labels loaded, using 'unknown' for all classes\n");
}
// Initialize camera
cv::VideoCapture cap;
cap.open(video_device, cv::CAP_V4L2);
if (!cap.isOpened()) {
printf("Failed to open camera: %s\n", video_device);
return -1;
}
// Set camera properties
cap.set(cv::CAP_PROP_FRAME_WIDTH, 640);
cap.set(cv::CAP_PROP_FRAME_HEIGHT, 480);
cap.set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('Y', 'U', 'Y', 'V'));
cap.set(cv::CAP_PROP_FPS, 30);
cap.set(cv::CAP_PROP_BUFFERSIZE, 3);
// Create a window
if (enable_display) {
cv::namedWindow("YOLOv8 Detection", cv::WINDOW_AUTOSIZE);
}
// 等待摄像头初始化
printf("Waiting for camera initialization...\n");
sleep(2);
printf("Camera format: %s\n", "YUYV");
printf("Camera resolution: %dx%d\n",
(int)cap.get(cv::CAP_PROP_FRAME_WIDTH),
(int)cap.get(cv::CAP_PROP_FRAME_HEIGHT));
printf("Camera FPS: %d\n", (int)cap.get(cv::CAP_PROP_FPS));
// Initialize RKNN context
rknn_app_context_t app_ctx;
memset(&app_ctx, 0, sizeof(rknn_app_context_t));
int ret = init_yolov8_model(model_path, &app_ctx);
if (ret < 0) {
printf("Init model failed\n");
return -1;
}
FPSCounter fps_counter;
cv::Mat frame, rgb_frame;
int retry_count = 0;
const int max_retries = 5;
while (true) {
if (!cap.read(frame)) {
printf("Failed to capture frame, retry %d/%d\n", retry_count + 1, max_retries);
retry_count++;
if (retry_count >= max_retries) {
printf("Failed to capture frame after %d retries\n", max_retries);
break;
}
sleep(1);
continue;
}
retry_count = 0;
if (frame.empty()) {
printf("Empty frame captured\n");
continue;
}
try {
// Convert YUYV to BGR
if (frame.channels() == 2) {
cv::cvtColor(frame, rgb_frame, cv::COLOR_YUV2BGR_YUYV);
} else if (frame.channels() == 3) {
rgb_frame = frame.clone();
} else {
printf("Unsupported number of channels: %d\n", frame.channels());
continue;
}
// Create a square canvas with padding
cv::Mat square_frame = cv::Mat::zeros(640, 640, CV_8UC3);
int y_offset = (640 - 480) / 2; // Center vertically
// Copy the original frame to the center of the square canvas
cv::Mat roi = square_frame(cv::Rect(0, y_offset, 640, 480));
cv::resize(rgb_frame, roi, cv::Size(640, 480));
// Convert BGR to RGB for model input
cv::Mat input_frame;
cv::cvtColor(square_frame, input_frame, cv::COLOR_BGR2RGB);
// Prepare image buffer
image_buffer_t img_buffer;
memset(&img_buffer, 0, sizeof(image_buffer_t));
img_buffer.width = input_frame.cols;
img_buffer.height = input_frame.rows;
img_buffer.format = IMAGE_FORMAT_RGB888;
img_buffer.size = input_frame.total() * input_frame.elemSize();
img_buffer.virt_addr = (unsigned char*)input_frame.data;
// Detect objects
object_detect_result_list detect_result_list;
ret = inference_yolov8_model(&app_ctx, &img_buffer, &detect_result_list);
if (ret < 0) {
printf("Inference failed\n");
break;
}
// Print and draw detection results
for (int i = 0; i < detect_result_list.count; i++) {
object_detect_result* det = &detect_result_list.results[i];
// Get class name from loaded labels
std::string class_name = "unknown";
if (det->cls_id >= 0 && det->cls_id < labels.size()) {
class_name = labels[det->cls_id];
}
// Adjust coordinates back to original frame
int x1 = det->box.left;
int y1 = det->box.top - y_offset;
int x2 = det->box.right;
int y2 = det->box.bottom - y_offset;
// Only draw if the object is in the visible area
if (y1 >= 0 && y2 < 480) {
printf("Object %d: class=%s, score=%.2f%%, box=(%d,%d,%d,%d)\n",
i, class_name.c_str(), det->prop * 100,
x1, y1, x2, y2);
if (enable_display) {
// Draw bounding box
cv::rectangle(rgb_frame,
cv::Point(x1, y1),
cv::Point(x2, y2),
cv::Scalar(255, 0, 0), 2);
// Draw label
char text[256];
sprintf(text, "%s %.1f%%", class_name.c_str(), det->prop * 100);
cv::putText(rgb_frame, text,
cv::Point(x1, y1 - 5),
cv::FONT_HERSHEY_SIMPLEX, 0.5,
cv::Scalar(0, 255, 0), 2);
}
}
}
if (enable_display) {
// Show frame
cv::imshow("YOLOv8 Detection", rgb_frame);
// Wait for key press with a small delay
int key = cv::waitKey(1);
if (key == 'q' || key == 27) { // 'q' or ESC
printf("User requested exit\n");
break;
}
}
} catch (const cv::Exception& e) {
printf("OpenCV error: %s\n", e.what());
continue;
}
}
// Cleanup
cap.release();
if (enable_display) {
cv::destroyAllWindows();
}
release_yolov8_model(&app_ctx);
return 0;
}
板端编译
进入rknn_model_zoo目录后，执行如下命令进行编译：
./build-linux.sh -t rk3588 -a aarch64 -d yolov8
编译完成后，在rknn_model_zoo/install/rk3588_linux_aarch64/rknn_yolov8_demo目录下会生成rknn_yolov8_demo可执行文件。
板端运行
执行如下命令运行rknn_yolov8_demo：
./rknn_yolov8_demo model/yolov8s.rknn /dev/video41
在本地可以查看到摄像头画面。
温馨提示
- 如果摄像头没有画面，请检查摄像头是否正常连接。
- 如果画面显示不正常，请检测数据格式是否正确。
- 如果想用其他模型，请修改rknn_yolov8_demo中的模型路径、名称
- 本项目代码仅供参考，如果需要更复杂或稳定的目标检测，请自行修改代码。