Is the Nx Cloud up? Visit our Status Page for the current health and performance of the Nx Cloud.

Status Page

Step 3. How to Add Object Detection

Nx Support
Updated

This guide explains how to implement single-object detection and tracking using the OpenCV framework and a pre-trained MobileNet SSD Caffe model.

Prerequisites and terminology

Before you begin, familiarize yourself with these key terms:

  • Object: A subset of pixels extracted from a video frame based on a detection algorithm.

  • Object metadata: Information about the detected object, represented in the Server by the ObjectMetadata class. This includes the bounding box, object type, and other attributes.

  • Object track: A sequence of digital frames showing the same real-world object over time. The integration determines whether distinct objects belong to the same track.

Install and link the OpenCV framework

Configure your build environment to automatically download, configure, and link the OpenCV library.

1. Configure the Conan profile

Add the Conan repository and update your default profile settings by running the following commands in your terminal:

conan remote add conancenter https://center.conan.io
conan profile new default
conan profile update settings.arch=x86_64 default
conan profile update settings.compiler=gcc default
conan profile update settings.compiler.version=12 default
conan profile update settings.compiler.libcxx=libstdc++11 default
conan profile update settings.build_type=Release default
conan profile update settings.os=Linux default

2. Update CMakeLists.txt

Open your CMakeLists.txt file. Locate the if(metadataSdkDir STREQUAL "") clause, and insert the following block directly underneath the set(metadataSdkDir...) statement:

# Download OpenCV automatically using Conan
if(NOT EXISTS "${CMAKE_BINARY_DIR}/conan.cmake")
    message(STATUS "Downloading conan.cmake from https://github.com/conan-io/cmake-conan")
    file(DOWNLOAD "https://raw.githubusercontent.com/conan-io/cmake-conan/master/conan.cmake" 
        "${CMAKE_BINARY_DIR}/conan.cmake")
endif()
include(${CMAKE_BINARY_DIR}/conan.cmake)

if(WIN32)
    set(CONAN_VS_RUNTIME_MT_SETTING "compiler.runtime=MT") 
endif()
set(OPENCV_OPTIONS
    opencv:parallel=False
    opencv:contrib=True
    opencv:contrib_freetype=False
    opencv:contrib_sfm=False
    opencv:with_jpeg=False
    opencv:with_png=False
    opencv:with_tiff=False
    opencv:with_jpeg2000=False
    opencv:with_openexr=False
    opencv:with_eigen=False
    opencv:with_webp=False
    opencv:with_quirc=False
    opencv:with_cuda=False
    opencv:with_cublas=False
    opencv:dnn=True
)

if(UNIX)
    set(OPENCV_OPTIONS
    ${OPENCV_OPTIONS}
    opencv:fPIC=True
    opencv:with_gtk=False
    opencv:with_cufft=False
    opencv:with_v4l=False
    )
endif()

conan_cmake_run(BUILD_TYPE "Release")
conan_cmake_configure(REQUIRES opencv/4.1.2
    GENERATORS cmake_find_package
    OPTIONS ${OPENCV_OPTIONS}
    SETTINGS ${CONAN_VS_RUNTIME_MT_SETTING} 
)

conan_cmake_install(PATH_OR_REFERENCE .
    OPTIONS ${OPENCV_OPTIONS}
    SETTINGS ${CONAN_VS_RUNTIME_MT_SETTING} 
    BUILD missing
)

find_package(OpenCV REQUIRED)

3. Link OpenCV libraries statically

Add the following snippet to CMakeLists.txt to collect and link the required submodules:

set(CMAKE_EXE_LINKER_FLAGS " -static")
target_link_libraries(opencv_object_detection_analytics_plugin
nx_kit
nx_sdk
opencv::core opencv::flann opencv::imgproc opencv::imgcodecs opencv::dnn opencv::opencv_dnn opencv::ml
opencv::plot opencv::opencv_features2d opencv::opencv_calib3d opencv::datasets opencv::video opencv::tracking
   -static-libgcc
   -static-libstdc++
)

4. Build the project

Run the following commands to clear previous artifacts and compile the project. The initial build will take several minutes to download OpenCV. Subsequent builds use the cached version.

rm -rf $BUILD_DIR/*
cmake -DmetadataSdkDir=~/develop/metadata_sdk -B $BUILD_DIR ./step3
cmake --build $BUILD_DIR --config Release -- -j

Install and link MobileNet SSD Caffe models

This integration uses a pre-trained MobileNet SSD Caffe model because it offers fast processing execution on standard CPUs.

1. Add download logic to CMakeLists.txt

Insert this script into your CMakeLists.txt file to automate model downloads:

if(NOT DEFINED ARTIFACTORY_URL)
        set(ARTIFACTORY_URL "https://resources.vmsproxy.com/nx_open_integrations")
endif()

# Download model files
set(model_file_names
    MobileNetSSD.caffemodel
    MobileNetSSD.prototxt
)
foreach(model_file_name IN LISTS model_file_names)
    set(model_file "${CMAKE_BINARY_DIR}/${model_file_name}")
    list(APPEND model_files "${model_file}")
    if(NOT EXISTS "${model_file}")
        message("Downloading ${model_file_name}")
        file(DOWNLOAD
                "${ARTIFACTORY_URL}/opencv/${model_file_name}"
                "${model_file}"
                SHOW_PROGRESS
                STATUS DOWNLOAD_STATUS
        )
        list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
        if(NOT STATUS_CODE EQUAL 0)
            file(REMOVE "${model_file}")
            message(FATAL_ERROR "Error downloading ${ARTIFACTORY_URL}/opencv/${model_file_name}")
        endif()
    elseif()
        message("${model_file_name} already exists")
    endif()
endforeach()

2. Download the files

In your IDE or build system, click Build > Run CMake to download the model assets.

Organize third-party library files

To ensure stable runtime loading, place all third-party binaries, libraries (.so, .a, .lib, .dll), and model files in a dedicated plugin directory on the Server.

Run the following commands to stop the server, create the directory structure, and copy the artifacts:

sudo systemctl stop networkoptix-mediaserver
sudo mkdir $SERVER_DIR/bin/plugins/opencv_object_detection_analytics_plugin
sudo rm $SERVER_DIR/bin/plugins/libopencv_object_detection_analytics_plugin.so
sudo cp $BUILD_DIR/libopencv_object_detection_analytics_plugin.so $SERVER_DIR/bin/plugins/opencv_object_detection_analytics_plugin
sudo cp $BUILD_DIR/MobileNetSSD.caffemodel $BUILD_DIR/MobileNetSSD.prototxt $SERVER_DIR/bin/plugins/opencv_object_detection_analytics_plugin

Verify that your directory structure matches the following layout:

$ ls -1 $SERVER_DIR/bin/plugins/opencv_object_detection_analytics_plugin
libopencv_object_detection_analytics_plugin.so
MobileNetSSD.caffemodel
MobileNetSSD.prototxt

Obtain the integration home directory

To prevent path resolution errors across different host systems, use the Metadata SDK's homeDir() method to discover the correct integration directory path dynamically.

Modify the following source files to pass the pluginHomeDir argument to the Engine and DeviceAgent components.

engine.h

class Engine: public nx::sdk::analytics::Engine
{
public:
    explicit Engine(std::filesystem::path pluginHomeDir);
// ...
private:
    std::filesystem::path m_pluginHomeDir;
};

engine.cpp

Engine::Engine(std::filesystem::path pluginHomeDir):
    nx::sdk::analytics::Engine(/*enableOutput*/ true),
    m_pluginHomeDir(pluginHomeDir)
{
}

void Engine::doObtainDeviceAgent(Result<IDeviceAgent*>* outResult, const IDeviceInfo* deviceInfo)
{
    *outResult = new DeviceAgent(deviceInfo, m_pluginHomeDir);
}

device_agent.h

public:
DeviceAgent(
     const nx::sdk::IDeviceInfo* deviceInfo,
     std::filesystem::path pluginHomeDir);

device_agent.cpp

DeviceAgent::DeviceAgent(
    const nx::sdk::IDeviceInfo* deviceInfo,
    std::filesystem::path pluginHomeDir):
VideoFrameProcessingDeviceAgent(deviceInfo, /*enableOutput*/ true)
{
}

integration.cpp

Result<IEngine*> Plugin::doObtainEngine()
{
    const auto utilityProvider = this->utilityProvider();
    const std::filesystem::path pluginHomeDir = utilityProvider->homeDir();
    return new Engine(pluginHomeDir);
}

Define data structures for object detection

OpenCV structures are not directly compatible with VMS Server telemetry formats. Create two helper structures to convert data between formats.

  • detection.h: Stores Server-specific metadata.

  • frame.h: Converts raw server frames into OpenCV matrices.

detection.h

#include <nx/sdk/analytics/rect.h>
#include <nx/sdk/uuid.h>
struct Detection
{
    nx::sdk::analytics::Rect boundingBox;
    std::string classLabel;
    float confidence;
    nx::sdk::Uuid trackId;
};

frame.h

#include <opencv2/core/core.hpp>
#include <nx/sdk/analytics/i_uncompressed_video_frame.h>
struct Frame
{
    const int width;
    const int height;
    const int64_t timestampUs;
    const int64_t index;
    const cv::Mat cvMat;
public:
    Frame(const nx::sdk::analytics::IUncompressedVideoFrame* frame, int64_t index):
        width(frame->width()),
        height(frame->height()),
        timestampUs(frame->timestampUs()),
        index(index),
        cvMat({
            /*_rows*/ height,
            /*_cols*/ width,
            /*_type*/ CV_8UC3, //< BGR color space default
            /*_data*/ (void*) frame->data(0),
            /*_step*/ (size_t) frame->lineSize(0),
        })
    {
    }
};
NOTE: The complete source code implementation details are located in the opencv_object_detection_analytics_plugin/step3/src/sample_company/vms_server_plugins/opencv_object_detection directory. The samples below illustrate core plugin concepts.

Implement error handling

To protect the Server from crashing when exceptions occur, catch all runtime errors inside DeviceAgent::processFrame() and convert them into either method result errors or diagnostic notifications via pushPluginDiagnosticEvent().

Define exception classes (exceptions.h)

class Error: public std::runtime_error { using std::runtime_error::runtime_error; };
class ObjectDetectorError: public Error { using Error::Error; };
class ObjectDetectorInitializationError: public ObjectDetectorError { using ObjectDetectorError::ObjectDetectorError; };
class ObjectDetectorIsTerminatedError: public ObjectDetectorError { using ObjectDetectorError::ObjectDetectorError; };
class ObjectDetectionError: public ObjectDetectorError { using ObjectDetectorError::ObjectDetectorError; };

inline std::string cvExceptionToStdString(const cv::Exception& e)
{
    return "OpenCV error: " + e.err + " (error code: " + std::to_string(e.code) + ")";
}

Prevent error flooding

If an integration encounters an unrecoverable error, it must halt further execution loop calls so it does not flood the client with repetitive notification messages. Use a boolean flag m_terminated to handle this state.

bool DeviceAgent::pushUncompressedVideoFrame(const IUncompressedVideoFrame* videoFrame)
{
    m_terminated = m_terminated || m_objectDetector->isTerminated();
    if (m_terminated)
    {
        if (!m_terminatedPrevious)
        {
             pushPluginDiagnosticEvent(
                 IPluginDiagnosticEvent::Level::error,
                 "integration is in broken state.",
                 "Disable the integration.");
             m_terminatedPrevious = true;
        }
        return true;
    }
    const MetadataPacketList metadataPackets = processFrame(videoFrame);
}

Skip frames during processing

Processing every frame can degrade host performance. Balance your computing resources by evaluating objects at a fixed frame interval using kDetectionFramePeriod.

  • 10 FPS is typically optimal for multi-object tracking.

  • 1 FPS is sufficient for general object classification.

bool DeviceAgent::pushUncompressedVideoFrame(const IUncompressedVideoFrame* videoFrame)
{
    m_lastVideoFrameTimestampUs = videoFrame->timestampUs();
    
    // Detect objects only on targeted intervals
    if (m_frameIndex % kDetectionFramePeriod == 0)
    {
        const MetadataPacketList metadataPackets = processFrame(videoFrame);
        for (const Ptr<IMetadataPacket>& metadataPacket: metadataPackets)
        {
            metadataPacket->addRef();
            pushMetadataPacket(metadataPacket.get());
        }
    }
    ++m_frameIndex;
    return true;
}

Initialize the ObjectDetector class

The ObjectDetector loads and manages the model. Invoke ensureInitialized() inside DeviceAgent::doSetNeededMetadataTypes() to ensure setup occurs only once when the Server updates expected metadata types.

1. Initialize inside DeviceAgent

void DeviceAgent::doSetNeededMetadataTypes(
    nx::sdk::Result<void>* outValue,
    const nx::sdk::analytics::IMetadataTypes* /*neededMetadataTypes*/)
{
    if (m_terminated)
         return;
    try
    {
         m_objectDetector->ensureInitialized();
    }
    catch (const ObjectDetectorInitializationError& e)
    {
         *outValue = {ErrorCode::otherError, new String(e.what())};
         m_terminated = true;
    }
    catch (const ObjectDetectorIsTerminatedError& /*e*/)
    {
         m_terminated = true;
    }
}

2. Implement check and load logic

void ObjectDetector::ensureInitialized()
{
    if (isTerminated())
    {
       throw ObjectDetectorIsTerminatedError(
            "Object detector initialization error: object detector is terminated.");
    }
    if (m_netLoaded)
        return;
    try
    {
        loadModel();
    }
    catch (const cv::Exception& e)
    {
        terminate();
        throw ObjectDetectorInitializationError("Loading model: " + cvExceptionToStdString(e));
    }
    catch (const std::exception& e)
    {
        terminate();
        throw ObjectDetectorInitializationError("Loading model: Error: "s + e.what());
    }
}

void ObjectDetector::loadModel()
{
    static const auto modelBin = m_modelPath / std::filesystem::path("MobileNetSSD.caffemodel");
    static const auto modelTxt = m_modelPath / std::filesystem::path("MobileNetSSD.prototxt");

    m_net = std::make_unique<Net>(readNetFromCaffe(modelTxt.string(), modelBin.string()));
    m_netLoaded = !m_net->empty();
    if (!m_netLoaded)
         throw ObjectDetectorInitializationError("Loading model: network is empty.");
}

Filter target object classes and confidence levels

The MobileNet SSD Caffe model includes pre-defined target categories in its network definition array (kClasses). Filter these predictions to only surface objects that match your intended classification scope (kClassesToDetect) and exceed your required confidence threshold.

  • Low confidence threshold (e.g., < 0.5): Generates more detections but increases false positives.

  • High confidence threshold (e.g., > 0.5): Increases accuracy but might lead to missed detections (false negatives).

static std::string kClasses[] = {
"background", "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat",
"chair", "cow", "dining table", "dog", "horse", "motorbike", "person", "potted plant",
"sheep", "sofa", "train", "tv monitor"
};
const std::vector<std::string> kClassesToDetect{"cat", "dog", "person"};

// Filter within detection execution
static constexpr float confidenceThreshold = 0.5F;
const int& i = detectionIndex;
const float confidence = rawDetections.at<float>(i, (int) OutputIndex::confidence);
const auto classIndex = (int) (rawDetections.at<float>(i, (int) OutputIndex::classIndex));
const std::string classLabel = kClasses[(size_t) classIndex];

const bool confidentDetection = confidence >= confidenceThreshold;
bool oneOfRequiredClasses = std::find(kClassesToDetect.begin(), kClassesToDetect.end(), classLabel) != kClassesToDetect.end();

if (confidentDetection && oneOfRequiredClasses)
{ 
    // Accept and process object metadata
}

Run frame processing and inference

When the server passes an uncompressed video frame to pushUncompressedVideoFrame(), use your inference processing block to run detection, convert the structures, and return telemetry back to the server pipeline.

DeviceAgent::MetadataPacketList DeviceAgent::processFrame(const IUncompressedVideoFrame* videoFrame)
{
    const Frame frame(videoFrame, m_frameIndex);
    try
    {
         DetectionList detections = m_objectDetector->run(frame);
         const auto& objectMetadataPacket = detectionsToObjectMetadataPacket(detections, frame.timestampUs);
         MetadataPacketList result;
         if (objectMetadataPacket)
             result.push_back(objectMetadataPacket);
         return result;
    }
    catch (const ObjectDetectionError& e)
    {
         pushPluginDiagnosticEvent(
             IPluginDiagnosticEvent::Level::error,
             "Object detection error.",
             e.what());
         m_terminated = true;
    }
    return {};
}

Let’s build your integration and see how it works.

Related articles

Comments

0 comments

Article is closed for comments.