TDA4VE-Q1: A camera image mask node cause frame rate drops

Joy Yang

Part Number: TDA4VE-Q1

Tool/software:

I have modified the surround view app by adding a camera image mask node before going to the surround view node. I would like to mask the front camera image to a black image, i.e., memset to 0, and remain the other 3 camera images (right, rear, left). Then, send these 4 images into surround view node. I used memset to provide black image, and memcpy to remain the same images. But, I found that the frame rate will drop from 30fps to about 26fps.

Any suggestion?

Here is my code below:

// vx_kernels_3way_mask_target.h

#ifndef __VX_KERNELS_3WAY_MASK_TARGET_H__
#define __VX_KERNELS_3WAY_MASK_TARGET_H__

#include <stdio.h>
#include <VX/vx.h>
#include <VX/vx_compatibility.h>

vx_status vxMaskCameraImagesKernel(vx_node node, const vx_reference *parameters, vx_uint32 num);
vx_status tivxMaskLoadKernels(vx_context context);
vx_status tivxMaskUnloadKernels(vx_context context);

#endif

// vx_kernels_3way_mask_target.c

#include "vx_kernels_3way_mask_target.h"

vx_status vxMaskCameraImagesKernel(vx_node node, const vx_reference *parameters, vx_uint32 num)
{
    vx_object_array input_images = (vx_object_array)parameters[0];
    vx_uint32 width, height, cam_num; // [1], [2], [3]
    vx_array flags = (vx_array)parameters[4];
    vx_object_array output_images = (vx_object_array)parameters[5];

    // Get the parameters
    vxCopyScalar((vx_scalar)parameters[1], &width, VX_READ_ONLY, VX_MEMORY_TYPE_HOST);
    vxCopyScalar((vx_scalar)parameters[2], &height, VX_READ_ONLY, VX_MEMORY_TYPE_HOST);
    vxCopyScalar((vx_scalar)parameters[3], &cam_num, VX_READ_ONLY, VX_MEMORY_TYPE_HOST);

    vx_status status = VX_SUCCESS;

    // Get is_mask_cam array
    vx_uint8 is_mask_cam_flags[32] = {0}; // Support up to 32 channels
    vx_size num_items = 0;
    vxQueryArray(flags, VX_ARRAY_ATTRIBUTE_NUMITEMS, &num_items, sizeof(num_items));
    if(num_items > 32) num_items = 32;
    vxCopyArrayRange(flags, 0, num_items, sizeof(vx_uint8), is_mask_cam_flags, VX_READ_ONLY, VX_MEMORY_TYPE_HOST);

    vx_rectangle_t rect_y = {0, 0, width, height};
    vx_rectangle_t rect_uv = {0, 0, width, height / 2};

    for (vx_uint32 i = 0; i < cam_num; i++) {
        vx_image dst = (vx_image)vxGetObjectArrayItem(output_images, i);
        if (!dst || vxGetStatus((vx_reference)dst) != VX_SUCCESS) {
            printf("dst image invalid at cam %d\n", i);
            continue;
        }

        uint8_t mask_this_cam = (i < num_items) ? is_mask_cam_flags[i] : 0;

        if (mask_this_cam) {
            vx_imagepatch_addressing_t dst_addr;
            void *dst_ptr = NULL;
            vx_map_id map_id;

            // Y plane = 0
            status = vxMapImagePatch(dst, &rect_y, 0, &map_id, &dst_addr, &dst_ptr, VX_WRITE_ONLY, VX_MEMORY_TYPE_HOST, 0);
            if (status == VX_SUCCESS) {
                memset(dst_ptr, 0x00, dst_addr.stride_y * dst_addr.dim_y);
                vxUnmapImagePatch(dst, map_id);
            }
            // UV plane = 128
            status = vxMapImagePatch(dst, &rect_uv, 1, &map_id, &dst_addr, &dst_ptr, VX_WRITE_ONLY, VX_MEMORY_TYPE_HOST, 0);
            if (status == VX_SUCCESS) {
                memset(dst_ptr, 0x80, dst_addr.stride_y * dst_addr.dim_y);
                vxUnmapImagePatch(dst, map_id);
            }
            vxReleaseImage(&dst);
        } else {
            vx_image src = (vx_image)vxGetObjectArrayItem(input_images, i);
            if (!src || vxGetStatus((vx_reference)src) != VX_SUCCESS) {
                printf("src image invalid at cam %d\n", i);
                continue;
            }

            vx_imagepatch_addressing_t src_addr, dst_addr;
            void *src_ptr = NULL, *dst_ptr = NULL;
            vx_map_id map_id_src, map_id_dst;

            // Copy Y plane
            status = vxMapImagePatch(src, &rect_y, 0, &map_id_src, &src_addr, &src_ptr, VX_READ_ONLY, VX_MEMORY_TYPE_HOST, 0);
            if (status == VX_SUCCESS) {
                status = vxMapImagePatch(dst, &rect_y, 0, &map_id_dst, &dst_addr, &dst_ptr, VX_WRITE_ONLY, VX_MEMORY_TYPE_HOST, 0);
                if (status == VX_SUCCESS) {
                    memcpy(dst_ptr, src_ptr, src_addr.stride_y * src_addr.dim_y);
                    vxUnmapImagePatch(dst, map_id_dst);
                }
                vxUnmapImagePatch(src, map_id_src);
            }
            // Copy UV plane
            src_ptr = dst_ptr = NULL;
            status = vxMapImagePatch(src, &rect_uv, 1, &map_id_src, &src_addr, &src_ptr, VX_READ_ONLY, VX_MEMORY_TYPE_HOST, 0);
            if (status == VX_SUCCESS) {
                status = vxMapImagePatch(dst, &rect_uv, 1, &map_id_dst, &dst_addr, &dst_ptr, VX_WRITE_ONLY, VX_MEMORY_TYPE_HOST, 0);
                if (status == VX_SUCCESS) {
                    memcpy(dst_ptr, src_ptr, src_addr.stride_y * src_addr.dim_y);
                    vxUnmapImagePatch(dst, map_id_dst);
                }
                vxUnmapImagePatch(src, map_id_src);
            }
            vxReleaseImage(&src);
            vxReleaseImage(&dst);
        }
    }

    return VX_SUCCESS;
}

vx_status tivxMaskLoadKernels(vx_context context)
{
    vx_kernel kernel = vxAddUserKernel(
        context,
        "mask_node",
        0x1001, // kernel id, please use a bigger and unique number
        vxMaskCameraImagesKernel,
        6, // 6 parameters
        NULL, NULL, NULL); // validate, input, output function, => NULL

    if (kernel == NULL) {
        printf("[MASK_KERNEL] vxAddUserKernel failed!\n");
        return VX_FAILURE;
    } else {
        printf("[MASK_KERNEL] vxAddUserKernel success!\n");
    }

    vxAddParameterToKernel(kernel, 0, VX_INPUT, VX_TYPE_OBJECT_ARRAY, VX_PARAMETER_STATE_REQUIRED); // input_images
    vxAddParameterToKernel(kernel, 1, VX_INPUT, VX_TYPE_SCALAR, VX_PARAMETER_STATE_REQUIRED); // width
    vxAddParameterToKernel(kernel, 2, VX_INPUT, VX_TYPE_SCALAR, VX_PARAMETER_STATE_REQUIRED); // height
    vxAddParameterToKernel(kernel, 3, VX_INPUT, VX_TYPE_SCALAR, VX_PARAMETER_STATE_REQUIRED); // cam_num
    vxAddParameterToKernel(kernel, 4, VX_INPUT, VX_TYPE_ARRAY, VX_PARAMETER_STATE_REQUIRED);  // is_mask_cam_array
    vxAddParameterToKernel(kernel, 5, VX_OUTPUT, VX_TYPE_OBJECT_ARRAY, VX_PARAMETER_STATE_REQUIRED); // output_images

    vxFinalizeKernel(kernel);
    vxReleaseKernel(&kernel);
    return VX_SUCCESS;
}

vx_status tivxMaskUnloadKernels(vx_context context)
{
    vx_kernel kernel = vxGetKernelByName(context, "mask_node");
    if (kernel) {
        vx_status s = vxRemoveKernel(kernel);
        if (s == VX_SUCCESS) {
            printf("[MASK_KERNEL] vxRemoveKernel success!\n");
            return VX_SUCCESS;
        } else {
            printf("[MASK_KERNEL] vxRemoveKernel failed with status: %d\n", s);
            return s;
        }
    } else {
        printf("[MASK_KERNEL] vxGetKernelByName failed, kernel not found!\n");
        return VX_FAILURE;
    }
}

// app_3way_mask_module.h

#ifndef _APP_3WAY_MASK_MODULE
#define _APP_3WAY_MASK_MODULE

#include <stdio.h>
#include "app_modules.h"

typedef struct {
    vx_node mask_node;
    vx_scalar s_width, s_height, s_camnum;
    vx_array flags;
    vx_object_array masked_images;
    uint8_t en_mask_overlay;
    uint8_t en_mask_node;
    uint8_t is_mask;
    vx_uint8 is_mask_cam0, is_mask_cam1, is_mask_cam2, is_mask_cam3;
    vx_image black_image;
} MaskObj;


vx_status app_init_3way_mask(vx_context context, MaskObj *maskObj, vx_uint32 width, vx_uint32 height, vx_uint32 cam_num);
vx_status app_create_graph_3way_mask(vx_graph graph, MaskObj *maskObj, vx_object_array input_images);
vx_status app_deinit_3way_mask(MaskObj *maskObj);
void app_delete_3way_mask(MaskObj *maskObj);

/* @} */

#endif

// app_3way_mask_module.c

#include "app_3way_mask_module.h"

vx_status app_init_3way_mask(vx_context context, MaskObj *maskObj, vx_uint32 width, vx_uint32 height, vx_uint32 cam_num)
{
    if (!maskObj->black_image)
        maskObj->black_image = vxCreateImage(context, width, height, VX_DF_IMAGE_NV12);

    maskObj->masked_images = vxCreateObjectArray(context, (vx_reference)maskObj->black_image, cam_num);

    maskObj->flags = vxCreateArray(context, VX_TYPE_UINT8, cam_num);
    vx_uint8 cam_flags[4] = {maskObj->is_mask_cam0, maskObj->is_mask_cam1, maskObj->is_mask_cam2, maskObj->is_mask_cam3};
    vxAddArrayItems(maskObj->flags, cam_num, cam_flags, sizeof(vx_uint8));

    maskObj->s_width = vxCreateScalar(context, VX_TYPE_UINT32, &width);
    maskObj->s_height = vxCreateScalar(context, VX_TYPE_UINT32, &height);
    maskObj->s_camnum = vxCreateScalar(context, VX_TYPE_UINT32, &cam_num);

    return VX_SUCCESS;
}

vx_status app_create_graph_3way_mask(vx_graph graph, MaskObj *maskObj, vx_object_array input_images)
{
    vx_kernel mask_kernel = vxGetKernelByName(vxGetContext((vx_reference)graph), "mask_node");
    maskObj->mask_node = vxCreateGenericNode(graph, mask_kernel);
    vxReleaseKernel(&mask_kernel);

    vxSetParameterByIndex(maskObj->mask_node, 0, (vx_reference)input_images);
    vxSetParameterByIndex(maskObj->mask_node, 1, (vx_reference)maskObj->s_width);
    vxSetParameterByIndex(maskObj->mask_node, 2, (vx_reference)maskObj->s_height);
    vxSetParameterByIndex(maskObj->mask_node, 3, (vx_reference)maskObj->s_camnum);
    vxSetParameterByIndex(maskObj->mask_node, 4, (vx_reference)maskObj->flags);
    vxSetParameterByIndex(maskObj->mask_node, 5, (vx_reference)maskObj->masked_images);

    if (vxGetStatus((vx_reference)maskObj->mask_node) != VX_SUCCESS) {
        APP_PRINTF("maskObj.mask_node create failed\n");
        return VX_FAILURE;
    }

    return VX_SUCCESS;
}

vx_status app_deinit_3way_mask(MaskObj *maskObj)
{
    // get cam_num integer
    vx_uint32 cam_num = 0;
    vxCopyScalar(maskObj->s_camnum, &cam_num, VX_READ_ONLY, VX_MEMORY_TYPE_HOST);
    if (cam_num == 0) {
        APP_PRINTF("cam_num is zero, nothing to deinit\n");
        return VX_SUCCESS;
    }

    vxReleaseScalar(&maskObj->s_width);
    vxReleaseScalar(&maskObj->s_height);
    vxReleaseScalar(&maskObj->s_camnum);
    for (vx_uint32 i = 0; i < cam_num; i++) {
        vx_image masked_image = (vx_image)vxGetObjectArrayItem(maskObj->masked_images, i);
        if (masked_image) {
            vxReleaseImage(&masked_image);
        }
    }
    vxReleaseObjectArray(&maskObj->masked_images);

    if (maskObj->black_image) {
        vxReleaseImage(&maskObj->black_image);
        maskObj->black_image = NULL;
    }

    return VX_SUCCESS;
}

void app_delete_3way_mask(MaskObj *maskObj)
{
    if(maskObj->mask_node != NULL)
    {
        vxReleaseNode(&maskObj->mask_node);
    }
}

// main.c

status = app_create_graph_3way_mask(obj->graph, &obj->maskObj, obj->ldcObj.output_arr);
status = app_create_graph_srv_proc(
                    obj->graph, &obj->srvObj, obj->maskObj.masked_images,
                    NULL, NULL);

11 months ago

0 Gokul S 11 months ago

TI__Mastermind 25550 points

Hi Joy,

1. Can you share the graph's dot output file to check if the buffers are created properly with appropriate buffer depths.

2. Also could you check the performance of each node's execution by using the performance print utils, pressing 'p' in the interactive mode gives the statistics. you can compare it with the default demo to check which node is the bottleneck.

3. Instead of memcpy of the camera channels (which you don't want to mask) you can swap the elements of the input and output object_array to avoid unnecessary memory copy overhead.
you can refer to this file tiovx/kernels/openvx-ext/target/vx_obj_array_split_target.c for swapping the elements of object_array, and implement similar method for swapping specific elements of obj_arr.

Regards,
Gokul

Processors

Processors forum

TDA4VE-Q1: A camera image mask node cause frame rate drops