J784S4XEVM: Zero Copy from V4L2 (CSI camera) to vx_image and put to VPAC

Ninh Nguyen

Part Number: J784S4XEVM

Tool/software:

Hi TI Experts,

I am working capture pipline and want to zero copy v4l2 buffer to vx_image and using VPAC LDC to process color format to NV12.

1. V4L2 setup

- Setup v4l2 with V4L2_MEMORY_MMAP

2. Create vx_image from V4l2 buffer

- Using vxCreateImageFromHandle to create vx_image

=> I can use vxMapImagePatch this vx_image and convert to cv::Mat and view this frame.

3. Processing vx_image with LDC VPAC

- Put the vx_image (created by vxCreateImageFromHandle) to LDC VPAC, the output is incorrect.

Ninh,

BRS

9 months ago

0 Nikhil Dasan 9 months ago

TI__Guru* 84926 points

Hi,

May I know the format of the image created in vx_image? Is it also NV12?

Could you share the code where you are doing this?

Regards,

Nikhil

0 Ninh Nguyen 9 months ago in reply to Nikhil Dasan

Prodigy 160 points

Hi Nikhil Dasan,

I using the camera with UYVY output format with v4l2 interface.

- V4l2 interface + memcpy uyvy buffer to UYVY vx_image

=> VPAC LDC output correct the NV12 format.

- V4l2 interface + createImagefromHandle with UYVY v4l2 buffer
=> VPAC LDC output incorrect.

The code like bellow

#define DEVICE "/dev/video0"
#define WIDTH  640
#define HEIGHT 480
#define FRAME_COUNT 100  // Số khung hình muốn chụp

struct buffer {
    void *start;
    size_t length;
};

int main() {
    int fd = open(DEVICE, O_RDWR);
    if (fd == -1) {
        perror("Opening video device");
        return 1;
    }

    struct v4l2_capability cap;
    if (ioctl(fd, VIDIOC_QUERYCAP, &cap) < 0) {
        perror("Querying Capabilities");
        return 1;
    }

    printf("Driver: %s\n", cap.driver);
    printf("Card: %s\n", cap.card);
    printf("Bus info: %s\n", cap.bus_info);

    struct v4l2_format fmt;
    fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    fmt.fmt.pix.width = WIDTH;
    fmt.fmt.pix.height = HEIGHT;
    fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV;  // Sử dụng định dạng YUYV
    fmt.fmt.pix.field = V4L2_FIELD_NONE;

    if (ioctl(fd, VIDIOC_S_FMT, &fmt) < 0) {
        perror("Setting Pixel Format");
        return 1;
    }

    struct v4l2_requestbuffers req;
    req.count = 1;
    req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    req.memory = V4L2_MEMORY_MMAP;

    if (ioctl(fd, VIDIOC_REQBUFS, &req) < 0) {
        perror("Requesting Buffer");
        return 1;
    }

    struct buffer buffer;
    struct v4l2_buffer buf;
    buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    buf.memory = V4L2_MEMORY_MMAP;
    buf.index = 0;

    if (ioctl(fd, VIDIOC_QUERYBUF, &buf) < 0) {
        perror("Querying Buffer");
        return 1;
    }

    buffer.length = buf.length;
    buffer.start = mmap(NULL, buf.length, PROT_READ | PROT_WRITE, MAP_SHARED, fd, buf.m.offset);

    if (buffer.start == MAP_FAILED) {
        perror("Mapping Buffer");
        return 1;
    }

    if (ioctl(fd, VIDIOC_QBUF, &buf) < 0) {
        perror("Queue Buffer");
        return 1;
    }

    int type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    if (ioctl(fd, VIDIOC_STREAMON, &type) < 0) {
        perror("Start Capture");
        return 1;
    }

    /* VPAC setup */

    vx_graph vpac_graph;
    vx_user_data_object userDataOb;
    vx_node ldc_node;

    // Create the image in NV12 format
    vx_image vx_ldc_nv12_output = vxCreateImage(context, WIDTH, HEIGHT, VX_DF_IMAGE_NV12);
    if (image == NULL) {
        printf("Failed to create OpenVX image\n");
        vxReleaseContext(&context);
        return -1;
    }

    if (ioctl(fd, VIDIOC_DQBUF, &buf) < 0) {
        perror("Retrieve Frame");
        return 1;
    }
    printf("Frame %d captured (%d bytes)\n", i, buf.bytesused);
    
    // Set up the V4L2 UYVY buffer addressing and pointer array
    vx_imagepatch_addressing_t addr = {0};
    addr.dim_x = WIDTH;            // Image width (in pixels)
    addr.dim_y = HEIGHT;           // Image height (in pixels)
    addr.stride_x = 2;                   // Each pixel (UYVY) is 2 bytes
    addr.stride_y = WIDTH * 2;     // Each row in memory takes `width * 2` bytes due to UYVY format
    void *ptrs[1];                       // Single plane for UYVY format
    ptrs[0] = buffer.start;         // Pointer to the captured V4L2 buffer (UYVY format)
    // Create a vx_image from the UYVY buffer using zero-copy
    vx_image vx_handle_img = vxCreateImageFromHandle(vpac_context, VX_DF_IMAGE_UYVY, &addr, ptrs, VX_MEMORY_TYPE_HOST);
    if (vxGetStatus((vx_reference)vx_img) != VX_SUCCESS) 
    {
        std::cerr << "Error creating vx_image from handle" << std::endl;
        return -1;
    }

    userDataOb = vxCreateUserDataObject(vpac_context, "tivx_vpac_ldc_params_t", sizeof(tivx_vpac_ldc_params_t), NULL);
    tivx_vpac_ldc_params_t tivxVpacLdcParamsConfig;
    tivxVpacLdcParamsConfig.luma_interpolation_type = 1;
    tivxVpacLdcParamsConfig.input_align_12bit = 0;
    tivxVpacLdcParamsConfig.yc_mode = 0;
    tivxVpacLdcParamsConfig.init_x = 0;
    tivxVpacLdcParamsConfig.init_y = 0;
    status = vxGetStatus((vx_reference)userDataOb);
    if(status == VX_SUCCESS)
    {
        vxSetReferenceName((vx_reference)userDataOb, "ldc_node_config");
        status = vxCopyUserDataObject(userDataOb, 0,
                            sizeof(tivx_vpac_ldc_params_t),
                            &tivxVpacLdcParams,
                            VX_WRITE_ONLY,
                            VX_MEMORY_TYPE_HOST);
        if(status != VX_SUCCESS)
        {
            std::cerr <<"Unable to copy ldc config params into buffer!" << std::endl;
        }
    }
    ldc_node = tivxVpacLdcNode(header_graph, userDataOb, NULL,
                                       NULL, NULL,
                                       NULL, NULL, vx_handle_img,
                                       vx_ldc_nv12_output, NULL);
    if (status != VX_SUCCESS)
    {
        std::cerr <<"Error: LDC Node creation failed with status" << std::endl;
    }
    vxSetNodeTarget(ldc_node, VX_TARGET_STRING, TIVX_TARGET_VPAC_LDC1);
    
    status = vxVerifyGraph(vpac_graph);
    if (VX_SUCCESS != status)
    {
        std::cerr << "Vpac: Can not run vxVerifyGraph success !" << std::endl;
        return status;
    }

    status = vxProcessGraph(vpac_graph);
    if (VX_SUCCESS != status)
    {
        std::cerr << "Error processing graph" << std::endl;
        return (int)status;
    }

    if (cv::waitKey(30) == 'q') {
        break;
    }

    if (ioctl(fd, VIDIOC_QBUF, &buf) < 0) {
        perror("Queue Buffer");
        return 1;
    }

    if (ioctl(fd, VIDIOC_STREAMOFF, &type) < 0) {
        perror("Stop Capture");
        return 1;
    }

    munmap(buffer.start, buffer.length);
    close(fd);

    return 0;
}

#define DEVICE "/dev/video0"
#define WIDTH  640
#define HEIGHT 480
#define FRAME_COUNT 100

struct buffer {
    void *start;
    size_t length;
};

int main() {
    int fd = open(DEVICE, O_RDWR);
    if (fd == -1) {
        perror("Opening video device");
        return 1;
    }

    struct v4l2_capability cap;
    if (ioctl(fd, VIDIOC_QUERYCAP, &cap) < 0) {
        perror("Querying Capabilities");
        return 1;
    }

    printf("Driver: %s\n", cap.driver);
    printf("Card: %s\n", cap.card);
    printf("Bus info: %s\n", cap.bus_info);

    struct v4l2_format fmt;
    fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    fmt.fmt.pix.width = WIDTH;
    fmt.fmt.pix.height = HEIGHT;
    fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_YUYV;  // Sử dụng định dạng YUYV
    fmt.fmt.pix.field = V4L2_FIELD_NONE;

    if (ioctl(fd, VIDIOC_S_FMT, &fmt) < 0) {
        perror("Setting Pixel Format");
        return 1;
    }

    struct v4l2_requestbuffers req;
    req.count = 1;
    req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    req.memory = V4L2_MEMORY_MMAP;

    if (ioctl(fd, VIDIOC_REQBUFS, &req) < 0) {
        perror("Requesting Buffer");
        return 1;
    }

    struct buffer buffer;
    struct v4l2_buffer buf;
    buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    buf.memory = V4L2_MEMORY_MMAP;
    buf.index = 0;

    if (ioctl(fd, VIDIOC_QUERYBUF, &buf) < 0) {
        perror("Querying Buffer");
        return 1;
    }

    buffer.length = buf.length;
    buffer.start = mmap(NULL, buf.length, PROT_READ | PROT_WRITE, MAP_SHARED, fd, buf.m.offset);

    if (buffer.start == MAP_FAILED) {
        perror("Mapping Buffer");
        return 1;
    }

    if (ioctl(fd, VIDIOC_QBUF, &buf) < 0) {
        perror("Queue Buffer");
        return 1;
    }

    int type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
    if (ioctl(fd, VIDIOC_STREAMON, &type) < 0) {
        perror("Start Capture");
        return 1;
    }

    /* VPAC setup */

    vx_graph vpac_graph;
    vx_user_data_object userDataOb;
    vx_node ldc_node;

    // Create the image in NV12 format
    vx_image vx_ldc_nv12_output = vxCreateImage(context, WIDTH, HEIGHT, VX_DF_IMAGE_NV12);
    if (image == NULL) {
        printf("Failed to create OpenVX image\n");
        vxReleaseContext(&context);
        return -1;
    }

    if (ioctl(fd, VIDIOC_DQBUF, &buf) < 0) {
        perror("Retrieve Frame");
        return 1;
    }
    printf("Frame %d captured (%d bytes)\n", i, buf.bytesused);
    
    // Set up the V4L2 UYVY buffer addressing and pointer array
    vx_imagepatch_addressing_t addr = {0};
    addr.dim_x = WIDTH;            // Image width (in pixels)
    addr.dim_y = HEIGHT;           // Image height (in pixels)
    addr.stride_x = 2;                   // Each pixel (UYVY) is 2 bytes
    addr.stride_y = WIDTH * 2;     // Each row in memory takes `width * 2` bytes due to UYVY format
    void *ptrs[1];                       // Single plane for UYVY format
    ptrs[0] = buffer.start;         // Pointer to the captured V4L2 buffer (UYVY format)
    // Create a vx_image from the UYVY buffer using zero-copy
    vx_image vx_handle_img = vxCreateImageFromHandle(vpac_context, VX_DF_IMAGE_UYVY, &addr, ptrs, VX_MEMORY_TYPE_HOST);
    if (vxGetStatus((vx_reference)vx_img) != VX_SUCCESS) 
    {
        std::cerr << "Error creating vx_image from handle" << std::endl;
        return -1;
    }

    userDataOb = vxCreateUserDataObject(vpac_context, "tivx_vpac_ldc_params_t", sizeof(tivx_vpac_ldc_params_t), NULL);
    tivx_vpac_ldc_params_t tivxVpacLdcParamsConfig;
    tivxVpacLdcParamsConfig.luma_interpolation_type = 1;
    tivxVpacLdcParamsConfig.input_align_12bit = 0;
    tivxVpacLdcParamsConfig.yc_mode = 0;
    tivxVpacLdcParamsConfig.init_x = 0;
    tivxVpacLdcParamsConfig.init_y = 0;
    status = vxGetStatus((vx_reference)userDataOb);
    if(status == VX_SUCCESS)
    {
        vxSetReferenceName((vx_reference)userDataOb, "ldc_node_config");
        status = vxCopyUserDataObject(userDataOb, 0,
                            sizeof(tivx_vpac_ldc_params_t),
                            &tivxVpacLdcParams,
                            VX_WRITE_ONLY,
                            VX_MEMORY_TYPE_HOST);
        if(status != VX_SUCCESS)
        {
            std::cerr <<"Unable to copy ldc config params into buffer!" << std::endl;
        }
    }
    ldc_node = tivxVpacLdcNode(header_graph, userDataOb, NULL,
                                       NULL, NULL,
                                       NULL, NULL, vx_handle_img,
                                       vx_ldc_nv12_output, NULL);
    if (status != VX_SUCCESS)
    {
        std::cerr <<"Error: LDC Node creation failed with status" << std::endl;
    }
    vxSetNodeTarget(ldc_node, VX_TARGET_STRING, TIVX_TARGET_VPAC_LDC1);
    
    status = vxVerifyGraph(vpac_graph);
    if (VX_SUCCESS != status)
    {
        std::cerr << "Vpac: Can not run vxVerifyGraph success !" << std::endl;
        return status;
    }

    status = vxProcessGraph(vpac_graph);
    if (VX_SUCCESS != status)
    {
        std::cerr << "Error processing graph" << std::endl;
        return (int)status;
    }

    if (cv::waitKey(30) == 'q') {
        break;
    }

    if (ioctl(fd, VIDIOC_QBUF, &buf) < 0) {
        perror("Queue Buffer");
        return 1;
    }

    if (ioctl(fd, VIDIOC_STREAMOFF, &type) < 0) {
        perror("Stop Capture");
        return 1;
    }

    munmap(buffer.start, buffer.length);
    close(fd);

    return 0;
}

0 Nikhil Dasan 9 months ago in reply to Ninh Nguyen

TI__Guru* 84926 points

Ninh Nguyen said:
- V4l2 interface + createImagefromHandle with UYVY v4l2 buffer
=> VPAC LDC output incorrect.

Hi,

As your buffer is not created using tiovxMemAlloc, it would not have a physical address mapped to the virtual address you are sharing.

To check this, inside you check what is the physical address mapped to mem_ptr->shared_ptr in createImagefromHandle()?

Regards,

nikhil

0 Ninh Nguyen 9 months ago in reply to Nikhil Dasan

Prodigy 160 points

Hi, Nikhil thanks for respond

Is there any way success to zero copy to vx_image from v4l2 ?
How about fd_dma export/import for buffer ?

0 Nikhil Dasan 9 months ago in reply to Ninh Nguyen

TI__Guru* 84926 points

Hi,

Can you try memory as buf.memory = V4L2_MEMORY_DMABUF instead of buf.memory = V4L2_MEMORY_MMAP; ?

You can refer the below

https://github.com/TexasInstruments/edgeai-tiovx-apps/blob/cf9150154a8cd9c417190d6554170de224564685/modules/src/v4l2_capture_module.c#L266

With this I believe you should get a dmabuf. Again, after this please check the below

Nikhil Dasan said:
To check this, inside you check what is the physical address mapped to mem_ptr->shared_ptr in createImagefromHandle()?

Regards,

Nikhil

0 Ninh Nguyen 9 months ago in reply to Nikhil Dasan

Prodigy 160 points

Hi Nikhil,
Refer to your link and other link about DMA-BUF v4l2, I can fix my issue.

Thanks,
Ninh Nguyễn

Processors

Processors forum

J784S4XEVM: Zero Copy from V4L2 (CSI camera) to vx_image and put to VPAC