AM5728: VIP driver issue

I tried the following changing the func vip_process_buffer_complete(struct vip_stream *stream)

static void vip_process_buffer_complete(struct vip_stream *stream)
{
	struct vip_dev *dev = stream->port->dev;
	struct vip_buffer *buf;
	struct vb2_v4l2_buffer *vb = NULL;
	unsigned long flags, fld;

	buf = list_first_entry(&stream->post_bufs, struct vip_buffer, list);

	if (stream->port->flags & FLAG_INTERLACED) {
		vpdma_unmap_desc_buf(dev->shared->vpdma,
				     &stream->desc_list.buf);

		fld = dtd_get_field(stream->write_desc);
		stream->field = fld ? V4L2_FIELD_BOTTOM : V4L2_FIELD_TOP;

		vpdma_map_desc_buf(dev->shared->vpdma, &stream->desc_list.buf);
	}

	if (buf) {
		vip_dbg(4, stream, "vip buffer complete 0x%x, 0x%x\n",
			(unsigned int)buf, buf->drop);

		vb = &buf->vb;
		vb->field = stream->field;
		vb->sequence = stream->sequence;
		vb->vb2_buf.timestamp = ktime_get_ns();

		if(dtd_get_field(stream->write_desc))
			buf->drop = true;

		if (buf->drop) {
			spin_lock_irqsave(&dev->slock, flags);
			list_move_tail(&buf->list, &stream->dropq);
			spin_unlock_irqrestore(&dev->slock, flags);
		} else {
			spin_lock_irqsave(&dev->slock, flags);
			list_del(&buf->list);
			spin_unlock_irqrestore(&dev->slock, flags);
			vb2_buffer_done(&vb->vb2_buf, VB2_BUF_STATE_DONE);
		}
	} else {
		vip_err(stream, "%s: buf is null!!!\n", __func__);
		return;
	}

	if(!dtd_get_field(stream->write_desc))
		stream->sequence++;
}

--- vip.c	2019-09-13 19:11:24.887545132 +0300
+++ vip_new.c	2019-09-13 18:40:23.391501417 +0300
@@ -1036,6 +1036,9 @@
 		vb->sequence = stream->sequence;
 		vb->vb2_buf.timestamp = ktime_get_ns();
 
+		if(dtd_get_field(stream->write_desc))
+			buf->drop = true;
+
 		if (buf->drop) {
 			spin_lock_irqsave(&dev->slock, flags);
 			list_move_tail(&buf->list, &stream->dropq);
@@ -1051,7 +1054,8 @@
 		return;
 	}
 
-	stream->sequence++;
+	if(!dtd_get_field(stream->write_desc))
+		stream->sequence++;
 }
 
 static int vip_reset_vpdma(struct vip_stream *stream)

It does not work stably and not for long. Sometimes it works fine 25fps but after a while it drops to 12fps, then the picture stops

gst-launch-1.0 v4l2src device=/dev/video1 io-mode=4 ! 'video/x-raw,format=(string)YUY2, width=(int)704, height=(int)280' ! vpe num-input-buffers=8 ! 'video/x-raw,format=(string)YUY2, width=(int)704, height=(int)560' ! queue ! fpsdisplaysink fps-update-interval=1000 video-sink=waylandsink

After the stop Gstreamer, I saw the next error

[  217.484572] WARNING: CPU: 0 PID: 1209 at drivers/media/v4l2-core/videobuf2-core.c:1663 __vb2_queue_cancel+0x174/0x1d8
[  217.496101] Modules linked in: sha512_generic sha512_arm sha256_generic sha1_generic sha1_arm_neon sha1_arm md5 cbc aes_arm_bs crypto_simd cryptd xfrm_user xfrm4_tunnel ipcomp xfrm_ipcomp esp4 ah4 af_key xfrm_algo bc_example(O) xhci_plat_hcd pru_rproc xhci_hcd pruss_intc rpmsg_rpc rpmsg_proto pruss dwc3 udc_core snd_soc_simple_card snd_soc_simple_card_utils snd_soc_omap_hdmi_audio pvrsrvkm(O) tw2964(O) omap_aes_driver pruss_soc_bus ahci_platform libahci_platform libahci omap_sham omap_wdt libata ti_vip(O) scsi_mod phy_omap_usb2 ti_vpe ti_sc ti_csc v4l2_fwnode ti_vpdma dwc3_omap rtc_omap mcp251x c_can_platform c_can can_dev extcon_palmas omap_des rtc_palmas des_generic snd_soc_tlv320aic3x crypto_engine at24 omap_crypto pixcir_i2c_ts rtc_ds1307 omap_remoteproc virtio_rpmsg_bus rpmsg_core remoteproc
[  217.567355]  sch_fq_codel uio_module_drv(O) uio ftdi_sio usbserial usbcore usb_common gdbserverproxy(O) cryptodev(O) cmemk(O)
[  217.583616] CPU: 1 PID: 1209 Comm: gst-launch-1.0 Tainted: G           O    4.14.79-gbde58ab01e #1
[  217.592613] Hardware name: Generic DRA74X (Flattened Device Tree)
[  217.598729] Backtrace: 
[  217.601194] [<c020b4dc>] (dump_backtrace) from [<c020b7c0>] (show_stack+0x18/0x1c)
[  217.608797]  r7:00000009 r6:600f0013 r5:00000000 r4:c1053d90
[  217.614486] [<c020b7a8>] (show_stack) from [<c0932528>] (dump_stack+0x90/0xa4)
[  217.621742] [<c0932498>] (dump_stack) from [<c022b9cc>] (__warn+0xec/0x104)
[  217.628733]  r7:00000009 r6:c0c0f3d4 r5:00000000 r4:00000000
[  217.634416] [<c022b8e0>] (__warn) from [<c022ba9c>] (warn_slowpath_null+0x28/0x30)
[  217.642018]  r9:00000000 r8:00000013 r7:ec7c688c r6:ed241800 r5:dfbbef00 r4:ec7c688c
[  217.649799] [<c022ba74>] (warn_slowpath_null) from [<c072ebf4>] (__vb2_queue_cancel+0x174/0x1d8)
[  217.658625] [<c072ea80>] (__vb2_queue_cancel) from [<c072f1f0>] (vb2_core_streamoff+0x3c/0xa4)
[  217.667272]  r10:dfbbef00 r9:00000000 r8:00000013 r7:c071944c r6:ed241800 r5:dfbbef00
[  217.675132]  r4:ec7c688c r3:00000001
[  217.678725] [<c072f1b4>] (vb2_core_streamoff) from [<c073208c>] (vb2_streamoff+0x38/0x58)
[  217.686936]  r5:dfbbef00 r4:00000001
[  217.690526] [<c0732054>] (vb2_streamoff) from [<c07320f0>] (vb2_ioctl_streamoff+0x44/0x48)
[  217.698831] [<c07320ac>] (vb2_ioctl_streamoff) from [<c0719470>] (v4l_streamoff+0x24/0x28)
[  217.707128]  r5:40045613 r4:c07320ac
[  217.710722] [<c071944c>] (v4l_streamoff) from [<c071bdd4>] (__video_do_ioctl+0x30c/0x314)
[  217.718933]  r5:40045613 r4:00000001
[  217.722526] [<c071bac8>] (__video_do_ioctl) from [<c071b688>] (video_usercopy+0x78/0x49c)
[  217.730737]  r10:00000000 r9:00000000 r8:cc8f9e28 r7:00000000 r6:00000004 r5:00000001
[  217.738598]  r4:40045613
[  217.741143] [<c071b610>] (video_usercopy) from [<c071bac4>] (video_ioctl2+0x18/0x1c)
[  217.748920]  r10:d4613210 r9:cc8f8000 r8:ec4f8e74 r7:00258504 r6:40045613 r5:dfbbef00
[  217.756781]  r4:ed241800
[  217.759326] [<c071baac>] (video_ioctl2) from [<c0717d0c>] (v4l2_ioctl+0x84/0xe0)
[  217.766758] [<c0717c88>] (v4l2_ioctl) from [<c034ed3c>] (do_vfs_ioctl+0xa8/0x790)
[  217.774272]  r9:cc8f8000 r8:0000000e r7:40045613 r6:0000000e r5:dfbbef00 r4:00258504
[  217.782050] [<c034ec94>] (do_vfs_ioctl) from [<c034f460>] (SyS_ioctl+0x3c/0x60)
[  217.789390]  r10:00000036 r9:cc8f8000 r8:00258504 r7:40045613 r6:0000000e r5:dfbbef00
[  217.797251]  r4:dfbbef01
[  217.799799] [<c034f424>] (SyS_ioctl) from [<c0207c40>] (ret_fast_syscall+0x0/0x4c)
[  217.807399]  r9:cc8f8000 r8:c0207e44 r7:00000036 r6:b4f0a28c r5:b6ad6000 r4:b4f0a0d0
[  217.815948] ---[ end trace a40e5e9a3a4e4a96 ]---

Hi Oleg,

Can you describe what kind of jitter you see? Is it out of order display or reversal of the field or something else? 

If you just capture the image through Yavta, save in a file and then playback offline, how does that video looks like?

What happens if you use VPE for de-interlacing? For every interlaced field submitted to VPE, it's de-interlacer engine provides de-interlaced frame for it. So if you are feeding interlaced field of resolution 704x240@30 fps, it will produce output of 704x480 @ 30 fps. Have you saved this output  in a file and then did offline playback? 

As I mentioned in my other e2e post https://e2e.ti.com/support/processors/f/791/t/835475, AM57x DSS IP doesn't have context on the top/bottom field id. It will display in the order, images are fed. You need to root cause where is this jitter getting introduced. 

Regards,

Manisha

Thank you very much for your responsiveness.

manisha said:

Can you describe what kind of jitter you see? Is it out of order display or reversal of the field or something else? 

I did not understand the question. 

One half-frame intelaced video differs from the other by shifting by one line (after scaling 2x with vpe already by two lines). Since half-frames become frames, the difference in the two line generates jitter.

Therefore, if i leave only the upper or lower half-frames, this leads to the disappearance of jitter. Losing half of the frames leads to a loss of quality, but for me the lack of jitter and 25fps are more important.

manisha said:

If you just capture the image through Yavta, save in a file and then playback offline, how does that video looks like?

What happens if you use VPE for de-interlacing? For every interlaced field submitted to VPE, it's de-interlacer engine provides de-interlaced frame for it. So if you are feeding interlaced field of resolution 704x240@30 fps, it will produce output of 704x480 @ 30 fps. Have you saved this output  in a file and then did offline playback?

I will definitely do everything you write about. I will write about the result in the near future. For a quick answer, it seems to me that I have the same jitter as in the case of https://e2e.ti.com/support/processors/f/791/t/618455?pi320966=4

Manisha, so as not to lose time, can you give advice, where did I make a mistake in my modification of vip.c?

manisha said:

If you just capture the image through Yavta, save in a file and then playback offline, how does that video looks like?

Jitter is also on Yavta.

on target-am5728 side:

echo '\n' |  yavta -c100 -p -F/home/root/my.yuv --skip 2 -t 1/50 -f YUYV -s 704x280 /dev/video2

https://e2e.ti.com/cfs-file/__key/communityserver-discussions-components-files/791/my.yuv.7z

on host-PC side:

ffplay -f rawvideo -pix_fmt yuyv422 -video_size 704x280 -framerate 50 -i my.yuv

manisha said:

What happens if you use VPE for de-interlacing? For every interlaced field submitted to VPE, it's de-interlacer engine provides de-interlaced frame for it. So if you are feeding interlaced field of resolution 704x240@30 fps, it will produce output of 704x480 @ 30 fps. Have you saved this output  in a file and then did offline playback?

I cannot perform such an experiment at 30fps. Only 50fps is working correctly for me now. If I do thinning of frames with the help of the gst ( ! videorate ! 'video/x-raw, framerate=25/1' ! ), then of course there will not be a jitter since there will be no second half frame.
And There is a problem with video compression and recording tools via gst ( ducatimpeg4enc and ducatih264enc ) I can’t compress 50fps, only <=30fps

PS. Manisha, so as not to lose time, can you give advice, where did I make a mistake in my modification of vip.c?

Hi Oleg,

VPE is suppose to do the deinterlacing, so the two lines shift that you are experiencing is not expected. I will check with the VIP driver author if the changes you did are recommended/sufficient or something is missing.  

Oleg Butko said:

I cannot perform such an experiment at 30fps. Only 50fps is working correctly for me now.

You do not have to do the display at 30 fps. That was just an example I used. Use the resolution and framerate as supported by your display. 

Oleg Butko said:

There is a problem with video compression and recording tools via gst ( ducatimpeg4enc and ducatih264enc ) I can’t compress 50fps, only <=30fps

Not sure where this limitation is coming from. The codec library are as such agnostic of frame rates. 

Oleg Butko said:

PS. Manisha, so as not to lose time, can you give advice, where did I make a mistake in my modification of vip.c?

I will check on that and get back

I also recommend you to follow the VPE documentation on de-interlacing usage.

Regards,

Manisha

Thanks for the extensive answer.

manisha said:

I will check with the VIP driver author if the changes you did are recommended/sufficient or something is missing.  

Waiting for your answer.

manisha said:

I also recommend you to follow the VPE documentation on de-interlacing usage.

Thanks for the advice. I will study the vpe documentation again.

I would really appreciate if you pointed to a working example.

Hi Oleg,

It is not recommended to modify VIP driver to drop alternate frames as the queue management for the drop and regular descriptors are tricky. Rather, you can achieve same thing at the user space level following the filed_id parameter.

Before going to dropping the alternate filed, I think there is something fundamentally wrong in your setup. The output of VPE should have given you right de-interlaced frame (without any jitter).

I would like you to check if you have pin-muxing set for field-id properly (snippet below from datasheet for vin1 port). If that's not set properly, VIP won't capture the status of the field type properly and this can mess up the de-interlacing process. Second thing I wanted to ensure you are capturing interlace in alternate field mode. Third thing I wanted you to try capture + di-interlacing by vpe using v4l2-test-m2m application. This application is packaged in processor SDK file system. Source code can be found here https://git.ti.com/glsdk/example-applications/blobs/master/v4l2-m2m/test-v4l2-m2m.c.

4.3.1 VIP

NOTE

For more information, see the Video Input Port (VIP) section of the device TRM.

CAUTION

The I/O timings provided in Section 5.9 Timing Requirements and Switching

Characteristics are applicable for all combinations of signals for vin1. However,

the timings are only valid for vin2, vin3, and vin4 if signals within a single

IOSET are used. The IOSETs are defined in the Table 5-35, Table 5-36 and

Table 5-37.

Table 4-2. VIP Signal Descriptions

SIGNAL NAME DESCRIPTION TYPE BALL

Video Input 1

vin1a_clk0 Video Input 1 Port A Clock input.Input clock for 8-bit 16-bit or 24-bit Port A video

capture. Input data is sampled on the CLK0 edge.

I AG8

vin1a_de0 Video Input 1 Data Enable input (1) I AD9

vin1a_fld0 Video Input 1 Port A Field ID input (1) I AF9

vin1a_hsync0 Video Input 1 Port A Horizontal Sync input (1) I AE9

vin1a_vsync0 Video Input 1 Port A Vertical Sync input (1) I AF8

vin1a_d0 Video Input 1 Port A Data input (1) I AE8

vin1a_d1 Video Input 1 Port A Data input (1) I AD8

vin1a_d2 Video Input 1 Port A Data input (1) I AG7

Once the VPE de-interlacer is doing the right job, display shouldn't be a problem because at that time you are displaying progressive frames.

manisha said:

It is not recommended to modify VIP driver to drop alternate frames as the queue management for the drop and regular descriptors are tricky

It's a pity! I was hoping you could explain to me how to make the code patch I need.

1.

manisha said:

I would like you to check if you have pin-muxing set for field-id properly (snippet below from datasheet for vin1 port).

I want to apologize. I forgot to say that I capture video in Bt.656. And that means that I don’t have output pin vin1a_fld0, only (0..7)data and clk.

/dts-v1/;
/plugin/;
#include <dt-bindings/gpio/gpio.h>
#include <dt-bindings/pinctrl/dra.h>

&dra7_pmx_core {
    vin3a_pins_default: vin3a_pins_default {
	pinctrl-single,pins = <
	   /* DRA7XX_CORE_IOPAD(0x3460, MUX_MODE15) */		/* N7: gpmc_a8.vin3a_hsync0 */
	   /* DRA7XX_CORE_IOPAD(0x3464, MUX_MODE15) */		/* R4: gpmc_a9.vin3a_vsync0 */
	   /* DRA7XX_CORE_IOPAD(0x346C, MUX_MODE15) */		/* P9: gpmc_a11.vin3a_fld0  */
	   /* DRA7XX_CORE_IOPAD(0x3468, MUX_MODE15) */		/* N9: gpmc_a10.vin3a_de0   */
	    DRA7XX_CORE_IOPAD(0x34BC, MUX_MODE2 | PIN_INPUT)	/* P1: gpmc_cs3.vin3a_clk0  */

	    DRA7XX_CORE_IOPAD(0x3400, MUX_MODE2 | PIN_INPUT)	/* M6: gpmc_ad0.vin3a_d0 */
	    DRA7XX_CORE_IOPAD(0x3404, MUX_MODE2 | PIN_INPUT)	/* M2: gpmc_ad1.vin3a_d1 */
	    DRA7XX_CORE_IOPAD(0x3408, MUX_MODE2 | PIN_INPUT)	/* L5: gpmc_ad2.vin3a_d2 */
	    DRA7XX_CORE_IOPAD(0x340C, MUX_MODE2 | PIN_INPUT)	/* M1: gpmc_ad3.vin3a_d3 */
	    DRA7XX_CORE_IOPAD(0x3410, MUX_MODE2 | PIN_INPUT)	/* L6: gpmc_ad4.vin3a_d4 */
	    DRA7XX_CORE_IOPAD(0x3414, MUX_MODE2 | PIN_INPUT)	/* L4: gpmc_ad5.vin3a_d5 */
	    DRA7XX_CORE_IOPAD(0x3418, MUX_MODE2 | PIN_INPUT)	/* L3: gpmc_ad6.vin3a_d6 */
	    DRA7XX_CORE_IOPAD(0x341C, MUX_MODE2 | PIN_INPUT)	/* L2: gpmc_ad7.vin3a_d7 */
	>;
    };

	vin2a_pins_default: vin2a_pins_default {
	pinctrl-single,pins = <
	    DRA7XX_CORE_IOPAD(0x3554, MUX_MODE0 | PIN_INPUT)	/* E1 vin2a_clk0.vin2a_clk0 */

	    DRA7XX_CORE_IOPAD(0x3568, MUX_MODE0 | PIN_INPUT)	/* F2 vin2a_d0.vin2a_d0 */
	    DRA7XX_CORE_IOPAD(0x356C, MUX_MODE0 | PIN_INPUT)	/* F3 vin2a_d1.vin2a_d1 */
	    DRA7XX_CORE_IOPAD(0x3570, MUX_MODE0 | PIN_INPUT)	/* D1 vin2a_d2.vin2a_d2 */
	    DRA7XX_CORE_IOPAD(0x3574, MUX_MODE0 | PIN_INPUT)	/* E2 vin2a_d3.vin2a_d3 */
	    DRA7XX_CORE_IOPAD(0x3578, MUX_MODE0 | PIN_INPUT)	/* D2 vin2a_d4.vin2a_d4 */
	    DRA7XX_CORE_IOPAD(0x357C, MUX_MODE0 | PIN_INPUT)	/* F4 vin2a_d5.vin2a_d5 */
	    DRA7XX_CORE_IOPAD(0x3580, MUX_MODE0 | PIN_INPUT)	/* C1 vin2a_d6.vin2a_d6 */
	    DRA7XX_CORE_IOPAD(0x3584, MUX_MODE0 | PIN_INPUT)	/* E4 vin2a_d7.vin2a_d7 */
	>;
    };
};

&gpio6 {
	p11 {
		gpio-hog;
		gpios = <11 GPIO_ACTIVE_LOW>;
		output-high;
		line-name = "cm-camen-gpio";
	};
};

&vip2 {
    status = "okay";
};

&vin3a {
	status = "okay";
	pinctrl-name = "default";
	pinctrl-0 = <&vin3a_pins_default>;
	
	endpoint@0 {
		slave-mode;
		remote-endpoint = <&tw2964_1>;
		bus_width = <8>;
	};
};

&vip1 {
    status = "okay";
};

&vin2a {
	status = "okay";
	pinctrl-name = "default";
	pinctrl-0 = <&vin2a_pins_default>;
	
	endpoint@1 {
		slave-mode;
		remote-endpoint = <&tw2964_2>;
		bus_width = <8>;
	};
};

&i2c5 {
	tw2964: tw2964@29 {
		compatible = "techwell,tw2964";
		reg = <0x29>;
		
		reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>;
		
		port@0 {
			tw2964_1: endpoint@0 {
				remote-endpoint = <&vin3a>;
				channels = <4>;
				ain_num = <0>;	
			};
		};

		port@1 {
			tw2964_2: endpoint@1 {
				remote-endpoint = <&vin2a>;
				channels = <2>;
				ain_num = <3>;
			};
		};
	};
};

2.

manisha said:

Second thing I wanted to ensure you are capturing interlace in alternate field mode.

Here is the function from subdriver v4l2(drv2964.c) responsible for setting the mode for field = V4L2_FIELD_ALTERNATE

static int tw2964_fill_fmt(	struct v4l2_subdev *sd,
							struct v4l2_subdev_pad_config *cfg,
							struct v4l2_subdev_format *format)
{
	struct v4l2_mbus_framefmt *f;
	struct tw2964 *decoder = to_tw2964(sd);

	//DeVdistress
#if(0)
	if (!format || (format->pad != DEMOD_PAD_VID_OUT))
		return -EINVAL;
#else
	if (!format || format->pad)
		return -EINVAL;
#endif

	f = &format->format;

	//DeVdistress
#if(1)
	f->width  = decoder->rect.width;
#else
	f->width  = 700;
#endif

	f->height = decoder->rect.height / 2;

	f->code = MEDIA_BUS_FMT_UYVY8_2X8;
	f->field = V4L2_FIELD_ALTERNATE;
	f->colorspace = V4L2_COLORSPACE_SMPTE170M;

	dev_dbg_lvl(sd->dev, 1, debug, "width = %d, height = %d\n", f->width,
			f->height);

	return 0;
}

I will check that it may not be correctly transferred to the vip.c. Add a printk and show the dmsg log.

Now my experience is limited to using gstreamer, yavta, capturevpedisplay.

capturevpedisplay does not work for me. This is correctly written in the topic. https://e2e.ti.com/support/processors/f/791/t/835475#pi320966=2

root@am57xx-evm:~# capturevpedisplay 704 280 yuyv 704 560 yuyv 1 3 -s 35:800x480
vpe:/dev/video0 open success!!!
vip open success!!!
using 1 connectors, 800x480 display, multiplanar: 1
Setting mode 800x480 on connector 35, crtc 39
ERROR:post_buffer:298: Could not post buffer on crtc 39: Permission denied (-13)
vip: G_FMT(start): width = 704, height = 280, 4cc = YUYV
vpe i/p: G_FMT: width = 704, height = 280, 4cc = YUYV
vpe o/p: G_FMT: width = 704, height = 560, 4cc = YUYV
ERROR:post_buffer:298: Could not post buffer on crtc 39: Permission denied (-13)
ERROR:post_buffer:325: Timeout waiting for flip complete: Permission denied (0)
streaming started...
ERROR:post_vid_buffer:419: failed to enable plane 48: Permission denied
disp post vid buf failed

launch yavta is given by four answers above

gstreamer i run like this:

gst-launch-1.0 v4l2src device=/dev/video1 io-mode=4 ! 'video/x-raw,format=(string)YUY2, width=(int)704, height=(int)280' ! vpe num-input-buffers=8 ! 'video/x-raw,format=(string)YUY2, width=(int)704, height=(int)560' ! queue ! fpsdisplaysink fps-update-interval=1000 video-sink=waylandsink

How else can I show that I turn on сapturing interlace in alternate field mode?

3.

manisha said:

Third thing I wanted you to try capture + di-interlacing by vpe using v4l2-test-m2m application. This application is packaged in processor SDK file system. Source code can be found here https://git.ti.com/glsdk/example-applications/blobs/master/v4l2-m2m/test-v4l2-m2m.c.

I will conduct an experiment and write the result later.

Sorry for the delay in responding. I can write more specifically after the weekend. I did not have time to make the necessary experiments. Please do not close the topic.

Hey. Sorry for my slowness. Let's move on.

2.

root@am57xx-evm:~# v4l2-ctl -d /dev/video1 -V
Format Video Capture:
	Width/Height      : 704/280
	Pixel Format      : 'YUYV'
	Field             : Alternating
	Bytes per Line    : 1408
	Size Image        : 394240
	Colorspace        : SMPTE 170M
	Transfer Function : Default
	YCbCr Encoding    : Default
	Quantization      : Default
	Flags             : 

3.

I did not have time for test-v4l2-m2m.
But I managed to execute with your help ( https://e2e.ti.com/support/processors/f/791/t/835475#pi320966=2 ):

root@am57xx-evm:~# /etc/init.d/weston stop && sleep 1 && capturevpedisplay 704 280 yuyv 704 560 yuyv 1 3 -s 35:800x480
Stopping Weston
vpe:/dev/video0 open success!!!
vip open success!!!
using 1 connectors, 800x480 display, multiplanar: 1
Setting mode 800x480 on connector 35, crtc 39
vip: G_FMT(start): width = 704, height = 280, 4cc = YUYV
vpe i/p: G_FMT: width = 704, height = 280, 4cc = YUYV
vpe o/p: G_FMT: width = 704, height = 560, 4cc = YUYV
Page flip: frame=278103, sec=6795, usec=159784, remaining=0
streaming started...

There is no jitter. Everything works just fine.

So I have two questions.
1.Do i need to do test-v4l2-m2m?
2.And why is jitter on gstreamer?

gst-launch-1.0 v4l2src device=/dev/video1 io-mode=4 ! 'video/x-raw,format=(string)YUY2, width=(int)704, height=(int)280' ! vpe num-input-buffers=8 ! 'video/x-raw,format=(string)YUY2, width=(int)704, height=(int)560' ! queue ! fpsdisplaysink fps-update-interval=1000 video-sink=waylandsink

Hi Manisha, I launched test-v4l2-m2m today. And I'm surprised jitter didn't go missing.

Now I will describe what I did.

1. recorded a file my.yuv using yavta https://e2e.ti.com/cfs-file/__key/communityserver-discussions-components-files/791/3730.my.yuv.tar.7z yavta.log.gz

yavta -c80 -p -F/home/root/my.yuv --skip 2 -t 1/50 -f YUYV -s 704x280 /dev/video2

2. I converted my interlaced my.yuv file to my deinterlaced my_out.yuv file https://e2e.ti.com/cfs-file/__key/communityserver-discussions-components-files/791/8611.my_5F00_out.yuv.tar.7z test-v4l2-m2m.log.gz

test-v4l2-m2m /dev/video0 /home/root/my.yuv 704 280 yuyv /home/root/my_out.yuv 704 560 yuyv 1 1 80

3. play on PC

ffplay -f rawvideo -pix_fmt yuyv422 -video_size 704x560 -framerate 50 -i my_out.yuv

Jitter has a place to be.

If you look at the yavta.log, you will see strange behavior in the order of fields top and bottom.
This is normal?

21 (5) [-] bottom 21 394240 B 812.432980 812.571694 49.940 fps ts mono/EoF
22 (6) [-] bottom 23 394240 B 812.472979 812.597091 25.001 fps ts mono/EoF
23 (7) [-] bottom 25 394240 B 812.512980 812.621873 24.999 fps ts mono/EoF
24 (0) [-] top 26 394240 B 812.532948 812.646567 50.080 fps ts mono/EoF
25 (1) [-] top 28 394240 B 812.572945 812.672224 25.002 fps ts mono/EoF
26 (2) [-] bottom 29 394240 B 812.592974 812.695732 49.928 fps ts mono/EoF
27 (3) [-] top 30 394240 B 812.612944 812.728540 50.075 fps ts mono/EoF
28 (4) [-] bottom 31 394240 B 812.632976 812.763627 49.920 fps ts mono/EoF
29 (5) [-] bottom 33 394240 B 812.672974 812.799864 25.001 fps ts mono/EoF
30 (6) [-] top 34 394240 B 812.692940 812.835919 50.085 fps ts mono/EoF
31 (7) [-] bottom 35 394240 B 812.712973 812.871175 49.918 fps ts mono/EoF
32 (0) [-] top 36 394240 B 812.732940 812.896113 50.083 fps ts mono/EoF
33 (1) [-] top 38 394240 B 812.772939 812.924419 25.001 fps ts mono/EoF
34 (2) [-] bottom 39 394240 B 812.792969 812.948082 49.925 fps ts mono/EoF
35 (3) [-] bottom 41 394240 B 812.832968 812.974008 25.001 fps ts mono/EoF
36 (4) [-] bottom 43 394240 B 812.872968 813.000471 25.000 fps ts mono/EoF
37 (5) [-] bottom 45 394240 B 812.912973 813.025150 24.997 fps ts mono/EoF

What to do next?

Manisha waiting for your answer, please answer

Hi Oleg,

I am out of office at the moment. I shall look into your logs when I am in office next week and get back to you.

Thank you for your patience.

Regards,

Manisha

manisha said:

Thank you for your patience.

Hi Manisha,

Waiting for your reply and thanks for your help.

Hi Oleg,

As you see from the Yavta logs, it is dropping fields as it is trying to do file writes (loosing real time operation due to file I/O). See the field numbers highlighted in yellow below. Now with the file write, there is no information on the field type stored as well. When this file is presented to test-v4l2-m2m, the application in absence of field id information with each field read,  assumes that the field id are in alternate top-bottom sequence (which we can see from the logs below that it is not) and hence causing the jitter.

21 (5) [-] bottom 21 394240 B 812.432980 812.571694 49.940 fps ts mono/EoF
22 (6) [-] bottom 23 394240 B 812.472979 812.597091 25.001 fps ts mono/EoF
23 (7) [-] bottom 25 394240 B 812.512980 812.621873 24.999 fps ts mono/EoF
24 (0) [-] top 26 394240 B 812.532948 812.646567 50.080 fps ts mono/EoF
25 (1) [-] top 28 394240 B 812.572945 812.672224 25.002 fps ts mono/EoF
26 (2) [-] bottom 29 394240 B 812.592974 812.695732 49.928 fps ts mono/EoF

This problem is not seen in the capturevpedisplay application because the field id is preserved by the application (capture driver populates it in the v4l2 buffer descriptor and it is passed to the VPE driver). As well there is no field drop in the application as there is no file write happening.

Regards,
Manisha

manisha said:

loosing real time operation due to file I/O

Hi Manisha,

Now let’s summarize

1. capturevpedisplay : works well, deinterlacing without jitter

2. yavta + test-v4l2-m2m : works well, deinterlacing without jitter.

PS. The first unsuccessful experiment I did was recording file my.yuv to NFS (Ethernet).

I conducted a second experiment with a record of file my.yuv to eMMC. The second experiment was successful.

yavta recorded files without skipping fields. I converted the recorded file using test-v4l2-m2m and played on PC using ffmpeg. When playing on a PC, there was no jitter

3. gstreamer : not work, deinterlacer with jitter

But what to do with gstreamer?

Hi Manisha,
I waiting for your reply.

Hi Manisha

manisha said:

Follow below link to understand different modes for interlace arrangements -

Thanks now it became clear

manisha said:

To get rid of jitter, you will need to modify the v4l2src plugin to support captured field in ALTERNATE mode as understood by VPE

Thanks for the help.

What is easier to throw half the frame in driver vip.c or fix plugin v4l2?

Can you tell me more about v4l2 plugin? ( What is the name of the plugin? Where to get the source? Can you help navigate the code? )