• State
  • Locked Locked
  • Replies 3 replies
  • Subscribers 98 subscribers
  • Views 186 views
  • Users 0 members are here
Support feedback
Options
Options
Related

This thread has been locked.

If you have a related question, please click the "Ask a related question" button in the top right corner. The newly created question will be automatically linked to this question.

AM62A7-Q1: Kernel Panic randomly when streaming

Ray Chang
Prodigy 10 points

Part Number: AM62A7-Q1

Tool/software:

Hi,

We are now using a deserializer (MAX96724) with two cameras attached to send csi2 data with two streams.

These two camera do work and show image with gstreamer.

However, the kernel may crash randomly. (sometimes the streams can last for more than 6 hours, sometimes it crashes at boot)

root@am62axx-evm:~# [ 2817.381902] Kernel panic - not syncing: stack-protector: Kernel stack is corrupted in: ti_csi2rx_start_dma+0x104/0x10c [j721e_csi2rx]
[ 2817.393903] CPU: 0 PID: 6184 Comm: sh Tainted: G           O       6.6.44-ti-01478-g541c20281af7-dirty #1
[ 2817.403460] Hardware name: Texas Instruments AM62A7 SK (DT)
[ 2817.409023] Call trace:
[ 2817.411463]  dump_backtrace+0x90/0xe8
[ 2817.415131]  show_stack+0x18/0x24
[ 2817.418445]  dump_stack_lvl+0x48/0x60
[ 2817.422110]  dump_stack+0x18/0x24
[ 2817.425424]  panic+0x324/0x380
[ 2817.428481]  __ktime_get_real_seconds+0x0/0xc
[ 2817.432838]  ti_csi2rx_start_dma+0x104/0x10c [j721e_csi2rx]
[ 2817.438417]  ti_csi2rx_dma_submit_pending+0x7c/0xe8 [j721e_csi2rx]
[ 2817.444602]  ti_csi2rx_dma_callback+0x8c/0xd0 [j721e_csi2rx]
[ 2817.450263]  udma_vchan_complete+0x218/0x2bc
[ 2817.454536]  tasklet_action_common.isra.0+0xf8/0x11c
[ 2817.459497]  tasklet_action+0x24/0x30
[ 2817.463155]  handle_softirqs+0x108/0x24c
[ 2817.467076]  __do_softirq+0x14/0x20
[ 2817.470560]  ____do_softirq+0x10/0x1c
[ 2817.474220]  call_on_irq_stack+0x24/0x4c
[ 2817.478139]  do_softirq_own_stack+0x1c/0x2c
[ 2817.482312]  irq_exit_rcu+0xc0/0xdc
[ 2817.485790]  el1_interrupt+0x38/0x68
[ 2817.489359]  el1h_64_irq_handler+0x18/0x24
[ 2817.493446]  el1h_64_irq+0x64/0x68
[ 2817.496839]  account_kernel_stack.isra.0+0x40/0x60
[ 2817.501619]  exit_task_stack_account+0x20/0x58
[ 2817.506053]  do_exit+0x55c/0x8c4
[ 2817.509276]  do_group_exit+0x34/0x90
[ 2817.512842]  __wake_up_parent+0x0/0x30
[ 2817.516582]  invoke_syscall+0x48/0x114
[ 2817.520330]  el0_svc_common.constprop.0+0xc0/0xe0
[ 2817.525031]  do_el0_svc+0x1c/0x28
[ 2817.528345]  el0_svc+0x2c/0x84
[ 2817.531397]  el0t_64_sync_handler+0x120/0x12c
[ 2817.535751]  el0t_64_sync+0x190/0x194
[ 2817.539407] SMP: stopping secondary CPUs
[ 2818.610932] SMP: failed to stop secondary CPUs 0,3
[ 2818.615716] Kernel Offset: 0x80000 from 0xffff800080000000
[ 2818.621186] PHYS_OFFSET: 0x80000000
[ 2818.624662] CPU features: 0x0,00000008,00020000,1000420b
[ 2818.629961] Memory Limit: none
[ 2818.633010] ---[ end Kernel panic - not syncing: stack-protector: Kernel stack is corrupted in: ti_csi2rx_start_dma+0x104/0x10c [j721e_csi2rx] ]---

In order to meet our requirements, we modified the default format and routes in j721e-csi2rx.c and cdns-csi2rx.c:

static int _ti_csi2rx_sd_set_routing(struct v4l2_subdev *sd,
				     struct v4l2_subdev_state *state,
				     struct v4l2_subdev_krouting *routing)
{
	int ret;
	//20250806 Ray Edited: Default resolution, format
	const struct v4l2_mbus_framefmt format = {
		.width = 1920,
		.height = 1280,
		.code = MEDIA_BUS_FMT_UYVY8_1X16,
		.field = V4L2_FIELD_NONE,
		.colorspace = V4L2_COLORSPACE_SRGB,
		.ycbcr_enc = V4L2_YCBCR_ENC_601,
		.quantization = V4L2_QUANTIZATION_LIM_RANGE,
		.xfer_func = V4L2_XFER_FUNC_SRGB,
	};

	ret = v4l2_subdev_routing_validate(sd, routing,
					   V4L2_SUBDEV_ROUTING_ONLY_1_TO_1 |
					   V4L2_SUBDEV_ROUTING_NO_SOURCE_MULTIPLEXING);

	if (ret)
		return ret;

	ret = v4l2_subdev_set_routing_with_fmt(sd, state, routing, &format);

	return ret;
}

static int ti_csi2rx_sd_set_routing(struct v4l2_subdev *sd,
				    struct v4l2_subdev_state *state,
				    enum v4l2_subdev_format_whence which,
				    struct v4l2_subdev_krouting *routing)
{
	return _ti_csi2rx_sd_set_routing(sd, state, routing);
}

static int ti_csi2rx_sd_init_cfg(struct v4l2_subdev *sd,
				 struct v4l2_subdev_state *state)
{
	struct v4l2_subdev_route routes[] = { 
		{
		.sink_pad = 0,
		.sink_stream = 0,
		.source_pad = TI_CSI2RX_PAD_FIRST_SOURCE+1,
		.source_stream = 0,
		.flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE,
		},
		//20250806 Ray Edited: Default routing
		{
		.sink_pad = 0,
		.sink_stream = 1,
		.source_pad = TI_CSI2RX_PAD_FIRST_SOURCE+2,
		.source_stream = 0,
		.flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE,
		},
	};

	struct v4l2_subdev_krouting routing = {
		.num_routes = 2,
		.routes = routes,
	};

	/* Initialize routing to single route to the fist source pad */
	return _ti_csi2rx_sd_set_routing(sd, state, &routing);
}

the media-ctl -p looks like:

root@am62axx-evm:~# media-ctl -p
Media controller API version 6.6.44

Media device information
------------------------
driver          j721e-csi2rx
model           TI-CSI2RX
serial          
bus info        platform:30102000.ticsi2rx
hw revision     0x1
driver version  6.6.44

Device topology
- entity 1: 30102000.ticsi2rx (7 pads, 7 links, 2 routes)
            type V4L2 subdev subtype Unknown flags 0
            device node name /dev/v4l-subdev0
	routes:
		0/0 -> 2/0 [ACTIVE]
		0/1 -> 3/0 [ACTIVE]
	pad0: Sink
		[stream:0 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb xfer:srgb ycbcr:601 quantization:lim-range]
		[stream:1 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb xfer:srgb ycbcr:601 quantization:lim-range]
		<- "cdns_csi2rx.30101000.csi-bridge":1 [ENABLED,IMMUTABLE]
	pad1: Source
		-> "30102000.ticsi2rx context 0":0 [ENABLED,IMMUTABLE]
	pad2: Source
		[stream:0 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb xfer:srgb ycbcr:601 quantization:lim-range]
		-> "30102000.ticsi2rx context 1":0 [ENABLED,IMMUTABLE]
	pad3: Source
		[stream:0 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb xfer:srgb ycbcr:601 quantization:lim-range]
		-> "30102000.ticsi2rx context 2":0 [ENABLED,IMMUTABLE]
	pad4: Source
		-> "30102000.ticsi2rx context 3":0 [ENABLED,IMMUTABLE]
	pad5: Source
		-> "30102000.ticsi2rx context 4":0 [ENABLED,IMMUTABLE]
	pad6: Source
		-> "30102000.ticsi2rx context 5":0 [ENABLED,IMMUTABLE]

- entity 9: cdns_csi2rx.30101000.csi-bridge (5 pads, 2 links, 2 routes)
            type V4L2 subdev subtype Unknown flags 0
            device node name /dev/v4l-subdev1
	routes:
		0/0 -> 1/0 [ACTIVE]
		0/1 -> 1/1 [ACTIVE]
	pad0: Sink
		[stream:0 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb xfer:srgb ycbcr:601 quantization:lim-range]
		[stream:1 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb xfer:srgb ycbcr:601 quantization:lim-range]
		<- "max96724 4-0027":4 [ENABLED,IMMUTABLE]
	pad1: Source
		[stream:0 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb xfer:srgb ycbcr:601 quantization:lim-range]
		[stream:1 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb xfer:srgb ycbcr:601 quantization:lim-range]
		-> "30102000.ticsi2rx":0 [ENABLED,IMMUTABLE]
	pad2: Source
	pad3: Source
	pad4: Source

- entity 15: max96724 4-0027 (6 pads, 3 links, 4 routes)
             type V4L2 subdev subtype Unknown flags 0
             device node name /dev/v4l-subdev2
	routes:
		0/0 -> 4/0 [ACTIVE]
		1/0 -> 4/1 [ACTIVE]
		2/0 -> 5/2 [ACTIVE]
		3/0 -> 5/3 [ACTIVE]
	pad0: Sink
		[stream:0 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb]
		<- "auo_cms_gmsl 5-0040":0 [ENABLED,IMMUTABLE]
	pad1: Sink
		[stream:0 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb]
		<- "auo_cms_gmsl 6-0040":0 [ENABLED,IMMUTABLE]
	pad2: Sink
		[stream:0 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb]
	pad3: Sink
		[stream:0 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb]
	pad4: Source
		[stream:0 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb]
		[stream:1 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb]
		-> "cdns_csi2rx.30101000.csi-bridge":0 [ENABLED,IMMUTABLE]
	pad5: Source
		[stream:2 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb]
		[stream:3 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb]

- entity 24: auo_cms_gmsl 5-0040 (1 pad, 1 link, 0 routes)
             type V4L2 subdev subtype Sensor flags 0
             device node name /dev/v4l-subdev3
	pad0: Source
		[stream:0 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb
		 crop:(0,0)/1920x1280]
		-> "max96724 4-0027":0 [ENABLED,IMMUTABLE]

- entity 28: auo_cms_gmsl 6-0040 (1 pad, 1 link, 0 routes)
             type V4L2 subdev subtype Sensor flags 0
             device node name /dev/v4l-subdev4
	pad0: Source
		[stream:0 fmt:UYVY8_1X16/1920x1280 field:none colorspace:srgb
		 crop:(0,0)/1920x1280]
		-> "max96724 4-0027":1 [ENABLED,IMMUTABLE]

- entity 34: 30102000.ticsi2rx context 0 (1 pad, 1 link)
             type Node subtype V4L flags 0
             device node name /dev/video3
	pad0: Sink
		<- "30102000.ticsi2rx":1 [ENABLED,IMMUTABLE]

- entity 40: 30102000.ticsi2rx context 1 (1 pad, 1 link)
             type Node subtype V4L flags 0
             device node name /dev/video4
	pad0: Sink
		<- "30102000.ticsi2rx":2 [ENABLED,IMMUTABLE]

- entity 46: 30102000.ticsi2rx context 2 (1 pad, 1 link)
             type Node subtype V4L flags 0
             device node name /dev/video5
	pad0: Sink
		<- "30102000.ticsi2rx":3 [ENABLED,IMMUTABLE]

- entity 52: 30102000.ticsi2rx context 3 (1 pad, 1 link)
             type Node subtype V4L flags 0
             device node name /dev/video6
	pad0: Sink
		<- "30102000.ticsi2rx":4 [ENABLED,IMMUTABLE]

- entity 58: 30102000.ticsi2rx context 4 (1 pad, 1 link)
             type Node subtype V4L flags 0
             device node name /dev/video7
	pad0: Sink
		<- "30102000.ticsi2rx":5 [ENABLED,IMMUTABLE]

- entity 64: 30102000.ticsi2rx context 5 (1 pad, 1 link)
             type Node subtype V4L flags 0
             device node name /dev/video8
	pad0: Sink
		<- "30102000.ticsi2rx":6 [ENABLED,IMMUTABLE]

We've skipped the context 0 (seen someone reported issues with this context), but nothing changed. 

The SDK we are using is 10_01_00 with FALCON mode

Is there anything I can do?

  • 0
    TI__Guru 60480 points

    Hello,

    I saw the following error message:

    Kernel panic - not syncing: stack-protector: Kernel stack is corrupted in: ti_csi2rx_start_dma+0x104/0x10c [j721e_csi2rx]

    This means there was likely memory corruption, i.e. something overwrote kernel stack memory beyond what it should.

    Ray Chang said:
    The SDK we are using is 10_01_00 with FALCON mode

    Did you run your application on the original SDK, or did you customize the SDK based on your hardware?

    When kernel crashed, did it always crash in ti_csi2rx_start_dma?

    Regards,

    Jianzhong

  • 0 in reply to Jianzhong Xu
    Prodigy 10 points

    Hi,

    thanks for the reply.

    Jianzhong Xu said:
    Did you run your application on the original SDK, or did you customize the SDK based on your hardware?

    We are basically using the edgeai yocto image in the original SDK with some unused features removed.

    Jianzhong Xu said:
    When kernel crashed, did it always crash in ti_csi2rx_start_dma?

    Yes, it does always crash in ti_csi2rx_strart_dma, but the condition to trigger this crash is not clear and the occurrence timing is unpredictable.

    The major modifications we did on yocto are shown below:

    disable.cfg:

    CONFIG_CMDLINE="console=ttyS2,115200n8 earlycon=ns16550a,mmio32,0x02800000 root=/dev/mmcblk0p2 rw rootwait rootfstype=ext4 quiet loglevel=0 systemd.show_status=0"
#CONFIG_CMDLINE="console=ttyS2,115200n8 earlycon=ns16550a,mmio32,0x02800000 root=/dev/mmcblk0p2 rw rootwait rootfstype=ext4 ignore_loglevel systemd.show_status=1"
CONFIG_CMDLINE_FORCE=y
CONFIG_MTD_SPI_NAND=y
CONFIG_SOUND=n
CONFIG_SND=n
CONFIG_SND_USB_AUDIO=n
CONFIG_SND_SOC=n
CONFIG_SND_SOC_DAVINCI_MCASP=n
CONFIG_SND_SOC_SIMPLE_CARD=n
CONFIG_SND_SOC_HDMI_CODEC=n
CONFIG_SND_SOC_TLV320AIC3X=n
CONFIG_SND_RAWMIDI=n
CONFIG_SND_USB_MIDI_LIB=n
CONFIG_SND_SOC_TLV320AIC3X=n
CONFIG_SND_SOC_TI_SDMA=n
CONFIG_SND_PCM=n
CONFIG_SND_JACK=n
CONFIG_SND_SOC_SIMPLE_AMPLIFIER=n
CONFIG_SND_SOC_SIMPLE_CARD_UTILS=n
CONFIG_SND_SOC_UTILS=n
CONFIG_SND_DMAENGINE_PCM=n
CONFIG_SND_SOC_GENERIC_DMAENGINE_PCM=n
CONFIG_SND_TIMER=n
CONFIG_WIRELESS=n
CONFIG_CFG80211=n
CONFIG_MAC80211=n
CONFIG_WLAN=n
CONFIG_WLAN_VENDOR_ADMTEK=n
CONFIG_ATH_COMMON=n
CONFIG_WLAN_VENDOR_ATH=n
CONFIG_ATH10K=n
CONFIG_ATH10K_CE=n
CONFIG_ATH10K_PCI=n
CONFIG_ATH10K_SDIO=n
CONFIG_WCN36XX=n
CONFIG_ATH11K=n
CONFIG_ATH11K_AHB=n
CONFIG_ATH11K_PCI=n
CONFIG_WLAN_VENDOR_ATMEL=n
CONFIG_WLAN_VENDOR_BROADCOM=n
CONFIG_BRCMUTIL=n
CONFIG_BRCMFMAC=n
CONFIG_BRCMFMAC_PROTO_BCDC=n
CONFIG_BRCMFMAC_SDIO=n
CONFIG_WLAN_VENDOR_CISCO=n
CONFIG_WLAN_VENDOR_INTEL=n
CONFIG_IWLWIFI=n
CONFIG_IWLWIFI_LEDS=n
CONFIG_IWLDVM=n
CONFIG_IWLMVM=n
CONFIG_WLAN_VENDOR_INTERSIL=n
CONFIG_WLAN_VENDOR_MARVELL=n
CONFIG_MWIFIEX=n
CONFIG_MWIFIEX_SDIO=n
CONFIG_MWIFIEX_PCIE=n
CONFIG_WLAN_VENDOR_MEDIATEK=n
CONFIG_MT76_CORE=n
CONFIG_MT76_LEDS=n
CONFIG_MT76_CONNAC_LIB=n
CONFIG_MT792x_LIB=n
CONFIG_MT7921_COMMON=n
CONFIG_MT7921E=n
CONFIG_WLAN_VENDOR_MICROCHIP=n
CONFIG_WLAN_VENDOR_PURELIFI=n
CONFIG_WLAN_VENDOR_RALINK=n
CONFIG_WLAN_VENDOR_REALTEK=n
CONFIG_RTL_CARDS=n
CONFIG_WLAN_VENDOR_RSI=n
CONFIG_RSI_91X=n
CONFIG_WLAN_VENDOR_SILABS=n
CONFIG_WLAN_VENDOR_ST=n
CONFIG_WLAN_VENDOR_TI=n
CONFIG_WL18XX=n
CONFIG_CC33XX=n
CONFIG_WLCORE=n
CONFIG_WLAN_VENDOR_ZYDAS=n
CONFIG_WLAN_VENDOR_QUANTENNA=n
CONFIG_BT_INTEL=n
CONFIG_BT_BCM=n
CONFIG_BT_RTL=n
CONFIG_BT_QCA=n
CONFIG_BT_MTK=n
CONFIG_BT_HCIBTUSB=n
BT_HCIBTUSB_POLL_SYNC=n
BT_HCIBTUSB_BCM=n
BT_HCIBTUSB_MTK=n
BT_HCIBTUSB_RTL=n
CONFIG_BT_HCIUART=n
CONFIG_BT_MRVL=n
CONFIG_BT_HCIRSI=n
CONFIG_BT_NXPUART=n
CONFIG_TI_K3_AM65_CPSW_NUSS=m
CONFIG_TI_K3_AM65_CPSW_SWITCHDEV=m

    tisdk-edgeai-image.bbappend:

    IMAGE_INSTALL:remove = "edgeai-gui-app"
#IMAGE_INSTALL:append = "delayed-eth0"
IMAGE_INSTALL:remove = "packagegroup-arago-base docker-moby weston-init"

    Regards,

    Ray

  • 0 in reply to Ray Chang
    TI__Guru 60480 points

    Hi Ray,

    Thanks for the additional information. I'll need to check internally. Please give me a few days.

    Regards,

    Jianzhong