This thread has been locked.

If you have a related question, please click the "Ask a related question" button in the top right corner. The newly created question will be automatically linked to this question.

(Bug report) the kernel crash when I run cyclictest on DM3730

The test command : 

# cyclictest -l100000000 -m -a0 -t1 -n -p99 -i200 -h200 -q
I run the following command to increase cpu's burden at the same time.
# while true; do hackbench; sleep 1;done

cyclictest and hackbench can be get in git://git.kernel.org/pub/scm/linux/kernel/git/clrkwllms/rt-tests.git

The linux kernel version :

the kernel I used is windriver linux kernel. but this bug can also be reproduced on the branch OMAPPSP_03.00.01.06 in git://arago-project.org/git/projects/linux-omap3.git. The kernel in linux-omap-2.6 git tree will hang and didn't give any useful Oops messages when I run this cyclictest.

The Oops messages (it crashed in many different places):

1, 

root@localhost:/root> cyclictest -l100000000 -m -a0 -t1 -n -p99 -i200 -h200 -q
Unhandled fault: external abort on non-linefetch (0x1028) at 0xfa3069e4
Internal error: : 1028 [#1] PREEMPT
last sysfs file: /sys/kernel/uevent_seqnum
Modules linked in: x_tables ip_tables ipv6 sctp binfmt_misc [last unloaded: scsi_wait_scan]
CPU: 0    Not tainted  (2.6.34.9-WR4.2.0.0_standard #3)
PC is at prm_read_mod_reg+0x34/0x44
LR is at pwrdm_wait_transition+0x40/0x90
pc : [<c004bf7c>]    lr : [<c004c76c>]    psr: a00000d3
sp : c072bec0  ip : c004bf5c  fp : c072becc
r10: 00000000  r9 : 413fc082  r8 : 00000000
r7 : 00000000  r6 : 00000000  r5 : c072efe0  r4 : 00000000
r3 : fa306800  r2 : fa004948  r1 : 000000e4  r0 : 000001e4
Flags: NzCv  IRQs off  FIQs off  Mode SVC_32  ISA ARM  Segment kernel
Control: 10c5387d  Table: 86858019  DAC: 00000017
Process swapper (pid: 0, stack limit = 0xc072a2e8)
Stack: (0xc072bec0 to 0xc072c000)
bec0: c072bee4 c072bed0 c004c76c c004bf54 c072f940 00000003 c072befc c072bee8
bee0: c004ce8c c004c738 00000003 c072f940 c072bf14 c072bf00 c0054b4c c004ce6c
bf00: c072efe0 00000001 c072bf3c c072bf18 c004fa90 c0054b0c c077e6bc c077e6a0
bf20: 00000003 00000003 8002f56c 0000001f c072bf74 c072bf40 c0051a38 c004f58c
bf40: 0000492b 00000000 c6bcee00 c0731340 386d4532 2ac85836 c0731340 c07313b0
bf60: c082a350 c072e420 c072bf9c c072bf78 c046dd58 c005198c c046dca0 c072a000
bf80: c0032018 c0032014 8002f56c 413fc082 c072bfb4 c072bfa0 c003e99c c046dcac
bfa0: c07f2630 00000002 c072bfcc c072bfb8 c0557d6c c003e944 00000000 c07f4368
bfc0: c072bff4 c072bfd0 c0008a74 c0557cc0 c0008584 00000000 00000000 c0032018
bfe0: 10c53c7d c077dd70 00000000 c072bff8 80008034 c00087dc 00000000 00000000
[<c004bf7c>] (prm_read_mod_reg+0x34/0x44) from [<c004c76c>] (pwrdm_wait_transition+0x40/0x90)
[<c004c76c>] (pwrdm_wait_transition+0x40/0x90) from [<c004ce8c>] (pwrdm_clkdm_state_switch+0x2c/0x40)
[<c004ce8c>] (pwrdm_clkdm_state_switch+0x2c/0x40) from [<c0054b4c>] (omap2_clkdm_allow_idle+0x4c/0x50)
[<c0054b4c>] (omap2_clkdm_allow_idle+0x4c/0x50) from [<c004fa90>] (omap_sram_idle+0x510/0x534)
[<c004fa90>] (omap_sram_idle+0x510/0x534) from [<c0051a38>] (omap3_enter_idle+0xb8/0x150)
[<c0051a38>] (omap3_enter_idle+0xb8/0x150) from [<c046dd58>] (cpuidle_idle_call+0xb8/0x198)
[<c046dd58>] (cpuidle_idle_call+0xb8/0x198) from [<c003e99c>] (cpu_idle+0x64/0xbc)
[<c003e99c>] (cpu_idle+0x64/0xbc) from [<c0557d6c>] (rest_init+0xb8/0xd4)
[<c0557d6c>] (rest_init+0xb8/0xd4) from [<c0008a74>] (start_kernel+0x2a4/0x30c)
[<c0008a74>] (start_kernel+0x2a4/0x30c) from [<80008034>] (0x80008034)
Code: e59f0014 e3a010aa ebffd57e e0810000 (e7930000) 

2,

 

root@localhost:/root> cyclictest -l100000000 -m -a0 -t1 -n -p99 -i200 -h200 -q
Unhandled fault: external abort on non-linefetch (0x1028) at 0xfa318034
Internal error: : 1028 [#1] PREEMPT
last sysfs file: /sys/kernel/uevent_seqnum
Modules linked in: x_tables ip_tables ipv6 sctp binfmt_misc [last unloaded: scsi_wait_scan]
CPU: 0    Not tainted  (2.6.34.9-WR4.2.0.0_standard #5)
PC is at omap_dm_timer_set_load_start+0x30/0x94
LR is at omap2_gp_timer_set_next_event+0x28/0x34
pc : [<c005a0a0>]    lr : [<c0047fb0>]    psr: 20000093
sp : c0709e58  ip : c005a084  fp : c0709e74
r10: c0709f10  r9 : 00000466  r8 : 3824a2bc
r7 : 00000000  r6 : ffffff91  r5 : 00000000  r4 : c0717564
r3 : fa318000  r2 : ffffff91  r1 : 00000000  r0 : c0717564
Flags: nzCv  IRQs off  FIQs on  Mode SVC_32  ISA ARM  Segment kernel
Control: 10c5387d  Table: 864cc019  DAC: 00000017
Process swapper (pid: 0, stack limit = 0xc07082e8)
Stack: (0xc0709e58 to 0xc070a000)
9e40:                                                       c075a4a0 c070ea38
<snip the stack data>
[<c005a0a0>] (omap_dm_timer_set_load_start+0x30/0x94) from [<c0047fb0>] (omap2_gp_timer_set_next_event+0x28/0x34)
[<c0047fb0>] (omap2_gp_timer_set_next_event+0x28/0x34) from [<c0098fd0>] (clockevents_program_event+0xec/0x104)
[<c0098fd0>] (clockevents_program_event+0xec/0x104) from [<c00999e8>] (tick_dev_program_event+0x4c/0x160)
[<c00999e8>] (tick_dev_program_event+0x4c/0x160) from [<c0099b84>] (tick_program_event+0x38/0x44)
[<c0099b84>] (tick_program_event+0x38/0x44) from [<c0090a70>] (__hrtimer_start_range_ns+0x248/0x31c)
[<c0090a70>] (__hrtimer_start_range_ns+0x248/0x31c) from [<c0090bb4>] (hrtimer_start_range_ns+0x34/0x3c)
[<c0090bb4>] (hrtimer_start_range_ns+0x34/0x3c) from [<c009ad54>] (tick_nohz_restart_sched_tick+0x1ac/0x214)
[<c009ad54>] (tick_nohz_restart_sched_tick+0x1ac/0x214) from [<c003c9d0>] (cpu_idle+0x98/0xbc)
[<c003c9d0>] (cpu_idle+0x98/0xbc) from [<c0541094>] (rest_init+0xb8/0xd4)
[<c0541094>] (rest_init+0xb8/0xd4) from [<c0008a74>] (start_kernel+0x2a4/0x30c)
[<c0008a74>] (start_kernel+0x2a4/0x30c) from [<80008034>] (0x80008034)
Code: e3130004 1a000000 ea000003 e5943010 (e5933034) 

 

3,

 

Unhandled fault: external abort on non-linefetch (0x1028) at 0xfa320010
Internal error: : 1028 [#2] PREEMPT
last sysfs file: /sys/devices/system/cpu/cpu0/cpufreq/scaling_setspeed
Modules linked in: x_tables ip_tables ipv6 sctp binfmt_misc [last unloaded: scsi_wait_scan]
CPU: 0    Not tainted  (2.6.34.9-WR4.2.0.0_standard #6)
PC is at omap_readl+0x18/0x20
LR is at omap34xx_32k_read+0x1c/0x38
pc : [<c0059bb4>]    lr : [<c005586c>]    psr: 600001d3
sp : c0721890  ip : c0059bb0  fp : c072189c
r10: c072f488  r9 : c0720000  r8 : 00000000
r7 : 0000026b  r6 : 0001ee12  r5 : c077a3c0  r4 : c07218e8
r3 : c0055850  r2 : 1fd67bb3  r1 : 00000000  r0 : fa320010
Flags: nZCv  IRQs off  FIQs off  Mode SVC_32  ISA ARM  Segment kernel
Control: 10c5387d  Table: 869b4019  DAC: 00000017
Process swapper (pid: 0, stack limit = 0xc07202e8)
Stack: (0xc0721890 to 0xc0722000)
1880:                                     c07218ac c07218a0 c005586c c0059ba8
<snip the stack dump>
[<c0059bb4>] (omap_readl+0x18/0x20) from [<c005586c>] (omap34xx_32k_read+0x1c/0x38)
[<c005586c>] (omap34xx_32k_read+0x1c/0x38) from [<c009b85c>] (ktime_get+0x80/0xec)
[<c009b85c>] (ktime_get+0x80/0xec) from [<c009f9d4>] (tick_check_idle+0x40/0xd4)
[<c009f9d4>] (tick_check_idle+0x40/0xd4) from [<c0079ed8>] (irq_enter+0x70/0x9c)
[<c0079ed8>] (irq_enter+0x70/0x9c) from [<c003b028>] (asm_do_IRQ+0x28/0x9c)
[<c003b028>] (asm_do_IRQ+0x28/0x9c) from [<c003bbb4>] (__irq_svc+0x54/0xbc)
Exception stack(0xc0721940 to 0xc0721988)

Analysis:

All the kernel crashes occurred at the time when registers were read. These register belongs to prm module, GPTIMER module and 32k sync timer module. All these module are in wakeup domain. I utilize a ICE to debug this problem. And I found all the registers in the wakeup domain cannot be read when the do_DataAbort() was called, whereas the registers in other domains still can be read. 

Taking consider of the wakeup domain is always active and this kernel crashes even when I disabled the CONFIG_PM, I think this problem could probably be a hardware bug. I guess there might be some problem with L4-wakeup interconnect which is used to read registers in wakeup domain.

Hope someone can give me advises to fix this problem or confirm this is a hardware bug.

 

 

Regards.

Stanley.

  • Hi,
    
    
    i think i have the same Error. The problem can be triggert bei using
    this command:  cangen can0 -g 100 -I 42A -L 1 -D i -v -v
    it sends CAN messages with increment payload every 100ms.
    
    
    !! New info: same bug with:  watch -n 1 echo "test"
    
    
    Kernel from TI am33x git repository (3.1.0) on Beaglebone platform.
    After a while the Kernel throws these Ooops:
    
    
    [ 9725.943072] Unable to handle kernel paging request at virtual address e9e31034
    [ 9725.950629] pgd = ce844000
    [ 9725.953454] [e9e31034] *pgd=00000000
    [ 9725.957198] Internal error: Oops: 5 [#1]
    [ 9725.961293] Modules linked in:
    [ 9725.964494] CPU: 0 Not tainted (3.1.0-00010-g66bfbd2-dirty #3)
    [ 9725.970973] PC is at omap2_gp_timer_set_next_event+0x14/0x74
    [ 9725.976899] LR is at clockevents_program_event+0xc0/0x130
    [ 9725.982543] pc : [<c0022a28>] lr : [<c0066fa8>] psr: 60000193
    [ 9725.982553] sp : c05b5d30 ip : c05b5d40 fp : c05b5d3c
    [ 9725.994540] r10: c050ccc0 r9 : 000008d8 r8 : 7f338790
    [ 9725.999998] r7 : 0003a599 r6 : 373bf910 r5 : 00000000 r4 : 0097f3f0
    [ 9726.006817] r3 : c0609d70 r2 : e9e31034 r1 : c05bd600 r0 : 0003a599
    [ 9726.013639] Flags: nZCv IRQs off FIQs on Mode SVC_32 ISA ARM Segment kernel
    [ 9726.021367] Control: 10c5387d Table: 8e844019 DAC: 00000015
    [ 9726.027369] Process swapper (pid: 0, stack limit = 0xc05b42f0)
    [ 9726.033463] Stack: (0xc05b5d30 to 0xc05b6000)
    [ 9726.038018] 5d20: c05b5d64 c05b5d40 c0066fa8 c0022a20
    [ 9726.046568] 5d40: c05bd600 00000000 00000001 c05b5d88 7f338790 000008d8 c05b5dbc c05b5d68
    [ 9726.055118] 5d60: c0067908 c0066ef4 7f338790 000008d8 c05d61e0 0098967f 000008d8 7fcb7b80
    [ 9726.063670] 5d80: 7f338790 000008d8 7f338790 000008d8 c020d3d8 c05b5e18 c05d5bc8 c05d5c70
    [ 9726.072220] 5da0: 00000003 7fcb7b80 00000001 000008d8 c05b5dd4 c05b5dc0 c0067a88 c00678ac
    [ 9726.080772] 5dc0: 00000000 c05d5bc8 c05b5e4c c05b5dd8 c005c6a4 c0067a68 7f3308ed 000008d8
    [ 9726.089322] 5de0: cfa6c340 7fffffff ffffffff 00000000 00000000 00000000 7f3308ed 000008d8
    [ 9726.097872] 5e00: 7f3308ed 000008d8 7fcb7b80 000008d8 7f3308ed 000008d8 7f3308ed 000008d8
    [ 9726.106421] 5e20: 000025fd c05bd700 00000043 00000000 00000000 00000043 c06291a4 c05d805c
    [ 9726.114972] 5e40: c05b5e5c c05b5e50 c0022a0c c005c558 c05b5e94 c05b5e60 c0073dc0 c00229e4
    [ 9726.123522] 5e60: 00000000 00000000 000008d8 c05d805c 00000043 00000000 c05b5f3c 80004059
    [ 9726.132072] 5e80: 413fc082 00000000 c05b5eac c05b5e98 c0073f40 c0073d78 c05d805c 00000043
    [ 9726.140622] 5ea0: c05b5ec4 c05b5eb0 c00762c8 c0073f1c c05e63ac 00000043 c05b5ed4 c05b5ec8
    [ 9726.149172] 5ec0: c0073a28 c007624c c05b5ef4 c05b5ed8 c0016018 c00739fc c00297e8 80000013
    [ 9726.157722] 5ee0: fa200000 c05b5f3c c05b5f04 c05b5ef8 c0008390 c0015fec c05b5f74 c05b5f08
    [ 9726.166273] 5f00: c0014cf4 c000838c ba846e68 0000aebc 386d697e 0000aebc 00000000 c05bdf30
    [ 9726.174823] 5f20: c063b1b0 00000000 80004059 413fc082 00000000 c05b5f74 386d697e c05b5f50
    [ 9726.183373] 5f40: c00629b4 c00297e8 80000013 ffffffff 386d697e 0000aebc 386d697e 0000991e
    [ 9726.191923] 5f60: 413fc082 c05bdf20 c05b5f9c c05b5f78 c030eb90 c0029790 c05b4000 c0609684
    [ 9726.200473] 5f80: c05baf2c c05baf24 80004059 413fc082 c05b5fbc c05b5fa0 c00167f4 c030ead8
    [ 9726.209023] 5fa0: c05b692c c05a5004 c05a4fd4 c05baf24 c05b5fcc c05b5fc0 c041a9a4 c0016798
    [ 9726.217577] 5fc0: c05b5ff4 c05b5fd0 c057a7f8 c041a93c c057a32c 00000000 00000000 c05a5004
    [ 9726.226128] 5fe0: 10c53c7d c05b6040 00000000 c05b5ff8 8000803c c057a588 00000000 00000000
    [ 9726.234668] Backtrace:
    [ 9726.237242] [<c0022a14>] (omap2_gp_timer_set_next_event+0x0/0x74) from [<c0066fa8>] (clockevents_program_event+0xc0/0x130)
    [ 9726.248794] [<c0066ee8>] (clockevents_program_event+0x0/0x130) from [<c0067908>] (tick_dev_program_event+0x68/0x174)
    [ 9726.259788] r9:000008d8 r8:7f338790 r7:c05b5d88 r6:00000001 r5:00000000
    [ 9726.266628] r4:c05bd600
    [ 9726.269380] [<c00678a0>] (tick_dev_program_event+0x0/0x174) from [<c0067a88>] (tick_program_event+0x2c/0x34)
    [ 9726.279669] [<c0067a5c>] (tick_program_event+0x0/0x34) from [<c005c6a4>] (hrtimer_interrupt+0x158/0x2f8)
    [ 9726.289583] [<c005c54c>] (hrtimer_interrupt+0x0/0x2f8) from [<c0022a0c>] (omap2_gp_timer_interrupt+0x34/0x3c)
    [ 9726.299960] [<c00229d8>] (omap2_gp_timer_interrupt+0x0/0x3c) from [<c0073dc0>] (handle_irq_event_percpu+0x54/0x1a4)
    [ 9726.310874] [<c0073d6c>] (handle_irq_event_percpu+0x0/0x1a4) from [<c0073f40>] (handle_irq_event+0x30/0x40)
    [ 9726.321064] [<c0073f10>] (handle_irq_event+0x0/0x40) from [<c00762c8>] (handle_level_irq+0x88/0xf0)
    [ 9726.330575] r5:00000043 r4:c05d805c
    [ 9726.334355] [<c0076240>] (handle_level_irq+0x0/0xf0) from [<c0073a28>] (generic_handle_irq+0x38/0x44)
    [ 9726.344047] r5:00000043 r4:c05e63ac
    [ 9726.347838] [<c00739f0>] (generic_handle_irq+0x0/0x44) from [<c0016018>] (handle_IRQ+0x38/0x8c)
    [ 9726.356996] [<c0015fe0>] (handle_IRQ+0x0/0x8c) from [<c0008390>] (asm_do_IRQ+0x10/0x14)
    [ 9726.365409] r7:c05b5f3c r6:fa200000 r5:80000013 r4:c00297e8
    [ 9726.371391] [<c0008380>] (asm_do_IRQ+0x0/0x14) from [<c0014cf4>] (__irq_svc+0x34/0xa0)
    [ 9726.379716] Exception stack(0xc05b5f08 to 0xc05b5f50)
    [ 9726.385030] 5f00: ba846e68 0000aebc 386d697e 0000aebc 00000000 c05bdf30
    [ 9726.393636] 5f20: c063b1b0 00000000 80004059 413fc082 00000000 c05b5f74 386d697e c05b5f50
    [ 9726.402237] 5f40: c00629b4 c00297e8 80000013 ffffffff
    [ 9726.407575] [<c0029784>] (am33xx_enter_idle+0x0/0x94) from [<c030eb90>] (cpuidle_idle_call+0xc4/0x138)
    [ 9726.417360] r4:c05bdf20
    [ 9726.420034] [<c030eacc>] (cpuidle_idle_call+0x0/0x138) from [<c00167f4>] (cpu_idle+0x68/0xb0)
    [ 9726.428997] r9:413fc082 r8:80004059 r7:c05baf24 r6:c05baf2c r5:c0609684
    [ 9726.435885] r4:c05b4000
    [ 9726.438665] [<c001678c>] (cpu_idle+0x0/0xb0) from [<c041a9a4>] (rest_init+0x74/0x78)
    [ 9726.446805] r7:c05baf24 r6:c05a4fd4 r5:c05a5004 r4:c05b692c
    [ 9726.452803] [<c041a930>] (rest_init+0x0/0x78) from [<c057a7f8>] (start_kernel+0x27c/0x2d0)
    [ 9726.461502] [<c057a57c>] (start_kernel+0x0/0x2d0) from [<8000803c>] (0x8000803c)
    [ 9726.469286] Code: e92dd800 e24cb004 e59f305c e5932028 (e5922000)
    [ 9726.475743] ---[ end trace 360b6b0f468e4aa2 ]---
    [ 9726.480603] Kernel panic - not syncing: Fatal exception in interrupt
    [ 9726.487299] Backtrace:
    [ 9726.489890] [<c0018b70>] (dump_backtrace+0x0/0x10c) from [<c0424a00>] (dump_stack+0x18/0x1c)
    [ 9726.498762] r7:00000001 r6:c0022a2a r5:c060a5a0 r4:c060a5a0
    [ 9726.504742] [<c04249e8>] (dump_stack+0x0/0x1c) from [<c0424a68>] (panic+0x64/0x198)
    [ 9726.512801] [<c0424a04>] (panic+0x0/0x198) from [<c0018ed4>] (die+0x258/0x2bc)
    [ 9726.520393] r3:00010000 r2:c05b5b58 r1:00000209 r0:c04f28f0
    [ 9726.526378] [<c0018c7c>] (die+0x0/0x2bc) from [<c001b60c>] (__do_kernel_fault+0x6c/0x8c)
    [ 9726.534897] [<c001b5a0>] (__do_kernel_fault+0x0/0x8c) from [<c001b874>] (do_bad_area+0x48/0x74)
    [ 9726.544042] r9:000008d8 r8:7f338790 r7:8e844019 r6:00000000 r5:c0004000
    [ 9726.550928] r4:60000193
    [ 9726.553695] [<c001b82c>] (do_bad_area+0x0/0x74) from [<c001b904>] (do_translation_fault+0x64/0xa8)
    [ 9726.563113] r4:00000005
    [ 9726.565789] [<c001b8a0>] (do_translation_fault+0x0/0xa8) from [<c00085c8>] (do_DataAbort+0x3c/0xa4)
    [ 9726.575300] r7:c05b5ce8 r6:e9e31034 r5:c05bb3a4 r4:00000005
    [ 9726.581282] [<c000858c>] (do_DataAbort+0x0/0xa4) from [<c0014c98>] (__dabt_svc+0x38/0x60)
    [ 9726.589880] Exception stack(0xc05b5ce8 to 0xc05b5d30)
    [ 9726.595196] 5ce0: 0003a599 c05bd600 e9e31034 c0609d70 0097f3f0 00000000
    [ 9726.603802] 5d00: 373bf910 0003a599 7f338790 000008d8 c050ccc0 c05b5d3c c05b5d40 c05b5d30
    [ 9726.612403] 5d20: c0066fa8 c0022a28 60000193 ffffffff
    [ 9726.617710] r7:c05b5d1c r6:ffffffff r5:60000193 r4:c0022a28
    [ 9726.623696] [<c0022a14>] (omap2_gp_timer_set_next_event+0x0/0x74) from [<c0066fa8>] (clockevents_program_event+0xc0/0x130)
    [ 9726.635322] [<c0066ee8>] (clockevents_program_event+0x0/0x130) from [<c0067908>] (tick_dev_program_event+0x68/0x174)
    [ 9726.646385] r9:000008d8 r8:7f338790 r7:c05b5d88 r6:00000001 r5:00000000
    [ 9726.653269] r4:c05bd600
    [ 9726.656040] [<c00678a0>] (tick_dev_program_event+0x0/0x174) from [<c0067a88>] (tick_program_event+0x2c/0x34)
    [ 9726.666386] [<c0067a5c>] (tick_program_event+0x0/0x34) from [<c005c6a4>] (hrtimer_interrupt+0x158/0x2f8)
    [ 9726.676366] [<c005c54c>] (hrtimer_interrupt+0x0/0x2f8) from [<c0022a0c>] (omap2_gp_timer_interrupt+0x34/0x3c)
    [ 9726.686804] [<c00229d8>] (omap2_gp_timer_interrupt+0x0/0x3c) from [<c0073dc0>] (handle_irq_event_percpu+0x54/0x1a4)
    [ 9726.697790] [<c0073d6c>] (handle_irq_event_percpu+0x0/0x1a4) from [<c0073f40>] (handle_irq_event+0x30/0x40)
    [ 9726.708044] [<c0073f10>] (handle_irq_event+0x0/0x40) from [<c00762c8>] (handle_level_irq+0x88/0xf0)
    [ 9726.717555] r5:00000043 r4:c05d805c
    [ 9726.721332] [<c0076240>] (handle_level_irq+0x0/0xf0) from [<c0073a28>] (generic_handle_irq+0x38/0x44)
    [ 9726.731026] r5:00000043 r4:c05e63ac
    [ 9726.734805] [<c00739f0>] (generic_handle_irq+0x0/0x44) from [<c0016018>] (handle_IRQ+0x38/0x8c)
    [ 9726.743962] [<c0015fe0>] (handle_IRQ+0x0/0x8c) from [<c0008390>] (asm_do_IRQ+0x10/0x14)
    [ 9726.752376] r7:c05b5f3c r6:fa200000 r5:80000013 r4:c00297e8
    [ 9726.758358] [<c0008380>] (asm_do_IRQ+0x0/0x14) from [<c0014cf4>] (__irq_svc+0x34/0xa0)
    [ 9726.766682] Exception stack(0xc05b5f08 to 0xc05b5f50)
    [ 9726.771997] 5f00: ba846e68 0000aebc 386d697e 0000aebc 00000000 c05bdf30
    [ 9726.780603] 5f20: c063b1b0 00000000 80004059 413fc082 00000000 c05b5f74 386d697e c05b5f50
    [ 9726.789204] 5f40: c00629b4 c00297e8 80000013 ffffffff
    [ 9726.794527] [<c0029784>] (am33xx_enter_idle+0x0/0x94) from [<c030eb90>] (cpuidle_idle_call+0xc4/0x138)
    [ 9726.804312] r4:c05bdf20
    [ 9726.806986] [<c030eacc>] (cpuidle_idle_call+0x0/0x138) from [<c00167f4>] (cpu_idle+0x68/0xb0)
    [ 9726.815948] r9:413fc082 r8:80004059 r7:c05baf24 r6:c05baf2c r5:c0609684
    [ 9726.822832] r4:c05b4000
    [ 9726.825600] [<c001678c>] (cpu_idle+0x0/0xb0) from [<c041a9a4>] (rest_init+0x74/0x78)
    [ 9726.833741] r7:c05baf24 r6:c05a4fd4 r5:c05a5004 r4:c05b692c
    [ 9726.839726] [<c041a930>] (rest_init+0x0/0x78) from [<c057a7f8>] (start_kernel+0x27c/0x2d0)
    [ 9726.848425] [<c057a57c>] (start_kernel+0x0/0x2d0) from [<8000803c>] (0x8000803c)