AM625: AM625 RTC issue

Hi Michael,

Do you mean this interrupt storm only happens on your custom board, not on the AM62 EVM?

I am not sure why the RTC interrupt happens but the IRQ status MMR doesn't show any RTC event, but can you please try the following kernel patch to see if it clears the interrupt storm?

diff --git a/drivers/rtc/rtc-ti-k3.c b/drivers/rtc/rtc-ti-k3.c
index 0d90fe923355..85ca9e086275 100644
--- a/drivers/rtc/rtc-ti-k3.c
+++ b/drivers/rtc/rtc-ti-k3.c
@@ -429,7 +429,9 @@ static irqreturn_t ti_k3_rtc_interrupt(s32 irq, void *dev_id)
                dev_err(dev,
                        HW_ERR
                        "Erratum i2327/IRQ trig: status: 0x%08x / 0x%08x\n", reg, raw);
-               return IRQ_NONE;
+               ti_k3_rtc_alarm_irq_enable(dev, 0);
+               ti_k3_rtc_alarm_irq_enable(dev, 1);
+               return IRQ_HANDLED;
        }

Hi,

Yes custom design but not all the part are affecting. For now only my hardware has this problem. I have applied this patch and this is the result

[  112.840422] ti_k3_rtc_interrupt: 29761 callbacks suppressed
[  112.840467] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  112.856495] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  112.866603] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  112.876649] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  112.886728] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  112.896850] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  112.906969] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  112.917056] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  112.927136] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  112.937217] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  117.844324] ti_k3_rtc_interrupt: 29737 callbacks suppressed
[  117.844367] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  117.860393] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  117.870502] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  117.880548] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  117.890628] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  117.900752] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  117.910871] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  117.920955] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  117.931031] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[  117.941112] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000

Slide change to not have console dead

--- a/drivers/rtc/rtc-ti-k3.c
+++ b/drivers/rtc/rtc-ti-k3.c
@@ -426,10 +426,12 @@ static irqreturn_t ti_k3_rtc_interrupt(s32 irq, void *dev_id)
        if (!reg) {
                u32 raw = k3rtc_field_read(priv, K3RTC_IRQ_STATUS_RAW);
 
-               dev_err(dev,
-                       HW_ERR
-                       "Erratum i2327/IRQ trig: status: 0x%08x / 0x%08x\n", reg, raw);
-               return IRQ_NONE;
+               dev_err_ratelimited(dev,
+                                   HW_ERR
+                                   "Erratum i2327/IRQ trig: status: 0x%08x / 0x%08x\n", reg, raw);
+               ti_k3_rtc_alarm_irq_enable(dev, 0);
+               ti_k3_rtc_alarm_irq_enable(dev, 1);
+               return IRQ_HANDLED;
        }

Hi,

One point is that I don't load and use cortex m4 firmware and only R5 one. Is the m4 have some role on it?

Hi Michael,

Thanks for the experiment.

I don't think the m4 firmware is relevant, but I just removed /lib/firmware/am62-mcu-m4f0_0-fw on my SK-AM62B setup, the issue doesn't happen.

I will check internally to understand what would cause the RTC interrupt storm.

Hi Michael,

The RTC driver only uses the RTC ON_OFF event (bit0 in IRQ_STATUS register), I am wondering if other event is enabled/generated. Can you please apply the following kernel debug patch and provide the console log when the interrupt storm happens?

diff --git a/drivers/rtc/rtc-ti-k3.c b/drivers/rtc/rtc-ti-k3.c
index 0d90fe923355..06bee9b0c7e9 100644
--- a/drivers/rtc/rtc-ti-k3.c
+++ b/drivers/rtc/rtc-ti-k3.c
@@ -94,9 +94,9 @@ static const struct reg_field ti_rtc_reg_fields[] = {
        /* We use on to off as alarm trigger */
        [K3RTC_ALM_S_CNT_LSW] = REG_FIELD(REG_K3RTC_ON_OFF_S_CNT_LSW, 0, 31),
        [K3RTC_ALM_S_CNT_MSW] = REG_FIELD(REG_K3RTC_ON_OFF_S_CNT_MSW, 0, 15),
-       [K3RTC_IRQ_STATUS_RAW] = REG_FIELD(REG_K3RTC_IRQSTATUS_RAW_SYS, 0, 0),
+       [K3RTC_IRQ_STATUS_RAW] = REG_FIELD(REG_K3RTC_IRQSTATUS_RAW_SYS, 0, 31),
        [K3RTC_IRQ_STATUS] = REG_FIELD(REG_K3RTC_IRQSTATUS_SYS, 0, 0),
-       [K3RTC_IRQ_ENABLE_SET] = REG_FIELD(REG_K3RTC_IRQENABLE_SET_SYS, 0, 0),
+       [K3RTC_IRQ_ENABLE_SET] = REG_FIELD(REG_K3RTC_IRQENABLE_SET_SYS, 0, 31),
        [K3RTC_IRQ_ENABLE_CLR] = REG_FIELD(REG_K3RTC_IRQENABLE_CLR_SYS, 0, 0),
        /* Off to on is alternate */
        [K3RTC_IRQ_STATUS_ALT] = REG_FIELD(REG_K3RTC_IRQSTATUS_SYS, 1, 1),
@@ -300,6 +300,8 @@ static int ti_k3_rtc_alarm_irq_enable(struct device *dev, unsigned int enabled)
        u32 offset = enabled ? K3RTC_IRQ_ENABLE_SET : K3RTC_IRQ_ENABLE_CLR;
 
        reg = k3rtc_field_read(priv, K3RTC_IRQ_ENABLE_SET);
+       dev_err(dev, "irq enable set: 0x%x\n", reg);
+       reg &= 1;
        if ((enabled && reg) || (!enabled && !reg))
                return 0;
 
@@ -323,7 +325,7 @@ static int ti_k3_rtc_read_alarm(struct device *dev, struct rtc_wkalrm *alarm)
 
        rtc_time64_to_tm((((time64_t)seconds_hi) << 32) | (time64_t)seconds_lo, &alarm->time);
 
-       alarm->enabled = k3rtc_field_read(priv, K3RTC_IRQ_ENABLE_SET);
+       alarm->enabled = k3rtc_field_read(priv, K3RTC_IRQ_ENABLE_SET) & 1;
 
        return 0;
 }

I'm not able to execute this step fast

/*                                         
 * RTC Erratum i2327 Workaround            
 * Due to a bug in initial synchronization out of cold power on,
 * IRQ status can get locked infinitely if we do not:
 * a) unlock RTC                           
 *                                         
 * This workaround *must* be applied within 1 second of power on,
 * So, this is closest point to be able to guarantee the max
 * timing.
 */
void rtc_erratumi2327_init(void)
{
        u32 counter;

        /*                                 
         * If counter has gone past 1, nothing we can do, leave
         * system locked! This is the only way we know if RTC   
         * can be used for all practical purposes.
         */
        counter = readl(REG_K3RTC_S_CNT_LSW);
        if (counter > 1)
                return;                    
        /*
         * Need to set this up at the very start
         * MUST BE DONE under 1 second of boot.
         */                                
        writel(K3RTC_KICK0_UNLOCK_VALUE, REG_K3RTC_KICK0);
        writel(K3RTC_KICK1_UNLOCK_VALUE, REG_K3RTC_KICK1);
        return;
}

Because I can only boot from USB

Hi Michael,

The step is done in R5 SPL, which should be done within a sec even boot from USB thumb drive.

And even if it is not done in SPL, kernel would disable RTC in the RTC driver init. So I don't think this is related the RTC interrupt storm.

DFU is a bit slow to execute and yes you are right, I'm now checking that code in linux driver that should avoid rtc working. I will test your patch too

rtcwake --seconds 4
rtcwake: assuming RTC uses UTC ...
[   24.430069] rtc-ti-k3 2b1f0000.rtc: irq enable set: 0x0
rtcwake: wakeup from "suspend" using /dev/rtc0 at Fri Mar  9 12:[   24.438113] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
35:23 2018
[   24.453743] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   24.464665] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   24.474772] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   24.484864] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   24.494946] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   24.505011] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   24.515092] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   24.525215] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   24.535282] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
rtcwake: write error
0;root@bsh-ccm-m3:~✘ 1 root@bsh-ccm-m3 ~ # [   29.441157] ti_k3_rtc_interrupt: 30872 callbacks suppressed
[   29.441227] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   29.457156] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   29.467211] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   29.477292] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   29.487489] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   29.497603] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   29.507661] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   29.517719] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   29.527796] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   29.537867] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000

0;root@bsh-ccm-m3:~✘ 1 root@bsh-ccm-m3 ~ # [   34.444984] ti_k3_rtc_interrupt: 30919 callbacks suppressed
[   34.445006] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   34.460636] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   34.470715] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   34.480876] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   34.490963] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   34.501015] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   34.511068] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   34.521136] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   34.531195] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   34.541252] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000

0;root@bsh-ccm-m3:~✘ 1 root@bsh-ccm-m3 ~ # dme[   39.449089] ti_k3_rtc_interrupt: 30953 callbacks suppressed
[   39.449161] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   39.465032] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   39.475095] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   39.485139] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   39.495203] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   39.505307] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   39.515396] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   39.525465] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   39.535524] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
[   39.545585] rtc-ti-k3 2b1f0000.rtc: [Hardware Error]: Erratum i2327/IRQ trig: status: 0x00000000 / 0x00000000
sg | [   40.687324] irq 289: nobody cared (try booting with the "irqpoll" option)
[   40.694115] CPU: 0 PID: 108 Comm: irq/289-2b1f000 Not tainted 6.1.33-g999 #1
[   40.701149] Hardware name: BSH - CCM-M3 (DT)
[   40.705406] Call trace:
[   40.707842]  dump_backtrace.part.0+0xdc/0xf0
[   40.712119]  show_stack+0x18/0x30
[   40.715426]  dump_stack_lvl+0x68/0x84
[   40.719083]  dump_stack+0x18/0x34
[   40.722389]  __report_bad_irq+0x4c/0xdc
[   40.726215]  note_interrupt+0x33c/0x3a0
[   40.730044]  handle_irq_event+0x9c/0xbc
[   40.733869]  handle_fasteoi_irq+0xa4/0x1f4
[   40.737956]  generic_handle_domain_irq+0x2c/0x44
[   40.742562]  gic_handle_irq+0x50/0x124
[   40.746307]  call_on_irq_stack+0x24/0x4c
[   40.750219]  do_interrupt_handler+0x80/0x84
[   40.754393]  el1_interrupt+0x34/0x70
[   40.757958]  el1h_64_irq_handler+0x18/0x2c
[   40.762044]  el1h_64_irq+0x64/0x68
[   40.765435]  _raw_spin_unlock_irq+0x18/0x50
[   40.769609]  irq_thread_fn+0x60/0xb0
[   40.773173]  irq_thread+0x168/0x24c
[   40.776652]  kthread+0x10c/0x110
[   40.779874]  ret_from_fork+0x10/0x20
[   40.783439] handlers:
[   40.785700] [<000000001f42b04b>] irq_default_primary_handler threaded [<00000000fb0fe625>] ti_k3_rtc_interrupt
[   40.795695] Disabling IRQ #289

Hi Michael,

Thanks for testing the patch. The log shows there is no any interrupt event seen in the RTC irq handler. I will check internally to understand how a spurious interrupt could happen.

Can you please do another quick test - after power on the board, wait for at least a couple seconds before send tiboot3.bin via DFU and check if RTC is not initialized in kernel? This would ensure the RTC unlock logic in both U-Boot and kernel have no issue. 

Hi

diff --git a/arch/arm/mach-k3/am625_init.c b/arch/arm/mach-k3/am625_init.c
index d5d741cb93..1f284f6a2c 100644
--- a/arch/arm/mach-k3/am625_init.c
+++ b/arch/arm/mach-k3/am625_init.c
@@ -92,7 +92,7 @@ static __maybe_unused void enable_mcu_esm_reset(void)
  * So, this is closest point to be able to guarantee the max
  * timing.
  */
-void rtc_erratumi2327_init(void)
+bool rtc_erratumi2327_init(void)
 {
        u32 counter;
 
@@ -103,14 +103,14 @@ void rtc_erratumi2327_init(void)
         */
        counter = readl(REG_K3RTC_S_CNT_LSW);
        if (counter > 1)
-               return;
+               return false;
        /*
         * Need to set this up at the very start
         * MUST BE DONE under 1 second of boot.
         */
        writel(K3RTC_KICK0_UNLOCK_VALUE, REG_K3RTC_KICK0);
        writel(K3RTC_KICK1_UNLOCK_VALUE, REG_K3RTC_KICK1);
-       return;
+       return true;
 }
 #endif
 
@@ -118,10 +118,11 @@ void board_init_f(ulong dummy)
 {
        struct udevice *dev;
        int ret;
-
 #if defined(CONFIG_CPU_V7R)
+       bool rtc_enabled = false;
+
        setup_k3_mpu_regions();
-       rtc_erratumi2327_init();
+       rtc_enabled = rtc_erratumi2327_init();
 #endif
 
        /*
@@ -223,6 +224,14 @@ void board_init_f(ulong dummy)
        }
 
        spl_enable_dcache();
+
+#if defined(CONFIG_CPU_V7R)
+       if (rtc_enabled) {
+               printf("RTC get enabled\n");
+       } else {
+               printf("RTC fail to get enabled\n");
+       }
+#endif
 }

I have applied this patch to uboot, to print out the fact if the rtc get enabled or not.
what I have always even I power few seconds before to flash is RTC get enabled. That is not possible according to the errata