We are using TSCH protocol with Contiki-NG version 4.6 with our cc1352p1f3rgzr mcu. We have based the design of the CC1352 Launchpad. The launchpad has the same problem as we do. We are having a firmware problem in the tsch operation that causes a crash, more precisely a PRECISE BUS FAULT and upon debugging that fault it shows that we have a fault when reading a register:
This is the line causing the bus fault:
HWREG(RFC_DBELL_BASE + RFC_DBELL_O_RFHWIFG) & RFC_DBELL_RFHWIFG_MDMSOFT)
This is called in prop-mode.c when receiving a tsch packet, right after turning ON the radio:
What is weird is that this crash can take hours before occuring (more than 3h).
We have successfully stopped the crash by making sure the radio is ON before reading that register (as you can see in this code) but now it seems the radio is stuck in an unknown state and unresponsive?
static int
receiving_packet(void)
{
if(!prop_radio.rf_is_on) {
return 0;
}
//Really check it
if(!rf_is_on_truely()){
count++;
if(count>10)
{
gpio_hal_arch_write_pin(GPIO_PORT, 19, 1);
}
prop_radio.rf_is_on = false;
return 0;
}
#if MAC_CONF_WITH_TSCH
/*
* Under TSCH operation, we rely on "hints" from the MDMSOFT interrupt
* flag. This flag is set by the radio upon sync word detection, but it is
* not cleared automatically by hardware. We store state in a variable after
* first call. The assumption is that the TSCH code will keep calling us
* until frame reception has completed, at which point we can clear MDMSOFT.
*/
if(!is_receiving_packet) {
/* Look for the modem synchronization word detection interrupt flag.
* This flag is raised when the synchronization word is received.
*/
if(HWREG(RFC_DBELL_BASE + RFC_DBELL_O_RFHWIFG) & RFC_DBELL_RFHWIFG_MDMSOFT) {
is_receiving_packet = 1;
}
} else {
/* After the start of the packet: reset the Rx flag once the channel gets clear */
is_receiving_packet = (cca_request() == CCA_STATE_BUSY);
if(!is_receiving_packet) {
/* Clear the modem sync flag */
RFCHwIntClear(RFC_DBELL_RFHWIFG_MDMSOFT);
}
}
return is_receiving_packet;
#else
/*
* Under CSMA operation, there is no immediately straightforward logic as to
* when it's OK to clear the MDMSOFT interrupt flag:
*
* - We cannot re-use the same logic as above, since CSMA may bail out of
* frame TX immediately after a single call this function here. In this
* scenario, is_receiving_packet would remain equal to one and we would
* therefore erroneously signal ongoing RX in subsequent calls to this
* function here, even _after_ reception has completed.
* - We can neither clear inside read_frame() nor inside the RX frame
* interrupt handler (remember, we are not in poll mode under CSMA),
* since we risk clearing MDMSOFT after we have seen a sync word for the
* _next_ frame. If this happens, this function here would incorrectly
* return 0 during RX of this next frame.
*
* So to avoid a very convoluted logic of how to handle MDMSOFT, we simply
* perform a clear channel assessment here: We interpret channel activity
* as frame reception.
*/
if(cca_request() == CCA_STATE_BUSY) {
return 1;
}
return 0;
#endif
This function is called in tsch-slot-operation.c : right at line 805.
Here is the full code of the process tsch_rx_slot :
static
PT_THREAD(tsch_rx_slot(struct pt *pt, struct rtimer *t))
{
/**
* RX slot:
* 1. Check if it is used for TIME_KEEPING
* 2. Sleep and wake up just before expected RX time (with a guard time: TS_LONG_GT)
* 3. Check for radio activity for the guard time: TS_LONG_GT
* 4. Prepare and send ACK if needed
* 5. Drift calculated in the ACK callback registered with the radio driver. Use it if receiving from a time source neighbor.
**/
struct tsch_neighbor *n;
static linkaddr_t source_address;
static linkaddr_t destination_address;
static int16_t input_index;
static int input_queue_drop = 0;
PT_BEGIN(pt);
TSCH_DEBUG_RX_EVENT();
input_index = ringbufindex_peek_put(&input_ringbuf);
if(input_index == -1) {
input_queue_drop++;
} else {
static struct input_packet *current_input;
/* Estimated drift based on RX time */
static int32_t estimated_drift;
/* Rx timestamps */
static rtimer_clock_t rx_start_time;
static rtimer_clock_t expected_rx_time;
static rtimer_clock_t packet_duration;
uint8_t packet_seen;
expected_rx_time = current_slot_start + tsch_timing[tsch_ts_tx_offset];
/* Default start time: expected Rx time */
rx_start_time = expected_rx_time;
current_input = &input_array[input_index];
/* Wait before starting to listen */
TSCH_SCHEDULE_AND_YIELD(pt, t, current_slot_start, tsch_timing[tsch_ts_rx_offset] - RADIO_DELAY_BEFORE_RX, "RxBeforeListen");
TSCH_DEBUG_RX_EVENT();
/* Start radio for at least guard time */
tsch_radio_on(TSCH_RADIO_CMD_ON_WITHIN_TIMESLOT);
packet_seen = NETSTACK_RADIO.receiving_packet() || NETSTACK_RADIO.pending_packet();
if(!packet_seen) {
/* Check if receiving within guard time */
RTIMER_BUSYWAIT_UNTIL_ABS((packet_seen = (NETSTACK_RADIO.receiving_packet() || NETSTACK_RADIO.pending_packet())),
current_slot_start, tsch_timing[tsch_ts_rx_offset] + tsch_timing[tsch_ts_rx_wait] + RADIO_DELAY_BEFORE_DETECT);
}
if(!packet_seen) {
/* no packets on air */
tsch_radio_off(TSCH_RADIO_CMD_OFF_FORCE);
} else {
TSCH_DEBUG_RX_EVENT();
/* Save packet timestamp */
rx_start_time = RTIMER_NOW() - RADIO_DELAY_BEFORE_DETECT;
/* Wait until packet is received, turn radio off */
RTIMER_BUSYWAIT_UNTIL_ABS(!NETSTACK_RADIO.receiving_packet(),
current_slot_start, tsch_timing[tsch_ts_rx_offset] + tsch_timing[tsch_ts_rx_wait] + tsch_timing[tsch_ts_max_tx]);
TSCH_DEBUG_RX_EVENT();
tsch_radio_off(TSCH_RADIO_CMD_OFF_WITHIN_TIMESLOT);
if(NETSTACK_RADIO.pending_packet()) {
static int frame_valid;
static int header_len;
static frame802154_t frame;
radio_value_t radio_last_rssi;
radio_value_t radio_last_lqi;
/* Read packet */
current_input->len = NETSTACK_RADIO.read((void *)current_input->payload, TSCH_PACKET_MAX_LEN);
NETSTACK_RADIO.get_value(RADIO_PARAM_LAST_RSSI, &radio_last_rssi);
current_input->rx_asn = tsch_current_asn;
current_input->rssi = (signed)radio_last_rssi;
current_input->channel = tsch_current_channel;
header_len = frame802154_parse((uint8_t *)current_input->payload, current_input->len, &frame);
frame_valid = header_len > 0 &&
frame802154_check_dest_panid(&frame) &&
frame802154_extract_linkaddr(&frame, &source_address, &destination_address);
#if TSCH_RESYNC_WITH_SFD_TIMESTAMPS
/* At the end of the reception, get an more accurate estimate of SFD arrival time */
NETSTACK_RADIO.get_object(RADIO_PARAM_LAST_PACKET_TIMESTAMP, &rx_start_time, sizeof(rtimer_clock_t));
#endif
packet_duration = TSCH_PACKET_DURATION(current_input->len);
/* limit packet_duration to its max value */
packet_duration = MIN(packet_duration, tsch_timing[tsch_ts_max_tx]);
if(!frame_valid) {
TSCH_LOG_ADD(tsch_log_message,
snprintf(log->message, sizeof(log->message),
"!failed to parse frame %u %u", header_len, current_input->len));
}
if(frame_valid) {
if(frame.fcf.frame_type != FRAME802154_DATAFRAME
&& frame.fcf.frame_type != FRAME802154_BEACONFRAME) {
TSCH_LOG_ADD(tsch_log_message,
snprintf(log->message, sizeof(log->message),
"!discarding frame with type %u, len %u", frame.fcf.frame_type, current_input->len));
frame_valid = 0;
}
}
#if LLSEC802154_ENABLED
/* Decrypt and verify incoming frame */
if(frame_valid) {
if(tsch_security_parse_frame(
current_input->payload, header_len, current_input->len - header_len - tsch_security_mic_len(&frame),
&frame, &source_address, &tsch_current_asn)) {
current_input->len -= tsch_security_mic_len(&frame);
} else {
TSCH_LOG_ADD(tsch_log_message,
snprintf(log->message, sizeof(log->message),
"!failed to authenticate frame %u", current_input->len));
frame_valid = 0;
}
}
#endif /* LLSEC802154_ENABLED */
if(frame_valid) {
/* Check that frome is for us or broadcast, AND that it is not from
* ourselves. This is for consistency with CSMA and to avoid adding
* ourselves to neighbor tables in case frames are being replayed. */
if((linkaddr_cmp(&destination_address, &linkaddr_node_addr)
|| linkaddr_cmp(&destination_address, &linkaddr_null))
&& !linkaddr_cmp(&source_address, &linkaddr_node_addr)) {
int do_nack = 0;
rx_count++;
estimated_drift = RTIMER_CLOCK_DIFF(expected_rx_time, rx_start_time);
tsch_stats_on_time_synchronization(estimated_drift);
#if TSCH_TIMESYNC_REMOVE_JITTER
/* remove jitter due to measurement errors */
if(ABS(estimated_drift) <= TSCH_TIMESYNC_MEASUREMENT_ERROR) {
estimated_drift = 0;
} else if(estimated_drift > 0) {
estimated_drift -= TSCH_TIMESYNC_MEASUREMENT_ERROR;
} else {
estimated_drift += TSCH_TIMESYNC_MEASUREMENT_ERROR;
}
#endif
#ifdef TSCH_CALLBACK_DO_NACK
if(frame.fcf.ack_required) {
do_nack = TSCH_CALLBACK_DO_NACK(current_link,
&source_address, &destination_address);
}
#endif
if(frame.fcf.ack_required) {
static uint8_t ack_buf[TSCH_PACKET_MAX_LEN];
static int ack_len;
/* Build ACK frame */
ack_len = tsch_packet_create_eack(ack_buf, sizeof(ack_buf),
&source_address, frame.seq, (int16_t)RTIMERTICKS_TO_US(estimated_drift), do_nack);
if(ack_len > 0) {
#if LLSEC802154_ENABLED
if(tsch_is_pan_secured) {
/* Secure ACK frame. There is only header and header IEs, therefore data len == 0. */
ack_len += tsch_security_secure_frame(ack_buf, ack_buf, ack_len, 0, &tsch_current_asn);
}
#endif /* LLSEC802154_ENABLED */
/* Copy to radio buffer */
NETSTACK_RADIO.prepare((const void *)ack_buf, ack_len);
/* Wait for time to ACK and transmit ACK */
TSCH_SCHEDULE_AND_YIELD(pt, t, rx_start_time,
packet_duration + tsch_timing[tsch_ts_tx_ack_delay] - RADIO_DELAY_BEFORE_TX, "RxBeforeAck");
TSCH_DEBUG_RX_EVENT();
NETSTACK_RADIO.transmit(ack_len);
tsch_radio_off(TSCH_RADIO_CMD_OFF_WITHIN_TIMESLOT);
/* Schedule a burst link iff the frame pending bit was set */
burst_link_scheduled = tsch_packet_get_frame_pending(current_input->payload, current_input->len);
}
}
/* If the sender is a time source, proceed to clock drift compensation */
n = tsch_queue_get_nbr(&source_address);
if(n != NULL && n->is_time_source) {
int32_t since_last_timesync = TSCH_ASN_DIFF(tsch_current_asn, last_sync_asn);
/* Keep track of last sync time */
last_sync_asn = tsch_current_asn;
tsch_last_sync_time = clock_time();
/* Save estimated drift */
drift_correction = -estimated_drift;
is_drift_correction_used = 1;
sync_count++;
tsch_timesync_update(n, since_last_timesync, -estimated_drift);
tsch_schedule_keepalive(0);
}
/* Add current input to ringbuf */
ringbufindex_put(&input_ringbuf);
/* If the neighbor is known, update its stats */
if(n != NULL) {
NETSTACK_RADIO.get_value(RADIO_PARAM_LAST_LINK_QUALITY, &radio_last_lqi);
tsch_stats_rx_packet(n, current_input->rssi, radio_last_lqi, tsch_current_channel);
}
/* Log every reception */
TSCH_LOG_ADD(tsch_log_rx,
linkaddr_copy(&log->rx.src, (linkaddr_t *)&frame.src_addr);
log->rx.is_unicast = frame.fcf.ack_required;
log->rx.datalen = current_input->len;
log->rx.drift = drift_correction;
log->rx.drift_used = is_drift_correction_used;
log->rx.is_data = frame.fcf.frame_type == FRAME802154_DATAFRAME;
log->rx.sec_level = frame.aux_hdr.security_control.security_level;
log->rx.estimated_drift = estimated_drift;
log->rx.seqno = frame.seq;
);
}
/* Poll process for processing of pending input and logs */
process_poll(&tsch_pending_events_process);
}
}
tsch_radio_off(TSCH_RADIO_CMD_OFF_END_OF_TIMESLOT);
}
if(input_queue_drop != 0) {
TSCH_LOG_ADD(tsch_log_message,
snprintf(log->message, sizeof(log->message),
"!queue full skipped %u", input_queue_drop);
);
input_queue_drop = 0;
}
}
TSCH_DEBUG_RX_EVENT();
PT_END(pt);
}
/*-------------------------
It really looks like the radio is still off when we try to read that register....
Any idea why this fault would occur so rarely? Do you know any fixes or workaround to prevent this crash from occuring?
Is there some changes to recent TI SDK that would help with this kind of problem?
Do you know what is the minimum timing between turning radio ON and reading a register?
How could we make the radio reset?
Is there some hot fix we can do to prevent this?
Thank you
