我正在努力理解一种很少发生的情况（MTBF 以几周到几个月为单位测量），但一旦发生就会导致致命的内核锁定。
该环境是内核 5.4.0，在 AMD/Xilinx Zynq 7020 SoC 中的双核 ARM A9 32 位 (armhf) 处理器上运行。我已经能够使用 Xilinx Vitis 系统调试器检查两个故障设备上的堆栈跟踪。在这两种情况下，两个内核都在等待（尝试锁定）同一个自旋锁。自旋锁是与线程 IRQ 处理程序关联的结构pi_lock的成员。struct task_struct
核心 #0 正在运行空闲进程，响应中断，尝试唤醒 IRQ 处理程序线程。核心 #1 正在读取与 IRQ 处理程序线程关联的 sysfs 伪文件（同时读取代表 的/proc/<PID>/stat所有 sysfs 伪文件，可能是应 的请求，由 调用）。/proc/<PID>/statpidofsessioncleanapache2
在一个设备上处理的中断用于 BQ25890 电池充电器（每秒断言约 0 到 10 次，而在另一台设备上则用于自定义内核模块（约 100 次/秒）。两个中断都使用默认的主 IRQ 处理程序。
目前的假设是，核心 #0 首先被卡住，一段时间后核心 #1 也卡住了，而不是两者同时发生，但这只是推测。
下面是两个内核的堆栈跟踪以及其中一个设备的线程 IRQ 处理程序源。
问题
堆栈跟踪中是否有足够的信息来提供有关出错原因的线索？如果是这样，出了什么问题？如果没有，还需要哪些额外信息？其中一台设备仍然可用，并且可以询问一些（不是全部）变量。禁用中断（两个内核的 cpsr.i == 1）这一事实是否重要？
请注意，由于故障间隔时间极长，黑盒测试技术（即进行更改以观察结果）并不实用。
ARM Cortex-A9 MPCore #0（已暂停）
0x80165d68: ./include/linux/compiler.h, line 199
0x80165d68 arch_spin_lock(): .../arch/arm/include/asm/spinlock.h, line 75
0x8076f774: ./include/linux/spinlock.h, line 193
0x8076f774: ...include/linux/spinlock_api_smp.h, line 119
0x8076f774 _raw_spin_lock_irqsave(): kernel/locking/spinlock.c, line 159
0x8014a394 try_to_wake_up(): kernel/sched/core.c, line 2551
0x8014a724 wake_up_process(): kernel/sched/core.c, line 2667
0x8016f600 __irq_wake_thread(): kernel/irq/handle.c, line 134
0x8016f820 __handle_irq_event_percpu(): kernel/irq/handle.c, line 167
0x8016f888 handle_irq_event_percpu(): kernel/irq/handle.c, line 189
0x8016f924 handle_irq_event(): kernel/irq/handle.c, line 206
0x80174230 handle_level_irq(): kernel/irq/chip.c, line 650
0x8016e808: ./include/linux/irqdesc.h, line 156
0x8016e808 generic_handle_irq(): kernel/irq/irqdesc.c, line 644
0x80447a20: drivers/gpio/gpio-zynq.c, line 632
0x80447a20 zynq_gpio_irqhandler(): drivers/gpio/gpio-zynq.c, line 661
0x8016e808: ./include/linux/irqdesc.h, line 156
0x8016e808 generic_handle_irq(): kernel/irq/irqdesc.c, line 644
0x8016eec4 __handle_domain_irq(): kernel/irq/irqdesc.c, line 681
0x80102300: ./include/linux/irqdesc.h, line 174
0x80102300 gic_handle_irq(): drivers/irqchip/irq-gic.c, line 383
0x80101a70 __irq_svc(): arch/arm/kernel/entry-armv.S, line 212
0x805827c0 cpuidle_enter_state(): .../arch/arm/include/asm/irqflags.h, line 39
0x805829d8 cpuidle_enter(): drivers/cpuidle/cpuidle.c, line 344
0x8014ecf0: kernel/sched/idle.c, line 117
0x8014ecf0: kernel/sched/idle.c, line 201
0x8014ecf0 do_idle(): kernel/sched/idle.c, line 263
0x8014eeac cpu_startup_entry(): kernel/sched/idle.c, line 355
0x80769828 rest_init(): init/main.c, line 451
0x80c00b30 arch_call_rest_init(): init/main.c, line 572
0x80c00f78 start_kernel(): init/main.c, line 784
0x00000000
0x00000000
违规的锁
lock arch_spinlock_t * {{slock=0x94449440, tickets={owner=0x9440, next=0x9444}}}
slock u32 0x94449440
tickets struct __raw_tickets {owner=0x9440, next=0x9444}
owner u16 0x9440
next u16 0x9444
线程化 IRQ 处理程序源
请注意，此代码不会出现在调用堆栈中，而是 core #0 尝试唤醒的线程的代码。
在其中一台设备上：
static irqreturn_t bq25890_irq_handler_thread(int irq, void *private)
{
struct bq25890_device *bq = private;
int ret;
struct bq25890_state state;
ret = bq25890_get_chip_state(bq, &state);
if (ret < 0)
goto handled;
if (!bq25890_state_changed(bq, &state))
goto handled;
bq25890_handle_state_change(bq, &state);
mutex_lock(&bq->lock);
bq->state = state;
mutex_unlock(&bq->lock);
power_supply_changed(bq->charger);
handled:
return IRQ_HANDLED;
}
在另一台设备上：
static irqreturn_t core100_irq_handler( int irq, void* data )
{
struct core100_drvdata* p_info;
struct core100_block_header* next;
unsigned long iflags;
p_info = ( struct core100_drvdata* ) data;
if ( ( p_info->state != state_reading ) && ( p_info->state != state_writing ) )
{
return IRQ_HANDLED;
}
next = &p_info->current_block->next->header;
next->count_at_pps = read_register( p_info->count_pps_register );
next->frc_at_pps = read_register( p_info->frc_pps_register );
next->count = adc_counter();
next->frc = free_running_counter();
spin_lock_irqsave( &p_info->state_lock, iflags );
p_info->dma_head += bytes_per_block( p_info );
if ( p_info->dma_head - p_info->dma_tail > bytes_per_buffer( p_info ) )
{
p_info->dma_tail = p_info->dma_head - bytes_per_buffer( p_info );
}
p_info->current_block = p_info->current_block->next;
wake_up_interruptible( &p_info->wq );
spin_unlock_irqrestore( &p_info->state_lock, iflags );
return IRQ_HANDLED;
}
内核源码：[struct task_struct][1]
ARM Cortex-A9 MPCore #1（已暂停）
0x80165d68: ./include/linux/compiler.h, line 199
0x80165d68 arch_spin_lock(): .../arch/arm/include/asm/spinlock.h, line 75
0x8076f774: ./include/linux/spinlock.h, line 193
0x8076f774: ...include/linux/spinlock_api_smp.h, line 119
0x8076f774 _raw_spin_lock_irqsave(): kernel/locking/spinlock.c, line 159
0x80148914 task_rq_lock(): kernel/sched/core.c, line 109
0x8014e22c: kernel/sched/cputime.c, line 281
0x8014e22c thread_group_cputime(): kernel/sched/cputime.c, line 326
0x8014e650 thread_group_cputime_adjusted(): kernel/sched/cputime.c, line 678
0x802ccb5c do_task_stat(): fs/proc/array.c, line 510
0x802cdaf4 proc_tgid_stat(): fs/proc/array.c, line 632
0x802c7c0c proc_single_show(): fs/proc/base.c, line 756
0x80281b18 seq_read(): fs/seq_file.c, line 229
0x8025dd1c __vfs_read(): fs/read_write.c, line 425
0x8025de60 vfs_read(): fs/read_write.c, line 461
0x8025e088 ksys_read(): fs/read_write.c, line 587
0x8025e0ec: fs/read_write.c, line 597
0x8025e0ec __se_sys_read(): fs/read_write.c, line 595
0x80101000 __idmap_text_end()
0x76e627e6
0x76edc616
这是另一个视图，显示与其调用进程关联的调用堆栈。 “Swaper”是空闲进程的一个古老术语：
Kernel
0 swapper/0 (Suspended), ARM Cortex-A9 MPCore #0
0x80165d68: ./include/linux/compiler.h, line 199
0x80165d68 arch_spin_lock(): .../arch/arm/include/asm/spinlock.h, line 75
0x8076f774: ./include/linux/spinlock.h, line 193
0x8076f774: ...include/linux/spinlock_api_smp.h, line 119
0x8076f774 _raw_spin_lock_irqsave(): kernel/locking/spinlock.c, line 159
11950 pidof
11950 (Suspended), ARM Cortex-A9 MPCore #1
0x80165d68: ./include/linux/compiler.h, line 199
0x80165d68 arch_spin_lock(): .../arch/arm/include/asm/spinlock.h, line 75
0x8076f774: ./include/linux/spinlock.h, line 193
0x8076f774: ...include/linux/spinlock_api_smp.h, line 119
0x8076f774 _raw_spin_lock_irqsave(): kernel/locking/spinlock.c, line 159
RCU配置
# RCU Subsystem
CONFIG_PREEMPT_RCU=y
# CONFIG_RCU_EXPERT is not set
CONFIG_SRCU=y
CONFIG_TREE_SRCU=y
CONFIG_TASKS_RCU=y
CONFIG_RCU_STALL_COMMON=y
CONFIG_RCU_NEED_SEGCBLIST=y
# end of RCU Subsystem
# RCU Debugging
# CONFIG_RCU_PERF_TEST is not set
# CONFIG_RCU_TORTURE_TEST is not set
CONFIG_RCU_CPU_STALL_TIMEOUT=21
# CONFIG_RCU_TRACE is not set
# CONFIG_RCU_EQS_DEBUG is not set
# end of RCU Debugging
内核线程优先级
# ps -eo pid,tid,class,rtprio,ni,pri,wchan:14,comm | grep -E '(PID|bq25890|raw_dma)'
PID TID CLS RTPRIO NI PRI WCHAN COMMAND
214 214 FF 50 - 90 irq_thread irq/59-bq25890_
229 229 FF 50 - 90 irq_thread irq/60-raw_dma
处理器状态寄存器
在至少一个发生故障的系统上，该interrupt disable位在两个处理器上均被设置。这似乎是错误的，尽管该fast interrupt disable位尚未设置。如果我在工作系统上设置断点arch_spin_lock()，则不会设置中断禁用位。似乎释放自旋锁的唯一方法是响应中断，因为两个内核上的活动线程都不会执行此操作。
Core #0
cpsr 200f0193 537854355
n 0 0 Negative condition code flag
z 0 0 Zero condition code flag
c 1 1 Carry condition code flag
v 0 0 Overflow condition code flag
q 0 0 Cumulative saturation flag
it 00 0 If-Then execution state bits
j 0 0 Jazelle bit
ge f 15 SIMD Greater than or Equal flags
e 0 0 Endianness execution state bit
a 1 1 Asynchronous abort disable bit
i 1 1 Interrupt disable bit
f 0 0 Fast interrupt disable bit
t 0 0 Thumb execution state bit
m 13 19 Mode field
irq
sp 80d94400 2161722368
lr 80101e20 2148539936
spsr 200f0193 537854355
n 0 0 Negative condition code flag
z 0 0 Zero condition code flag
c 1 1 Carry condition code flag
v 0 0 Overflow condition code flag
q 0 0 Cumulative saturation flag
it 00 0 If-Then execution state bits
j 0 0 Jazelle bit
ge f 15 SIMD Greater than or Equal flags
e 0 0 Endianness execution state bit
a 1 1 Asynchronous abort disable bit
i 1 1 Interrupt disable bit
f 0 0 Fast interrupt disable bit
t 0 0 Thumb execution state bit
m 13 19 Mode field
Core #1
cpsr 200f0093 537854099
n 0 0 Negative condition code flag
z 0 0 Zero condition code flag
c 1 1 Carry condition code flag
v 0 0 Overflow condition code flag
q 0 0 Cumulative saturation flag
it 00 0 If-Then execution state bits
j 0 0 Jazelle bit
ge f 15 SIMD Greater than or Equal flags
e 0 0 Endianness execution state bit
a 0 0 Asynchronous abort disable bit
i 1 1 Interrupt disable bit
f 0 0 Fast interrupt disable bit
t 0 0 Thumb execution state bit
m 13 19 Mode field
irq
sp 80d94440 2161722432
lr 80101a00 2148538880
spsr 60030193 1610809747
n 0 0 Negative condition code flag
z 1 1 Zero condition code flag
c 1 1 Carry condition code flag
v 0 0 Overflow condition code flag
q 0 0 Cumulative saturation flag
it 00 0 If-Then execution state bits
j 0 0 Jazelle bit
ge 3 3 SIMD Greater than or Equal flags
e 0 0 Endianness execution state bit
a 1 1 Asynchronous abort disable bit
i 1 1 Interrupt disable bit
f 0 0 Fast interrupt disable bit
t 0 0 Thumb execution state bit
m 13 19 Mode field