Linux 内存使用量高于进程总和

Linux 内存使用量高于进程总和

我正在尝试找出是什么占用了我的内存。我猜测,如果它未被用户空间进程使用,那么它可能来自内核方面。

这是一些命令的输出。我知道 Linux 使用的是写时复制,因此预期的内存使用量应为 514MB 或更少(共享段总共只有 7.75 MB)。free -m 报告已使用 2804 MB。我尝试使用 python 来处理它,当使用量超过 7324 MB 时,它确实开始交换(free -m 报告)。那么我如何才能找出使用了 2804-514 MB 内存的是什么?我没有找到任何简单的方法来按内核模块细分已使用的内存。

基本系统信息:Linux Mint 17,带有 BTRFS 文件系统的 ThinkPad T530。

$ free -m
             total       used       free     shared    buffers     cached
Mem:          7669       4055       3614         66          0       1250
-/+ buffers/cache:       2804       4864
Swap:         5367          0       5367


$ ipcs -m|tail -n +4|awk '{sum+=$5} END {print sum / 1024/1024}'
7.75

$ sudo ps axo pid,rss,cmd  | awk '{sum+=$2} END {print sum / 1024}'
514.43

$ smem -twk
Area                           Used      Cache   Noncache
firmware/hardware                 0          0          0
kernel image                      0          0          0
kernel dynamic memory          3.6G       1.2G       2.3G
userspace memory             412.3M      52.6M     359.7M
free memory                    3.5G       3.5G          0
----------------------------------------------------------
                               7.5G       4.8G       2.7G

meminfo(注意,数字以 MB 为单位)

cat /proc/meminfo |awk '{print $1"\t"($2/1024)"\t"$3}'
MemTotal:       7669.72 kB
MemFree:        3615.05 kB
Buffers:        0.078125        kB
Cached: 1250.41 kB
SwapCached:     0       kB
Active: 825.223 kB
Inactive:       781.68  kB
Active(anon):   124.059 kB
Inactive(anon): 298.477 kB
Active(file):   701.164 kB
Inactive(file): 483.203 kB
Unevictable:    0       kB
Mlocked:        0       kB
SwapTotal:      5367.33 kB
SwapFree:       5367.33 kB
Dirty:  0       kB
Writeback:      0       kB
AnonPages:      356.391 kB
Mapped: 52.7031 kB
Shmem:  66.1211 kB
Slab:   100.266 kB
SReclaimable:   54.0039 kB
SUnreclaim:     46.2617 kB
KernelStack:    3.35156 kB
PageTables:     18.0586 kB
NFS_Unstable:   0       kB
Bounce: 0       kB
WritebackTmp:   0       kB
CommitLimit:    9202.19 kB
Committed_AS:   1343.93 kB
VmallocTotal:   3.35544e+07     kB
VmallocUsed:    352.312 kB
VmallocChunk:   3.35541e+07     kB
HardwareCorrupted:      0       kB
AnonHugePages:  58      kB
HugePages_Total:        0
HugePages_Free: 0
HugePages_Rsvd: 0
HugePages_Surp: 0
Hugepagesize:   2       kB
DirectMap4k:    294.938 kB
DirectMap2M:    7578    kB

未改变的 meminfo 和其他请求的信息(请注意,自第一批信息以来已经过去了一些时间,但我试图使其尽可能相同)

sudo ps axo pid,rss,cmd | awk '{sum+=$2} END {print sum / 1024}'
518.551

$ free -m
             total       used       free     shared    buffers     cached
Mem:          7669       4155       3513         68          0       1202
-/+ buffers/cache:       2953       4716
Swap:         5367          0       5367

$ sudo slabtop -o -s c
 Active / Total Objects (% used)    : 344353 / 409584 (84.1%)
 Active / Total Slabs (% used)      : 12149 / 12149 (100.0%)
 Active / Total Caches (% used)     : 80 / 112 (71.4%)
 Active / Total Size (% used)       : 92748.52K / 105317.22K (88.1%)
 Minimum / Average / Maximum Object : 0.01K / 0.26K / 15.75K

  OBJS ACTIVE  USE OBJ SIZE  SLABS OBJ/SLAB CACHE SIZE NAME               
 13371  13371 100%    1.04K    452       30     14464K btrfs_inode        
 56280  49088  87%    0.19K   2680       21     10720K dentry             
 17010  14121  83%    0.55K    627       28     10032K radix_tree_node    
 10175   9575  94%    0.63K    407       25      6512K proc_inode_cache   
 29652  19386  65%    0.19K   1412       21      5648K kmalloc-192        
  9856   9452  95%    0.57K    352       28      5632K inode_cache        
   745    481  64%    5.98K    149        5      4768K task_struct        
  1104   1068  96%    4.00K    138        8      4416K kmalloc-4096       
 11200  11018  98%    0.31K    448       25      3584K btrfs_extent_buffer
  1746   1679  96%    2.00K    110       16      3520K kmalloc-2048       
 30096  30048  99%    0.11K    836       36      3344K sysfs_dir_cache    
 43456  23262  53%    0.06K    679       64      2716K kmalloc-64         
 10368   7228  69%    0.25K    324       32      2592K kmalloc-256        
  2144   1911  89%    1.00K     67       32      2144K kmalloc-1024       
 13650   9943  72%    0.15K    525       26      2100K btrfs_extent_map   
  2013   2013 100%    0.96K     61       33      1952K ext4_inode_cache   
  3040   2472  81%    0.50K     95       32      1520K kmalloc-512        
  2232   1938  86%    0.66K     93       24      1488K shmem_inode_cache  
  1755   1735  98%    0.81K     45       39      1440K task_xstate        
  1584   1584 100%    0.88K     44       36      1408K mm_struct          
  1026    886  86%    1.06K     36       30      1152K signal_cache        
   525    442  84%    2.06K     35       15      1120K sighand_cache      
   133    127  95%    8.00K     34        4      1088K kmalloc-8192        
  7744   6829  88%    0.12K    242       32       968K kmalloc-128         
 14720  14260  96%    0.06K    230       64       920K ext4_free_data      
  8610   6966  80%    0.09K    205       42       820K btrfs_extent_state 
 12736   6989  54%    0.06K    199       64       796K anon_vma           
  1225   1069  87%    0.62K     49       25       784K sock_inode_cache   
   345    345 100%    2.06K     23       15       736K idr_layer_cache    
  2254   1321  58%    0.29K     84       27       672K btrfs_delayed_node 
 18688  14340  76%    0.03K    146      128       584K kmalloc-32         
  3870   3870 100%    0.13K    129       30       516K ext4_allocation_context
  4875   4647  95%    0.10K    125       39       500K buffer_head        
  6272   6272 100%    0.07K    112       56       448K Acpi-ParseExt      
   252    163  64%    1.75K     14       18       448K TCP                
  1155   1155 100%    0.37K     55       21       440K btrfs_ordered_extent
  4116   4116 100%    0.09K     98       42       392K kmalloc-96         
  4140   4140 100%    0.09K     90       46       360K ext4_xattr         
  4539   4539 100%    0.08K     89       51       356K btrfs_prelim_ref   
  1050    907  86%    0.31K     42       25       336K mnt_cache          
   882    647  73%    0.38K     42       21       336K blkdev_requests

$ top -n 1 -o %MEM

top - 19:38:58 up 2 days,  4:48, 21 users,  load average: 0.07, 0.09, 0.13
Tasks: 267 total,   1 running, 266 sleeping,   0 stopped,   0 zombie
%Cpu(s):  1.1 us,  0.8 sy,  0.0 ni, 97.5 id,  0.6 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem:   7853796 total,  4270896 used,  3582900 free,       60 buffers
KiB Swap:  5496148 total,        0 used,  5496148 free.  1238332 cached Mem

  PID USER      PR  NI    VIRT    RES    SHR S  %CPU %MEM     TIME+ COMMAND                                                
 1710 root      20   0  450176  37480  23708 S   6.3  0.5  23:25.64 Xorg                                                   
 3233 loj       20   0  564556  37128   8868 S   6.3  0.5   4:15.48 xfce4-terminal                                         
 1889 root      20   0  182980  33740   1500 S   0.0  0.4   0:04.07 puppet                                                 
 2604 loj       20   0  457136  29468   3096 S   0.0  0.4   0:08.20 xfdesktop                                              
 2398 root      20   0  926976  28652   1244 S   0.0  0.4   1:01.01 docker.io                                              
 2605 loj       20   0  421612  24312   2612 S   0.0  0.3   1:00.18 redshift-gtk                                           
 2634 loj       20   0  568552  21900   5252 S   0.0  0.3   0:03.30 mintUpdate                                             
 2919 loj       20   0  483152  12052   4608 S   0.0  0.2   0:05.71 panel-7-indicat                                        
 2742 loj       20   0  545364  11548   5864 S   0.0  0.1   0:06.72 panel-1-whisker                                        
 2600 loj       20   0  487296  11444   5264 S   0.0  0.1   0:43.00 xfce4-panel                                            
 2645 loj       20   0  248504  10576   1408 S   0.0  0.1   0:00.92 applet.py                                              
 2596 loj       20   0  167352   9924   4176 S   0.0  0.1   2:47.19 xfwm4                                                  
 2628 loj       20   0  683744   9776   3352 S   0.0  0.1   0:15.63 nm-applet                                              
 2616 loj       20   0  382528   6832   3388 S   0.0  0.1   0:11.27 xfce4-power-man                                        
 2872 loj       20   0  382416   5944   1636 S   0.0  0.1   0:02.91 gvfs-udisks2-vo                                        
10166 root      20   0  358828   5840   1884 S   0.0  0.1   0:16.78 NetworkManager                                         
 1099 root      20   0  295828   5556   1272 S   0.0  0.1   0:01.56 polkitd                                                
 2923 loj       20   0  164404   5348   3008 S   0.0  0.1   0:02.78 panel-5-datetim                                        
14257 loj       20   0   25384   5340   1768 S   0.0  0.1   0:00.12 bash                                                   
20503 loj       20   0   25380   5340   1772 S   0.0  0.1   0:00.16 bash                                                   
17690 loj       20   0   25396   5328   1748 S   0.0  0.1   0:00.05 bash                                                   
18732 loj       20   0   25396   5328   1744 S   0.0  0.1   0:00.05 bash                                                   
18538 loj       20   0   25396   5324   1740 S   0.0  0.1   0:00.04 bash                                                   
16501 loj       20   0   25380   5308   1744 S   0.0  0.1   0:00.06 bash                                                   
27045 loj       20   0   26316   5100    592 S   0.0  0.1   0:00.87 bash                                                   
 2913 loj       20   0  149164   5096   3132 S   0.0  0.1   0:11.82 panel-4-genmon                                         
28187 loj       20   0   25396   5028   1440 S   0.0  0.1   0:00.09 bash                                                   
 2917 loj       20   0  149160   4996   3036 S   0.0  0.1   0:29.28 panel-9-genmon                                         
16558 loj       20   0   26160   4948    592 S   0.0  0.1   0:01.02 bash                                                   
 2641 loj        9 -11  371060   4940   2532 S   0.0  0.1  13:13.11 pulseaudio                                             
 2904 loj       20   0   26100   4884    592 S   0.0  0.1   0:00.21 bash                                                   
12720 loj       20   0   26076   4856    592 S   0.0  0.1   0:00.25 bash                                                   
19240 loj       20   0   25572   4844   1156 S   0.0  0.1   0:00.18 bash                                                   
11583 loj       20   0   25380   4708   1140 S   0.0  0.1   0:00.15 bash                                                   
18425 loj       20   0   25508   4692   1044 S   0.0  0.1   0:00.08 bash                                                   
 4865 loj       20   0   25368   4556    996 S   0.0  0.1   0:00.30 bash                                                   
11552 loj       20   0   25384   4516    944 S   0.0  0.1   0:00.05 bash                                                   
 6966 loj       20   0   51180   4332   1620 S   0.0  0.1   0:00.15 vim                                                    
 2888 root      20   0  451912   4288   1624 S   0.0  0.1   0:28.69 udisksd                               
11310 loj       20   0   25452   4216    592 S   0.0  0.1   0:01.20 bash
 2679 loj       20   0  343428   4196   1752 S   0.0  0.1   0:02.56 xfsettingsd
31494 loj       20   0   25380   4164    596 S   0.0  0.1   0:00.16 bash



$ cat /proc/meminfo
MemTotal:        7853796 kB
MemFree:         3593092 kB
Buffers:              60 kB
Cached:          1237532 kB
SwapCached:            0 kB
Active:           667844 kB
Inactive:         934244 kB
Active(anon):      28480 kB
Inactive(anon):   412756 kB
Active(file):     639364 kB
Inactive(file):   521488 kB
Unevictable:           0 kB
Mlocked:               0 kB
SwapTotal:       5496148 kB
SwapFree:        5496148 kB
Dirty:                 0 kB
Writeback:             0 kB
AnonPages:        364556 kB
Mapped:            48308 kB
Shmem:             76740 kB
Slab:             107128 kB
SReclaimable:      61012 kB
SUnreclaim:        46116 kB
KernelStack:        3344 kB
PageTables:        18088 kB
NFS_Unstable:          0 kB
Bounce:                0 kB
WritebackTmp:          0 kB
CommitLimit:     9423044 kB
Committed_AS:    1382404 kB
VmallocTotal:   34359738367 kB
VmallocUsed:      360768 kB
VmallocChunk:   34359364252 kB
HardwareCorrupted:     0 kB
AnonHugePages:     61440 kB
HugePages_Total:       0
HugePages_Free:        0
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:       2048 kB
DirectMap4k:      304064 kB
DirectMap2M:     7757824 kB

$ sudo cat /proc/vmstat 
nr_free_pages 807707
nr_alloc_batch 3485
nr_inactive_anon 103817
nr_active_anon 14695
nr_inactive_file 157126
nr_active_file 183051
nr_unevictable 0
nr_mlock 0
nr_anon_pages 96859
nr_mapped 19398
nr_file_pages 361889
nr_dirty 0
nr_writeback 0
nr_slab_reclaimable 16123
nr_slab_unreclaimable 11793
nr_page_table_pages 4772
nr_kernel_stack 429
nr_unstable 0
nr_bounce 0
nr_vmscan_write 100293
nr_vmscan_immediate_reclaim 3432
nr_writeback_temp 0
nr_isolated_anon 0
nr_isolated_file 0
nr_shmem 21712
nr_dirtied 55931244
nr_written 55847537
numa_hit 335064087
numa_miss 0
numa_foreign 0
numa_interleave 24522
numa_local 335064087
numa_other 0
nr_anon_transparent_hugepages 28
nr_free_cma 0
nr_dirty_threshold 111416
nr_dirty_background_threshold 55708
pgpgin 129334306
pgpgout 233190636
pswpin 80715
pswpout 100268
pgalloc_dma 0
pgalloc_dma32 159890087
pgalloc_normal 257522463
pgalloc_movable 0
pgfree 424750256
pgactivate 23419593
pgdeactivate 11177646
pgfault 508757717
pgmajfault 19562
pgrefill_dma 0
pgrefill_dma32 3369566
pgrefill_normal 7975279
pgrefill_movable 0
pgsteal_kswapd_dma 0
pgsteal_kswapd_dma32 20040131
pgsteal_kswapd_normal 34430518
pgsteal_kswapd_movable 0
pgsteal_direct_dma 0
pgsteal_direct_dma32 1161023
pgsteal_direct_normal 1344398
pgsteal_direct_movable 0
pgscan_kswapd_dma 0
pgscan_kswapd_dma32 20077734
pgscan_kswapd_normal 34646412
pgscan_kswapd_movable 0
pgscan_direct_dma 0
pgscan_direct_dma32 1164470
pgscan_direct_normal 1414283
pgscan_direct_movable 0
pgscan_direct_throttle 0
zone_reclaim_failed 0
pginodesteal 63
slabs_scanned 2262272
kswapd_inodesteal 168
kswapd_low_wmark_hit_quickly 107411
kswapd_high_wmark_hit_quickly 94396
pageoutrun 231397
allocstall 37862
pgrotated 113947
numa_pte_updates 0
numa_huge_pte_updates 0
numa_hint_faults 0
numa_hint_faults_local 0
numa_pages_migrated 0
pgmigrate_success 2038063
pgmigrate_fail 3670
compact_migrate_scanned 18509229
compact_free_scanned 464282669
compact_isolated 9111435
compact_stall 3146
compact_fail 2215
compact_success 931
htlb_buddy_alloc_success 0
htlb_buddy_alloc_fail 0
unevictable_pgs_culled 0
unevictable_pgs_scanned 0
unevictable_pgs_rescued 6
unevictable_pgs_mlocked 6
unevictable_pgs_munlocked 6
unevictable_pgs_cleared 0
unevictable_pgs_stranded 0
thp_fault_alloc 5220
thp_fault_fallback 7309
thp_collapse_alloc 2740
thp_collapse_alloc_failed 1229
thp_split 2320
thp_zero_page_alloc 3
thp_zero_page_alloc_failed 0
nr_tlb_remote_flush 1119474
nr_tlb_remote_flush_received 1416528
nr_tlb_local_flush_all 3215771
nr_tlb_local_flush_one 161032407

$ sudo cat /proc/loadavg 
0.52 0.35 0.27 2/1124 22236

答案1

这看起来非常像内核内存泄漏。我大致预计 File+Anon+Free+Slab 会增加您的总物理 RAM,但还差 2-3 GB。如果您一直在做大量磁盘密集型工作,那么可能是 BTRFS(例如这里)。您需要一个启用了 kmemleak 的内核来进行检查。

相关内容