Ubuntu 主机上的虚拟化 Centos 7:I/O 非常慢

Ubuntu 主机上的虚拟化 Centos 7:I/O 非常慢

我在 Ubuntu 20.04 中运行虚拟化的 Centos 7。与主机相比,虚拟机的 I/O 性能下降幅度很大。主机有足够的 RAM。

主持人:

* MB/s = 1,000,000 bytes/s
* KB = 1000 bytes, KiB = 1024 bytes

   Sequential Read (Q= 32,T= 1) :   4080MB/s [   124 IOPS]
  Sequential Write (Q= 32,T= 1) :   2799MB/s [   85 IOPS]
  Random Read 4KiB (Q=  8,T= 8) :   2932MB/s [   733039 IOPS]
 Random Write 4KiB (Q=  8,T= 8) :   1631MB/s [   407961 IOPS]
  Random Read 4KiB (Q= 32,T= 1) :   600MB/s [   150036 IOPS]
 Random Write 4KiB (Q= 32,T= 1) :   344MB/s [   86050 IOPS]
  Random Read 4KiB (Q=  1,T= 1) :   587MB/s [   146810 IOPS]
 Random Write 4KiB (Q=  1,T= 1) :   339MB/s [   84979 IOPS]

虚拟机:

* MB/s = 1,000,000 bytes/s
* KB = 1000 bytes, KiB = 1024 bytes

   Sequential Read (Q= 32,T= 1) :   633MB/s [   19 IOPS]
  Sequential Write (Q= 32,T= 1) :   4660MB/s [   142 IOPS]
  Random Read 4KiB (Q=  8,T= 8) :   447MB/s [   111872 IOPS]
 Random Write 4KiB (Q=  8,T= 8) :   358MB/s [   89671 IOPS]
  Random Read 4KiB (Q= 32,T= 1) :   282MB/s [   70620 IOPS]
 Random Write 4KiB (Q= 32,T= 1) :   287MB/s [   71859 IOPS]
  Random Read 4KiB (Q=  1,T= 1) :   27MB/s [   6805 IOPS]
 Random Write 4KiB (Q=  1,T= 1) :   33MB/s [   8415 IOPS]

VM 磁盘是 ZVOL(我也使用 ZFS 数据集中的 .qcow2 进行了测试,结果同样糟糕):

# zfs get all ssd1/vm/isp-master
NAME                PROPERTY              VALUE                  SOURCE
ssd1/vm/isp-master  type                  volume                 -
ssd1/vm/isp-master  creation              Tue Feb 22 21:44 2022  -
ssd1/vm/isp-master  used                  703G                   -
ssd1/vm/isp-master  available             164G                   -
ssd1/vm/isp-master  referenced            393G                   -
ssd1/vm/isp-master  compressratio         1.09x                  -
ssd1/vm/isp-master  reservation           none                   default
ssd1/vm/isp-master  volsize               900G                   local
ssd1/vm/isp-master  volblocksize          8K                     default
ssd1/vm/isp-master  checksum              on                     default
ssd1/vm/isp-master  compression           lz4                    inherited from ssd1
ssd1/vm/isp-master  readonly              off                    default
ssd1/vm/isp-master  createtxg             9025875                -
ssd1/vm/isp-master  copies                1                      default
ssd1/vm/isp-master  refreservation        none                   default
ssd1/vm/isp-master  guid                  14573562710197222671   -
ssd1/vm/isp-master  primarycache          metadata               local
ssd1/vm/isp-master  secondarycache        all                    default
ssd1/vm/isp-master  usedbysnapshots       310G                   -
ssd1/vm/isp-master  usedbydataset         393G                   -
ssd1/vm/isp-master  usedbychildren        0B                     -
ssd1/vm/isp-master  usedbyrefreservation  0B                     -
ssd1/vm/isp-master  logbias               latency                default
ssd1/vm/isp-master  objsetid              660                    -
ssd1/vm/isp-master  dedup                 off                    default
ssd1/vm/isp-master  mlslabel              none                   default
ssd1/vm/isp-master  sync                  standard               default
ssd1/vm/isp-master  refcompressratio      1.11x                  -
ssd1/vm/isp-master  written               76.6M                  -
ssd1/vm/isp-master  logicalused           763G                   -
ssd1/vm/isp-master  logicalreferenced     436G                   -
ssd1/vm/isp-master  volmode               default                default
ssd1/vm/isp-master  snapshot_limit        none                   default
ssd1/vm/isp-master  snapshot_count        none                   default
ssd1/vm/isp-master  snapdev               hidden                 default
ssd1/vm/isp-master  context               none                   default
ssd1/vm/isp-master  fscontext             none                   default
ssd1/vm/isp-master  defcontext            none                   default
ssd1/vm/isp-master  rootcontext           none                   default
ssd1/vm/isp-master  redundant_metadata    all                    default
ssd1/vm/isp-master  encryption            off                    default
ssd1/vm/isp-master  keylocation           none                   default
ssd1/vm/isp-master  keyformat             none                   default
ssd1/vm/isp-master  pbkdf2iters           0                      default

我使用 libvirt 进行虚拟化,Qemu 客户代理安装在运行最新内核(5.4.231-1.el7.elrepo.x86_64)的 Centos7 中:

<domain type='kvm'>
  <name>isp-master</name>
  <uuid>[redacted]</uuid>
  <metadata>
    <libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
      <libosinfo:os id="http://centos.org/centos/7.0"/>
    </libosinfo:libosinfo>
  </metadata>
  <memory unit='KiB'>33554432</memory>
  <currentMemory unit='KiB'>33554432</currentMemory>
  <vcpu placement='static'>12</vcpu>
  <os>
    <type arch='x86_64' machine='pc-i440fx-bionic'>hvm</type>
  </os>
  <features>
    <acpi/>
    <apic/>
    <vmport state='off'/>
  </features>
  <cpu mode='host-passthrough' check='none'/>
  <clock offset='utc'>
    <timer name='rtc' tickpolicy='catchup'/>
    <timer name='pit' tickpolicy='delay'/>
    <timer name='hpet' present='no'/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <pm>
    <suspend-to-mem enabled='no'/>
    <suspend-to-disk enabled='no'/>
  </pm>
  <devices>
    <emulator>/usr/bin/kvm-spice</emulator>
    <disk type='block' device='disk'>
      <driver name='qemu' type='raw' cache='none' io='native'/>
      <source dev='/dev/zvol/ssd1/vm/isp-master'/>
      <target dev='vda' bus='virtio'/>
      <boot order='1'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
    </disk>
    <disk type='block' device='disk'>
      <driver name='qemu' type='raw' cache='none' io='native'/>
      <source dev='/dev/zvol/pool1/vm/isp-master-backups'/>
      <target dev='vdb' bus='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x09' function='0x0'/>
    </disk>
    <disk type='file' device='disk'>
      <driver name='qemu' type='qcow2'/>
      <source file='/var/lib/libvirt/images-ssd1/vm-isp-master.qcow2'/>
      <target dev='vdc' bus='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x0a' function='0x0'/>
    </disk>
    <disk type='file' device='cdrom'>
      <driver name='qemu' type='raw'/>
      <target dev='hda' bus='ide'/>
      <readonly/>
      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
    </disk>
    <controller type='usb' index='0' model='ich9-ehci1'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x7'/>
    </controller>
    <controller type='usb' index='0' model='ich9-uhci1'>
      <master startport='0'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0' multifunction='on'/>
    </controller>
    <controller type='usb' index='0' model='ich9-uhci2'>
      <master startport='2'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x1'/>
    </controller>
    <controller type='usb' index='0' model='ich9-uhci3'>
      <master startport='4'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x2'/>
    </controller>
    <controller type='pci' index='0' model='pci-root'/>
    <controller type='ide' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
    </controller>
    <controller type='virtio-serial' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
    </controller>
    <controller type='scsi' index='0' model='virtio-scsi'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
    </controller>
    <interface type='bridge'>
      <mac address='[redacted]'/>
      <source bridge='br0'/>
      <model type='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>
    <serial type='pty'>
      <target type='isa-serial' port='0'>
        <model name='isa-serial'/>
      </target>
    </serial>
    <console type='pty'>
      <target type='serial' port='0'/>
    </console>
    <channel type='unix'>
      <target type='virtio' name='org.qemu.guest_agent.0'/>
      <address type='virtio-serial' controller='0' bus='0' port='1'/>
    </channel>
    <channel type='spicevmc'>
      <target type='virtio' name='com.redhat.spice.0'/>
      <address type='virtio-serial' controller='0' bus='0' port='2'/>
    </channel>
    <input type='tablet' bus='usb'>
      <address type='usb' bus='0' port='1'/>
    </input>
    <input type='mouse' bus='ps2'/>
    <input type='keyboard' bus='ps2'/>
    <graphics type='spice' autoport='yes'>
      <listen type='address'/>
    </graphics>
    <video>
      <model type='qxl' ram='65536' vram='65536' vgamem='16384' heads='1' primary='yes'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </video>
    <redirdev bus='usb' type='spicevmc'>
      <address type='usb' bus='0' port='2'/>
    </redirdev>
    <redirdev bus='usb' type='spicevmc'>
      <address type='usb' bus='0' port='3'/>
    </redirdev>
    <memballoon model='virtio'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
    </memballoon>
  </devices>
</domain>

我不知道为什么 I/O 性能这么差。有什么想法吗?

相关内容