KVM : hot add PCI devices ( storage )

[ software info ]

KVM host : ubuntu 11.04

root@hat1:~# kvm --version QEMU emulator version 0.14.0 (qemu-kvm-0.14.0), Copyright (c) 2003-2008 Fabrice Bellard root@hat1:~# tail -1 /etc/lsb-release DISTRIB_DESCRIPTION="Ubuntu 11.04" root@hat1:~# uname -r 2.6.38-11-server

VM : CentOS 5.5 32bit

# uname -r 2.6.18-194.el5 # cat /etc/redhat-release CentOS release 5.5 (Final)

[ procedures including trial and error ]

prepare a disk image to add it to CentOS VM.

create image disk on KVM host

# kvm-img create -f qcow2 /media/disk1/images/zzz.img 10G Formatting '/media/disk1/images/zzz.img', fmt=qcow2 size=10737418240 encryption=off cluster_size=0 # chown libvirt-qemu.kvm /media/disk1/images/zzz.img # file /media/disk1/images/zzz.img /media/disk1/images/zzz.img: QEMU QCOW Image (v2), 10737418240 bytes

add zzz.img to runnning VM.

before adding zzz.img

virsh # qemu-monitor-command centos5-vm1 'info block' drive-virtio-disk0: type=hd removable=0 file=/media/disk1/images/centos5-vm1.img ro=0 drv=raw encrypted=0 drive-ide0-1-0: type=cdrom removable=1 locked=0 [not inserted]

add zzz.img to a VM

virsh # qemu-monitor-command centos5-vm1 'pci_add auto storage file=/media/disk1/images/zzz.img,if=scsi' could not open disk image /media/disk1/images/zzz.img: Permission denied failed to add file=/media/disk1/images/zzz.img,if=scsi

permission denied... nnn
Seen from syslog , this seems to be caused by apparmor.

hat1 kernel: [ 9411.832993] type=1400 audit(1314440847.600:62): apparmor="DENIED" operation="open" parent=1 profile="libvirt-597771da-79f1-6d57-1e45-5906c22417aa" name="/media/disk1/images/zzz.img" pid=12312 comm="kvm" requested_mask="r" denied_mask="r" fsuid=102 ouid=102 hat1 kernel: [ 9411.833029] type=1400 audit(1314440847.600:63): apparmor="DENIED" operation="open" parent=1 profile="libvirt-597771da-79f1-6d57-1e45-5906c22417aa" name="/media/disk1/images/zzz.img" pid=12312 comm="kvm" requested_mask="r" denied_mask="r" fsuid=102 ouid=102 hat1 kernel: [ 9411.833077] type=1400 audit(1314440847.600:64): apparmor="DENIED" operation="open" parent=1 profile="libvirt-597771da-79f1-6d57-1e45-5906c22417aa" name="/media/disk1/images/zzz.img" pid=12312 comm="kvm" requested_mask="r" denied_mask="r" fsuid=102 ouid=102

on KVM host

root@hat1:~# apparmor_status apparmor module is loaded. 13 profiles are loaded. 13 profiles are in enforce mode.   /sbin/dhclient   /usr/bin/evince   /usr/bin/evince-previewer   /usr/bin/evince-thumbnailer   /usr/lib/NetworkManager/nm-dhcp-client.action   /usr/lib/connman/scripts/dhclient-script   /usr/lib/cups/backend/cups-pdf   /usr/lib/libvirt/virt-aa-helper   /usr/sbin/cupsd   /usr/sbin/libvirtd   /usr/sbin/tcpdump   /usr/share/gdm/guest-session/Xsession   libvirt-597771da-79f1-6d57-1e45-5906c22417aa 0 profiles are in complain mode. 3 processes have profiles defined. 3 processes are in enforce mode :   /usr/sbin/cupsd (1307)   /usr/sbin/libvirtd (1288)   libvirt-597771da-79f1-6d57-1e45-5906c22417aa (12312) 0 processes are in complain mode. 0 processes are unconfined but have a profile defined.

I’m unfamiliar with apparmor...
Let’s see apparomor logs under /etc/apparmor.d/

root@hat1:.../libvirt# cat libvirt-597771da-79f1-6d57-1e45-5906c22417aa # # This profile is for the domain whose UUID matches this file. # #include <tunables/global> profile libvirt-597771da-79f1-6d57-1e45-5906c22417aa {  #include <abstractions/libvirt-qemu>  #include <libvirt/libvirt-597771da-79f1-6d57-1e45-5906c22417aa.files> } root@hat1:.../libvirt# root@hat1:.../libvirt# cat libvirt-597771da-79f1-6d57-1e45-5906c22417aa.files # DO NOT EDIT THIS FILE DIRECTLY. IT IS MANAGED BY LIBVIRT.  "/var/log/libvirt/**/centos5-vm1.log" w,  "/var/lib/libvirt/**/centos5-vm1.monitor" rw,  "/var/run/libvirt/**/centos5-vm1.pid" rwk,  "/media/disk1/images/centos5-vm1.img" rw,

seen from http://libvirt.org/drvqemu.html , I need to modify :

/etc/apparmor.d/usr.sbin.libvirtd
or
/etc/apparmor.d/usr.sbin.libvirtd/usr.lib.libvirt.virt-aa-helper
or
/etc/apparmor.d/abstractions/libvirt-qemu....

nnnn...Okay , esay way is turning off apparmor :)

shutdown a VM

virsh # shutdown centos5-vm1 ドメイン centos5-vm1 をシャットダウンしています

add security_driver=”none” option in /etc/libvirt/qemu.conf

# egrep security_driver /etc/libvirt/qemu.conf # security_driver = "selinux" security_driver="none"

start a VM

# virsh start centos5-vm1 ドメイン centos5-vm1 が起動されました

try it agin.
nnn , same error..

virsh # qemu-monitor-command centos5-vm1 'pci_add auto storage file=/media/disk1/images/zzz.img,if=scsi' could not open disk image /media/disk1/images/zzz.img: Permission denied failed to add file=/media/disk1/images/zzz.img,if=scsi

oh,, I forget to restart libvirt.

shutdown VM and restart libvirtd.

virsh # shutdown centos5-vm1

restart libvirtd.

# /etc/init.d/libvirt-bin restart

start a VM

virsh # start centos5-vm1

before adding zzz.img

virsh # qemu-monitor-command centos5-vm1 'info block' drive-virtio-disk0: type=hd removable=0 file=/media/disk1/images/centos5-vm1.img ro=0 drv=raw encrypted=0 drive-ide0-1-0: type=cdrom removable=1 locked=0 [not inserted]

add zzz.img

virsh # qemu-monitor-command centos5-vm1 'pci_add auto storage file=/media/disk1/images/zzz.img,if=scsi' OK domain 0, bus 0, slot 4, function 0

after adding zzz.img

virsh # qemu-monitor-command centos5-vm1 'info block' drive-virtio-disk0: type=hd removable=0 file=/media/disk1/images/centos5-vm1.img ro=0 drv=raw encrypted=0 drive-ide0-1-0: type=cdrom removable=1 locked=0 [not inserted] scsi0-hd0: type=hd removable=0 file=/media/disk1/images/zzz.img ro=0 drv=qcow2 encrypted=0

on VM.
nnn , no new disk has appeared.

# fdisk -l Disk /dev/vda: 8589 MB, 8589934592 bytes 255 heads, 63 sectors/track, 1044 cylinders Units = シリンダ数 of 16065 * 512 = 8225280 bytes デバイス Boot      Start         End      Blocks   Id  System /dev/vda1   *           1          13      104391   83  Linux /dev/vda2              14        1044     8281507+  8e  Linux LVM

It seems that VM needs to load acpiphp module.

# modprobe acpiphp

reboot VM and load acpiphp

# modprobe acpiphp

virsh # qemu-monitor-command centos5-vm1 'pci_add auto storage file=/media/disk1/images/zzz.img,if=scsi' OK domain 0, bus 0, slot 4, function 0

dmesg on VM

ACPI Exception (pci_bind-0302): AE_NOT_FOUND, Unable to get data from device S4 [20060707] decode_hpp: Could not get hotplug parameters. Use defaults PCI: Enabling device 0000:00:04.0 (0000 -> 0003) ACPI: PCI Interrupt 0000:00:04.0[A] -> Link [LNKD] -> GSI 11 (level, high) -> IRQ 11 sym0: <895a> rev 0x0 at pci 0000:00:04.0 irq 11 sym0: No NVRAM, ID 7, Fast-40, LVD, parity checking sym0: SCSI BUS has been reset. scsi0 : sym-2.2.3 sym0: unknown interrupt(s) ignored, ISTAT=0x1 DSTAT=0x80 SIST=0x0  Vendor: QEMU      Model: QEMU HARDDISK     Rev: 0.14  Type:   Direct-Access                      ANSI SCSI revision: 05 target0:0:0: tagged command queuing enabled, command queue depth 16. target0:0:0: Beginning Domain Validation target0:0:0: Domain Validation skipping write tests target0:0:0: Ending Domain Validation SCSI device sda: 20971520 512-byte hdwr sectors (10737 MB) sda: Write Protect is off sda: Mode Sense: 1f 00 00 08 SCSI device sda: drive cache: write through SCSI device sda: 20971520 512-byte hdwr sectors (10737 MB) sda: Write Protect is off sda: Mode Sense: 1f 00 00 08 SCSI device sda: drive cache: write through sda: unknown partition table sd 0:0:0:0: Attached scsi disk sda sd 0:0:0:0: Attached scsi generic sg0 type 0

on VM
/dev/sda has been added :)

# LANG=C fdisk -l Disk /dev/vda: 8589 MB, 8589934592 bytes 255 heads, 63 sectors/track, 1044 cylinders Units = cylinders of 16065 * 512 = 8225280 bytes   Device Boot      Start         End      Blocks   Id  System /dev/vda1   *           1          13      104391   83  Linux /dev/vda2              14        1044     8281507+  8e  Linux LVM Disk /dev/sda: 10.7 GB, 10737418240 bytes 64 heads, 32 sectors/track, 10240 cylinders Units = cylinders of 2048 * 512 = 1048576 bytes Disk /dev/sda doesn't contain a valid partition table

remove /dev/sda 

pci_del ‘slot number’
You can check slot number with virsh qemu-monitor-command VM ‘info pci’.

virsh # qemu-monitor-command centos5-vm1 'pci_del 4'

/dev/sda has been removed

# LANG=C fdisk -l Disk /dev/vda: 8589 MB, 8589934592 bytes 255 heads, 63 sectors/track, 1044 cylinders Units = cylinders of 16065 * 512 = 8225280 bytes   Device Boot      Start         End      Blocks   Id  System /dev/vda1   *           1          13      104391   83  Linux /dev/vda2              14        1044     8281507+  8e  Linux LVM

dmesg on VM

sym0: detaching ... sym0: resetting chip ACPI: PCI interrupt for device 0000:00:04.0 disabled

virsh # qemu-monitor-command centos5-vm1 'info block' drive-virtio-disk0: type=hd removable=0 file=/media/disk1/images/centos5-vm1.img ro=0 drv=raw encrypted=0 drive-ide0-1-0: type=cdrom removable=1 locked=0 [not inserted]

# apparmor_status apparmor module is loaded. 12 profiles are loaded. 12 profiles are in enforce mode.   /sbin/dhclient   /usr/bin/evince   /usr/bin/evince-previewer   /usr/bin/evince-thumbnailer   /usr/lib/NetworkManager/nm-dhcp-client.action   /usr/lib/connman/scripts/dhclient-script   /usr/lib/cups/backend/cups-pdf   /usr/lib/libvirt/virt-aa-helper   /usr/sbin/cupsd   /usr/sbin/libvirtd   /usr/sbin/tcpdump   /usr/share/gdm/guest-session/Xsession 0 profiles are in complain mode. 2 processes have profiles defined. 2 processes are in enforce mode :   /usr/sbin/cupsd (1307)   /usr/sbin/libvirtd (19414) 0 processes are in complain mode. 0 processes are unconfined but have a profile defined.