run multiple VMs with vagrant

$ vagrant --version Vagrant 1.7.2 $ tail -1 /etc/lsb-release DISTRIB_DESCRIPTION="Ubuntu 14.04.2 LTS"

initialize a project.

$ vagrant box list trusty64 (libvirt, 0) trusty64 (virtualbox, 0) $ vagrant init trusty64

edit Vagrantfile

$ cat Vagrantfile Vagrant.configure(2) do |config|  config.vm.define "server01" do |server01|    server01.vm.box = "trusty64"  end  config.vm.define "server02" do |server02|    server02.vm.box = "trusty64"  end end

run VMs

I have used libvirt as its provider.

$ vagrant up server01 --provider=libvirt $ vagrant up server02 --provider=libvirt $ virsh list --all Id    Name                           State ---------------------------------------------------- 2     Vagrant_works_server01             running 3     Vagrant_works_server02             running $ vagrant status Current machine states: server01                  running (libvirt) server02                  running (libvirt)

access to the VMs

$ vagrant ssh server01 $ vagrant ssh server02

Ubuntu 13.10: manage LXC containers with libvirt

Here is how to manage LXC with libvirt.

reference

http://blog.scottlowe.org/2013/11/27/linux-containers-via-lxc-and-libvirt/

install LXC and create a container.

# tail -1 /etc/lsb-release;uname -ri DISTRIB_DESCRIPTION="Ubuntu 13.10" 3.11.0-17-generic x86_64 # apt-get install lxc lxc-templates # lxc-create -t ubuntu -n ubuntu1310-cn1 -- -r saucy ## # The default user is 'ubuntu' with password 'ubuntu'! # Use the 'sudo' command to run tasks as root in the container. ##

The container “ubuntu1310-cn1” has been installed under /var/lib/lxc directory.

# ls /var/lib/lxc/ubuntu1310-cn1/rootfs/ bin   dev  home  lib64  mnt  proc  run   srv  tmp  var boot  etc  lib   media  opt  root  sbin  sys  usr

download a template file.

# wget https://gist.githubusercontent.com/lowescott/7647800/raw/76ca9a923773b577f6d8ed0cd5171f675b61eef6/lxc-template

edit the file to suite my environment.

# sed 's/REPLACE/ubuntu1310-cn1/g' lxc-template > ubuntu1310-cn1.xml # cat ubuntu1310-cn1.xml <domain type='lxc'>  <name>ubuntu1310-cn1</name>  <memory>327680</memory>  <os>    <type>exe</type>    <init>/sbin/init</init>  </os>  <vcpu>1</vcpu>  <clock offset='utc'/>  <on_poweroff>destroy</on_poweroff>  <on_reboot>restart</on_reboot>  <on_crash>destroy</on_crash>  <devices>    <emulator>/usr/lib/libvirt/libvirt_lxc</emulator>    <filesystem type='mount'>      <source dir='/var/lib/lxc/ubuntu1310-cn1/rootfs'/>      <target dir='/'/>    </filesystem>    <interface type='network'>      <source network='default'/>    </interface>    <console type='pty'/>  </devices>

I am using openvswitch, so edit the file  to use openvswitch.

Here is how to install openvswitch.

//lost-and-found-narihiro.blogspot.jp/2014/02/ubuntu-1310-install-use-openvswitch.html

# cat ubuntu1310-cn1.xml <domain type='lxc'>  <name>ubuntu1310-cn1</name>  <memory>327680</memory>  <os>    <type>exe</type>    <init>/sbin/init</init>  </os>  <vcpu>1</vcpu>  <clock offset='utc'/>  <on_poweroff>destroy</on_poweroff>  <on_reboot>restart</on_reboot>  <on_crash>destroy</on_crash>  <devices>    <emulator>/usr/lib/libvirt/libvirt_lxc</emulator>    <filesystem type='mount'>      <source dir='/var/lib/lxc/ubuntu1310-cn1/rootfs'/>      <target dir='/'/>    </filesystem>    <interface type='bridge'>      <source bridge='br0'/>      <virtualport type='openvswitch'/>    </interface>    <console type='pty'/>  </devices> </domain>

deploy the container with libvirt.

# virsh -c lxc:// define ubuntu1310-cn1.xml ドメイン ubuntu1310-cn1 が ubuntu1310-cn1.xml から定義されました # virsh -c lxc:// list --all Id    名前                         状態 ---------------------------------------------------- -     ubuntu1310-cn1                 シャットオフ

start

# virsh -c lxc:// start ubuntu1310-cn1 ドメイン ubuntu1310-cn1 が起動されました

access to the container via virsh console

# virsh -c lxc:// console ubuntu1310-cn1 Ubuntu 13.10 ubuntu1310-cn1 tty1 ubuntu1310-cn1 login: # ping www.google.com -c 1 PING www.google.com (74.125.235.178) 56(84) bytes of data. 64 bytes from nrt19s12-in-f18.1e100.net (74.125.235.178): icmp_seq=1 ttl=53 time=10.4 ms --- www.google.com ping statistics --- 1 packets transmitted, 1 received, 0% packet loss, time 0ms rtt min/avg/max/mdev = 10.427/10.427/10.427/0.000 ms

Fedora 19: install open vswitch

just referred to http://kashyapc.wordpress.com/2013/07/13/configuring-libvirt-guests-with-an-open-vswitch-bridge/

# cat /etc/fedora-release ;uname -ri Fedora release 19 (Schrödinger’s Cat) 3.11.1-200.fc19.x86_64 x86_64

Fedora 19 box has one NIC called p10p1.

p10p1 gets IP address over DHCP.

# cat /etc/sysconfig/network-scripts/ifcfg-p10p1 DEVICE=p10p1 ONBOOT=yes #BRIDGE=br0 BOOTPROTO=dhcp #

install open vswitch via yum

# yum install openvswitch

start open vswitch

[root@localhost ~]# systemctl enable openvswitch.service ln -s '/usr/lib/systemd/system/openvswitch.service' '/etc/systemd/system/multi-user.target.wants/openvswitch.service' [root@localhost ~]# systemctl start openvswitch.service [root@localhost ~]# systemctl status openvswitch.service openvswitch.service - Open vSwitch Unit   Loaded: loaded (/usr/lib/systemd/system/openvswitch.service; enabled)   Active: active (exited) since 月 2013-09-30 00:09:20 JST; 5s ago  Process: 4117 ExecStart=/bin/true (code=exited, status=0/SUCCESS)

configure open vswitch bridge

# ovs-vsctl add-br ovsbr0

associate ovsbr0 to phsical interface p10p1

Please note that you will lost internet connectivity after issuing following command.

# ovs-vsctl add-port ovsbr0 p10p1

after issuing following commands, you will get internet connectivity again.

[root@localhost ~]# ifconfig p10p1 0.0.0.0 [root@localhost ~]# ifconfig ovsbr0 192.168.11.100 [root@localhost ~]# dhclient ovsbr0 & [1] 4206 # ip addr show dev ovsbr0 8: ovsbr0: <BROADCAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN    link/ether 78:e7:d1:7f:84:b7 brd ff:ff:ff:ff:ff:ff    inet 192.168.11.100/24 brd 192.168.11.255 scope global ovsbr0       valid_lft forever preferred_lft forever    inet 192.168.11.5/24 brd 192.168.11.255 scope global secondary ovsbr0       valid_lft forever preferred_lft forever    inet6 fe80::e8ec:5aff:fe59:8655/64 scope link       valid_lft forever preferred_lft forever [root@localhost ~]# [1]+  終了                  dhclient ovsbr0 [root@localhost ~]# ovs-vsctl show 6e8e87c4-700b-4556-b977-cc3fa103d893    Bridge "ovsbr0"        Port "p10p1"            Interface "p10p1"        Port "ovsbr0"            Interface "ovsbr0"                type: internal

edit VM’s xml file to use open vswitch.

# virsh dumpxml f19-vm1    <interface type='bridge'>      <mac address='52:54:00:d0:86:2e'/>      <source bridge='ovsbr0'/>      <virtualport type='openvswitch'>

start the VM

# virsh start f19-vm1

on the VM, confirm that the VM gets an IP address over DHCP.

[root@localhost ~]# ip addr show dev eth0 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000    link/ether 52:54:00:d0:86:2e brd ff:ff:ff:ff:ff:ff    inet 192.168.11.11/24 brd 192.168.11.255 scope global eth0       valid_lft forever preferred_lft forever    inet6 fe80::5054:ff:fed0:862e/64 scope link       valid_lft forever preferred_lft forever [root@localhost ~]# [root@localhost ~]# ping www.google.com -c 1 PING www.google.com (173.194.38.114) 56(84) bytes of data. 64 bytes from nrt19s18-in-f18.1e100.net (173.194.38.114): icmp_seq=1 ttl=53 time=11.2 ms --- www.google.com ping statistics --- 1 packets transmitted, 1 received, 0% packet loss, time 0ms rtt min/avg/max/mdev = 11.256/11.256/11.256/0.000 ms

how to install / use libvirt-snmp on CentOS6

libvirt-snmp allows you to get KVM-related info via snmpget , snmptrap.
Add to that , you can shutdown , pause , start VMs via snmpset.
Here’s an explanation of how to install / use libvirt-snmp.

I just referred to http://wiki.libvirt.org/page/Libvirt-snmp#libvirt-snmp.
many thanks!

# uname –ri 2.6.32-279.19.1.el6.x86_64 x86_64 # cat /etc/centos-release CentOS release 6.3 (Final) # /usr/libexec/qemu-kvm --version QEMU PC emulator version 0.12.1 (qemu-kvm-0.12.1.2), Copyright (c) 2003-2008 Fabrice Bellard # libvirtd --version libvirtd (libvirt) 0.9.10

[ installation logs of libvirt-snmp including trial and error ]

download libvirt-snmp via git

# git clone git://libvirt.org/libvirt-snmp.git

# ls libvirt-snmp/ AUTHORS  INSTALL.1st  NEWS    autobuild.sh  configure.ac  libvirt-snmp.spec.in COPYING  Makefile.am  README  autogen.sh    docs          src

install some packages which are required to build libvirt-snmp.

# yum install net-snmp-perl net-snmp net-snmp-utils net-snmp-devel libvirt-devel rpm-build -y

build libvirt-snmp

# cd libvirt-snmp/

# ./autobuild.sh test -n "$1" && RESULTS=$1 || RESULTS=results.log : ${AUTOBUILD_INSTALL_ROOT=$HOME/builder} # Make things clean. test -f Makefile && make -k distclean || : rm -rf build mkdir build cd build MIBDIR=$AUTOBUILD_INSTALL_ROOT/share/snmp/mibs mkdir -p $MIBDIR ../autogen.sh --prefix=$AUTOBUILD_INSTALL_ROOT \  --with-mibdir=$MIBDIR ../autogen.sh: line 22: autoreconf: command not found

need autoreconf command to build libvirt-snmp. install autoreconf via yum.

# yum install -y autoconf

run autobuild.sh again.
nn , error

# ./autobuild.sh test -n "$1" && RESULTS=$1 || RESULTS=results.log : ${AUTOBUILD_INSTALL_ROOT=$HOME/builder} # Make things clean. test -f Makefile && make -k distclean || : rm -rf build mkdir build cd build MIBDIR=$AUTOBUILD_INSTALL_ROOT/share/snmp/mibs mkdir -p $MIBDIR ../autogen.sh --prefix=$AUTOBUILD_INSTALL_ROOT \  --with-mibdir=$MIBDIR Can't exec "aclocal": No such file or directory at /usr/share/autoconf/Autom4te/FileUtils.pm line 326. autoreconf: failed to run aclocal: No such file or directory

I googled this error and this might be solved by installing automake.
install automake

# yum install automake –y

try again. okay.

# ./autobuild.sh <snip> Wrote: /root/rpmbuild/RPMS/x86_64/libvirt-snmp-0.0.3-1.el6.root1358398204.x86_64.rpm Executing(%clean): /bin/sh -e /var/tmp/rpm-tmp.Sd7MN0 + umask 022 + cd /root/rpmbuild/BUILD + cd libvirt-snmp-0.0.3 + /bin/rm -rf /root/rpmbuild/BUILDROOT/libvirt-snmp-0.0.3-1.el6.root1358398204.x86_64 + exit 0 Executing(--clean): /bin/sh -e /var/tmp/rpm-tmp.SPQKCv + umask 022 + cd /root/rpmbuild/BUILD + rm -rf libvirt-snmp-0.0.3 + exit 0 # echo $? 0 #

libvirt-snmp rpm package will be created under ~/rpmbuild/RPMS/<arch>/ directory.

# ls ~/rpmbuild/RPMS/x86_64/ libvirt-snmp-0.0.3-1.el6.root1358398204.x86_64.rpm

install libvirt-snmp via yum

# yum --nogpgcheck localinstall ~/rpmbuild/RPMS/x86_64/libvirt-snmp-0.0.3-1.el6.root1358398204.x86_64.rpm –y <snip> Running Transaction  Installing : libvirt-snmp-0.0.3-1.el6.root1358398204.x86_64               1/1  Verifying  : libvirt-snmp-0.0.3-1.el6.root1358398204.x86_64               1/1 Installed:  libvirt-snmp.x86_64 0:0.0.3-1.el6.root1358398204 Complete!

check options

# /usr/bin/libvirtMib_subagent --help /usr/bin/libvirtMib_subagent: invalid option -- '-' unknown option ? usage: libvirtGuestTable [-D<tokens>] [-f] [-L] [-M] [-H] [LISTENING ADDRESSES]        -f      Do not fork() from the calling shell.        -DTOKEN[,TOKEN,...]                Turn on debugging output for the given TOKEN(s).                Without any tokens specified, it defaults to printing                all the tokens (which is equivalent to the keyword 'ALL').                You might want to try ALL for extremely verbose output.                Note: You can't put a space between the -D and the TOKENs.        -H      Display a list of configuration file directives                understood by the agent and then exit.        -M      Run as a normal SNMP Agent instead of an AgentX sub-agent.        -x ADDRESS      connect to master agent at ADDRESS (default /var/agentx/master).        -L      Do not open a log file; print all messages to stderr.

display configuration file directives to stderr.

# /usr/bin/libvirtMib_subagent -H -L No log handling enabled - turning on stderr logging Warning: no access control information configured.  It's unlikely this agent can serve any useful purpose in this state.  Run "snmpconf -g basic_setup" to help you configure the libvirtGuestTable.conf file for this agent. Configuration directives understood:  In libvirtGuestTable.conf and libvirtGuestTable.local.conf:    authtrapenable           1 | 2              (1 = enable, 2 = disable)    trapsink                 host [community] [port]    trap2sink                host [community] [port]    informsink               host [community] [port]    trapsess                 [snmpcmdargs] host    trapcommunity            community-string    v1trapaddress            string    agentuser                userid <snip>

a MIB file for libvirt-snmp

# head -5 /usr/share/snmp/mibs/LIBVIRT-MIB.txt LIBVIRT-MIB DEFINITIONS ::= BEGIN -- -- libvirt.org MIB definition --

[ SNMP configuration ]

- snmp get and set configuration

# cd /etc/snmp/ # cp snmpd.conf snmpd.conf.orig

edit snmpd.conf like this:

# egrep -v ^# /etc/snmp/snmpd.conf | grep -v ^$ com2sec notConfigUser  default       public group   notConfigGroup v1           notConfigUser group   notConfigGroup v2c           notConfigUser view    systemview    included   .1 access  notConfigGroup ""      any       noauth    exact  systemview systemview none access MyRWGroup ""      any       noauth    0      all    all    all syslocation Unknown (edit /etc/snmp/snmpd.conf) syscontact Root <root@localhost> (configure /etc/snmp/snmp.local.conf) dontLogTCPWrappersConnects yes master agentx agentX Socket /var/agentx/master trap2sink localhost public

start snmpd

# /etc/init.d/snmpd restart

run libvirtMib_subagent

# LIBVIRT_DEFAULT_URI="qemu:///system" libvirtMib_subagent -f -L NET-SNMP version 5.5 AgentX subagent connected

try snmpbuldwalk to confirm you can get info about libvrit.

# snmpbulkwalk -v 2c -c public localhost -OX libvirtMIB -m all LIBVIRT-MIB::libvirtGuestName[STRING: bc90f9d2-7366-2c60-891b-23224db5538] = STRING: "centos6-64-1" LIBVIRT-MIB::libvirtGuestState[STRING: bc90f9d2-7366-2c60-891b-23224db5538] = INTEGER: running(1) LIBVIRT-MIB::libvirtGuestCpuCount[STRING: bc90f9d2-7366-2c60-891b-23224db5538] = Gauge32: 1 LIBVIRT-MIB::libvirtGuestMemoryCurrent[STRING: bc90f9d2-7366-2c60-891b-23224db5538] = Gauge32: 1024 LIBVIRT-MIB::libvirtGuestMemoryLimit[STRING: bc90f9d2-7366-2c60-891b-23224db5538] = Gauge32: 1024 LIBVIRT-MIB::libvirtGuestCpuTime[STRING: bc90f9d2-7366-2c60-891b-23224db5538] = Counter64: 119410000000 LIBVIRT-MIB::libvirtGuestRowStatus[STRING: bc90f9d2-7366-2c60-891b-23224db5538] = INTEGER: active(1)

display snmp table

# snmptable -m ALL -v 2c -c public -Cb localhost libvirtGuestTable SNMP table: LIBVIRT-MIB::libvirtGuestTable           Name   State CpuCount MemoryCurrent MemoryLimit      CpuTime RowStatus "centos6-64-1" running        1          1024        1024 120270000000    active

pause the VM “centos6-64-1” via snmpset.

before sending snmpset

# virsh list --all Id    Name                           State ---------------------------------------------------- 1     centos6-64-1                   running

pause the VM via snmp

# snmpset -m ALL -v 2c -c public localhost libvirtGuestState.\'bc90f9d2-7366-2c60-891b-23224db5538\' = paused LIBVIRT-MIB::libvirtGuestState.'....sf,`...2$.U8' = INTEGER: paused(3)

# virsh list --all Id    Name                           State ---------------------------------------------------- 1     centos6-64-1                   paused

run

# snmpset -m ALL -v 2c -c public localhost libvirtGuestState.\'bc90f9d2-7366-2c60-891b-23224db5538\' = running LIBVIRT-MIB::libvirtGuestState.'....sf,`...2$.U8' = INTEGER: running(1) # virsh list --all Id    Name                           State ---------------------------------------------------- 1     centos6-64-1                   running

shutdown

# snmpset -m ALL -v 2c -c public localhost libvirtGuestState.\'bc90f9d2-7366-2c60-891b-23224db5538\' = shutdown

# virsh list --all Id    Name                           State ---------------------------------------------------- -     centos6-64-1                   shut off

start the VM via snmp

at first check UUID

# snmpbulkwalk -v 2c -c public localhost -OX libvirtMIB -m all LIBVIRT-MIB::libvirtGuestName[STRING: bc90f9d2-7366-2c60-891b-23224db5538] = STRING: "centos6-64-1"

start the VM

# virsh list –all Id    Name                           State ---------------------------------------------------- -     centos6-64-1                   shut off # snmpset -m ALL -v 2c -c public localhost libvirtGuestRowStat us.\'bc90f9d2-7366-2c60-891b-23224db5538\' = createAndGo LIBVIRT-MIB::libvirtGuestRowStatus.'....sf,`...2$.U8' = INTEGER: createAndGo(4) # virsh list --all Id    Name                           State ---------------------------------------------------- 2     centos6-64-1                   running

- snmp trap configuration

# cat /etc/snmp/snmptrapd.conf # Example configuration file for snmptrapd # # No traps are handled by default, you must edit this file! # # authCommunity   log,execute,net public # traphandle SNMPv2-MIB::coldStart    /usr/bin/bin/my_great_script cold authCommunity log test

run snmptrapd with foreground

# snmptrapd -f -Lo -c /etc/snmp/snmptrapd.conf --disableAuthorization=yes -m ALL NET-SNMP version 5.5 AgentX subagent connected NET-SNMP version 5.5

pause the VM

# snmpset -m ALL -v 2c -c public localhost libvirtGuestState.\'bc90f9d2-7366-2c60-891b-23224db5538\' = paused LIBVIRT-MIB::libvirtGuestState.'....sf,`...2$.U8' = INTEGER: paused(3)

# snmptrapd -f -Lo -c /etc/snmp/snmptrapd.conf --disableAuthorization=yes -m ALL NET-SNMP version 5.5 AgentX subagent connected NET-SNMP version 5.5 2013-01-17 21::01:12 localhost.localdomain [UDP: [127.0.0.1]:43882->[127.0.0.1]]: DISMAN-EVENT-MIB::sysUpTimeInstance = Timeticks: (98514) 0:16:25.14     SNMPv2-MIB::snmpTrapOID.0 = OID: LIBVIRT-MIB::libvirtGuestNotif LIBVIRT-MIB::libvirtGuestName.0 = STRING: "centos6-64-1"        LIBVIRT-MIB::libvirtGuestUUID.1 = STRING: bc90f9d2-7366-2c60-891b-23224db5538   LIBVIRT-MIB::libvirtGuestState.2 = INTEGER: paused(3)   LIBVIRT-MIB::libvirtGuestRowStatus.3 = INTEGER: active(1)

shutdown the VM

# snmpset -m ALL -v 2c -c public localhost libvirtGuestState.\'bc90f9d2-7366-2c60-891b-23224db5538\' = shutdown LIBVIRT-MIB::libvirtGuestState.'....sf,`...2$.U8' = INTEGER: shutdown(4)

trap message

2013-01-17 21::02:41 localhost.localdomain [UDP: [127.0.0.1]:43882->[127.0.0.1]]: DISMAN-EVENT-MIB::sysUpTimeInstance = Timeticks: (107358) 0:17:53.58    SNMPv2-MIB::snmpTrapOID.0 = OID: LIBVIRT-MIB::libvirtGuestNotif LIBVIRT-MIB::libvirtGuestName.0 = STRING: "centos6-64-1"        LIBVIRT-MIB::libvirtGuestUUID.1 = STRING: bc90f9d2-7366-2c60-891b-23224db5538   LIBVIRT-MIB::libvirtGuestState.2 = INTEGER: shutdown(4) LIBVIRT-MIB::libvirtGuestRowStatus.3 = INTEGER: notInService(2) 2013-01-17 21::02:41 localhost.localdomain [UDP: [127.0.0.1]:43882->[127.0.0.1]]: DISMAN-EVENT-MIB::sysUpTimeInstance = Timeticks: (107414) 0:17:54.14    SNMPv2-MIB::snmpTrapOID.0 = OID: LIBVIRT-MIB::libvirtGuestNotif LIBVIRT-MIB::libvirtGuestName.0 = STRING: "centos6-64-1"        LIBVIRT-MIB::libvirtGuestUUID.1 = STRING: bc90f9d2-7366-2c60-891b-23224db5538   LIBVIRT-MIB::libvirtGuestState.2 = INTEGER: shutoff(5)  LIBVIRT-MIB::libvirtGuestRowStatus.3 = INTEGER: notInService(2)