################################################################################ # OVS DPDK VM 安装 ################################################################################
#vim ~/.bash_profile #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> #DPDK_SDK=/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk #export DPDK_SDK # #LD_LIBRARY_PATH=$DPDK_SDK/lib:$MYTOOLS/lib:$MYTOOLS/libexec:$LD_LIBRARY_PATH #export LD_LIBRARY_PATH #<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #source ~/.bash_profile # # #sudo vim /etc/grub2.cfg #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> #linux16 /vmlinuz-3.10.0-1127.19.1.el7.x86_64 root=/dev/mapper/centos-root ro crashkernel=auto rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb LANG=zh_CN.UTF-8 iommu=pt intel_iommu=on default_hugepagesz=1G hugepagesz=1G hugepages=6 #<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< # # #/home/liufeng/workspace/src/ovs-dpdk #git clone git@github.com:DPDK/dpdk.git #git clone git@github.com:openvswitch/ovs.git # # #cd /home/liufeng/workspace/src/ovs-dpdk/dpdk #git checkout v19.11 -b v19.11 #vim config/common_base #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> #CONFIG_RTE_BUILD_SHARED_LIB=y #<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#make config T=x86_64-native-linuxapp-gcc #sed -ri 's,(PMD_PCAP=).*,\1y,' build/.config #mkdir /home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk #make install T=x86_64-native-linuxapp-gcc DESTDIR=/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk ##sudo make install T=x86_64-native-linuxapp-gcc DESTDIR=/usr #sudo make install T=x86_64-native-linuxapp-gcc DESTDIR=/home/liufeng/workspace/tools/ # #vim /etc/ld.so.conf.d/dpdk.conf #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> #/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk/lib #<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #ldconfig # #sudo yum install autoconf.noarch automake.noarch libtool.x86_64 python3-pip.noarch
#cd /home/liufeng/workspace/src/open/ovs #git checkout v2.13.0 -b v2.13.0 #./boot.sh #./configure \ #--with-dpdk=/home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk \ #--prefix=/home/liufeng/workspace/tools/ \ #--exec-prefix=/home/liufeng/workspace/tools/ \ #--sysconfdir=/home/liufeng/workspace/tools/etc \ #--localstatedir=/home/liufeng/workspace/tools/var
#make #sudo make install
su
sudo modprobe vfio sudo modprobe vfio-pci chmod a+x /dev/vfio chmod 0666 /dev/vfio/*
sudo modprobe uio_pci_generic #sudo insmod /home/liufeng/workspace/src/ovs-dpdk/dpdk/sdk/lib/modules/3.10.0-1127.19.1.el7.x86_64/extra/dpdk/igb_uio.ko
dpdk-devbind --status sudo ifconfig enp1s0f0 down sudo ifconfig enp1s0f1 down sudo ifconfig enp1s0f2 down sudo ifconfig enp1s0f3 down
sudo dpdk-devbind --bind=vfio-pci enp1s0f0 sudo dpdk-devbind --bind=vfio-pci enp1s0f1 sudo dpdk-devbind --bind=vfio-pci enp1s0f2 sudo dpdk-devbind --bind=vfio-pci enp1s0f3
dpdk-devbind --status
mount -t hugetlbfs -o pagesize=1G none /dev/hugepages grep HugePages_ /proc/meminfo
#mkdir -p /home/liufeng/workspace/tools/etc/openvswitch #sudo mkdir -p /home/liufeng/workspace/tools/var/run/openvswitch #sudo mkdir -p /home/liufeng/workspace/tools/var/log/openvswitch/
#rm /home/liufeng/workspace/tools/etc/openvswitch/conf.db
#sudo /home/liufeng/workspace/tools/bin/ovsdb-tool create /home/liufeng/workspace/tools/etc/openvswitch/conf.db /home/liufeng/workspace/tools/share/openvswitch/vswitch.ovsschema
# 启动ovsdb server ovsdb-server \ /home/liufeng/workspace/tools/etc/openvswitch/conf.db \ -vconsole:emer -vsyslog:err -vfile:info \ --remote=punix:/home/liufeng/workspace/tools/var/run/openvswitch/db.sock \ --remote=db:Open_vSwitch,Open_vSwitch,manager_options \ --private-key=db:Open_vSwitch,SSL,private_key \ --certificate=db:Open_vSwitch,SSL,certificate \ --bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert \ --log-file=/home/liufeng/workspace/tools/var/log/openvswitch/ovsdb-server.log \ --pidfile=/home/liufeng/workspace/tools/var/run/openvswitch/ovsdb-server.pid \ --detach --monitor
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
# 初始化dpdk #sudo /home/liufeng/workspace/tools/bin/ovs-vsctl --no-wait init #sudo /home/liufeng/workspace/tools/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true # 指定的sockets从hugepages预先分配的内存 #sudo /home/liufeng/workspace/tools/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem="1024,0" # 指定在某些core上运行 #sudo /home/liufeng/workspace/tools/bin/ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x02
#查看自定义的dpdk参数 #sudo /home/liufeng/workspace/tools/bin/ovs-vsctl get Open_vSwitch . other_config:dpdk-socket-mem #sudo /home/liufeng/workspace/tools/bin/ovs-vsctl get Open_vSwitch . other_config:pmd-cpu-mask #sudo /home/liufeng/workspace/tools/bin/ovs-vsctl get Open_vSwitch . other_config:dpdk-init
#启动vswitchd进程 ovs-vswitchd \ unix:/home/liufeng/workspace/tools/var/run/openvswitch/db.sock \ -vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir \ --log-file=/home/liufeng/workspace/tools/var/log/openvswitch/ovs-vswitchd.log \ --pidfile=/home/liufeng/workspace/tools/var/run/openvswitch/ovs-vswitchd.pid \ --detach --monitor
top -H ps -eLo pid,psr,comm | grep pmd
ovs-vsctl --version ovs-vsctl show #把dpdk端口加入网桥 #ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev #ovs-vsctl add-port br0 dpdk0 -- set Interface dpdk0 type=dpdk options:dpdk-devargs=0000:01:00.0 #ovs-vsctl add-port br0 dpdk1 -- set Interface dpdk1 type=dpdk options:dpdk-devargs=0000:01:00.1 #ovs-vsctl add-port br0 vhost-user-1 -- set Interface vhost-user-1 type=dpdkvhostuserclient options:vhost-server-path="/tmp/sock0" #ovs-vsctl add-port br0 vhost-user-1-1 -- set Interface vhost-user-1-1 type=dpdkvhostuserclient options:vhost-server-path="/tmp/sock0-1" #ovs-vsctl add-port br0 vhost-user-2 -- set Interface vhost-user-2 type=dpdkvhostuserclient options:vhost-server-path="/tmp/sock2"
ovs-ofctl dump-flows br0 #sudo /home/liufeng/workspace/tools/bin/ovs-vsctl list controller br0
ovsdb-client dump
#git clone https://git.qemu.org/git/qemu.git #/home/liufeng/workspace/src/ovs-dpdk/qemu #git checkout v3.0.0-rc4 -b v3.0.0-rc4 #./configure --prefix=/home/liufeng/workspace/tools/ --target-list=x86_64-softmmu --disable-strip --enable-debug --enable-debug-info --disable-sdl --disable-git-update #scripts/git-submodule.sh update ui/keycodemapdb dtc #make #sudo make install
#pushd /home/kvm/disk #qemu-img create -f qcow2 -o size=50G vm3.img #popd
#vim /etc/yum.repos.d/nux-misc.repo #>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> #[nux-misc] #name=Nux Misc #baseurl=http://li.nux.ro/download/nux/misc/el7/x86_64/ #enabled=0 #gpgcheck=1 #gpgkey=http://li.nux.ro/download/nux/RPM-GPG-KEY-nux.ro #<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< #yum --enablerepo=nux-misc install tunctl
#建立VM网络 tunctl -t tap30 tunctl -t tap33 brctl addif br1 tap30 brctl addif br1 tap33 brctl show ifconfig tap30 promisc up ifconfig tap33 promisc up
#安装： #qemu-system-x86_64 -m 2048 --enable-kvm -cpu kvm64 -smp 4 -boot d \ #-hda /home/kvm/disk/vm3.img -cdrom /home/kvm/iso/CentOS-7-x86_64-Everything-2003.iso \ #-vnc 0.0.0.0:30 -monitor stdio \ #-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:38
# 普通启动 VM3 #qemu-system-x86_64 -m 2048 --enable-kvm -cpu kvm64 -smp 4 \ #-hda /home/kvm/disk/vm3.img -cdrom /home/kvm/iso/CentOS-7-x86_64-Everything-2003.iso \ #-vnc 0.0.0.0:30 -monitor stdio \ #-netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:38
# 查看OVS状态 tailf /home/liufeng/workspace/tools/var/log/openvswitch/ovs-vswitchd.log
# vhost user 启动 VM0 qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 2048M \ -object memory-backend-file,id=mem,size=2048M,mem-path=/dev/hugepages,share=on \ -hda /home/kvm/disk/vm0.img -mem-prealloc -numa node,memdev=mem \ -vnc 0.0.0.0:00 -monitor stdio --enable-kvm \ -netdev type=tap,id=eth0,ifname=tap30,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:08 \ -chardev socket,id=char1,path=/tmp/sock0,server \ -netdev type=vhost-user,id=mynet1,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet1,id=net1,mac=00:00:00:00:00:01
#进入VM0 service NetworkManager stop ifconfig eth0 192.168.5.1/24 up tcpdump -i eth0
#进入VM进行配置 #ifconfig eth0 192.168.3.1/24 up #ping 192.168.3.2 （可以ping通） # #ifconfig eth0 192.168.4.1/24 up #ping 192.168.4.2 （可以ping通）
# vhost user 启动 VM3 qemu-system-x86_64 -machine accel=kvm -cpu host -smp sockets=2,cores=2,threads=1 -m 2048M \ -object memory-backend-file,id=mem,size=2048M,mem-path=/dev/hugepages,share=on \ -hda /home/kvm/disk/vm3.img -mem-prealloc -numa node,memdev=mem \ -vnc 0.0.0.0:30 -monitor stdio --enable-kvm \ -netdev type=tap,id=eth0,ifname=tap33,script=no,downscript=no -device e1000,netdev=eth0,mac=12:03:04:05:06:38 \ -chardev socket,id=char1,path=/tmp/sock2,server \ -netdev type=vhost-user,id=mynet1,chardev=char1,vhostforce -device virtio-net-pci,netdev=mynet1,id=net1,mac=00:00:00:00:00:03
# 进入VM3 # service NetworkManager stop ifconfig eth0 192.168.5.2/24 up ping 192.168.5.1
#进入VM进行配置 #ifconfig eth0 192.168.3.1/24 up #ping 192.168.3.2 （可以ping通） # #ifconfig eth0 192.168.4.1/24 up #ping 192.168.4.2 （可以ping通）
#删除VM网络 tunctl -d tap30
# 解开内核SRC RPM包 rpm2cpio kernel-3.10.0-1127.el7.src.rpm | cpio -idmv
