Storage (ceph)

$ sudo aptitude install ceph
$ sudo mkdir -p /srv/ceph/{osd,mon,mds}
In server node
$ sudo emacs /etc/ceph/ceph.conf
[global]
    auth supported = cephx
    keyring = /etc/ceph/keyring.admin
    log file = /var/log/ceph/$name.log
    pid file = /var/run/ceph/$name.pid
[mon]
    ;; sudo mkdir /srv/ceph/mon /srv/ceph/osd
    ;; sudo mkdir /srv/ceph/mon/mon.$id
    mon data = /srv/ceph/mon/mon.$id    
[mds]
    keyring = /etc/ceph/keyring.$name
[osd]
    ;; sudo mkdir /srv/ceph/osd/osd.$host
    ;; sudo mount -t btrfs /dev/sdb /srv/ceph/osd/osd.$host
    osd data = /srv/ceph/osd/osd.$host
    osd journal = /srv/ceph/osd/osd.$id.journal
    osd journal size = 1000 ; journal size, in megabytes
    keyring = /etc/ceph/keyring.$name
[mon.scss]
    host = Storage-Server
    mon addr = 192.168.121.250:6789
[mds.scss]
    host = Storage-Server
[osd.0]
    host = Storage-Server
[osd.1]
    host = Storage-Cloud01
[osd.2]
    host = Storage-Cloud02
...
[osd.8]
    host = Storage-Cloud08

On OSD node
# Load the btrfs module.
$ sudo modprobe btrfs
$ diff /etc/modules /etc/modules.orig
8,9d7
< btrfs
< ceph
$ sudo mkfs.btrfs /dev/sdb

WARNING! - Btrfs Btrfs v0.19 IS EXPERIMENTAL
WARNING! - see http://btrfs.wiki.kernel.org before using
fs created label (null) on /dev/sdb
	nodesize 4096 leafsize 4096 sectorsize 4096 size 465.76GB
Btrfs Btrfs v0.19

The next line for mounting (btrfs) object storage is added to /etc/rc.local.
mount -t btrfs /dev/sdb /srv/ceph/osd/osd.$host
Permit Root login so that mkcephfs can be executed as root.
$ diff /etc/ssh/sshd_config /etc/ssh/sshd_config.bkp
26c26
< PermitRootLogin yes
---
> PermitRootLogin no
$ ssh root@192.168.121.250
# exit
# Back to server node.
$ su
# ssh-keygen
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa): 
/root/.ssh/id_rsa already exists.
Overwrite (y/n)? y
Enter passphrase (empty for no passphrase): 
Enter same passphrase again: 
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
22:ba:19:4d:0e:74:5f:4e:93:57:14:c7:68:35:1e:0b root@CSIE-Server
The key's randomart image is:
+--[ RSA 2048]----+
|           .E==  |
|         . .o+.+ |
|  . .   = ..  o  |
| . . . + o       |
|  . o o S        |
|   * . .         |
|  o o            |
|   +             |
|  o              |
+-----------------+

# ssh-copy-id root@scss
# ssh-copy-id root@sc01
# ssh-copy-id root@sc02
...
Create a new Ceph cluster file system
# mkcephfs -c /etc/ceph/ceph.conf -a -k /etc/ceph/keyring.admin
# echo $?
0
# for host in sc01 sc02 sc03 sc04 sc05 sc06 sc07 sc08 
  do
    scp /etc/ceph/keyring.admin root@$host:/etc/ceph/keyring.admin;
  done
# /etc/init.d/ceph -a start
## In any client
$ sudo ceph-authtool -l /etc/ceph/keyring.admin
[client.admin]
	key = AQAjb2VQ6I4HNhAAlxStlBRj8DJV7YfkFsERvQ==

$ sudo mount -t ceph scss:6789:/ /mnt/tmp/ -vv -o name=admin,secret=AQBZn2lQYAEFFxAAmLaNiBfXJQF3GQUGp2gYeA==
mount: error writing /etc/mtab: Invalid argument
$ df -h
/dev/sdb                     466G  3.2M  464G   1% /srv/ceph/osd/osd.Storage-Server
192.168.121.250:6789:/     4.1T   19G  4.1T   1% /mnt/tmp

$ sudo umount /mnt/tmp
$ sudo mkdir /mnt/Storage
$ sudo mount -t ceph scss:6789:/ /mnt/Storage -vv -o name=admin,secret=AQBZn2lQYAEFFxAAmLaNiBfXJQF3GQUGp2gYeA==
$ df -h

KVM Compilation

Check hardware assisted virtualization support
$ egrep '(vmx|svm)' --color=always /proc/cpuinfo
$ sudo modprobe kvm_amd #(svm) or kvm_intel (vmx)
Download KVM latest from sourceforge
$ cd ;cd Downloads/
$ tar zxvf qemu-kvm-*.tar.gz
$ cd qemu-kvm-*
$ sudo apt-get install gcc libsdl1.2-dev zlib1g-dev libasound2-dev \
linux-kernel-headers pkg-config libgnutls-dev libpci-dev gawk
$ sudo aptitude install libvdeplug2-dev
The following NEW packages will be installed:
  libvdeplug2-dev{b} 
0 packages upgraded, 1 newly installed, 0 to remove and 55 not upgraded.
Need to get 14.2 kB of archives. After unpacking 90.1 kB will be used.
The following packages have unmet dependencies:
 libvdeplug2-dev : Depends: libvdeplug2 (= 2.2.3-3) but 2.3.2-4 is installed.
The following actions will resolve these dependencies:
     Keep the following packages at their current version:
1)     libvdeplug2-dev [Not Installed]                    
Accept this solution? [Y/n/q/?] n
The following actions will resolve these dependencies:
     Downgrade the following packages:                         
1)     libvdeplug2 [2.3.2-4 (now, testing) -> 2.2.3-3 (stable)]
Accept this solution? [Y/n/q/?] Y

$ sudo aptitude install uml-utilities screen socat vde2
$ sudo aptitude install librbd-dev 
curses
$ sudo aptitude install libcurses-ocaml-dev
VirtFS
$ sudo aptitude install libcap-dev libattr1-dev
iSCSI
$ sudo aptitude install libiscsi-dev
$ mkdir objects; cd objects
$ ../configure --prefix=/usr/local --source-path=.. \
--target-list=x86_64-linux-user,x86_64-softmmu --enable-kvm --enable-linux-user \
--enable-vde --enable-vhost-net --enable-libiscsi \
--enable-docs --enable-rbd --enable-curses --enable-sdl --enable-virtfs \
--enable-vnc-tls --enable-vnc-jpeg --enable-vnc-png --enable-vnc
$ make
$ sudo make install
$ ls -l /usr/local/bin/qemu-*
-rwxr-xr-x 1 root staff 1284639 Sep 28 02:33 /usr/local/bin/qemu-ga
-rwxr-xr-x 1 root staff 2475055 Sep 28 02:33 /usr/local/bin/qemu-img
-rwxr-xr-x 1 root staff 2530996 Sep 28 02:33 /usr/local/bin/qemu-io
-rwxr-xr-x 1 root staff 2428912 Sep 28 02:33 /usr/local/bin/qemu-nbd
-rwxr-xr-x 1 root staff 4734240 Sep 28 02:33 /usr/local/bin/qemu-system-x86_64
-rwxr-xr-x 1 root staff 2447088 Sep 28 02:33 /usr/local/bin/qemu-x86_64
$ qemu-system-x86_64 --version
QEMU emulator version 1.2.0 (qemu-kvm-1.2.0), Copyright (c) 2003-2008 Fabrice Bellard
$ cd /usr/local/bin/
$ sudo ln -s qemu-system-x86_64 kvm
$ diff /etc/initramfs-tools/modules /etc/initramfs-tools/modules.orig
12d11
< kvm-amd nested=1

建立環境所需檔案

或許使用雙硬碟比較簡單而且妥當。
$ cd /mnt/Storage/
$ sudo mkdir KVM
$ sudo chown cloud:cloud KVM/
$ cd KVM/
$ tar zxvf /backup/KVM-tool/KVM-tool-*.tgz 
$ rm -rf Hurd*
$ mkdir 2012-10-Day
$ mkdir 2012-10-Work
$ cp /src3/KVM/img/Debian-sid.img.gz .
$ cp Debian-sid.img.gz 001.img.gz
$ gunzip 001.img.gz 
$ qemu-img resize 001.img 40G
$ ls -lh
total 41G
-rw-r--r-- 1 cloud cloud  40G Oct  2 00:08 001.img
-rw-r--r-- 1 cloud cloud 811M Oct  2 00:06 Debian-sid.img.gz
$ sudo aptitude install gparted 
$ sudo modprobe nbd max_part=8
$ sudo qemu-nbd -c /dev/nbd0 001.img 
$ ls -l /dev/nbd0*
brw-rw---T 1 root disk 43, 0 Oct  2 00:14 /dev/nbd0
brw-rw---T 1 root disk 43, 1 Oct  2 00:14 /dev/nbd0p1
brw-rw---T 1 root disk 43, 2 Oct  2 00:14 /dev/nbd0p2
$ gksudo gparted /dev/nbd0 
$ sudo qemu-nbd -d /dev/nbd0
/dev/nbd0 disconnected
$ sudo rmmod nbd

Upgrade Virtual Machine

$ kvm 001.img 
Update /etc/apt/sources.list replacing `squeeze' to `wheezy'.
# aptitude update
# aptitude safe-upgrade
# apt-get upgrade
# aptitude install bash-completion deborphan fail2ban sudo
# deborphan
# dpkg -P `deborphan`
# aptitude clean
# init 6
Purge old kernel, and then halt the VM.

佈署 root filesystem

$ gzip 001.img 
$ ls -l
$ cp 001.img.gz backup.img.gz
$ cp 001.img.gz test.img.gz

建立 VNC 遠端連線環境

$ sudo aptitude install tightvncserver
$ vncpasswd 
Using password file /home/cloud/.vnc/passwd
VNC directory /home/cloud/.vnc does not exist, creating.
Password: 
Verify:   
Would you like to enter a view-only password (y/n)? n

Enable nested KVM

$ sudo modprobe -r kvm_amd
$ sudo modprobe kvm_amd nested=1
$ dmesg | grep kvm
[344651.400477] kvm: Nested Virtualization enabled
[344651.400505] kvm: Nested Paging enabled

Configure Root File System

$ cd /mnt/Storage/KVM/bin/
$ Config-Kvm /mnt/Storage/KVM/2012-10-Day/test.img test 140.120.16.80 eth1 taptest

備用方案

$ cd /src4
$ sudo mkdir KVM
$ sudo chown cloud:cloud KVM
$ cd KVM
$ scp css:/backup/KVM-tool/KVM-tool-Cloud-2012-10-03.tgz .
$ tar zxvf KVM-tool-Cloud-2012-10-03.tgz 
$ rm KVM-tool-Cloud-2012-10-03.tgz
$ cd img

References

Ceph FS