[Pacemaker] help with configuration for Xen domU on two DRBD devices

Sun Sep 12 23:56:12 UTC 2010

Hi

I'm new to pacemaker configurations and am trying to replace my old heartbeat two node cluster setup using haresources to pacemaker/corosync. I have mostly followed the instructions from http://www.clusterlabs.org/doc/Cluster_from_Scratch.pdf.

My configuration is for two drbd devices which are used as the filesystems for a xen DomU so they both need to be on the same node and in master role before the domU can be started. But I think I still have something wrong with my config (configs are below) as I had some trouble getting my colocation and order commands to work with master/slave resources.

Could you please have a look at my configuration and tell me what's wrong with it?

The problem:
During testing most actions seemed to work as expected(shutdown active node, failover of resources in correct order succeed), however pulling the power from the active node did not produce the results expected, which where that node alpha would take over resources.

# crm_mon -1
Online: [ alpha ]
OFFLINE: [ bravo ]

Master/Slave Set: drbdRoot
    Slaves: [ alpha ]
    Stopped: [ drbd0:1 ]
Master/Slave Set: drbdTmp
    Masters: [ alpha ]
    Stopped: [ drbd1:0 ]

Failed actions:
   testDomU_start_0 (node=alpha, call=17, rc=1, status=complete): unknown error
   drbd0:1_monitor_10000 (node=alpha, call=31, rc=8, status=complete): master

# cat /proc/drbd
version: 8.3.8 (api:88/proto:86-94)
GIT-hash: d78846e52224fd00562f7c225bcc25b2d422321d build by mockbuild at builder10.
centos.org, 2010-06-04 08:04:31
0: cs:WFConnection ro:Secondary/Unknown ds:UpToDate/Outdated C r----
   ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0
1: cs:WFConnection ro:Primary/Unknown ds:UpToDate/Outdated C r----
   ns:112 nr:128 dw:128 dr:112 al:0 bm:4 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0

I expected drbd fencing to add both these rules but only one was added, the one for drbdTmp.

location drbd-fence-by-handler-drbdRoot drbdRoot \
	rule $id="drbd-fence-by-handler-rule-drbdRoot" $role="Master" -inf: #uname ne alpha
location drbd-fence-by-handler-drbdTmp drbdTmp \
	rule $id="drbd-fence-by-handler-rule-drbdTmp" $role="Master" -inf: #uname ne alpha

<CONFIGURATIONS BELOW>
CentOS 5.5
# rpm -q pacemaker
pacemaker-1.0.9.1-1.15.el5
# rpm -q corosync
corosync-1.2.7-1.1.el5
# rpm -q drbd83
drbd83-8.3.8-1.el5.centos
# rpm -q heartbeat
heartbeat-3.0.3-2.3.el5

# crm configuration show
node alpha \
	attributes standby="off"
node bravo \
	attributes standby="off"
primitive drbd0 ocf:linbit:drbd \
	params drbd_resource="r0" \
	op monitor interval="3s" role="Master" timeout="20s" \
	op monitor interval="10s" role="Slave" timeout="20s" \
	op start interval="0" timeout="240s" \
	op stop interval="0" timeout="100s"
primitive drbd1 ocf:linbit:drbd \
	params drbd_resource="r1" \
	op monitor interval="3s" role="Master" timeout="20s" \
	op monitor interval="10s" role="Slave" timeout="20s" \
	op start interval="0" timeout="240s" \
	op stop interval="0" timeout="100s"
primitive testDomU ocf:heartbeat:Xen \
	params xmfile="/etc/xen/test.sxp" \
	op monitor interval="120" timeout="30" on-fail="restart" start-delay="60" \
	op start interval="0" timeout="120" start-delay="0" \
	op stop interval="0" timeout="120" \
	op migrate_from interval="0" timeout="240" \
	op migrate_to interval="0" timeout="240" \
	meta target-role="Started"
ms drbdRoot drbd0 \
	meta globally-unique="false" master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true" target-role="Started"
ms drbdTmp drbd1 \
	meta globally-unique="false" master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true" target-role="Started"
colocation testDomU-with-drbdRoot inf: testDomU drbdRoot:Master
colocation testDomU-with-drbdTmp inf: testDomU drbdTmp:Master
colocation drbdTmp-with-drbdRoot inf: drbdRoot:Master drbdTmp:Master
order order-drbdRoot-before-testDomU inf: drbdRoot:promote testDomU:start
order order-drbdTmp-and-drbdRoot inf: drbdTmp:promote drbdRoot:promote
property $id="cib-bootstrap-options" \
	dc-version="1.0.9-89bd754939df5150de7cd76835f98fe90851b677" \
	cluster-infrastructure="openais" \
	expected-quorum-votes="2" \
	stonith-enabled="false" \
	no-quorum-policy="ignore" \
	default-resource-stickiness="1000"

# cat /etc/drbd.conf
global {
       usage-count no;
}
common {
       protocol C;
 net {
       cram-hmac-alg "sha1";
       shared-secret "secret";

       # Disable for production
       data-integrity-alg sha1;
 }
 disk {
       fencing resource-only;
 }
 handlers {
       # these handlers are necessary for drbd 8.3 + pacemaker compatibility
       fence-peer "/usr/lib/drbd/crm-fence-peer.sh";
       after-resync-target "/usr/lib/drbd/crm-unfence-peer.sh";

 }
 syncer {
       rate 90M;
	al-extents 257;
  }
}
resource r0 {
 on alpha {
       device  /dev/drbd0;
       disk    /dev/VGxen/LVRoot;
       address 192.168.1.1:7789;
       meta-disk       internal;
       }
 on bravo {
       device  /dev/drbd0;
       disk    /dev/VGxen/LVRoot;
       address 192.168.1.2:7789;
       meta-disk       internal;
	}
}
resource r1 {
 on alpha {
       device  /dev/drbd1;
       disk    /dev/VGxen/LVTmp;
       address 192.168.1.1:7790;
       meta-disk       internal;
       }
 on bravo {
       device  /dev/drbd1;
       disk    /dev/VGxen/LVTmp;
       address 192.168.1.2:7790;
       meta-disk       internal;
	}
}

# cat /etc/xen/test.sxp
name = "testDomU"
memory = 3000
bootloader = "/usr/bin/pygrub"
on_poweroff = "destroy"
on_reboot = "restart"
on_crash = "restart"
vfb = [ "type=vnc,vncunused=1" ]
disk = [ "phy:/dev/drbd0,xvda1,w", 
	"phy:/dev/drbd1,xvda2,w",
       "phy:/dev/VGsys/LVswap1,xvda3,w", ]
vif = [ "bridge=xenbr0", ]

Thanks.