[Pacemaker] Recovery after lost quorum
Denis Witt
denis.witt at concepts-and-training.de
Tue Jun 4 21:55:20 EDT 2013
Am 05.06.2013 um 03:34 schrieb Andrew Beekhof <andrew at beekhof.net>:
>> But no resources are started, so I suspect there really is quorum.
>
> Can you send me the output of cibadmin -Ql please?
> Perhaps those two resources are blocked for other reasons.
Hi Andrew,
here we go:
<cib epoch="66" num_updates="28" admin_epoch="0" validate-with="pacemaker-1.2" crm_feature_set="3.0.6" update-origin="test4" update-client="crmd" cib-last-written="Wed Jun 5 03:44:38 2013" have-quorum="1" dc-uuid="test4">
<configuration>
<crm_config>
<cluster_property_set id="cib-bootstrap-options">
<nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.7-ee0730e13d124c3d58f00016c3376a1de5323cff"/>
<nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="openais"/>
<nvpair id="cib-bootstrap-options-expected-quorum-votes" name="expected-quorum-votes" value="3"/>
<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="stop"/>
<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
<nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1370396678"/>
<nvpair id="cib-bootstrap-options-default-resource-stickiness" name="default-resource-stickiness" value="100"/>
<nvpair id="cib-bootstrap-options-maintenance-mode" name="maintenance-mode" value="false"/>
</cluster_property_set>
</crm_config>
<nodes>
<node id="test3" type="normal" uname="test3"/>
<node id="test4" type="normal" uname="test4"/>
<node id="backup3" type="normal" uname="backup3">
<instance_attributes id="nodes-backup3">
<nvpair id="nodes-backup3-standby" name="standby" value="on"/>
</instance_attributes>
</node>
</nodes>
<resources>
<clone id="clone_pingtest">
<primitive class="ocf" id="pingtest" provider="pacemaker" type="ping">
<instance_attributes id="pingtest-instance_attributes">
<nvpair id="pingtest-instance_attributes-multiplier" name="multiplier" value="1000"/>
<nvpair id="pingtest-instance_attributes-host_list" name="host_list" value="192.168.100.19"/>
</instance_attributes>
<operations>
<op id="pingtest-monitor-5" interval="5" name="monitor"/>
</operations>
</primitive>
</clone>
<master id="ms_drbd">
<meta_attributes id="ms_drbd-meta_attributes">
<nvpair id="ms_drbd-meta_attributes-master-max" name="master-max" value="1"/>
<nvpair id="ms_drbd-meta_attributes-master-node-max" name="master-node-max" value="1"/>
<nvpair id="ms_drbd-meta_attributes-clone-max" name="clone-max" value="2"/>
<nvpair id="ms_drbd-meta_attributes-clone-node-max" name="clone-node-max" value="1"/>
<nvpair id="ms_drbd-meta_attributes-notify" name="notify" value="true"/>
</meta_attributes>
<primitive class="ocf" id="drbd" provider="linbit" type="drbd">
<instance_attributes id="drbd-instance_attributes">
<nvpair id="drbd-instance_attributes-drbd_resource" name="drbd_resource" value="www_r0"/>
</instance_attributes>
<operations>
<op id="drbd-monitor-10" interval="10" name="monitor"/>
</operations>
</primitive>
</master>
<group id="grp_all">
<primitive class="ocf" id="sip" provider="heartbeat" type="IPaddr2">
<instance_attributes id="sip-instance_attributes">
<nvpair id="sip-instance_attributes-ip" name="ip" value="192.168.100.30"/>
<nvpair id="sip-instance_attributes-nic" name="nic" value="eth0"/>
</instance_attributes>
<operations>
<op id="sip-monitor-10" interval="10" name="monitor" timeout="20"/>
</operations>
<meta_attributes id="sip-meta_attributes">
<nvpair id="sip-meta_attributes-target-role" name="target-role" value="Started"/>
</meta_attributes>
</primitive>
<primitive class="ocf" id="fs_drbd" provider="heartbeat" type="Filesystem">
<instance_attributes id="fs_drbd-instance_attributes">
<nvpair id="fs_drbd-instance_attributes-device" name="device" value="/dev/drbd0"/>
<nvpair id="fs_drbd-instance_attributes-directory" name="directory" value="/var/www"/>
<nvpair id="fs_drbd-instance_attributes-fstype" name="fstype" value="ext4"/>
</instance_attributes>
<operations>
<op id="fs_drbd-monitor-5" interval="5" name="monitor"/>
</operations>
<meta_attributes id="fs_drbd-meta_attributes">
<nvpair id="fs_drbd-meta_attributes-target-role" name="target-role" value="Started"/>
</meta_attributes>
</primitive>
<primitive class="lsb" id="apache" type="apache2">
<operations>
<op id="apache-monitor-10" interval="10" name="monitor" timeout="20"/>
</operations>
<meta_attributes id="apache-meta_attributes">
<nvpair id="apache-meta_attributes-target-role" name="target-role" value="Started"/>
</meta_attributes>
</primitive>
</group>
</resources>
<constraints>
<rsc_colocation id="coloc_all_on_drbd" rsc="grp_all" score="INFINITY" with-rsc="ms_drbd" with-rsc-role="Master"/>
<rsc_order first="ms_drbd" first-action="promote" id="order_all_after_drbd" score="INFINITY" then="grp_all" then-action="start"/>
<rsc_location id="loc_all_on_best_ping" rsc="grp_all">
<rule boolean-op="or" id="loc_all_on_best_ping-rule" score="-INFINITY">
<expression attribute="pingd" id="loc_all_on_best_ping-expression" operation="not_defined"/>
<expression attribute="pingd" id="loc_all_on_best_ping-expression-0" operation="lt" value="1000"/>
</rule>
</rsc_location>
</constraints>
</configuration>
<status>
<node_state id="test4" uname="test4" ha="active" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="post_cache_update" shutdown="0">
<transient_attributes id="test4">
<instance_attributes id="status-test4">
<nvpair id="status-test4-probe_complete" name="probe_complete" value="true"/>
</instance_attributes>
</transient_attributes>
<lrm id="test4">
<lrm_resources>
<lrm_resource id="pingtest:0" type="ping" class="ocf" provider="pacemaker">
<lrm_rsc_op id="pingtest:0_last_0" operation_key="pingtest:0_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="8:15:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;8:15:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="48" rc-code="0" op-status="0" interval="0" op-digest="7d41580263769d86524e63413353cd0b"/>
</lrm_resource>
<lrm_resource id="pingtest:1" type="ping" class="ocf" provider="pacemaker">
<lrm_rsc_op id="pingtest:1_last_0" operation_key="pingtest:1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="8:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;8:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="141" rc-code="0" op-status="0" interval="0" op-digest="7d41580263769d86524e63413353cd0b" last-run="1370396742" last-rc-change="1370396742" exec-time="90" queue-time="0"/>
<lrm_rsc_op id="pingtest:1_monitor_5000" operation_key="pingtest:1_monitor_5000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="10:46:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;10:46:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="103" rc-code="0" op-status="0" interval="5000" op-digest="ea75753f1e9e0cac00432d83a812f4c8"/>
</lrm_resource>
<lrm_resource id="sip" type="IPaddr2" class="ocf" provider="heartbeat">
<lrm_rsc_op id="sip_last_0" operation_key="sip_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="41:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;41:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="144" rc-code="0" op-status="0" interval="0" op-digest="b009a03f7cd980a54dff5e570c8de74d" last-run="1370396743" last-rc-change="1370396743" exec-time="40" queue-time="0"/>
<lrm_rsc_op id="sip_monitor_10000" operation_key="sip_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="44:48:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;44:48:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="108" rc-code="0" op-status="0" interval="10000" op-digest="f92644ce891641f5d8a4019b289dbbd0"/>
</lrm_resource>
<lrm_resource id="drbd:0" type="drbd" class="ocf" provider="linbit">
<lrm_rsc_op id="drbd:0_last_failure_0" operation_key="drbd:0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="8:59:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:8;8:59:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="120" rc-code="8" op-status="0" interval="0" op-digest="f2655e5354fe9acd9270c82254b11693"/>
<lrm_rsc_op id="drbd:0_last_0" operation_key="drbd:0_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="10:84:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;10:84:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="148" rc-code="0" op-status="0" interval="0" last-run="1370396744" last-rc-change="1370396744" exec-time="110" queue-time="0" op-digest="f2655e5354fe9acd9270c82254b11693"/>
</lrm_resource>
<lrm_resource id="fs_drbd" type="Filesystem" class="ocf" provider="heartbeat">
<lrm_rsc_op id="fs_drbd_last_failure_0" operation_key="fs_drbd_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="7:76:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;7:76:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="132" rc-code="0" op-status="0" interval="0" last-run="1370396658" last-rc-change="1370396658" exec-time="30" queue-time="0" op-digest="e0cde594906f5ecf8fa72d58000e0c45"/>
<lrm_rsc_op id="fs_drbd_monitor_5000" operation_key="fs_drbd_monitor_5000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="46:77:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;46:77:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="133" rc-code="0" op-status="0" interval="5000" last-rc-change="1370396658" exec-time="30" queue-time="0" op-digest="58701fb56792cdf0fbe729ee4f043a87"/>
<lrm_rsc_op id="fs_drbd_last_0" operation_key="fs_drbd_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="43:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;43:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="143" rc-code="0" op-status="0" interval="0" last-run="1370396743" last-rc-change="1370396743" exec-time="70" queue-time="0" op-digest="e0cde594906f5ecf8fa72d58000e0c45"/>
</lrm_resource>
<lrm_resource id="apache" type="apache2" class="lsb">
<lrm_rsc_op id="apache_last_failure_0" operation_key="apache_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="7:81:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;7:81:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="138" rc-code="0" op-status="0" interval="0" last-run="1370396677" last-rc-change="1370396677" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
<lrm_rsc_op id="apache_monitor_10000" operation_key="apache_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="48:82:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;48:82:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="139" rc-code="0" op-status="0" interval="10000" last-rc-change="1370396677" exec-time="20" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
<lrm_rsc_op id="apache_last_0" operation_key="apache_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="45:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;45:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="140" rc-code="0" op-status="0" interval="0" last-run="1370396742" last-rc-change="1370396742" exec-time="1160" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
</lrm_resource>
</lrm_resources>
</lrm>
</node_state>
<node_state id="backup3" uname="backup3" crmd="online" crm-debug-origin="post_cache_update" ha="active" in_ccm="true" join="member" expected="member" shutdown="0"/>
</status>
</cib>
Quorum was lost, then I restarted corosync on backup3 without pacemaker.
When I enable pacemaker on backup3 the resources are started after a while. Then I got the following:
<cib epoch="66" num_updates="81" admin_epoch="0" validate-with="pacemaker-1.2" crm_feature_set="3.0.6" update-origin="test4" update-client="crmd" cib-last-written="Wed Jun 5 03:44:38 2013" have-quorum="1" dc-uuid="test4">
<configuration>
<crm_config>
<cluster_property_set id="cib-bootstrap-options">
<nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.7-ee0730e13d124c3d58f00016c3376a1de5323cff"/>
<nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="openais"/>
<nvpair id="cib-bootstrap-options-expected-quorum-votes" name="expected-quorum-votes" value="3"/>
<nvpair id="cib-bootstrap-options-no-quorum-policy" name="no-quorum-policy" value="stop"/>
<nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
<nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1370396678"/>
<nvpair id="cib-bootstrap-options-default-resource-stickiness" name="default-resource-stickiness" value="100"/>
<nvpair id="cib-bootstrap-options-maintenance-mode" name="maintenance-mode" value="false"/>
</cluster_property_set>
</crm_config>
<nodes>
<node id="test3" type="normal" uname="test3"/>
<node id="test4" type="normal" uname="test4"/>
<node id="backup3" type="normal" uname="backup3">
<instance_attributes id="nodes-backup3">
<nvpair id="nodes-backup3-standby" name="standby" value="on"/>
</instance_attributes>
</node>
</nodes>
<resources>
<clone id="clone_pingtest">
<primitive class="ocf" id="pingtest" provider="pacemaker" type="ping">
<instance_attributes id="pingtest-instance_attributes">
<nvpair id="pingtest-instance_attributes-multiplier" name="multiplier" value="1000"/>
<nvpair id="pingtest-instance_attributes-host_list" name="host_list" value="192.168.100.19"/>
</instance_attributes>
<operations>
<op id="pingtest-monitor-5" interval="5" name="monitor"/>
</operations>
</primitive>
</clone>
<master id="ms_drbd">
<meta_attributes id="ms_drbd-meta_attributes">
<nvpair id="ms_drbd-meta_attributes-master-max" name="master-max" value="1"/>
<nvpair id="ms_drbd-meta_attributes-master-node-max" name="master-node-max" value="1"/>
<nvpair id="ms_drbd-meta_attributes-clone-max" name="clone-max" value="2"/>
<nvpair id="ms_drbd-meta_attributes-clone-node-max" name="clone-node-max" value="1"/>
<nvpair id="ms_drbd-meta_attributes-notify" name="notify" value="true"/>
</meta_attributes>
<primitive class="ocf" id="drbd" provider="linbit" type="drbd">
<instance_attributes id="drbd-instance_attributes">
<nvpair id="drbd-instance_attributes-drbd_resource" name="drbd_resource" value="www_r0"/>
</instance_attributes>
<operations>
<op id="drbd-monitor-10" interval="10" name="monitor"/>
</operations>
</primitive>
</master>
<group id="grp_all">
<primitive class="ocf" id="sip" provider="heartbeat" type="IPaddr2">
<instance_attributes id="sip-instance_attributes">
<nvpair id="sip-instance_attributes-ip" name="ip" value="192.168.100.30"/>
<nvpair id="sip-instance_attributes-nic" name="nic" value="eth0"/>
</instance_attributes>
<operations>
<op id="sip-monitor-10" interval="10" name="monitor" timeout="20"/>
</operations>
<meta_attributes id="sip-meta_attributes">
<nvpair id="sip-meta_attributes-target-role" name="target-role" value="Started"/>
</meta_attributes>
</primitive>
<primitive class="ocf" id="fs_drbd" provider="heartbeat" type="Filesystem">
<instance_attributes id="fs_drbd-instance_attributes">
<nvpair id="fs_drbd-instance_attributes-device" name="device" value="/dev/drbd0"/>
<nvpair id="fs_drbd-instance_attributes-directory" name="directory" value="/var/www"/>
<nvpair id="fs_drbd-instance_attributes-fstype" name="fstype" value="ext4"/>
</instance_attributes>
<operations>
<op id="fs_drbd-monitor-5" interval="5" name="monitor"/>
</operations>
<meta_attributes id="fs_drbd-meta_attributes">
<nvpair id="fs_drbd-meta_attributes-target-role" name="target-role" value="Started"/>
</meta_attributes>
</primitive>
<primitive class="lsb" id="apache" type="apache2">
<operations>
<op id="apache-monitor-10" interval="10" name="monitor" timeout="20"/>
</operations>
<meta_attributes id="apache-meta_attributes">
<nvpair id="apache-meta_attributes-target-role" name="target-role" value="Started"/>
</meta_attributes>
</primitive>
</group>
</resources>
<constraints>
<rsc_colocation id="coloc_all_on_drbd" rsc="grp_all" score="INFINITY" with-rsc="ms_drbd" with-rsc-role="Master"/>
<rsc_order first="ms_drbd" first-action="promote" id="order_all_after_drbd" score="INFINITY" then="grp_all" then-action="start"/>
<rsc_location id="loc_all_on_best_ping" rsc="grp_all">
<rule boolean-op="or" id="loc_all_on_best_ping-rule" score="-INFINITY">
<expression attribute="pingd" id="loc_all_on_best_ping-expression" operation="not_defined"/>
<expression attribute="pingd" id="loc_all_on_best_ping-expression-0" operation="lt" value="1000"/>
</rule>
</rsc_location>
</constraints>
</configuration>
<status>
<node_state id="test4" uname="test4" ha="active" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="do_update_resource" shutdown="0">
<transient_attributes id="test4">
<instance_attributes id="status-test4">
<nvpair id="status-test4-probe_complete" name="probe_complete" value="true"/>
<nvpair id="status-test4-master-drbd.0" name="master-drbd:0" value="10000"/>
<nvpair id="status-test4-pingd" name="pingd" value="1000"/>
</instance_attributes>
</transient_attributes>
<lrm id="test4">
<lrm_resources>
<lrm_resource id="pingtest:0" type="ping" class="ocf" provider="pacemaker">
<lrm_rsc_op id="pingtest:0_last_0" operation_key="pingtest:0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="5:90:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;5:90:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="149" rc-code="0" op-status="0" interval="0" op-digest="7d41580263769d86524e63413353cd0b" last-run="1370396943" last-rc-change="1370396943" exec-time="2080" queue-time="0"/>
<lrm_rsc_op id="pingtest:0_monitor_5000" operation_key="pingtest:0_monitor_5000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="7:91:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;7:91:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="152" rc-code="0" op-status="0" interval="5000" last-rc-change="1370396945" exec-time="2060" queue-time="0" op-digest="ea75753f1e9e0cac00432d83a812f4c8"/>
</lrm_resource>
<lrm_resource id="pingtest:1" type="ping" class="ocf" provider="pacemaker">
<lrm_rsc_op id="pingtest:1_last_0" operation_key="pingtest:1_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="8:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;8:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="141" rc-code="0" op-status="0" interval="0" op-digest="7d41580263769d86524e63413353cd0b"/>
</lrm_resource>
<lrm_resource id="fs_drbd" type="Filesystem" class="ocf" provider="heartbeat">
<lrm_rsc_op id="fs_drbd_last_0" operation_key="fs_drbd_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="42:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;42:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="158" rc-code="0" op-status="0" interval="0" op-digest="e0cde594906f5ecf8fa72d58000e0c45" last-run="1370396950" last-rc-change="1370396950" exec-time="110" queue-time="0"/>
<lrm_rsc_op id="fs_drbd_last_failure_0" operation_key="fs_drbd_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="7:76:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;7:76:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="132" rc-code="0" op-status="0" interval="0" op-digest="e0cde594906f5ecf8fa72d58000e0c45"/>
<lrm_rsc_op id="fs_drbd_monitor_5000" operation_key="fs_drbd_monitor_5000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="43:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;43:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="159" rc-code="0" op-status="0" interval="5000" last-rc-change="1370396950" exec-time="100" queue-time="0" op-digest="58701fb56792cdf0fbe729ee4f043a87"/>
</lrm_resource>
<lrm_resource id="apache" type="apache2" class="lsb">
<lrm_rsc_op id="apache_last_0" operation_key="apache_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="44:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;44:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="160" rc-code="0" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" last-run="1370396950" last-rc-change="1370396950" exec-time="50" queue-time="0"/>
<lrm_rsc_op id="apache_last_failure_0" operation_key="apache_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="7:81:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;7:81:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="138" rc-code="0" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
<lrm_rsc_op id="apache_monitor_10000" operation_key="apache_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="45:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;45:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="161" rc-code="0" op-status="0" interval="10000" last-rc-change="1370396950" exec-time="30" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
</lrm_resource>
<lrm_resource id="sip" type="IPaddr2" class="ocf" provider="heartbeat">
<lrm_rsc_op id="sip_last_0" operation_key="sip_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="40:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;40:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="156" rc-code="0" op-status="0" interval="0" op-digest="b009a03f7cd980a54dff5e570c8de74d" last-run="1370396950" last-rc-change="1370396950" exec-time="120" queue-time="0"/>
<lrm_rsc_op id="sip_monitor_10000" operation_key="sip_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="41:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;41:93:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="157" rc-code="0" op-status="0" interval="10000" last-rc-change="1370396950" exec-time="60" queue-time="0" op-digest="f92644ce891641f5d8a4019b289dbbd0"/>
</lrm_resource>
<lrm_resource id="drbd:0" type="drbd" class="ocf" provider="linbit">
<lrm_rsc_op id="drbd:0_last_0" operation_key="drbd:0_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="15:91:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;15:91:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="154" rc-code="0" op-status="0" interval="0" op-digest="f2655e5354fe9acd9270c82254b11693" last-run="1370396945" last-rc-change="1370396945" exec-time="70" queue-time="0"/>
<lrm_rsc_op id="drbd:0_last_failure_0" operation_key="drbd:0_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="8:59:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:8;8:59:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="120" rc-code="8" op-status="0" interval="0" op-digest="f2655e5354fe9acd9270c82254b11693"/>
</lrm_resource>
</lrm_resources>
</lrm>
</node_state>
<node_state id="backup3" uname="backup3" crmd="online" crm-debug-origin="do_update_resource" ha="active" in_ccm="true" join="member" expected="member" shutdown="0">
<lrm id="backup3">
<lrm_resources>
<lrm_resource id="drbd:0" type="drbd" class="ocf" provider="linbit">
<lrm_rsc_op id="drbd:0_last_failure_0" operation_key="drbd:0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="6:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:5;6:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="3" rc-code="5" op-status="0" interval="0" last-run="1370396962" last-rc-change="1370396962" exec-time="20" queue-time="0" op-digest="f2655e5354fe9acd9270c82254b11693"/>
</lrm_resource>
<lrm_resource id="pingtest:0" type="ping" class="ocf" provider="pacemaker">
<lrm_rsc_op id="pingtest:0_last_0" operation_key="pingtest:0_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="5:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:7;5:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1370396962" last-rc-change="1370396962" exec-time="60" queue-time="0" op-digest="7d41580263769d86524e63413353cd0b"/>
</lrm_resource>
<lrm_resource id="fs_drbd" type="Filesystem" class="ocf" provider="heartbeat">
<lrm_rsc_op id="fs_drbd_last_failure_0" operation_key="fs_drbd_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="8:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:5;8:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="5" rc-code="5" op-status="0" interval="0" last-run="1370396962" last-rc-change="1370396962" exec-time="120" queue-time="0" op-digest="e0cde594906f5ecf8fa72d58000e0c45"/>
</lrm_resource>
<lrm_resource id="sip" type="IPaddr2" class="ocf" provider="heartbeat">
<lrm_rsc_op id="sip_last_0" operation_key="sip_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="7:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:7;7:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="4" rc-code="7" op-status="0" interval="0" last-run="1370396962" last-rc-change="1370396962" exec-time="140" queue-time="0" op-digest="b009a03f7cd980a54dff5e570c8de74d"/>
</lrm_resource>
<lrm_resource id="apache" type="apache2" class="lsb">
<lrm_rsc_op id="apache_last_failure_0" operation_key="apache_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.6" transition-key="9:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:5;9:89:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="6" rc-code="5" op-status="0" interval="0" last-run="1370396964" last-rc-change="1370396964" exec-time="0" queue-time="1010" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
</lrm_resource>
</lrm_resources>
</lrm>
<transient_attributes id="backup3">
<instance_attributes id="status-backup3">
<nvpair id="status-backup3-probe_complete" name="probe_complete" value="true"/>
</instance_attributes>
</transient_attributes>
</node_state>
</status>
</cib>
The differences (without last-rc-change):
root at test4:~# diff cibadmin1 cibadmin2 | grep -v last-rc-change
1c1
< <cib epoch="66" num_updates="28" admin_epoch="0" validate-with="pacemaker-1.2" crm_feature_set="3.0.6" update-origin="test4" update-client="crmd" cib-last-written="Wed Jun 5 03:44:38 2013" have-quorum="1" dc-uuid="test4">
---
> <cib epoch="66" num_updates="81" admin_epoch="0" validate-with="pacemaker-1.2" crm_feature_set="3.0.6" update-origin="test4" update-client="crmd" cib-last-written="Wed Jun 5 03:44:38 2013" have-quorum="1" dc-uuid="test4">
101c101
< <node_state id="test4" uname="test4" ha="active" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="post_cache_update" shutdown="0">
---
> <node_state id="test4" uname="test4" ha="active" in_ccm="true" crmd="online" join="member" expected="member" crm-debug-origin="do_update_resource" shutdown="0">
104a105,106
> <nvpair id="status-test4-master-drbd.0" name="master-drbd:0" value="10000"/>
> <nvpair id="status-test4-pingd" name="pingd" value="1000"/>
110c112,113
< <lrm_rsc_op id="pingtest:0_last_0" operation_key="pingtest:0_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="8:15:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;8:15:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="48" rc-code="0" op-status="0" interval="0" op-digest="7d41580263769d86524e63413353cd0b"/>
---
113,114c116,126
< <lrm_rsc_op id="pingtest:1_monitor_5000" operation_key="pingtest:1_monitor_5000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="10:46:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;10:46:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="103" rc-code="0" op-status="0" interval="5000" op-digest="ea75753f1e9e0cac00432d83a812f4c8"/>
---
> <lrm_rsc_op id="pingtest:1_last_0" operation_key="pingtest:1_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="8:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;8:83:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="141" rc-code="0" op-status="0" interval="0" op-digest="7d41580263769d86524e63413353cd0b"/>
> </lrm_resource>
> <lrm_resource id="fs_drbd" type="Filesystem" class="ocf" provider="heartbeat">
> <lrm_rsc_op id="fs_drbd_last_failure_0" operation_key="fs_drbd_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="7:76:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;7:76:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="132" rc-code="0" op-status="0" interval="0" op-digest="e0cde594906f5ecf8fa72d58000e0c45"/>
> </lrm_resource>
> <lrm_resource id="apache" type="apache2" class="lsb">
> <lrm_rsc_op id="apache_last_failure_0" operation_key="apache_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="7:81:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;7:81:7:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="138" rc-code="0" op-status="0" interval="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
117,118c129,130
< <lrm_rsc_op id="sip_monitor_10000" operation_key="sip_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.6" transition-key="44:48:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" transition-magic="0:0;44:48:0:ca6fd6dd-bf3c-4b75-9214-26e3a36b2f1c" call-id="108" rc-code="0" op-status="0" interval="10000" op-digest="f92644ce891641f5d8a4019b289dbbd0"/>
---
120a133
122c135,146
---
> </lrm_resource>
> </lrm_resources>
> </lrm>
> </node_state>
> <node_state id="backup3" uname="backup3" crmd="online" crm-debug-origin="do_update_resource" ha="active" in_ccm="true" join="member" expected="member" shutdown="0">
> <lrm id="backup3">
> <lrm_resources>
> <lrm_resource id="drbd:0" type="drbd" class="ocf" provider="linbit">
> </lrm_resource>
> <lrm_resource id="pingtest:0" type="ping" class="ocf" provider="pacemaker">
125,127c149,152
---
> </lrm_resource>
> <lrm_resource id="sip" type="IPaddr2" class="ocf" provider="heartbeat">
130,132c155
---
135a159,163
> <transient_attributes id="backup3">
> <instance_attributes id="status-backup3">
> <nvpair id="status-backup3-probe_complete" name="probe_complete" value="true"/>
> </instance_attributes>
> </transient_attributes>
137d164
< <node_state id="backup3" uname="backup3" crmd="online" crm-debug-origin="post_cache_update" ha="active" in_ccm="true" join="member" expected="member" shutdown="0"/>
Best regards
Denis Witt
More information about the Pacemaker
mailing list