Diff hostcache... no /tmp/report/*/hostcache :/
Diff members.txt... OK
Diff cib.xml... <diff>
  <diff-removed admin_epoch="0" epoch="199" num_updates="42049">
    <cib num_updates="42049">
      <status>
        <node_state id="ha1.itactics.com">
          <transient_attributes id="ha1.itactics.com">
            <instance_attributes id="status-ha1.itactics.com">
              <nvpair value="4823" id="status-ha1.itactics.com-fail-count-ha2.itactics.com-stonith"/>
              <nvpair value="1286385930" id="status-ha1.itactics.com-last-failure-ha2.itactics.com-stonith"/>
            </instance_attributes>
          </transient_attributes>
          <lrm id="ha1.itactics.com">
            <lrm_resources>
              <lrm_resource id="ha2.itactics.com-stonith">
                <lrm_rsc_op exec-time="30" transition-key="2:8258:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" transition-magic="0:0;2:8258:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" call-id="14489" last-run="1286385931" last-rc-change="1286385931" id="ha2.itactics.com-stonith_stop_0"/>
                <lrm_rsc_op transition-key="5:8257:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" transition-magic="0:0;5:8257:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" call-id="14487" last-run="1286385928" last-rc-change="1286385928" id="ha2.itactics.com-stonith_start_0"/>
                <lrm_rsc_op transition-key="6:8257:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" transition-magic="0:1;6:8257:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" call-id="14488" last-run="1286385928" last-rc-change="1286385928" exec-time="1070" id="ha2.itactics.com-stonith_monitor_20000"/>
              </lrm_resource>
            </lrm_resources>
          </lrm>
        </node_state>
        <node_state id="ha2.itactics.com">
          <transient_attributes id="ha2.itactics.com">
            <instance_attributes id="status-ha2.itactics.com">
              <nvpair value="4085" id="status-ha2.itactics.com-fail-count-ha1.itactics.com-stonith"/>
              <nvpair value="1286385928" id="status-ha2.itactics.com-last-failure-ha1.itactics.com-stonith"/>
            </instance_attributes>
          </transient_attributes>
          <lrm id="ha2.itactics.com">
            <lrm_resources>
              <lrm_resource id="ha1.itactics.com-stonith">
                <lrm_rsc_op transition-key="4:8256:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" transition-magic="0:0;4:8256:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" call-id="12317" last-run="1286385928" last-rc-change="1286385928" exec-time="30" id="ha1.itactics.com-stonith_stop_0"/>
                <lrm_rsc_op transition-key="7:8257:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" transition-magic="0:0;7:8257:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" call-id="12318" last-run="1286385928" last-rc-change="1286385928" id="ha1.itactics.com-stonith_start_0"/>
                <lrm_rsc_op transition-key="3:8255:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" transition-magic="0:1;3:8255:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" call-id="12316" last-run="1286385926" last-rc-change="1286385926" exec-time="1050" id="ha1.itactics.com-stonith_monitor_20000"/>
              </lrm_resource>
            </lrm_resources>
          </lrm>
        </node_state>
      </status>
    </cib>
  </diff-removed>
  <diff-added admin_epoch="0" epoch="199" num_updates="42025">
    <cib num_updates="42025">
      <status>
        <node_state id="ha1.itactics.com">
          <transient_attributes id="ha1.itactics.com">
            <instance_attributes id="status-ha1.itactics.com">
              <nvpair value="4820" id="status-ha1.itactics.com-fail-count-ha2.itactics.com-stonith"/>
              <nvpair value="1286385926" id="status-ha1.itactics.com-last-failure-ha2.itactics.com-stonith"/>
            </instance_attributes>
          </transient_attributes>
          <lrm id="ha1.itactics.com">
            <lrm_resources>
              <lrm_resource id="ha2.itactics.com-stonith">
                <lrm_rsc_op transition-key="2:8253:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" transition-magic="0:0;2:8253:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" call-id="14480" last-run="1286385925" last-rc-change="1286385925" exec-time="20" id="ha2.itactics.com-stonith_stop_0"/>
                <lrm_rsc_op transition-key="5:8254:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" transition-magic="0:0;5:8254:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" call-id="14481" last-run="1286385925" last-rc-change="1286385925" id="ha2.itactics.com-stonith_start_0"/>
                <lrm_rsc_op transition-key="9:8252:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" transition-magic="0:1;9:8252:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" call-id="14479" last-run="1286385924" last-rc-change="1286385924" exec-time="1050" id="ha2.itactics.com-stonith_monitor_20000"/>
              </lrm_resource>
            </lrm_resources>
          </lrm>
        </node_state>
        <node_state id="ha2.itactics.com">
          <transient_attributes id="ha2.itactics.com">
            <instance_attributes id="status-ha2.itactics.com">
              <nvpair value="4083" id="status-ha2.itactics.com-fail-count-ha1.itactics.com-stonith"/>
              <nvpair value="1286385926" id="status-ha2.itactics.com-last-failure-ha1.itactics.com-stonith"/>
            </instance_attributes>
          </transient_attributes>
          <lrm id="ha2.itactics.com">
            <lrm_resources>
              <lrm_resource id="ha1.itactics.com-stonith">
                <lrm_rsc_op transition-key="4:8253:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" transition-magic="0:0;4:8253:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" call-id="12311" last-run="1286385925" last-rc-change="1286385925" exec-time="20" id="ha1.itactics.com-stonith_stop_0"/>
                <lrm_rsc_op transition-key="7:8254:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" transition-magic="0:0;7:8254:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" call-id="12312" last-run="1286385925" last-rc-change="1286385925" id="ha1.itactics.com-stonith_start_0"/>
                <lrm_rsc_op transition-key="1:8252:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" transition-magic="0:1;1:8252:0:6f0d6669-49f9-48b9-bb80-179ce76545ac" call-id="12310" last-run="1286385924" last-rc-change="1286385924" exec-time="1060" id="ha1.itactics.com-stonith_monitor_20000"/>
              </lrm_resource>
            </lrm_resources>
          </lrm>
        </node_state>
      </status>
    </cib>
  </diff-added>
</diff>
Diff crm_mon.txt... --- /tmp/report/ha1.itactics.com/crm_mon.txt	2010-10-06 13:25:26.000000000 -0400
+++ /tmp/report/ha2.itactics.com/crm_mon.txt	2010-10-06 13:25:32.000000000 -0400
@@ -8,9 +8,8 @@
 
 Online: [ ha2.itactics.com ha1.itactics.com ]
 
- ha2.itactics.com-stonith	(stonith:external/safe/ipmi):	Started ha1.itactics.com
  ha1.itactics.com-stonith	(stonith:external/safe/ipmi):	Started ha2.itactics.com
 
 Failed actions:
-    ha2.itactics.com-stonith_monitor_20000 (node=ha1.itactics.com, call=14479, rc=1, status=complete): unknown error
-    ha1.itactics.com-stonith_monitor_20000 (node=ha2.itactics.com, call=12310, rc=1, status=complete): unknown error
+    ha2.itactics.com-stonith_monitor_20000 (node=ha1.itactics.com, call=14488, rc=1, status=complete): unknown error
+    ha1.itactics.com-stonith_monitor_20000 (node=ha2.itactics.com, call=12316, rc=1, status=complete): unknown error

Diff corosync.conf... OK
Diff logd.cf... no /tmp/report/*/logd.cf :/
Diff sysinfo.txt... OK
WARN: problem with permissions/ownership at ha1.itactics.com:
wrong permissions or ownership for /var/lib/pengine:
drwxr-xr-x 2 hacluster haclient 15212544 Oct  6 13:25 /var/lib/pengine
wrong permissions or ownership for /var/lib/heartbeat/crm:
drwxr-xr-x 2 hacluster haclient 12288 Oct  6 10:56 /var/lib/heartbeat/crm
WARN: problem with permissions/ownership at ha2.itactics.com:
wrong permissions or ownership for /var/lib/pengine:
drwxr-xr-x 2 hacluster haclient 155648 Oct  6 10:30 /var/lib/pengine
wrong permissions or ownership for /var/lib/heartbeat/crm:
drwxr-xr-x 2 hacluster haclient 12288 Oct  6 10:56 /var/lib/heartbeat/crm

Log patterns:
