ubuntuusers.de

via DuckDuckGo

Corosync Pacemaker DRBD errror: Connection to the CPG API failed: Library error (2)

Status: Ungelöst | Ubuntu-Version: Server 14.04 (Trusty Tahr)
Antworten |

Kiess

Anmeldungsdatum:
13. November 2015

Beiträge: 1
Zitieren
13. November 2015 15:20

Hallo,

wir haben folgendes Situation:

Das ganze ist eine Teststellung für

  • einen Apache Webserver als reverseproxy

  • auf 2 Nodes (revproxy1 und revproxy2)

  • mit einem Linux HA Cluser Active/Passive (Corosync und Pacemaker)

  • mit DRBD primary/secondary (r0 /dev/drbd0 mountpoint /data) als root-Verzeichniss für Apache

  • einer failover_ip (abhängig von der Lokation der IP wird auf dem Node wo die IP ist DRBD R0 gemounted und dann der Apache gestartet)

  • Die Server sind Vms in einem VmWare ESX-Cluser

  • zum Test ist STONITH über ssh eingerichtet.

Jetzt das Problem:

In unregelmäßigen für mich nicht nachvollziehbaren Abständen hängt sich (so wie ich das Log deute) der crmd Dienst auf und startet neu. Was für Pacemaker der anlass ist einen Failover zu machen. Was ja aus der Sicht von Pacemaker auch richtig ist. Allerdings startet der Dienst so schnell wieder das ich gelegentlich 2 Primarys bei DRBD habe und somit ein Splitbrain. IP Failover und der Rest klappen trotzdem korrekt und STONITH dann auch aber wohl manchmal nicht schnell genug um den DRBD Splitbrain zu verhindern.

Meine Frage:

Fällt jemand was ein woran der error "Connection to the CPG API failed: Library error (2)" liegen könnte und wo ich noch nach fehlern suchen könnte. Mir fehlen jetzt die einfälle wo ich noch suchen kann.

Systeme:

  • Ubuntu 14.04.3 LTS (Server)

  • Apache/2.4.7 (Ubuntu)

  • Corosync Cluster Engine, version '2.3.3'

  • Pacemaker 1.1.10

  • DRBD_KERNEL_VERSION_CODE=0x080403

  • DRBDADM_VERSION=8.4.4

drbd cfg:

/etc/drbd.conf

1
2
3
4
# You can find an example in  /usr/share/doc/drbd.../drbd.conf.example

include "drbd.d/global_common.conf";
include "drbd.d/*.res";

/etc/drbd.d/global_common.conf

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
global  {
        usage-count no;
        }
common  {
        syncer {
                c-plan-ahead 10;
                c-min-rate 50M;
                c-max-rate 100M;
                c-fill-target 2M;
                verify-alg md5;
                al-extents 3389;
        }
        net {
                #ko-count 10;
                ko-count 0;
                max-buffers 128k;
                max-epoch-size 16000;
                sndbuf-size 8m;
                rcvbuf-size 8m;
                unplug-watermark 16001;
        }
        disk {
                no-md-flushes;
                no-disk-barrier;
                no-disk-flushes;
        }
}

/etc/drbd.d/drbd0.res

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
resource r0 {
        device /dev/drbd0;
        disk /dev/sdb1;
        meta-disk internal;
        protocol C;
        startup {
                wfc-timeout  120;
                degr-wfc-timeout 60;
        }
        on revproxy1 {
                device      /dev/drbd0;
                disk        /dev/sdb1;
                address     10.129.88.2:7789;
                meta-disk   internal;
        }

        on revproxy2 {
                device      /dev/drbd0;
                disk        /dev/sdb1;
                address     10.129.88.3:7789;
                meta-disk   internal;
        }
}

crm cfg:

crm configure show

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
node $id="176248834" revproxy1 \
        attributes standby="off"
node $id="176248835" revproxy2 \
        attributes standby="off"
primitive apache2_res ocf:heartbeat:apache \
        params configfile="/etc/apache2/apache2.conf" httpd="/usr/sbin/apache2ctl" \
        op monitor interval="60s" timeout="10s" \
        op start timeout="40s" interval="0" \
        op stop timeout="60s" interval="0" \
        meta target-role="started" is-managed="true"
primitive drbd_res ocf:linbit:drbd \
        params drbd_resource="r0" \
        op monitor interval="29s" role="Master" \
        op monitor interval="31s" role="Slave" \
        meta target-role="started" is-managed="true"
primitive failover_ip ocf:heartbeat:IPaddr2 \
        params ip="10.129.88.1" cidr_netmask="32" \
        op monitor interval="30s" \
        meta target-role="started" is-managed="true"
primitive fs_res ocf:heartbeat:Filesystem \
        params device="/dev/drbd0" directory="/data" fstype="ext4" \
        meta target-role="started" is-managed="true"
primitive resSTONITH1 stonith:external/ssh \
        params hostlist="revproxy1" \
        meta target-role="started" is-managed="true"
primitive resSTONITH2 stonith:external/ssh \
        params hostlist="revproxy2" \
        meta target-role="started" is-managed="true"
ms drbd_master_slave drbd_res \
        meta master-max="1" master-node-max="1" clone-max="2" clone-node-max="1" notify="true"
location locSTONITH1 resSTONITH1 -inf: revproxy1
location locSTONITH2 resSTONITH2 -inf: revproxy2
colocation apache2_fs_colo inf: apache2_res fs_res
colocation col_drbd_master_slave_apache2_res inf: apache2_res drbd_master_slave:Master
colocation col_failover_ip_apache2_res inf: apache2_res failover_ip
colocation col_fs_res_failover_ip inf: fs_res failover_ip
colocation fs_drbd_colo inf: fs_res drbd_master_slave:Master
order fs_after_drbd inf: drbd_master_slave:promote fs_res:start
order ord_drbd_master_slave_apache2_res inf: drbd_master_slave:promote apache2_res:start
order ord_failover_ip_apache2_res inf: failover_ip apache2_res
order ord_failover_ip_fs_res inf: failover_ip fs_res
order ord_fs_res_apache2_res inf: fs_res apache2_res
property $id="cib-bootstrap-options" \
        cluster-recheck-interval="1" \
        stonith-enabled="true" \
        no-quorum-policy="ignore" \
        dc-version="1.1.10-42f2063" \
        cluster-infrastructure="corosync" \
        last-lrm-refresh="1446742332"
rsc_defaults $id="rsc-options" \
        target-role="started"

Corosync cfg:

/etc/corosync/corosync.conf

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
totem {
        version: 2

        # How long before declaring a token lost (ms)
        #token: 3000
        token: 9000

        # How many token retransmits before forming a new configuration
        token_retransmits_before_loss_const: 10

        # How long to wait for join messages in the membership protocol (ms)
        join: 60

        # How long to wait for consensus to be achieved before starting a new round of membership configuration (ms)
        consensus: 3600

        # Turn off the virtual synchrony filter
        vsftype: none

        # Number of messages that may be sent by one processor on receipt of the token
        max_messages: 20

        # Limit generated nodeids to 31-bits (positive signed integers)
        clear_node_high_bit: yes

        # Disable encryption
        secauth: off

        # How many threads to use for encryption/decryption
        threads: 0

        # Optionally assign a fixed node id (integer)
        # nodeid: 1234

        # This specifies the mode of redundant ring, which may be none, active, or passive.
        rrp_mode: none

        interface {
                # The following values need to be set based on your environment
                ringnumber: 0
                bindnetaddr: 10.129.88.0
                mcastaddr: 226.94.1.1
                mcastport: 5405
        }
}

amf {
        mode: disabled
}

quorum {
        # Quorum for the Pacemaker Cluster Resource Manager
        provider: corosync_votequorum
        expected_votes: 1
}

aisexec {
        user:   root
        group:  root
}

logging {
        fileline: off
        to_stderr: yes
        to_logfile: yes
        logfile: /var/log/corosync/corosync.log
        to_syslog: yes
        syslog_facility: daemon
        #debug: off
        debug: on
        timestamp: on
        logger_subsys {
                subsys: AMF
                debug: off
                tags: enter|leave|trace1|trace2|trace3|trace4|trace6
        }
}

Corosync log:

/var/log/corosync/corosync.log

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [QB    ] IPC credentials authenticated (1041-27391-27)
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [QB    ] connecting to client [27391]
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [MAIN  ] connection created
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [QB    ] lib_init_fn: conn=0x7fc5d4a713a0
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [QB    ] HUP conn (1041-27391-27)
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [QB    ] qb_ipcs_disconnect(1041-27391-27) state:2
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [QB    ] epoll_ctl(del): Bad file descriptor (9)
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_closed()
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [QB    ] exit_fn for conn=0x7fc5d4a713a0
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_destroyed()
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-response-1041-27391-27-header
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-event-1041-27391-27-header
Nov 12 16:03:03 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-request-1041-27391-27-header
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [QB    ] IPC credentials authenticated (1041-27663-27)
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [QB    ] connecting to client [27663]
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [MAIN  ] connection created
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [QB    ] lib_init_fn: conn=0x7fc5d4a713a0
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [QB    ] HUP conn (1041-27663-27)
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [QB    ] qb_ipcs_disconnect(1041-27663-27) state:2
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [QB    ] epoll_ctl(del): Bad file descriptor (9)
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_closed()
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [QB    ] exit_fn for conn=0x7fc5d4a713a0
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_destroyed()
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-response-1041-27663-27-header
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-event-1041-27663-27-header
Nov 12 16:03:05 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-request-1041-27663-27-header
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [QB    ] IPC credentials authenticated (1041-27938-27)
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [QB    ] connecting to client [27938]
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [MAIN  ] connection created
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [QB    ] lib_init_fn: conn=0x7fc5d4a713a0
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [QB    ] HUP conn (1041-27938-27)
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [QB    ] qb_ipcs_disconnect(1041-27938-27) state:2
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [QB    ] epoll_ctl(del): Bad file descriptor (9)
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_closed()
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [QB    ] exit_fn for conn=0x7fc5d4a713a0
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_destroyed()
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-response-1041-27938-27-header
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-event-1041-27938-27-header
Nov 12 16:03:13 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-request-1041-27938-27-header
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [QB    ] IPC credentials authenticated (1041-28210-27)
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [QB    ] connecting to client [28210]
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [MAIN  ] connection created
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [QB    ] lib_init_fn: conn=0x7fc5d4a713a0
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [QB    ] HUP conn (1041-28210-27)
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [QB    ] qb_ipcs_disconnect(1041-28210-27) state:2
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [QB    ] epoll_ctl(del): Bad file descriptor (9)
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_closed()
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [QB    ] exit_fn for conn=0x7fc5d4a713a0
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_destroyed()
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-response-1041-28210-27-header
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-event-1041-28210-27-header
Nov 12 16:03:16 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-request-1041-28210-27-header
Nov 12 16:03:23 [1239] revproxy1       lrmd:    debug: recurring_action_timer:  Scheduling another invokation of drbd_res_monitor_31000
Nov 12 16:03:23 [28340] revproxy1   crm_node:     info: get_cluster_type:       Verifying cluster type: 'corosync'
Nov 12 16:03:23 [28340] revproxy1   crm_node:     info: get_cluster_type:       Assuming an active 'corosync' cluster
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] IPC credentials authenticated (1041-28340-27)
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] connecting to client [28340]
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [MAIN  ] connection created
Nov 12 16:03:23 [28340] revproxy1   crm_node:    debug: qb_rb_open_2:   shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [CPG   ] lib_init_fn: conn=0x7fc5d4a713a0, cpd=0x7fc5d4a70e34
Nov 12 16:03:23 [28340] revproxy1   crm_node:    debug: qb_rb_open_2:   shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:23 [28340] revproxy1   crm_node:    debug: qb_rb_open_2:   shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [CPG   ] cpg finalize for conn=0x7fc5d4a713a0
Nov 12 16:03:23 [28340] revproxy1   crm_node:    debug: qb_ipcc_disconnect:     qb_ipcc_disconnect()
Nov 12 16:03:23 [28340] revproxy1   crm_node:    debug: qb_rb_close:    Closing ringbuffer: /dev/shm/qb-cpg-request-1041-28340-27-header
Nov 12 16:03:23 [28340] revproxy1   crm_node:    debug: qb_rb_close:    Closing ringbuffer: /dev/shm/qb-cpg-response-1041-28340-27-header
Nov 12 16:03:23 [28340] revproxy1   crm_node:    debug: qb_rb_close:    Closing ringbuffer: /dev/shm/qb-cpg-event-1041-28340-27-header
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] HUP conn (1041-28340-27)
Nov 12 16:03:23 [28340] revproxy1   crm_node:    debug: get_local_nodeid:       Local nodeid is 176248834
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] qb_ipcs_disconnect(1041-28340-27) state:2
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] epoll_ctl(del): Bad file descriptor (9)
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_closed()
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [CPG   ] exit_fn for conn=0x7fc5d4a713a0
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_destroyed()
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cpg-response-1041-28340-27-header
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cpg-event-1041-28340-27-header
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cpg-request-1041-28340-27-header
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] IPC credentials authenticated (1041-28340-27)
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] connecting to client [28340]
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [MAIN  ] connection created
Nov 12 16:03:23 [28340] revproxy1   crm_node:    debug: qb_rb_open_2:   shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] lib_init_fn: conn=0x7fc5d4a713a0
Nov 12 16:03:23 [28340] revproxy1   crm_node:    debug: qb_rb_open_2:   shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:23 [28340] revproxy1   crm_node:    debug: qb_rb_open_2:   shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:23 [28340] revproxy1   crm_node:    debug: qb_ipcc_disconnect:     qb_ipcc_disconnect()
Nov 12 16:03:23 [28340] revproxy1   crm_node:    debug: qb_rb_close:    Closing ringbuffer: /dev/shm/qb-cmap-request-1041-28340-27-header
Nov 12 16:03:23 [28340] revproxy1   crm_node:    debug: qb_rb_close:    Closing ringbuffer: /dev/shm/qb-cmap-response-1041-28340-27-header
Nov 12 16:03:23 [28340] revproxy1   crm_node:    debug: qb_rb_close:    Closing ringbuffer: /dev/shm/qb-cmap-event-1041-28340-27-header
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] HUP conn (1041-28340-27)
Nov 12 16:03:23 [28340] revproxy1   crm_node:   notice: corosync_node_name:     Unable to get node name for nodeid 176248834
Nov 12 16:03:23 [28340] revproxy1   crm_node:   notice: get_node_name:  Defaulting to uname -n for the local corosync node name
Nov 12 16:03:23 [28340] revproxy1   crm_node:     info: crm_xml_cleanup:        Cleaning up memory from libxml2
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] qb_ipcs_disconnect(1041-28340-27) state:2
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] epoll_ctl(del): Bad file descriptor (9)
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_closed()
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] exit_fn for conn=0x7fc5d4a713a0
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_destroyed()
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-response-1041-28340-27-header
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-event-1041-28340-27-header
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-request-1041-28340-27-header
Nov 12 16:03:23 [1237] revproxy1        cib:     info: crm_client_new:  Connecting 0x7fb404f8a620 for uid=0 gid=0 pid=28341 id=8b33a4d4-6341-470f-be3c-f2e889b00f59
Nov 12 16:03:23 [1237] revproxy1        cib:    debug: handle_new_connection:   IPC credentials authenticated (1237-28341-15)
Nov 12 16:03:23 [1237] revproxy1        cib:    debug: qb_ipcs_shm_connect:     connecting to client [28341]
Nov 12 16:03:23 [1237] revproxy1        cib:    debug: qb_rb_open_2:    shm size:524301; real_size:528384; rb->word_size:132096
Nov 12 16:03:23 [1237] revproxy1        cib:    debug: qb_rb_open_2:    shm size:524301; real_size:528384; rb->word_size:132096
Nov 12 16:03:23 [1237] revproxy1        cib:    debug: qb_rb_open_2:    shm size:524301; real_size:528384; rb->word_size:132096
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: qb_rb_open_2:        shm size:524301; real_size:528384; rb->word_size:132096
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: qb_rb_open_2:        shm size:524301; real_size:528384; rb->word_size:132096
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: qb_rb_open_2:        shm size:524301; real_size:528384; rb->word_size:132096
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: cib_native_signon_raw:       Connection to CIB successful
Nov 12 16:03:23 [1237] revproxy1        cib:     info: cib_process_request:     Completed cib_query operation for section nodes: OK (rc=0, origin=local/crm_attribute/2, version=0.605.34)
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: get_cluster_node_uuid:       Result section   <nodes>
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: get_cluster_node_uuid:       Result section     <node uname="revproxy1" id="176248834">
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: get_cluster_node_uuid:       Result section       <instance_attributes id="revproxy1-instance_attributes">
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: get_cluster_node_uuid:       Result section         <nvpair name="standby" value="off" id="revproxy1-instance_attributes-standby"/>
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: get_cluster_node_uuid:       Result section       </instance_attributes>
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: get_cluster_node_uuid:       Result section     </node>
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: get_cluster_node_uuid:       Result section     <node id="176248835" uname="revproxy2">
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: get_cluster_node_uuid:       Result section       <instance_attributes id="nodes-176248835">
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: get_cluster_node_uuid:       Result section         <nvpair id="nodes-176248835-standby" name="standby" value="off"/>
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: get_cluster_node_uuid:       Result section       </instance_attributes>
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: get_cluster_node_uuid:       Result section     </node>
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: get_cluster_node_uuid:       Result section   </nodes>
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:     info: query_node_uuid:     Mapped revproxy1 to 176248834
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:     info: attrd_update_delegate:       Connecting to cluster... 5 retries remaining
Nov 12 16:03:23 [1240] revproxy1      attrd:     info: crm_client_new:  Connecting 0x7f5cebd0f7f0 for uid=0 gid=0 pid=28341 id=21167e3b-9e73-43f9-8791-43dcdc612dc5
Nov 12 16:03:23 [1240] revproxy1      attrd:    debug: handle_new_connection:   IPC credentials authenticated (1240-28341-11)
Nov 12 16:03:23 [1240] revproxy1      attrd:    debug: qb_ipcs_shm_connect:     connecting to client [28341]
Nov 12 16:03:23 [1240] revproxy1      attrd:    debug: qb_rb_open_2:    shm size:51213; real_size:53248; rb->word_size:13312
Nov 12 16:03:23 [1240] revproxy1      attrd:    debug: qb_rb_open_2:    shm size:51213; real_size:53248; rb->word_size:13312
Nov 12 16:03:23 [1240] revproxy1      attrd:    debug: qb_rb_open_2:    shm size:51213; real_size:53248; rb->word_size:13312
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: qb_rb_open_2:        shm size:51213; real_size:53248; rb->word_size:13312
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: qb_rb_open_2:        shm size:51213; real_size:53248; rb->word_size:13312
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: qb_rb_open_2:        shm size:51213; real_size:53248; rb->word_size:13312
Nov 12 16:03:23 [1240] revproxy1      attrd:    debug: attrd_local_callback:    update message from crm_attribute: master-drbd_res=1000
Nov 12 16:03:23 [1240] revproxy1      attrd:    debug: attrd_local_callback:    Supplied: 1000, Current: 1000, Stored: 1000
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: attrd_update_delegate:       Sent update: master-drbd_res=1000 for revproxy1
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:     info: main:        Update master-drbd_res=1000 sent via attrd
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: cib_native_signoff:  Signing out of the CIB Service
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: qb_ipcc_disconnect:  qb_ipcc_disconnect()
Nov 12 16:03:23 [1237] revproxy1        cib:    debug: qb_ipcs_dispatch_connection_request:     HUP conn (1237-28341-15)
Nov 12 16:03:23 [1237] revproxy1        cib:    debug: qb_ipcs_disconnect:      qb_ipcs_disconnect(1237-28341-15) state:2
Nov 12 16:03:23 [1237] revproxy1        cib:     info: crm_client_destroy:      Destroying 0 events
Nov 12 16:03:23 [1237] revproxy1        cib:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-cib_rw-response-1237-28341-15-header
Nov 12 16:03:23 [1237] revproxy1        cib:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-cib_rw-event-1237-28341-15-header
Nov 12 16:03:23 [1237] revproxy1        cib:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-cib_rw-request-1237-28341-15-header
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: qb_rb_close:         Closing ringbuffer: /dev/shm/qb-cib_rw-request-1237-28341-15-header
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: qb_rb_close:         Closing ringbuffer: /dev/shm/qb-cib_rw-response-1237-28341-15-header
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:    debug: qb_rb_close:         Closing ringbuffer: /dev/shm/qb-cib_rw-event-1237-28341-15-header
Nov 12 16:03:23 [28341] revproxy1 crm_attribute:     info: crm_xml_cleanup:     Cleaning up memory from libxml2
Nov 12 16:03:23 [1240] revproxy1      attrd:    debug: qb_ipcs_dispatch_connection_request:     HUP conn (1240-28341-11)
Nov 12 16:03:23 [1240] revproxy1      attrd:    debug: qb_ipcs_disconnect:      qb_ipcs_disconnect(1240-28341-11) state:2
Nov 12 16:03:23 [1240] revproxy1      attrd:     info: crm_client_destroy:      Destroying 0 events
Nov 12 16:03:23 [1240] revproxy1      attrd:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-attrd-response-1240-28341-11-header
Nov 12 16:03:23 [1240] revproxy1      attrd:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-attrd-event-1240-28341-11-header
Nov 12 16:03:23 [1240] revproxy1      attrd:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-attrd-request-1240-28341-11-header
Nov 12 16:03:23 [1239] revproxy1       lrmd:    debug: operation_finished:      drbd_res_monitor_31000:28311 - exited with rc=0
Nov 12 16:03:23 [1239] revproxy1       lrmd:    debug: operation_finished:      drbd_res_monitor_31000:28311:stderr [ -- empty -- ]
Nov 12 16:03:23 [1239] revproxy1       lrmd:    debug: operation_finished:      drbd_res_monitor_31000:28311:stdout [  ]
Nov 12 16:03:23 [1239] revproxy1       lrmd:    debug: log_finished:    finished - rsc:drbd_res action:monitor call_id:43 pid:28311 exit-code:0 exec-time:0ms queue-time:0ms
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] IPC credentials authenticated (1041-28524-27)
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] connecting to client [28524]
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [MAIN  ] connection created
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] lib_init_fn: conn=0x7fc5d4a713a0
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] HUP conn (1041-28524-27)
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] qb_ipcs_disconnect(1041-28524-27) state:2
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] epoll_ctl(del): Bad file descriptor (9)
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_closed()
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] exit_fn for conn=0x7fc5d4a713a0
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_destroyed()
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-response-1041-28524-27-header
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-event-1041-28524-27-header
Nov 12 16:03:23 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-request-1041-28524-27-header
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [QB    ] IPC credentials authenticated (1041-28795-27)
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [QB    ] connecting to client [28795]
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [MAIN  ] connection created
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [QB    ] lib_init_fn: conn=0x7fc5d4a713a0
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [QB    ] HUP conn (1041-28795-27)
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [QB    ] qb_ipcs_disconnect(1041-28795-27) state:2
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [QB    ] epoll_ctl(del): Bad file descriptor (9)
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_closed()
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [QB    ] exit_fn for conn=0x7fc5d4a713a0
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_destroyed()
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-response-1041-28795-27-header
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-event-1041-28795-27-header
Nov 12 16:03:26 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-request-1041-28795-27-header
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [QB    ] IPC credentials authenticated (1041-29085-27)
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [QB    ] connecting to client [29085]
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [MAIN  ] connection created
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [QB    ] lib_init_fn: conn=0x7fc5d4a713a0
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [QB    ] HUP conn (1041-29085-27)
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [QB    ] qb_ipcs_disconnect(1041-29085-27) state:2
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [QB    ] epoll_ctl(del): Bad file descriptor (9)
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_closed()
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [QB    ] exit_fn for conn=0x7fc5d4a713a0
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_destroyed()
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-response-1041-29085-27-header
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-event-1041-29085-27-header
Nov 12 16:03:33 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-request-1041-29085-27-header
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [QB    ] IPC credentials authenticated (1041-29395-27)
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [QB    ] connecting to client [29395]
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [QB    ] shm size:1048589; real_size:1052672; rb->word_size:263168
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [MAIN  ] connection created
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [QB    ] lib_init_fn: conn=0x7fc5d4a713a0
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [QB    ] HUP conn (1041-29395-27)
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [QB    ] qb_ipcs_disconnect(1041-29395-27) state:2
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [QB    ] epoll_ctl(del): Bad file descriptor (9)
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_closed()
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [QB    ] exit_fn for conn=0x7fc5d4a713a0
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [MAIN  ] cs_ipcs_connection_destroyed()
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-response-1041-29395-27-header
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-event-1041-29395-27-header
Nov 12 16:03:36 [1018] revproxy1 corosync debug   [QB    ] Free'ing ringbuffer: /dev/shm/qb-cmap-request-1041-29395-27-header
Nov 12 16:03:44 [1240] revproxy1      attrd:    debug: qb_ipc_us_ready:         poll(fd 6) got POLLHUP
Nov 12 16:03:44 [1240] revproxy1      attrd:    debug: _check_connection_state_with:    interpreting result -107 (from socket) as a disconnect: Transport endpoint is not connected (107)
Nov 12 16:03:44 [1240] revproxy1      attrd:    error: pcmk_cpg_dispatch:       Connection to the CPG API failed: Library error (2)
Nov 12 16:03:44 [1240] revproxy1      attrd:     crit: attrd_cs_destroy:        Lost connection to Corosync service!
Nov 12 16:03:44 [1240] revproxy1      attrd:   notice: main:    Exiting...
Nov 12 16:03:44 [1240] revproxy1      attrd:   notice: main:    Disconnecting client 0x7f5cebcff4d0, pid=1242...
Nov 12 16:03:44 [1240] revproxy1      attrd:    debug: qb_ipcs_disconnect:      qb_ipcs_disconnect(1240-1242-10) state:2
Nov 12 16:03:44 [1240] revproxy1      attrd:     info: crm_client_destroy:      Destroying 0 events
Nov 12 16:03:44 [1240] revproxy1      attrd:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-attrd-response-1240-1242-10-header
Nov 12 16:03:44 [1240] revproxy1      attrd:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-attrd-event-1240-1242-10-header
Nov 12 16:03:44 [1240] revproxy1      attrd:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-attrd-request-1240-1242-10-header
Nov 12 16:03:44 [1240] revproxy1      attrd:     info: qb_ipcs_us_withdraw:     withdrawing server sockets
Nov 12 16:03:44 [1240] revproxy1      attrd:    debug: qb_ipcs_unref:   qb_ipcs_unref() - destroying
Nov 12 16:03:44 [1240] revproxy1      attrd:    debug: cib_native_signoff:      Signing out of the CIB Service
Nov 12 16:03:44 [1240] revproxy1      attrd:    debug: qb_ipcc_disconnect:      qb_ipcc_disconnect()
Nov 12 16:03:44 [1240] revproxy1      attrd:    debug: qb_rb_close:     Closing ringbuffer: /dev/shm/qb-cib_rw-request-1237-1240-13-header
Nov 12 16:03:44 [1240] revproxy1      attrd:    debug: qb_rb_close:     Closing ringbuffer: /dev/shm/qb-cib_rw-response-1237-1240-13-header
Nov 12 16:03:44 [1240] revproxy1      attrd:    debug: qb_rb_close:     Closing ringbuffer: /dev/shm/qb-cib_rw-event-1237-1240-13-header
Nov 12 16:03:44 [1240] revproxy1      attrd:    debug: cib_native_signoff:      Signing out of the CIB Service
Nov 12 16:03:44 [1240] revproxy1      attrd:    error: attrd_cib_connection_destroy:    Connection to the CIB terminated...
Nov 12 16:03:44 [1240] revproxy1      attrd:     info: crm_xml_cleanup:         Cleaning up memory from libxml2
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: qb_ipc_us_ready:         poll(fd 9) got POLLHUP
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: _check_connection_state_with:    interpreting result -107 (from socket) as a disconnect: Transport endpoint is not connected (107)
Nov 12 16:03:44 [1242] revproxy1       crmd:    error: crmd_quorum_destroy:     connection terminated
Nov 12 16:03:44 [1242] revproxy1       crmd:     info: qb_ipcs_us_withdraw:     withdrawing server sockets
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: qb_ipcs_unref:   qb_ipcs_unref() - destroying
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: qb_ipcc_disconnect:      qb_ipcc_disconnect()
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: qb_ipc_us_ready:         poll(fd 13) got POLLHUP
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: _check_connection_state_with:    interpreting result -107 (from socket) as a disconnect: Transport endpoint is not connected (107)
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: qb_rb_force_close:       Force free'ing ringbuffer: /dev/shm/qb-attrd-request-1240-1242-10-data: No such file or directory (2)
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: qb_rb_force_close:       Force free'ing ringbuffer: /dev/shm/qb-attrd-request-1240-1242-10-header: No such file or directory (2)
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: qb_rb_force_close:       Force free'ing ringbuffer: /dev/shm/qb-attrd-response-1240-1242-10-data: No such file or directory (2)
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: qb_rb_force_close:       Force free'ing ringbuffer: /dev/shm/qb-attrd-response-1240-1242-10-header: No such file or directory (2)
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: qb_rb_force_close:       Force free'ing ringbuffer: /dev/shm/qb-attrd-event-1240-1242-10-data: No such file or directory (2)
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: qb_rb_force_close:       Force free'ing ringbuffer: /dev/shm/qb-attrd-event-1240-1242-10-header: No such file or directory (2)
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: stonith_api_signoff:     Signing out of the STONITH Service
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: qb_ipcc_disconnect:      qb_ipcc_disconnect()
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: qb_rb_close:     Closing ringbuffer: /dev/shm/qb-stonith-ng-request-1238-1242-10-header
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: qb_rb_close:     Closing ringbuffer: /dev/shm/qb-stonith-ng-response-1238-1242-10-header
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: qb_rb_close:     Closing ringbuffer: /dev/shm/qb-stonith-ng-event-1238-1242-10-header
Nov 12 16:03:44 [1242] revproxy1       crmd:    debug: get_xpath_object:        No match for //st_notify_disconnect in /notify
Nov 12 16:03:44 [1242] revproxy1       crmd:     info: tengine_stonith_connection_destroy:      Fencing daemon disconnected
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: qb_ipc_us_ready:         poll(fd 6) got POLLHUP
Nov 12 16:03:44 [1242] revproxy1       crmd:   notice: crmd_exit:       Forcing immediate exit: Link has been severed (67)
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: _check_connection_state_with:    interpreting result -107 (from socket) as a disconnect: Transport endpoint is not connected (107)
Nov 12 16:03:44 [1242] revproxy1       crmd:     info: crm_xml_cleanup:         Cleaning up memory from libxml2
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    error: pcmk_cpg_dispatch:       Connection to the CPG API failed: Library error (2)
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    error: stonith_peer_cs_destroy:         Corosync connection terminated
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:     info: stonith_shutdown:        Terminating with  2 clients
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: cib_native_signoff:      Signing out of the CIB Service
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: qb_ipcc_disconnect:      qb_ipcc_disconnect()
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: qb_rb_close:     Closing ringbuffer: /dev/shm/qb-cib_rw-request-1237-1238-12-header
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: qb_rb_close:     Closing ringbuffer: /dev/shm/qb-cib_rw-response-1237-1238-12-header
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: qb_rb_close:     Closing ringbuffer: /dev/shm/qb-cib_rw-event-1237-1238-12-header
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:     info: cib_connection_destroy:  Connection to the CIB closed.
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: qb_ipcs_disconnect:      qb_ipcs_disconnect(1238-1239-11) state:2
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:     info: crm_client_destroy:      Destroying 0 events
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-stonith-ng-response-1238-1239-11-header
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-stonith-ng-event-1238-1239-11-header
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-stonith-ng-request-1238-1239-11-header
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: qb_ipcs_disconnect:      qb_ipcs_disconnect(1238-1242-10) state:2
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:     info: crm_client_destroy:      Destroying 0 events
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-stonith-ng-response-1238-1242-10-header
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-stonith-ng-event-1238-1242-10-header
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-stonith-ng-request-1238-1242-10-header
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:     info: qb_ipcs_us_withdraw:     withdrawing server sockets
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:    debug: qb_ipcs_unref:   qb_ipcs_unref() - destroying
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:     info: main:    Done
Nov 12 16:03:44 [1238] revproxy1 stonith-ng:     info: crm_xml_cleanup:         Cleaning up memory from libxml2
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:    error: pcmk_child_exit:         Child process attrd (1240) exited: Transport endpoint is not connected (107)
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:    debug: update_node_processes:   Node revproxy1 now has process list: 00000000000000000000000000110310 (was 00000000000000000000000000111310)
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_ipc_us_ready:         poll(fd 6) got POLLHUP
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: _check_connection_state_with:    interpreting result -107 (from socket) as a disconnect: Transport endpoint is not connected (107)
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:    debug: _check_connection_state:         interpreting result -107 as a disconnect: Transport endpoint is not connected (107)
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:     info: crm_cs_flush:    Sent 0 CPG messages  (1 remaining, last=8): Library error (2)
Nov 12 16:03:44 [1237] revproxy1        cib:    error: pcmk_cpg_dispatch:       Connection to the CPG API failed: Library error (2)
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: qb_ipc_us_ready:         poll(fd 16) got POLLHUP
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: _check_connection_state_with:    interpreting result -107 (from socket) as a disconnect: Transport endpoint is not connected (107)
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: qb_ipc_us_ready:         poll(fd 16) got POLLHUP
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: _check_connection_state_with:    interpreting result -107 (from socket) as a disconnect: Transport endpoint is not connected (107)
Nov 12 16:03:44 [1239] revproxy1       lrmd:    error: crm_ipc_read:    Connection to stonith-ng failed
Nov 12 16:03:44 [1239] revproxy1       lrmd:    error: mainloop_gio_callback:   Connection to stonith-ng[0x7f6b1cf8fa90] closed (I/O condition=17)
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: qb_ipcc_disconnect:      qb_ipcc_disconnect()
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: qb_ipc_us_ready:         poll(fd 16) got POLLHUP
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: _check_connection_state_with:    interpreting result -107 (from socket) as a disconnect: Transport endpoint is not connected (107)
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: qb_rb_force_close:       Force free'ing ringbuffer: /dev/shm/qb-stonith-ng-request-1238-1239-11-data: No such file or directory (2)
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: qb_rb_force_close:       Force free'ing ringbuffer: /dev/shm/qb-stonith-ng-request-1238-1239-11-header: No such file or directory (2)
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: qb_rb_force_close:       Force free'ing ringbuffer: /dev/shm/qb-stonith-ng-response-1238-1239-11-data: No such file or directory (2)
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: qb_rb_force_close:       Force free'ing ringbuffer: /dev/shm/qb-stonith-ng-response-1238-1239-11-header: No such file or directory (2)
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: qb_rb_force_close:       Force free'ing ringbuffer: /dev/shm/qb-stonith-ng-event-1238-1239-11-data: No such file or directory (2)
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: qb_rb_force_close:       Force free'ing ringbuffer: /dev/shm/qb-stonith-ng-event-1238-1239-11-header: No such file or directory (2)
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: get_xpath_object:        No match for //st_notify_disconnect in /notify
Nov 12 16:03:44 [1239] revproxy1       lrmd:    error: stonith_connection_destroy_cb:   LRMD lost STONITH connection
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: qb_ipcs_dispatch_connection_request:     HUP conn (1239-1242-7)
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: qb_ipcs_disconnect:      qb_ipcs_disconnect(1239-1242-7) state:2
Nov 12 16:03:44 [1239] revproxy1       lrmd:     info: cancel_recurring_action:         Cancelling operation drbd_res_monitor_31000
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: log_finished:    finished - rsc:drbd_res action:monitor call_id:43  exit-code:0 exec-time:0ms queue-time:0ms
Nov 12 16:03:44 [1239] revproxy1       lrmd:  warning: qb_ipcs_event_sendv:     new_event_notification (1239-1242-7): Bad file descriptor (9)
Nov 12 16:03:44 [1239] revproxy1       lrmd:  warning: send_client_notify:      Notification of client crmd/47a499c5-6476-4eec-9d1d-01ffce296a34 failed
Nov 12 16:03:44 [1239] revproxy1       lrmd:     info: crm_client_destroy:      Destroying 1 events
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-lrmd-response-1239-1242-7-header
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-lrmd-event-1239-1242-7-header
Nov 12 16:03:44 [1239] revproxy1       lrmd:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-lrmd-request-1239-1242-7-header
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:   notice: pcmk_process_exit:       Respawning failed child process: attrd
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:     info: start_child:     Using uid=108 and group=117 for process attrd
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:     info: start_child:     Forked child 29695 for process attrd
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:    debug: update_node_processes:   Node revproxy1 now has process list: 00000000000000000000000000111310 (was 00000000000000000000000000110310)
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:    error: pcmk_child_exit:         Child process crmd (1242) exited: Link has been severed (67)
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:    debug: update_node_processes:   Node revproxy1 now has process list: 00000000000000000000000000111110 (was 00000000000000000000000000111310)
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:   notice: pcmk_process_exit:       Respawning failed child process: crmd
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:     info: start_child:     Using uid=108 and group=117 for process crmd
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:     info: start_child:     Forked child 29696 for process crmd
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:    debug: update_node_processes:   Node revproxy1 now has process list: 00000000000000000000000000111310 (was 00000000000000000000000000111110)
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:     info: pcmk_child_exit:         Child process stonith-ng (1238) exited: OK (0)
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:    debug: update_node_processes:   Node revproxy1 now has process list: 00000000000000000000000000011310 (was 00000000000000000000000000111310)
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:   notice: pcmk_process_exit:       Respawning failed child process: stonith-ng
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:     info: start_child:     Forked child 29697 for process stonith-ng
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:    debug: update_node_processes:   Node revproxy1 now has process list: 00000000000000000000000000111310 (was 00000000000000000000000000011310)
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:    debug: qb_ipc_us_ready:         poll(fd 11) got POLLHUP
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:    debug: _check_connection_state_with:    interpreting result -107 (from socket) as a disconnect: Transport endpoint is not connected (107)
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:     info: mcp_quorum_destroy:      connection closed
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:    debug: qb_ipc_us_ready:         poll(fd 9) got POLLHUP
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:    debug: _check_connection_state_with:    interpreting result -107 (from socket) as a disconnect: Transport endpoint is not connected (107)
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:    error: pcmk_cpg_dispatch:       Connection to the CPG API failed: Library error (2)
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:    error: mcp_cpg_destroy:         Connection destroyed
Nov 12 16:03:44 [1126] revproxy1 pacemakerd:     info: crm_xml_cleanup:         Cleaning up memory from libxml2
Nov 12 16:03:44 [1237] revproxy1        cib:    error: cib_cs_destroy:  Corosync connection lost!  Exiting.
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: uninitializeCib:         Deallocating the CIB.
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: uninitializeCib:         The CIB has been deallocated.
Nov 12 16:03:44 [1237] revproxy1        cib:     info: terminate_cib:   cib_cs_destroy: Exiting fast...
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_ipcs_disconnect:      qb_ipcs_disconnect(1237-1977-14) state:2
Nov 12 16:03:44 [1237] revproxy1        cib:     info: crm_client_destroy:      Destroying 0 events
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-cib_ro-response-1237-1977-14-header
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-cib_ro-event-1237-1977-14-header
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-cib_ro-request-1237-1977-14-header
Nov 12 16:03:44 [1237] revproxy1        cib:     info: qb_ipcs_us_withdraw:     withdrawing server sockets
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_ipcs_unref:   qb_ipcs_unref() - destroying
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_ipcs_disconnect:      qb_ipcs_disconnect(1237-1240-13) state:2
Nov 12 16:03:44 [1237] revproxy1        cib:     info: crm_client_destroy:      Destroying 0 events
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-cib_rw-response-1237-1240-13-header
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-cib_rw-event-1237-1240-13-header
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-cib_rw-request-1237-1240-13-header
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_ipcs_disconnect:      qb_ipcs_disconnect(1237-1238-12) state:2
Nov 12 16:03:44 [1237] revproxy1        cib:     info: crm_client_destroy:      Destroying 0 events
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-cib_rw-response-1237-1238-12-header
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-cib_rw-event-1237-1238-12-header
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-cib_rw-request-1237-1238-12-header
Nov 12 16:03:44 [1237] revproxy1        cib:     info: qb_ipcs_us_withdraw:     withdrawing server sockets
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_ipcs_unref:   qb_ipcs_unref() - destroying
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_ipcs_disconnect:      qb_ipcs_disconnect(1237-1242-11) state:2
Nov 12 16:03:44 [1237] revproxy1        cib:     info: crm_client_destroy:      Destroying 0 events
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-cib_shm-response-1237-1242-11-header
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-cib_shm-event-1237-1242-11-header
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_rb_close:     Free'ing ringbuffer: /dev/shm/qb-cib_shm-request-1237-1242-11-header
Nov 12 16:03:44 [1237] revproxy1        cib:     info: qb_ipcs_us_withdraw:     withdrawing server sockets
Nov 12 16:03:44 [1237] revproxy1        cib:    debug: qb_ipcs_unref:   qb_ipcs_unref() - destroying
Nov 12 16:03:44 [1237] revproxy1        cib:     info: crm_xml_cleanup:         Cleaning up memory from libxml2
Nov 12 16:03:44 [29695] revproxy1      attrd:     info: crm_log_init:   Changed active directory to /var/lib/heartbeat/cores/hacluster
Nov 12 16:03:44 [29695] revproxy1      attrd:     info: main:   Starting up
Nov 12 16:03:44 [29695] revproxy1      attrd:     info: get_cluster_type:       Verifying cluster type: 'corosync'
Nov 12 16:03:44 [29695] revproxy1      attrd:     info: get_cluster_type:       Assuming an active 'corosync' cluster
Nov 12 16:03:44 [29695] revproxy1      attrd:   notice: crm_cluster_connect:    Connecting to cluster infrastructure: corosync
Nov 12 16:03:44 [29695] revproxy1      attrd:    debug: qb_ipcc_disconnect:     qb_ipcc_disconnect()
Nov 12 16:03:44 [29695] revproxy1      attrd:    error: cluster_connect_cpg:    Could not connect to the Cluster Process Group API: 2
Nov 12 16:03:44 [29695] revproxy1      attrd:    error: main:   HA Signon failed
Nov 12 16:03:44 [29695] revproxy1      attrd:     info: main:   Cluster connection active
Nov 12 16:03:44 [29695] revproxy1      attrd:     info: main:   Accepting attribute updates
Nov 12 16:03:44 [29695] revproxy1      attrd:    error: main:   Aborting startup
Nov 12 16:03:44 [29696] revproxy1       crmd:     info: crm_log_init:   Changed active directory to /var/lib/heartbeat/cores/hacluster
Nov 12 16:03:44 [29696] revproxy1       crmd:   notice: main:   CRM Git Version: 42f2063
Nov 12 16:03:44 [29696] revproxy1       crmd:    debug: crmd_init:      Starting crmd
Nov 12 16:03:44 [29696] revproxy1       crmd:    debug: s_crmd_fsa:     Processing I_STARTUP: [ state=S_STARTING cause=C_STARTUP origin=crmd_init ]
Nov 12 16:03:44 [29696] revproxy1       crmd:     info: do_log:         FSA: Input I_STARTUP from crmd_init() received in state S_STARTING
Nov 12 16:03:44 [29696] revproxy1       crmd:    debug: do_startup:     Registering Signal Handlers
Nov 12 16:03:44 [29696] revproxy1       crmd:    debug: do_startup:     Creating CIB and LRM objects
Nov 12 16:03:44 [29696] revproxy1       crmd:     info: get_cluster_type:       Verifying cluster type: 'corosync'
Nov 12 16:03:44 [29696] revproxy1       crmd:     info: get_cluster_type:       Assuming an active 'corosync' cluster
Nov 12 16:03:44 [29696] revproxy1       crmd:     info: crm_ipc_connect:        Could not establish cib_shm connection: Connection refused (111)
Nov 12 16:03:44 [29696] revproxy1       crmd:    debug: cib_native_signon_raw:  Connection unsuccessful (0 (nil))
Nov 12 16:03:44 [29696] revproxy1       crmd:    debug: cib_native_signon_raw:  Connection to CIB failed: Transport endpoint is not connected
Nov 12 16:03:44 [29696] revproxy1       crmd:    debug: cib_native_signoff:     Signing out of the CIB Service
Nov 12 16:03:44 [29697] revproxy1 stonith-ng:     info: crm_log_init:   Changed active directory to /var/lib/heartbeat/cores/root
Nov 12 16:03:44 [29697] revproxy1 stonith-ng:     info: get_cluster_type:       Verifying cluster type: 'corosync'
Nov 12 16:03:44 [29697] revproxy1 stonith-ng:     info: get_cluster_type:       Assuming an active 'corosync' cluster
Nov 12 16:03:44 [29697] revproxy1 stonith-ng:   notice: crm_cluster_connect:    Connecting to cluster infrastructure: corosync
Nov 12 16:03:44 [29697] revproxy1 stonith-ng:    debug: qb_ipcc_disconnect:     qb_ipcc_disconnect()
Nov 12 16:03:44 [29697] revproxy1 stonith-ng:    error: cluster_connect_cpg:    Could not connect to the Cluster Process Group API: 2
Nov 12 16:03:44 [29697] revproxy1 stonith-ng:     crit: main:   Cannot sign in to the cluster... terminating
Nov 12 16:03:44 [29697] revproxy1 stonith-ng:     info: crm_xml_cleanup:        Cleaning up memory from libxml2
Nov 12 16:03:45 [29696] revproxy1       crmd:     info: crm_ipc_connect:        Could not establish cib_shm connection: Connection refused (111)
Nov 12 16:03:45 [29696] revproxy1       crmd:    debug: cib_native_signon_raw:  Connection unsuccessful (0 (nil))
Nov 12 16:03:45 [29696] revproxy1       crmd:    debug: cib_native_signon_raw:  Connection to CIB failed: Transport endpoint is not connected
Nov 12 16:03:45 [29696] revproxy1       crmd:    debug: cib_native_signoff:     Signing out of the CIB Service
Nov 12 16:03:45 [29696] revproxy1       crmd:     info: do_cib_control:         Could not connect to the CIB service: Transport endpoint is not connected
Nov 12 16:03:45 [29696] revproxy1       crmd:  warning: do_cib_control:         Couldn't complete CIB registration 1 times... pause and retry
Nov 12 16:03:45 [29696] revproxy1       crmd:    debug: crm_timer_start:        Started Wait Timer (I_NULL:2000ms), src=7
Nov 12 16:03:45 [29696] revproxy1       crmd:    debug: register_fsa_input_adv:         Stalling the FSA pending further input: source=do_cib_control cause=C_STARTUP data=(nil) queue=0
Nov 12 16:03:45 [29696] revproxy1       crmd:    debug: s_crmd_fsa:     Exiting the FSA: queue=0, fsa_actions=0x180021000000006, stalled=true

hofmeis_jo

Anmeldungsdatum:
15. Januar 2016

Beiträge: 1
Zitieren
15. Januar 2016 09:39

Wir haben das gleiche Problem. Wir verwenden die gleiche Ubunut-Version, die gleiche Pacemaker- und Corosync-Version. Der einzige Unterschied besteht darin, dass wir das Cluster für Postfix verwenden möchten. Nach etwa zwei Tagen Laufzeit kommt auch bei uns der Fehler " Connection to the CPG API failed: Library error (2)".

/var/log/corosync/corosync.log:

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
Jan 14 21:41:55 [1230] mail1       crmd:     info: do_state_transition: 	State transition S_POLICY_ENGINE -> S_TRANSITION_ENGINE [ input=I_PE_SUCCESS cause=C_IPC_MESSAGE origin=handle_response ]
Jan 14 21:41:55 [1230] mail1       crmd:     info: do_te_invoke: 	Processing graph 259 (ref=pe_calc-dc-1452804115-509) derived from /var/lib/pacemaker/pengine/pe-input-318.bz2
Jan 14 21:41:55 [1230] mail1       crmd:   notice: run_graph: 	Transition 259 (Complete=0, Pending=0, Fired=0, Skipped=0, Incomplete=0, Source=/var/lib/pacemaker/pengine/pe-input-318.bz2): Complete
Jan 14 21:41:55 [1230] mail1       crmd:     info: do_log: 	FSA: Input I_TE_SUCCESS from notify_crmd() received in state S_TRANSITION_ENGINE
Jan 14 21:41:55 [1230] mail1       crmd:   notice: do_state_transition: 	State transition S_TRANSITION_ENGINE -> S_IDLE [ input=I_TE_SUCCESS cause=C_FSA_INTERNAL origin=notify_crmd ]
Jan 14 21:41:55 [1229] mail1    pengine:   notice: process_pe_message: 	Calculated Transition 259: /var/lib/pacemaker/pengine/pe-input-318.bz2
Jan 14 21:55:58 [1010] mail1 corosync notice  [TOTEM ] A processor failed, forming new configuration.
Jan 14 21:55:59 [1010] mail1 corosync notice  [TOTEM ] A new membership (192.168.0.10:400) was formed. Members left: 2
Jan 14 21:55:59 [1225] mail1        cib:     info: pcmk_cpg_membership: 	Left[2.0] cib.2 
Jan 14 21:55:59 [1225] mail1        cib:     info: crm_update_peer_proc: 	pcmk_cpg_membership: Node mail2[2] - corosync-cpg is now offline
Jan 14 21:55:59 [1225] mail1        cib:     info: pcmk_cpg_membership: 	Member[2.0] cib.1 
Jan 14 21:55:59 [1230] mail1       crmd:     info: pcmk_cpg_membership: 	Left[1.0] crmd.2 
Jan 14 21:55:59 [1230] mail1       crmd:     info: crm_update_peer_proc: 	pcmk_cpg_membership: Node mail2[2] - corosync-cpg is now offline
Jan 14 21:55:59 [1230] mail1       crmd:     info: peer_update_callback: 	Client mail2/peer now has status [offline] (DC=true)
Jan 14 21:55:59 [1230] mail1       crmd:  warning: match_down_event: 	No match for shutdown action on 2
Jan 14 21:55:59 [1230] mail1       crmd:   notice: peer_update_callback: 	Stonith/shutdown of mail2 not matched
Jan 14 21:55:59 [1230] mail1       crmd:     info: crm_update_peer_join: 	peer_update_callback: Node mail2[2] - join-1 phase 4 -> 0
Jan 14 21:55:59 [1230] mail1       crmd:     info: abort_transition_graph: 	peer_update_callback:214 - Triggered transition abort (complete=1) : Node failure
Jan 14 21:55:59 [1230] mail1       crmd:     info: pcmk_cpg_membership: 	Member[1.0] crmd.1 
Jan 14 21:55:59 [1230] mail1       crmd:   notice: do_state_transition: 	State transition S_IDLE -> S_POLICY_ENGINE [ input=I_PE_CALC cause=C_FSA_INTERNAL origin=abort_transition_graph ]
Jan 14 21:55:59 [1225] mail1        cib:     info: cib_process_request: 	Completed cib_modify operation for section status: OK (rc=0, origin=local/crmd/342, version=0.184.7)
Jan 14 21:55:59 [1225] mail1        cib:     info: crm_cs_flush: 	Sent 0 CPG messages  (1 remaining, last=200): Try again (6)
Jan 14 21:55:59 [1225] mail1        cib:     info: cib_process_request: 	Completed cib_query operation for section 'all': OK (rc=0, origin=local/crmd/343, version=0.184.7)
Jan 14 21:55:59 [1229] mail1    pengine:   notice: unpack_config: 	On loss of CCM Quorum: Ignore
Jan 14 21:55:59 [1229] mail1    pengine:  warning: pe_fence_node: 	Node mail2 is unclean because it is partially and/or un-expectedly down
Jan 14 21:55:59 [1229] mail1    pengine:     info: determine_online_status_no_fencing: 		in_cluster=true, is_peer=offline, join=member, expected=member
Jan 14 21:55:59 [1229] mail1    pengine:  warning: determine_online_status: 	Node mail2 is unclean
Jan 14 21:55:59 [1229] mail1    pengine:     info: determine_online_status: 	Node mail1 is online
Jan 14 21:55:59 [1229] mail1    pengine:   notice: unpack_rsc_op: 	Operation monitor found resource res_drbd_1:0 active in master mode on mail1
Jan 14 21:55:59 [1229] mail1    pengine:     info: clone_print: 	 Master/Slave Set: ms_drbd_1 [res_drbd_1]
Jan 14 21:55:59 [1229] mail1    pengine:     info: short_print: 	     Masters: [ mail1 ]
Jan 14 21:55:59 [1229] mail1    pengine:     info: short_print: 	     Stopped: [ mail2 ]
Jan 14 21:55:59 [1229] mail1    pengine:     info: native_print: 	res_Filesystem_1	(ocf::heartbeat:Filesystem):	Started mail1 
Jan 14 21:55:59 [1229] mail1    pengine:     info: native_print: 	res_IPaddr2_1	(ocf::heartbeat:IPaddr2):	Started mail1 
Jan 14 21:55:59 [1229] mail1    pengine:     info: native_print: 	res_postfix_1	(lsb:postfix):	Started mail1 
Jan 14 21:55:59 [1229] mail1    pengine:     info: native_color: 	Resource res_drbd_1:1 cannot run anywhere
Jan 14 21:55:59 [1229] mail1    pengine:     info: master_color: 	Promoting res_drbd_1:0 (Master mail1)
Jan 14 21:55:59 [1229] mail1    pengine:     info: master_color: 	ms_drbd_1: Promoted 1 instances of a possible 1 to master
Jan 14 21:55:59 [1225] mail1        cib:     info: cib_process_request: 	Completed cib_query operation for section 'all': OK (rc=0, origin=local/cibmon/2, version=0.184.7)
Jan 14 21:55:59 [1229] mail1    pengine:  warning: stage6: 	Node mail2 is unclean!
Jan 14 21:55:59 [1229] mail1    pengine:  warning: stage6: 	YOUR RESOURCES ARE NOW LIKELY COMPROMISED
Jan 14 21:55:59 [1229] mail1    pengine:    error: stage6: 	ENABLE STONITH TO KEEP YOUR RESOURCES SAFE
Jan 14 21:55:59 [1229] mail1    pengine:     info: LogActions: 	Leave   res_drbd_1:0	(Master mail1)
Jan 14 21:55:59 [1229] mail1    pengine:     info: LogActions: 	Leave   res_drbd_1:1	(Stopped)
Jan 14 21:55:59 [1229] mail1    pengine:     info: LogActions: 	Leave   res_Filesystem_1	(Started mail1)
Jan 14 21:55:59 [1229] mail1    pengine:     info: LogActions: 	Leave   res_IPaddr2_1	(Started mail1)
Jan 14 21:55:59 [1229] mail1    pengine:     info: LogActions: 	Leave   res_postfix_1	(Started mail1)
Jan 14 21:55:59 [1223] mail1 pacemakerd:     info: crm_cs_flush: 	Sent 0 CPG messages  (1 remaining, last=7): Try again (6)
Jan 14 21:55:59 [1226] mail1 stonith-ng:     info: pcmk_cpg_membership: 	Left[2.0] stonith-ng.2 
Jan 14 21:55:59 [1226] mail1 stonith-ng:     info: crm_update_peer_proc: 	pcmk_cpg_membership: Node mail2[2] - corosync-cpg is now offline
Jan 14 21:55:59 [1226] mail1 stonith-ng:     info: crm_cs_flush: 	Sent 0 CPG messages  (1 remaining, last=2): Try again (6)
Jan 14 21:55:59 [1226] mail1 stonith-ng:     info: pcmk_cpg_membership: 	Member[2.0] stonith-ng.1 
Jan 14 21:55:59 [1010] mail1 corosync notice  [QUORUM] Members[1]: 1
Jan 14 21:55:59 [1230] mail1       crmd:     info: pcmk_quorum_notification: 	Membership 400: quorum retained (1)
Jan 14 21:55:59 [1230] mail1       crmd:   notice: crm_update_peer_state: 	pcmk_quorum_notification: Node mail2[2] - state is now lost (was member)
Jan 14 21:55:59 [1230] mail1       crmd:     info: peer_update_callback: 	mail2 is now lost (was member)
Jan 14 21:55:59 [1230] mail1       crmd:  warning: match_down_event: 	No match for shutdown action on 2
Jan 14 21:55:59 [1230] mail1       crmd:   notice: peer_update_callback: 	Stonith/shutdown of mail2 not matched
Jan 14 21:55:59 [1230] mail1       crmd:     info: abort_transition_graph: 	peer_update_callback:214 - Triggered transition abort (complete=1) : Node failure
Jan 14 21:55:59 [1230] mail1       crmd:     info: crm_cs_flush: 	Sent 0 CPG messages  (1 remaining, last=100): Try again (6)
Jan 14 21:55:59 [1225] mail1        cib:     info: cib_process_request: 	Completed cib_modify operation for section status: OK (rc=0, origin=local/crmd/344, version=0.184.7)
Jan 14 21:55:59 [1225] mail1        cib:     info: cib_process_request: 	Completed cib_modify operation for section nodes: OK (rc=0, origin=local/crmd/345, version=0.184.7)
Jan 14 21:55:59 [1223] mail1 pacemakerd:     info: pcmk_quorum_notification: 	Membership 400: quorum retained (1)
Jan 14 21:55:59 [1223] mail1 pacemakerd:   notice: crm_update_peer_state: 	pcmk_quorum_notification: Node mail2[2] - state is now lost (was member)
Jan 14 21:55:59 [1010] mail1 corosync notice  [MAIN  ] Completed service synchronization, ready to provide service.
Jan 14 21:55:59 [1225] mail1        cib:     info: cib_process_request: 	Completed cib_modify operation for section status: OK (rc=0, origin=local/crmd/346, version=0.184.8)
Jan 14 21:55:59 [1225] mail1        cib:     info: cib_process_request: 	Completed cib_query operation for section 'all': OK (rc=0, origin=local/crmd/347, version=0.184.8)
Jan 14 21:55:59 [1229] mail1    pengine:    error: process_pe_message: 	Calculated Transition 260: /var/lib/pacemaker/pengine/pe-error-12.bz2
Jan 14 21:55:59 [1229] mail1    pengine:   notice: process_pe_message: 	Configuration ERRORs found during PE processing.  Please run "crm_verify -L" to identify issues.
Jan 14 21:55:59 [1225] mail1        cib:     info: crm_client_new: 	Connecting 0x7f882ec54590 for uid=0 gid=0 pid=18273 id=5196915d-cf34-4008-8711-9f5ce26f08fb
Jan 14 21:55:59 [1225] mail1        cib:     info: cib_process_request: 	Completed cib_query operation for section 'all': OK (rc=0, origin=local/crm_simulate/2, version=0.184.8)
Jan 14 21:55:59 [1225] mail1        cib:     info: crm_client_destroy: 	Destroying 0 events
Jan 14 21:55:59 [1225] mail1        cib:     info: crm_client_new: 	Connecting 0x7f882ec54590 for uid=0 gid=0 pid=18277 id=0fc29697-a89a-49b1-8988-9c7fa02f683f
Jan 14 21:55:59 [1225] mail1        cib:     info: cib_process_request: 	Completed cib_query operation for section 'all': OK (rc=0, origin=local/cibadmin/2, version=0.184.8)
Jan 14 21:55:59 [1225] mail1        cib:     info: crm_client_destroy: 	Destroying 0 events
Jan 14 21:55:59 [1225] mail1        cib:     info: crm_client_new: 	Connecting 0x7f882ec54590 for uid=0 gid=0 pid=18280 id=3261235b-6b0c-42b1-a4ed-c7e32b85e465
Jan 14 21:55:59 [1225] mail1        cib:     info: cib_process_request: 	Completed cib_query operation for section 'all': OK (rc=0, origin=local/crm_simulate/2, version=0.184.8)
Jan 14 21:55:59 [1225] mail1        cib:     info: crm_client_destroy: 	Destroying 0 events
Jan 14 21:55:59 [1225] mail1        cib:     info: crm_client_new: 	Connecting 0x7f882ec54590 for uid=0 gid=0 pid=18284 id=c5d27b6e-c6f6-4476-8468-0c6ffe62bff2
Jan 14 21:55:59 [1225] mail1        cib:     info: cib_process_request: 	Completed cib_query operation for section 'all': OK (rc=0, origin=local/cibadmin/2, version=0.184.8)
Jan 14 21:55:59 [1225] mail1        cib:     info: crm_client_destroy: 	Destroying 0 events
Jan 14 21:55:59 [1225] mail1        cib:     info: crm_cs_flush: 	Sent 2 CPG messages  (0 remaining, last=202): OK (1)
Jan 14 21:56:00 [1230] mail1       crmd:     info: handle_response: 	pe_calc calculation pe_calc-dc-1452804959-510 is obsolete
Jan 14 21:56:00 [1225] mail1        cib:     info: cib_process_request: 	Completed cib_query operation for section 'all': OK (rc=0, origin=local/crmd/348, version=0.184.8)
Jan 14 21:56:00 [1229] mail1    pengine:   notice: unpack_config: 	On loss of CCM Quorum: Ignore
Jan 14 21:56:00 [1229] mail1    pengine:     info: determine_online_status: 	Node mail1 is online
Jan 14 21:56:00 [1229] mail1    pengine:   notice: unpack_rsc_op: 	Operation monitor found resource res_drbd_1:0 active in master mode on mail1
Jan 14 21:56:00 [1229] mail1    pengine:     info: clone_print: 	 Master/Slave Set: ms_drbd_1 [res_drbd_1]
Jan 14 21:56:00 [1229] mail1    pengine:     info: short_print: 	     Masters: [ mail1 ]
Jan 14 21:56:00 [1229] mail1    pengine:     info: short_print: 	     Stopped: [ mail2 ]
Jan 14 21:56:00 [1229] mail1    pengine:     info: native_print: 	res_Filesystem_1	(ocf::heartbeat:Filesystem):	Started mail1 
Jan 14 21:56:00 [1229] mail1    pengine:     info: native_print: 	res_IPaddr2_1	(ocf::heartbeat:IPaddr2):	Started mail1 
Jan 14 21:56:00 [1229] mail1    pengine:     info: native_print: 	res_postfix_1	(lsb:postfix):	Started mail1 
Jan 14 21:56:00 [1229] mail1    pengine:     info: native_color: 	Resource res_drbd_1:1 cannot run anywhere
Jan 14 21:56:00 [1229] mail1    pengine:     info: master_color: 	Promoting res_drbd_1:0 (Master mail1)
Jan 14 21:56:00 [1229] mail1    pengine:     info: master_color: 	ms_drbd_1: Promoted 1 instances of a possible 1 to master
Jan 14 21:56:00 [1229] mail1    pengine:     info: LogActions: 	Leave   res_drbd_1:0	(Master mail1)
Jan 14 21:56:00 [1229] mail1    pengine:     info: LogActions: 	Leave   res_drbd_1:1	(Stopped)
Jan 14 21:56:00 [1229] mail1    pengine:     info: LogActions: 	Leave   res_Filesystem_1	(Started mail1)
Jan 14 21:56:00 [1229] mail1    pengine:     info: LogActions: 	Leave   res_IPaddr2_1	(Started mail1)
Jan 14 21:56:00 [1229] mail1    pengine:     info: LogActions: 	Leave   res_postfix_1	(Started mail1)
Jan 14 21:56:00 [1230] mail1       crmd:     info: do_state_transition: 	State transition S_POLICY_ENGINE -> S_TRANSITION_ENGINE [ input=I_PE_SUCCESS cause=C_IPC_MESSAGE origin=handle_response ]
Jan 14 21:56:00 [1230] mail1       crmd:     info: do_te_invoke: 	Processing graph 261 (ref=pe_calc-dc-1452804960-512) derived from /var/lib/pacemaker/pengine/pe-input-319.bz2
Jan 14 21:56:00 [1230] mail1       crmd:   notice: run_graph: 	Transition 261 (Complete=0, Pending=0, Fired=0, Skipped=0, Incomplete=0, Source=/var/lib/pacemaker/pengine/pe-input-319.bz2): Complete
Jan 14 21:56:00 [1230] mail1       crmd:     info: do_log: 	FSA: Input I_TE_SUCCESS from notify_crmd() received in state S_TRANSITION_ENGINE
Jan 14 21:56:00 [1230] mail1       crmd:   notice: do_state_transition: 	State transition S_TRANSITION_ENGINE -> S_IDLE [ input=I_TE_SUCCESS cause=C_FSA_INTERNAL origin=notify_crmd ]
Jan 14 21:56:00 [1229] mail1    pengine:   notice: process_pe_message: 	Calculated Transition 261: /var/lib/pacemaker/pengine/pe-input-319.bz2
Jan 14 22:11:00 [1230] mail1       crmd:     info: crm_timer_popped: 	PEngine Recheck Timer (I_PE_CALC) just popped (900000ms)
Jan 14 22:11:00 [1230] mail1       crmd:   notice: do_state_transition: 	State transition S_IDLE -> S_POLICY_ENGINE [ input=I_PE_CALC cause=C_TIMER_POPPED origin=crm_timer_popped ]
Jan 14 22:11:00 [1230] mail1       crmd:     info: do_state_transition: 	Progressed to state S_POLICY_ENGINE after C_TIMER_POPPED
Jan 14 22:11:00 [1225] mail1        cib:     info: cib_process_request: 	Completed cib_query operation for section 'all': OK (rc=0, origin=local/crmd/349, version=0.184.8)
Jan 14 22:11:00 [1229] mail1    pengine:     info: process_pe_message: 	Input has not changed since last time, not saving to disk
Jan 14 22:11:00 [1229] mail1    pengine:   notice: unpack_config: 	On loss of CCM Quorum: Ignore
Jan 14 22:11:00 [1229] mail1    pengine:     info: determine_online_status: 	Node mail1 is online
Jan 14 22:11:00 [1229] mail1    pengine:   notice: unpack_rsc_op: 	Operation monitor found resource res_drbd_1:0 active in master mode on mail1
Jan 14 22:11:00 [1229] mail1    pengine:     info: clone_print: 	 Master/Slave Set: ms_drbd_1 [res_drbd_1]
Jan 14 22:11:00 [1229] mail1    pengine:     info: short_print: 	     Masters: [ mail1 ]
Jan 14 22:11:00 [1229] mail1    pengine:     info: short_print: 	     Stopped: [ mail2 ]
Jan 14 22:11:00 [1229] mail1    pengine:     info: native_print: 	res_Filesystem_1	(ocf::heartbeat:Filesystem):	Started mail1 
Jan 14 22:11:00 [1229] mail1    pengine:     info: native_print: 	res_IPaddr2_1	(ocf::heartbeat:IPaddr2):	Started mail1 
Jan 14 22:11:00 [1229] mail1    pengine:     info: native_print: 	res_postfix_1	(lsb:postfix):	Started mail1 
Jan 14 22:11:00 [1229] mail1    pengine:     info: native_color: 	Resource res_drbd_1:1 cannot run anywhere
Jan 14 22:11:00 [1229] mail1    pengine:     info: master_color: 	Promoting res_drbd_1:0 (Master mail1)
Jan 14 22:11:00 [1229] mail1    pengine:     info: master_color: 	ms_drbd_1: Promoted 1 instances of a possible 1 to master
Jan 14 22:11:00 [1229] mail1    pengine:     info: LogActions: 	Leave   res_drbd_1:0	(Master mail1)
Jan 14 22:11:00 [1229] mail1    pengine:     info: LogActions: 	Leave   res_drbd_1:1	(Stopped)
Jan 14 22:11:00 [1229] mail1    pengine:     info: LogActions: 	Leave   res_Filesystem_1	(Started mail1)
Jan 14 22:11:00 [1229] mail1    pengine:     info: LogActions: 	Leave   res_IPaddr2_1	(Started mail1)
Jan 14 22:11:00 [1229] mail1    pengine:     info: LogActions: 	Leave   res_postfix_1	(Started mail1)
Jan 14 22:11:00 [1230] mail1       crmd:     info: do_state_transition: 	State transition S_POLICY_ENGINE -> S_TRANSITION_ENGINE [ input=I_PE_SUCCESS cause=C_IPC_MESSAGE origin=handle_response ]
Jan 14 22:11:00 [1230] mail1       crmd:     info: do_te_invoke: 	Processing graph 262 (ref=pe_calc-dc-1452805860-513) derived from /var/lib/pacemaker/pengine/pe-input-319.bz2
Jan 14 22:11:00 [1230] mail1       crmd:   notice: run_graph: 	Transition 262 (Complete=0, Pending=0, Fired=0, Skipped=0, Incomplete=0, Source=/var/lib/pacemaker/pengine/pe-input-319.bz2): Complete
Jan 14 22:11:00 [1230] mail1       crmd:     info: do_log: 	FSA: Input I_TE_SUCCESS from notify_crmd() received in state S_TRANSITION_ENGINE
Jan 14 22:11:00 [1230] mail1       crmd:   notice: do_state_transition: 	State transition S_TRANSITION_ENGINE -> S_IDLE [ input=I_TE_SUCCESS cause=C_FSA_INTERNAL origin=notify_crmd ]
Jan 14 22:11:00 [1229] mail1    pengine:   notice: process_pe_message: 	Calculated Transition 262: /var/lib/pacemaker/pengine/pe-input-319.bz2
Jan 14 22:26:00 [1230] mail1       crmd:     info: crm_timer_popped: 	PEngine Recheck Timer (I_PE_CALC) just popped (900000ms)
Jan 14 22:26:00 [1230] mail1       crmd:   notice: do_state_transition: 	State transition S_IDLE -> S_POLICY_ENGINE [ input=I_PE_CALC cause=C_TIMER_POPPED origin=crm_timer_popped ]
Jan 14 22:26:00 [1230] mail1       crmd:     info: do_state_transition: 	Progressed to state S_POLICY_ENGINE after C_TIMER_POPPED
Jan 14 22:26:00 [1225] mail1        cib:     info: cib_process_request: 	Completed cib_query operation for section 'all': OK (rc=0, origin=local/crmd/350, version=0.184.8)
Jan 14 22:26:00 [1229] mail1    pengine:     info: process_pe_message: 	Input has not changed since last time, not saving to disk
Jan 14 22:26:00 [1229] mail1    pengine:   notice: unpack_config: 	On loss of CCM Quorum: Ignore
Jan 14 22:26:00 [1229] mail1    pengine:     info: determine_online_status: 	Node mail1 is online
Jan 14 22:26:00 [1229] mail1    pengine:   notice: unpack_rsc_op: 	Operation monitor found resource res_drbd_1:0 active in master mode on mail1
Jan 14 22:26:00 [1229] mail1    pengine:     info: clone_print: 	 Master/Slave Set: ms_drbd_1 [res_drbd_1]
Jan 14 22:26:00 [1229] mail1    pengine:     info: short_print: 	     Masters: [ mail1 ]
Jan 14 22:26:00 [1229] mail1    pengine:     info: short_print: 	     Stopped: [ mail2 ]
Jan 14 22:26:00 [1229] mail1    pengine:     info: native_print: 	res_Filesystem_1	(ocf::heartbeat:Filesystem):	Started mail1 
Jan 14 22:26:00 [1229] mail1    pengine:     info: native_print: 	res_IPaddr2_1	(ocf::heartbeat:IPaddr2):	Started mail1 
Jan 14 22:26:00 [1229] mail1    pengine:     info: native_print: 	res_postfix_1	(lsb:postfix):	Started mail1 
Jan 14 22:26:00 [1229] mail1    pengine:     info: native_color: 	Resource res_drbd_1:1 cannot run anywhere
Jan 14 22:26:00 [1229] mail1    pengine:     info: master_color: 	Promoting res_drbd_1:0 (Master mail1)
Jan 14 22:26:00 [1229] mail1    pengine:     info: master_color: 	ms_drbd_1: Promoted 1 instances of a possible 1 to master
Jan 14 22:26:00 [1229] mail1    pengine:     info: LogActions: 	Leave   res_drbd_1:0	(Master mail1)
Jan 14 22:26:00 [1229] mail1    pengine:     info: LogActions: 	Leave   res_drbd_1:1	(Stopped)
Jan 14 22:26:00 [1229] mail1    pengine:     info: LogActions: 	Leave   res_Filesystem_1	(Started mail1)
Jan 14 22:26:00 [1229] mail1    pengine:     info: LogActions: 	Leave   res_IPaddr2_1	(Started mail1)
Jan 14 22:26:00 [1229] mail1    pengine:     info: LogActions: 	Leave   res_postfix_1	(Started mail1)
Jan 14 22:26:00 [1230] mail1       crmd:     info: do_state_transition: 	State transition S_POLICY_ENGINE -> S_TRANSITION_ENGINE [ input=I_PE_SUCCESS cause=C_IPC_MESSAGE origin=handle_response ]
Jan 14 22:26:00 [1230] mail1       crmd:     info: do_te_invoke: 	Processing graph 263 (ref=pe_calc-dc-1452806760-514) derived from /var/lib/pacemaker/pengine/pe-input-319.bz2
Jan 14 22:26:00 [1230] mail1       crmd:   notice: run_graph: 	Transition 263 (Complete=0, Pending=0, Fired=0, Skipped=0, Incomplete=0, Source=/var/lib/pacemaker/pengine/pe-input-319.bz2): Complete
Jan 14 22:26:00 [1230] mail1       crmd:     info: do_log: 	FSA: Input I_TE_SUCCESS from notify_crmd() received in state S_TRANSITION_ENGINE
Jan 14 22:26:00 [1230] mail1       crmd:   notice: do_state_transition: 	State transition S_TRANSITION_ENGINE -> S_IDLE [ input=I_TE_SUCCESS cause=C_FSA_INTERNAL origin=notify_crmd ]
Jan 14 22:26:00 [1229] mail1    pengine:   notice: process_pe_message: 	Calculated Transition 263: /var/lib/pacemaker/pengine/pe-input-319.bz2
Jan 14 22:29:20 [1230] mail1       crmd:    error: crmd_quorum_destroy: 	connection terminated
Jan 14 22:29:20 [1230] mail1       crmd:     info: qb_ipcs_us_withdraw: 	withdrawing server sockets
Jan 14 22:29:20 [1230] mail1       crmd:     info: tengine_stonith_connection_destroy: 	Fencing daemon disconnected
Jan 14 22:29:20 [1230] mail1       crmd:   notice: crmd_exit: 	Forcing immediate exit: Link has been severed (67)
Jan 14 22:29:20 [1230] mail1       crmd:     info: crm_xml_cleanup: 	Cleaning up memory from libxml2
Jan 14 22:29:20 [1229] mail1    pengine:     info: crm_client_destroy: 	Destroying 0 events
Jan 14 22:29:20 [1226] mail1 stonith-ng:     info: crm_client_destroy: 	Destroying 0 events
Jan 14 22:29:20 [1225] mail1        cib:    error: pcmk_cpg_dispatch: 	Connection to the CPG API failed: Library error (2)
Jan 14 22:29:20 [1225] mail1        cib:    error: cib_cs_destroy: 	Corosync connection lost!  Exiting.
Jan 14 22:29:20 [1225] mail1        cib:     info: terminate_cib: 	cib_cs_destroy: Exiting fast...
Jan 14 22:29:20 [1225] mail1        cib:     info: crm_client_destroy: 	Destroying 0 events
Jan 14 22:29:20 [1225] mail1        cib:     info: qb_ipcs_us_withdraw: 	withdrawing server sockets
Jan 14 22:29:20 [1225] mail1        cib:     info: crm_client_destroy: 	Destroying 0 events
Jan 14 22:29:20 [1225] mail1        cib:     info: crm_client_destroy: 	Destroying 0 events
Jan 14 22:29:20 [1225] mail1        cib:     info: qb_ipcs_us_withdraw: 	withdrawing server sockets
Jan 14 22:29:20 [1225] mail1        cib:     info: crm_client_destroy: 	Destroying 0 events
Jan 14 22:29:20 [1225] mail1        cib:     info: qb_ipcs_us_withdraw: 	withdrawing server sockets
Jan 14 22:29:20 [1225] mail1        cib:     info: crm_xml_cleanup: 	Cleaning up memory from libxml2
Jan 14 22:29:20 [1228] mail1      attrd:    error: crm_ipc_read: 	Connection to cib_rw failed
Jan 14 22:29:20 [1228] mail1      attrd:    error: mainloop_gio_callback: 	Connection to cib_rw[0x7f04d1718820] closed (I/O condition=17)
Jan 14 22:29:20 [1228] mail1      attrd:    error: attrd_cib_connection_destroy: 	Connection to the CIB terminated...
Jan 14 22:29:20 [1223] mail1 pacemakerd:    error: pcmk_child_exit: 	Child process attrd (1228) exited: Transport endpoint is not connected (107)
Jan 14 22:29:20 [1223] mail1 pacemakerd:   notice: pcmk_process_exit: 	Respawning failed child process: attrd
Jan 14 22:29:20 [1223] mail1 pacemakerd:     info: start_child: 	Using uid=107 and group=114 for process attrd
Jan 14 22:29:20 [1223] mail1 pacemakerd:     info: start_child: 	Forked child 18715 for process attrd
Jan 14 22:29:20 [1226] mail1 stonith-ng:    error: crm_ipc_read: 	Connection to cib_rw failed
Jan 14 22:29:20 [1226] mail1 stonith-ng:    error: mainloop_gio_callback: 	Connection to cib_rw[0x7fb48340fd80] closed (I/O condition=17)
Jan 14 22:29:20 [1226] mail1 stonith-ng:   notice: cib_connection_destroy: 	Connection to the CIB terminated. Shutting down.
Jan 14 22:29:20 [1226] mail1 stonith-ng:     info: stonith_shutdown: 	Terminating with  0 clients
Jan 14 22:29:20 [1226] mail1 stonith-ng:     info: qb_ipcs_us_withdraw: 	withdrawing server sockets
Jan 14 22:29:20 [1226] mail1 stonith-ng:     info: main: 	Done
Jan 14 22:29:20 [1226] mail1 stonith-ng:     info: crm_xml_cleanup: 	Cleaning up memory from libxml2
Jan 14 22:29:20 [1223] mail1 pacemakerd:     info: pcmk_child_exit: 	Child process stonith-ng (1226) exited: OK (0)
Jan 14 22:29:20 [1223] mail1 pacemakerd:   notice: pcmk_process_exit: 	Respawning failed child process: stonith-ng
Jan 14 22:29:20 [1223] mail1 pacemakerd:     info: start_child: 	Forked child 18716 for process stonith-ng
Jan 14 22:29:20 [1223] mail1 pacemakerd:     info: mcp_quorum_destroy: 	connection closed
Jan 14 22:29:20 [1223] mail1 pacemakerd:    error: pcmk_cpg_dispatch: 	Connection to the CPG API failed: Library error (2)
Jan 14 22:29:20 [1223] mail1 pacemakerd:    error: mcp_cpg_destroy: 	Connection destroyed
Jan 14 22:29:20 [1223] mail1 pacemakerd:     info: crm_xml_cleanup: 	Cleaning up memory from libxml2
Jan 14 22:29:20 [1227] mail1       lrmd:     info: cancel_recurring_action: 	Cancelling operation res_IPaddr2_1_monitor_10000
Jan 14 22:29:20 [1227] mail1       lrmd:  warning: qb_ipcs_event_sendv: 	new_event_notification (1227-1230-7): Bad file descriptor (9)
Jan 14 22:29:20 [1227] mail1       lrmd:  warning: send_client_notify: 	Notification of client crmd/68ce58a2-d1f1-4935-b112-f17c92cc4796 failed
Jan 14 22:29:20 [1227] mail1       lrmd:     info: cancel_recurring_action: 	Cancelling operation res_Filesystem_1_monitor_20000
Jan 14 22:29:20 [1227] mail1       lrmd:  warning: send_client_notify: 	Notification of client crmd/68ce58a2-d1f1-4935-b112-f17c92cc4796 failed
Jan 14 22:29:20 [1227] mail1       lrmd:     info: cancel_recurring_action: 	Cancelling operation res_postfix_1_status_15000
Jan 14 22:29:20 [1227] mail1       lrmd:  warning: send_client_notify: 	Notification of client crmd/68ce58a2-d1f1-4935-b112-f17c92cc4796 failed
Jan 14 22:29:20 [1227] mail1       lrmd:     info: crm_client_destroy: 	Destroying 3 events
Jan 14 22:29:20 [18715] mail1      attrd:   notice: crm_cluster_connect: 	Connecting to cluster infrastructure: corosync
Jan 14 22:29:20 [18715] mail1      attrd:    error: cluster_connect_cpg: 	Could not connect to the Cluster Process Group API: 2
Jan 14 22:29:20 [18715] mail1      attrd:    error: main: 	HA Signon failed
Jan 14 22:29:20 [18715] mail1      attrd:    error: main: 	Aborting startup
Jan 14 22:29:20 [18716] mail1 stonith-ng:     info: crm_log_init: 	Changed active directory to /var/lib/heartbeat/cores/root
Jan 14 22:29:20 [18716] mail1 stonith-ng:     info: get_cluster_type: 	Verifying cluster type: 'corosync'
Jan 14 22:29:20 [18716] mail1 stonith-ng:     info: get_cluster_type: 	Assuming an active 'corosync' cluster
Jan 14 22:29:20 [18716] mail1 stonith-ng:   notice: crm_cluster_connect: 	Connecting to cluster infrastructure: corosync
Jan 14 22:29:20 [18716] mail1 stonith-ng:    error: cluster_connect_cpg: 	Could not connect to the Cluster Process Group API: 2
Jan 14 22:29:20 [18716] mail1 stonith-ng:     crit: main: 	Cannot sign in to the cluster... terminating
Jan 14 22:29:20 [18716] mail1 stonith-ng:     info: crm_xml_cleanup: 	Cleaning up memory from libxml2

Bei mir ist das Problem nach dem Neustart von beiden Hosts gelöst, DRBD meldet keinen Split-Brain.

Die /etc/corosync/corosync.conf sieht so aus:

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 totem {
      version: 2
      cluster_name: mailcluster
      transport: udpu
      interface {
        ringnumber: 0
        bindnetaddr: 192.168.0.0
        broadcast: yes
        mcastport: 5405
      }
    }

    quorum {
      provider: corosync_votequorum
      two_node: 1
    }

    nodelist {
      node {
        ring0_addr: 192.168.0.10
        name: mail1
        nodeid: 1
      }
      node {
        ring0_addr: 192.168.0.20
        name: mail2
        nodeid: 2
      }
    }

    logging {
      to_logfile: yes
      logfile: /var/log/corosync/corosync.log
      to_syslog: yes
      timestamp: on
    }

Ich hoffe, uns kann jemand helfen.
Antworten |